From 8b296b148e2e62d9bd618849a4e072f05375714c Mon Sep 17 00:00:00 2001 From: Steve Bazyl Date: Mon, 19 Oct 2015 15:36:24 -0700 Subject: [PATCH] #286 - Send content-id in batch requests --- lib/google/apis/core/batch.rb | 33 +++++++++++++++++++------- lib/google/apis/core/multipart.rb | 26 ++++++++++++++++----- spec/google/apis/core/batch_spec.rb | 36 ++++++++++++++++++++--------- 3 files changed, 70 insertions(+), 25 deletions(-) diff --git a/lib/google/apis/core/batch.rb b/lib/google/apis/core/batch.rb index 5a2e94e02..e95ce68ce 100644 --- a/lib/google/apis/core/batch.rb +++ b/lib/google/apis/core/batch.rb @@ -31,7 +31,7 @@ require 'google/apis/core/http_command' require 'google/apis/core/upload' require 'google/apis/core/download' require 'addressable/uri' - +require 'securerandom' module Google module Apis module Core @@ -47,6 +47,7 @@ module Google def initialize(method, url) super(method, url) @calls = [] + @base_id = SecureRandom.uuid end ## @@ -78,9 +79,12 @@ module Google parts = split_parts(body, m[1]) deserializer = CallDeserializer.new parts.each_index do |index| - call, callback = @calls[index] + response = deserializer.to_http_response(parts[index]) + outer_header = response.shift + call_id = header_to_id(outer_header[:content_id]) || index + call, callback = @calls[call_id] begin - result = call.process_response(*deserializer.to_http_response(parts[index])) unless call.nil? + result = call.process_response(*response) unless call.nil? success(result, &callback) rescue => e error(e, &callback) @@ -103,9 +107,11 @@ module Google serializer = CallSerializer.new multipart = Multipart.new(boundary: BATCH_BOUNDARY, content_type: MULTIPART_MIXED) - @calls.each do |(call, _)| + @calls.each_index do |index| + call, _ = @calls[index] + content_id = id_to_header(index) io = serializer.to_upload_io(call) - multipart.add_upload(io) + multipart.add_upload(io, content_id: content_id) end self.body = multipart.assemble @@ -120,6 +126,17 @@ module Google end fail Google::Apis::ClientError, 'Invalid command object' unless command.is_a?(HttpCommand) end + + def id_to_header(call_id) + return sprintf('<%s+%i>', @base_id, call_id) + end + + def header_to_id(content_id) + match = //.match(content_id) + return match[1].to_i if match + return nil + end + end # Wrapper request for batching multiple uploads in a single server request @@ -180,19 +197,19 @@ module Google # Deconstructs a raw HTTP response part # @private class CallDeserializer - # Convert a single batched response into a BatchedCallResponse object. + # Parse a batched response. # # @param [String] call_response # the response to parse. # @return [Array<(Fixnum, Hurley::Header, String)>] # Status, header, and response body. def to_http_response(call_response) - _, outer_body = split_header_and_body(call_response) + outer_header, outer_body = split_header_and_body(call_response) status_line, payload = outer_body.split(/\n/, 2) _, status = status_line.split(' ', 3) header, body = split_header_and_body(payload) - [status.to_i, header, body] + [outer_header, status.to_i, header, body] end protected diff --git a/lib/google/apis/core/multipart.rb b/lib/google/apis/core/multipart.rb index 7eb4c43f2..c6063fac3 100644 --- a/lib/google/apis/core/multipart.rb +++ b/lib/google/apis/core/multipart.rb @@ -31,7 +31,7 @@ module Google # Multipart boundary # @param [String] value # JSON content - def initialize(boundary, value) + def initialize(boundary, value, header = {}) @part = build_part(boundary, value) @length = @part.bytesize @io = StringIO.new(@part) @@ -95,19 +95,25 @@ module Google # @param [Hash] header # Headers for the part def build_head(boundary, type, content_len, header) + content_id = '' + if header[:content_id] + content_id = sprintf(CID_FORMAT, header[:content_id]) + end sprintf(HEAD_FORMAT, boundary, content_len.to_i, + content_id, header[:content_type] || type, header[:content_transfer_encoding] || DEFAULT_TR_ENCODING) end DEFAULT_TR_ENCODING = 'binary'.freeze FOOT = "\r\n".freeze + CID_FORMAT = "Content-ID: %s\r\n" HEAD_FORMAT = <<-END --%s\r Content-Length: %d\r -Content-Type: %s\r +%sContent-Type: %s\r Content-Transfer-Encoding: %s\r \r END @@ -137,9 +143,12 @@ Content-Transfer-Encoding: %s\r # # @param [String] body # JSON text + # @param [String] content_id + # Optional unique ID of this part # @return [self] - def add_json(body) - @parts << Google::Apis::Core::JsonPart.new(@boundary, body) + def add_json(body, content_id: nil) + header = { :content_id => content_id } + @parts << Google::Apis::Core::JsonPart.new(@boundary, body, header) self end @@ -147,9 +156,14 @@ Content-Transfer-Encoding: %s\r # # @param [Google::Apis::Core::UploadIO] upload_io # IO stream + # @param [String] content_id + # Optional unique ID of this part # @return [self] - def add_upload(upload_io) - @parts << Google::Apis::Core::FilePart.new(@boundary, upload_io) + def add_upload(upload_io, content_id: nil) + header = { :content_id => content_id } + @parts << Google::Apis::Core::FilePart.new(@boundary, + upload_io, + header) self end diff --git a/spec/google/apis/core/batch_spec.rb b/spec/google/apis/core/batch_spec.rb index 16ef7aa87..f784a03fa 100644 --- a/spec/google/apis/core/batch_spec.rb +++ b/spec/google/apis/core/batch_spec.rb @@ -42,9 +42,12 @@ RSpec.describe Google::Apis::Core::BatchCommand do end before(:example) do + allow(SecureRandom).to receive(:uuid).and_return('ffe23d1b-e8f7-47f5-8c01-2a30cf8ecb8f') + response = < HTTP/1.1 200 OK Content-Type: text/plain; charset=UTF-8 @@ -52,18 +55,20 @@ Content-Type: text/plain; charset=UTF-8 Hello --batch123 Content-Type: application/http - -HTTP/1.1 200 OK -Content-Type: text/plain; charset=UTF-8 - -world ---batch123 -Content-Type: application/http +Content-ID: HTTP/1.1 500 Server Error Content-Type: text/plain; charset=UTF-8 Error! +--batch123 +Content-Type: application/http +Content-ID: + +HTTP/1.1 200 OK +Content-Type: text/plain; charset=UTF-8 + +world --batch123-- EOF stub_request(:post, 'https://www.googleapis.com/batch') @@ -80,6 +85,7 @@ EOF expected_body = < Content-Type: application/http Content-Transfer-Encoding: binary @@ -89,6 +95,7 @@ Host: www.googleapis.com --RubyApiBatchRequest Content-Length: 96 +Content-ID: Content-Type: application/http Content-Transfer-Encoding: binary @@ -99,6 +106,7 @@ Host: www.googleapis.com Hello world --RubyApiBatchRequest Content-Length: 93 +Content-ID: Content-Type: application/http Content-Transfer-Encoding: binary @@ -115,11 +123,17 @@ EOF it 'should send decode responses' do expect do |b| - command.add(get_command, &b) - command.add(post_with_string_command, &b) - command.add(post_with_io_command, &b) + command.add(get_command) do |res, err| + b.to_proc.call(1, res, err) + end + command.add(post_with_string_command) do |res, err| + b.to_proc.call(2, res, err) + end + command.add(post_with_io_command) do |res, err| + b.to_proc.call(3, res, err) + end command.execute(client) - end.to yield_successive_args(['Hello', nil], ['world', nil], [nil, an_instance_of(Google::Apis::ServerError)]) + end.to yield_successive_args([1, 'Hello', nil], [3, nil, an_instance_of(Google::Apis::ServerError)], [2, 'world', nil],) end it 'should raise error if batch is empty' do