Adding batch support

This commit is contained in:
Sergio Gomes 2012-05-16 20:05:24 +01:00
parent c44ca3afc1
commit a666d52770
7 changed files with 691 additions and 39 deletions

View File

@ -29,6 +29,7 @@ Gem::Specification.new do |s|
s.add_runtime_dependency(%q<multi_json>, [">= 1.3.0"])
s.add_runtime_dependency(%q<extlib>, [">= 0.9.15"])
s.add_runtime_dependency(%q<launchy>, [">= 2.0.0"])
s.add_runtime_dependency(%q<uuidtools>, [">= 2.1.0"])
s.add_development_dependency(%q<sinatra>, [">= 1.2.0"])
s.add_development_dependency(%q<rake>, [">= 0.9.0"])
s.add_development_dependency(%q<rspec>, ["~> 1.2.9"])
@ -41,6 +42,7 @@ Gem::Specification.new do |s|
s.add_dependency(%q<multi_json>, [">= 1.3.0"])
s.add_dependency(%q<extlib>, [">= 0.9.15"])
s.add_dependency(%q<launchy>, [">= 2.0.0"])
s.add_dependency(%q<uuidtools>, [">= 2.1.0"])
s.add_dependency(%q<sinatra>, [">= 1.2.0"])
s.add_dependency(%q<rake>, [">= 0.9.0"])
s.add_dependency(%q<rspec>, ["~> 1.2.9"])
@ -54,6 +56,7 @@ Gem::Specification.new do |s|
s.add_dependency(%q<multi_json>, [">= 1.3.0"])
s.add_dependency(%q<extlib>, [">= 0.9.15"])
s.add_dependency(%q<launchy>, [">= 2.0.0"])
s.add_dependency(%q<uuidtools>, [">= 2.1.0"])
s.add_dependency(%q<sinatra>, [">= 1.2.0"])
s.add_dependency(%q<rake>, [">= 0.9.0"])
s.add_dependency(%q<rspec>, ["~> 1.2.9"])

View File

@ -26,6 +26,7 @@ require 'google/api_client/discovery'
require 'google/api_client/reference'
require 'google/api_client/result'
require 'google/api_client/media'
require 'google/api_client/batch'
module Google
# TODO(bobaman): Document all this stuff.
@ -663,22 +664,34 @@ module Google
##
# Executes a request, wrapping it in a Result object.
#
# @param [Google::APIClient::Method, String] api_method
# The method object or the RPC name of the method being executed.
# @param [Hash, Array] parameters
# The parameters to send to the method.
# @param [String] body The body of the request.
# @param [Hash, Array] headers The HTTP headers for the request.
# @option options [String] :version ("v1")
# The service version. Only used if `api_method` is a `String`.
# @option options [#generate_authenticated_request] :authorization
# The authorization mechanism for the response. Used only if
# `:authenticated` is `true`.
# @option options [TrueClass, FalseClass] :authenticated (true)
# `true` if the request must be signed or somehow
# authenticated, `false` otherwise.
# @param [Google::APIClient::BatchRequest, Hash, Array] params
# Either a Google::APIClient::BatchRequest, a Hash, or an Array.
#
# @return [Google::APIClient::Result] The result from the API.
# If a Google::APIClient::BatchRequest, no other parameters are expected.
#
# If a Hash, the below parameters are handled. If an Array, the
# parameters are assumed to be in the below order:
#
# - (Google::APIClient::Method, String) api_method:
# The method object or the RPC name of the method being executed.
# - (Hash, Array) parameters:
# The parameters to send to the method.
# - (String) body: The body of the request.
# - (Hash, Array) headers: The HTTP headers for the request.
# - (Hash) options: A set of options for the request, of which:
# - (String) :version (default: "v1") -
# The service version. Only used if `api_method` is a `String`.
# - (#generate_authenticated_request) :authorization (default: true) -
# The authorization mechanism for the response. Used only if
# `:authenticated` is `true`.
# - (TrueClass, FalseClass) :authenticated (default: true) -
# `true` if the request must be signed or somehow
# authenticated, `false` otherwise.
#
# @return [Google::APIClient::Result] The result from the API, nil if batch.
#
# @example
# result = client.execute(batch_request)
#
# @example
# result = client.execute(
@ -688,28 +701,64 @@ module Google
#
# @see Google::APIClient#generate_request
def execute(*params)
# This block of code allows us to accept multiple parameter passing
# styles, and maintaining some backwards compatibility.
#
# Note: I'm extremely tempted to deprecate this style of execute call.
if params.last.respond_to?(:to_hash) && params.size == 1
options = params.pop
else
options = {}
end
options[:api_method] = params.shift if params.size > 0
options[:parameters] = params.shift if params.size > 0
options[:body] = params.shift if params.size > 0
options[:headers] = params.shift if params.size > 0
options[:client] = self
if params.last.kind_of?(Google::APIClient::BatchRequest) &&
params.size == 1
batch = params.pop
options = batch.options
http_request = batch.to_http_request
request = nil
reference = Google::APIClient::Reference.new(options)
request = self.generate_request(reference)
response = self.transmit(
:request => request,
:connection => options[:connection]
)
return Google::APIClient::Result.new(reference, request, response)
if @authorization
method, uri, headers, body = http_request
method = method.to_s.downcase.to_sym
faraday_request = Faraday::Request.create(method) do |req|
req.url(uri.to_s)
req.headers = Faraday::Utils::Headers.new(headers)
req.body = body
end
request = {
:request => self.generate_authenticated_request(
:request => faraday_request,
:connection => options[:connection]
),
:connection => options[:connection]
}
else
request = {
:request => http_request,
:connection => options[:connection]
}
end
response = self.transmit(request)
batch.process_response(response)
return nil
else
# This block of code allows us to accept multiple parameter passing
# styles, and maintaining some backwards compatibility.
#
# Note: I'm extremely tempted to deprecate this style of execute call.
if params.last.respond_to?(:to_hash) && params.size == 1
options = params.pop
else
options = {}
end
options[:api_method] = params.shift if params.size > 0
options[:parameters] = params.shift if params.size > 0
options[:body] = params.shift if params.size > 0
options[:headers] = params.shift if params.size > 0
options[:client] = self
reference = Google::APIClient::Reference.new(options)
request = self.generate_request(reference)
response = self.transmit(
:request => request,
:connection => options[:connection]
)
return Google::APIClient::Result.new(reference, request, response)
end
end
##

View File

@ -0,0 +1,300 @@
# Copyright 2012 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
require 'addressable/uri'
require 'uuidtools'
module Google
class APIClient
# Error class for problems in batch requests.
class BatchError < StandardError
end
# Helper class to contain a response to an individual batched call.
class BatchedCallResponse
attr_reader :call_id
attr_accessor :status, :headers, :body
def initialize(call_id, status = nil, headers = nil, body = nil)
@call_id, @status, @headers, @body = call_id, status, headers, body
end
end
##
# Wraps multiple API calls into a single over-the-wire HTTP request.
class BatchRequest
BATCH_BOUNDARY = "-----------RubyApiBatchRequest".freeze
attr_accessor :options
attr_reader :calls, :callbacks
##
# Creates a new batch request.
#
# @param [Hash] options
# Set of options for this request, the only important one being
# :connection, which specifies an HTTP connection to use.
# @param [Proc] block
# Callback for every call's response. Won't be called if a call defined
# a callback of its own.
#
# @return [Google::APIClient::BatchRequest] The constructed object.
def initialize(options = {}, &block)
# Request options, ignoring method and parameters.
@options = options
# Batched calls to be made, indexed by call ID.
@calls = {}
# Callbacks per batched call, indexed by call ID.
@callbacks = {}
# Order for the call IDs, since Ruby 1.8 hashes are unordered.
@order = []
# Global callback to be used for every call. If a specific callback
# has been defined for a request, this won't be called.
@global_callback = block if block_given?
# The last auto generated ID.
@last_auto_id = 0
# Base ID for the batch request.
@base_id = nil
end
##
# Add a new call to the batch request.
# Each call must have its own call ID; if not provided, one will
# automatically be generated, avoiding collisions. If duplicate call IDs
# are provided, an error will be thrown.
#
# @param [Hash, Google::APIClient::Reference] call: the call to be added.
# @param [String] call_id: the ID to be used for this call. Must be unique
# @param [Proc] block: callback for this call's response.
#
# @return [Google::APIClient::BatchRequest] The BatchRequest, for chaining
def add(call, call_id = nil, &block)
unless call.kind_of?(Google::APIClient::Reference)
call = Google::APIClient::Reference.new(call)
end
if call_id.nil?
call_id = new_id
end
if @calls.include?(call_id)
raise BatchError,
'A call with this ID already exists: %s' % call_id
end
@calls[call_id] = call
@order << call_id
if block_given?
@callbacks[call_id] = block
elsif @global_callback
@callbacks[call_id] = @global_callback
end
return self
end
##
# Convert this batch request into an HTTP request.
#
# @return [Array<String, String, Hash, String>]
# An array consisting of, in order: HTTP method, request path, request
# headers and request body.
def to_http_request
return ['POST', request_uri, request_headers, request_body]
end
##
# Processes the HTTP response to the batch request, issuing callbacks.
#
# @param [Faraday::Response] response: the HTTP response.
def process_response(response)
content_type = find_header('Content-Type', response.headers)
boundary = /.*boundary=(.+)/.match(content_type)[1]
parts = response.body.split(/--#{Regexp.escape(boundary)}/)
parts = parts[1...-1]
parts.each do |part|
call_response = deserialize_call_response(part)
callback = @callbacks[call_response.call_id]
call = @calls[call_response.call_id]
result = Google::APIClient::Result.new(call, nil, call_response)
callback.call(result) if callback
end
end
private
##
# Helper method to find a header from its name, regardless of case.
#
# @param [String] name: The name of the header to find.
# @param [Hash] headers: The hash of headers and their values.
#
# @return [String] The value of the desired header.
def find_header(name, headers)
_, header = headers.detect do |h, v|
h.downcase == name.downcase
end
return header
end
##
# Create a new call ID. Uses an auto-incrementing, conflict-avoiding ID.
#
# @return [String] the new, unique ID.
def new_id
@last_auto_id += 1
while @calls.include?(@last_auto_id)
@last_auto_id += 1
end
return @last_auto_id.to_s
end
##
# Convert an id to a Content-ID header value.
#
# @param [String] call_id: identifier of individual call.
#
# @return [String]
# A Content-ID header with the call_id encoded into it. A UUID is
# prepended to the value because Content-ID headers are supposed to be
# universally unique.
def id_to_header(call_id)
if @base_id.nil?
# TODO(sgomes): Use SecureRandom.uuid, drop UUIDTools when we drop 1.8
@base_id = UUIDTools::UUID.random_create.to_s
end
return '<%s+%s>' % [@base_id, Addressable::URI.encode(call_id)]
end
##
# Convert a Content-ID header value to an id. Presumes the Content-ID
# header conforms to the format that id_to_header() returns.
#
# @param [String] header: Content-ID header value.
#
# @return [String] The extracted ID value.
def header_to_id(header)
if !header.start_with?('<') || !header.end_with?('>') ||
!header.include?('+')
raise BatchError, 'Invalid value for Content-ID: "%s"' % header
end
base, call_id = header[1...-1].split('+')
return Addressable::URI.unencode(call_id)
end
##
# Convert a single batched call into a string.
#
# @param [Google::APIClient::Reference] call: the call to serialize.
#
# @return [String] The request as a string in application/http format.
def serialize_call(call)
http_request = call.to_request
method = http_request.method.to_s.upcase
path = http_request.path.to_s
status_line = method + " " + path + " HTTP/1.1"
serialized_call = status_line
if http_request.headers
http_request.headers.each do |header, value|
serialized_call << "\r\n%s: %s" % [header, value]
end
end
if http_request.body
serialized_call << "\r\n\r\n"
serialized_call << http_request.body
end
return serialized_call
end
##
# Auxiliary method to split the headers from the body in an HTTP response.
#
# @param [String] response: the response to parse.
#
# @return [Array<Hash>, String] The headers and the body, separately.
def split_headers_and_body(response)
headers = {}
payload = response.lstrip
while payload
line, payload = payload.split("\n", 2)
line.sub!(/\s+\z/, '')
break if line.empty?
match = /\A([^:]+):\s*/.match(line)
if match
headers[match[1]] = match.post_match
else
raise BatchError, 'Invalid header line in response: %s' % line
end
end
return headers, payload
end
##
# Convert a single batched response into a BatchedCallResponse object.
#
# @param [Google::APIClient::Reference] response:
# the request to deserialize.
#
# @return [BatchedCallResponse] The parsed and converted response.
def deserialize_call_response(call_response)
outer_headers, outer_body = split_headers_and_body(call_response)
status_line, payload = outer_body.split("\n", 2)
protocol, status, reason = status_line.split(' ', 3)
headers, body = split_headers_and_body(payload)
content_id = find_header('Content-ID', outer_headers)
call_id = header_to_id(content_id)
return BatchedCallResponse.new(call_id, status.to_i, headers, body)
end
##
# Return the request headers for the BatchRequest's HTTP request.
#
# @return [Hash] The HTTP headers.
def request_headers
return {
'Content-Type' => 'multipart/mixed; boundary=%s' % BATCH_BOUNDARY
}
end
##
# Return the request path for the BatchRequest's HTTP request.
#
# @return [String] The request path.
def request_uri
if @calls.nil? || @calls.empty?
raise BatchError, 'Cannot make an empty batch request'
end
# All APIs have the same batch path, so just get the first one.
return @calls.first[1].api_method.api.batch_path
end
##
# Return the request body for the BatchRequest's HTTP request.
#
# @return [String] The request body.
def request_body
body = ""
@order.each do |call_id|
body << "--" + BATCH_BOUNDARY + "\r\n"
body << "Content-Type: application/http\r\n"
body << "Content-ID: %s\r\n\r\n" % id_to_header(call_id)
body << serialize_call(@calls[call_id]) + "\r\n\r\n"
end
body << "--" + BATCH_BOUNDARY + "--"
return body
end
end
end
end

View File

@ -171,6 +171,21 @@ module Google
end
end
##
# Returns the base URI for batch calls to this service.
#
# @return [Addressable::URI] The base URI that methods are joined to.
def batch_path
if @discovery_document['batchPath']
return @batch_path ||= (
self.document_base.join(Addressable::URI.parse('/' +
@discovery_document['batchPath']))
).normalize
else
return nil
end
end
##
# A list of schemas available for this version of the API.
#

View File

@ -28,6 +28,8 @@ module Google
##
# Creates a description of a particular method.
#
# @param [Google::APIClient::API] api
# The API this method belongs to.
# @param [Addressable::URI] method_base
# The base URI for the service.
# @param [String] method_name
@ -43,6 +45,12 @@ module Google
@discovery_document = discovery_document
end
##
# Returns the API this method belongs to.
#
# @return [Google::APIClient::API] The API this method belongs to.
attr_reader :api
##
# Returns the identifier for the method.
#
@ -111,7 +119,7 @@ module Google
return nil
end
end
##
# Returns the Schema object for the method's request, if any.
#
@ -183,7 +191,7 @@ module Google
end
case upload_type.last
when 'media', 'multipart', 'resumable'
uri = self.media_upload.uri_template.expand(parameters)
uri = self.media_upload.uri_template.expand(parameters)
else
raise ArgumentException, "Invalid uploadType '#{upload_type}'"
end
@ -232,7 +240,7 @@ module Google
req.body = body
end
end
##
# Returns a <code>Hash</code> of the parameter descriptions for

View File

@ -0,0 +1,237 @@
# Copyright 2012 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
require 'spec_helper'
require 'google/api_client'
require 'google/api_client/version'
describe Google::APIClient::BatchRequest do
before do
@client = Google::APIClient.new
end
it 'should raise an error if making an empty batch request' do
batch = Google::APIClient::BatchRequest.new
(lambda do
@client.execute(batch)
end).should raise_error(Google::APIClient::BatchError)
end
describe 'with the discovery API' do
before do
@client.authorization = nil
@discovery = @client.discovered_api('discovery', 'v1')
end
describe 'with two valid requests' do
before do
@call1 = {
:api_method => @discovery.apis.get_rest,
:parameters => {
'api' => 'adsense',
'version' => 'v1'
}
}
@call2 = {
:api_method => @discovery.apis.get_rest,
:parameters => {
'api' => 'discovery',
'version' => 'v1'
}
}
end
it 'should execute both when using a global callback' do
block_called = 0
ids = ['first_call', 'second_call']
expected_ids = ids.clone
batch = Google::APIClient::BatchRequest.new do |result|
block_called += 1
result.status.should == 200
expected_ids.should include(result.response.call_id)
expected_ids.delete(result.response.call_id)
end
batch.add(@call1, ids[0])
batch.add(@call2, ids[1])
@client.execute(batch)
block_called.should == 2
end
it 'should execute both when using individual callbacks' do
batch = Google::APIClient::BatchRequest.new
call1_returned, call2_returned = false, false
batch.add(@call1) do |result|
call1_returned = true
result.status.should == 200
end
batch.add(@call2) do |result|
call2_returned = true
result.status.should == 200
end
@client.execute(batch)
call1_returned.should == true
call2_returned.should == true
end
it 'should raise an error if using the same call ID more than once' do
batch = Google::APIClient::BatchRequest.new
(lambda do
batch.add(@call1, 'my_id')
batch.add(@call2, 'my_id')
end).should raise_error(Google::APIClient::BatchError)
end
end
describe 'with a valid request and an invalid one' do
before do
@call1 = {
:api_method => @discovery.apis.get_rest,
:parameters => {
'api' => 'adsense',
'version' => 'v1'
}
}
@call2 = {
:api_method => @discovery.apis.get_rest,
:parameters => {
'api' => 0,
'version' => 1
}
}
end
it 'should execute both when using a global callback' do
block_called = 0
ids = ['first_call', 'second_call']
expected_ids = ids.clone
batch = Google::APIClient::BatchRequest.new do |result|
block_called += 1
expected_ids.should include(result.response.call_id)
expected_ids.delete(result.response.call_id)
if result.response.call_id == ids[0]
result.status.should == 200
else
result.status.should >= 400
result.status.should < 500
end
end
batch.add(@call1, ids[0])
batch.add(@call2, ids[1])
@client.execute(batch)
block_called.should == 2
end
it 'should execute both when using individual callbacks' do
batch = Google::APIClient::BatchRequest.new
call1_returned, call2_returned = false, false
batch.add(@call1) do |result|
call1_returned = true
result.status.should == 200
end
batch.add(@call2) do |result|
call2_returned = true
result.status.should >= 400
result.status.should < 500
end
@client.execute(batch)
call1_returned.should == true
call2_returned.should == true
end
end
end
describe 'with the calendar API' do
before do
@client.authorization = nil
@calendar = @client.discovered_api('calendar', 'v3')
end
describe 'with two valid requests' do
before do
event1 = {
'summary' => 'Appointment 1',
'location' => 'Somewhere',
'start' => {
'dateTime' => '2011-01-01T10:00:00.000-07:00'
},
'end' => {
'dateTime' => '2011-01-01T10:25:00.000-07:00'
},
'attendees' => [
{
'email' => 'myemail@mydomain.tld'
}
]
}
event2 = {
'summary' => 'Appointment 2',
'location' => 'Somewhere as well',
'start' => {
'dateTime' => '2011-01-02T10:00:00.000-07:00'
},
'end' => {
'dateTime' => '2011-01-02T10:25:00.000-07:00'
},
'attendees' => [
{
'email' => 'myemail@mydomain.tld'
}
]
}
@call1 = {
:api_method => @calendar.events.insert,
:parameters => {'calendarId' => 'myemail@mydomain.tld'},
:body => JSON.dump(event1),
:headers => {'Content-Type' => 'application/json'}
}
@call2 = {
:api_method => @calendar.events.insert,
:parameters => {'calendarId' => 'myemail@mydomain.tld'},
:body => JSON.dump(event2),
:headers => {'Content-Type' => 'application/json'}
}
end
it 'should convert to a correct HTTP request' do
batch = Google::APIClient::BatchRequest.new { |result| }
batch.add(@call1, '1').add(@call2, '2')
method, uri, headers, body = batch.to_http_request
boundary = Google::APIClient::BatchRequest::BATCH_BOUNDARY
method.to_s.downcase.should == 'post'
uri.to_s.should == 'https://www.googleapis.com/batch'
headers.should == {
"Content-Type"=>"multipart/mixed; boundary=#{boundary}"
}
expected_body = /--#{Regexp.escape(boundary)}\nContent-Type: +application\/http\nContent-ID: +<[\w-]+\+1>\n\nPOST +https:\/\/www.googleapis.com\/calendar\/v3\/calendars\/myemail@mydomain.tld\/events +HTTP\/1.1\nContent-Type: +application\/json\n\n#{Regexp.escape(@call1[:body])}\n\n--#{boundary}\nContent-Type: +application\/http\nContent-ID: +<[\w-]+\+2>\n\nPOST +https:\/\/www.googleapis.com\/calendar\/v3\/calendars\/myemail@mydomain.tld\/events HTTP\/1.1\nContent-Type: +application\/json\n\n#{Regexp.escape(@call2[:body])}\n\n--#{Regexp.escape(boundary)}--/
body.gsub("\r", "").should =~ expected_body
end
end
end
end

View File

@ -142,6 +142,12 @@ describe Google::APIClient do
).name.should == 'delete'
end
it 'should define the origin API in discovered methods' do
@client.discovered_method(
'prediction.training.insert', 'prediction', 'v1.2'
).api.name.should == 'prediction'
end
it 'should not find methods that are not in the discovery document' do
@client.discovered_method(
'prediction.bogus', 'prediction', 'v1.2'
@ -165,6 +171,10 @@ describe Google::APIClient do
@client.preferred_version(:prediction).version.should_not == 'v1'
end
it 'should return a batch path' do
@client.discovered_api('prediction', 'v1.2').batch_path.should_not be_nil
end
it 'should generate valid requests' do
request = @client.generate_request(
:api_method => @prediction.training.insert,
@ -324,6 +334,12 @@ describe Google::APIClient do
).name.should == 'list'
end
it 'should define the origin API in discovered methods' do
@client.discovered_method(
'plus.activities.list', 'plus'
).api.name.should == 'plus'
end
it 'should not find methods that are not in the discovery document' do
@client.discovered_method('plus.bogus', 'plus').should == nil
end
@ -381,12 +397,22 @@ describe Google::APIClient do
@client.discovered_api('latitude').version.should == 'v1'
end
it 'should return a batch path' do
@client.discovered_api('latitude').batch_path.should_not be_nil
end
it 'should find methods that are in the discovery document' do
@client.discovered_method(
'latitude.currentLocation.get', 'latitude'
).name.should == 'get'
end
it 'should define the origin API in discovered methods' do
@client.discovered_method(
'latitude.currentLocation.get', 'latitude'
).api.name.should == 'latitude'
end
it 'should not find methods that are not in the discovery document' do
@client.discovered_method('latitude.bogus', 'latitude').should == nil
end
@ -440,10 +466,20 @@ describe Google::APIClient do
).name.should == 'get'
end
it 'should define the origin API in discovered methods' do
@client.discovered_method(
'moderator.profiles.get', 'moderator'
).api.name.should == 'moderator'
end
it 'should not find methods that are not in the discovery document' do
@client.discovered_method('moderator.bogus', 'moderator').should == nil
end
it 'should return a batch path' do
@client.discovered_api('moderator').batch_path.should_not be_nil
end
it 'should generate requests against the correct URIs' do
request = @client.generate_request(
:api_method => 'moderator.profiles.get',
@ -490,6 +526,10 @@ describe Google::APIClient do
@client.discovered_api('adsense').version.should == 'v1'
end
it 'should return a batch path' do
@client.discovered_api('adsense').batch_path.should_not be_nil
end
it 'should find methods that are in the discovery document' do
@client.discovered_method(
'adsense.reports.generate', 'adsense'