From b8301b0dd57e3a77871af3292bbe275ae92d5d3f Mon Sep 17 00:00:00 2001 From: Steven Bazyl Date: Thu, 1 Mar 2012 17:23:36 -0800 Subject: [PATCH] Media upload support --- lib/google/api_client.rb | 18 ++- lib/google/api_client/discovery/api.rb | 5 +- lib/google/api_client/discovery/media.rb | 77 ++++++++++ lib/google/api_client/discovery/method.rb | 34 ++++- lib/google/api_client/media.rb | 172 ++++++++++++++++++++++ lib/google/api_client/reference.rb | 82 +++++++++-- lib/google/api_client/result.rb | 20 ++- 7 files changed, 372 insertions(+), 36 deletions(-) create mode 100644 lib/google/api_client/discovery/media.rb create mode 100644 lib/google/api_client/media.rb diff --git a/lib/google/api_client.rb b/lib/google/api_client.rb index 090dda26a..ba7c1f7b7 100644 --- a/lib/google/api_client.rb +++ b/lib/google/api_client.rb @@ -25,7 +25,7 @@ require 'google/api_client/environment' require 'google/api_client/discovery' require 'google/api_client/reference' require 'google/api_client/result' - +require 'google/api_client/media' module Google # TODO(bobaman): Document all this stuff. @@ -679,13 +679,15 @@ module Google # @see Google::APIClient#execute def execute!(*params) result = self.execute(*params) - if result.data.respond_to?(:error) && - result.data.error.respond_to?(:message) - # You're going to get a terrible error message if the response isn't - # parsed successfully as an error. - error_message = result.data.error.message - elsif result.data['error'] && result.data['error']['message'] - error_message = result.data['error']['message'] + if result.data? + if result.data.respond_to?(:error) && + result.data.error.respond_to?(:message) + # You're going to get a terrible error message if the response isn't + # parsed successfully as an error. + error_message = result.data.error.message + elsif result.data['error'] && result.data['error']['message'] + error_message = result.data['error']['message'] + end end if result.response.status >= 400 case result.response.status diff --git a/lib/google/api_client/discovery/api.rb b/lib/google/api_client/discovery/api.rb index 9979ed79d..db5e97ea7 100644 --- a/lib/google/api_client/discovery/api.rb +++ b/lib/google/api_client/discovery/api.rb @@ -18,7 +18,7 @@ require 'addressable/uri' require 'google/inflection' require 'google/api_client/discovery/resource' require 'google/api_client/discovery/method' - +require 'google/api_client/discovery/media' module Google class APIClient @@ -149,8 +149,7 @@ module Google def method_base if @discovery_document['basePath'] return @method_base ||= ( - self.document_base + - Addressable::URI.parse(@discovery_document['basePath']) + self.document_base.join(Addressable::URI.parse(@discovery_document['basePath'])) ).normalize else return nil diff --git a/lib/google/api_client/discovery/media.rb b/lib/google/api_client/discovery/media.rb new file mode 100644 index 000000000..34408dc63 --- /dev/null +++ b/lib/google/api_client/discovery/media.rb @@ -0,0 +1,77 @@ +# Copyright 2010 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +require 'addressable/uri' +require 'addressable/template' + +require 'google/api_client/errors' + + +module Google + class APIClient + ## + # Media upload elements for discovered methods + class MediaUpload + + ## + # Creates a description of a particular method. + # + # @param [Google::APIClient::API] api + # Base discovery document for the API + # @param [Addressable::URI] method_base + # The base URI for the service. + # @param [Hash] discovery_document + # The media upload section of the discovery document. + # + # @return [Google::APIClient::Method] The constructed method object. + def initialize(api, method_base, discovery_document) + @api = api + @method_base = method_base + @discovery_document = discovery_document + end + + ## + # List of acceptable mime types + # + # @return [Array] + # List of acceptable mime types for uploaded content + def accepted_types + @discovery_document['accept'] + end + + ## + # Maximum size of an uplad + # TODO: Parse & convert to numeric value + # + # @return [String] + def max_size + @discovery_document['maxSize'] + end + + ## + # Returns the URI template for the method. A parameter list can be + # used to expand this into a URI. + # + # @return [Addressable::Template] The URI template. + def uri_template + return @uri_template ||= Addressable::Template.new( + @api.method_base.join(Addressable::URI.parse(@discovery_document['protocols']['simple']['path'])) + ) + end + + end + + end +end diff --git a/lib/google/api_client/discovery/method.rb b/lib/google/api_client/discovery/method.rb index 8f0198507..8a66e3476 100644 --- a/lib/google/api_client/discovery/method.rb +++ b/lib/google/api_client/discovery/method.rb @@ -95,15 +95,23 @@ module Google # # @return [Addressable::Template] The URI template. def uri_template - # TODO(bobaman) We shouldn't be calling #to_s here, this should be - # a join operation on a URI, but we have to treat these as Strings - # because of the way the discovery document provides the URIs. - # This should be fixed soon. return @uri_template ||= Addressable::Template.new( - self.method_base + @discovery_document['path'] + self.method_base.join(Addressable::URI.parse(@discovery_document['path'])) ) end + ## + # Returns media upload information for this method, if supported + # + # @return [Google::APIClient::MediaUpload] Description of upload endpoints + def media_upload + if @discovery_document['mediaUpload'] + return @media_upload ||= Google::APIClient::MediaUpload.new(self, self.method_base, @discovery_document['mediaUpload']) + else + return nil + end + end + ## # Returns the Schema object for the method's request, if any. # @@ -168,7 +176,20 @@ module Google parameters = self.normalize_parameters(parameters) self.validate_parameters(parameters) template_variables = self.uri_template.variables - uri = self.uri_template.expand(parameters) + upload_type = parameters.assoc('uploadType') || parameters.assoc('upload_type') + if upload_type + unless self.media_upload + raise ArgumentException, "Media upload not supported for this method" + end + case upload_type.last + when 'media', 'multipart', 'resumable' + uri = self.media_upload.uri_template.expand(parameters) + else + raise ArgumentException, "Invalid uploadType '#{upload_type}'" + end + else + uri = self.uri_template.expand(parameters) + end query_parameters = parameters.reject do |k, v| template_variables.include?(k) end @@ -211,6 +232,7 @@ module Google req.body = body end end + ## # Returns a Hash of the parameter descriptions for diff --git a/lib/google/api_client/media.rb b/lib/google/api_client/media.rb new file mode 100644 index 000000000..e1e3ad549 --- /dev/null +++ b/lib/google/api_client/media.rb @@ -0,0 +1,172 @@ +# Copyright 2010 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module Google + class APIClient + ## + # Uploadable media support. Holds an IO stream & content type. + # + # @see Faraday::UploadIO + # @example + # media = Google::APIClient::UploadIO.new('mymovie.m4v', 'video/mp4') + class UploadIO < Faraday::UploadIO + ## + # Get the length of the stream + # @return [Integer] + # Length of stream, in bytes + def length + io.respond_to?(:length) ? io.length : File.size(local_path) + end + end + + ## + # Resumable uploader. + # + class ResumableUpload + attr_reader :result + attr_accessor :client + attr_accessor :chunk_size + attr_accessor :media + attr_accessor :location + + ## + # Creates a new uploader. + # + # @param [Google::APIClient::Result] result + # Result of the initial request that started the upload + # @param [Google::APIClient::UploadIO] media + # Media to upload + # @param [String] location + # URL to upload to + def initialize(result, media, location) + self.media = media + self.location = location + self.chunk_size = 256 * 1024 + + @api_method = result.reference.api_method + @result = result + @offset = 0 + @complete = false + end + + ## + # Sends all remaining chunks to the server + # + # @param [Google::APIClient] api_client + # API Client instance to use for sending + def send_all(api_client) + until complete? + send_chunk(api_client) + break unless result.status == 308 + end + return result + end + + + ## + # Sends the next chunk to the server + # + # @param [Google::APIClient] api_client + # API Client instance to use for sending + def send_chunk(api_client) + if @offset.nil? + return resync_range(api_client) + end + + start_offset = @offset + self.media.io.pos = start_offset + chunk = self.media.io.read(chunk_size) + content_length = chunk.bytesize + + end_offset = start_offset + content_length - 1 + @result = api_client.execute( + :uri => self.location, + :http_method => :put, + :headers => { + 'Content-Length' => "#{content_length}", + 'Content-Type' => self.media.content_type, + 'Content-Range' => "bytes #{start_offset}-#{end_offset}/#{media.length}" }, + :body => chunk) + return process_result(@result) + end + + ## + # Check if upload is complete + # + # @return [TrueClass, FalseClass] + # Whether or not the upload complete successfully + def complete? + return @complete + end + + ## + # Check if the upload URL expired (upload not completed in alotted time.) + # Expired uploads must be restarted from the beginning + # + # @return [TrueClass, FalseClass] + # Whether or not the upload has expired and can not be resumed + def expired? + return @result.status == 404 || @result.status == 410 + end + + ## + # Get the last saved range from the server in case an error occurred + # and the offset is not known. + # + # @param [Google::APIClient] api_client + # API Client instance to use for sending + def resync_range(api_client) + r = api_client.execute( + :uri => self.location, + :http_method => :put, + :headers => { + 'Content-Length' => "0", + 'Content-Range' => "bytes */#{media.length}" }) + return process_result(r) + end + + ## + # Check the result from the server, updating the offset and/or location + # if available. + # + # @param [Google::APIClient::Result] r + # Result of a chunk upload or range query + def process_result(result) + case result.status + when 200...299 + @complete = true + if @api_method + # Inject the original API method so data is parsed correctly + result.reference.api_method = @api_method + end + return result + when 308 + range = result.headers['range'] + if range + @offset = range.scan(/\d+/).collect{|x| Integer(x)}.last + 1 + end + if result.headers['location'] + self.location = result.headers['location'] + end + when 500...599 + # Invalidate the offset to mark it needs to be queried on the + # next request + @offset = nil + end + return nil + end + + end + end +end \ No newline at end of file diff --git a/lib/google/api_client/reference.rb b/lib/google/api_client/reference.rb index 4c67bca1e..8575314fa 100644 --- a/lib/google/api_client/reference.rb +++ b/lib/google/api_client/reference.rb @@ -25,6 +25,8 @@ require 'google/api_client/discovery' module Google class APIClient class Reference + + MULTIPART_BOUNDARY = "-----------RubyApiMultipartPost".freeze def initialize(options={}) # We only need this to do lookups on method ID String values # It's optional, but method ID lookups will fail if the client is @@ -39,20 +41,53 @@ module Google # parameters to the API method, but rather to the API system. self.parameters['key'] ||= options[:key] if options[:key] self.parameters['userIp'] ||= options[:user_ip] if options[:user_ip] - self.headers = options[:headers] || [] - if options[:body] + self.headers = options[:headers] || {} + + if options[:media] + self.media = options[:media] + upload_type = parameters['uploadType'] || parameters['upload_type'] + case upload_type + when "media" + if options[:body] || options[:body_object] + raise ArgumentError, "Can not specify body & body object for simple uploads" + end + self.headers['Content-Type'] ||= self.media.content_type + self.body = self.media + when "multipart" + unless options[:body_object] + raise ArgumentError, "Multipart requested but no body object" + end + # This is all a bit of a hack due to signet requiring body to be a string + # Ideally, update signet to delay serialization so we can just pass + # streams all the way down through to the HTTP lib + metadata = StringIO.new(serialize_body(options[:body_object])) + env = { + :request_headers => {'Content-Type' => "multipart/related;boundary=#{MULTIPART_BOUNDARY}"}, + :request => { :boundary => MULTIPART_BOUNDARY } + } + multipart = Faraday::Request::Multipart.new + self.body = multipart.create_multipart(env, { + :metadata => Faraday::UploadIO.new(metadata, 'application/json'), + :content => self.media}) + self.headers.update(env[:request_headers]) + when "resumable" + file_length = self.media.length + self.headers['X-Upload-Content-Type'] = self.media.content_type + self.headers['X-Upload-Content-Length'] = file_length.to_s + if options[:body_object] + self.headers['Content-Type'] ||= 'application/json' + self.body = serialize_body(options[:body_object]) + else + self.body = '' + end + else + raise ArgumentError, "Invalid uploadType for media" + end + elsif options[:body] self.body = options[:body] elsif options[:body_object] - if options[:body_object].respond_to?(:to_json) - serialized_body = options[:body_object].to_json - elsif options[:body_object].respond_to?(:to_hash) - serialized_body = MultiJson.encode(options[:body_object].to_hash) - else - raise TypeError, - 'Could not convert body object to JSON.' + - 'Must respond to :to_json or :to_hash.' - end - self.body = serialized_body + self.headers['Content-Type'] ||= 'application/json' + self.body = serialize_body(options[:body_object]) else self.body = '' end @@ -65,7 +100,22 @@ module Google end end end - + + def serialize_body(body) + return body.to_json if body.respond_to?(:to_json) + return MultiJson.encode(options[:body_object].to_hash) if body.respond_to?(:to_hash) + raise TypeError, 'Could not convert body object to JSON.' + + 'Must respond to :to_json or :to_hash.' + end + + def media + return @media + end + + def media=(media) + @media = (media) + end + def connection return @connection end @@ -132,18 +182,20 @@ module Google def body=(new_body) if new_body.respond_to?(:to_str) @body = new_body.to_str + elsif new_body.respond_to?(:read) + @body = new_body.read() elsif new_body.respond_to?(:inject) @body = (new_body.inject(StringIO.new) do |accu, chunk| accu.write(chunk) accu end).string else - raise TypeError, "Expected body to be String or Enumerable chunks." + raise TypeError, "Expected body to be String, IO, or Enumerable chunks." end end def headers - return @headers ||= [] + return @headers ||= {} end def headers=(new_headers) diff --git a/lib/google/api_client/result.rb b/lib/google/api_client/result.rb index 05fa3c7b6..de1bf6014 100644 --- a/lib/google/api_client/result.rb +++ b/lib/google/api_client/result.rb @@ -42,12 +42,24 @@ module Google return @response.body end + def resumable_upload + @media_upload ||= Google::APIClient::ResumableUpload.new(self, reference.media, self.headers['location']) + end + + def media_type + _, content_type = self.headers.detect do |h, v| + h.downcase == 'Content-Type'.downcase + end + content_type[/^([^;]*);?.*$/, 1].strip.downcase + end + + def data? + self.media_type == 'application/json' + end + def data return @data ||= (begin - _, content_type = self.headers.detect do |h, v| - h.downcase == 'Content-Type'.downcase - end - media_type = content_type[/^([^;]*);?.*$/, 1].strip.downcase + media_type = self.media_type data = self.body case media_type when 'application/json'