From 2046e00f14d2924880782dcb6537fad1a7f3fc05 Mon Sep 17 00:00:00 2001 From: Steve Bazyl Date: Wed, 17 Aug 2016 13:51:09 -0700 Subject: [PATCH] Remove Hurley as a dependency --- google-api-client.gemspec | 3 +- lib/google/apis/core/api_command.rb | 4 +- lib/google/apis/core/base_service.rb | 36 +++-- lib/google/apis/core/batch.rb | 31 ++--- lib/google/apis/core/composite_io.rb | 97 ++++++++++++++ lib/google/apis/core/download.rb | 36 ++--- lib/google/apis/core/http_client_adapter.rb | 82 ------------ lib/google/apis/core/http_command.rb | 73 ++++++----- lib/google/apis/core/multipart.rb | 138 ++++++-------------- lib/google/apis/core/upload.rb | 123 +++++++---------- spec/google/apis/core/api_command_spec.rb | 1 - spec/google/apis/core/batch_spec.rb | 26 ++-- spec/google/apis/core/composite_io_spec.rb | 76 +++++++++++ spec/google/apis/core/download_spec.rb | 1 - spec/google/apis/core/http_command_spec.rb | 1 - spec/google/apis/core/service_spec.rb | 42 +++++- spec/google/apis/core/upload_spec.rb | 73 +---------- third_party/hurley_patches.rb | 103 --------------- 18 files changed, 417 insertions(+), 529 deletions(-) create mode 100644 lib/google/apis/core/composite_io.rb delete mode 100644 lib/google/apis/core/http_client_adapter.rb create mode 100644 spec/google/apis/core/composite_io_spec.rb delete mode 100644 third_party/hurley_patches.rb diff --git a/google-api-client.gemspec b/google-api-client.gemspec index 707b750c7..ce6330bb6 100644 --- a/google-api-client.gemspec +++ b/google-api-client.gemspec @@ -24,9 +24,8 @@ Gem::Specification.new do |spec| spec.add_runtime_dependency 'retriable', '~> 2.0' spec.add_runtime_dependency 'addressable', '~> 2.3' spec.add_runtime_dependency 'mime-types', '>= 1.6' - spec.add_runtime_dependency 'hurley', '~> 0.1' spec.add_runtime_dependency 'googleauth', '~> 0.5' spec.add_runtime_dependency 'thor', '~> 0.19' - spec.add_runtime_dependency 'httpclient', '~> 2.7' + spec.add_runtime_dependency 'httpclient', '>= 2.8.1', '< 3.0' spec.add_runtime_dependency 'memoist', '~> 0.11' end diff --git a/lib/google/apis/core/api_command.rb b/lib/google/apis/core/api_command.rb index 519c0c97e..e76e3512c 100644 --- a/lib/google/apis/core/api_command.rb +++ b/lib/google/apis/core/api_command.rb @@ -50,7 +50,7 @@ module Google def prepare! query[FIELDS_PARAM] = normalize_fields_param(query[FIELDS_PARAM]) if query.key?(FIELDS_PARAM) if request_representation && request_object - header[:content_type] ||= JSON_CONTENT_TYPE + header['Content-Type'] ||= JSON_CONTENT_TYPE self.body = request_representation.new(request_object).to_json(skip_undefined: true) end super @@ -78,7 +78,7 @@ module Google # # @param [Fixnum] status # HTTP status code of response - # @param [Hurley::Header] header + # @param [Hash] header # HTTP response headers # @param [String] body # HTTP response body diff --git a/lib/google/apis/core/base_service.rb b/lib/google/apis/core/base_service.rb index 9065a389a..93699fafa 100644 --- a/lib/google/apis/core/base_service.rb +++ b/lib/google/apis/core/base_service.rb @@ -19,11 +19,8 @@ require 'google/apis/core/api_command' require 'google/apis/core/batch' require 'google/apis/core/upload' require 'google/apis/core/download' -require 'google/apis/core/http_client_adapter' require 'google/apis/options' require 'googleauth' -require 'hurley' -require 'hurley/addressable' module Google module Apis @@ -96,7 +93,7 @@ module Google attr_accessor :batch_path # HTTP client - # @return [Hurley::Client] + # @return [HTTPClient] attr_accessor :client # General settings @@ -198,7 +195,7 @@ module Google end # Get the current HTTP client - # @return [Hurley::Client] + # @return [HTTPClient] def client @client ||= new_client end @@ -368,19 +365,30 @@ module Google end # Create a new HTTP client - # @return [Hurley::Client] + # @return [HTTPClient] def new_client - client = Hurley::Client.new - client.connection = Google::Apis::Core::HttpClientAdapter.new unless client_options.use_net_http - client.request_options.timeout = request_options.timeout_sec - client.request_options.open_timeout = request_options.open_timeout_sec - client.request_options.proxy = client_options.proxy_url - client.request_options.query_class = Hurley::Query::Flat - client.ssl_options.ca_file = File.join(Google::Apis::ROOT, 'lib', 'cacerts.pem') - client.header[:user_agent] = user_agent + client = ::HTTPClient.new + + client.transparent_gzip_decompression = true + + client.proxy = client_options.proxy_url if client_options.proxy_url + + if request_options.timeout_sec + client.connect_timeout = request_options.timeout_sec + client.receive_timeout = request_options.timeout_sec + client.send_timeout = request_options.timeout_sec + end + + if request_options.open_timeout_sec + client.connect_timeout = request_options.open_timeout_sec + client.send_timeout = request_options.open_timeout_sec + end + client.follow_redirect_count = 5 + client.default_header = { 'User-Agent' => user_agent } client end + # Build the user agent header # @return [String] def user_agent diff --git a/lib/google/apis/core/batch.rb b/lib/google/apis/core/batch.rb index e95ce68ce..af60ce649 100644 --- a/lib/google/apis/core/batch.rb +++ b/lib/google/apis/core/batch.rb @@ -25,19 +25,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -require 'hurley' require 'google/apis/core/multipart' require 'google/apis/core/http_command' require 'google/apis/core/upload' require 'google/apis/core/download' +require 'google/apis/core/composite_io' require 'addressable/uri' require 'securerandom' + module Google module Apis module Core # Wrapper request for batching multiple calls in a single server request class BatchCommand < HttpCommand - BATCH_BOUNDARY = 'RubyApiBatchRequest'.freeze MULTIPART_MIXED = 'multipart/mixed' # @param [symbol] method @@ -81,7 +81,7 @@ module Google parts.each_index do |index| response = deserializer.to_http_response(parts[index]) outer_header = response.shift - call_id = header_to_id(outer_header[:content_id]) || index + call_id = header_to_id(outer_header['Content-ID'].first) || index call, callback = @calls[call_id] begin result = call.process_response(*response) unless call.nil? @@ -106,17 +106,16 @@ module Google fail BatchError, 'Cannot make an empty batch request' if @calls.empty? serializer = CallSerializer.new - multipart = Multipart.new(boundary: BATCH_BOUNDARY, content_type: MULTIPART_MIXED) + multipart = Multipart.new(content_type: MULTIPART_MIXED) @calls.each_index do |index| call, _ = @calls[index] content_id = id_to_header(index) - io = serializer.to_upload_io(call) - multipart.add_upload(io, content_id: content_id) + io = serializer.to_part(call) + multipart.add_upload(io, content_type: 'application/http', content_id: content_id) end self.body = multipart.assemble - header[:content_type] = multipart.content_type - header[:content_length] = "#{body.length}" + header['Content-Type'] = multipart.content_type super end @@ -155,24 +154,20 @@ module Google # Serializes a command for embedding in a multipart batch request # @private class CallSerializer - HTTP_CONTENT_TYPE = 'application/http' - ## # Serialize a single batched call for assembling the multipart message # # @param [Google::Apis::Core::HttpCommand] call # the call to serialize. - # @return [Hurley::UploadIO] + # @return [IO] # the serialized request - def to_upload_io(call) + def to_part(call) call.prepare! parts = [] parts << build_head(call) parts << build_body(call) unless call.body.nil? length = parts.inject(0) { |a, e| a + e.length } - Hurley::UploadIO.new(Hurley::CompositeReadIO.new(length, *parts), - HTTP_CONTENT_TYPE, - 'ruby-api-request') + Google::Apis::Core::CompositeIO.new(*parts) end protected @@ -201,7 +196,7 @@ module Google # # @param [String] call_response # the response to parse. - # @return [Array<(Fixnum, Hurley::Header, String)>] + # @return [Array<(Fixnum, Hash, String)>] # Status, header, and response body. def to_http_response(call_response) outer_header, outer_body = split_header_and_body(call_response) @@ -218,10 +213,10 @@ module Google # # @param [String] response # the response to parse. - # @return [Array<(Hurley::Header, String)>] + # @return [Array<(HTTP::Message::Headers, String)>] # the header and the body, separately. def split_header_and_body(response) - header = Hurley::Header.new + header = HTTP::Message::Headers.new payload = response.lstrip while payload line, payload = payload.split(/\n/, 2) diff --git a/lib/google/apis/core/composite_io.rb b/lib/google/apis/core/composite_io.rb new file mode 100644 index 000000000..0f3e03ebc --- /dev/null +++ b/lib/google/apis/core/composite_io.rb @@ -0,0 +1,97 @@ +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'google/apis/core/http_command' +require 'google/apis/core/upload' +require 'google/apis/core/download' +require 'addressable/uri' +require 'securerandom' +module Google + module Apis + module Core + class CompositeIO + def initialize(*ios) + @ios = ios.flatten + @pos = 0 + @index = 0 + @sizes = @ios.map(&:size) + end + + def read(length = nil, buf = nil) + buf = buf ? buf.replace('') : '' + + begin + io = @ios[@index] + break if io.nil? + result = io.read(length) + if result + buf << result + if length + length -= result.length + break if length == 0 + end + end + @index += 1 + end while @index < @ios.length + buf.length > 0 ? buf : nil + end + + def size + @sizes.reduce(:+) + end + + alias_method :length, :size + + def pos + @pos + end + + def pos=(pos) + fail ArgumentError, "Position can not be negative" if pos < 0 + @pos = pos + new_index = nil + @ios.each_with_index do |io,idx| + size = io.size + if pos <= size + new_index ||= idx + io.pos = pos + pos = 0 + else + io.pos = size + pos -= size + end + end + @index = new_index unless new_index.nil? + end + + def rewind + self.pos = 0 + end + end + end + end +end \ No newline at end of file diff --git a/lib/google/apis/core/download.rb b/lib/google/apis/core/download.rb index db55509a7..0078005b4 100644 --- a/lib/google/apis/core/download.rb +++ b/lib/google/apis/core/download.rb @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -require 'google/apis/core/multipart' require 'google/apis/core/api_command' require 'google/apis/errors' require 'addressable/uri' @@ -22,7 +21,7 @@ module Google module Core # Streaming/resumable media download support class DownloadCommand < ApiCommand - RANGE_HEADER = 'range' + RANGE_HEADER = 'Range' # File or IO to write content to # @return [String, File, #write] @@ -57,7 +56,7 @@ module Google # of file content. # # @private - # @param [Hurley::Client] client + # @param [HTTPClient] client # HTTP client # @yield [result, err] Result or error if block supplied # @return [Object] @@ -65,34 +64,41 @@ module Google # @raise [Google::Apis::ClientError] The request is invalid and should not be retried without modification # @raise [Google::Apis::AuthorizationError] Authorization is required def execute_once(client, &block) - client.get(@download_url || url) do |req| - apply_request_options(req) - check_if_rewind_needed = false - if @offset > 0 - logger.debug { sprintf('Resuming download from offset %d', @offset) } - req.header[RANGE_HEADER] = sprintf('bytes=%d-', @offset) - check_if_rewind_needed = true - end - req.on_body(200, 201, 206) do |res, chunk| - check_status(res.status_code, chunk) unless res.status_code.nil? - if check_if_rewind_needed && res.status_code != 206 + request_header = header.dup + apply_request_options(request_header) + + check_if_rewind_needed = false + if @offset > 0 + logger.debug { sprintf('Resuming download from offset %d', @offset) } + request_header[RANGE_HEADER] = sprintf('bytes=%d-', @offset) + check_if_rewind_needed = true + end + + http_res = client.get(url.to_s, + query: query, + header: request_header, + follow_redirect: true) do |res, chunk| + status = res.http_header.status_code.to_i + if [200, 201, 206].include?(status) + if check_if_rewind_needed && status != 206 # Oh no! Requested a chunk, but received the entire content # Attempt to rewind the stream @download_io.rewind check_if_rewind_needed = false end - logger.debug { sprintf('Writing chunk (%d bytes)', chunk.length) } @offset += chunk.length @download_io.write(chunk) @download_io.flush end end + if @close_io_on_finish result = nil else result = @download_io end + check_status(http_res.status.to_i, http_res.header, http_res.body) success(result, &block) rescue => e error(e, rethrow: true, &block) diff --git a/lib/google/apis/core/http_client_adapter.rb b/lib/google/apis/core/http_client_adapter.rb deleted file mode 100644 index a58fd4b11..000000000 --- a/lib/google/apis/core/http_client_adapter.rb +++ /dev/null @@ -1,82 +0,0 @@ -require 'httpclient' -require 'hurley' -require 'hurley/client' - -module Google - module Apis - module Core - # HTTPClient adapter for Hurley. - class HttpClientAdapter - - def call(request) - client = ::HTTPClient.new - configure_client(client, request) - - begin - ::Hurley::Response.new(request) do |res| - http_res = client.request(request.verb.to_s.upcase, request.url.to_s, nil, request.body_io, request.header.to_hash, false) do |http_res, chunk| - copy_response(http_res, res) - res.receive_body(chunk) - end - copy_response(http_res, res) - end - rescue ::HTTPClient::TimeoutError, Errno::ETIMEDOUT - raise ::Hurley::Timeout, $! - rescue ::HTTPClient::BadResponseError => err - if err.message.include?('status 407') - raise ::Hurley::ConnectionFailed, %{407 "Proxy Authentication Required "} - else - raise Hurley::ClientError, $! - end - rescue Errno::ECONNREFUSED, EOFError - raise ::Hurley::ConnectionFailed, $! - rescue => err - if defined?(OpenSSL) && OpenSSL::SSL::SSLError === err - raise Hurley::SSLError, err - else - raise - end - end - end - - def copy_response(http_res, res) - unless res.status_code - res.status_code = http_res.status.to_i - http_res.header.all.each do |(k,v)| - res.header[k] = v - end - end - end - - def configure_client(client, request) - client.transparent_gzip_decompression = true - if request.options.proxy - proxy = request.options.proxy - client.proxy = sprintf('%s:%d', proxy.host, proxy.port) - if proxy.user && proxy.password - client.set_proxy_auth proxy.user, proxy.password - end - end - if request.options.timeout - client.connect_timeout = request.options.timeout - client.receive_timeout = request.options.timeout - client.send_timeout = request.options.timeout - end - if request.options.open_timeout - client.connect_timeout = request.options.open_timeout - client.send_timeout = request.options.open_timeout - end - ssl_config = client.ssl_config - ssl_opts = request.ssl_options - ssl_config.verify_mode = ssl_opts.openssl_verify_mode - ssl_config.cert_store = ssl_opts.openssl_cert_store - ssl_config.add_trust_ca ssl_opts.ca_file if ssl_opts.ca_file - ssl_config.add_trust_ca ssl_opts.ca_path if ssl_opts.ca_path - ssl_config.client_cert = ssl_opts.openssl_client_cert if ssl_opts.openssl_client_cert - ssl_config.client_key = ssl_opts.openssl_client_key if ssl_opts.openssl_client_key - ssl_config.verify_depth = ssl_opts.verify_depth if ssl_opts.verify_depth - end - end - end - end -end diff --git a/lib/google/apis/core/http_command.rb b/lib/google/apis/core/http_command.rb index bcea5c53a..fb57d0f00 100644 --- a/lib/google/apis/core/http_command.rb +++ b/lib/google/apis/core/http_command.rb @@ -17,9 +17,6 @@ require 'addressable/template' require 'google/apis/options' require 'google/apis/errors' require 'retriable' -require 'hurley' -require 'hurley/addressable' -require 'hurley_patches' require 'google/apis/core/logging' require 'pp' @@ -41,7 +38,7 @@ module Google attr_accessor :url # HTTP headers - # @return [Hurley::Header] + # @return [Hash] attr_accessor :header # Request body @@ -53,7 +50,7 @@ module Google attr_accessor :method # HTTP Client - # @return [Hurley::Client] + # @return [HTTPClient] attr_accessor :connection # Query params @@ -75,7 +72,7 @@ module Google self.url = url self.url = Addressable::Template.new(url) if url.is_a?(String) self.method = method - self.header = Hurley::Header.new + self.header = Hash.new self.body = body self.query = {} self.params = {} @@ -83,7 +80,7 @@ module Google # Execute the command, retrying as necessary # - # @param [Hurley::Client] client + # @param [HTTPClient] client # HTTP client # @yield [result, err] Result or error if block supplied # @return [Object] @@ -166,7 +163,7 @@ module Google # # @param [Fixnum] status # HTTP status code of response - # @param [Hurley::Header] header + # @param [Hash] header # Response headers # @param [String, #read] body # Response body @@ -177,7 +174,7 @@ module Google # @raise [Google::Apis::AuthorizationError] Authorization is required def process_response(status, header, body) check_status(status, header, body) - decode_response_body(header[:content_type], body) + decode_response_body(header['Content-Type'].first, body) end # Check the response and raise error if needed @@ -185,7 +182,7 @@ module Google # @param [Fixnum] status # HTTP status code of response # @param - # @param [Hurley::Header] header + # @param [Hash] header # HTTP response headers # @param [String] body # HTTP response body @@ -201,7 +198,7 @@ module Google when 200...300 nil when 301, 302, 303, 307 - message ||= sprintf('Redirect to %s', header[:location]) + message ||= sprintf('Redirect to %s', header['Location']) raise Google::Apis::RedirectError.new(message, status_code: status, header: header, body: body) when 401 message ||= 'Unauthorized' @@ -251,7 +248,16 @@ module Google # @raise [StandardError] if no block def error(err, rethrow: false, &block) logger.debug { sprintf('Error - %s', PP.pp(err, '')) } - err = Google::Apis::TransmissionError.new(err) if err.is_a?(Hurley::ClientError) || err.is_a?(SocketError) + if err.is_a?(HTTPClient::BadResponseError) + begin + res = err.res + check_status(res.status.to_i, res.header, res.body) + rescue Google::Apis::Error => e + err = e + end + elsif err.is_a?(HTTPClient::TimeoutError) || err.is_a?(SocketError) + err = Google::Apis::TransmissionError.new(err) + end block.call(nil, err) if block_given? fail err if rethrow || block.nil? end @@ -259,7 +265,7 @@ module Google # Execute the command once. # # @private - # @param [Hurley::Client] client + # @param [HTTPClient] client # HTTP client # @return [Object] # @raise [Google::Apis::ServerError] An error occurred on the server and the request can be retried @@ -269,21 +275,18 @@ module Google body.rewind if body.respond_to?(:rewind) begin logger.debug { sprintf('Sending HTTP %s %s', method, url) } - response = client.send(method, url, body) do |req| - # Temporary workaround for Hurley bug where the connection preference - # is ignored and it uses nested anyway - unless form_encoded? - req.url.query_class = Hurley::Query::Flat - query.each do | k, v| - req.url.query[k] = normalize_query_value(v) - end - end - # End workaround - apply_request_options(req) - end - logger.debug { response.status_code } - logger.debug { response.inspect } - response = process_response(response.status_code, response.header, response.body) + request_header = header.dup + apply_request_options(request_header) + + http_res = client.request(method.to_s.upcase, + url.to_s, + query: nil, + body: body, + header: request_header, + follow_redirect: true) + logger.debug { http_res.status } + logger.debug { http_res.inspect } + response = process_response(http_res.status.to_i, http_res.header, http_res.body) success(response) rescue => e logger.debug { sprintf('Caught error %s', e) } @@ -292,18 +295,16 @@ module Google end # Update the request with any specified options. - # @param [Hurley::Request] req - # HTTP request + # @param [Hash] header + # HTTP headers # @return [void] - def apply_request_options(req) + def apply_request_options(req_header) if options.authorization.respond_to?(:apply!) - options.authorization.apply!(req.header) + options.authorization.apply!(req_header) elsif options.authorization.is_a?(String) - req.header[:authorization] = sprintf('Bearer %s', options.authorization) + req_header['Authorization'] = sprintf('Bearer %s', options.authorization) end - req.header.update(header) - req.options.timeout = options.timeout_sec - req.options.open_timeout = options.open_timeout_sec + req_header.update(header) end private diff --git a/lib/google/apis/core/multipart.rb b/lib/google/apis/core/multipart.rb index c6063fac3..d2d678d67 100644 --- a/lib/google/apis/core/multipart.rb +++ b/lib/google/apis/core/multipart.rb @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -require 'hurley' module Google module Apis @@ -21,108 +20,60 @@ module Google # # @private class JsonPart - include Hurley::Multipart::Part - # @return [Fixnum] - # Length of part - attr_reader :length - - # @param [String] boundary - # Multipart boundary # @param [String] value # JSON content - def initialize(boundary, value, header = {}) - @part = build_part(boundary, value) - @length = @part.bytesize - @io = StringIO.new(@part) + # @param [Hash] header + # Additional headers + def initialize(value, header = {}) + @value = value + @header = header end - private - - # Format the part - # - # @param [String] boundary - # Multipart boundary - # @param [String] value - # JSON content - # @return [String] - def build_part(boundary, value) + def to_io(boundary) part = '' part << "--#{boundary}\r\n" part << "Content-Type: application/json\r\n" + @header.each do |(k, v)| + part << "#{k}: #{v}\r\n" + end part << "\r\n" - part << "#{value}\r\n" + part << "#{@value}\r\n" + StringIO.new(part) end + end - # Part of a multipart request for holding arbitrary content. Modified - # from Hurley::Multipart::FilePart to remove Content-Disposition + # Part of a multipart request for holding arbitrary content. # # @private class FilePart - include Hurley::Multipart::Part - - # @return [Fixnum] - # Length of part - attr_reader :length - - # @param [String] boundary - # Multipart boundary - # @param [Google::Apis::Core::UploadIO] io + # @param [IO] io # IO stream # @param [Hash] header # Additional headers - def initialize(boundary, io, header = {}) - file_length = io.respond_to?(:length) ? io.length : File.size(io.local_path) - - @head = build_head(boundary, io.content_type, file_length, - io.respond_to?(:opts) ? io.opts.merge(header) : header) - - @length = @head.bytesize + file_length + FOOT.length - @io = Hurley::CompositeReadIO.new(@length, StringIO.new(@head), io, StringIO.new(FOOT)) + def initialize(io, header = {}) + @io = io + @header = header + @length = io.respond_to?(:size) ? io.size : nil end - private - - # Construct the header for the part - # - # @param [String] boundary - # Multipart boundary - # @param [String] type - # Content type for the part - # @param [Fixnum] content_len - # Length of the part - # @param [Hash] header - # Headers for the part - def build_head(boundary, type, content_len, header) - content_id = '' - if header[:content_id] - content_id = sprintf(CID_FORMAT, header[:content_id]) + def to_io(boundary) + head = '' + head << "--#{boundary}\r\n" + @header.each do |(k, v)| + head << "#{k}: #{v}\r\n" end - sprintf(HEAD_FORMAT, - boundary, - content_len.to_i, - content_id, - header[:content_type] || type, - header[:content_transfer_encoding] || DEFAULT_TR_ENCODING) + head << "Content-Length: #{@length}\r\n" unless @length.nil? + head << "Content-Transfer-Encoding: binary\r\n" + head << "\r\n" + Google::Apis::Core::CompositeIO.new(StringIO.new(head), @io, StringIO.new("\r\n")) end - - DEFAULT_TR_ENCODING = 'binary'.freeze - FOOT = "\r\n".freeze - CID_FORMAT = "Content-ID: %s\r\n" - HEAD_FORMAT = <<-END ---%s\r -Content-Length: %d\r -%sContent-Type: %s\r -Content-Transfer-Encoding: %s\r -\r - END end # Helper for building multipart requests class Multipart MULTIPART_RELATED = 'multipart/related' - DEFAULT_BOUNDARY = 'RubyApiClientMultiPart' # @return [String] # Content type header @@ -135,8 +86,8 @@ Content-Transfer-Encoding: %s\r def initialize(content_type: MULTIPART_RELATED, boundary: nil) @parts = [] - @boundary = boundary || DEFAULT_BOUNDARY - @content_type = "#{content_type}; boundary=#{boundary}" + @boundary = boundary || Digest::SHA1.hexdigest(SecureRandom.random_bytes(8)) + @content_type = "#{content_type}; boundary=#{@boundary}" end # Append JSON data part @@ -147,23 +98,26 @@ Content-Transfer-Encoding: %s\r # Optional unique ID of this part # @return [self] def add_json(body, content_id: nil) - header = { :content_id => content_id } - @parts << Google::Apis::Core::JsonPart.new(@boundary, body, header) + header = {} + header['Content-ID'] = content_id unless content_id.nil? + @parts << Google::Apis::Core::JsonPart.new(body, header).to_io(@boundary) self end # Append arbitrary data as a part # - # @param [Google::Apis::Core::UploadIO] upload_io + # @param [IO] upload_io # IO stream # @param [String] content_id # Optional unique ID of this part # @return [self] - def add_upload(upload_io, content_id: nil) - header = { :content_id => content_id } - @parts << Google::Apis::Core::FilePart.new(@boundary, - upload_io, - header) + def add_upload(upload_io, content_type: nil, content_id: nil) + header = { + 'Content-Type' => content_type || 'application/octet-stream' + } + header['Content-Id'] = content_id unless content_id.nil? + @parts << Google::Apis::Core::FilePart.new(upload_io, + header).to_io(@boundary) self end @@ -172,16 +126,10 @@ Content-Transfer-Encoding: %s\r # @return [IO] # IO stream def assemble - @parts << Hurley::Multipart::EpiloguePart.new(@boundary) - ios = [] - len = 0 - @parts.each do |part| - len += part.length - ios << part.to_io - end - Hurley::CompositeReadIO.new(len, *ios) + @parts << StringIO.new("--#{@boundary}--\r\n\r\n") + Google::Apis::Core::CompositeIO.new(*@parts) end end end end -end +end \ No newline at end of file diff --git a/lib/google/apis/core/upload.rb b/lib/google/apis/core/upload.rb index 4d97773d5..9c2cd3151 100644 --- a/lib/google/apis/core/upload.rb +++ b/lib/google/apis/core/upload.rb @@ -29,41 +29,6 @@ end module Google module Apis module Core - # Extension of Hurley's UploadIO to add length accessor - class UploadIO < Hurley::UploadIO - OCTET_STREAM_CONTENT_TYPE = 'application/octet-stream' - - # Get the length of the stream - # @return [Fixnum] - def length - io.respond_to?(:length) ? io.length : File.size(local_path) - end - - # Create a new instance given a file path - # @param [String, File] file_name - # Path to file - # @param [String] content_type - # Optional content type. If nil, will attempt to auto-detect - # @return [Google::Apis::Core::UploadIO] - def self.from_file(file_name, content_type: nil) - if content_type.nil? - type = MIME::Types.of(file_name) - content_type = type.first.content_type unless type.nil? || type.empty? - end - new(file_name, content_type || OCTET_STREAM_CONTENT_TYPE) - end - - # Wraps an IO stream in UploadIO - # @param [#read] io - # IO to wrap - # @param [String] content_type - # Optional content type. - # @return [Google::Apis::Core::UploadIO] - def self.from_io(io, content_type: OCTET_STREAM_CONTENT_TYPE) - new(io, content_type) - end - end - # Base upload command. Not intended to be used directly # @private class BaseUploadCommand < ApiCommand @@ -83,17 +48,17 @@ module Google # @return [Google::Apis::Core::UploadIO] attr_accessor :upload_io - # Ensure the content is readable and wrapped in an {{Google::Apis::Core::UploadIO}} instance. + # Ensure the content is readable and wrapped in an IO instance. # # @return [void] # @raise [Google::Apis::ClientError] if upload source is invalid def prepare! super if streamable?(upload_source) - self.upload_io = UploadIO.from_io(upload_source, content_type: upload_content_type) + self.upload_io = upload_source @close_io_on_finish = false elsif upload_source.is_a?(String) - self.upload_io = UploadIO.from_file(upload_source, content_type: upload_content_type) + self.upload_io = File.new(upload_source, 'r') @close_io_on_finish = true else fail Google::Apis::ClientError, 'Invalid upload source' @@ -124,13 +89,12 @@ module Google super self.body = upload_io header[UPLOAD_PROTOCOL_HEADER] = RAW_PROTOCOL - header[UPLOAD_CONTENT_TYPE_HEADER] = upload_io.content_type + header[UPLOAD_CONTENT_TYPE_HEADER] = upload_content_type end end # Implementation of the multipart upload protocol class MultipartUploadCommand < BaseUploadCommand - UPLOAD_BOUNDARY = 'RubyApiClientUpload' MULTIPART_PROTOCOL = 'multipart' MULTIPART_RELATED = 'multipart/related' @@ -140,11 +104,11 @@ module Google # @raise [Google::Apis::ClientError] if upload source is invalid def prepare! super - @multipart = Multipart.new(boundary: UPLOAD_BOUNDARY, content_type: MULTIPART_RELATED) - @multipart.add_json(body) - @multipart.add_upload(upload_io) - self.body = @multipart.assemble - header[:content_type] = @multipart.content_type + multipart = Multipart.new + multipart.add_json(body) + multipart.add_upload(upload_io, content_type: upload_content_type) + self.body = multipart.assemble + header['Content-Type'] = multipart.content_type header[UPLOAD_PROTOCOL_HEADER] = MULTIPART_PROTOCOL end end @@ -179,7 +143,7 @@ module Google # # @param [Fixnum] status # HTTP status code of response - # @param [Hurley::Header] header + # @param [HTTP::Message::Headers] header # Response headers # @param [String, #read] body # Response body @@ -189,9 +153,9 @@ module Google # @raise [Google::Apis::ClientError] The request is invalid and should not be retried without modification # @raise [Google::Apis::AuthorizationError] Authorization is required def process_response(status, header, body) - @offset = Integer(header[BYTES_RECEIVED_HEADER]) if header.key?(BYTES_RECEIVED_HEADER) - @upload_url = header[UPLOAD_URL_HEADER] if header.key?(UPLOAD_URL_HEADER) - upload_status = header[UPLOAD_STATUS_HEADER] + @offset = Integer(header[BYTES_RECEIVED_HEADER].first) unless header[BYTES_RECEIVED_HEADER].empty? + @upload_url = header[UPLOAD_URL_HEADER].first unless header[UPLOAD_URL_HEADER].empty? + upload_status = header[UPLOAD_STATUS_HEADER].first logger.debug { sprintf('Upload status %s', upload_status) } if upload_status == STATUS_ACTIVE @state = :active @@ -204,61 +168,68 @@ module Google super(status, header, body) end - # Send the start command to initiate the upload - # - # @param [Hurley::Client] client - # HTTP client - # @return [Hurley::Response] - # @raise [Google::Apis::ServerError] Unable to send the request def send_start_command(client) logger.debug { sprintf('Sending upload start command to %s', url) } - client.send(method, url, body) do |req| - apply_request_options(req) - req.header[UPLOAD_PROTOCOL_HEADER] = RESUMABLE - req.header[UPLOAD_COMMAND_HEADER] = START_COMMAND - req.header[UPLOAD_CONTENT_LENGTH] = upload_io.length.to_s - req.header[UPLOAD_CONTENT_TYPE_HEADER] = upload_io.content_type - end + + request_header = header.dup + apply_request_options(request_header) + request_header[UPLOAD_PROTOCOL_HEADER] = RESUMABLE + request_header[UPLOAD_COMMAND_HEADER] = START_COMMAND + request_header[UPLOAD_CONTENT_LENGTH] = upload_io.size.to_s + request_header[UPLOAD_CONTENT_TYPE_HEADER] = upload_content_type + + client.request(method.to_s.upcase, + url.to_s, query: nil, + body: body, + header: request_header, + follow_redirect: true) rescue => e raise Google::Apis::ServerError, e.message end # Query for the status of an incomplete upload # - # @param [Hurley::Client] client + # @param [HTTPClient] client # HTTP client - # @return [Hurley::Response] + # @return [HTTP::Message] # @raise [Google::Apis::ServerError] Unable to send the request def send_query_command(client) logger.debug { sprintf('Sending upload query command to %s', @upload_url) } - client.post(@upload_url, nil) do |req| - apply_request_options(req) - req.header[UPLOAD_COMMAND_HEADER] = QUERY_COMMAND - end + + request_header = header.dup + apply_request_options(request_header) + request_header[UPLOAD_COMMAND_HEADER] = QUERY_COMMAND + + client.post(@upload_url, header: request_header, follow_redirect: true) end + # Send the actual content # - # @param [Hurley::Client] client + # @param [HTTPClient] client # HTTP client - # @return [Hurley::Response] + # @return [HTTP::Message] # @raise [Google::Apis::ServerError] Unable to send the request def send_upload_command(client) logger.debug { sprintf('Sending upload command to %s', @upload_url) } + content = upload_io content.pos = @offset - client.post(@upload_url, content) do |req| - apply_request_options(req) - req.header[UPLOAD_COMMAND_HEADER] = UPLOAD_COMMAND - req.header[UPLOAD_OFFSET_HEADER] = @offset.to_s - end + + request_header = header.dup + apply_request_options(request_header) + request_header[UPLOAD_COMMAND_HEADER] = QUERY_COMMAND + request_header[UPLOAD_COMMAND_HEADER] = UPLOAD_COMMAND + request_header[UPLOAD_OFFSET_HEADER] = @offset.to_s + + client.post(@upload_url, body: content, header: request_header, follow_redirect: true) end # Execute the upload request once. This will typically perform two HTTP requests -- one to initiate or query # for the status of the upload, the second to send the (remaining) content. # # @private - # @param [Hurley::Client] client + # @param [HTTPClient] client # HTTP client # @yield [result, err] Result or error if block supplied # @return [Object] diff --git a/spec/google/apis/core/api_command_spec.rb b/spec/google/apis/core/api_command_spec.rb index 4920c496c..93122fae8 100644 --- a/spec/google/apis/core/api_command_spec.rb +++ b/spec/google/apis/core/api_command_spec.rb @@ -15,7 +15,6 @@ require 'spec_helper' require 'google/apis/core/api_command' require 'google/apis/core/json_representation' -require 'hurley/test' RSpec.describe Google::Apis::Core::HttpCommand do include TestHelpers diff --git a/spec/google/apis/core/batch_spec.rb b/spec/google/apis/core/batch_spec.rb index f784a03fa..46e6e65de 100644 --- a/spec/google/apis/core/batch_spec.rb +++ b/spec/google/apis/core/batch_spec.rb @@ -15,7 +15,6 @@ require 'spec_helper' require 'google/apis/core/batch' require 'google/apis/core/json_representation' -require 'hurley/test' RSpec.describe Google::Apis::Core::BatchCommand do include TestHelpers @@ -30,19 +29,20 @@ RSpec.describe Google::Apis::Core::BatchCommand do let(:post_with_string_command) do command = Google::Apis::Core::HttpCommand.new(:post, 'https://www.googleapis.com/zoo/animals/2') command.body = 'Hello world' - command.header[:content_type] = 'text/plain' + command.header['Content-Type'] = 'text/plain' command end let(:post_with_io_command) do command = Google::Apis::Core::HttpCommand.new(:post, 'https://www.googleapis.com/zoo/animals/3') command.body = StringIO.new('Goodbye!') - command.header[:content_type] = 'text/plain' + command.header['Content-Type'] = 'text/plain' command end before(:example) do allow(SecureRandom).to receive(:uuid).and_return('ffe23d1b-e8f7-47f5-8c01-2a30cf8ecb8f') + allow(Digest::SHA1).to receive(:hexdigest).and_return('123abc') response = < +--123abc Content-Type: application/http +Content-Id: +Content-Length: 58 Content-Transfer-Encoding: binary GET /zoo/animals/1? HTTP/1.1 Host: www.googleapis.com ---RubyApiBatchRequest -Content-Length: 96 -Content-ID: +--123abc Content-Type: application/http +Content-Id: +Content-Length: 96 Content-Transfer-Encoding: binary POST /zoo/animals/2? HTTP/1.1 @@ -104,10 +104,10 @@ Content-Type: text/plain Host: www.googleapis.com Hello world ---RubyApiBatchRequest -Content-Length: 93 -Content-ID: +--123abc Content-Type: application/http +Content-Id: +Content-Length: 93 Content-Transfer-Encoding: binary POST /zoo/animals/3? HTTP/1.1 @@ -115,7 +115,7 @@ Content-Type: text/plain Host: www.googleapis.com Goodbye! ---RubyApiBatchRequest-- +--123abc-- EOF expect(a_request(:post, 'https://www.googleapis.com/batch').with(body: expected_body)).to have_been_made diff --git a/spec/google/apis/core/composite_io_spec.rb b/spec/google/apis/core/composite_io_spec.rb new file mode 100644 index 000000000..9bf6da21b --- /dev/null +++ b/spec/google/apis/core/composite_io_spec.rb @@ -0,0 +1,76 @@ +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'spec_helper' +require 'google/apis/core/composite_io' + +RSpec.describe Google::Apis::Core::CompositeIO do + + shared_examples 'should act like IO' do + it 'should read from all IOs' do + expect(io.read).to eq 'Hello Cruel World' + end + + it 'should respond to size' do + expect(io.size).to eq 17 + end + + it 'should respond to pos=' do + io.pos = 6 + expect(io.read).to eq('Cruel World') + end + + it 'should reject negative positions' do + expect { io.pos = -1 }.to raise_error(ArgumentError) + end + + + it 'should return nil if position beyond size' do + io.pos = 20 + expect(io.read).to be_nil + end + + it 'should be readable after rewinding' do + expect(io.read).to eq 'Hello Cruel World' + expect(io.read).to be_nil + io.rewind + expect(io.read).to eq 'Hello Cruel World' + end + end + + context 'with StringIOs' do + let(:io) do + Google::Apis::Core::CompositeIO.new( + StringIO.new("Hello "), + StringIO.new("Cruel "), + StringIO.new("World")) + end + include_examples 'should act like IO' + end + + context 'with Files' do + let(:io) do + files = [] + dir = Dir.mktmpdir + ['Hello ', 'Cruel ', 'World'].each_with_index do |text, index| + name = File.join(dir, "f#{index}") + File.open(name, 'w') { |f| f.write(text) } + files << name + end + Google::Apis::Core::CompositeIO.new(files.map { |name| File.open(name, 'r') }) + end + include_examples 'should act like IO' + end + +end \ No newline at end of file diff --git a/spec/google/apis/core/download_spec.rb b/spec/google/apis/core/download_spec.rb index 9f56beb0f..93620017c 100644 --- a/spec/google/apis/core/download_spec.rb +++ b/spec/google/apis/core/download_spec.rb @@ -15,7 +15,6 @@ require 'spec_helper' require 'google/apis/core/download' require 'google/apis/core/json_representation' -require 'hurley/test' require 'tempfile' require 'tmpdir' diff --git a/spec/google/apis/core/http_command_spec.rb b/spec/google/apis/core/http_command_spec.rb index e9e9ead39..b64a638b7 100644 --- a/spec/google/apis/core/http_command_spec.rb +++ b/spec/google/apis/core/http_command_spec.rb @@ -14,7 +14,6 @@ require 'spec_helper' require 'google/apis/core/http_command' -require 'hurley/test' RSpec.describe Google::Apis::Core::HttpCommand do include TestHelpers diff --git a/spec/google/apis/core/service_spec.rb b/spec/google/apis/core/service_spec.rb index ae5fa6d44..38c674b7e 100644 --- a/spec/google/apis/core/service_spec.rb +++ b/spec/google/apis/core/service_spec.rb @@ -16,8 +16,6 @@ require 'spec_helper' require 'google/apis/options' require 'google/apis/core/base_service' require 'google/apis/core/json_representation' -require 'hurley/test' -require 'ostruct' RSpec.describe Google::Apis::Core::BaseService do include TestHelpers @@ -190,6 +188,8 @@ EOF context 'with batch uploads' do before(:example) do + allow(SecureRandom).to receive(:uuid).and_return('b1981e17-f622-49af-b2eb-203308b1b17d') + allow(Digest::SHA1).to receive(:hexdigest).and_return('outer', 'inner') response = < +Content-Length: 303 +Content-Transfer-Encoding: binary + +POST /upload/zoo/animals? HTTP/1.1 +Content-Type: multipart/related; boundary=inner +X-Goog-Upload-Protocol: multipart +Host: www.googleapis.com + +--inner +Content-Type: application/json + + +--inner +Content-Type: text/plain +Content-Length: 4 +Content-Transfer-Encoding: binary + +test +--inner-- + + +--outer-- + +EOF + expect(a_request(:put, 'https://www.googleapis.com/upload/').with(body: expected_body)).to have_been_made + end + it 'should disallow downloads in batch' do expect do |b| service.batch_upload do |service| diff --git a/spec/google/apis/core/upload_spec.rb b/spec/google/apis/core/upload_spec.rb index 99039f8f0..6e338a15d 100644 --- a/spec/google/apis/core/upload_spec.rb +++ b/spec/google/apis/core/upload_spec.rb @@ -15,70 +15,6 @@ require 'spec_helper' require 'google/apis/core/upload' require 'google/apis/core/json_representation' -require 'hurley/test' - -# TODO: JSON Response decoding -# TODO: Upload from IO -# TODO: Upload from file - -RSpec.describe Google::Apis::Core::UploadIO do - context 'from_file' do - let(:upload_io) { Google::Apis::Core::UploadIO.from_file(file) } - - context 'with text file' do - let(:file) { File.join(FIXTURES_DIR, 'files', 'test.txt') } - it 'should infer content type from file' do - expect(upload_io.content_type).to eql('text/plain') - end - - it 'should allow overriding the mime type' do - io = Google::Apis::Core::UploadIO.from_file(file, content_type: 'application/json') - expect(io.content_type).to eql('application/json') - end - end - - context 'with unknown type' do - let(:file) { File.join(FIXTURES_DIR, 'files', 'test.blah') } - it 'should use the default mime type' do - expect(upload_io.content_type).to eql('application/octet-stream') - end - - it 'should allow overriding the mime type' do - io = Google::Apis::Core::UploadIO.from_file(file, content_type: 'application/json') - expect(io.content_type).to eql('application/json') - end - - it 'should setup length of the stream' do - upload_io = Google::Apis::Core::UploadIO.from_file(file) - expect(upload_io.length).to eq File.size(file) - end - - end - end - - context 'from_io' do - - context 'with i/o stream' do - let(:io) { StringIO.new 'Hello google' } - - it 'should setup default content-type' do - upload_io = Google::Apis::Core::UploadIO.from_io(io) - expect(upload_io.content_type).to eql Google::Apis::Core::UploadIO::OCTET_STREAM_CONTENT_TYPE - end - - it 'should allow overring the mime type' do - upload_io = Google::Apis::Core::UploadIO.from_io(io, content_type: 'application/x-gzip') - expect(upload_io.content_type).to eq('application/x-gzip') - end - - it 'should setup length of the stream' do - upload_io = Google::Apis::Core::UploadIO.from_io(io) - expect(upload_io.length).to eq 'Hello google'.length - end - end - - end -end RSpec.describe Google::Apis::Core::RawUploadCommand do include TestHelpers @@ -170,21 +106,22 @@ RSpec.describe Google::Apis::Core::MultipartUploadCommand do before(:example) do stub_request(:post, 'https://www.googleapis.com/zoo/animals').to_return(body: %(Hello world)) + allow(Digest::SHA1).to receive(:hexdigest).and_return('123abc') end it 'should send content' do expected_body = < err - if defined?(OpenSSL) && OpenSSL::SSL::SSLError === err - raise SSLError, err - else - raise ConnectionFailed, err - end - end - end - - rescue ::Timeout::Error => err - raise Timeout, err - end - - private - - def net_http_request(request) - http_req = Net::HTTPGenericRequest.new( - request.verb.to_s.upcase, # request method - !!request.body, # is there a request body - :head != request.verb, # is there a response body - request.url.request_uri, # request uri path - request.header, # request headers - ) - - if body = request.body_io - http_req.body_stream = body - end - - http_req - end - - def perform_request(http, request, res) - http.request(net_http_request(request)) do |http_res| - res.status_code = http_res.code.to_i - http_res.each_header do |key, value| - res.header[key] = value - end - - if :get == request.verb - http_res.read_body { |chunk| res.receive_body(chunk) } - else - res.receive_body(http_res.body) - end - end - end - end - end -end