From 2046e00f14d2924880782dcb6537fad1a7f3fc05 Mon Sep 17 00:00:00 2001 From: Steve Bazyl Date: Wed, 17 Aug 2016 13:51:09 -0700 Subject: [PATCH 01/16] Remove Hurley as a dependency --- google-api-client.gemspec | 3 +- lib/google/apis/core/api_command.rb | 4 +- lib/google/apis/core/base_service.rb | 36 +++-- lib/google/apis/core/batch.rb | 31 ++--- lib/google/apis/core/composite_io.rb | 97 ++++++++++++++ lib/google/apis/core/download.rb | 36 ++--- lib/google/apis/core/http_client_adapter.rb | 82 ------------ lib/google/apis/core/http_command.rb | 73 ++++++----- lib/google/apis/core/multipart.rb | 138 ++++++-------------- lib/google/apis/core/upload.rb | 123 +++++++---------- spec/google/apis/core/api_command_spec.rb | 1 - spec/google/apis/core/batch_spec.rb | 26 ++-- spec/google/apis/core/composite_io_spec.rb | 76 +++++++++++ spec/google/apis/core/download_spec.rb | 1 - spec/google/apis/core/http_command_spec.rb | 1 - spec/google/apis/core/service_spec.rb | 42 +++++- spec/google/apis/core/upload_spec.rb | 73 +---------- third_party/hurley_patches.rb | 103 --------------- 18 files changed, 417 insertions(+), 529 deletions(-) create mode 100644 lib/google/apis/core/composite_io.rb delete mode 100644 lib/google/apis/core/http_client_adapter.rb create mode 100644 spec/google/apis/core/composite_io_spec.rb delete mode 100644 third_party/hurley_patches.rb diff --git a/google-api-client.gemspec b/google-api-client.gemspec index 707b750c7..ce6330bb6 100644 --- a/google-api-client.gemspec +++ b/google-api-client.gemspec @@ -24,9 +24,8 @@ Gem::Specification.new do |spec| spec.add_runtime_dependency 'retriable', '~> 2.0' spec.add_runtime_dependency 'addressable', '~> 2.3' spec.add_runtime_dependency 'mime-types', '>= 1.6' - spec.add_runtime_dependency 'hurley', '~> 0.1' spec.add_runtime_dependency 'googleauth', '~> 0.5' spec.add_runtime_dependency 'thor', '~> 0.19' - spec.add_runtime_dependency 'httpclient', '~> 2.7' + spec.add_runtime_dependency 'httpclient', '>= 2.8.1', '< 3.0' spec.add_runtime_dependency 'memoist', '~> 0.11' end diff --git a/lib/google/apis/core/api_command.rb b/lib/google/apis/core/api_command.rb index 519c0c97e..e76e3512c 100644 --- a/lib/google/apis/core/api_command.rb +++ b/lib/google/apis/core/api_command.rb @@ -50,7 +50,7 @@ module Google def prepare! query[FIELDS_PARAM] = normalize_fields_param(query[FIELDS_PARAM]) if query.key?(FIELDS_PARAM) if request_representation && request_object - header[:content_type] ||= JSON_CONTENT_TYPE + header['Content-Type'] ||= JSON_CONTENT_TYPE self.body = request_representation.new(request_object).to_json(skip_undefined: true) end super @@ -78,7 +78,7 @@ module Google # # @param [Fixnum] status # HTTP status code of response - # @param [Hurley::Header] header + # @param [Hash] header # HTTP response headers # @param [String] body # HTTP response body diff --git a/lib/google/apis/core/base_service.rb b/lib/google/apis/core/base_service.rb index 9065a389a..93699fafa 100644 --- a/lib/google/apis/core/base_service.rb +++ b/lib/google/apis/core/base_service.rb @@ -19,11 +19,8 @@ require 'google/apis/core/api_command' require 'google/apis/core/batch' require 'google/apis/core/upload' require 'google/apis/core/download' -require 'google/apis/core/http_client_adapter' require 'google/apis/options' require 'googleauth' -require 'hurley' -require 'hurley/addressable' module Google module Apis @@ -96,7 +93,7 @@ module Google attr_accessor :batch_path # HTTP client - # @return [Hurley::Client] + # @return [HTTPClient] attr_accessor :client # General settings @@ -198,7 +195,7 @@ module Google end # Get the current HTTP client - # @return [Hurley::Client] + # @return [HTTPClient] def client @client ||= new_client end @@ -368,19 +365,30 @@ module Google end # Create a new HTTP client - # @return [Hurley::Client] + # @return [HTTPClient] def new_client - client = Hurley::Client.new - client.connection = Google::Apis::Core::HttpClientAdapter.new unless client_options.use_net_http - client.request_options.timeout = request_options.timeout_sec - client.request_options.open_timeout = request_options.open_timeout_sec - client.request_options.proxy = client_options.proxy_url - client.request_options.query_class = Hurley::Query::Flat - client.ssl_options.ca_file = File.join(Google::Apis::ROOT, 'lib', 'cacerts.pem') - client.header[:user_agent] = user_agent + client = ::HTTPClient.new + + client.transparent_gzip_decompression = true + + client.proxy = client_options.proxy_url if client_options.proxy_url + + if request_options.timeout_sec + client.connect_timeout = request_options.timeout_sec + client.receive_timeout = request_options.timeout_sec + client.send_timeout = request_options.timeout_sec + end + + if request_options.open_timeout_sec + client.connect_timeout = request_options.open_timeout_sec + client.send_timeout = request_options.open_timeout_sec + end + client.follow_redirect_count = 5 + client.default_header = { 'User-Agent' => user_agent } client end + # Build the user agent header # @return [String] def user_agent diff --git a/lib/google/apis/core/batch.rb b/lib/google/apis/core/batch.rb index e95ce68ce..af60ce649 100644 --- a/lib/google/apis/core/batch.rb +++ b/lib/google/apis/core/batch.rb @@ -25,19 +25,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -require 'hurley' require 'google/apis/core/multipart' require 'google/apis/core/http_command' require 'google/apis/core/upload' require 'google/apis/core/download' +require 'google/apis/core/composite_io' require 'addressable/uri' require 'securerandom' + module Google module Apis module Core # Wrapper request for batching multiple calls in a single server request class BatchCommand < HttpCommand - BATCH_BOUNDARY = 'RubyApiBatchRequest'.freeze MULTIPART_MIXED = 'multipart/mixed' # @param [symbol] method @@ -81,7 +81,7 @@ module Google parts.each_index do |index| response = deserializer.to_http_response(parts[index]) outer_header = response.shift - call_id = header_to_id(outer_header[:content_id]) || index + call_id = header_to_id(outer_header['Content-ID'].first) || index call, callback = @calls[call_id] begin result = call.process_response(*response) unless call.nil? @@ -106,17 +106,16 @@ module Google fail BatchError, 'Cannot make an empty batch request' if @calls.empty? serializer = CallSerializer.new - multipart = Multipart.new(boundary: BATCH_BOUNDARY, content_type: MULTIPART_MIXED) + multipart = Multipart.new(content_type: MULTIPART_MIXED) @calls.each_index do |index| call, _ = @calls[index] content_id = id_to_header(index) - io = serializer.to_upload_io(call) - multipart.add_upload(io, content_id: content_id) + io = serializer.to_part(call) + multipart.add_upload(io, content_type: 'application/http', content_id: content_id) end self.body = multipart.assemble - header[:content_type] = multipart.content_type - header[:content_length] = "#{body.length}" + header['Content-Type'] = multipart.content_type super end @@ -155,24 +154,20 @@ module Google # Serializes a command for embedding in a multipart batch request # @private class CallSerializer - HTTP_CONTENT_TYPE = 'application/http' - ## # Serialize a single batched call for assembling the multipart message # # @param [Google::Apis::Core::HttpCommand] call # the call to serialize. - # @return [Hurley::UploadIO] + # @return [IO] # the serialized request - def to_upload_io(call) + def to_part(call) call.prepare! parts = [] parts << build_head(call) parts << build_body(call) unless call.body.nil? length = parts.inject(0) { |a, e| a + e.length } - Hurley::UploadIO.new(Hurley::CompositeReadIO.new(length, *parts), - HTTP_CONTENT_TYPE, - 'ruby-api-request') + Google::Apis::Core::CompositeIO.new(*parts) end protected @@ -201,7 +196,7 @@ module Google # # @param [String] call_response # the response to parse. - # @return [Array<(Fixnum, Hurley::Header, String)>] + # @return [Array<(Fixnum, Hash, String)>] # Status, header, and response body. def to_http_response(call_response) outer_header, outer_body = split_header_and_body(call_response) @@ -218,10 +213,10 @@ module Google # # @param [String] response # the response to parse. - # @return [Array<(Hurley::Header, String)>] + # @return [Array<(HTTP::Message::Headers, String)>] # the header and the body, separately. def split_header_and_body(response) - header = Hurley::Header.new + header = HTTP::Message::Headers.new payload = response.lstrip while payload line, payload = payload.split(/\n/, 2) diff --git a/lib/google/apis/core/composite_io.rb b/lib/google/apis/core/composite_io.rb new file mode 100644 index 000000000..0f3e03ebc --- /dev/null +++ b/lib/google/apis/core/composite_io.rb @@ -0,0 +1,97 @@ +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'google/apis/core/http_command' +require 'google/apis/core/upload' +require 'google/apis/core/download' +require 'addressable/uri' +require 'securerandom' +module Google + module Apis + module Core + class CompositeIO + def initialize(*ios) + @ios = ios.flatten + @pos = 0 + @index = 0 + @sizes = @ios.map(&:size) + end + + def read(length = nil, buf = nil) + buf = buf ? buf.replace('') : '' + + begin + io = @ios[@index] + break if io.nil? + result = io.read(length) + if result + buf << result + if length + length -= result.length + break if length == 0 + end + end + @index += 1 + end while @index < @ios.length + buf.length > 0 ? buf : nil + end + + def size + @sizes.reduce(:+) + end + + alias_method :length, :size + + def pos + @pos + end + + def pos=(pos) + fail ArgumentError, "Position can not be negative" if pos < 0 + @pos = pos + new_index = nil + @ios.each_with_index do |io,idx| + size = io.size + if pos <= size + new_index ||= idx + io.pos = pos + pos = 0 + else + io.pos = size + pos -= size + end + end + @index = new_index unless new_index.nil? + end + + def rewind + self.pos = 0 + end + end + end + end +end \ No newline at end of file diff --git a/lib/google/apis/core/download.rb b/lib/google/apis/core/download.rb index db55509a7..0078005b4 100644 --- a/lib/google/apis/core/download.rb +++ b/lib/google/apis/core/download.rb @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -require 'google/apis/core/multipart' require 'google/apis/core/api_command' require 'google/apis/errors' require 'addressable/uri' @@ -22,7 +21,7 @@ module Google module Core # Streaming/resumable media download support class DownloadCommand < ApiCommand - RANGE_HEADER = 'range' + RANGE_HEADER = 'Range' # File or IO to write content to # @return [String, File, #write] @@ -57,7 +56,7 @@ module Google # of file content. # # @private - # @param [Hurley::Client] client + # @param [HTTPClient] client # HTTP client # @yield [result, err] Result or error if block supplied # @return [Object] @@ -65,34 +64,41 @@ module Google # @raise [Google::Apis::ClientError] The request is invalid and should not be retried without modification # @raise [Google::Apis::AuthorizationError] Authorization is required def execute_once(client, &block) - client.get(@download_url || url) do |req| - apply_request_options(req) - check_if_rewind_needed = false - if @offset > 0 - logger.debug { sprintf('Resuming download from offset %d', @offset) } - req.header[RANGE_HEADER] = sprintf('bytes=%d-', @offset) - check_if_rewind_needed = true - end - req.on_body(200, 201, 206) do |res, chunk| - check_status(res.status_code, chunk) unless res.status_code.nil? - if check_if_rewind_needed && res.status_code != 206 + request_header = header.dup + apply_request_options(request_header) + + check_if_rewind_needed = false + if @offset > 0 + logger.debug { sprintf('Resuming download from offset %d', @offset) } + request_header[RANGE_HEADER] = sprintf('bytes=%d-', @offset) + check_if_rewind_needed = true + end + + http_res = client.get(url.to_s, + query: query, + header: request_header, + follow_redirect: true) do |res, chunk| + status = res.http_header.status_code.to_i + if [200, 201, 206].include?(status) + if check_if_rewind_needed && status != 206 # Oh no! Requested a chunk, but received the entire content # Attempt to rewind the stream @download_io.rewind check_if_rewind_needed = false end - logger.debug { sprintf('Writing chunk (%d bytes)', chunk.length) } @offset += chunk.length @download_io.write(chunk) @download_io.flush end end + if @close_io_on_finish result = nil else result = @download_io end + check_status(http_res.status.to_i, http_res.header, http_res.body) success(result, &block) rescue => e error(e, rethrow: true, &block) diff --git a/lib/google/apis/core/http_client_adapter.rb b/lib/google/apis/core/http_client_adapter.rb deleted file mode 100644 index a58fd4b11..000000000 --- a/lib/google/apis/core/http_client_adapter.rb +++ /dev/null @@ -1,82 +0,0 @@ -require 'httpclient' -require 'hurley' -require 'hurley/client' - -module Google - module Apis - module Core - # HTTPClient adapter for Hurley. - class HttpClientAdapter - - def call(request) - client = ::HTTPClient.new - configure_client(client, request) - - begin - ::Hurley::Response.new(request) do |res| - http_res = client.request(request.verb.to_s.upcase, request.url.to_s, nil, request.body_io, request.header.to_hash, false) do |http_res, chunk| - copy_response(http_res, res) - res.receive_body(chunk) - end - copy_response(http_res, res) - end - rescue ::HTTPClient::TimeoutError, Errno::ETIMEDOUT - raise ::Hurley::Timeout, $! - rescue ::HTTPClient::BadResponseError => err - if err.message.include?('status 407') - raise ::Hurley::ConnectionFailed, %{407 "Proxy Authentication Required "} - else - raise Hurley::ClientError, $! - end - rescue Errno::ECONNREFUSED, EOFError - raise ::Hurley::ConnectionFailed, $! - rescue => err - if defined?(OpenSSL) && OpenSSL::SSL::SSLError === err - raise Hurley::SSLError, err - else - raise - end - end - end - - def copy_response(http_res, res) - unless res.status_code - res.status_code = http_res.status.to_i - http_res.header.all.each do |(k,v)| - res.header[k] = v - end - end - end - - def configure_client(client, request) - client.transparent_gzip_decompression = true - if request.options.proxy - proxy = request.options.proxy - client.proxy = sprintf('%s:%d', proxy.host, proxy.port) - if proxy.user && proxy.password - client.set_proxy_auth proxy.user, proxy.password - end - end - if request.options.timeout - client.connect_timeout = request.options.timeout - client.receive_timeout = request.options.timeout - client.send_timeout = request.options.timeout - end - if request.options.open_timeout - client.connect_timeout = request.options.open_timeout - client.send_timeout = request.options.open_timeout - end - ssl_config = client.ssl_config - ssl_opts = request.ssl_options - ssl_config.verify_mode = ssl_opts.openssl_verify_mode - ssl_config.cert_store = ssl_opts.openssl_cert_store - ssl_config.add_trust_ca ssl_opts.ca_file if ssl_opts.ca_file - ssl_config.add_trust_ca ssl_opts.ca_path if ssl_opts.ca_path - ssl_config.client_cert = ssl_opts.openssl_client_cert if ssl_opts.openssl_client_cert - ssl_config.client_key = ssl_opts.openssl_client_key if ssl_opts.openssl_client_key - ssl_config.verify_depth = ssl_opts.verify_depth if ssl_opts.verify_depth - end - end - end - end -end diff --git a/lib/google/apis/core/http_command.rb b/lib/google/apis/core/http_command.rb index bcea5c53a..fb57d0f00 100644 --- a/lib/google/apis/core/http_command.rb +++ b/lib/google/apis/core/http_command.rb @@ -17,9 +17,6 @@ require 'addressable/template' require 'google/apis/options' require 'google/apis/errors' require 'retriable' -require 'hurley' -require 'hurley/addressable' -require 'hurley_patches' require 'google/apis/core/logging' require 'pp' @@ -41,7 +38,7 @@ module Google attr_accessor :url # HTTP headers - # @return [Hurley::Header] + # @return [Hash] attr_accessor :header # Request body @@ -53,7 +50,7 @@ module Google attr_accessor :method # HTTP Client - # @return [Hurley::Client] + # @return [HTTPClient] attr_accessor :connection # Query params @@ -75,7 +72,7 @@ module Google self.url = url self.url = Addressable::Template.new(url) if url.is_a?(String) self.method = method - self.header = Hurley::Header.new + self.header = Hash.new self.body = body self.query = {} self.params = {} @@ -83,7 +80,7 @@ module Google # Execute the command, retrying as necessary # - # @param [Hurley::Client] client + # @param [HTTPClient] client # HTTP client # @yield [result, err] Result or error if block supplied # @return [Object] @@ -166,7 +163,7 @@ module Google # # @param [Fixnum] status # HTTP status code of response - # @param [Hurley::Header] header + # @param [Hash] header # Response headers # @param [String, #read] body # Response body @@ -177,7 +174,7 @@ module Google # @raise [Google::Apis::AuthorizationError] Authorization is required def process_response(status, header, body) check_status(status, header, body) - decode_response_body(header[:content_type], body) + decode_response_body(header['Content-Type'].first, body) end # Check the response and raise error if needed @@ -185,7 +182,7 @@ module Google # @param [Fixnum] status # HTTP status code of response # @param - # @param [Hurley::Header] header + # @param [Hash] header # HTTP response headers # @param [String] body # HTTP response body @@ -201,7 +198,7 @@ module Google when 200...300 nil when 301, 302, 303, 307 - message ||= sprintf('Redirect to %s', header[:location]) + message ||= sprintf('Redirect to %s', header['Location']) raise Google::Apis::RedirectError.new(message, status_code: status, header: header, body: body) when 401 message ||= 'Unauthorized' @@ -251,7 +248,16 @@ module Google # @raise [StandardError] if no block def error(err, rethrow: false, &block) logger.debug { sprintf('Error - %s', PP.pp(err, '')) } - err = Google::Apis::TransmissionError.new(err) if err.is_a?(Hurley::ClientError) || err.is_a?(SocketError) + if err.is_a?(HTTPClient::BadResponseError) + begin + res = err.res + check_status(res.status.to_i, res.header, res.body) + rescue Google::Apis::Error => e + err = e + end + elsif err.is_a?(HTTPClient::TimeoutError) || err.is_a?(SocketError) + err = Google::Apis::TransmissionError.new(err) + end block.call(nil, err) if block_given? fail err if rethrow || block.nil? end @@ -259,7 +265,7 @@ module Google # Execute the command once. # # @private - # @param [Hurley::Client] client + # @param [HTTPClient] client # HTTP client # @return [Object] # @raise [Google::Apis::ServerError] An error occurred on the server and the request can be retried @@ -269,21 +275,18 @@ module Google body.rewind if body.respond_to?(:rewind) begin logger.debug { sprintf('Sending HTTP %s %s', method, url) } - response = client.send(method, url, body) do |req| - # Temporary workaround for Hurley bug where the connection preference - # is ignored and it uses nested anyway - unless form_encoded? - req.url.query_class = Hurley::Query::Flat - query.each do | k, v| - req.url.query[k] = normalize_query_value(v) - end - end - # End workaround - apply_request_options(req) - end - logger.debug { response.status_code } - logger.debug { response.inspect } - response = process_response(response.status_code, response.header, response.body) + request_header = header.dup + apply_request_options(request_header) + + http_res = client.request(method.to_s.upcase, + url.to_s, + query: nil, + body: body, + header: request_header, + follow_redirect: true) + logger.debug { http_res.status } + logger.debug { http_res.inspect } + response = process_response(http_res.status.to_i, http_res.header, http_res.body) success(response) rescue => e logger.debug { sprintf('Caught error %s', e) } @@ -292,18 +295,16 @@ module Google end # Update the request with any specified options. - # @param [Hurley::Request] req - # HTTP request + # @param [Hash] header + # HTTP headers # @return [void] - def apply_request_options(req) + def apply_request_options(req_header) if options.authorization.respond_to?(:apply!) - options.authorization.apply!(req.header) + options.authorization.apply!(req_header) elsif options.authorization.is_a?(String) - req.header[:authorization] = sprintf('Bearer %s', options.authorization) + req_header['Authorization'] = sprintf('Bearer %s', options.authorization) end - req.header.update(header) - req.options.timeout = options.timeout_sec - req.options.open_timeout = options.open_timeout_sec + req_header.update(header) end private diff --git a/lib/google/apis/core/multipart.rb b/lib/google/apis/core/multipart.rb index c6063fac3..d2d678d67 100644 --- a/lib/google/apis/core/multipart.rb +++ b/lib/google/apis/core/multipart.rb @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -require 'hurley' module Google module Apis @@ -21,108 +20,60 @@ module Google # # @private class JsonPart - include Hurley::Multipart::Part - # @return [Fixnum] - # Length of part - attr_reader :length - - # @param [String] boundary - # Multipart boundary # @param [String] value # JSON content - def initialize(boundary, value, header = {}) - @part = build_part(boundary, value) - @length = @part.bytesize - @io = StringIO.new(@part) + # @param [Hash] header + # Additional headers + def initialize(value, header = {}) + @value = value + @header = header end - private - - # Format the part - # - # @param [String] boundary - # Multipart boundary - # @param [String] value - # JSON content - # @return [String] - def build_part(boundary, value) + def to_io(boundary) part = '' part << "--#{boundary}\r\n" part << "Content-Type: application/json\r\n" + @header.each do |(k, v)| + part << "#{k}: #{v}\r\n" + end part << "\r\n" - part << "#{value}\r\n" + part << "#{@value}\r\n" + StringIO.new(part) end + end - # Part of a multipart request for holding arbitrary content. Modified - # from Hurley::Multipart::FilePart to remove Content-Disposition + # Part of a multipart request for holding arbitrary content. # # @private class FilePart - include Hurley::Multipart::Part - - # @return [Fixnum] - # Length of part - attr_reader :length - - # @param [String] boundary - # Multipart boundary - # @param [Google::Apis::Core::UploadIO] io + # @param [IO] io # IO stream # @param [Hash] header # Additional headers - def initialize(boundary, io, header = {}) - file_length = io.respond_to?(:length) ? io.length : File.size(io.local_path) - - @head = build_head(boundary, io.content_type, file_length, - io.respond_to?(:opts) ? io.opts.merge(header) : header) - - @length = @head.bytesize + file_length + FOOT.length - @io = Hurley::CompositeReadIO.new(@length, StringIO.new(@head), io, StringIO.new(FOOT)) + def initialize(io, header = {}) + @io = io + @header = header + @length = io.respond_to?(:size) ? io.size : nil end - private - - # Construct the header for the part - # - # @param [String] boundary - # Multipart boundary - # @param [String] type - # Content type for the part - # @param [Fixnum] content_len - # Length of the part - # @param [Hash] header - # Headers for the part - def build_head(boundary, type, content_len, header) - content_id = '' - if header[:content_id] - content_id = sprintf(CID_FORMAT, header[:content_id]) + def to_io(boundary) + head = '' + head << "--#{boundary}\r\n" + @header.each do |(k, v)| + head << "#{k}: #{v}\r\n" end - sprintf(HEAD_FORMAT, - boundary, - content_len.to_i, - content_id, - header[:content_type] || type, - header[:content_transfer_encoding] || DEFAULT_TR_ENCODING) + head << "Content-Length: #{@length}\r\n" unless @length.nil? + head << "Content-Transfer-Encoding: binary\r\n" + head << "\r\n" + Google::Apis::Core::CompositeIO.new(StringIO.new(head), @io, StringIO.new("\r\n")) end - - DEFAULT_TR_ENCODING = 'binary'.freeze - FOOT = "\r\n".freeze - CID_FORMAT = "Content-ID: %s\r\n" - HEAD_FORMAT = <<-END ---%s\r -Content-Length: %d\r -%sContent-Type: %s\r -Content-Transfer-Encoding: %s\r -\r - END end # Helper for building multipart requests class Multipart MULTIPART_RELATED = 'multipart/related' - DEFAULT_BOUNDARY = 'RubyApiClientMultiPart' # @return [String] # Content type header @@ -135,8 +86,8 @@ Content-Transfer-Encoding: %s\r def initialize(content_type: MULTIPART_RELATED, boundary: nil) @parts = [] - @boundary = boundary || DEFAULT_BOUNDARY - @content_type = "#{content_type}; boundary=#{boundary}" + @boundary = boundary || Digest::SHA1.hexdigest(SecureRandom.random_bytes(8)) + @content_type = "#{content_type}; boundary=#{@boundary}" end # Append JSON data part @@ -147,23 +98,26 @@ Content-Transfer-Encoding: %s\r # Optional unique ID of this part # @return [self] def add_json(body, content_id: nil) - header = { :content_id => content_id } - @parts << Google::Apis::Core::JsonPart.new(@boundary, body, header) + header = {} + header['Content-ID'] = content_id unless content_id.nil? + @parts << Google::Apis::Core::JsonPart.new(body, header).to_io(@boundary) self end # Append arbitrary data as a part # - # @param [Google::Apis::Core::UploadIO] upload_io + # @param [IO] upload_io # IO stream # @param [String] content_id # Optional unique ID of this part # @return [self] - def add_upload(upload_io, content_id: nil) - header = { :content_id => content_id } - @parts << Google::Apis::Core::FilePart.new(@boundary, - upload_io, - header) + def add_upload(upload_io, content_type: nil, content_id: nil) + header = { + 'Content-Type' => content_type || 'application/octet-stream' + } + header['Content-Id'] = content_id unless content_id.nil? + @parts << Google::Apis::Core::FilePart.new(upload_io, + header).to_io(@boundary) self end @@ -172,16 +126,10 @@ Content-Transfer-Encoding: %s\r # @return [IO] # IO stream def assemble - @parts << Hurley::Multipart::EpiloguePart.new(@boundary) - ios = [] - len = 0 - @parts.each do |part| - len += part.length - ios << part.to_io - end - Hurley::CompositeReadIO.new(len, *ios) + @parts << StringIO.new("--#{@boundary}--\r\n\r\n") + Google::Apis::Core::CompositeIO.new(*@parts) end end end end -end +end \ No newline at end of file diff --git a/lib/google/apis/core/upload.rb b/lib/google/apis/core/upload.rb index 4d97773d5..9c2cd3151 100644 --- a/lib/google/apis/core/upload.rb +++ b/lib/google/apis/core/upload.rb @@ -29,41 +29,6 @@ end module Google module Apis module Core - # Extension of Hurley's UploadIO to add length accessor - class UploadIO < Hurley::UploadIO - OCTET_STREAM_CONTENT_TYPE = 'application/octet-stream' - - # Get the length of the stream - # @return [Fixnum] - def length - io.respond_to?(:length) ? io.length : File.size(local_path) - end - - # Create a new instance given a file path - # @param [String, File] file_name - # Path to file - # @param [String] content_type - # Optional content type. If nil, will attempt to auto-detect - # @return [Google::Apis::Core::UploadIO] - def self.from_file(file_name, content_type: nil) - if content_type.nil? - type = MIME::Types.of(file_name) - content_type = type.first.content_type unless type.nil? || type.empty? - end - new(file_name, content_type || OCTET_STREAM_CONTENT_TYPE) - end - - # Wraps an IO stream in UploadIO - # @param [#read] io - # IO to wrap - # @param [String] content_type - # Optional content type. - # @return [Google::Apis::Core::UploadIO] - def self.from_io(io, content_type: OCTET_STREAM_CONTENT_TYPE) - new(io, content_type) - end - end - # Base upload command. Not intended to be used directly # @private class BaseUploadCommand < ApiCommand @@ -83,17 +48,17 @@ module Google # @return [Google::Apis::Core::UploadIO] attr_accessor :upload_io - # Ensure the content is readable and wrapped in an {{Google::Apis::Core::UploadIO}} instance. + # Ensure the content is readable and wrapped in an IO instance. # # @return [void] # @raise [Google::Apis::ClientError] if upload source is invalid def prepare! super if streamable?(upload_source) - self.upload_io = UploadIO.from_io(upload_source, content_type: upload_content_type) + self.upload_io = upload_source @close_io_on_finish = false elsif upload_source.is_a?(String) - self.upload_io = UploadIO.from_file(upload_source, content_type: upload_content_type) + self.upload_io = File.new(upload_source, 'r') @close_io_on_finish = true else fail Google::Apis::ClientError, 'Invalid upload source' @@ -124,13 +89,12 @@ module Google super self.body = upload_io header[UPLOAD_PROTOCOL_HEADER] = RAW_PROTOCOL - header[UPLOAD_CONTENT_TYPE_HEADER] = upload_io.content_type + header[UPLOAD_CONTENT_TYPE_HEADER] = upload_content_type end end # Implementation of the multipart upload protocol class MultipartUploadCommand < BaseUploadCommand - UPLOAD_BOUNDARY = 'RubyApiClientUpload' MULTIPART_PROTOCOL = 'multipart' MULTIPART_RELATED = 'multipart/related' @@ -140,11 +104,11 @@ module Google # @raise [Google::Apis::ClientError] if upload source is invalid def prepare! super - @multipart = Multipart.new(boundary: UPLOAD_BOUNDARY, content_type: MULTIPART_RELATED) - @multipart.add_json(body) - @multipart.add_upload(upload_io) - self.body = @multipart.assemble - header[:content_type] = @multipart.content_type + multipart = Multipart.new + multipart.add_json(body) + multipart.add_upload(upload_io, content_type: upload_content_type) + self.body = multipart.assemble + header['Content-Type'] = multipart.content_type header[UPLOAD_PROTOCOL_HEADER] = MULTIPART_PROTOCOL end end @@ -179,7 +143,7 @@ module Google # # @param [Fixnum] status # HTTP status code of response - # @param [Hurley::Header] header + # @param [HTTP::Message::Headers] header # Response headers # @param [String, #read] body # Response body @@ -189,9 +153,9 @@ module Google # @raise [Google::Apis::ClientError] The request is invalid and should not be retried without modification # @raise [Google::Apis::AuthorizationError] Authorization is required def process_response(status, header, body) - @offset = Integer(header[BYTES_RECEIVED_HEADER]) if header.key?(BYTES_RECEIVED_HEADER) - @upload_url = header[UPLOAD_URL_HEADER] if header.key?(UPLOAD_URL_HEADER) - upload_status = header[UPLOAD_STATUS_HEADER] + @offset = Integer(header[BYTES_RECEIVED_HEADER].first) unless header[BYTES_RECEIVED_HEADER].empty? + @upload_url = header[UPLOAD_URL_HEADER].first unless header[UPLOAD_URL_HEADER].empty? + upload_status = header[UPLOAD_STATUS_HEADER].first logger.debug { sprintf('Upload status %s', upload_status) } if upload_status == STATUS_ACTIVE @state = :active @@ -204,61 +168,68 @@ module Google super(status, header, body) end - # Send the start command to initiate the upload - # - # @param [Hurley::Client] client - # HTTP client - # @return [Hurley::Response] - # @raise [Google::Apis::ServerError] Unable to send the request def send_start_command(client) logger.debug { sprintf('Sending upload start command to %s', url) } - client.send(method, url, body) do |req| - apply_request_options(req) - req.header[UPLOAD_PROTOCOL_HEADER] = RESUMABLE - req.header[UPLOAD_COMMAND_HEADER] = START_COMMAND - req.header[UPLOAD_CONTENT_LENGTH] = upload_io.length.to_s - req.header[UPLOAD_CONTENT_TYPE_HEADER] = upload_io.content_type - end + + request_header = header.dup + apply_request_options(request_header) + request_header[UPLOAD_PROTOCOL_HEADER] = RESUMABLE + request_header[UPLOAD_COMMAND_HEADER] = START_COMMAND + request_header[UPLOAD_CONTENT_LENGTH] = upload_io.size.to_s + request_header[UPLOAD_CONTENT_TYPE_HEADER] = upload_content_type + + client.request(method.to_s.upcase, + url.to_s, query: nil, + body: body, + header: request_header, + follow_redirect: true) rescue => e raise Google::Apis::ServerError, e.message end # Query for the status of an incomplete upload # - # @param [Hurley::Client] client + # @param [HTTPClient] client # HTTP client - # @return [Hurley::Response] + # @return [HTTP::Message] # @raise [Google::Apis::ServerError] Unable to send the request def send_query_command(client) logger.debug { sprintf('Sending upload query command to %s', @upload_url) } - client.post(@upload_url, nil) do |req| - apply_request_options(req) - req.header[UPLOAD_COMMAND_HEADER] = QUERY_COMMAND - end + + request_header = header.dup + apply_request_options(request_header) + request_header[UPLOAD_COMMAND_HEADER] = QUERY_COMMAND + + client.post(@upload_url, header: request_header, follow_redirect: true) end + # Send the actual content # - # @param [Hurley::Client] client + # @param [HTTPClient] client # HTTP client - # @return [Hurley::Response] + # @return [HTTP::Message] # @raise [Google::Apis::ServerError] Unable to send the request def send_upload_command(client) logger.debug { sprintf('Sending upload command to %s', @upload_url) } + content = upload_io content.pos = @offset - client.post(@upload_url, content) do |req| - apply_request_options(req) - req.header[UPLOAD_COMMAND_HEADER] = UPLOAD_COMMAND - req.header[UPLOAD_OFFSET_HEADER] = @offset.to_s - end + + request_header = header.dup + apply_request_options(request_header) + request_header[UPLOAD_COMMAND_HEADER] = QUERY_COMMAND + request_header[UPLOAD_COMMAND_HEADER] = UPLOAD_COMMAND + request_header[UPLOAD_OFFSET_HEADER] = @offset.to_s + + client.post(@upload_url, body: content, header: request_header, follow_redirect: true) end # Execute the upload request once. This will typically perform two HTTP requests -- one to initiate or query # for the status of the upload, the second to send the (remaining) content. # # @private - # @param [Hurley::Client] client + # @param [HTTPClient] client # HTTP client # @yield [result, err] Result or error if block supplied # @return [Object] diff --git a/spec/google/apis/core/api_command_spec.rb b/spec/google/apis/core/api_command_spec.rb index 4920c496c..93122fae8 100644 --- a/spec/google/apis/core/api_command_spec.rb +++ b/spec/google/apis/core/api_command_spec.rb @@ -15,7 +15,6 @@ require 'spec_helper' require 'google/apis/core/api_command' require 'google/apis/core/json_representation' -require 'hurley/test' RSpec.describe Google::Apis::Core::HttpCommand do include TestHelpers diff --git a/spec/google/apis/core/batch_spec.rb b/spec/google/apis/core/batch_spec.rb index f784a03fa..46e6e65de 100644 --- a/spec/google/apis/core/batch_spec.rb +++ b/spec/google/apis/core/batch_spec.rb @@ -15,7 +15,6 @@ require 'spec_helper' require 'google/apis/core/batch' require 'google/apis/core/json_representation' -require 'hurley/test' RSpec.describe Google::Apis::Core::BatchCommand do include TestHelpers @@ -30,19 +29,20 @@ RSpec.describe Google::Apis::Core::BatchCommand do let(:post_with_string_command) do command = Google::Apis::Core::HttpCommand.new(:post, 'https://www.googleapis.com/zoo/animals/2') command.body = 'Hello world' - command.header[:content_type] = 'text/plain' + command.header['Content-Type'] = 'text/plain' command end let(:post_with_io_command) do command = Google::Apis::Core::HttpCommand.new(:post, 'https://www.googleapis.com/zoo/animals/3') command.body = StringIO.new('Goodbye!') - command.header[:content_type] = 'text/plain' + command.header['Content-Type'] = 'text/plain' command end before(:example) do allow(SecureRandom).to receive(:uuid).and_return('ffe23d1b-e8f7-47f5-8c01-2a30cf8ecb8f') + allow(Digest::SHA1).to receive(:hexdigest).and_return('123abc') response = < +--123abc Content-Type: application/http +Content-Id: +Content-Length: 58 Content-Transfer-Encoding: binary GET /zoo/animals/1? HTTP/1.1 Host: www.googleapis.com ---RubyApiBatchRequest -Content-Length: 96 -Content-ID: +--123abc Content-Type: application/http +Content-Id: +Content-Length: 96 Content-Transfer-Encoding: binary POST /zoo/animals/2? HTTP/1.1 @@ -104,10 +104,10 @@ Content-Type: text/plain Host: www.googleapis.com Hello world ---RubyApiBatchRequest -Content-Length: 93 -Content-ID: +--123abc Content-Type: application/http +Content-Id: +Content-Length: 93 Content-Transfer-Encoding: binary POST /zoo/animals/3? HTTP/1.1 @@ -115,7 +115,7 @@ Content-Type: text/plain Host: www.googleapis.com Goodbye! ---RubyApiBatchRequest-- +--123abc-- EOF expect(a_request(:post, 'https://www.googleapis.com/batch').with(body: expected_body)).to have_been_made diff --git a/spec/google/apis/core/composite_io_spec.rb b/spec/google/apis/core/composite_io_spec.rb new file mode 100644 index 000000000..9bf6da21b --- /dev/null +++ b/spec/google/apis/core/composite_io_spec.rb @@ -0,0 +1,76 @@ +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'spec_helper' +require 'google/apis/core/composite_io' + +RSpec.describe Google::Apis::Core::CompositeIO do + + shared_examples 'should act like IO' do + it 'should read from all IOs' do + expect(io.read).to eq 'Hello Cruel World' + end + + it 'should respond to size' do + expect(io.size).to eq 17 + end + + it 'should respond to pos=' do + io.pos = 6 + expect(io.read).to eq('Cruel World') + end + + it 'should reject negative positions' do + expect { io.pos = -1 }.to raise_error(ArgumentError) + end + + + it 'should return nil if position beyond size' do + io.pos = 20 + expect(io.read).to be_nil + end + + it 'should be readable after rewinding' do + expect(io.read).to eq 'Hello Cruel World' + expect(io.read).to be_nil + io.rewind + expect(io.read).to eq 'Hello Cruel World' + end + end + + context 'with StringIOs' do + let(:io) do + Google::Apis::Core::CompositeIO.new( + StringIO.new("Hello "), + StringIO.new("Cruel "), + StringIO.new("World")) + end + include_examples 'should act like IO' + end + + context 'with Files' do + let(:io) do + files = [] + dir = Dir.mktmpdir + ['Hello ', 'Cruel ', 'World'].each_with_index do |text, index| + name = File.join(dir, "f#{index}") + File.open(name, 'w') { |f| f.write(text) } + files << name + end + Google::Apis::Core::CompositeIO.new(files.map { |name| File.open(name, 'r') }) + end + include_examples 'should act like IO' + end + +end \ No newline at end of file diff --git a/spec/google/apis/core/download_spec.rb b/spec/google/apis/core/download_spec.rb index 9f56beb0f..93620017c 100644 --- a/spec/google/apis/core/download_spec.rb +++ b/spec/google/apis/core/download_spec.rb @@ -15,7 +15,6 @@ require 'spec_helper' require 'google/apis/core/download' require 'google/apis/core/json_representation' -require 'hurley/test' require 'tempfile' require 'tmpdir' diff --git a/spec/google/apis/core/http_command_spec.rb b/spec/google/apis/core/http_command_spec.rb index e9e9ead39..b64a638b7 100644 --- a/spec/google/apis/core/http_command_spec.rb +++ b/spec/google/apis/core/http_command_spec.rb @@ -14,7 +14,6 @@ require 'spec_helper' require 'google/apis/core/http_command' -require 'hurley/test' RSpec.describe Google::Apis::Core::HttpCommand do include TestHelpers diff --git a/spec/google/apis/core/service_spec.rb b/spec/google/apis/core/service_spec.rb index ae5fa6d44..38c674b7e 100644 --- a/spec/google/apis/core/service_spec.rb +++ b/spec/google/apis/core/service_spec.rb @@ -16,8 +16,6 @@ require 'spec_helper' require 'google/apis/options' require 'google/apis/core/base_service' require 'google/apis/core/json_representation' -require 'hurley/test' -require 'ostruct' RSpec.describe Google::Apis::Core::BaseService do include TestHelpers @@ -190,6 +188,8 @@ EOF context 'with batch uploads' do before(:example) do + allow(SecureRandom).to receive(:uuid).and_return('b1981e17-f622-49af-b2eb-203308b1b17d') + allow(Digest::SHA1).to receive(:hexdigest).and_return('outer', 'inner') response = < +Content-Length: 303 +Content-Transfer-Encoding: binary + +POST /upload/zoo/animals? HTTP/1.1 +Content-Type: multipart/related; boundary=inner +X-Goog-Upload-Protocol: multipart +Host: www.googleapis.com + +--inner +Content-Type: application/json + + +--inner +Content-Type: text/plain +Content-Length: 4 +Content-Transfer-Encoding: binary + +test +--inner-- + + +--outer-- + +EOF + expect(a_request(:put, 'https://www.googleapis.com/upload/').with(body: expected_body)).to have_been_made + end + it 'should disallow downloads in batch' do expect do |b| service.batch_upload do |service| diff --git a/spec/google/apis/core/upload_spec.rb b/spec/google/apis/core/upload_spec.rb index 99039f8f0..6e338a15d 100644 --- a/spec/google/apis/core/upload_spec.rb +++ b/spec/google/apis/core/upload_spec.rb @@ -15,70 +15,6 @@ require 'spec_helper' require 'google/apis/core/upload' require 'google/apis/core/json_representation' -require 'hurley/test' - -# TODO: JSON Response decoding -# TODO: Upload from IO -# TODO: Upload from file - -RSpec.describe Google::Apis::Core::UploadIO do - context 'from_file' do - let(:upload_io) { Google::Apis::Core::UploadIO.from_file(file) } - - context 'with text file' do - let(:file) { File.join(FIXTURES_DIR, 'files', 'test.txt') } - it 'should infer content type from file' do - expect(upload_io.content_type).to eql('text/plain') - end - - it 'should allow overriding the mime type' do - io = Google::Apis::Core::UploadIO.from_file(file, content_type: 'application/json') - expect(io.content_type).to eql('application/json') - end - end - - context 'with unknown type' do - let(:file) { File.join(FIXTURES_DIR, 'files', 'test.blah') } - it 'should use the default mime type' do - expect(upload_io.content_type).to eql('application/octet-stream') - end - - it 'should allow overriding the mime type' do - io = Google::Apis::Core::UploadIO.from_file(file, content_type: 'application/json') - expect(io.content_type).to eql('application/json') - end - - it 'should setup length of the stream' do - upload_io = Google::Apis::Core::UploadIO.from_file(file) - expect(upload_io.length).to eq File.size(file) - end - - end - end - - context 'from_io' do - - context 'with i/o stream' do - let(:io) { StringIO.new 'Hello google' } - - it 'should setup default content-type' do - upload_io = Google::Apis::Core::UploadIO.from_io(io) - expect(upload_io.content_type).to eql Google::Apis::Core::UploadIO::OCTET_STREAM_CONTENT_TYPE - end - - it 'should allow overring the mime type' do - upload_io = Google::Apis::Core::UploadIO.from_io(io, content_type: 'application/x-gzip') - expect(upload_io.content_type).to eq('application/x-gzip') - end - - it 'should setup length of the stream' do - upload_io = Google::Apis::Core::UploadIO.from_io(io) - expect(upload_io.length).to eq 'Hello google'.length - end - end - - end -end RSpec.describe Google::Apis::Core::RawUploadCommand do include TestHelpers @@ -170,21 +106,22 @@ RSpec.describe Google::Apis::Core::MultipartUploadCommand do before(:example) do stub_request(:post, 'https://www.googleapis.com/zoo/animals').to_return(body: %(Hello world)) + allow(Digest::SHA1).to receive(:hexdigest).and_return('123abc') end it 'should send content' do expected_body = < err - if defined?(OpenSSL) && OpenSSL::SSL::SSLError === err - raise SSLError, err - else - raise ConnectionFailed, err - end - end - end - - rescue ::Timeout::Error => err - raise Timeout, err - end - - private - - def net_http_request(request) - http_req = Net::HTTPGenericRequest.new( - request.verb.to_s.upcase, # request method - !!request.body, # is there a request body - :head != request.verb, # is there a response body - request.url.request_uri, # request uri path - request.header, # request headers - ) - - if body = request.body_io - http_req.body_stream = body - end - - http_req - end - - def perform_request(http, request, res) - http.request(net_http_request(request)) do |http_res| - res.status_code = http_res.code.to_i - http_res.each_header do |key, value| - res.header[key] = value - end - - if :get == request.verb - http_res.read_body { |chunk| res.receive_body(chunk) } - else - res.receive_body(http_res.body) - end - end - end - end - end -end From 2578736b44e2fb5234834a5d531c8063004ca41a Mon Sep 17 00:00:00 2001 From: Steve Bazyl Date: Wed, 17 Aug 2016 13:58:07 -0700 Subject: [PATCH 02/16] #445 - Treat 429 status codes as rate limit errors --- lib/google/apis/core/http_command.rb | 3 +++ spec/google/apis/core/http_command_spec.rb | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/lib/google/apis/core/http_command.rb b/lib/google/apis/core/http_command.rb index fb57d0f00..9e376660b 100644 --- a/lib/google/apis/core/http_command.rb +++ b/lib/google/apis/core/http_command.rb @@ -203,6 +203,9 @@ module Google when 401 message ||= 'Unauthorized' raise Google::Apis::AuthorizationError.new(message, status_code: status, header: header, body: body) + when 429 + message ||= 'Rate limit exceeded' + raise Google::Apis::RateLimitError.new(message, status_code: status, header: header, body: body) when 304, 400, 402...500 message ||= 'Invalid request' raise Google::Apis::ClientError.new(message, status_code: status, header: header, body: body) diff --git a/spec/google/apis/core/http_command_spec.rb b/spec/google/apis/core/http_command_spec.rb index b64a638b7..22e78e2ab 100644 --- a/spec/google/apis/core/http_command_spec.rb +++ b/spec/google/apis/core/http_command_spec.rb @@ -299,4 +299,11 @@ RSpec.describe Google::Apis::Core::HttpCommand do command.options.retries = 0 expect { command.execute(client) }.to raise_error(Google::Apis::TransmissionError) end + + it 'should raise rate limit error for 429 status codes' do + stub_request(:get, 'https://www.googleapis.com/zoo/animals').to_return(status: [429, '']) + command = Google::Apis::Core::HttpCommand.new(:get, 'https://www.googleapis.com/zoo/animals') + command.options.retries = 0 + expect { command.execute(client) }.to raise_error(Google::Apis::RateLimitError) + end end From e6acda71180733eb3d8d435787c068d710ee37dd Mon Sep 17 00:00:00 2001 From: Steve Bazyl Date: Wed, 17 Aug 2016 14:18:41 -0700 Subject: [PATCH 03/16] Move dependencies used at generation time only to dev dependencies --- Gemfile | 1 - bin/generate-api | 8 +++++++- google-api-client.gemspec | 4 +++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/Gemfile b/Gemfile index 9c1d9e077..42ae10e3c 100644 --- a/Gemfile +++ b/Gemfile @@ -21,7 +21,6 @@ group :development do gem 'rmail', '~> 1.1' gem 'sinatra', '~> 1.4' gem 'redis', '~> 3.2' - gem 'activesupport', '>= 3.2', '< 5.0' end platforms :jruby do diff --git a/bin/generate-api b/bin/generate-api index ff600b452..bebeea5e3 100755 --- a/bin/generate-api +++ b/bin/generate-api @@ -1,6 +1,12 @@ #!/usr/bin/env ruby -require 'thor' +begin + require 'thor' +rescue LoadError => e + puts "Thor is required. Please install the gem with development dependencies." + exit 1 +end + require 'open-uri' require 'google/apis/discovery_v1' require 'logger' diff --git a/google-api-client.gemspec b/google-api-client.gemspec index ce6330bb6..e094441d2 100644 --- a/google-api-client.gemspec +++ b/google-api-client.gemspec @@ -25,7 +25,9 @@ Gem::Specification.new do |spec| spec.add_runtime_dependency 'addressable', '~> 2.3' spec.add_runtime_dependency 'mime-types', '>= 1.6' spec.add_runtime_dependency 'googleauth', '~> 0.5' - spec.add_runtime_dependency 'thor', '~> 0.19' spec.add_runtime_dependency 'httpclient', '>= 2.8.1', '< 3.0' spec.add_runtime_dependency 'memoist', '~> 0.11' + spec.add_development_dependency 'thor', '~> 0.14' + spec.add_development_dependency 'activesupport', '>= 3.2', '< 5.0' + end From 2c190e9745f09492da1d9d41a553598ced80c12b Mon Sep 17 00:00:00 2001 From: Steve Bazyl Date: Wed, 17 Aug 2016 14:38:25 -0700 Subject: [PATCH 04/16] Update representable, mimetype dependencies + others --- Gemfile | 4 ++-- google-api-client.gemspec | 4 ++-- lib/google/apis/core/api_command.rb | 2 +- lib/google/apis/core/json_representation.rb | 4 ++-- lib/google/apis/core/upload.rb | 12 +++++------- spec/google/apis/core/json_representation_spec.rb | 4 ++-- spec/google/apis/errors_spec.rb | 5 ++--- 7 files changed, 16 insertions(+), 19 deletions(-) diff --git a/Gemfile b/Gemfile index 42ae10e3c..bb5be5597 100644 --- a/Gemfile +++ b/Gemfile @@ -10,8 +10,8 @@ group :development do gem 'rspec', '~> 3.1' gem 'json_spec', '~> 1.1' gem 'webmock', '~> 1.21' - gem 'simplecov', '~> 0.9' - gem 'coveralls', '~> 0.7.11' + gem 'simplecov', '~> 0.12' + gem 'coveralls', '~> 0.8' gem 'rubocop', '~> 0.29' gem 'launchy', '~> 2.4' gem 'dotenv', '~> 2.0' diff --git a/google-api-client.gemspec b/google-api-client.gemspec index e094441d2..b3d45ec56 100644 --- a/google-api-client.gemspec +++ b/google-api-client.gemspec @@ -20,10 +20,10 @@ Gem::Specification.new do |spec| spec.required_ruby_version = '~> 2.0' - spec.add_runtime_dependency 'representable', '~> 2.3.0' + spec.add_runtime_dependency 'representable', '~> 3.0' spec.add_runtime_dependency 'retriable', '~> 2.0' spec.add_runtime_dependency 'addressable', '~> 2.3' - spec.add_runtime_dependency 'mime-types', '>= 1.6' + spec.add_runtime_dependency 'mime-types', '>= 3.0' spec.add_runtime_dependency 'googleauth', '~> 0.5' spec.add_runtime_dependency 'httpclient', '>= 2.8.1', '< 3.0' spec.add_runtime_dependency 'memoist', '~> 0.11' diff --git a/lib/google/apis/core/api_command.rb b/lib/google/apis/core/api_command.rb index e76e3512c..62dbf7e52 100644 --- a/lib/google/apis/core/api_command.rb +++ b/lib/google/apis/core/api_command.rb @@ -51,7 +51,7 @@ module Google query[FIELDS_PARAM] = normalize_fields_param(query[FIELDS_PARAM]) if query.key?(FIELDS_PARAM) if request_representation && request_object header['Content-Type'] ||= JSON_CONTENT_TYPE - self.body = request_representation.new(request_object).to_json(skip_undefined: true) + self.body = request_representation.new(request_object).to_json(user_options: { skip_undefined: true }) end super end diff --git a/lib/google/apis/core/json_representation.rb b/lib/google/apis/core/json_representation.rb index 6194efcb5..bb3d9906a 100644 --- a/lib/google/apis/core/json_representation.rb +++ b/lib/google/apis/core/json_representation.rb @@ -55,7 +55,7 @@ module Google def if_fn(name) ivar_name = "@#{name}".to_sym lambda do |opts| - if opts[:skip_undefined] + if opts[:user_options] && opts[:user_options][:skip_undefined] if respond_to?(:key?) self.key?(name) || instance_variable_defined?(ivar_name) else @@ -137,7 +137,7 @@ module Google def to_json representation = self.class.const_get(:Representation) - representation.new(self).to_json(skip_undefined: true) + representation.new(self).to_json(user_options: { skip_undefined: true }) end end end diff --git a/lib/google/apis/core/upload.rb b/lib/google/apis/core/upload.rb index 9c2cd3151..6aa88ec41 100644 --- a/lib/google/apis/core/upload.rb +++ b/lib/google/apis/core/upload.rb @@ -18,13 +18,7 @@ require 'google/apis/core/api_command' require 'google/apis/errors' require 'addressable/uri' require 'tempfile' -begin - require 'mime/types/columnar' -rescue LoadError - # Temporary until next major bump when we can tighten - # dependency to mime-types >-=3.0 - require 'mime-types' -end +require 'mime-types' module Google module Apis @@ -59,6 +53,10 @@ module Google @close_io_on_finish = false elsif upload_source.is_a?(String) self.upload_io = File.new(upload_source, 'r') + if upload_content_type.nil? + type = MIME::Types.of(upload_source) + upload_content_type = type.first.content_type unless type.nil? || type.empty? + end @close_io_on_finish = true else fail Google::Apis::ClientError, 'Invalid upload source' diff --git a/spec/google/apis/core/json_representation_spec.rb b/spec/google/apis/core/json_representation_spec.rb index 005dabf39..7d5fbc462 100644 --- a/spec/google/apis/core/json_representation_spec.rb +++ b/spec/google/apis/core/json_representation_spec.rb @@ -109,7 +109,7 @@ RSpec.describe Google::Apis::Core::JsonRepresentation do end context 'with model object' do - let(:json) { representer_class.new(model).to_json(skip_undefined: true) } + let(:json) { representer_class.new(model).to_json(user_options: { skip_undefined: true }) } let(:model) do model = model_class.new model.nil_value = nil @@ -132,7 +132,7 @@ RSpec.describe Google::Apis::Core::JsonRepresentation do end context 'with hash' do - let(:json) { representer_class.new(model).to_json(skip_undefined: true) } + let(:json) { representer_class.new(model).to_json(user_options: { skip_undefined: true }) } let(:model) do { nil_value: nil, diff --git a/spec/google/apis/errors_spec.rb b/spec/google/apis/errors_spec.rb index ad2a576e3..394ce7d1f 100644 --- a/spec/google/apis/errors_spec.rb +++ b/spec/google/apis/errors_spec.rb @@ -80,9 +80,8 @@ RSpec.describe Google::Apis::Error do context '@cause is falsy' do before do - subject.class.superclass.any_instance.stub(:backtrace) do - "super class's #backtrace called" - end + expect_any_instance_of(subject.class.superclass).to receive(:backtrace).and_return( + "super class's #backtrace called") end it "calls super class's #backtrace" do From 39d0a164b575e248024f70ea643dbb030ee0f7b8 Mon Sep 17 00:00:00 2001 From: Steve Bazyl Date: Wed, 17 Aug 2016 15:27:23 -0700 Subject: [PATCH 05/16] Update generator to correctly handle int64/uint64 types as fixnum/bignum instead of strings --- lib/google/apis/core/base_service.rb | 1 + lib/google/apis/core/json_representation.rb | 4 ++++ lib/google/apis/generator/model.rb | 2 ++ .../apis/generator/templates/_representation.tmpl | 2 +- spec/google/apis/core/json_representation_spec.rb | 14 ++++++++++++++ 5 files changed, 22 insertions(+), 1 deletion(-) diff --git a/lib/google/apis/core/base_service.rb b/lib/google/apis/core/base_service.rb index 93699fafa..cc01ac78c 100644 --- a/lib/google/apis/core/base_service.rb +++ b/lib/google/apis/core/base_service.rb @@ -21,6 +21,7 @@ require 'google/apis/core/upload' require 'google/apis/core/download' require 'google/apis/options' require 'googleauth' +require 'httpclient' module Google module Apis diff --git a/lib/google/apis/core/json_representation.rb b/lib/google/apis/core/json_representation.rb index bb3d9906a..bfc1e49d9 100644 --- a/lib/google/apis/core/json_representation.rb +++ b/lib/google/apis/core/json_representation.rb @@ -72,6 +72,10 @@ module Google options[:render_filter] = ->(value, _doc, *_args) { value.nil? ? nil : Base64.urlsafe_encode64(value) } options[:parse_filter] = ->(fragment, _doc, *_args) { Base64.urlsafe_decode64(fragment) } end + if options[:numeric_string] + options[:render_filter] = ->(value, _doc, *_args) { value.nil? ? nil : value.to_s} + options[:parse_filter] = ->(fragment, _doc, *_args) { fragment.to_i } + end if options[:type] == DateTime options[:render_filter] = ->(value, _doc, *_args) { value.nil? ? nil : value.is_a?(DateTime) ? value.rfc3339(3) : value.to_s } options[:parse_filter] = ->(fragment, _doc, *_args) { DateTime.parse(fragment) } diff --git a/lib/google/apis/generator/model.rb b/lib/google/apis/generator/model.rb index b2b59080f..6d30ff24d 100644 --- a/lib/google/apis/generator/model.rb +++ b/lib/google/apis/generator/model.rb @@ -51,6 +51,8 @@ module Google when 'string', 'boolean', 'number', 'integer', 'any' return 'DateTime' if format == 'date-time' return 'Date' if format == 'date' + return 'Fixnum' if format == 'int64' + return 'Fixnum' if format == 'uint64' return TYPE_MAP[type] when 'array' return sprintf('Array<%s>', items.generated_type) diff --git a/lib/google/apis/generator/templates/_representation.tmpl b/lib/google/apis/generator/templates/_representation.tmpl index c9320fcb4..125bed5ea 100644 --- a/lib/google/apis/generator/templates/_representation.tmpl +++ b/lib/google/apis/generator/templates/_representation.tmpl @@ -33,7 +33,7 @@ class <%= cls.generated_class_name %> <% elsif property.type == 'array' -%> collection :<%= property.generated_name %>, as: '<%= property.name %>'<%= include('representation_type', :lead => ', ', :type => property.items, :api => api) %> <% else -%> - property :<%= property.generated_name %>,<% if property.format == 'byte' %> :base64 => true,<%end%> as: '<%= property.name %>'<%= include('representation_type', :lead => ', ', :type => property, :api => api) %> + property :<%= property.generated_name %>,<% if ['uint64', 'int64'].include?(property.format) %> :numeric_string => true,<%end%><% if property.format == 'byte' %> :base64 => true,<%end%> as: '<%= property.name %>'<%= include('representation_type', :lead => ', ', :type => property, :api => api) %> <% end -%> <% end -%> <% end -%> diff --git a/spec/google/apis/core/json_representation_spec.rb b/spec/google/apis/core/json_representation_spec.rb index 7d5fbc462..585b03b76 100644 --- a/spec/google/apis/core/json_representation_spec.rb +++ b/spec/google/apis/core/json_representation_spec.rb @@ -33,6 +33,7 @@ RSpec.describe Google::Apis::Core::JsonRepresentation do attr_accessor :date_value attr_accessor :nil_date_value attr_accessor :bytes_value + attr_accessor :big_value attr_accessor :items attr_accessor :child attr_accessor :children @@ -51,6 +52,7 @@ RSpec.describe Google::Apis::Core::JsonRepresentation do property :date_value, as: 'dateValue', type: DateTime property :nil_date_value, as: 'nullDateValue', type: DateTime property :bytes_value, as: 'bytesValue', base64: true + property :big_value, as: 'bigValue', numeric_string: true property :items property :child, class: klass do property :value @@ -106,6 +108,10 @@ RSpec.describe Google::Apis::Core::JsonRepresentation do it 'serializes object collections' do expect(json).to be_json_eql(%([{"value" : "child"}])).at_path('children') end + + it 'serializes numeric strings' do + expect(json).to be_json_eql(%("1208925819614629174706176")).at_path('bigValue') + end end context 'with model object' do @@ -124,6 +130,7 @@ RSpec.describe Google::Apis::Core::JsonRepresentation do model.child.value = 'child' model.children = [model.child] model.nil_date_value = nil + model.big_value = 1208925819614629174706176 model end @@ -143,6 +150,7 @@ RSpec.describe Google::Apis::Core::JsonRepresentation do boolean_value_true: true, boolean_value_false: false, bytes_value: 'Hello world', + big_value: 1208925819614629174706176, items: [1, 2, 3], child: { value: 'child' @@ -165,6 +173,7 @@ RSpec.describe Google::Apis::Core::JsonRepresentation do "numericValue": 123, "dateValue": "2015-05-01T12:00:00+00:00", "bytesValue": "SGVsbG8gd29ybGQ=", + "bigValue": "1208925819614629174706176", "items": [1,2,3], "child": {"value" : "hello"}, "children": [{"value" : "hello"}] @@ -204,5 +213,10 @@ EOF it 'serializes object collections' do expect(model.children[0].value).to eql 'hello' end + + it 'deserializes numeric strings' do + expect(model.big_value).to eql 1208925819614629174706176 + end + end end From c61d64c150edef37161541f4fab1912503f1537e Mon Sep 17 00:00:00 2001 From: Steve Bazyl Date: Wed, 17 Aug 2016 15:32:56 -0700 Subject: [PATCH 06/16] Drop rails 3.x from test since no longer supported --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 57023b0cd..8a82bfef3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,6 @@ rvm: - 2.1 - jruby-9000 env: - - RAILS_VERSION="~>3.2" - RAILS_VERSION="~>4.0.0" - RAILS_VERSION="~>4.1.0" - RAILS_VERSION="~>4.2.0" From ecc107f805004fa7be9a978259c7598f3e59f099 Mon Sep 17 00:00:00 2001 From: Steve Bazyl Date: Wed, 17 Aug 2016 15:40:17 -0700 Subject: [PATCH 07/16] Update changelog/version --- CHANGELOG.md | 16 ++++++++++++++-- lib/google/apis/version.rb | 2 +- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a360e2662..9b7385351 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,20 @@ +# 0.10.0 +* *Breaking Change* - Fix handling of large numbers during code generation. Previously the + uint64/int64 formats were passed through as strings. They are now coerced to/from Fixnum/Bignum types +* Remove Hurley as dependency. May cause inor breaking changes if directly accessing the underlying + client connection. +* Drop compatibility with Rails 3.x since that is no longer supported by the Rails community. +* Upgrade representable to 3.0 +* Upgrade mime-types to 3.0 +* Move Thor & ActiveSupport to development depenencies. Using the code gengerator + now requires using the Bundle file or install the gem with dev dependencies. +* Treat 429 status codes as rate limit errors +* Fix a potential download corruption if download interrupted and retried against a URL + that does not return partial content. + # 0.9.12 * Regenerated APIs - # 0.9.11 * Add DFA Reporting 2.5, regenerate APIs @@ -10,7 +23,6 @@ * Reduce memory footprint used by mimetypes library * Fix bug with pagination when items collection is nil - # 0.9.9 * Add monitoring v3, regenerate APIs * Add samples for sheets, bigquery diff --git a/lib/google/apis/version.rb b/lib/google/apis/version.rb index 94eef67a5..52eeec93d 100644 --- a/lib/google/apis/version.rb +++ b/lib/google/apis/version.rb @@ -15,7 +15,7 @@ module Google module Apis # Client library version - VERSION = '0.9.12' + VERSION = '0.10.0' # Current operating system # @private From 33a30a3dee5d727b507041a3e956eb17ec3660f5 Mon Sep 17 00:00:00 2001 From: Abdelkader Boudih Date: Thu, 18 Aug 2016 00:06:59 +0100 Subject: [PATCH 08/16] remove memoist --- google-api-client.gemspec | 1 - 1 file changed, 1 deletion(-) diff --git a/google-api-client.gemspec b/google-api-client.gemspec index b3d45ec56..c3b67f872 100644 --- a/google-api-client.gemspec +++ b/google-api-client.gemspec @@ -26,7 +26,6 @@ Gem::Specification.new do |spec| spec.add_runtime_dependency 'mime-types', '>= 3.0' spec.add_runtime_dependency 'googleauth', '~> 0.5' spec.add_runtime_dependency 'httpclient', '>= 2.8.1', '< 3.0' - spec.add_runtime_dependency 'memoist', '~> 0.11' spec.add_development_dependency 'thor', '~> 0.14' spec.add_development_dependency 'activesupport', '>= 3.2', '< 5.0' From 93281749c3cc970e18934f84b3a489e9107cf1e6 Mon Sep 17 00:00:00 2001 From: Steve Bazyl Date: Wed, 17 Aug 2016 16:23:34 -0700 Subject: [PATCH 09/16] Update webmock dependency --- Gemfile | 6 +++--- google-api-client.gemspec | 2 -- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/Gemfile b/Gemfile index bb5be5597..238047f1a 100644 --- a/Gemfile +++ b/Gemfile @@ -6,13 +6,13 @@ gemspec group :development do gem 'bundler', '~> 1.7' - gem 'rake', '~> 10.0' + gem 'rake', '~> 11.2' gem 'rspec', '~> 3.1' gem 'json_spec', '~> 1.1' - gem 'webmock', '~> 1.21' + gem 'webmock', '~> 2.1' gem 'simplecov', '~> 0.12' gem 'coveralls', '~> 0.8' - gem 'rubocop', '~> 0.29' + gem 'rubocop', '~> 0.42.0' gem 'launchy', '~> 2.4' gem 'dotenv', '~> 2.0' gem 'fakefs', '~> 0.6', require: "fakefs/safe" diff --git a/google-api-client.gemspec b/google-api-client.gemspec index b3d45ec56..75737a6cd 100644 --- a/google-api-client.gemspec +++ b/google-api-client.gemspec @@ -26,8 +26,6 @@ Gem::Specification.new do |spec| spec.add_runtime_dependency 'mime-types', '>= 3.0' spec.add_runtime_dependency 'googleauth', '~> 0.5' spec.add_runtime_dependency 'httpclient', '>= 2.8.1', '< 3.0' - spec.add_runtime_dependency 'memoist', '~> 0.11' spec.add_development_dependency 'thor', '~> 0.14' spec.add_development_dependency 'activesupport', '>= 3.2', '< 5.0' - end From 23d52bd3e98de8f7dd912fa2450254739672530b Mon Sep 17 00:00:00 2001 From: Abdelkader Boudih Date: Thu, 18 Aug 2016 08:04:17 +0100 Subject: [PATCH 10/16] faster build. Test only supported version of rails. Test in Rails 5.0 --- .travis.yml | 22 ++++++++++++---------- Gemfile | 1 - Rakefile | 1 + google-api-client.gemspec | 2 +- samples/web/Gemfile | 1 - 5 files changed, 14 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8a82bfef3..66663df1e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,19 +1,21 @@ language: ruby +sudo: false rvm: - - 2.3.0 - - 2.2 + - 2.3.1 + - 2.2.5 - 2.0.0 - 2.1 - - jruby-9000 + - jruby-9.0.5.0 env: - - RAILS_VERSION="~>4.0.0" - - RAILS_VERSION="~>4.1.0" - RAILS_VERSION="~>4.2.0" -script: "bundle exec rake spec:all" -before_install: - - sudo apt-get update - - sudo apt-get install idn - - gem update bundler + - RAILS_VERSION="~>5.0.0" +matrix: + exclude: + - env: RAILS_VERSION="~>5.0.0" + rvm: 2.0.0 + - env: RAILS_VERSION="~>5.0.0" + rvm: 2.1 +before_install: gem install bundler notifications: email: recipients: diff --git a/Gemfile b/Gemfile index 238047f1a..8d0eb2c4b 100644 --- a/Gemfile +++ b/Gemfile @@ -19,7 +19,6 @@ group :development do gem 'google-id-token', '~> 1.3' gem 'os', '~> 0.9' gem 'rmail', '~> 1.1' - gem 'sinatra', '~> 1.4' gem 'redis', '~> 3.2' end diff --git a/Rakefile b/Rakefile index 809eb5616..dc9a4dcd2 100644 --- a/Rakefile +++ b/Rakefile @@ -1,2 +1,3 @@ require "bundler/gem_tasks" +task default: :spec diff --git a/google-api-client.gemspec b/google-api-client.gemspec index 75737a6cd..fba6622ee 100644 --- a/google-api-client.gemspec +++ b/google-api-client.gemspec @@ -27,5 +27,5 @@ Gem::Specification.new do |spec| spec.add_runtime_dependency 'googleauth', '~> 0.5' spec.add_runtime_dependency 'httpclient', '>= 2.8.1', '< 3.0' spec.add_development_dependency 'thor', '~> 0.14' - spec.add_development_dependency 'activesupport', '>= 3.2', '< 5.0' + spec.add_development_dependency 'activesupport', '>= 4.2', '< 5.1' end diff --git a/samples/web/Gemfile b/samples/web/Gemfile index 9c86e389f..98418b3e4 100644 --- a/samples/web/Gemfile +++ b/samples/web/Gemfile @@ -2,6 +2,5 @@ source 'https://rubygems.org' gem 'google-api-client', '~> 0.9' gem 'google-id-token', '~> 1.3' -gem 'sinatra', '~> 1.4' gem 'redis', '~> 3.2' gem 'dotenv' From 5447ddb7c0419d394c15756888f3a5d9a0fcb47f Mon Sep 17 00:00:00 2001 From: Steve Bazyl Date: Thu, 30 Mar 2017 12:33:49 -0700 Subject: [PATCH 11/16] Don't normalize unicode in templates (fixed #557 + GCS issues) --- CHANGELOG.md | 3 +++ MIGRATING.md | 2 +- google-api-client.gemspec | 2 +- lib/google/apis/core/http_command.rb | 8 ++++++-- lib/google/apis/options.rb | 7 +++++-- spec/google/apis/core/http_command_spec.rb | 20 ++++++++++++++++++++ 6 files changed, 36 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 625c6a53b..6e5ffbb0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,9 @@ # 0.11.0 * *Breaking Change* - Fix handling of large numbers during code generation. Previously the uint64/int64 formats were passed through as strings. They are now coerced to/from Fixnum/Bignum types +* *Breaking Change* - No longer normalize unicode strings in URI templates. Mostly affects + Cloud Storage, but other APIs with unicode strings in paths may be affected. Old behavior + can be restored using the `normalize_unicode` request option. * Remove Hurley as dependency. May cause minor breaking changes if directly accessing the underlying client connection. * Drop compatibility with Rails 3.x since that is no longer supported by the Rails community. diff --git a/MIGRATING.md b/MIGRATING.md index bbbbe242c..709e7cfbf 100644 --- a/MIGRATING.md +++ b/MIGRATING.md @@ -1,4 +1,4 @@ -# Migrating from version `0.8.x` to `0.9` +# Migrating from version `0.8.x` to `0.9` and above Many changes and improvements have been made to the `google-api-ruby-client` library to bring it to `0.9`. If you are starting a new project or haven't used diff --git a/google-api-client.gemspec b/google-api-client.gemspec index e115caae5..44a6622a0 100644 --- a/google-api-client.gemspec +++ b/google-api-client.gemspec @@ -22,7 +22,7 @@ Gem::Specification.new do |spec| spec.add_runtime_dependency 'representable', '~> 3.0' spec.add_runtime_dependency 'retriable', '>= 2.0', '< 4.0' - spec.add_runtime_dependency 'addressable', '~> 2.3' + spec.add_runtime_dependency 'addressable', '>= 2.5.1' spec.add_runtime_dependency 'mime-types', '>= 3.0' spec.add_runtime_dependency 'googleauth', '~> 0.5' spec.add_runtime_dependency 'httpclient', '>= 2.8.1', '< 3.0' diff --git a/lib/google/apis/core/http_command.rb b/lib/google/apis/core/http_command.rb index 16df06a24..9eb123468 100644 --- a/lib/google/apis/core/http_command.rb +++ b/lib/google/apis/core/http_command.rb @@ -139,8 +139,12 @@ module Google # @private # @return [void] def prepare! - header.update(options.header) if options && options.header - self.url = url.expand(params) if url.is_a?(Addressable::Template) + normalize_unicode = true + if options + header.update(options.header) if options.header + normalize_unicode = options.normalize_unicode + end + self.url = url.expand(params, nil, normalize_unicode) if url.is_a?(Addressable::Template) url.query_values = query.merge(url.query_values || {}) if allow_form_encoding? diff --git a/lib/google/apis/options.rb b/lib/google/apis/options.rb index f790e0cd1..066558a62 100644 --- a/lib/google/apis/options.rb +++ b/lib/google/apis/options.rb @@ -26,7 +26,8 @@ module Google :retries, :header, :timeout_sec, - :open_timeout_sec) + :open_timeout_sec, + :normalize_unicode) # General client options class ClientOptions @@ -56,6 +57,8 @@ module Google # @return [Fixnum] How long, in seconds, before failed connections time out # @!attribute [rw] header # @return [Hash Date: Fri, 31 Mar 2017 13:12:59 -0700 Subject: [PATCH 12/16] Add test for #509 - setting proxy url as string --- spec/google/apis/core/service_spec.rb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/spec/google/apis/core/service_spec.rb b/spec/google/apis/core/service_spec.rb index dbf29dcfb..d9cbab0ea 100644 --- a/spec/google/apis/core/service_spec.rb +++ b/spec/google/apis/core/service_spec.rb @@ -93,6 +93,14 @@ RSpec.describe Google::Apis::Core::BaseService do end end + context 'with proxy' do + + it 'should allow proxy URLs as strings' do + Google::Apis::ClientOptions.default.proxy_url = 'http://gateway.example.com:1234' + service.client + end + end + context 'when making simple commands' do let(:command) { service.send(:make_simple_command, :get, 'zoo/animals', authorization: 'foo') } From 489378db5852d5e26a98295ab5fb8abe9653d710 Mon Sep 17 00:00:00 2001 From: Steve Bazyl Date: Fri, 31 Mar 2017 14:59:07 -0700 Subject: [PATCH 13/16] Ensure content length when empty body, hook up httpclient to logger when debugging --- lib/google/apis/core/base_service.rb | 4 ++++ lib/google/apis/core/http_command.rb | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/lib/google/apis/core/base_service.rb b/lib/google/apis/core/base_service.rb index da9dccacd..81799bced 100644 --- a/lib/google/apis/core/base_service.rb +++ b/lib/google/apis/core/base_service.rb @@ -84,6 +84,8 @@ module Google # Base service for all APIs. Not to be used directly. # class BaseService + include Logging + # Root URL (host/port) for the API # @return [Addressable::URI] attr_accessor :root_url @@ -393,6 +395,8 @@ module Google end client.follow_redirect_count = 5 client.default_header = { 'User-Agent' => user_agent } + + client.debug_dev = logger if logger.level == Logger::DEBUG client end diff --git a/lib/google/apis/core/http_command.rb b/lib/google/apis/core/http_command.rb index 9eb123468..aff138317 100644 --- a/lib/google/apis/core/http_command.rb +++ b/lib/google/apis/core/http_command.rb @@ -155,6 +155,12 @@ module Google else @form_encoded = false end + + unless body + self.header['Content-Type'] = 'application/json' + self.header['Content-Length'] = 0 + end + end # Release any resources used by this command From 97ff7f4d9a009d657de65691a7af4635d8d90524 Mon Sep 17 00:00:00 2001 From: Steve Bazyl Date: Fri, 31 Mar 2017 15:26:51 -0700 Subject: [PATCH 14/16] #475, #488 -- allow raw JSON if explicitly requested, fix empty body handling & test cleanup --- lib/google/apis/core/api_command.rb | 7 +++++- lib/google/apis/core/http_command.rb | 5 +--- lib/google/apis/options.rb | 10 +++++++- spec/google/apis/core/api_command_spec.rb | 29 +++++++++++++++++++++++ spec/google/apis/core/service_spec.rb | 4 ++++ 5 files changed, 49 insertions(+), 6 deletions(-) diff --git a/lib/google/apis/core/api_command.rb b/lib/google/apis/core/api_command.rb index 27430e712..1df7d6240 100644 --- a/lib/google/apis/core/api_command.rb +++ b/lib/google/apis/core/api_command.rb @@ -55,7 +55,11 @@ module Google query[FIELDS_PARAM] = normalize_fields_param(query[FIELDS_PARAM]) if query.key?(FIELDS_PARAM) if request_representation && request_object header['Content-Type'] ||= JSON_CONTENT_TYPE - self.body = request_representation.new(request_object).to_json(user_options: { skip_undefined: true }) + if options && options.skip_serialization + self.body = request_object + else + self.body = request_representation.new(request_object).to_json(user_options: { skip_undefined: true }) + end end super end @@ -71,6 +75,7 @@ module Google # noinspection RubyUnusedLocalVariable def decode_response_body(content_type, body) return super unless response_representation + return super if options && options.skip_deserialization return super if content_type.nil? return nil unless content_type.start_with?(JSON_CONTENT_TYPE) instance = response_class.new diff --git a/lib/google/apis/core/http_command.rb b/lib/google/apis/core/http_command.rb index aff138317..4e544b719 100644 --- a/lib/google/apis/core/http_command.rb +++ b/lib/google/apis/core/http_command.rb @@ -156,10 +156,7 @@ module Google @form_encoded = false end - unless body - self.header['Content-Type'] = 'application/json' - self.header['Content-Length'] = 0 - end + self.body = '' unless self.body end diff --git a/lib/google/apis/options.rb b/lib/google/apis/options.rb index 066558a62..1287fef6d 100644 --- a/lib/google/apis/options.rb +++ b/lib/google/apis/options.rb @@ -27,7 +27,9 @@ module Google :header, :timeout_sec, :open_timeout_sec, - :normalize_unicode) + :normalize_unicode, + :skip_serialization, + :skip_deserialization) # General client options class ClientOptions @@ -59,6 +61,10 @@ module Google # @return [Hash 'application/json' }, body: %({})) + end + + it 'should allow raw JSON if skip_serialization = true' do + command.execute(client) + expect(a_request(:post, 'https://www.googleapis.com/zoo/animals').with do |req| + be_json_eql(%({"value":"hello"})).matches?(req.body) + end).to have_been_made + end + end + context('with a JSON response') do let(:command) do command = Google::Apis::Core::ApiCommand.new(:get, 'https://www.googleapis.com/zoo/animals') @@ -85,6 +108,12 @@ RSpec.describe Google::Apis::Core::ApiCommand do result = command.execute(client) expect(result.value).to eql 'hello' end + + it 'should return a raw JSON if skip_deserialization true' do + command.options.skip_deserialization = true + result = command.execute(client) + expect(result).to eql %({"value" : "hello"}) + end end context('with an invalid content-type response') do diff --git a/spec/google/apis/core/service_spec.rb b/spec/google/apis/core/service_spec.rb index d9cbab0ea..673b8311c 100644 --- a/spec/google/apis/core/service_spec.rb +++ b/spec/google/apis/core/service_spec.rb @@ -95,6 +95,10 @@ RSpec.describe Google::Apis::Core::BaseService do context 'with proxy' do + after(:example) do + Google::Apis::ClientOptions.default.proxy_url = nil + end + it 'should allow proxy URLs as strings' do Google::Apis::ClientOptions.default.proxy_url = 'http://gateway.example.com:1234' service.client From 4682267161c434a9fa4645cebe04b6b194605c00 Mon Sep 17 00:00:00 2001 From: Steve Bazyl Date: Mon, 3 Apr 2017 11:10:54 -0700 Subject: [PATCH 15/16] Move timeouts from request to clientoptions, update migration/changelog --- CHANGELOG.md | 7 +++--- MIGRATING.md | 33 ++++++++++++++++++++++++++++ dl.rb | 0 lib/google/apis/core/base_service.rb | 19 ++++++++-------- lib/google/apis/core/download.rb | 8 ++++--- lib/google/apis/core/upload.rb | 14 ++++++++---- lib/google/apis/options.rb | 23 +++++++++---------- 7 files changed, 74 insertions(+), 30 deletions(-) create mode 100644 dl.rb diff --git a/CHANGELOG.md b/CHANGELOG.md index e68f6d94f..d90919c68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,10 @@ # 0.11.0 -* *Breaking Change* - Fix handling of large numbers during code generation. Previously the +* *Breaking change* - Fix handling of large numbers during code generation. Previously the uint64/int64 formats were passed through as strings. They are now coerced to/from Fixnum/Bignum types -* *Breaking Change* - No longer normalize unicode strings in URI templates. Mostly affects +* *Breaking change* - No longer normalize unicode strings in URI templates. Mostly affects Cloud Storage, but other APIs with unicode strings in paths may be affected. Old behavior can be restored using the `normalize_unicode` request option. +* *Breaking change* -- Moved timeout options from `RequestOptions` to `ClientOptions` * Remove Hurley as dependency. May cause minor breaking changes if directly accessing the underlying client connection. * Drop compatibility with Rails 3.x since that is no longer supported by the Rails community. @@ -17,7 +18,7 @@ # 0.10.3 * Regenerate APIs * Enable additional API: - * `acceleratedmobilepageurl:v1`` + * `acceleratedmobilepageurl:v1` * `appengine:v1` * `clouderrorreporting:v1beta1` * `cloudfunctions:v1` diff --git a/MIGRATING.md b/MIGRATING.md index b0f99b2f4..96a5b5b9a 100644 --- a/MIGRATING.md +++ b/MIGRATING.md @@ -1,3 +1,36 @@ +# Migrating from version`0.10` to `0.11` + +## Unicode normalization + +The client no longer normalizes unicode strings in path parameters. This may affect +some applications using multibyte strings that were previously normalized.: +To restore the previous behavior, set the following option prior to creating a service. + +```ruby +ClientOptions.default.normalize_unicode = true +``` + +## Type change for large numbers + +Previously, types declared as 64 bit numbers were mapped to strings. These are now mapped to +`Fixednum`/`Bignum`. + +## Timeouts + +Timeout options have been moved from `RequestOptions` to `ClientOptions`. + +Old | New +----------------------------------|----------------- +`RequestOptions.open_timeout_sec` | `ClentOptions.open_timeout_sec` +`RequestOptions.timeout_sec` | `ClentOptions.read_timeout_sec` +`RequestOptions.timeout_sec` | `ClentOptions.send_timeout_sec` + +## Batch requests across services no longer supported + +It is no longer possible to combine multiple services (e.g. Gail & Drive) +in a batch request. If batching requests that span services, group +requests for each service in their own batch request. + # Migrating from version `0.9.x` to `0.10` Only one minor breaking change was introduced in the `to_json` method due to a version bump for the `representable` gem from `2.3` to `3.0`. If you used the `skip_undefined` in `to_json`, you should replace that with `user_options: { skip_undefined: true }`. diff --git a/dl.rb b/dl.rb new file mode 100644 index 000000000..e69de29bb diff --git a/lib/google/apis/core/base_service.rb b/lib/google/apis/core/base_service.rb index 81799bced..e7b9e806b 100644 --- a/lib/google/apis/core/base_service.rb +++ b/lib/google/apis/core/base_service.rb @@ -380,23 +380,24 @@ module Google client = ::HTTPClient.new client.transparent_gzip_decompression = true - client.proxy = client_options.proxy_url if client_options.proxy_url - if request_options.timeout_sec - client.connect_timeout = request_options.timeout_sec - client.receive_timeout = request_options.timeout_sec - client.send_timeout = request_options.timeout_sec + if client_options.open_timeout_sec + client.connect_timeout = client_options.open_timeout_sec end - if request_options.open_timeout_sec - client.connect_timeout = request_options.open_timeout_sec - client.send_timeout = request_options.open_timeout_sec + if client_options.read_timeout_sec + client.receive_timeout = client_options.read_timeout_sec end + + if client_options.send_timeout_sec + client.send_timeout = client_options.send_timeout_sec + end + client.follow_redirect_count = 5 client.default_header = { 'User-Agent' => user_agent } - client.debug_dev = logger if logger.level == Logger::DEBUG + client.debug_dev = logger if client_options.log_http_requests client end diff --git a/lib/google/apis/core/download.rb b/lib/google/apis/core/download.rb index 3cc23311b..de4f6bf9f 100644 --- a/lib/google/apis/core/download.rb +++ b/lib/google/apis/core/download.rb @@ -87,13 +87,14 @@ module Google @download_io.rewind check_if_rewind_needed = false end - logger.debug { sprintf('Writing chunk (%d bytes)', chunk.length) } - @offset += chunk.length + # logger.debug { sprintf('Writing chunk (%d bytes, %d total)', chunk.length, bytes_read) } @download_io.write(chunk) - @download_io.flush + @offset += chunk.length end end + @download_io.flush + if @close_io_on_finish result = nil else @@ -102,6 +103,7 @@ module Google check_status(http_res.status.to_i, http_res.header, http_res.body) success(result, &block) rescue => e + @download_io.flush error(e, rethrow: true, &block) end end diff --git a/lib/google/apis/core/upload.rb b/lib/google/apis/core/upload.rb index 6aa88ec41..7ad5fed97 100644 --- a/lib/google/apis/core/upload.rb +++ b/lib/google/apis/core/upload.rb @@ -29,6 +29,7 @@ module Google UPLOAD_PROTOCOL_HEADER = 'X-Goog-Upload-Protocol' UPLOAD_CONTENT_TYPE_HEADER = 'X-Goog-Upload-Header-Content-Type' UPLOAD_CONTENT_LENGTH = 'X-Goog-Upload-Header-Content-Length' + CONTENT_TYPE_HEADER = 'Content-Type' # File name or IO containing the content to upload # @return [String, File, #read] @@ -51,16 +52,20 @@ module Google if streamable?(upload_source) self.upload_io = upload_source @close_io_on_finish = false - elsif upload_source.is_a?(String) + elsif self.upload_source.is_a?(String) self.upload_io = File.new(upload_source, 'r') - if upload_content_type.nil? + if self.upload_content_type.nil? type = MIME::Types.of(upload_source) - upload_content_type = type.first.content_type unless type.nil? || type.empty? + self.upload_content_type = type.first.content_type unless type.nil? || type.empty? end @close_io_on_finish = true else fail Google::Apis::ClientError, 'Invalid upload source' end + if self.upload_content_type.nil? || self.upload_content_type.empty? + self.upload_content_type = 'application/octet-stream' + end + puts self.upload_content_type.inspect end # Close IO stream when command done. Only closes the stream if it was opened by the command. @@ -198,7 +203,7 @@ module Google apply_request_options(request_header) request_header[UPLOAD_COMMAND_HEADER] = QUERY_COMMAND - client.post(@upload_url, header: request_header, follow_redirect: true) + client.post(@upload_url, body: '', header: request_header, follow_redirect: true) end @@ -219,6 +224,7 @@ module Google request_header[UPLOAD_COMMAND_HEADER] = QUERY_COMMAND request_header[UPLOAD_COMMAND_HEADER] = UPLOAD_COMMAND request_header[UPLOAD_OFFSET_HEADER] = @offset.to_s + request_header[CONTENT_TYPE_HEADER] = upload_content_type client.post(@upload_url, body: content, header: request_header, follow_redirect: true) end diff --git a/lib/google/apis/options.rb b/lib/google/apis/options.rb index 1287fef6d..83f6def9e 100644 --- a/lib/google/apis/options.rb +++ b/lib/google/apis/options.rb @@ -15,18 +15,19 @@ module Google module Apis # General options for API requests - ClientOptions = Struct.new( + ClientOptions = Struct.new( :application_name, :application_version, :proxy_url, - :use_net_http) + :open_timeout_sec, + :read_timeout_sec, + :send_timeout_sec, + :log_http_requests) RequestOptions = Struct.new( :authorization, :retries, :header, - :timeout_sec, - :open_timeout_sec, :normalize_unicode, :skip_serialization, :skip_deserialization) @@ -39,7 +40,12 @@ module Google # @return [String] Version of the application, for identification in the User-Agent header # @!attribute [rw] proxy_url # @return [String] URL of a proxy server - + # @!attribute [rw] log_http_requests + # @return [Boolean] True if raw HTTP requests should be logged + # @!attribute [rw] open_timeout_sec + # @return [Fixnum] How long, in seconds, before failed connections time out + # @!attribute [rw] read_timeout_sec + # @return [Fixnum] How long, in seconds, before requests time out # Get the default options # @return [Google::Apis::ClientOptions] def self.default @@ -53,10 +59,6 @@ module Google # @return [Signet::OAuth2::Client, #apply(Hash)] OAuth2 credentials # @!attribute [rw] retries # @return [Fixnum] Number of times to retry requests on server error - # @!attribute [rw] timeout_sec - # @return [Fixnum] How long, in seconds, before requests time out - # @!attribute [rw] open_timeout_sec - # @return [Fixnum] How long, in seconds, before failed connections time out # @!attribute [rw] header # @return [Hash Date: Mon, 3 Apr 2017 11:20:00 -0700 Subject: [PATCH 16/16] Update readme to describe raw json support --- README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 839cc81a5..be33d6e8b 100644 --- a/README.md +++ b/README.md @@ -191,6 +191,14 @@ file = drive.create_file(file) # Raises ArgumentError: unknown keywords: id, tit file = drive.create_file(file, {}) # Returns a Drive::File instance ``` +### Using raw JSON + +To handle JSON serialization or deserialization in the application, set `skip_serialization` or +or `skip_deserializaton` options respectively. When setting `skip_serialization` in a request, +the body object must be a string representing the serialized JSON. + +When setting `skip_deserialization` to true, the response from the API will likewise +be a string containing the raw JSON from the server. ## Authorization @@ -283,7 +291,7 @@ The second is to set the environment variable `GOOGLE_API_USE_RAILS_LOGGER` to a ## Samples -See the [samples](samples) for examples on how to use the client library for various +See the [samples](https://github.com/google/google-api-ruby-client/tree/master/samples) for examples on how to use the client library for various services. Contributions for additional samples are welcome. See [CONTRIBUTING](CONTRIBUTING.md).