chiu
/
google-api-ruby-client
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove Hurley as a dependency

This commit is contained in:
Steve Bazyl 2016-08-17 13:51:09 -07:00
parent c5327670d0
commit 2046e00f14
18 changed files with 417 additions and 529 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
google-api-client.gemspec
View File

@ -24,9 +24,8 @@ Gem::Specification.new do |spec|
spec.add_runtime_dependency 'retriable', '~> 2.0'
spec.add_runtime_dependency 'addressable', '~> 2.3'
spec.add_runtime_dependency 'mime-types', '>= 1.6'
spec.add_runtime_dependency 'hurley', '~> 0.1'
spec.add_runtime_dependency 'googleauth', '~> 0.5'
spec.add_runtime_dependency 'thor', '~> 0.19'
spec.add_runtime_dependency 'httpclient', '~> 2.7'
spec.add_runtime_dependency 'httpclient', '>= 2.8.1', '< 3.0'
spec.add_runtime_dependency 'memoist', '~> 0.11'
end

4
lib/google/apis/core/api_command.rb
View File

@ -50,7 +50,7 @@ module Google
def prepare!
query[FIELDS_PARAM] = normalize_fields_param(query[FIELDS_PARAM]) if query.key?(FIELDS_PARAM)
if request_representation && request_object
header[:content_type] ||= JSON_CONTENT_TYPE
header['Content-Type'] ||= JSON_CONTENT_TYPE
self.body = request_representation.new(request_object).to_json(skip_undefined: true)
end
super
@ -78,7 +78,7 @@ module Google
#
# @param [Fixnum] status
# HTTP status code of response
# @param [Hurley::Header] header
# @param [Hash] header
# HTTP response headers
# @param [String] body
# HTTP response body

36
lib/google/apis/core/base_service.rb
View File

@ -19,11 +19,8 @@ require 'google/apis/core/api_command'
require 'google/apis/core/batch'
require 'google/apis/core/upload'
require 'google/apis/core/download'
require 'google/apis/core/http_client_adapter'
require 'google/apis/options'
require 'googleauth'
require 'hurley'
require 'hurley/addressable'
module Google
module Apis
@ -96,7 +93,7 @@ module Google
attr_accessor :batch_path
# HTTP client
# @return [Hurley::Client]
# @return [HTTPClient]
attr_accessor :client
# General settings
@ -198,7 +195,7 @@ module Google
end
# Get the current HTTP client
# @return [Hurley::Client]
# @return [HTTPClient]
def client
@client ||= new_client
end
@ -368,19 +365,30 @@ module Google
end
# Create a new HTTP client
# @return [Hurley::Client]
# @return [HTTPClient]
def new_client
client = Hurley::Client.new
client.connection = Google::Apis::Core::HttpClientAdapter.new unless client_options.use_net_http
client.request_options.timeout = request_options.timeout_sec
client.request_options.open_timeout = request_options.open_timeout_sec
client.request_options.proxy = client_options.proxy_url
client.request_options.query_class = Hurley::Query::Flat
client.ssl_options.ca_file = File.join(Google::Apis::ROOT, 'lib', 'cacerts.pem')
client.header[:user_agent] = user_agent
client = ::HTTPClient.new
client.transparent_gzip_decompression = true
client.proxy = client_options.proxy_url if client_options.proxy_url
if request_options.timeout_sec
client.connect_timeout = request_options.timeout_sec
client.receive_timeout = request_options.timeout_sec
client.send_timeout = request_options.timeout_sec
end
if request_options.open_timeout_sec
client.connect_timeout = request_options.open_timeout_sec
client.send_timeout = request_options.open_timeout_sec
end
client.follow_redirect_count = 5
client.default_header = { 'User-Agent' => user_agent }
client
end
# Build the user agent header
# @return [String]
def user_agent

31
lib/google/apis/core/batch.rb
View File

@ -25,19 +25,19 @@
# See the License for the specific language governing permissions and
# limitations under the License.
require 'hurley'
require 'google/apis/core/multipart'
require 'google/apis/core/http_command'
require 'google/apis/core/upload'
require 'google/apis/core/download'
require 'google/apis/core/composite_io'
require 'addressable/uri'
require 'securerandom'
module Google
module Apis
module Core
# Wrapper request for batching multiple calls in a single server request
class BatchCommand < HttpCommand
BATCH_BOUNDARY = 'RubyApiBatchRequest'.freeze
MULTIPART_MIXED = 'multipart/mixed'
# @param [symbol] method
@ -81,7 +81,7 @@ module Google
parts.each_index do |index|
response = deserializer.to_http_response(parts[index])
outer_header = response.shift
call_id = header_to_id(outer_header[:content_id]) || index
call_id = header_to_id(outer_header['Content-ID'].first) || index
call, callback = @calls[call_id]
begin
result = call.process_response(*response) unless call.nil?
@ -106,17 +106,16 @@ module Google
fail BatchError, 'Cannot make an empty batch request' if @calls.empty?
serializer = CallSerializer.new
multipart = Multipart.new(boundary: BATCH_BOUNDARY, content_type: MULTIPART_MIXED)
multipart = Multipart.new(content_type: MULTIPART_MIXED)
@calls.each_index do |index|
call, _ = @calls[index]
content_id = id_to_header(index)
io = serializer.to_upload_io(call)
multipart.add_upload(io, content_id: content_id)
io = serializer.to_part(call)
multipart.add_upload(io, content_type: 'application/http', content_id: content_id)
end
self.body = multipart.assemble
header[:content_type] = multipart.content_type
header[:content_length] = "#{body.length}"
header['Content-Type'] = multipart.content_type
super
end
@ -155,24 +154,20 @@ module Google
# Serializes a command for embedding in a multipart batch request
# @private
class CallSerializer
HTTP_CONTENT_TYPE = 'application/http'
##
# Serialize a single batched call for assembling the multipart message
#
# @param [Google::Apis::Core::HttpCommand] call
# the call to serialize.
# @return [Hurley::UploadIO]
# @return [IO]
# the serialized request
def to_upload_io(call)
def to_part(call)
call.prepare!
parts = []
parts << build_head(call)
parts << build_body(call) unless call.body.nil?
length = parts.inject(0) { |a, e| a + e.length }
Hurley::UploadIO.new(Hurley::CompositeReadIO.new(length, *parts),
HTTP_CONTENT_TYPE,
'ruby-api-request')
Google::Apis::Core::CompositeIO.new(*parts)
end
protected
@ -201,7 +196,7 @@ module Google
#
# @param [String] call_response
# the response to parse.
# @return [Array<(Fixnum, Hurley::Header, String)>]
# @return [Array<(Fixnum, Hash, String)>]
# Status, header, and response body.
def to_http_response(call_response)
outer_header, outer_body = split_header_and_body(call_response)
@ -218,10 +213,10 @@ module Google
#
# @param [String] response
# the response to parse.
# @return [Array<(Hurley::Header, String)>]
# @return [Array<(HTTP::Message::Headers, String)>]
# the header and the body, separately.
def split_header_and_body(response)
header = Hurley::Header.new
header = HTTP::Message::Headers.new
payload = response.lstrip
while payload
line, payload = payload.split(/\n/, 2)

97
lib/google/apis/core/composite_io.rb Normal file
View File

@ -0,0 +1,97 @@
# Copyright 2015 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Copyright 2015 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
require 'google/apis/core/http_command'
require 'google/apis/core/upload'
require 'google/apis/core/download'
require 'addressable/uri'
require 'securerandom'
module Google
module Apis
module Core
class CompositeIO
def initialize(*ios)
@ios = ios.flatten
@pos = 0
@index = 0
@sizes = @ios.map(&:size)
end
def read(length = nil, buf = nil)
buf = buf ? buf.replace('') : ''
begin
io = @ios[@index]
break if io.nil?
result = io.read(length)
if result
buf << result
if length
length -= result.length
break if length == 0
end
end
@index += 1
end while @index < @ios.length
buf.length > 0 ? buf : nil
end
def size
@sizes.reduce(:+)
end
alias_method :length, :size
def pos
@pos
end
def pos=(pos)
fail ArgumentError, "Position can not be negative" if pos < 0
@pos = pos
new_index = nil
@ios.each_with_index do |io,idx|
size = io.size
if pos <= size
new_index ||= idx
io.pos = pos
pos = 0
else
io.pos = size
pos -= size
end
end
@index = new_index unless new_index.nil?
end
def rewind
self.pos = 0
end
end
end
end
end

36
lib/google/apis/core/download.rb
View File

@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
require 'google/apis/core/multipart'
require 'google/apis/core/api_command'
require 'google/apis/errors'
require 'addressable/uri'
@ -22,7 +21,7 @@ module Google
module Core
# Streaming/resumable media download support
class DownloadCommand < ApiCommand
RANGE_HEADER = 'range'
RANGE_HEADER = 'Range'
# File or IO to write content to
# @return [String, File, #write]
@ -57,7 +56,7 @@ module Google
# of file content.
#
# @private
# @param [Hurley::Client] client
# @param [HTTPClient] client
# HTTP client
# @yield [result, err] Result or error if block supplied
# @return [Object]
@ -65,34 +64,41 @@ module Google
# @raise [Google::Apis::ClientError] The request is invalid and should not be retried without modification
# @raise [Google::Apis::AuthorizationError] Authorization is required
def execute_once(client, &block)
client.get(@download_url || url) do |req|
apply_request_options(req)
check_if_rewind_needed = false
if @offset > 0
logger.debug { sprintf('Resuming download from offset %d', @offset) }
req.header[RANGE_HEADER] = sprintf('bytes=%d-', @offset)
check_if_rewind_needed = true
end
req.on_body(200, 201, 206) do |res, chunk|
check_status(res.status_code, chunk) unless res.status_code.nil?
if check_if_rewind_needed && res.status_code != 206
request_header = header.dup
apply_request_options(request_header)
check_if_rewind_needed = false
if @offset > 0
logger.debug { sprintf('Resuming download from offset %d', @offset) }
request_header[RANGE_HEADER] = sprintf('bytes=%d-', @offset)
check_if_rewind_needed = true
end
http_res = client.get(url.to_s,
query: query,
header: request_header,
follow_redirect: true) do |res, chunk|
status = res.http_header.status_code.to_i
if [200, 201, 206].include?(status)
if check_if_rewind_needed && status != 206
# Oh no! Requested a chunk, but received the entire content
# Attempt to rewind the stream
@download_io.rewind
check_if_rewind_needed = false
end
logger.debug { sprintf('Writing chunk (%d bytes)', chunk.length) }
@offset += chunk.length
@download_io.write(chunk)
@download_io.flush
end
end
if @close_io_on_finish
result = nil
else
result = @download_io
end
check_status(http_res.status.to_i, http_res.header, http_res.body)
success(result, &block)
rescue => e
error(e, rethrow: true, &block)

82
lib/google/apis/core/http_client_adapter.rb
View File

@ -1,82 +0,0 @@
require 'httpclient'
require 'hurley'
require 'hurley/client'
module Google
module Apis
module Core
# HTTPClient adapter for Hurley.
class HttpClientAdapter
def call(request)
client = ::HTTPClient.new
configure_client(client, request)
begin
::Hurley::Response.new(request) do |res|
http_res = client.request(request.verb.to_s.upcase, request.url.to_s, nil, request.body_io, request.header.to_hash, false) do |http_res, chunk|
copy_response(http_res, res)
res.receive_body(chunk)
end
copy_response(http_res, res)
end
rescue ::HTTPClient::TimeoutError, Errno::ETIMEDOUT
raise ::Hurley::Timeout, $!
rescue ::HTTPClient::BadResponseError => err
if err.message.include?('status 407')
raise ::Hurley::ConnectionFailed, %{407 "Proxy Authentication Required "}
else
raise Hurley::ClientError, $!
end
rescue Errno::ECONNREFUSED, EOFError
raise ::Hurley::ConnectionFailed, $!
rescue => err
if defined?(OpenSSL) && OpenSSL::SSL::SSLError === err
raise Hurley::SSLError, err
else
raise
end
end
end
def copy_response(http_res, res)
unless res.status_code
res.status_code = http_res.status.to_i
http_res.header.all.each do |(k,v)|
res.header[k] = v
end
end
end
def configure_client(client, request)
client.transparent_gzip_decompression = true
if request.options.proxy
proxy = request.options.proxy
client.proxy = sprintf('%s:%d', proxy.host, proxy.port)
if proxy.user && proxy.password
client.set_proxy_auth proxy.user, proxy.password
end
end
if request.options.timeout
client.connect_timeout = request.options.timeout
client.receive_timeout = request.options.timeout
client.send_timeout = request.options.timeout
end
if request.options.open_timeout
client.connect_timeout = request.options.open_timeout
client.send_timeout = request.options.open_timeout
end
ssl_config = client.ssl_config
ssl_opts = request.ssl_options
ssl_config.verify_mode = ssl_opts.openssl_verify_mode
ssl_config.cert_store = ssl_opts.openssl_cert_store
ssl_config.add_trust_ca ssl_opts.ca_file if ssl_opts.ca_file
ssl_config.add_trust_ca ssl_opts.ca_path if ssl_opts.ca_path
ssl_config.client_cert = ssl_opts.openssl_client_cert if ssl_opts.openssl_client_cert
ssl_config.client_key = ssl_opts.openssl_client_key if ssl_opts.openssl_client_key
ssl_config.verify_depth = ssl_opts.verify_depth if ssl_opts.verify_depth
end
end
end
end
end

73
lib/google/apis/core/http_command.rb
View File

@ -17,9 +17,6 @@ require 'addressable/template'
require 'google/apis/options'
require 'google/apis/errors'
require 'retriable'
require 'hurley'
require 'hurley/addressable'
require 'hurley_patches'
require 'google/apis/core/logging'
require 'pp'
@ -41,7 +38,7 @@ module Google
attr_accessor :url
# HTTP headers
# @return [Hurley::Header]
# @return [Hash]
attr_accessor :header
# Request body
@ -53,7 +50,7 @@ module Google
attr_accessor :method
# HTTP Client
# @return [Hurley::Client]
# @return [HTTPClient]
attr_accessor :connection
# Query params
@ -75,7 +72,7 @@ module Google
self.url = url
self.url = Addressable::Template.new(url) if url.is_a?(String)
self.method = method
self.header = Hurley::Header.new
self.header = Hash.new
self.body = body
self.query = {}
self.params = {}
@ -83,7 +80,7 @@ module Google
# Execute the command, retrying as necessary
#
# @param [Hurley::Client] client
# @param [HTTPClient] client
# HTTP client
# @yield [result, err] Result or error if block supplied
# @return [Object]
@ -166,7 +163,7 @@ module Google
#
# @param [Fixnum] status
# HTTP status code of response
# @param [Hurley::Header] header
# @param [Hash] header
# Response headers
# @param [String, #read] body
# Response body
@ -177,7 +174,7 @@ module Google
# @raise [Google::Apis::AuthorizationError] Authorization is required
def process_response(status, header, body)
check_status(status, header, body)
decode_response_body(header[:content_type], body)
decode_response_body(header['Content-Type'].first, body)
end
# Check the response and raise error if needed
@ -185,7 +182,7 @@ module Google
# @param [Fixnum] status
# HTTP status code of response
# @param
# @param [Hurley::Header] header
# @param [Hash] header
# HTTP response headers
# @param [String] body
# HTTP response body
@ -201,7 +198,7 @@ module Google
when 200...300
nil
when 301, 302, 303, 307
message ||= sprintf('Redirect to %s', header[:location])
message ||= sprintf('Redirect to %s', header['Location'])
raise Google::Apis::RedirectError.new(message, status_code: status, header: header, body: body)
when 401
message ||= 'Unauthorized'
@ -251,7 +248,16 @@ module Google
# @raise [StandardError] if no block
def error(err, rethrow: false, &block)
logger.debug { sprintf('Error - %s', PP.pp(err, '')) }
err = Google::Apis::TransmissionError.new(err) if err.is_a?(Hurley::ClientError) || err.is_a?(SocketError)
if err.is_a?(HTTPClient::BadResponseError)
begin
res = err.res
check_status(res.status.to_i, res.header, res.body)
rescue Google::Apis::Error => e
err = e
end
elsif err.is_a?(HTTPClient::TimeoutError) || err.is_a?(SocketError)
err = Google::Apis::TransmissionError.new(err)
end
block.call(nil, err) if block_given?
fail err if rethrow || block.nil?
end
@ -259,7 +265,7 @@ module Google
# Execute the command once.
#
# @private
# @param [Hurley::Client] client
# @param [HTTPClient] client
# HTTP client
# @return [Object]
# @raise [Google::Apis::ServerError] An error occurred on the server and the request can be retried
@ -269,21 +275,18 @@ module Google
body.rewind if body.respond_to?(:rewind)
begin
logger.debug { sprintf('Sending HTTP %s %s', method, url) }
response = client.send(method, url, body) do |req|
# Temporary workaround for Hurley bug where the connection preference
# is ignored and it uses nested anyway
unless form_encoded?
req.url.query_class = Hurley::Query::Flat
query.each do | k, v|
req.url.query[k] = normalize_query_value(v)
end
end
# End workaround
apply_request_options(req)
end
logger.debug { response.status_code }
logger.debug { response.inspect }
response = process_response(response.status_code, response.header, response.body)
request_header = header.dup
apply_request_options(request_header)
http_res = client.request(method.to_s.upcase,
url.to_s,
query: nil,
body: body,
header: request_header,
follow_redirect: true)
logger.debug { http_res.status }
logger.debug { http_res.inspect }
response = process_response(http_res.status.to_i, http_res.header, http_res.body)
success(response)
rescue => e
logger.debug { sprintf('Caught error %s', e) }
@ -292,18 +295,16 @@ module Google
end
# Update the request with any specified options.
# @param [Hurley::Request] req
# HTTP request
# @param [Hash] header
# HTTP headers
# @return [void]
def apply_request_options(req)
def apply_request_options(req_header)
if options.authorization.respond_to?(:apply!)
options.authorization.apply!(req.header)
options.authorization.apply!(req_header)
elsif options.authorization.is_a?(String)
req.header[:authorization] = sprintf('Bearer %s', options.authorization)
req_header['Authorization'] = sprintf('Bearer %s', options.authorization)
end
req.header.update(header)
req.options.timeout = options.timeout_sec
req.options.open_timeout = options.open_timeout_sec
req_header.update(header)
end
private

136
lib/google/apis/core/multipart.rb
View File

@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
require 'hurley'
module Google
module Apis
@ -21,108 +20,60 @@ module Google
#
# @private
class JsonPart
include Hurley::Multipart::Part
# @return [Fixnum]
# Length of part
attr_reader :length
# @param [String] boundary
# Multipart boundary
# @param [String] value
# JSON content
def initialize(boundary, value, header = {})
@part = build_part(boundary, value)
@length = @part.bytesize
@io = StringIO.new(@part)
# @param [Hash] header
# Additional headers
def initialize(value, header = {})
@value = value
@header = header
end
private
# Format the part
#
# @param [String] boundary
# Multipart boundary
# @param [String] value
# JSON content
# @return [String]
def build_part(boundary, value)
def to_io(boundary)
part = ''
part << "--#{boundary}\r\n"
part << "Content-Type: application/json\r\n"
@header.each do |(k, v)|
part << "#{k}: #{v}\r\n"
end
part << "\r\n"
part << "#{value}\r\n"
part << "#{@value}\r\n"
StringIO.new(part)
end
end
# Part of a multipart request for holding arbitrary content. Modified
# from Hurley::Multipart::FilePart to remove Content-Disposition
# Part of a multipart request for holding arbitrary content.
#
# @private
class FilePart
include Hurley::Multipart::Part
# @return [Fixnum]
# Length of part
attr_reader :length
# @param [String] boundary
# Multipart boundary
# @param [Google::Apis::Core::UploadIO] io
# @param [IO] io
# IO stream
# @param [Hash] header
# Additional headers
def initialize(boundary, io, header = {})
file_length = io.respond_to?(:length) ? io.length : File.size(io.local_path)
@head = build_head(boundary, io.content_type, file_length,
io.respond_to?(:opts) ? io.opts.merge(header) : header)
@length = @head.bytesize + file_length + FOOT.length
@io = Hurley::CompositeReadIO.new(@length, StringIO.new(@head), io, StringIO.new(FOOT))
def initialize(io, header = {})
@io = io
@header = header
@length = io.respond_to?(:size) ? io.size : nil
end
private
# Construct the header for the part
#
# @param [String] boundary
# Multipart boundary
# @param [String] type
# Content type for the part
# @param [Fixnum] content_len
# Length of the part
# @param [Hash] header
# Headers for the part
def build_head(boundary, type, content_len, header)
content_id = ''
if header[:content_id]
content_id = sprintf(CID_FORMAT, header[:content_id])
def to_io(boundary)
head = ''
head << "--#{boundary}\r\n"
@header.each do |(k, v)|
head << "#{k}: #{v}\r\n"
end
sprintf(HEAD_FORMAT,
boundary,
content_len.to_i,
content_id,
header[:content_type] || type,
header[:content_transfer_encoding] || DEFAULT_TR_ENCODING)
head << "Content-Length: #{@length}\r\n" unless @length.nil?
head << "Content-Transfer-Encoding: binary\r\n"
head << "\r\n"
Google::Apis::Core::CompositeIO.new(StringIO.new(head), @io, StringIO.new("\r\n"))
end
DEFAULT_TR_ENCODING = 'binary'.freeze
FOOT = "\r\n".freeze
CID_FORMAT = "Content-ID: %s\r\n"
HEAD_FORMAT = <<-END
--%s\r
Content-Length: %d\r
%sContent-Type: %s\r
Content-Transfer-Encoding: %s\r
\r
END
end
# Helper for building multipart requests
class Multipart
MULTIPART_RELATED = 'multipart/related'
DEFAULT_BOUNDARY = 'RubyApiClientMultiPart'
# @return [String]
# Content type header
@ -135,8 +86,8 @@ Content-Transfer-Encoding: %s\r
def initialize(content_type: MULTIPART_RELATED, boundary: nil)
@parts = []
@boundary = boundary || DEFAULT_BOUNDARY
@content_type = "#{content_type}; boundary=#{boundary}"
@boundary = boundary || Digest::SHA1.hexdigest(SecureRandom.random_bytes(8))
@content_type = "#{content_type}; boundary=#{@boundary}"
end
# Append JSON data part
@ -147,23 +98,26 @@ Content-Transfer-Encoding: %s\r
# Optional unique ID of this part
# @return [self]
def add_json(body, content_id: nil)
header = { :content_id => content_id }
@parts << Google::Apis::Core::JsonPart.new(@boundary, body, header)
header = {}
header['Content-ID'] = content_id unless content_id.nil?
@parts << Google::Apis::Core::JsonPart.new(body, header).to_io(@boundary)
self
end
# Append arbitrary data as a part
#
# @param [Google::Apis::Core::UploadIO] upload_io
# @param [IO] upload_io
# IO stream
# @param [String] content_id
# Optional unique ID of this part
# @return [self]
def add_upload(upload_io, content_id: nil)
header = { :content_id => content_id }
@parts << Google::Apis::Core::FilePart.new(@boundary,
upload_io,
header)
def add_upload(upload_io, content_type: nil, content_id: nil)
header = {
'Content-Type' => content_type || 'application/octet-stream'
}
header['Content-Id'] = content_id unless content_id.nil?
@parts << Google::Apis::Core::FilePart.new(upload_io,
header).to_io(@boundary)
self
end
@ -172,14 +126,8 @@ Content-Transfer-Encoding: %s\r
# @return [IO]
# IO stream
def assemble
@parts << Hurley::Multipart::EpiloguePart.new(@boundary)
ios = []
len = 0
@parts.each do |part|
len += part.length
ios << part.to_io
end
Hurley::CompositeReadIO.new(len, *ios)
@parts << StringIO.new("--#{@boundary}--\r\n\r\n")
Google::Apis::Core::CompositeIO.new(*@parts)
end
end
end

123
lib/google/apis/core/upload.rb
View File

@ -29,41 +29,6 @@ end
module Google
module Apis
module Core
# Extension of Hurley's UploadIO to add length accessor
class UploadIO < Hurley::UploadIO
OCTET_STREAM_CONTENT_TYPE = 'application/octet-stream'
# Get the length of the stream
# @return [Fixnum]
def length
io.respond_to?(:length) ? io.length : File.size(local_path)
end
# Create a new instance given a file path
# @param [String, File] file_name
# Path to file
# @param [String] content_type
# Optional content type. If nil, will attempt to auto-detect
# @return [Google::Apis::Core::UploadIO]
def self.from_file(file_name, content_type: nil)
if content_type.nil?
type = MIME::Types.of(file_name)
content_type = type.first.content_type unless type.nil? || type.empty?
end
new(file_name, content_type || OCTET_STREAM_CONTENT_TYPE)
end
# Wraps an IO stream in UploadIO
# @param [#read] io
# IO to wrap
# @param [String] content_type
# Optional content type.
# @return [Google::Apis::Core::UploadIO]
def self.from_io(io, content_type: OCTET_STREAM_CONTENT_TYPE)
new(io, content_type)
end
end
# Base upload command. Not intended to be used directly
# @private
class BaseUploadCommand < ApiCommand
@ -83,17 +48,17 @@ module Google
# @return [Google::Apis::Core::UploadIO]
attr_accessor :upload_io
# Ensure the content is readable and wrapped in an {{Google::Apis::Core::UploadIO}} instance.
# Ensure the content is readable and wrapped in an IO instance.
#
# @return [void]
# @raise [Google::Apis::ClientError] if upload source is invalid
def prepare!
super
if streamable?(upload_source)
self.upload_io = UploadIO.from_io(upload_source, content_type: upload_content_type)
self.upload_io = upload_source
@close_io_on_finish = false
elsif upload_source.is_a?(String)
self.upload_io = UploadIO.from_file(upload_source, content_type: upload_content_type)
self.upload_io = File.new(upload_source, 'r')
@close_io_on_finish = true
else
fail Google::Apis::ClientError, 'Invalid upload source'
@ -124,13 +89,12 @@ module Google
super
self.body = upload_io
header[UPLOAD_PROTOCOL_HEADER] = RAW_PROTOCOL
header[UPLOAD_CONTENT_TYPE_HEADER] = upload_io.content_type
header[UPLOAD_CONTENT_TYPE_HEADER] = upload_content_type
end
end
# Implementation of the multipart upload protocol
class MultipartUploadCommand < BaseUploadCommand
UPLOAD_BOUNDARY = 'RubyApiClientUpload'
MULTIPART_PROTOCOL = 'multipart'
MULTIPART_RELATED = 'multipart/related'
@ -140,11 +104,11 @@ module Google
# @raise [Google::Apis::ClientError] if upload source is invalid
def prepare!
super
@multipart = Multipart.new(boundary: UPLOAD_BOUNDARY, content_type: MULTIPART_RELATED)
@multipart.add_json(body)
@multipart.add_upload(upload_io)
self.body = @multipart.assemble
header[:content_type] = @multipart.content_type
multipart = Multipart.new
multipart.add_json(body)
multipart.add_upload(upload_io, content_type: upload_content_type)
self.body = multipart.assemble
header['Content-Type'] = multipart.content_type
header[UPLOAD_PROTOCOL_HEADER] = MULTIPART_PROTOCOL
end
end
@ -179,7 +143,7 @@ module Google
#
# @param [Fixnum] status
# HTTP status code of response
# @param [Hurley::Header] header
# @param [HTTP::Message::Headers] header
# Response headers
# @param [String, #read] body
# Response body
@ -189,9 +153,9 @@ module Google
# @raise [Google::Apis::ClientError] The request is invalid and should not be retried without modification
# @raise [Google::Apis::AuthorizationError] Authorization is required
def process_response(status, header, body)
@offset = Integer(header[BYTES_RECEIVED_HEADER]) if header.key?(BYTES_RECEIVED_HEADER)
@upload_url = header[UPLOAD_URL_HEADER] if header.key?(UPLOAD_URL_HEADER)
upload_status = header[UPLOAD_STATUS_HEADER]
@offset = Integer(header[BYTES_RECEIVED_HEADER].first) unless header[BYTES_RECEIVED_HEADER].empty?
@upload_url = header[UPLOAD_URL_HEADER].first unless header[UPLOAD_URL_HEADER].empty?
upload_status = header[UPLOAD_STATUS_HEADER].first
logger.debug { sprintf('Upload status %s', upload_status) }
if upload_status == STATUS_ACTIVE
@state = :active
@ -204,61 +168,68 @@ module Google
super(status, header, body)
end
# Send the start command to initiate the upload
#
# @param [Hurley::Client] client
# HTTP client
# @return [Hurley::Response]
# @raise [Google::Apis::ServerError] Unable to send the request
def send_start_command(client)
logger.debug { sprintf('Sending upload start command to %s', url) }
client.send(method, url, body) do |req|
apply_request_options(req)
req.header[UPLOAD_PROTOCOL_HEADER] = RESUMABLE
req.header[UPLOAD_COMMAND_HEADER] = START_COMMAND
req.header[UPLOAD_CONTENT_LENGTH] = upload_io.length.to_s
req.header[UPLOAD_CONTENT_TYPE_HEADER] = upload_io.content_type
end
request_header = header.dup
apply_request_options(request_header)
request_header[UPLOAD_PROTOCOL_HEADER] = RESUMABLE
request_header[UPLOAD_COMMAND_HEADER] = START_COMMAND
request_header[UPLOAD_CONTENT_LENGTH] = upload_io.size.to_s
request_header[UPLOAD_CONTENT_TYPE_HEADER] = upload_content_type
client.request(method.to_s.upcase,
url.to_s, query: nil,
body: body,
header: request_header,
follow_redirect: true)
rescue => e
raise Google::Apis::ServerError, e.message
end
# Query for the status of an incomplete upload
#
# @param [Hurley::Client] client
# @param [HTTPClient] client
# HTTP client
# @return [Hurley::Response]
# @return [HTTP::Message]
# @raise [Google::Apis::ServerError] Unable to send the request
def send_query_command(client)
logger.debug { sprintf('Sending upload query command to %s', @upload_url) }
client.post(@upload_url, nil) do |req|
apply_request_options(req)
req.header[UPLOAD_COMMAND_HEADER] = QUERY_COMMAND
end
request_header = header.dup
apply_request_options(request_header)
request_header[UPLOAD_COMMAND_HEADER] = QUERY_COMMAND
client.post(@upload_url, header: request_header, follow_redirect: true)
end
# Send the actual content
#
# @param [Hurley::Client] client
# @param [HTTPClient] client
# HTTP client
# @return [Hurley::Response]
# @return [HTTP::Message]
# @raise [Google::Apis::ServerError] Unable to send the request
def send_upload_command(client)
logger.debug { sprintf('Sending upload command to %s', @upload_url) }
content = upload_io
content.pos = @offset
client.post(@upload_url, content) do |req|
apply_request_options(req)
req.header[UPLOAD_COMMAND_HEADER] = UPLOAD_COMMAND
req.header[UPLOAD_OFFSET_HEADER] = @offset.to_s
end
request_header = header.dup
apply_request_options(request_header)
request_header[UPLOAD_COMMAND_HEADER] = QUERY_COMMAND
request_header[UPLOAD_COMMAND_HEADER] = UPLOAD_COMMAND
request_header[UPLOAD_OFFSET_HEADER] = @offset.to_s
client.post(@upload_url, body: content, header: request_header, follow_redirect: true)
end
# Execute the upload request once. This will typically perform two HTTP requests -- one to initiate or query
# for the status of the upload, the second to send the (remaining) content.
#
# @private
# @param [Hurley::Client] client
# @param [HTTPClient] client
# HTTP client
# @yield [result, err] Result or error if block supplied
# @return [Object]

1
spec/google/apis/core/api_command_spec.rb
View File

@ -15,7 +15,6 @@
require 'spec_helper'
require 'google/apis/core/api_command'
require 'google/apis/core/json_representation'
require 'hurley/test'
RSpec.describe Google::Apis::Core::HttpCommand do
include TestHelpers

26
spec/google/apis/core/batch_spec.rb
View File

@ -15,7 +15,6 @@
require 'spec_helper'
require 'google/apis/core/batch'
require 'google/apis/core/json_representation'
require 'hurley/test'
RSpec.describe Google::Apis::Core::BatchCommand do
include TestHelpers
@ -30,19 +29,20 @@ RSpec.describe Google::Apis::Core::BatchCommand do
let(:post_with_string_command) do
command = Google::Apis::Core::HttpCommand.new(:post, 'https://www.googleapis.com/zoo/animals/2')
command.body = 'Hello world'
command.header[:content_type] = 'text/plain'
command.header['Content-Type'] = 'text/plain'
command
end
let(:post_with_io_command) do
command = Google::Apis::Core::HttpCommand.new(:post, 'https://www.googleapis.com/zoo/animals/3')
command.body = StringIO.new('Goodbye!')
command.header[:content_type] = 'text/plain'
command.header['Content-Type'] = 'text/plain'
command
end
before(:example) do
allow(SecureRandom).to receive(:uuid).and_return('ffe23d1b-e8f7-47f5-8c01-2a30cf8ecb8f')
allow(Digest::SHA1).to receive(:hexdigest).and_return('123abc')
response = <<EOF
--batch123
@ -83,20 +83,20 @@ EOF
command.execute(client)
expected_body = <<EOF.gsub(/\n/, "\r\n")
--RubyApiBatchRequest
Content-Length: 58
Content-ID: <ffe23d1b-e8f7-47f5-8c01-2a30cf8ecb8f+0>
--123abc
Content-Type: application/http
Content-Id: <ffe23d1b-e8f7-47f5-8c01-2a30cf8ecb8f+0>
Content-Length: 58
Content-Transfer-Encoding: binary
GET /zoo/animals/1? HTTP/1.1
Host: www.googleapis.com
--RubyApiBatchRequest
Content-Length: 96
Content-ID: <ffe23d1b-e8f7-47f5-8c01-2a30cf8ecb8f+1>
--123abc
Content-Type: application/http
Content-Id: <ffe23d1b-e8f7-47f5-8c01-2a30cf8ecb8f+1>
Content-Length: 96
Content-Transfer-Encoding: binary
POST /zoo/animals/2? HTTP/1.1
@ -104,10 +104,10 @@ Content-Type: text/plain
Host: www.googleapis.com
Hello world
--RubyApiBatchRequest
Content-Length: 93
Content-ID: <ffe23d1b-e8f7-47f5-8c01-2a30cf8ecb8f+2>
--123abc
Content-Type: application/http
Content-Id: <ffe23d1b-e8f7-47f5-8c01-2a30cf8ecb8f+2>
Content-Length: 93
Content-Transfer-Encoding: binary
POST /zoo/animals/3? HTTP/1.1
@ -115,7 +115,7 @@ Content-Type: text/plain
Host: www.googleapis.com
Goodbye!
--RubyApiBatchRequest--
--123abc--
EOF
expect(a_request(:post, 'https://www.googleapis.com/batch').with(body: expected_body)).to have_been_made

76
spec/google/apis/core/composite_io_spec.rb Normal file
View File

@ -0,0 +1,76 @@
# Copyright 2015 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
require 'spec_helper'
require 'google/apis/core/composite_io'
RSpec.describe Google::Apis::Core::CompositeIO do
shared_examples 'should act like IO' do
it 'should read from all IOs' do
expect(io.read).to eq 'Hello Cruel World'
end
it 'should respond to size' do
expect(io.size).to eq 17
end
it 'should respond to pos=' do
io.pos = 6
expect(io.read).to eq('Cruel World')
end
it 'should reject negative positions' do
expect { io.pos = -1 }.to raise_error(ArgumentError)
end
it 'should return nil if position beyond size' do
io.pos = 20
expect(io.read).to be_nil
end
it 'should be readable after rewinding' do
expect(io.read).to eq 'Hello Cruel World'
expect(io.read).to be_nil
io.rewind
expect(io.read).to eq 'Hello Cruel World'
end
end
context 'with StringIOs' do
let(:io) do
Google::Apis::Core::CompositeIO.new(
StringIO.new("Hello "),
StringIO.new("Cruel "),
StringIO.new("World"))
end
include_examples 'should act like IO'
end
context 'with Files' do
let(:io) do
files = []
dir = Dir.mktmpdir
['Hello ', 'Cruel ', 'World'].each_with_index do |text, index|
name = File.join(dir, "f#{index}")
File.open(name, 'w') { |f| f.write(text) }
files << name
end
Google::Apis::Core::CompositeIO.new(files.map { |name| File.open(name, 'r') })
end
include_examples 'should act like IO'
end
end

1
spec/google/apis/core/download_spec.rb
View File

@ -15,7 +15,6 @@
require 'spec_helper'
require 'google/apis/core/download'
require 'google/apis/core/json_representation'
require 'hurley/test'
require 'tempfile'
require 'tmpdir'

1
spec/google/apis/core/http_command_spec.rb
View File

@ -14,7 +14,6 @@
require 'spec_helper'
require 'google/apis/core/http_command'
require 'hurley/test'
RSpec.describe Google::Apis::Core::HttpCommand do
include TestHelpers

42
spec/google/apis/core/service_spec.rb
View File

@ -16,8 +16,6 @@ require 'spec_helper'
require 'google/apis/options'
require 'google/apis/core/base_service'
require 'google/apis/core/json_representation'
require 'hurley/test'
require 'ostruct'
RSpec.describe Google::Apis::Core::BaseService do
include TestHelpers
@ -190,6 +188,8 @@ EOF
context 'with batch uploads' do
before(:example) do
allow(SecureRandom).to receive(:uuid).and_return('b1981e17-f622-49af-b2eb-203308b1b17d')
allow(Digest::SHA1).to receive(:hexdigest).and_return('outer', 'inner')
response = <<EOF.gsub(/\n/, "\r\n")
--batch123
Content-Type: application/http
@ -227,6 +227,44 @@ EOF
end.to yield_with_args('Hello', nil)
end
it 'should send nested multipart' do
service.batch_upload do |service|
command = service.send(:make_upload_command, :post, 'zoo/animals', {})
command.upload_source = StringIO.new('test')
command.upload_content_type = 'text/plain'
service.send(:execute_or_queue_command, command)
end
expected_body = <<EOF.gsub(/\n/, "\r\n")
--outer
Content-Type: application/http
Content-Id: <b1981e17-f622-49af-b2eb-203308b1b17d+0>
Content-Length: 303
Content-Transfer-Encoding: binary
POST /upload/zoo/animals? HTTP/1.1
Content-Type: multipart/related; boundary=inner
X-Goog-Upload-Protocol: multipart
Host: www.googleapis.com
--inner
Content-Type: application/json
--inner
Content-Type: text/plain
Content-Length: 4
Content-Transfer-Encoding: binary
test
--inner--
--outer--
EOF
expect(a_request(:put, 'https://www.googleapis.com/upload/').with(body: expected_body)).to have_been_made
end
it 'should disallow downloads in batch' do
expect do |b|
service.batch_upload do |service|

73
spec/google/apis/core/upload_spec.rb
View File

@ -15,70 +15,6 @@
require 'spec_helper'
require 'google/apis/core/upload'
require 'google/apis/core/json_representation'
require 'hurley/test'
# TODO: JSON Response decoding
# TODO: Upload from IO
# TODO: Upload from file
RSpec.describe Google::Apis::Core::UploadIO do
context 'from_file' do
let(:upload_io) { Google::Apis::Core::UploadIO.from_file(file) }
context 'with text file' do
let(:file) { File.join(FIXTURES_DIR, 'files', 'test.txt') }
it 'should infer content type from file' do
expect(upload_io.content_type).to eql('text/plain')
end
it 'should allow overriding the mime type' do
io = Google::Apis::Core::UploadIO.from_file(file, content_type: 'application/json')
expect(io.content_type).to eql('application/json')
end
end
context 'with unknown type' do
let(:file) { File.join(FIXTURES_DIR, 'files', 'test.blah') }
it 'should use the default mime type' do
expect(upload_io.content_type).to eql('application/octet-stream')
end
it 'should allow overriding the mime type' do
io = Google::Apis::Core::UploadIO.from_file(file, content_type: 'application/json')
expect(io.content_type).to eql('application/json')
end
it 'should setup length of the stream' do
upload_io = Google::Apis::Core::UploadIO.from_file(file)
expect(upload_io.length).to eq File.size(file)
end
end
end
context 'from_io' do
context 'with i/o stream' do
let(:io) { StringIO.new 'Hello google' }
it 'should setup default content-type' do
upload_io = Google::Apis::Core::UploadIO.from_io(io)
expect(upload_io.content_type).to eql Google::Apis::Core::UploadIO::OCTET_STREAM_CONTENT_TYPE
end
it 'should allow overring the mime type' do
upload_io = Google::Apis::Core::UploadIO.from_io(io, content_type: 'application/x-gzip')
expect(upload_io.content_type).to eq('application/x-gzip')
end
it 'should setup length of the stream' do
upload_io = Google::Apis::Core::UploadIO.from_io(io)
expect(upload_io.length).to eq 'Hello google'.length
end
end
end
end
RSpec.describe Google::Apis::Core::RawUploadCommand do
include TestHelpers
@ -170,21 +106,22 @@ RSpec.describe Google::Apis::Core::MultipartUploadCommand do
before(:example) do
stub_request(:post, 'https://www.googleapis.com/zoo/animals').to_return(body: %(Hello world))
allow(Digest::SHA1).to receive(:hexdigest).and_return('123abc')
end
it 'should send content' do
expected_body = <<EOF.gsub(/\n/, "\r\n")
--RubyApiClientUpload
--123abc
Content-Type: application/json
metadata
--RubyApiClientUpload
Content-Length: 11
--123abc
Content-Type: text/plain
Content-Length: 11
Content-Transfer-Encoding: binary
Hello world
--RubyApiClientUpload--
--123abc--
EOF
command.execute(client)

103
third_party/hurley_patches.rb vendored
View File

@ -1,103 +0,0 @@
#
# Copyright (c) 2015 Rick Olson
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
require 'hurley'
require 'hurley/client'
require 'hurley/connection'
require 'net/https'
# Temporary monkey patch for streaming downloads. These are fixed in HEAD,
# but pending a 0.3 release.
if Hurley::VERSION == '0.2'
module Hurley
class Response
def location
@location ||= begin
return unless loc = @header[:location]
verb = STATUS_FORCE_GET.include?(status_code) ? :get : request.verb
statuses, receiver = request.send(:body_receiver)
new_request = Request.new(verb, request.url.join(Url.parse(loc)), request.header, request.body, request.options, request.ssl_options)
new_request.on_body(*statuses, &receiver) unless receiver.is_a?(Hurley::BodyReceiver)
new_request
end
end
end
class Connection
def call(request)
net_http_connection(request) do |http|
begin
Response.new(request) do |res|
perform_request(http, request, res)
# net/http only raises exception on 407 with ssl...?
if res.status_code == 407
raise ConnectionFailed, %(407 "Proxy Authentication Required")
end
end
rescue *NET_HTTP_EXCEPTIONS => err
if defined?(OpenSSL) && OpenSSL::SSL::SSLError === err
raise SSLError, err
else
raise ConnectionFailed, err
end
end
end
rescue ::Timeout::Error => err
raise Timeout, err
end
private
def net_http_request(request)
http_req = Net::HTTPGenericRequest.new(
request.verb.to_s.upcase, # request method
!!request.body, # is there a request body
:head != request.verb, # is there a response body
request.url.request_uri, # request uri path
request.header, # request headers
)
if body = request.body_io
http_req.body_stream = body
end
http_req
end
def perform_request(http, request, res)
http.request(net_http_request(request)) do |http_res|
res.status_code = http_res.code.to_i
http_res.each_header do |key, value|
res.header[key] = value
end
if :get == request.verb
http_res.read_body { |chunk| res.receive_body(chunk) }
else
res.receive_body(http_res.body)
end
end
end
end
end
end