From a1b5f6c2d2d8876b46b0aa5b9cb0c96e868c5a41 Mon Sep 17 00:00:00 2001 From: Steve Bazyl Date: Wed, 2 Dec 2015 15:49:51 -0800 Subject: [PATCH] Issue #290 - Fix redirects during downloads, only stream body content on 20x response. Includes temporary patch to Hurley until 0.3 released --- lib/google/apis/core/download.rb | 2 +- lib/google/apis/core/http_command.rb | 1 + spec/google/apis/core/download_spec.rb | 13 ++++ third_party/hurley_patches.rb | 103 +++++++++++++++++++++++++ 4 files changed, 118 insertions(+), 1 deletion(-) create mode 100644 third_party/hurley_patches.rb diff --git a/lib/google/apis/core/download.rb b/lib/google/apis/core/download.rb index 207ff0099..6ddbc3984 100644 --- a/lib/google/apis/core/download.rb +++ b/lib/google/apis/core/download.rb @@ -71,7 +71,7 @@ module Google logger.debug { sprintf('Resuming download from offset %d', @offset) } req.header[RANGE_HEADER] = sprintf('bytes=%d-', @offset) end - req.on_body do |res, chunk| + req.on_body(200, 201) do |res, chunk| check_status(res.status_code, chunk) unless res.status_code.nil? logger.debug { sprintf('Writing chunk (%d bytes)', chunk.length) } @offset += chunk.length diff --git a/lib/google/apis/core/http_command.rb b/lib/google/apis/core/http_command.rb index 4827f7d1f..ded04e2e7 100644 --- a/lib/google/apis/core/http_command.rb +++ b/lib/google/apis/core/http_command.rb @@ -19,6 +19,7 @@ require 'google/apis/errors' require 'retriable' require 'hurley' require 'hurley/addressable' +require 'hurley_patches' require 'google/apis/core/logging' require 'pp' diff --git a/spec/google/apis/core/download_spec.rb b/spec/google/apis/core/download_spec.rb index 51e5fbf16..b7516ce36 100644 --- a/spec/google/apis/core/download_spec.rb +++ b/spec/google/apis/core/download_spec.rb @@ -59,6 +59,19 @@ RSpec.describe Google::Apis::Core::DownloadCommand do expect(received).to eql('Hello world') end end + + context 'with redirect' do + before(:example) do + stub_request(:get, 'https://www.googleapis.com/zoo/animals') + .to_return(status: [302, 'Content moved'], headers: { 'Location' => 'https://content.googleapis.com/files/12345' }, body: %(Content moved)) + stub_request(:get, 'https://content.googleapis.com/files/12345') + .to_return(headers: { 'Content-Type' => 'application/json' }, body: %(Hello world)) + end + + it 'should receive content' do + expect(received).to eql 'Hello world' + end + end end context 'with default destination' do diff --git a/third_party/hurley_patches.rb b/third_party/hurley_patches.rb new file mode 100644 index 000000000..c445e67d1 --- /dev/null +++ b/third_party/hurley_patches.rb @@ -0,0 +1,103 @@ +# +# Copyright (c) 2015 Rick Olson +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +require 'hurley' +require 'hurley/client' +require 'hurley/connection' +require 'net/https' + +# Temporary monkey patch for streaming downloads. These are fixed in HEAD, +# but pending a 0.3 release. +if Hurley::VERSION == '0.2' + module Hurley + class Response + def location + @location ||= begin + return unless loc = @header[:location] + verb = STATUS_FORCE_GET.include?(status_code) ? :get : request.verb + statuses, receiver = request.send(:body_receiver) + new_request = Request.new(verb, request.url.join(Url.parse(loc)), request.header, request.body, request.options, request.ssl_options) + new_request.on_body(*statuses, &receiver) unless receiver.is_a?(Hurley::BodyReceiver) + new_request + end + end + end + + class Connection + def call(request) + net_http_connection(request) do |http| + begin + Response.new(request) do |res| + perform_request(http, request, res) + + # net/http only raises exception on 407 with ssl...? + if res.status_code == 407 + raise ConnectionFailed, %(407 "Proxy Authentication Required") + end + end + rescue *NET_HTTP_EXCEPTIONS => err + if defined?(OpenSSL) && OpenSSL::SSL::SSLError === err + raise SSLError, err + else + raise ConnectionFailed, err + end + end + end + + rescue ::Timeout::Error => err + raise Timeout, err + end + + private + + def net_http_request(request) + http_req = Net::HTTPGenericRequest.new( + request.verb.to_s.upcase, # request method + !!request.body, # is there a request body + :head != request.verb, # is there a response body + request.url.request_uri, # request uri path + request.header, # request headers + ) + + if body = request.body_io + http_req.body_stream = body + end + + http_req + end + + def perform_request(http, request, res) + http.request(net_http_request(request)) do |http_res| + res.status_code = http_res.code.to_i + http_res.each_header do |key, value| + res.header[key] = value + end + + if :get == request.verb + http_res.read_body { |chunk| res.receive_body(chunk) } + else + res.receive_body(http_res.body) + end + end + end + end + end +end