diff --git a/.kokoro/release/all-generated.cfg b/.kokoro/release/all-generated.cfg new file mode 100644 index 000000000..ec149aa10 --- /dev/null +++ b/.kokoro/release/all-generated.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "PACKAGE" + value: "ALL_GENERATED" +} diff --git a/.kokoro/release/common.cfg b/.kokoro/release/common.cfg new file mode 100644 index 000000000..34a76ea43 --- /dev/null +++ b/.kokoro/release/common.cfg @@ -0,0 +1,89 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Fetch the token needed for reporting release status to GitHub +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "yoshi-automation-github-key" + } + } +} + +# Fetch magictoken to use with Magic Github Proxy +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "releasetool-magictoken" + backend_type: FASTCONFIGPUSH + } + } +} + +# Fetch api key to use with Magic Github Proxy +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "magic-github-proxy-api-key" + backend_type: FASTCONFIGPUSH + } + } +} + +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "docuploader_service_account" + } + } +} + +# Download resources for system tests (service account key, etc.) +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/google-cloud-ruby" + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "google-api-ruby-client/.kokoro/trampoline.sh" + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/yoshi-ruby/release" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/google-api-ruby-client/.kokoro/build.sh" +} + +env_vars: { + key: "TRAMPOLINE_SCRIPT" + value: "trampoline_v1.py" +} + +env_vars: { + key: "JOB_TYPE" + value: "release" +} + +env_vars: { + key: "OS" + value: "linux" +} + +env_vars: { + key: "REPO_DIR" + value: "github/google-api-ruby-client" +} diff --git a/.kokoro/release/core.cfg b/.kokoro/release/core.cfg new file mode 100644 index 000000000..dd9b0cb3b --- /dev/null +++ b/.kokoro/release/core.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "PACKAGE" + value: "google-apis-core" +} diff --git a/.kokoro/release/generator.cfg b/.kokoro/release/generator.cfg new file mode 100644 index 000000000..6cb93bc27 --- /dev/null +++ b/.kokoro/release/generator.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "PACKAGE" + value: "google-apis-generator" +} diff --git a/.kokoro/release/umbrella.cfg b/.kokoro/release/umbrella.cfg new file mode 100644 index 000000000..cd64ac141 --- /dev/null +++ b/.kokoro/release/umbrella.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "PACKAGE" + value: "google-api-client" +} diff --git a/Rakefile b/Rakefile index f6724290e..b9559d5ff 100644 --- a/Rakefile +++ b/Rakefile @@ -5,6 +5,10 @@ namespace :kokoro do load_env_vars end + task :dry_run do + ENV["DRY_RUN"] = "true" + end + task :presubmit do Rake::Task["kokoro:run_tests"].invoke end @@ -26,8 +30,27 @@ namespace :kokoro do end task :release do - Rake::Task["kokoro:publish_gem"].invoke - Rake::Task["kokoro:publish_docs"].invoke + require_relative "rakelib/releaser.rb" + + load_env_vars if ENV["KOKORO_GFILE_DIR"] + package = ENV["PACKAGE"] + raise "PACKAGE missing" unless package + dry_run = ENV["DRY_RUN"] + + if package == "ALL_GENERATED" + Dir.glob("generated/google-apis-*") do |gem_dir| + gem_name = File.basename(gem_dir) + releaser = Releaser.new(gem_name, gem_dir, dry_run: dry_run) + if releaser.needs_gem_publish? + releaser.publish_gem + releaser.publish_docs + end + end + else + releaser = Releaser.new(package, package, dry_run: dry_run) + releaser.publish_gem if releaser.needs_gem_publish? + releaser.publish_docs + end end task :run_tests do @@ -42,40 +65,6 @@ namespace :kokoro do end end end - - task :publish_gem do - require "gems" - - load_env_vars if ENV["KOKORO_GFILE_DIR"] - api_token = ENV["RUBYGEMS_API_TOKEN"] - if api_token - ::Gems.configure do |config| - config.key = api_token - end - end - - cd "google-api-client" do - Bundler.with_clean_env do - rm_rf "pkg" - sh "bundle update" - sh "bundle exec rake build" - built_gem_path = Dir.glob("pkg/google-api-client-*.gem").last - - response = ::Gems.push File.new built_gem_path - puts response - raise "Failed to release gem" unless response.include? "Successfully registered gem:" - end - end - end - - desc "Publish docs for the latest git tag" - task :publish_docs do - require_relative "rakelib/devsite/devsite_builder.rb" - - cd "google-api-client" do - DevsiteBuilder.new.publish ENV["DOCS_BUILD_TAG"] - end - end end def load_env_vars diff --git a/rakelib/releaser.rb b/rakelib/releaser.rb new file mode 100644 index 000000000..ec6fe232e --- /dev/null +++ b/rakelib/releaser.rb @@ -0,0 +1,146 @@ +require "fileutils" +require "gems" +require "rubygems" + +class Releaser + def initialize gem_name, gem_dir, + rubygems_api_token: nil, + docs_staging_bucket: nil, + docuploader_credentials: nil, + dry_run: false + @gem_name = gem_name + @gem_dir = File.expand_path(gem_dir) + @rubygems_api_token = rubygems_api_token || ENV["RUBYGEMS_API_TOKEN"] + @docs_staging_bucket = docs_staging_bucket || ENV["STAGING_BUCKET"] || "docs-staging" + @docuploader_credentials = docuploader_credentials + if @docuploader_credentials.nil? && ENV["KOKORO_KEYSTORE_DIR"] + @docuploader_credentials = File.join(ENV["KOKORO_KEYSTORE_DIR"], "73713_docuploader_service_account") + end + @dry_run = dry_run ? true : false + @rubygems_api_token_configured = false + @bundle_updated = false + end + + attr_reader :gem_name + attr_reader :gem_dir + attr_reader :rubygems_api_token + attr_reader :docs_staging_bucket + attr_reader :docuploader_credentials + + def dry_run? + @dry_run + end + + def needs_gem_publish? + Gem::Version.new(gem_version) > Gem::Version.new(current_rubygems_version) + end + + def publish_gem + configure_rubygems_api_token + bundle_dir do + FileUtils.rm_rf("pkg") + execute "bundle exec rake build" + built_gem_path = "pkg/#{gem_name}-#{gem_version}.gem" + raise "Failed to build #{built_gem_path}" unless File.file?(built_gem_path) + if dry_run? + puts "**** In dry run mode. Skipping gem publish of #{gem_name}" + return + end + response = gems_client.push(File.new(built_gem_path)) + puts response + raise "Failed to publish gem" unless response.include? "Successfully registered gem:" + end + end + + def publish_docs + bundle_dir do + FileUtils.rm_rf("doc") + FileUtils.rm_rf(".yardoc") + execute "bundle exec rake yard" + Dir.chdir("doc") do + execute "python3 -m docuploader create-metadata" \ + " --name #{gem_name}" \ + " --distribution-name #{gem_name}" \ + " --language ruby" \ + " --version v#{gem_version}" + unless docuploader_credentials + puts "**** No credentials available. Skipping doc upload of #{gem_name}" + return + end + if dry_run? + puts "**** In dry run mode. Skipping doc upload of #{gem_name}" + return + end + execute "python3 -m docuploader upload ." \ + " --credentials=#{docuploader_credentials}" \ + " --staging-bucket=#{docs_staging_bucket}" \ + " --metadata-file=./docs.metadata" + end + end + end + + def current_rubygems_version + @current_rubygems_version ||= begin + gems_client.info(gem_name)["version"] + rescue Gems::NotFound + "0.0.0" + end + end + + def gem_version + @gem_version ||= begin + version_content = File.read(version_file_path) + match = /\s(?:GEM_)?VERSION = "([\d\.]+)"/.match(version_content) + raise "Unable to find version constant in #{version_file_path}" unless match + match[1] + end + end + + def version_file_path + @version_file_path ||= begin + path = Dir.glob("#{gem_dir}/lib/google/apis/*/version.rb").first || + Dir.glob("#{gem_dir}/lib/google/apis/*/gem_version.rb").first || + Dir.glob("#{gem_dir}/lib/google/apis/version.rb").first + raise "Unable to find version file in #{gem_dir}" unless path + path + end + end + + def gems_client + @gems_client ||= Gems::Client.new + end + + def configure_rubygems_api_token + return if @rubygems_api_token_configured + @rubygems_api_token_configured = true + if rubygems_api_token + Gems.configure do |config| + config.key = rubygems_api_token + end + end + end + + def bundle_dir + block = proc do + execute "bundle update" unless @bundle_updated + @bundle_updated = true + yield + end + Dir.chdir(gem_dir) do + if defined?(Bundler) + if Bundler.respond_to?(:with_unbundled_env) + Bundler.with_unbundled_env(&block) + else + Bundler.with_clean_env(&block) + end + else + block.call + end + end + end + + def execute(cmd) + puts cmd + raise "Error executing command" unless system(cmd) + end +end