diff --git a/.kokoro/continuous/linux.cfg b/.kokoro/continuous/linux.cfg index 84bc77a67..8f8a846c5 100644 --- a/.kokoro/continuous/linux.cfg +++ b/.kokoro/continuous/linux.cfg @@ -3,10 +3,10 @@ build_file: "google-api-ruby-client/.kokoro/trampoline.sh" # Configure the docker image for kokoro-trampoline. -# Dockerfile is maintained at https://github.com/googleapis/google-cloud-ruby/tree/master/.kokoro/docker/ruby-multi +# Dockerfile is maintained at https://github.com/googleapis/google-cloud-ruby/tree/master/.kokoro/docker/multi env_vars: { key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/yoshi-ruby/ruby-multi" + value: "gcr.io/cloud-devrel-kokoro-resources/yoshi-ruby/multi" } env_vars: { diff --git a/.kokoro/presubmit/linux.cfg b/.kokoro/presubmit/linux.cfg index 2dc8a4a0e..77689157d 100644 --- a/.kokoro/presubmit/linux.cfg +++ b/.kokoro/presubmit/linux.cfg @@ -5,7 +5,7 @@ build_file: "google-api-ruby-client/.kokoro/trampoline.sh" # Configure the docker image for kokoro-trampoline. env_vars: { key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/yoshi-ruby/ruby-multi" + value: "gcr.io/cloud-devrel-kokoro-resources/yoshi-ruby/multi" } env_vars: { diff --git a/.kokoro/release.cfg b/.kokoro/release.cfg new file mode 100644 index 000000000..d7cff25fc --- /dev/null +++ b/.kokoro/release.cfg @@ -0,0 +1,94 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Fetch the token needed for reporting release status to GitHub +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "yoshi-automation-github-key" + } + } +} + +# Fetch magictoken to use with Magic Github Proxy +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "releasetool-magictoken" + backend_type: FASTCONFIGPUSH + } + } +} + +# Fetch api key to use with Magic Github Proxy +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "magic-github-proxy-api-key" + backend_type: FASTCONFIGPUSH + } + } +} + +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "docuploader_service_account" + } + } +} + +# Download resources for system tests (service account key, etc.) +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/google-cloud-ruby" + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "google-api-ruby-client/.kokoro/trampoline.sh" + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/yoshi-ruby/release" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/google-api-ruby-client/.kokoro/build.sh" +} + +env_vars: { + key: "TRAMPOLINE_SCRIPT" + value: "trampoline_v1.py" +} + +env_vars: { + key: "JOB_TYPE" + value: "release" +} + +env_vars: { + key: "OS" + value: "linux" +} + +env_vars: { + key: "REPO_DIR" + value: "github/google-api-ruby-client" +} + +env_vars: { + key: "PACKAGE" + value: "google-api-client" +} diff --git a/.repo-metadata.json b/.repo-metadata.json new file mode 100644 index 000000000..454fcdada --- /dev/null +++ b/.repo-metadata.json @@ -0,0 +1,5 @@ +{ + "name": "google-api-client", + "language": "ruby", + "distribution-name": "google-api-client" +} diff --git a/.rubocop.yml b/.rubocop.yml index 33d647ecf..73d7d7353 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -5,3 +5,8 @@ Metrics/LineLength: Style/FormatString: EnforcedStyle: sprintf + +AllCops: + Exclude: + - "Rakefile" + - "rakelib/**/*" diff --git a/Gemfile b/Gemfile index 4c1fd2aee..adfdfae42 100644 --- a/Gemfile +++ b/Gemfile @@ -22,6 +22,7 @@ group :development do gem 'redis', '~> 3.2' gem 'logging', '~> 2.2' gem 'opencensus', '~> 0.4' + gem 'httparty' end platforms :jruby do diff --git a/Rakefile b/Rakefile index bcb9ab422..fb9588500 100644 --- a/Rakefile +++ b/Rakefile @@ -1,4 +1,51 @@ require "bundler/gem_tasks" +require "json" + +task :release_gem, :tag do |_t, args| + tag = args[:tag] + raise "You must provide a tag to release." if tag.nil? + + # Verify the tag format "vVERSION" + m = tag.match(/v(?\S*)/) + raise "Tag #{tag} does not match the expected format." if m.nil? + + version = m[:version] + raise "You must provide a version." if version.nil? + + api_token = ENV["RUBYGEMS_API_TOKEN"] + + require "gems" + if api_token + ::Gems.configure do |config| + config.key = api_token + end + end + + Bundler.with_clean_env do + sh "rm -rf pkg" + sh "bundle update" + sh "bundle exec rake build" + end + + path_to_be_pushed = "pkg/google-api-client-#{version}.gem" + gem_was_published = nil + if File.file? path_to_be_pushed + begin + response = ::Gems.push File.new(path_to_be_pushed) + puts response + raise unless response.include? "Successfully registered gem:" + gem_was_published = true + puts "Successfully built and pushed google-api-client for version #{version}" + rescue StandardError => e + gem_was_published = false + puts "Error while releasing google-api-client version #{version}: #{e.message}" + end + else + raise "Cannot build google-api-client for version #{version}" + end + + Rake::Task["kokoro:publish_docs"].invoke if gem_was_published +end task default: :spec @@ -22,13 +69,35 @@ namespace :kokoro do task :nightly do Rake::Task["spec"].invoke end + + task :post do + require_relative "rakelib/link_checker.rb" + + link_checker = LinkChecker.new + link_checker.run + exit link_checker.exit_status + end + + task :release do + # Until code generation process is updated and release-please is set up, just publish docs + require_relative "rakelib/devsite/devsite_builder.rb" + + DevsiteBuilder.new.publish_if_missing ENV["DOCS_BUILD_TAG"] + end + + desc "Publish docs for the latest git tag" + task :publish_docs do + require_relative "rakelib/devsite/devsite_builder.rb" + + DevsiteBuilder.new.publish ENV["DOCS_BUILD_TAG"] + end end def header str, token = "#" line_length = str.length + 8 - puts "" + puts puts token * line_length puts "#{token * 3} #{str} #{token * 3}" puts token * line_length - puts "" + puts end diff --git a/rakelib/devsite/devsite_builder.rb b/rakelib/devsite/devsite_builder.rb new file mode 100644 index 000000000..c174166f6 --- /dev/null +++ b/rakelib/devsite/devsite_builder.rb @@ -0,0 +1,126 @@ +require "pathname" +require "tmpdir" + +require_relative "repo_metadata.rb" + +class DevsiteBuilder + def initialize build_tag = nil + @build_tag = build_tag || latest_tag + @output_dir = "doc" + end + + def build tag + checkout_tag tag + doc_path = tmp_dir + @output_dir + FileUtils.remove_dir doc_path if Dir.exist? doc_path + markup = "--markup markdown --markup-provider redcarpet" + + Dir.chdir tmp_dir do + cmds = ["-o #{@output_dir}", markup] + cmd "yard --verbose #{cmds.join ' '}" + end + metadata.build doc_path + end + + def upload + Dir.chdir tmp_dir + @output_dir do + opts = [ + "--credentials=#{ENV['KOKORO_KEYSTORE_DIR']}/73713_docuploader_service_account", + "--staging-bucket=#{ENV.fetch 'STAGING_BUCKET', 'docs-staging'}", + "--metadata-file=./docs.metadata" + ] + cmd "python3 -m docuploader upload . #{opts.join ' '}" + end + end + + def publish tag = nil + build(tag || @build_tag) + upload + end + + def publish_if_missing tag = nil + tag ||= @build_tag + puts tag + puts missing? tag + publish tag if missing? tag + end + + def missing? tag + require "httparty" + + url = "https://googleapis.dev/ruby/google-api-client/v#{version tag}/index.html" + response = HTTParty.get url + response.code != 200 + rescue StandardError + true + end + + def cmd line + puts line + output = `#{line}` + puts output + output + end + + def metadata + return @metadata if @metadata + + metadata_json = "#{tmp_dir}/.repo-metadata.json" + @metadata = RepoMetadata.from_source metadata_json if File.file? metadata_json + @metadata ||= RepoMetadata.from_source "name" => "google-api-client", + "distribution-name" => "google-api-client", + "language" => "ruby" + @metadata["version"] = version @build_tag + @metadata + end + + def checkout_tag git_tag + Dir.chdir tmp_dir do + `git checkout tags/#{git_tag} -b v#{version git_tag}` + end + end + + def version git_tag + m = git_tag.match(/(\d+\.\d+\.\d+)/) + return m if m.nil? + m[0] + end + + def versions + Dir.chdir tmp_dir do + tags.map { |t| version t }.reject(&:nil?).sort_by { |v| Gem::Version.new v }.reverse + end + end + + def tags + Dir.chdir tmp_dir do + `git tag`.split "\n" + end + end + + def latest_version + @latest_version ||= versions.first + end + + def latest_tag + @latest_tag ||= tags.select { |t| t.include? latest_version }.min_by(&:size) + end + + def tmp_dir + return @tmp_dir if @tmp_dir + + tmp = Dir.tmpdir + dir_name = "google-api-ruby-client" + @tmp_dir = Pathname.new(tmp) + dir_name + FileUtils.remove_dir @tmp_dir if Dir.exist? @tmp_dir + + Dir.chdir tmp do + `git clone https://github.com/googleapis/google-api-ruby-client.git` + end + Dir.chdir @tmp_dir do + `git fetch` + end + + @tmp_dir + end +end diff --git a/rakelib/devsite/link_checker.rb b/rakelib/devsite/link_checker.rb new file mode 100644 index 000000000..5842b8577 --- /dev/null +++ b/rakelib/devsite/link_checker.rb @@ -0,0 +1,64 @@ +require "open3" + +class LinkChecker + def initialize + @failed = false + end + + def run + job_info + git_commit = ENV.fetch "KOKORO_GITHUB_COMMIT", "master" + + markdown_files = Dir.glob "**/*.md" + broken_markdown_links = check_links(markdown_files, + "https://github.com/googleapis/google-api-ruby-client/tree/#{git_commit}", + " --skip '^(?!(\\Wruby.*google|.*google.*\\Wruby|.*cloud\\.google\\.com))'") + + broken_devsite_links = check_links(["google-api-client"], + "https://googleapis.dev/ruby", + "/latest/ --recurse --skip https:.*github.*") + + puts_broken_links broken_markdown_links + puts_broken_links broken_devsite_links + end + + def check_links location_list, base, tail + broken_links = Hash.new { |h, k| h[k] = [] } + location_list.each do |location| + out, err, st = Open3.capture3 "npx linkinator #{base}/#{location}#{tail}" + puts out + unless st.to_i.zero? + @failed = true + puts err + end + checked_links = out.split "\n" + checked_links.select! { |link| link =~ /\[\d+\]/ && !link.include?("[200]") } + unless checked_links.empty? + @failed = true + broken_links[location] += checked_links + end + end + broken_links + end + + def puts_broken_links link_hash + link_hash.each do |location, links| + puts "#{location} contains the following broken links:" + links.each { |link| puts " #{link}" } + puts "" + end + end + + def job_info + line_length = "Using Ruby - #{RUBY_VERSION}".length + 8 + puts + puts "#" * line_length + puts "### Using Ruby - #{RUBY_VERSION} ###" + puts "#" * line_length + puts + end + + def exit_status + @failed ? 1 : 0 + end +end diff --git a/rakelib/devsite/repo_metadata.rb b/rakelib/devsite/repo_metadata.rb new file mode 100644 index 000000000..fa06a1fca --- /dev/null +++ b/rakelib/devsite/repo_metadata.rb @@ -0,0 +1,56 @@ +require "json" + +class RepoMetadata + attr_accessor :data + + def initialize data + @data = data + normalize_data! + end + + def allowed_fields + [ + "name", "version", "language", "distribution-name", + "product-page", "github-repository", "issue-tracker" + ] + end + + def build output_directory + fields = @data.to_a.map { |kv| "--#{kv[0]} #{kv[1]}" } + Dir.chdir output_directory do + cmd "python3 -m docuploader create-metadata #{fields.join ' '}" + end + end + + def normalize_data! + @data.delete_if { |k, _| !allowed_fields.include?(k) } + end + + def [] key + data[key] + end + + def []= key, value + @data[key] = value + end + + def cmd line + puts line + output = `#{line}` + puts output + output + end + + def self.from_source source + if source.is_a? RepoMetadata + data = source.data + elsif source.is_a? Hash + data = source + elsif File.file? source + data = JSON.parse File.read(source) + else + raise "Source must be a path, hash, or RepoMetadata instance" + end + RepoMetadata.new data + end +end