chore: add release and continuous/post kokoro jobs (#836)
This commit is contained in:
parent
85763ece2f
commit
363b23570e
|
@ -3,10 +3,10 @@
|
|||
build_file: "google-api-ruby-client/.kokoro/trampoline.sh"
|
||||
|
||||
# Configure the docker image for kokoro-trampoline.
|
||||
# Dockerfile is maintained at https://github.com/googleapis/google-cloud-ruby/tree/master/.kokoro/docker/ruby-multi
|
||||
# Dockerfile is maintained at https://github.com/googleapis/google-cloud-ruby/tree/master/.kokoro/docker/multi
|
||||
env_vars: {
|
||||
key: "TRAMPOLINE_IMAGE"
|
||||
value: "gcr.io/cloud-devrel-kokoro-resources/yoshi-ruby/ruby-multi"
|
||||
value: "gcr.io/cloud-devrel-kokoro-resources/yoshi-ruby/multi"
|
||||
}
|
||||
|
||||
env_vars: {
|
||||
|
|
|
@ -5,7 +5,7 @@ build_file: "google-api-ruby-client/.kokoro/trampoline.sh"
|
|||
# Configure the docker image for kokoro-trampoline.
|
||||
env_vars: {
|
||||
key: "TRAMPOLINE_IMAGE"
|
||||
value: "gcr.io/cloud-devrel-kokoro-resources/yoshi-ruby/ruby-multi"
|
||||
value: "gcr.io/cloud-devrel-kokoro-resources/yoshi-ruby/multi"
|
||||
}
|
||||
|
||||
env_vars: {
|
||||
|
|
|
@ -0,0 +1,94 @@
|
|||
# Format: //devtools/kokoro/config/proto/build.proto
|
||||
|
||||
# Build logs will be here
|
||||
action {
|
||||
define_artifacts {
|
||||
regex: "**/*sponge_log.xml"
|
||||
}
|
||||
}
|
||||
|
||||
# Fetch the token needed for reporting release status to GitHub
|
||||
before_action {
|
||||
fetch_keystore {
|
||||
keystore_resource {
|
||||
keystore_config_id: 73713
|
||||
keyname: "yoshi-automation-github-key"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Fetch magictoken to use with Magic Github Proxy
|
||||
before_action {
|
||||
fetch_keystore {
|
||||
keystore_resource {
|
||||
keystore_config_id: 73713
|
||||
keyname: "releasetool-magictoken"
|
||||
backend_type: FASTCONFIGPUSH
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Fetch api key to use with Magic Github Proxy
|
||||
before_action {
|
||||
fetch_keystore {
|
||||
keystore_resource {
|
||||
keystore_config_id: 73713
|
||||
keyname: "magic-github-proxy-api-key"
|
||||
backend_type: FASTCONFIGPUSH
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
before_action {
|
||||
fetch_keystore {
|
||||
keystore_resource {
|
||||
keystore_config_id: 73713
|
||||
keyname: "docuploader_service_account"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Download resources for system tests (service account key, etc.)
|
||||
gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/google-cloud-ruby"
|
||||
|
||||
# Download trampoline resources.
|
||||
gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline"
|
||||
|
||||
# Use the trampoline script to run in docker.
|
||||
build_file: "google-api-ruby-client/.kokoro/trampoline.sh"
|
||||
|
||||
# Configure the docker image for kokoro-trampoline.
|
||||
env_vars: {
|
||||
key: "TRAMPOLINE_IMAGE"
|
||||
value: "gcr.io/cloud-devrel-kokoro-resources/yoshi-ruby/release"
|
||||
}
|
||||
|
||||
env_vars: {
|
||||
key: "TRAMPOLINE_BUILD_FILE"
|
||||
value: "github/google-api-ruby-client/.kokoro/build.sh"
|
||||
}
|
||||
|
||||
env_vars: {
|
||||
key: "TRAMPOLINE_SCRIPT"
|
||||
value: "trampoline_v1.py"
|
||||
}
|
||||
|
||||
env_vars: {
|
||||
key: "JOB_TYPE"
|
||||
value: "release"
|
||||
}
|
||||
|
||||
env_vars: {
|
||||
key: "OS"
|
||||
value: "linux"
|
||||
}
|
||||
|
||||
env_vars: {
|
||||
key: "REPO_DIR"
|
||||
value: "github/google-api-ruby-client"
|
||||
}
|
||||
|
||||
env_vars: {
|
||||
key: "PACKAGE"
|
||||
value: "google-api-client"
|
||||
}
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"name": "google-api-client",
|
||||
"language": "ruby",
|
||||
"distribution-name": "google-api-client"
|
||||
}
|
|
@ -5,3 +5,8 @@ Metrics/LineLength:
|
|||
|
||||
Style/FormatString:
|
||||
EnforcedStyle: sprintf
|
||||
|
||||
AllCops:
|
||||
Exclude:
|
||||
- "Rakefile"
|
||||
- "rakelib/**/*"
|
||||
|
|
1
Gemfile
1
Gemfile
|
@ -22,6 +22,7 @@ group :development do
|
|||
gem 'redis', '~> 3.2'
|
||||
gem 'logging', '~> 2.2'
|
||||
gem 'opencensus', '~> 0.4'
|
||||
gem 'httparty'
|
||||
end
|
||||
|
||||
platforms :jruby do
|
||||
|
|
73
Rakefile
73
Rakefile
|
@ -1,4 +1,51 @@
|
|||
require "bundler/gem_tasks"
|
||||
require "json"
|
||||
|
||||
task :release_gem, :tag do |_t, args|
|
||||
tag = args[:tag]
|
||||
raise "You must provide a tag to release." if tag.nil?
|
||||
|
||||
# Verify the tag format "vVERSION"
|
||||
m = tag.match(/v(?<version>\S*)/)
|
||||
raise "Tag #{tag} does not match the expected format." if m.nil?
|
||||
|
||||
version = m[:version]
|
||||
raise "You must provide a version." if version.nil?
|
||||
|
||||
api_token = ENV["RUBYGEMS_API_TOKEN"]
|
||||
|
||||
require "gems"
|
||||
if api_token
|
||||
::Gems.configure do |config|
|
||||
config.key = api_token
|
||||
end
|
||||
end
|
||||
|
||||
Bundler.with_clean_env do
|
||||
sh "rm -rf pkg"
|
||||
sh "bundle update"
|
||||
sh "bundle exec rake build"
|
||||
end
|
||||
|
||||
path_to_be_pushed = "pkg/google-api-client-#{version}.gem"
|
||||
gem_was_published = nil
|
||||
if File.file? path_to_be_pushed
|
||||
begin
|
||||
response = ::Gems.push File.new(path_to_be_pushed)
|
||||
puts response
|
||||
raise unless response.include? "Successfully registered gem:"
|
||||
gem_was_published = true
|
||||
puts "Successfully built and pushed google-api-client for version #{version}"
|
||||
rescue StandardError => e
|
||||
gem_was_published = false
|
||||
puts "Error while releasing google-api-client version #{version}: #{e.message}"
|
||||
end
|
||||
else
|
||||
raise "Cannot build google-api-client for version #{version}"
|
||||
end
|
||||
|
||||
Rake::Task["kokoro:publish_docs"].invoke if gem_was_published
|
||||
end
|
||||
|
||||
task default: :spec
|
||||
|
||||
|
@ -22,13 +69,35 @@ namespace :kokoro do
|
|||
task :nightly do
|
||||
Rake::Task["spec"].invoke
|
||||
end
|
||||
|
||||
task :post do
|
||||
require_relative "rakelib/link_checker.rb"
|
||||
|
||||
link_checker = LinkChecker.new
|
||||
link_checker.run
|
||||
exit link_checker.exit_status
|
||||
end
|
||||
|
||||
task :release do
|
||||
# Until code generation process is updated and release-please is set up, just publish docs
|
||||
require_relative "rakelib/devsite/devsite_builder.rb"
|
||||
|
||||
DevsiteBuilder.new.publish_if_missing ENV["DOCS_BUILD_TAG"]
|
||||
end
|
||||
|
||||
desc "Publish docs for the latest git tag"
|
||||
task :publish_docs do
|
||||
require_relative "rakelib/devsite/devsite_builder.rb"
|
||||
|
||||
DevsiteBuilder.new.publish ENV["DOCS_BUILD_TAG"]
|
||||
end
|
||||
end
|
||||
|
||||
def header str, token = "#"
|
||||
line_length = str.length + 8
|
||||
puts ""
|
||||
puts
|
||||
puts token * line_length
|
||||
puts "#{token * 3} #{str} #{token * 3}"
|
||||
puts token * line_length
|
||||
puts ""
|
||||
puts
|
||||
end
|
||||
|
|
|
@ -0,0 +1,126 @@
|
|||
require "pathname"
|
||||
require "tmpdir"
|
||||
|
||||
require_relative "repo_metadata.rb"
|
||||
|
||||
class DevsiteBuilder
|
||||
def initialize build_tag = nil
|
||||
@build_tag = build_tag || latest_tag
|
||||
@output_dir = "doc"
|
||||
end
|
||||
|
||||
def build tag
|
||||
checkout_tag tag
|
||||
doc_path = tmp_dir + @output_dir
|
||||
FileUtils.remove_dir doc_path if Dir.exist? doc_path
|
||||
markup = "--markup markdown --markup-provider redcarpet"
|
||||
|
||||
Dir.chdir tmp_dir do
|
||||
cmds = ["-o #{@output_dir}", markup]
|
||||
cmd "yard --verbose #{cmds.join ' '}"
|
||||
end
|
||||
metadata.build doc_path
|
||||
end
|
||||
|
||||
def upload
|
||||
Dir.chdir tmp_dir + @output_dir do
|
||||
opts = [
|
||||
"--credentials=#{ENV['KOKORO_KEYSTORE_DIR']}/73713_docuploader_service_account",
|
||||
"--staging-bucket=#{ENV.fetch 'STAGING_BUCKET', 'docs-staging'}",
|
||||
"--metadata-file=./docs.metadata"
|
||||
]
|
||||
cmd "python3 -m docuploader upload . #{opts.join ' '}"
|
||||
end
|
||||
end
|
||||
|
||||
def publish tag = nil
|
||||
build(tag || @build_tag)
|
||||
upload
|
||||
end
|
||||
|
||||
def publish_if_missing tag = nil
|
||||
tag ||= @build_tag
|
||||
puts tag
|
||||
puts missing? tag
|
||||
publish tag if missing? tag
|
||||
end
|
||||
|
||||
def missing? tag
|
||||
require "httparty"
|
||||
|
||||
url = "https://googleapis.dev/ruby/google-api-client/v#{version tag}/index.html"
|
||||
response = HTTParty.get url
|
||||
response.code != 200
|
||||
rescue StandardError
|
||||
true
|
||||
end
|
||||
|
||||
def cmd line
|
||||
puts line
|
||||
output = `#{line}`
|
||||
puts output
|
||||
output
|
||||
end
|
||||
|
||||
def metadata
|
||||
return @metadata if @metadata
|
||||
|
||||
metadata_json = "#{tmp_dir}/.repo-metadata.json"
|
||||
@metadata = RepoMetadata.from_source metadata_json if File.file? metadata_json
|
||||
@metadata ||= RepoMetadata.from_source "name" => "google-api-client",
|
||||
"distribution-name" => "google-api-client",
|
||||
"language" => "ruby"
|
||||
@metadata["version"] = version @build_tag
|
||||
@metadata
|
||||
end
|
||||
|
||||
def checkout_tag git_tag
|
||||
Dir.chdir tmp_dir do
|
||||
`git checkout tags/#{git_tag} -b v#{version git_tag}`
|
||||
end
|
||||
end
|
||||
|
||||
def version git_tag
|
||||
m = git_tag.match(/(\d+\.\d+\.\d+)/)
|
||||
return m if m.nil?
|
||||
m[0]
|
||||
end
|
||||
|
||||
def versions
|
||||
Dir.chdir tmp_dir do
|
||||
tags.map { |t| version t }.reject(&:nil?).sort_by { |v| Gem::Version.new v }.reverse
|
||||
end
|
||||
end
|
||||
|
||||
def tags
|
||||
Dir.chdir tmp_dir do
|
||||
`git tag`.split "\n"
|
||||
end
|
||||
end
|
||||
|
||||
def latest_version
|
||||
@latest_version ||= versions.first
|
||||
end
|
||||
|
||||
def latest_tag
|
||||
@latest_tag ||= tags.select { |t| t.include? latest_version }.min_by(&:size)
|
||||
end
|
||||
|
||||
def tmp_dir
|
||||
return @tmp_dir if @tmp_dir
|
||||
|
||||
tmp = Dir.tmpdir
|
||||
dir_name = "google-api-ruby-client"
|
||||
@tmp_dir = Pathname.new(tmp) + dir_name
|
||||
FileUtils.remove_dir @tmp_dir if Dir.exist? @tmp_dir
|
||||
|
||||
Dir.chdir tmp do
|
||||
`git clone https://github.com/googleapis/google-api-ruby-client.git`
|
||||
end
|
||||
Dir.chdir @tmp_dir do
|
||||
`git fetch`
|
||||
end
|
||||
|
||||
@tmp_dir
|
||||
end
|
||||
end
|
|
@ -0,0 +1,64 @@
|
|||
require "open3"
|
||||
|
||||
class LinkChecker
|
||||
def initialize
|
||||
@failed = false
|
||||
end
|
||||
|
||||
def run
|
||||
job_info
|
||||
git_commit = ENV.fetch "KOKORO_GITHUB_COMMIT", "master"
|
||||
|
||||
markdown_files = Dir.glob "**/*.md"
|
||||
broken_markdown_links = check_links(markdown_files,
|
||||
"https://github.com/googleapis/google-api-ruby-client/tree/#{git_commit}",
|
||||
" --skip '^(?!(\\Wruby.*google|.*google.*\\Wruby|.*cloud\\.google\\.com))'")
|
||||
|
||||
broken_devsite_links = check_links(["google-api-client"],
|
||||
"https://googleapis.dev/ruby",
|
||||
"/latest/ --recurse --skip https:.*github.*")
|
||||
|
||||
puts_broken_links broken_markdown_links
|
||||
puts_broken_links broken_devsite_links
|
||||
end
|
||||
|
||||
def check_links location_list, base, tail
|
||||
broken_links = Hash.new { |h, k| h[k] = [] }
|
||||
location_list.each do |location|
|
||||
out, err, st = Open3.capture3 "npx linkinator #{base}/#{location}#{tail}"
|
||||
puts out
|
||||
unless st.to_i.zero?
|
||||
@failed = true
|
||||
puts err
|
||||
end
|
||||
checked_links = out.split "\n"
|
||||
checked_links.select! { |link| link =~ /\[\d+\]/ && !link.include?("[200]") }
|
||||
unless checked_links.empty?
|
||||
@failed = true
|
||||
broken_links[location] += checked_links
|
||||
end
|
||||
end
|
||||
broken_links
|
||||
end
|
||||
|
||||
def puts_broken_links link_hash
|
||||
link_hash.each do |location, links|
|
||||
puts "#{location} contains the following broken links:"
|
||||
links.each { |link| puts " #{link}" }
|
||||
puts ""
|
||||
end
|
||||
end
|
||||
|
||||
def job_info
|
||||
line_length = "Using Ruby - #{RUBY_VERSION}".length + 8
|
||||
puts
|
||||
puts "#" * line_length
|
||||
puts "### Using Ruby - #{RUBY_VERSION} ###"
|
||||
puts "#" * line_length
|
||||
puts
|
||||
end
|
||||
|
||||
def exit_status
|
||||
@failed ? 1 : 0
|
||||
end
|
||||
end
|
|
@ -0,0 +1,56 @@
|
|||
require "json"
|
||||
|
||||
class RepoMetadata
|
||||
attr_accessor :data
|
||||
|
||||
def initialize data
|
||||
@data = data
|
||||
normalize_data!
|
||||
end
|
||||
|
||||
def allowed_fields
|
||||
[
|
||||
"name", "version", "language", "distribution-name",
|
||||
"product-page", "github-repository", "issue-tracker"
|
||||
]
|
||||
end
|
||||
|
||||
def build output_directory
|
||||
fields = @data.to_a.map { |kv| "--#{kv[0]} #{kv[1]}" }
|
||||
Dir.chdir output_directory do
|
||||
cmd "python3 -m docuploader create-metadata #{fields.join ' '}"
|
||||
end
|
||||
end
|
||||
|
||||
def normalize_data!
|
||||
@data.delete_if { |k, _| !allowed_fields.include?(k) }
|
||||
end
|
||||
|
||||
def [] key
|
||||
data[key]
|
||||
end
|
||||
|
||||
def []= key, value
|
||||
@data[key] = value
|
||||
end
|
||||
|
||||
def cmd line
|
||||
puts line
|
||||
output = `#{line}`
|
||||
puts output
|
||||
output
|
||||
end
|
||||
|
||||
def self.from_source source
|
||||
if source.is_a? RepoMetadata
|
||||
data = source.data
|
||||
elsif source.is_a? Hash
|
||||
data = source
|
||||
elsif File.file? source
|
||||
data = JSON.parse File.read(source)
|
||||
else
|
||||
raise "Source must be a path, hash, or RepoMetadata instance"
|
||||
end
|
||||
RepoMetadata.new data
|
||||
end
|
||||
end
|
Loading…
Reference in New Issue