chore: Add synth scripts for client generation (#912)

.gitignore

@@ -19,3 +19,5 @@ tmp/
 *.iml
 atlassian*
 
+__pycache__/
+/vendor/

api_list_config.yaml

@@ -1 +1,9 @@
+include:
+  - name: appsmarket
+    version: v2
+    discovery_rest_url: https://content.googleapis.com/discovery/v1/apis/appsmarket/v2/rest
+  - name: youtubePartner
+    version: v1
+    discovery_rest_url: https://content.googleapis.com/discovery/v1/apis/youtubePartner/v1/rest
 exclude: []
+pause: []

bin/generate-api

@@ -28,32 +28,34 @@ module Google
 
     desc 'gen OUTDIR', 'Generate ruby API from an API description'
     method_options url: :array, file: :array, from_discovery: :boolean, preferred_only: :boolean,
-                   verbose: :boolean, names: :string, names_out: :string
+                   verbose: :boolean, names: :string, names_out: :string, api: :string, clean: :boolean
     def gen(dir)
       ensure_active_support
       require 'google/apis/generator'
 
       self.destination_root = dir
       Google::Apis.logger.level = Logger::DEBUG if options[:verbose]
-      generate_from_url(options[:url]) if options[:url]
+      count = 0
-      generate_from_file(options[:file]) if options[:file]
+      count += generate_from_url(options[:url]) if options[:url]
-      generate_from_discovery(preferred_only: options[:preferred_only]) if options[:from_discovery]
+      count += generate_from_file(options[:file]) if options[:file]
-      create_file(options[:names_out]) { |*| generator.dump_api_names } if options[:names_out]
+      count += generate_specific_apis(options[:api]) if options[:api]
+      count += generate_from_discovery(preferred_only: options[:preferred_only]) if options[:from_discovery]
+      count += clean_from_discovery if options[:clean]
+      create_file(options[:names_out]) { |*| generator.dump_api_names } if count > 0 && options[:names_out]
     end
 
     desc 'list', 'List public APIs'
     method_options verbose: :boolean, preferred_only: :boolean
     def list
       Google::Apis.logger.level = Logger::DEBUG if options[:verbose]
-      discovery = Discovery::DiscoveryService.new
+      discovery_api_list.each do |api|
-      apis = discovery.list_apis
-      apis.items.each do |api|
         say sprintf('%s - %s', api.id, api.description).strip unless options[:preferred_only] && !api.preferred?
       end
     end
 
     no_commands do
-      def generate_from_url(urls)
+      def generate_from_url(urls, first_only: false)
+        count = 0
         Array(urls).each do |url|
           begin
             json = discovery.http(:get, url)
@@ -62,7 +64,10 @@ module Google
             next
           end
           generate_api(json)
+          return 1 if first_only
+          count += 1
         end
+        count
       end
 
       def generate_from_file(files)
@@ -71,15 +76,38 @@ module Google
             generate_api(f.read)
           end
         end
+        Array(files).size
+      end
+
+      def generate_specific_apis(apis)
+        discovery_apis = discovery_api_list.each_with_object({}) do |api, hash|
+          hash["#{api.name}.#{api.version}"] = api
+        end
+        paused_apis = Array(api_list_config["pause"])
+        count = 0
+        Array(apis).each do |name_version|
+          api = discovery_apis[name_version]
+          if api.nil?
+            say "API #{api} is not in the discovery list."
+          elsif paused_apis.include? name_version
+            say "Ignoring paused API #{api.name} #{api.version}"
+          else
+            discovery_rest_url = "https://raw.githubusercontent.com/googleapis/discovery-artifact-manager/master/discoveries/#{api.name}.#{api.version}.json"
+            say sprintf('Loading %s, version %s from %s', api.name, api.version, discovery_rest_url)
+            generate_from_url([discovery_rest_url, api.discovery_rest_url], first_only: true)
+            count += 1
+          end
+        end
+        count
       end
 
       def generate_from_discovery(preferred_only: false)
         say 'Fetching API list'
-        apis = discovery.list_apis
+        paused_apis = Array(api_list_config["pause"])
-        exclude_apis = api_list_config["exclude"] || []
+        count = 0
-        apis.items.each do |api|
+        discovery_api_list.each do |api|
-          if exclude_apis.include? "#{api.name}.#{api.version}"
+          if paused_apis.include? "#{api.name}.#{api.version}"
-            say "Ignoring excluded API #{api.name} #{api.version}"
+            say "Ignoring paused API #{api.name} #{api.version}"
           elsif (preferred_only && !api.preferred?)
             say sprintf('Ignoring disoverable API %s', api.id)
           else
@@ -91,9 +119,27 @@ module Google
 	    # by the Discovery index.
             discovery_rest_url = "https://raw.githubusercontent.com/googleapis/discovery-artifact-manager/master/discoveries/#{api.name}.#{api.version}.json"
             say sprintf('Loading %s, version %s from %s', api.name, api.version, discovery_rest_url)
-            generate_from_url(discovery_rest_url)
+            generate_from_url([discovery_rest_url, api.discovery_rest_url], first_only: true)
+            count += 1
           end
         end
+        count
+      end
+
+      def clean_from_discovery
+        count = 0
+        apis = discovery_api_list.map { |api| "#{api.name.underscore}_#{api.version.tr '.', '_'}" }
+        Dir.chdir("#{destination_root}/google/apis") do
+          Dir.glob("*.rb").each do |filename|
+            filename = File.basename(filename, ".rb")
+            unless apis.include? filename
+              FileUtils.rm_r filename
+              FileUtils.rm "#{filename}.rb"
+              count += 1
+            end
+          end
+        end
+        count
       end
 
       def generate_api(json)
@@ -107,6 +153,20 @@ module Google
         @discovery ||= Discovery::DiscoveryService.new
       end
 
+      def discovery_api_list
+        @discovery_api_list ||= begin
+          items = discovery.list_apis.items
+          excluded = Array(api_list_config["exclude"])
+          items.delete_if { |item| excluded.include? "#{item.name}.#{item.version}" }
+          Array(api_list_config["include"]).each do |include_hash|
+            include_hash.symbolize_keys!
+            include_item = Discovery::DirectoryList::Item.new(**include_hash)
+            items << include_item unless items.any? { |item| item.name == include_item.name && item.version == include_item.version }
+          end
+          items
+        end
+      end
+
       def generator
         @generator ||= Google::Apis::Generator.new(api_names: options[:names], api_names_out: options[:names_out])
       end
@@ -117,7 +177,9 @@ module Google
 
       def ensure_active_support
         begin
+          require 'active_support'
           require 'active_support/inflector'
+          require 'active_support/core_ext'
         rescue LoadError => e
           error 'ActiveSupport is required, please run:'
           error 'gem install activesupport'

script/generate

@@ -4,10 +4,4 @@
 
 DIR=$(dirname $( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ))
 
-URLS=(https://content.googleapis.com/discovery/v1/apis/appsmarket/v2/rest \
+echo 'a' | bundle exec bin/generate-api gen generated --from-discovery --no-preferred-only --names-out=$DIR/api_names_out.yaml
-      https://content.googleapis.com/discovery/v1/apis/youtubePartner/v1/rest \
-      https://content.googleapis.com/discovery/v1/apis/compute/beta/rest \
-      https://monitoring.googleapis.com/\$discovery/rest?version=v3
-)
-
-echo 'a' | bundle exec bin/generate-api gen generated --from-discovery --no-preferred-only --names-out=$DIR/api_names_out.yaml --url=${URLS[*]}

script/synth.rb

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+
+require "fileutils"
+
+DIR = File.dirname __dir__
+
+def execute cmd
+  puts cmd
+  abort unless system cmd
+end
+
+execute "bundle install"
+
+if ARGV.empty?
+  execute "echo a | bundle exec bin/generate-api gen generated --from-discovery --no-preferred-only --names-out=#{DIR}/api_names_out.yaml"
+elsif ARGV == ["--clean"]
+  execute "bundle exec bin/generate-api gen generated --clean"
+elsif ARGV.size == 2
+  api, version = ARGV
+  execute "echo a | bundle exec bin/generate-api gen generated --api=#{api}.#{version} --names-out=#{DIR}/api_names_out.yaml"
+else
+  abort "Bad arguments: #{ARGV}"
+end

synth.py

@@ -0,0 +1,40 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""This script is used to synthesize generated parts of this library."""
+
+from synthtool.__main__ import extra_args
+import synthtool as s
+import synthtool.log as log
+import synthtool.shell as shell
+import logging
+import os
+
+logging.basicConfig(level=logging.DEBUG)
+s.metadata.set_track_obsolete_files(False)  # TODO: enable again.
+
+command = [
+    "docker",
+    "run",
+    "--rm",
+    f"-v{os.getcwd()}:/workspace",
+    "-v/var/run/docker.sock:/var/run/docker.sock",
+    "-w", "/workspace",
+    "--entrypoint", "script/synth.rb",
+    "gcr.io/cloud-devrel-kokoro-resources/yoshi-ruby/autosynth"]
+if extra_args():
+    command.extend(extra_args())
+
+log.debug(f"Running: {' '.join(command)}")
+shell.run(command, hide_output=False)