require 'net/ssh' require 'pathname' require 'fileutils' require "resolv" namespace :create_site do desc "Detect sites" task :detect_sites,[:detect_name] => :environment do |task,args| Multithread.where(:key=>'detect_sites').each{|thread| thread.destroy if (thread.status["status"] == "error" || thread.status["status"] == "finish")} Multithread.where(:key=>'detect_sites').destroy @thread = Multithread.where(:key=>'detect_sites').first @type = "exec_all" if @thread.nil? begin @thread = Multithread.create(:key=>'detect_sites',:status=>{"infos"=>[],"status"=>"detecting"}) if( args.detect_name.nil? rescue true) site_servers = SiteServer.all.where(:active=>true).to_a else site_servers = SiteServer.where(:server_name=>args.detect_name).to_a end site_servers.each do |site_server| @site_server = site_server update_thread_infos(""+@site_server.server_name+"") @password = @site_server.password begin begin Net::SSH.start(@site_server.ip , @site_server.account , { password: @site_server.password, port: @site_server.port}) do |ssh| end rescue Net::SSH::HostKeyMismatch system("ssh-keygen -f \"$HOME/.ssh/known_hosts\" -R #{@site_server.ip}") rescue Errno::ENOTTY system("ssh-add \"$HOME/.ssh/id_rsa\"") rescue => e update_thread_infos(e.to_s) next end @no_stdout = true Net::SSH.start(@site_server.ip , @site_server.account , { password: @site_server.password, port: @site_server.port}) do |ssh| certbot_path = exec_ssh_command_by_sudo_and_see_output(ssh,"bash -l -c 'which certbot certbot-auto'",false,true).strip.split("\n")[0] @site_server.has_certbot = certbot_path.present? @site_server.save if @site_server.has_certbot update_thread_infos("Checking certbot renew cronjob...") crontab_lines = exec_ssh_command_by_sudo_and_see_output(ssh,"sudo -p 'sudo password:' crontab -l",false,false) certbot_renew_command = crontab_lines.select{|l| l.include?("certbot") && l.include?("renew")}[0] unless certbot_renew_command update_thread_infos("Add certbot renew cronjob!") certbot_renew_command = "30 2 * * 1 #{certbot_path} renew --no-self-upgrade --post-hook 'sudo service nginx restart' > /var/log/le-renew.log" update_thread_infos(certbot_renew_command) exec_ssh_command_by_sudo_and_see_output(ssh,"CRON=\"#{certbot_renew_command}\" && (sudo -p 'sudo password:' crontab -l; echo \"\$CRON\" ) | sudo -p 'sudo password:' crontab -",false) end end nginx_include_dir = exec_command_by_user(ssh,'grep include /etc/nginx/nginx.conf | grep -v "\#\|include /etc/nginx/mime.types\|include /etc/nginx/conf.d/\*.conf\|/etc/nginx/sites-enabled/\*"') nginx_include_dir = nginx_include_dir.gsub(/include|;|\n/,'').strip domain_name = @site_server.domain_name#'serv.rulingcom.com' server_names = exec_command_by_user(ssh,"grep -H 'server_name' -r #{nginx_include_dir}") server_names_array = server_names.scan(/(.*):[ \t]*server_name[ \t]+([^;]+);/) server_names_array = server_names_array.group_by{|v| v[0]} server_names_array.each do |nginx_file, server_name_with_file| org_server_names_for_site = server_name_with_file.map{|v| v[1].split(/[ |\t]+/)}.flatten.uniq server_names_for_site = org_server_names_for_site - ["localhost","127.0.0.1"] server_name_list = [] server_names_for_site.each do |server_name_for_site| if !server_name_for_site.include?(domain_name) if server_name_for_site.match(Regexp.union(Resolv::IPv4::Regex, Resolv::IPv6::Regex)).nil? #Not filter IP address next if !`nslookup "#{server_name_for_site}"`.include?(@site_server.ip) end end server_name_list << server_name_for_site end if server_name_list.blank? tmp = org_server_names_for_site & ["localhost","127.0.0.1"] if (tmp).length != 0 server_name_list = tmp + [@site_server.ip] end end server_name = server_name_list.join(' ') nginx_file_content = exec_ssh_command_by_sudo_and_see_output(ssh,"cat #{nginx_file}",false,true) site_path = Pathname.new(exec_ssh_command_by_sudo_and_see_output(ssh,"echo `grep root #{nginx_file} | grep -v -e '#.*root'`",false,true).to_s.split("\n").first.to_s.strip.split(";").select{|s| s.present? && s.include?("root")}.first.split("root").last.to_s.strip).dirname.to_s rescue nil next if site_path.nil? if site_path.present? && exec_ssh_command_by_sudo(ssh,"ls #{site_path}").split.length != 0 && exec_ssh_command_by_sudo(ssh,"ls #{site_path}/Gemfile").include?("No such file or directory") SiteConstruct.where(:server_type => @site_server.server_name , :domain_name=>server_name).destroy next end path = Pathname.new(site_path).dirname.to_s site_name = Pathname.new(site_path).basename.to_s server_type = @site_server.server_name port = exec_ssh_command_by_sudo_and_see_output(ssh,"grep 'listen' #{nginx_file} | grep -v -e '#.*listen'",false).flat_map{|s| s.split(/(\r\n|\n)/)}.map{|s| s.match(/listen\s+(\d+)/)[1] rescue nil}.select{|s| s.present?}.uniq db_name = exec_ssh_command_by_sudo(ssh,"echo `cat #{site_path}/config/mongoid.yml | grep 'database'`").split("database:").last.strip db_name = site_name.gsub("-","_") if db_name.include?("No such file or directory") unicorn_sock_path = exec_ssh_command_by_sudo(ssh,"echo `grep 'server unix' #{nginx_file} | grep -v -e '#.*server unix'`").strip.split("server unix").last.strip.split(":").last.strip rescue (site_path+"/tmp/unicorn.sock") if Pathname.new(path).basename.to_s == "orbit_sites" || Pathname.new(path).basename.to_s == "housing_sites" || Pathname.new(path).basename.to_s == "dev_sites" || Pathname.new(path).dirname.to_s == "/home" school_name = nil site_type = "Gravity" else school_name = Pathname.new(path).basename.to_s site_type = "School" end pid_infos = exec_ssh_command_by_sudo(ssh,"fuser #{unicorn_sock_path}").to_s.strip.gsub("\n","") if pid_infos.length != 0 && !pid_infos.include?("not exist") && !pid_infos.include?("No such file or directory") status = "finish" else bundle_show_info = exec_ssh_command_by_sudo(ssh,"bash -l -c 'cd #{site_path};bundle show'") if bundle_show_info.include?("is not yet checked out") || bundle_show_info.include?("No such file or directory") status = "" else status = "closed" end end site_constructs = SiteConstruct.where(:server_type => server_type , :nginx_file=>nginx_file).to_a site_construct = site_constructs[0] Array(site_constructs[1..-1]).each do |s| s.destroy end cert_ver_added_text = exec_command_by_user(ssh,"cat #{nginx_file}").scan(/[ \t]*location(?:(?!location.*{).)+{#add_by_site_module.*}/m)[0] update_thread_infos("Detect #{server_name}".html_safe) rescue nil if site_construct.nil? site_construct = SiteConstruct.create(:server_type=>server_type,:site_name=>site_name,:domain_name=>server_name,:nginx_file=>nginx_file,:db_name=>db_name,:port=>port,:path=>path,:site_type=>site_type,:school_name=>school_name,:user_id=>User.first.id,:status=>status,:cert_ver_added_text=>cert_ver_added_text) else site_construct.update(:server_type=>server_type,:site_name=>site_name,:domain_name=>server_name,:nginx_file=>nginx_file,:db_name=>db_name,:port=>port,:path=>path,:site_type=>site_type,:school_name=>school_name,:user_id=>User.first.id,:status=>status,:cert_ver_added_text=>cert_ver_added_text) end default_rails_env = 'development' enable_rails_env = ['development','production'] rails_env = default_rails_env cmd_output = exec_ssh_command_by_sudo_and_see_output(ssh,"ps -o args= -p `cat #{site_construct.path}/#{site_construct.site_name}/tmp/pids/unicorn.pid`",false) rescue [] enable_rails_env.each do |env| if( cmd_output[0].include?(env) rescue false) rails_env = env break elsif( cmd_output[0].include?("No such file or directory") rescue false) site_construct.update(:status => "closed") break end end site_construct.update(:rails_env => rails_env) crt_file_path = nginx_file_content.match(/(?<=!#|^)\s*ssl_certificate\s+(.*)/)[1].split(';').first rescue '' private_key_path = nginx_file_content.match(/(?<=!#|^)\s*ssl_certificate_key\s+(.*)/)[1].split(';').first rescue '' site_cert = nil if crt_file_path.present? || private_key_path.present? site_cert = site_construct.site_cert if site_cert.nil? site_cert = SiteCert.where(:source_paths=>[crt_file_path,private_key_path]).first site_cert = SiteCert.new if site_cert.nil? end if true #site_cert.source_paths.count == 0 site_cert["cert_file"] = File.basename(crt_file_path) cert_file_store_path = site_cert.cert_file.file.file crt_file_content = exec_ssh_command_by_sudo_and_see_output(ssh,"cat #{crt_file_path}",false).select{|s| s.present?}.join("\n").strip.split(/(\r\n|\n)/).select{|s| s.present?}.join("\n") FileUtils.mkdir_p(File.dirname(cert_file_store_path)) unless Dir.exist?(File.dirname(cert_file_store_path)) File.open(cert_file_store_path,'w+'){|f| f.write(crt_file_content)} site_cert["private_key"] = File.basename(private_key_path) private_key_store_path = site_cert.private_key.file.file private_key_content = exec_ssh_command_by_sudo_and_see_output(ssh,"cat #{private_key_path}",false).select{|s| s.present?}.join("\n").strip.split(/(\r\n|\n)/).select{|s| s.present?}.join("\n") FileUtils.mkdir_p(File.dirname(private_key_store_path)) unless Dir.exist?(File.dirname(private_key_store_path)) File.open(private_key_store_path,'w+'){|f| f.write(private_key_content)} site_cert.source_paths = [crt_file_path,private_key_path] site_cert.is_certbot = private_key_path.include?("letsencrypt") site_cert.save site_construct.update(:site_cert=>site_cert) end if nginx_file_content.match(/\s*return\s+30[12]\s+https:\/\/\$host\$request_uri\s*;/) site_construct.update(:redirect_to_https => true) else site_construct.update(:redirect_to_https => false) end end end update_thread_infos("Detecting Backup Setting on #{site_server.server_name}...") crontab_lines = exec_ssh_command_by_sudo_and_see_output(ssh,"sudo -p 'sudo password:' crontab -l;crontab -l",false,false) file_backups = crontab_lines.select{|s| s.match(/^\s*[^\s#]/) && s.include?('rsnapshot')}.map{|s| SiteServerFileBackup.read_from_crontab_line(s)}.compact rsnapshot_confs_path = ([SiteServerFileBackup::DefaultConf] + file_backups.map{|time, rsnapshot_conf_path, period| rsnapshot_conf_path}).uniq retain_settings = {} backup_dir_info = {} snapshot_root_info = {} rsnapshot_confs_path.each do |rsnapshot_conf_path| backup_retain_lines = exec_ssh_command_by_sudo_and_see_output(ssh,"sudo -p 'sudo password:' grep -E '^\\s*retain\\s+' #{rsnapshot_conf_path}",false,false).map{|s| s.strip}.select{|s| s.present?} backup_retain_lines = backup_retain_lines next if backup_retain_lines.blank? part_retain_settings = backup_retain_lines.map{|s| (s.strip.split(/\s+/)[1..2] rescue nil)}.compact.to_h retain_settings[rsnapshot_conf_path] = part_retain_settings backup_dir_subinfo = exec_ssh_command_by_sudo_and_see_output(ssh,"sudo -p 'sudo password:' grep -E '^\s*backup' #{rsnapshot_conf_path}",false,false).map{|s| s.strip}.select{|s| s.present?} # ex: ["backup\t/home/rulingcom\tlocalhost/"] backup_dir_info[rsnapshot_conf_path] = backup_dir_subinfo.map{|s| s.strip.split(/\s+/)[1..2]} snapshot_root_subinfo = exec_ssh_command_by_sudo_and_see_output(ssh,"sudo -p 'sudo password:' grep -E '^\s*snapshot_root' #{rsnapshot_conf_path}",false,false).map{|s| s.strip}.select{|s| s.present?}.last # ex: "snapshot_root\t/home/backup/orbit" snapshot_root_info[rsnapshot_conf_path] = snapshot_root_subinfo.strip.split(/\s+/)[1] rescue nil #only single snapshot_root in a conf end file_backups_group = file_backups.group_by{|time, rsnapshot_conf_path, period| rsnapshot_conf_path} file_backup_ids = [] retain_settings.each do |rsnapshot_conf_path, part_retain_settings| path = snapshot_root_info[rsnapshot_conf_path] if path.nil? path = '/home/backup/orbit' end backup_dir_subinfo = backup_dir_info[rsnapshot_conf_path] tmp = file_backups_group[rsnapshot_conf_path].to_a unused_period = part_retain_settings.map{|period_text, retain_count| SiteServerFileBackup::PeriodsTypes.index(period_text).to_i} - tmp.map{|time, rsnapshot_conf_path, period| period} unused_retain_settings = unused_period.map{|period| [period, part_retain_settings[SiteServerFileBackup::PeriodsTypes[period]]]} backup_dir_subinfo.each do |backup_dir, backup_prefix| unused_retain_settings.each do |period, retain_count| file_backup_data = {:site_server=>@site_server, :disable=>true, :period=>period, :path=>path, :retain_count=>retain_count, :rsnapshot_conf_path=>rsnapshot_conf_path, :backup_dir=>backup_dir, :backup_prefix=>backup_prefix} file_backup = SiteServerFileBackup.where(file_backup_data.except(:disable)).first if file_backup.nil? file_backup = SiteServerFileBackup.new(file_backup_data) file_backup.instance_variable_set(:@skip_callback, true) file_backup.save else file_backup.instance_variable_set(:@skip_callback, true) file_backup.update_attributes(file_backup_data) end file_backup_ids << file_backup.id end tmp.each do |time, rsnapshot_conf_path, period| retain_count = part_retain_settings[SiteServerFileBackup::PeriodsTypes[period]] if retain_count file_backup_data = {:site_server=>@site_server, :disable=>false, :backup_time=>time, :path=>path, :period=>period, :retain_count=>retain_count, :rsnapshot_conf_path=>rsnapshot_conf_path, :backup_dir=>backup_dir, :backup_prefix=>backup_prefix} file_backup = SiteServerFileBackup.where(file_backup_data.except(:disable)).first if file_backup.nil? file_backup = SiteServerFileBackup.create(file_backup_data) file_backup.instance_variable_set(:@skip_callback, true) file_backup.save else file_backup.instance_variable_set(:@skip_callback, true) file_backup.update_attributes(file_backup_data) end file_backup_ids << file_backup.id end end end end SiteServerFileBackup.where(:id.in=>file_backup_ids).update_all(:need_rewrite=>false) db_backup_ids = [] db_backup_infos = crontab_lines.select{|s| s.match(/^\s*[^\s#]/) && s.include?('mongodump')}.map{|s| SiteServerDbBackup.read_from_crontab_line(s, crontab_lines)}.compact db_backup_infos.each do |backup_time, path, period, retain_count| next if backup_time.nil? db_backup_data = {:site_server=>@site_server, :disable=>false, :period=>period, :path=>path, :backup_time=>backup_time, :retain_count=>retain_count} db_backup = SiteServerDbBackup.where(db_backup_data.except(:disable)).first if db_backup.nil? db_backup = SiteServerDbBackup.create(db_backup_data) db_backup.instance_variable_set(:@skip_callback, true) db_backup.save else db_backup.instance_variable_set(:@skip_callback, true) db_backup.update_attributes(db_backup_data) end db_backup_ids << db_backup.id end SiteServerDbBackup.where(:id.in=>db_backup_ids).update_all(:need_rewrite=>false) update_thread_infos("Finish detecting Backup Setting on #{site_server.server_name}!") end end end SiteConstruct.where(:domain_name.in => [nil,'','localhost','127.0.0.1']).destroy @thread.update(:status=>@thread.status.merge({"status"=>"finish"})) rescue => e puts [e,e.backtrace] @thread.update(:status=>{"infos"=>@thread.status["infos"].push(e.to_s),"status"=>"error"}) end end end def exec_ssh_command_by_sudo(session,command) output = session.exec!("echo '#{@password}' | sudo -S #{command}") # output = session.exec!("echo '#{@password}' | sudo -S -s #{command}") if output.include?("sudo:") && output.include?("command not found") output = session.exec!(command) end return output.encode!("UTF-8", :invalid => :replace, :undef => :replace, :replace => '') end def update_thread_infos(info) puts info @thread.status["infos"] = @thread.status["infos"].push(info) @thread.save! return @thread.status["infos"] end end