diff --git a/app/controllers/admin/site_panel_controller.rb b/app/controllers/admin/site_panel_controller.rb index 597efb3..27cce94 100644 --- a/app/controllers/admin/site_panel_controller.rb +++ b/app/controllers/admin/site_panel_controller.rb @@ -439,6 +439,10 @@ class Admin::SitePanelController < OrbitAdminController end end end + when 'apply_change_backup_setting' + Thread.new do + system("bundle exec rake exec_commands:change_backup_setting[#{params[:id]}]") + end else Thread.new do cmds = params[:commands].split('////') @@ -516,6 +520,27 @@ class Admin::SitePanelController < OrbitAdminController @site_server = SiteServer.find(params[:id]) end def update_backup_setting + @site_server = SiteServer.find(params[:id]) + site_server_params = params.require(:site_server).permit! rescue {} + need_rewrite_backup_setting = false + site_server_params["site_server_file_backups_attributes"].each do |k, v| + if v["_destroy"] == 'true' + need_rewrite_backup_setting = true + end + end if site_server_params["site_server_file_backups_attributes"] + site_server_params["site_server_db_backups_attributes"].each do |k, v| + if v["_destroy"] == 'true' + need_rewrite_backup_setting = true + end + end if site_server_params["site_server_db_backups_attributes"] + @site_server.update_attributes(site_server_params) + unless need_rewrite_backup_setting + need_rewrite_backup_setting = ((@site_server.site_server_file_backups.where(:need_rewrite=>true).count + @site_server.site_server_db_backups.where(:need_rewrite=>true).count) != 0) + end + if need_rewrite_backup_setting + @site_server.update(:need_rewrite_backup_setting=>true) + end + redirect_back_custom end private def site_cert_params diff --git a/app/models/site_server.rb b/app/models/site_server.rb index 4d7eed9..f0773fd 100644 --- a/app/models/site_server.rb +++ b/app/models/site_server.rb @@ -17,7 +17,12 @@ class SiteServer field :need_update_site_ids, type: Array, default: [] field :super_user_changed, type: Boolean, type: false field :removed_super_users, type: Array, default: [] + field :need_rewrite_backup_setting, type: Boolean, default: false + has_many :site_server_file_backups, :autosave => true, :dependent => :destroy + has_many :site_server_db_backups, :autosave => true, :dependent => :destroy has_many :site_super_users, :autosave => true, :dependent => :destroy + accepts_nested_attributes_for :site_server_file_backups, :allow_destroy => true + accepts_nested_attributes_for :site_server_db_backups, :allow_destroy => true accepts_nested_attributes_for :site_super_users, :allow_destroy => true def check_super_user_changed self.update(:super_user_changed => (self.site_super_users.where(:is_changed=>true).count != 0 || self.removed_super_users.count != 0)) diff --git a/app/models/site_server_db_backup.rb b/app/models/site_server_db_backup.rb new file mode 100644 index 0000000..0cd04da --- /dev/null +++ b/app/models/site_server_db_backup.rb @@ -0,0 +1,143 @@ +class SiteServerDbBackup + include Mongoid::Document + include Mongoid::Timestamps + PeriodsTypes=['day','week','month'] + belongs_to :site_server + field :disable, type: Boolean, default: false + field :backup_time, type: String, default: '03:00' + field :path , type: String ,default: '/home/backup/db' + field :period, type: Integer, default: 0 # 0 => daily , 1 => weekly , 2 => monthly + field :retain_count, type: Integer, default: 7 + field :need_rewrite, type: Boolean, default: true + + before_save do + if self.changed? + self.need_rewrite = true + end + end + def to_crontab_script(return_string=true) + # minute , hour, day(of month, 1~31), month(1~12), day(of week,0~6) + crontab_time = ['*', '*', '*', '*', '*'] + hour, minute = self.backup_time.split(':').map{|s| s.to_i} + crontab_time[0] = minute.to_s rescue '0' + crontab_time[1] = hour.to_s rescue '0' + if self.period == 1 #weekly + crontab_time[4] = '0' #every Sunday + elsif self.period == 2 #monthly + crontab_time[2] = '1' #day 1 of every month + end + if self.retain_count + retain_text = "1 week ago" + if self.period != 0 || self.retain_count != 7 + retain_text = "#{self.retain_count} #{PeriodsTypes[self.period]} ago" + end + scripts = ["#{crontab_time.join(' ')} mongodump -o #{self.path}/`date \"+\\%Y\\%m\\%d\"`_db"] + if hour < 23 + next_hour = hour + 1 + crontab_time[1] = next_hour.to_s + else + crontab_time[1] = ((hour + 1) % 24).to_s + if self.period == 1 #weekly + crontab_time[4] = '1' #every Sunday + 1 = Monday + elsif self.period == 2 #monthly + crontab_time[2] = '2' #day 1 of every month + 1 = 2 + end + retain_text += " 1 day ago" + end + scripts << "#{crontab_time.join(' ')} rm -fr #{self.path}/`date --date=\"#{retain_text}\" \"+\\%Y\\%m\\%d\"`_db" + end + if return_string + scripts.join("\n") + else + scripts + end + end + def self.read_from_crontab_line(crontab_line, crontab_lines) + if crontab_line.include?('mongodump') + tmp = crontab_line.split(/\s+[^\s]*mongodump\s+/) # ex: ["0 3 * * *", "-o /home/backup/db/`date \"+\\%Y\\%m\\%d\"`_db"] + if tmp.count == 2 + crontab_time = tmp[0].split(/\s+/) + path = File.dirname(tmp[1].split('-o')[1].strip) rescue nil + if path + if crontab_time[4] != '*' #weekly + period = 1 + period_unit = 1.week + elsif crontab_time[2] != '*' #monthly + period = 2 + period_unit = 1.month + else #daily + period = 0 + period_unit = 1.day + end + time = "#{crontab_time[1].rjust(2, "0")}:#{crontab_time[0].rjust(2, "0")}" + regex_path = /\s+#{::Regexp.escape(path)}\// + match_line = crontab_lines.select{|l| l.match(/\s+(|\/usr\/bin\/)rm/) && l.match(regex_path)}.first + if match_line.nil? + retain_count = nil + else + retain_text = match_line.scan(/`\s*date\s+\-\-date=[\'\"]([^\'\"]+)[\'\"]/).flatten.first + tmp_retain_count, tmp_period = retain_text.sub(/\s+ago/, '').split(/\s+/) + tmp_retain_count = tmp_retain_count.to_i + retain_count = (tmp_retain_count.send(tmp_period) / period_unit rescue nil) + end + return [time, path, period, retain_count] + else + return nil + end + else + return nil + end + else + return nil + end + end + def self.clean_crontab_setting(crontab_lines_str) + crontab_lines = crontab_lines_str.split("\n") + related_path = [] + crontab_lines.each do |crontab_line| + if crontab_line.include?('mongodump') + tmp = crontab_line.split(/\s+[^\s]*mongodump\s+/) # ex: ["0 3 * * *", "-o /home/backup/db/`date \"+\\%Y\\%m\\%d\"`_db"] + output_dir = File.dirname(tmp[1].split('-o')[1].strip) rescue nil + if output_dir + related_path << output_dir + end + end + end + related_path_reg = Regexp.union(related_path.map{|s| /\s+#{::Regexp.escape(s)}\//}) + crontab_lines_str = crontab_lines.map do |crontab_line| + if crontab_line.match(related_path_reg) + nil + else + crontab_line + end + end.compact.join("\n") + end + def self.write_crontab_setting(site_server, crontab_lines_str) + new_crontab_lines = self.where(:site_server_id=> site_server.id, :disable=>false).map{|db_backup| db_backup.to_crontab_script(false)} + crontab_lines = crontab_lines_str.split("\n") + exists_indices = new_crontab_lines.map do |sub_crontab_lines| + match_reg = ::Regexp.new(sub_crontab_lines[0].gsub(/\s+/, '\s+')) + crontab_lines.index{|l| l.match(match_reg)} + end + removed_related_path = [] + crontab_lines.each_with_index do |crontab_line, i| + if crontab_line.include?('mongodump') && exists_indices.exclude?(i) + tmp = crontab_line.split(/\s+[^\s]*mongodump\s+/) # ex: ["0 3 * * *", "-o /home/backup/db/`date \"+\\%Y\\%m\\%d\"`_db"] + output_dir = File.dirname(tmp[1].split('-o')[1].strip) rescue nil + if output_dir + removed_related_path << output_dir + end + end + end + removed_related_path.each_with_index do |path| + regex_path = /\s+#{::Regexp.escape(path)}\// + crontab_lines = crontab_lines.select{|l| l.match(regex_path).nil?} + end + exists_indices.each_with_index do |idx, i| + if idx.nil? + crontab_lines << new_crontab_lines[i] + end + end + crontab_lines_str = crontab_lines.compact.join("\n") + end +end \ No newline at end of file diff --git a/app/models/site_server_file_backup.rb b/app/models/site_server_file_backup.rb new file mode 100644 index 0000000..14b1509 --- /dev/null +++ b/app/models/site_server_file_backup.rb @@ -0,0 +1,176 @@ +class SiteServerFileBackup + include Mongoid::Document + include Mongoid::Timestamps + PeriodsTypes=['daily','weekly','monthly'] + DefaultConf='/etc/rsnapshot.conf' + belongs_to :site_server + field :disable, type: Boolean, default: false + field :backup_time, type: String, default: '04:15' + field :path , type: String ,default: '/home/backup/orbit' + field :period, type: Integer, default: 0 # 0 => daily , 1 => weekly , 2 => monthly + field :retain_count, type: Integer, default: 7 + field :rsnapshot_conf_path, type: String, default: DefaultConf + field :backup_dir, type: String, default: '/home/{{user_name}}' + field :backup_prefix, type: String, default: 'localhost/' + field :need_rewrite, type: Boolean, default: true + + before_create do + other_rsnapshot_conf_path = self.class.pluck(:rsnapshot_conf_path) + if other_rsnapshot_conf_path.include?(self.rsnapshot_conf_path) + max_postfix = other_rsnapshot_conf_path.select{|s| s.start_with?('/etc/')}.map{|s| tmp = File.basename(s).split('.conf')[0].match(/\d+/); tmp ? tmp[0].to_i : 0}.max + self.rsnapshot_conf_path = "#{self.rsnapshot_conf_path.split('.conf')[0]}#{max_postfix + 1}.conf" + end + end + before_save do + if self.changed? + self.need_rewrite = true + end + end + def to_crontab_script + # minute , hour, day(of month, 1~31), month(1~12), day(of week,0~6) + crontab_time = ['*', '*', '*', '*', '*'] + time = self.backup_time.split(':').map{|s| s.sub(/0(\d)/){$1}} rescue [] + crontab_time[0] = time[1].to_i.to_s rescue '0' + crontab_time[1] = time[0].to_i.to_s rescue '0' + if self.period == 1 #weekly + crontab_time[4] = '0' #every Sunday + elsif self.period == 2 #monthly + crontab_time[2] = '1' #day 1 of every month + end + extra_arg = '' + if self.rsnapshot_conf_path != DefaultConf + extra_arg = " -c #{rsnapshot_conf_path} " + end + "#{crontab_time.join(' ')} /usr/bin/rsnapshot #{extra_arg}#{period_text}" + end + def period_text + PeriodsTypes[self.period] + end + def retain_text + "retain\t#{self.period_text}\t#{self.retain_count}" + end + def override_text(contents, org_regex, new_text) + exist_flag = false + contents = contents.gsub(org_regex).with_index do |s, i| + if i == 0 + exist_flag = true + "#{new_text}#{(s[-1] == "\n" ? "\n" : '')}" + else + "##{s}" + end + end + unless exist_flag + contents += "\n#{new_text}" + end + contents + end + def gsub_rsnapshot_conf(contents) + contents = self.override_text(contents, /^retain\s+[^\n]*(\n|$)/m, self.retain_text) + contents = self.override_text(contents, /^snapshot_root\s+[^\n]*(\n|$)/m, self.snapshot_root) + contents = self.override_text(contents, /^backup\s+[^\n]*(\n|$)/m, self.backup_path) + end + def backup_path + home_dir = self.backup_dir + if home_dir.include?('{{user_name}}') + user_name = self.site_server.account + home_dir = home_dir.sub('{{user_name}}', user_name) + self.backup_dir = home_dir + self.save + end + backup_text = "backup\t#{home_dir}/\t#{self.backup_prefix}" + extra_exclude_path = [] + if self.class.class_variable_defined?(:@@db_backup_paths) + extra_exclude_path = @@db_backup_paths.select{|p| p.start_with?(home_dir)} + end + if self.path.start_with?(home_dir) + extra_exclude_path << self.path if extra_exclude_path.exclude?(self.path) + end + if extra_exclude_path.count != 0 + tmp = [] + extra_exclude_path.each do |s| + if tmp.count == 0 + tmp << s + else + if tmp.exclude?(s) + s2 = File.dirname(s) + idx = tmp.index{|ss| File.dirname(ss) == s2} + if idx + tmp[idx] = s2 + else + tmp << s + end + end + end + end + backup_text += "\t#{tmp.map{|p| "exclude=#{p}"}.join(',')}" + end + backup_text + end + def snapshot_root + "snapshot_root\t#{self.path}" + end + def self.read_from_crontab_line(crontab_line) + if crontab_line.include?('rsnapshot') + tmp = crontab_line.split(/\s+[^\s]*rsnapshot\s+/) # ex: ["15 4 * * *", "daily"], ["15 4 * * *", "-c /etc/rsnapshot.conf daily"] + if tmp.count == 2 + crontab_time = tmp[0].split(/\s+/) + period_text = tmp[1] + rsnapshot_conf_path = DefaultConf + if period_text.start_with?('-c') + tmp2 = period_text.split(/\s+/) + if tmp2.count == 3 + rsnapshot_conf_path = tmp2[1] + period_text = tmp2[2] + else + period_text = tmp2.last + end + end + if PeriodsTypes.include?(period_text) + period = PeriodsTypes.index(period_text) + else + period = 0 + end + time = "#{crontab_time[1].rjust(2, "0")}:#{crontab_time[0].rjust(2, "0")}" + return [time, rsnapshot_conf_path, period] + else + return nil + end + else + return nil + end + end + def self.clean_crontab_setting(crontab_lines_str) + crontab_lines = crontab_lines_str.split("\n") + crontab_lines_str = crontab_lines.map do |crontab_line| + if crontab_line.include?('rsnapshot') + nil + else + crontab_line + end + end.compact.join("\n") + end + def self.init_class_variables(site_server) + @@db_backup_paths = SiteServerDbBackup.where(:site_server_id=> site_server.id, :disable=>false).pluck(:path) + end + def self.write_crontab_setting(site_server, crontab_lines_str) + new_crontab_lines = self.where(:site_server_id=> site_server.id, :disable=>false).flat_map{|file_backup| file_backup.to_crontab_script.split("\n")} + crontab_lines = crontab_lines_str.split("\n") + exists_indices = new_crontab_lines.map do |new_crontab_line| + match_reg = ::Regexp.new(new_crontab_line.gsub(/\s+/, '\s+')) + crontab_lines.index{|l| l.match(match_reg)} + end + crontab_lines = crontab_lines.map.with_index do |crontab_line, i| + if crontab_line.include?('rsnapshot') && exists_indices.exclude?(i) + nil + else + crontab_line + end + end + exists_indices.each_with_index do |idx, i| + if idx.nil? + crontab_lines << new_crontab_lines[i] + end + end + crontab_lines_str = crontab_lines.compact.join("\n") + end +end \ No newline at end of file diff --git a/app/views/admin/site_panel/_backup_form.html.erb b/app/views/admin/site_panel/_backup_form.html.erb new file mode 100644 index 0000000..a42cedd --- /dev/null +++ b/app/views/admin/site_panel/_backup_form.html.erb @@ -0,0 +1,77 @@ + +<% + relation_field = "#{f.object_name}_#{type.pluralize}" + records = f.object.send(relation_field) +%> +
+ +
+ <% records.each do |record| %> + <%= f.fields_for relation_field, record do |f| %> + <%= render :partial => "backup_form_block", :locals=>{:f=>f} %> + <% end %> + <% end %> +
+

+ <%= hidden_field_tag "#{relation_field}_count", records.count %> + <%= t("client_management.add_backup_setting") %> +

+
+ + \ No newline at end of file diff --git a/app/views/admin/site_panel/_backup_form_block.html.erb b/app/views/admin/site_panel/_backup_form_block.html.erb new file mode 100644 index 0000000..9009564 --- /dev/null +++ b/app/views/admin/site_panel/_backup_form_block.html.erb @@ -0,0 +1,45 @@ +<% object_name_underscore = f.object_name.gsub(/[\[\]]/,'_') %> +
+ X +
+
+ <%= f.label :disable ,t("client_management.disable"), :class => "control-label muted" %> +
+ <%= f.check_box :disable, :title=> t("client_management.disable") %> +
+
+
+ <% default_backup_time = f.object.fields["backup_time"].options[:default] %> + <%= f.label :backup_time ,t("client_management.backup_time"), :class => "control-label muted" %> +
+ <%= f.text_field :backup_time, :placeholder => default_backup_time, :class=>"backup_timers" %> +
Eg: <%=default_backup_time%>
+
+
+
+ <% default_path = f.object.fields["path"].options[:default] %> + <%= f.label :path ,"Path", :class => "control-label muted" %> +
+ <%= f.text_field :path, :placeholder => "Path(ex: #{default_path})" %> +
Eg: <%=default_path%>
+
+
+
+ <%= f.label :period , t("client_management.period"), :class => "control-label muted" %> +
+ <%= f.select :period, f.object.class::PeriodsTypes.map.with_index{|c, i| [t("client_management.period_type.#{c}"),i]} %> +
+
+
+ <% default_retain_count = f.object.fields["retain_count"].options[:default] %> + <%= f.label :retain_count , t("client_management.retain_count"), :class => "control-label muted" %> +
+ <%= f.number_field :retain_count, :placeholder => "ex: #{default_retain_count}" %> +
Eg: <%=default_retain_count%>
+
+
+ <% unless f.object.new_record? %> + <%= f.hidden_field :id %> + <%= f.hidden_field :_destroy, :value => nil, :class => 'should_destroy' %> + <% end %> +
\ No newline at end of file diff --git a/app/views/admin/site_panel/_server_manager_index.html.erb b/app/views/admin/site_panel/_server_manager_index.html.erb index e3e880e..8b5b8ce 100644 --- a/app/views/admin/site_panel/_server_manager_index.html.erb +++ b/app/views/admin/site_panel/_server_manager_index.html.erb @@ -68,6 +68,7 @@ "><%=t(:edit)%> ';}"><%=t(:remove)%> "><%= t('client_management.see_sites') %> + <%= t('client_management.backup_setting') %> Detect sites <%=t('client_management.super_manager_management')%> diff --git a/app/views/admin/site_panel/backup_setting.html.erb b/app/views/admin/site_panel/backup_setting.html.erb new file mode 100644 index 0000000..9f719fa --- /dev/null +++ b/app/views/admin/site_panel/backup_setting.html.erb @@ -0,0 +1,145 @@ +<%= form_for @site_server, :url => {:action=>"update_backup_setting"}, :html => {:class => 'form-horizontal main-forms'} do |f| %> +<% content_for :page_specific_css do %> + <%= stylesheet_link_tag "lib/main-forms" %> +<% end %> +<% content_for :page_specific_javascript do %> +<% end %> + +<%#= f.error_messages %> +
+

<%= "#{f.object.server_name} (IP: #{f.object.ip}) " %>

+
+ <% if f.object.need_rewrite_backup_setting %> + <%= t('client_management.apply_change') %> +
+ <% end %> +
+ +
+ + +
<%= t(:type) %>
+ + + +
+ <%= render :partial => "backup_form", :locals=>{:f=>f,:type=>"file_backup",:@active_class=>"in active"} %> + <%= render :partial => "backup_form", :locals=>{:f=>f,:type=>"db_backup",:@active_class=>""} %> +
+
+ <%= f.submit t('submit'), class: 'btn btn-primary' %> +
+
+
+ + +<% end %> \ No newline at end of file diff --git a/config/locales/en.yml b/config/locales/en.yml index f1d1ea1..1b9d8e1 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -15,6 +15,13 @@ en: confirm_new_password: "Confirm new password" super_manager_management: "Super Manager Management" create_super_manager: Create Super Manager For Site + backup_setting: "Backup Setting" + add_backup_setting: "Add Backup Setting" + file_backup: "File Backup" + db_backup: "DB Backup" + backup_time: "Backup Time" + period: "Period" + retain_count: "Retain Count" enable_api: Enable API api_key: API Key setting: Setting diff --git a/config/locales/zh_tw.yml b/config/locales/zh_tw.yml index dc886b4..6f15aae 100644 --- a/config/locales/zh_tw.yml +++ b/config/locales/zh_tw.yml @@ -15,6 +15,13 @@ zh_tw: confirm_new_password: "確認新密碼" super_manager_management: "管理網站超級管理者" create_super_manager: 創建網站超級管理者 + backup_setting: "備份設定" + add_backup_setting: "新增備份設定" + file_backup: "檔案備份" + db_backup: "資料庫備份" + backup_time: "備份時間" + period: "週期" + retain_count: "保留" enable_api: 開啟API api_key: API Key setting: 設定 diff --git a/config/routes.rb b/config/routes.rb index feb563b..e8a23cc 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -68,6 +68,9 @@ Rails.application.routes.draw do patch 'create_cert' end member do + get 'backup_setting' + post 'update_backup_setting' + patch 'update_backup_setting' get "super_manager_management" patch "update_super_manager_management" get "super_manager_management_for_site" diff --git a/lib/tasks/change_backup_setting.rake b/lib/tasks/change_backup_setting.rake new file mode 100644 index 0000000..38e9370 --- /dev/null +++ b/lib/tasks/change_backup_setting.rake @@ -0,0 +1,96 @@ +require 'net/ssh' +require 'pathname' +require 'json' +require 'base64' +namespace :exec_commands do + desc "Change Server Backup Setting Script" + task :change_backup_setting,[:site_server_id] => :environment do |task,args| + if args.site_server_id.present? + site_server = SiteServer.find(args.site_server_id) + else + raise StandardError.new('Please Specify Server ID!') + end + thread_key = "change_backup_setting_#{site_server.id}" + @thread = Multithread.where(:key=>thread_key).first + begin + if @thread.nil? + @thread = Multithread.create(:key=>thread_key,:status=>{"infos"=>[],"status"=>"execing"}) + else + @thread.update(:status=>{"infos"=>[],"status"=>"execing"}) + end + ip = site_server.ip + user = site_server.account + password = site_server.password + @password = password + begin + Net::SSH.start(ip , user , password: password) do |ssh| + end + rescue Net::SSH::HostKeyMismatch + system("ssh-keygen -f \"$HOME/.ssh/known_hosts\" -R #{ip}") + rescue Errno::ENOTTY + system("ssh-add \"$HOME/.ssh/id_rsa\"") + end + Net::SSH.start(ip , user , password: password) do |ssh| + @no_stdout = true + crontab_lines_str = exec_ssh_command_by_sudo_and_see_output(ssh,"sudo -p 'sudo password:' crontab -l", false, true) + if site_server.site_server_file_backups.count != 0 + SiteServerFileBackup.init_class_variables(site_server) + rsnapshot_conf_default = SiteServerFileBackup::DefaultConf + rsnapshot_conf_exist = check_file_exist_for_ssh(ssh, rsnapshot_conf_default) + update_thread_infos_for_exec("Checking file backups config...") + if rsnapshot_conf_exist + rsnapshot_conf_contents = read_file_for_ssh(ssh, rsnapshot_conf_default) + else + rsnapshot_sample_conf = File.expand_path("../../../rsnapshot_sample.conf", __FILE__) + rsnapshot_conf_contents = File.read(rsnapshot_sample_conf) + write_file_for_ssh(ssh, rsnapshot_conf_default, rsnapshot_conf_contents) + end + site_server.site_server_file_backups.each do |file_backup| + rsnapshot_conf_path = file_backup.rsnapshot_conf_path + if check_file_exist_for_ssh(ssh, rsnapshot_conf_path) + tmp = read_file_for_ssh(ssh, rsnapshot_conf_path) + else + tmp = rsnapshot_conf_contents.clone + end + tmp = file_backup.gsub_rsnapshot_conf(tmp) + write_file_for_ssh(ssh, rsnapshot_conf_path, tmp) + mkdir_for_ssh(ssh, file_backup.path) + end + update_thread_infos_for_exec("Finish writing file backups config!") + end + site_server.site_server_db_backups do |db_backup| + mkdir_for_ssh(ssh, db_backup.path) + end + crontab_lines_str = SiteServerFileBackup.write_crontab_setting(site_server, crontab_lines_str) + crontab_lines_str = SiteServerDbBackup.write_crontab_setting(site_server, crontab_lines_str) + write_crontab_for_ssh(ssh, crontab_lines_str) + update_thread_infos_for_exec("Finish setting backups!") + end + site_server.site_server_file_backups.update_all(:need_rewrite=>false) + site_server.site_server_db_backups.update_all(:need_rewrite=>false) + site_server.update(:need_rewrite_backup_setting=>false) + @thread.update(:status=>@thread.status.merge({"status"=>"finish"})) + rescue => e + @thread.update(:status=>{"infos"=>@thread.status["infos"].push(e.message),"status"=>"error"}) + @thread.update(:status=>{"infos"=>@thread.status["infos"].push(e.backtrace.join("\n")),"status"=>"error"}) + end + end + def mkdir_for_ssh(ssh, dir) + exec_ssh_command_by_sudo_and_see_output(ssh,"sudo -p 'sudo password:' sh -c \"mkdir -p #{dir}\"", false) + end + def write_crontab_for_ssh(ssh, crontab_lines_str) + exec_ssh_command_by_sudo_and_see_output(ssh,"x='#{crontab_lines_str.gsub("\n", '\n').gsub("'","'\"'\"'")}'; sudo -p 'sudo password:' sh -c \"echo '$x'| crontab -\"", false) + end + def read_file_for_ssh(ssh, file_name) + exec_ssh_command_by_sudo_and_see_output(ssh,"sudo -p 'sudo password:' sh -c \"cat #{file_name}\"", false, true) + end + def write_file_for_ssh(ssh, file_name, contents) + exec_ssh_command_by_sudo_and_see_output(ssh,"x='#{contents.gsub("\n", '\n').gsub("'","'\"'\"'")}';sudo -p 'sudo password:' sh -c \"echo '$x' > #{file_name}\"", false) + end + def copy_file_for_ssh(ssh, src, dst) + exec_ssh_command_by_sudo_and_see_output(ssh,"sudo -p 'sudo password:' bash -l -c 'cp #{src} #{dst}'", 1) + end + def check_file_exist_for_ssh(ssh, file_name) + exec_ssh_command_by_sudo_and_see_output(ssh,"sudo -p 'sudo password:' bash -l -c 'if [ -e #{file_name} ]; then echo 1; else echo 0; fi'", false, true).include?('1') + end +end \ No newline at end of file diff --git a/lib/tasks/detect_sites.rake b/lib/tasks/detect_sites.rake index c736e5b..0dc25fb 100644 --- a/lib/tasks/detect_sites.rake +++ b/lib/tasks/detect_sites.rake @@ -164,6 +164,79 @@ namespace :create_site do end end end + update_thread_infos("Detecting Backup Setting on #{site_server.server_name}...") + crontab_lines = exec_ssh_command_by_sudo_and_see_output(ssh,"sudo -p 'sudo password:' crontab -l;crontab -l",false,false) + file_backups = crontab_lines.select{|s| s.match(/^\s*[^\s#]/) && s.include?('rsnapshot')}.map{|s| SiteServerFileBackup.read_from_crontab_line(s)}.compact + rsnapshot_confs_path = ([SiteServerFileBackup::DefaultConf] + file_backups.map{|time, rsnapshot_conf_path, period| rsnapshot_conf_path}).uniq + retain_settings = {} + backup_dir_info = {} + snapshot_root_info = {} + rsnapshot_confs_path.each do |rsnapshot_conf_path| + backup_retain_lines = exec_ssh_command_by_sudo_and_see_output(ssh,"sudo -p 'sudo password:' grep -E '^\\s*retain\\s+' #{rsnapshot_conf_path}",false,false).map{|s| s.strip}.select{|s| s.present?} + backup_retain_lines = backup_retain_lines + next if backup_retain_lines.blank? + part_retain_settings = backup_retain_lines.map{|s| (s.strip.split(/\s+/)[1..2] rescue nil)}.compact.to_h + retain_settings[rsnapshot_conf_path] = part_retain_settings + + backup_dir_subinfo = exec_ssh_command_by_sudo_and_see_output(ssh,"sudo -p 'sudo password:' grep -E '^\s*backup' #{rsnapshot_conf_path}",false,false).map{|s| s.strip}.select{|s| s.present?} # ex: ["backup\t/home/rulingcom\tlocalhost/"] + backup_dir_info[rsnapshot_conf_path] = backup_dir_subinfo.map{|s| s.strip.split(/\s+/)[1..2]} + + snapshot_root_subinfo = exec_ssh_command_by_sudo_and_see_output(ssh,"sudo -p 'sudo password:' grep -E '^\s*snapshot_root' #{rsnapshot_conf_path}",false,false).map{|s| s.strip}.select{|s| s.present?}.last # ex: "snapshot_root\t/home/backup/orbit" + snapshot_root_info[rsnapshot_conf_path] = snapshot_root_subinfo.strip.split(/\s+/)[1] rescue nil #only single snapshot_root in a conf + end + file_backups_group = file_backups.group_by{|time, rsnapshot_conf_path, period| rsnapshot_conf_path} + file_backup_ids = [] + retain_settings.each do |rsnapshot_conf_path, part_retain_settings| + path = snapshot_root_info[rsnapshot_conf_path] + if path.nil? + path = '/home/backup/orbit' + end + backup_dir_subinfo = backup_dir_info[rsnapshot_conf_path] + tmp = file_backups_group[rsnapshot_conf_path] + unused_period = part_retain_settings.map{|period_text, retain_count| SiteServerFileBackup::PeriodsTypes.index(period_text).to_i} - tmp.map{|time, rsnapshot_conf_path, period| period} + unused_retain_settings = unused_period.map{|period| [period, part_retain_settings[SiteServerFileBackup::PeriodsTypes[period]]]} + backup_dir_subinfo.each do |backup_dir, backup_prefix| + unused_retain_settings.each do |period, retain_count| + file_backup_data = {:site_server=>@site_server, :disable=>true, :period=>period, :path=>path, :retain_count=>retain_count, :rsnapshot_conf_path=>rsnapshot_conf_path, :backup_dir=>backup_dir, :backup_prefix=>backup_prefix} + file_backup = SiteServerFileBackup.where(file_backup_data.except(:disable)).first + if file_backup.nil? + file_backup = SiteServerFileBackup.create(file_backup_data) + else + file_backup.update_attributes(file_backup_data) + end + file_backup_ids << file_backup.id + end + tmp.each do |time, rsnapshot_conf_path, period| + retain_count = part_retain_settings[SiteServerFileBackup::PeriodsTypes[period]] + if retain_count + file_backup_data = {:site_server=>@site_server, :disable=>false, :backup_time=>time, :path=>path, :period=>period, :retain_count=>retain_count, :rsnapshot_conf_path=>rsnapshot_conf_path, :backup_dir=>backup_dir, :backup_prefix=>backup_prefix} + file_backup = SiteServerFileBackup.where(file_backup_data.except(:disable)).first + if file_backup.nil? + file_backup = SiteServerFileBackup.create(file_backup_data) + else + file_backup.update_attributes(file_backup_data) + end + file_backup_ids << file_backup.id + end + end + end + end + SiteServerFileBackup.where(:id.in=>file_backup_ids).update_all(:need_rewrite=>false) + db_backup_ids = [] + db_backup_infos = crontab_lines.select{|s| s.match(/^\s*[^\s#]/) && s.include?('mongodump')}.map{|s| SiteServerDbBackup.read_from_crontab_line(s, crontab_lines)}.compact + db_backup_infos.each do |backup_time, path, period, retain_count| + next if backup_time.nil? + db_backup_data = {:site_server=>@site_server, :disable=>false, :period=>period, :path=>path, :backup_time=>backup_time, :retain_count=>retain_count} + db_backup = SiteServerDbBackup.where(db_backup_data.except(:disable)).first + if db_backup.nil? + db_backup = SiteServerDbBackup.create(db_backup_data) + else + db_backup.update_attributes(db_backup_data) + end + db_backup_ids << db_backup.id + end + SiteServerDbBackup.where(:id.in=>db_backup_ids).update_all(:need_rewrite=>false) + update_thread_infos("Finish detecting Backup Setting on #{site_server.server_name}!") end end end diff --git a/rsnapshot_sample.conf b/rsnapshot_sample.conf new file mode 100644 index 0000000..657eb23 --- /dev/null +++ b/rsnapshot_sample.conf @@ -0,0 +1,238 @@ +################################################# +# rsnapshot.conf - rsnapshot configuration file # +################################################# +# # +# PLEASE BE AWARE OF THE FOLLOWING RULE: # +# # +# This file requires tabs between elements # +# # +################################################# + +####################### +# CONFIG FILE VERSION # +####################### + +config_version 1.2 + +########################### +# SNAPSHOT ROOT DIRECTORY # +########################### + +# All snapshots will be stored under this root directory. +# +#snapshot_root /var/cache/rsnapshot/ +snapshot_root /home/backup/orbit + +# If no_create_root is enabled, rsnapshot will not automatically create the +# snapshot_root directory. This is particularly useful if you are backing +# up to removable media, such as a FireWire or USB drive. +# +#no_create_root 1 + +################################# +# EXTERNAL PROGRAM DEPENDENCIES # +################################# + +# LINUX USERS: Be sure to uncomment "cmd_cp". This gives you extra features. +# EVERYONE ELSE: Leave "cmd_cp" commented out for compatibility. +# +# See the README file or the man page for more details. +# +cmd_cp /bin/cp + +# uncomment this to use the rm program instead of the built-in perl routine. +# +cmd_rm /bin/rm +# rsync must be enabled for anything to work. This is the only command that +# must be enabled. +# +cmd_rsync /usr/bin/rsync + +# Uncomment this to enable remote ssh backups over rsync. +# +#cmd_ssh /usr/bin/ssh + +# Comment this out to disable syslog support. +# +cmd_logger /usr/bin/logger + +# Uncomment this to specify the path to "du" for disk usage checks. +# If you have an older version of "du", you may also want to check the +# "du_args" parameter below. +# +#cmd_du /usr/bin/du + +# Uncomment this to specify the path to rsnapshot-diff. +# +#cmd_rsnapshot_diff /usr/bin/rsnapshot-diff + +# Specify the path to a script (and any optional arguments) to run right +# before rsnapshot syncs files +# +#cmd_preexec /path/to/preexec/script + +# Specify the path to a script (and any optional arguments) to run right +# after rsnapshot syncs files +# +#cmd_postexec /path/to/postexec/script + +# Paths to lvcreate, lvremove, mount and umount commands, for use with +# Linux LVMs. +# +#linux_lvm_cmd_lvcreate /sbin/lvcreate +#linux_lvm_cmd_lvremove /sbin/lvremove +#linux_lvm_cmd_mount /bin/mount +#linux_lvm_cmd_umount /bin/umount + +######################################### +# BACKUP LEVELS / INTERVALS # +# Must be unique and in ascending order # +# e.g. alpha, beta, gamma, etc. # +######################################### + +#retain alpha 6 +#retain beta 7 +#retain gamma 4 +#retain delta 3 +retain daily 7 + +############################################ +# GLOBAL OPTIONS # +# All are optional, with sensible defaults # +############################################ +# Verbose level, 1 through 5. +# 1 Quiet Print fatal errors only +# 2 Default Print errors and warnings only +# 3 Verbose Show equivalent shell commands being executed +# 4 Extra Verbose Show extra verbose information +# 5 Debug mode Everything +# +verbose 2 +# Same as "verbose" above, but controls the amount of data sent to the +# logfile, if one is being used. The default is 3. +# If you want the rsync output, you have to set it to 4 +# +loglevel 3 +# If you enable this, data will be written to the file you specify. The +# amount of data written is controlled by the "loglevel" parameter. +# +#logfile /var/log/rsnapshot.log + +# If enabled, rsnapshot will write a lockfile to prevent two instances +# from running simultaneously (and messing up the snapshot_root). +# If you enable this, make sure the lockfile directory is not world +# writable. Otherwise anyone can prevent the program from running. +# +lockfile /var/run/rsnapshot.pid + +# By default, rsnapshot check lockfile, check if PID is running +# and if not, consider lockfile as stale, then start +# Enabling this stop rsnapshot if PID in lockfile is not running +# +#stop_on_stale_lockfile 0 +# Default rsync args. All rsync commands have at least these options set. +# +#rsync_short_args -a +#rsync_long_args --delete --numeric-ids --relative --delete-excluded +# ssh has no args passed by default, but you can specify some here. +# +#ssh_args -p 22 + +# Default arguments for the "du" program (for disk space reporting). +# The GNU version of "du" is preferred. See the man page for more details. +# If your version of "du" doesnt support the -h flag, try -k flag instead.n#n#du_args -cshnn# If this is enabled, rsync wont span filesystem partitions within a +# backup point. This essentially passes the -x option to rsync. +# The default is 0 (off). +# +#one_fs 0 + +# The include and exclude parameters, if enabled, simply get passed directly +# to rsync. If you have multiple include/exclude patterns, put each one on a +# separate line. Please look up the --include and --exclude options in the +# rsync man page for more details on how to specify file name patterns. +# +#include ??? +#include ??? +#exclude ??? +#exclude ??? + +# The include_file and exclude_file parameters, if enabled, simply get +# passed directly to rsync. Please look up the --include-from and +# --exclude-from options in the rsync man page for more details. +# +#include_file /path/to/include/file +#exclude_file /path/to/exclude/file +# If your version of rsync supports --link-dest, consider enabling this. +# This is the best way to support special files (FIFOs, etc) cross-platform. +# The default is 0 (off). +# +#link_dest 0 + +# When sync_first is enabled, it changes the default behaviour of rsnapshot. +# Normally, when rsnapshot is called with its lowest interval +# (i.e.: "rsnapshot alpha"), it will sync files AND rotate the lowest +# intervals. With sync_first enabled, "rsnapshot sync" handles the file sync, +# and all interval calls simply rotate files. See the man page for more +# details. The default is 0 (off). +# +#sync_first 0 +# If enabled, rsnapshot will move the oldest directory for each interval +# to [interval_name].delete, then it will remove the lockfile and delete +# that directory just before it exits. The default is 0 (off). +# +#use_lazy_deletes 0 + +# Number of rsync re-tries. If you experience any network problems or +# network card issues that tend to cause ssh to fail with errors like +# "Corrupted MAC on input", for example, set this to a non-zero value +# to have the rsync operation re-tried. +# +#rsync_numtries 0 +# LVM parameters. Used to backup with creating lvm snapshot before backup +# and removing it after. This should ensure consistency of data in some special +# cases +# +# LVM snapshot(s) size (lvcreate --size option). +# +#linux_lvm_snapshotsize 100M +# Name to be used when creating the LVM logical volume snapshot(s). +# +#linux_lvm_snapshotname rsnapshot + +# Path to the LVM Volume Groups. +# +#linux_lvm_vgpath /dev +# Mount point to use to temporarily mount the snapshot(s). +# +#linux_lvm_mountpath /path/to/mount/lvm/snapshot/during/backup + +############################### +### BACKUP POINTS / SCRIPTS ### +############################### + +# LOCALHOST +backup /home/rulingcom/ localhost/ +#backup /home/ localhost/ +#backup /etc/ localhost/ +#backup /usr/local/ localhost/ +#backup /var/log/rsnapshot localhost/ +#backup /etc/passwd localhost/ +#backup /home/foo/My Documents/ localhost/ +#backup /foo/bar/ localhost/ one_fs=1,rsync_short_args=-urltvpog +#backup_script /usr/local/bin/backup_pgsql.sh localhost/postgres/ +# You must set linux_lvm_* parameters below before using lvm snapshots +#backup lvm://vg0/xen-home/ lvm-vg0/xen-home/ +# EXAMPLE.COM +#backup_exec /bin/date "+ backup of example.com started at %c" +#backup root@example.com:/home/ example.com/ +rsync_long_args=--bwlimit=16,exclude=core +#backup root@example.com:/etc/ example.com/ exclude=mtab,exclude=core +#backup_exec ssh root@example.com "mysqldump -A > /var/db/dump/mysql.sql" +#backup root@example.com:/var/db/dump/ example.com/ +#backup_exec /bin/date "+ backup of example.com ended at %c" + +# CVS.SOURCEFORGE.NET +#backup_script /usr/local/bin/backup_rsnapshot_cvsroot.sh rsnapshot.cvs.sourceforge.net/ + +# RSYNC.SAMBA.ORG +#backup rsync://rsync.samba.org/rsyncftp/ rsync.samba.org/rsyncftp/ +