class SiteFeed include Mongoid::Document include Mongoid::Timestamps ModuleAppSyncFields = ["feeds_model", "feeds_uid_field", "feeds_update_callback", "feeds_time_field", "feeds_finish_callback", "feeds_update_statuses_callback"] field :remote_site_url field :merge_with_category field :channel_name field :channel_title, :localize => true field :channel_key field :feed_name, localize: true field :disabled, type: Boolean, default: false field :feed_url field :feed_uid field :feeds_model field :feeds_uid_field field :feeds_update_callback field :feeds_time_field, type: String, default: "postdate" field :feeds_finish_callback field :feeds_update_statuses_callback field :enable_notify, type: Boolean, default: false field :is_rss, type: Boolean, default: false require 'feed_model/cache' require 'fileutils' include FeedModel::Cache Category.send(:include,FeedModel::Cache) after_create do if !self.is_rss self.add_notify end crontab_list = `crontab -l`.split("\n") rescue [] site_root = Rails.root.to_s if crontab_list.select{|s| s.include?(site_root) && s.include?("feeds_module:make_cache")} == [] `(crontab -l ; echo "*/5 * * * * /bin/bash -l -c 'cd #{site_root} && bundle exec rake feeds_module:make_cache > /dev/null'") | crontab` end end before_destroy do if !self.is_rss self.remove_notify end tmp = SiteFeedAnnc.where(feed_id: self.id) main_directory = File.join("#{Rails.root}","public","site_feeds") feed_directory = File.join(main_directory.to_s, self.id.to_s) FileUtils.rm_rf(feed_directory.to_s) if tmp.count!=0 tmp.destroy end end after_save do if @skip_callback unless @skip_fix_data update_url = self.remote_site_url_changed? if self.channel_title_changed? self.update_channel_title(update_url) elsif update_url self.update_remote_site_url end end end @site_feed_annc = nil end scope :enabled, ->{where(:disabled => false)} def site_feed_annc @site_feed_annc ||= SiteFeedAnnc.where(:feed_id=>self.id).first end def update_channel_title(update_url=false) #update_url=true will also fix remote_site_url in data site_feed_annc.update_channel_title(update_url) if site_feed_annc end def update_remote_site_url site_feed_annc.update_remote_site_url if site_feed_annc end def sync_data_to_annc(site_feed_annc=nil) category_title = self.category[:title] rescue {} tmp_channel_title = self.channel_title_for_cache clone_fields =["channel_key", "merge_with_category", "remote_site_url", "feeds_model", "feeds_uid_field", "feeds_update_callback", "feeds_time_field", "feeds_update_statuses_callback"] (site_feed_annc ? [site_feed_annc] : SiteFeedAnnc.where(:feed_id=>self.id)).each do |tmp| clone_fields.each do |f| tmp.send("#{f}=", self.send(f)) end tmp[:feed_name] = self[:feed_name] tmp.category_title = category_title tmp.channel_title = tmp_channel_title tmp.save end end def get_annc(force_refresh=false) def to_I18n(title, have_key=false, key="title_translations") value = I18n.available_locales.map{|v| [v,title]}.to_h if have_key { key => value } else value end end main_directory = File.join("#{Rails.root}","public","site_feeds") feed_directory = File.join(main_directory.to_s, self.id.to_s) if !force_refresh && File.exists?(feed_directory) anns = JSON.parse(File.read(File.join(feed_directory.to_s, self.feed_uid + ".json")))[self.channel_key.pluralize] rescue [] else uri = URI(self.feed_url) res = get_response_body(uri) rescue '' if self.is_rss rss_data = Hash.from_xml(res) category_title = rss_data['rss']['channel']['title'] category_title_trans = to_I18n(category_title, true) res_to_json = rss_data['rss']['channel']["item"].collect do |item| tag_titles = Array(item['類別']).collect{|title| to_I18n(title, true, "name_translations")} id = item['link'].split("&s=")[-1] title = Array(item['title'])[0] text = Array(item['description'])[0] files = [] if text.include?('附件') files = Nokogiri::HTML(get_response_body(URI(item['link']))).search('#RelData1_liFile ol a').collect do |v| file_title = v.attr('title') { "description_translations" => to_I18n(file_title), "title_translations" => to_I18n(file_title.sub(/^\[(?:(?!\]).)*\]/,'').rpartition(".")[0]), "url" => v.attr('href') } end end { "show_url" => item['link'], "org_is_top" => 0, "id" => id, "title_translations" => to_I18n(title), "subtitle_translations" => {}, "text_translations" => to_I18n(text), "postdate" => DateTime.parse(item['pubDate']).to_s, "image_description_translations" => {}, "image" => {}, "display_img" => false, "tags" => tag_titles, "category" => category_title_trans, "author" => item['author'], "params" => "#{title}-#{id}", "bulletin_links" => [], "bulletin_files" => files, "bulletin_carousel_images" => [], "external_link" => nil } end res = JSON.dump({"announcements" => res_to_json}) end FileUtils.mkdir_p(feed_directory) if !File.exists?(feed_directory) File.open(File.join(feed_directory.to_s,self.feed_uid + ".json"),"w") do |file| res.force_encoding("utf-8") file.write(res) end anns = JSON.parse(res)[self.channel_key.pluralize] rescue [] end anns end def get_response_body(uri) res = Net::HTTP.start(uri.host, uri.port, :use_ssl => uri.scheme == 'https', open_timeout: 60,read_timeout: 60, verify_mode: OpenSSL::SSL::VERIFY_NONE) do |http| req = Net::HTTP::Get.new(uri) req['User-Agent'] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36' http.request(req) end if res.code == "302" || res.code == "301" location = res['Location'] cookie = res['Set-Cookie'] if location[0] == "/" uri = URI.parse("#{uri.scheme}://#{uri.host}#{location}") else uri = URI.parse(location) end begin res = Net::HTTP.start(uri.host, uri.port, :use_ssl => uri.scheme == 'https', open_timeout: 60,read_timeout: 60, verify_mode: OpenSSL::SSL::VERIFY_NONE) do |http| req = Net::HTTP::Get.new(uri) req['Cookie'] = cookie http.request(req) end res_body = res.body rescue => e res_body = '' puts e puts e.backtrace end return res_body else return res.body end end def category Category.find(self.merge_with_category) rescue nil end def channel_title_for_cache #empty? must for hash !self[:channel_title].blank? ? self[:channel_title] : I18n.available_locales.collect{|v| [v,I18n.t("feed.source")]}.to_h end def http_request(http, request) response = http.request(request) if response.code.to_i == 301 || response.code.to_i == 302 location = response["location"] new_uri = URI(location) http = Net::HTTP.new(new_uri.host, new_uri.port) if location.include?('https') http.use_ssl = true http.verify_mode = OpenSSL::SSL::VERIFY_NONE end request.instance_variable_set(:@path, new_uri.path) response = http_request(http, request) end response end def add_notify unless self.enable_notify root_url = Site.first.root_url rescue "" if root_url.present? uri = URI(self.remote_site_url) http_req = Net::HTTP.new(uri.host, uri.port) if self.remote_site_url.include?('https') http_req.use_ssl = true http_req.verify_mode = OpenSSL::SSL::VERIFY_NONE end http_req.open_timeout = 10 request = Net::HTTP::Post.new("/xhr/#{self.channel_key.pluralize}/feed_add_remote/#{self.feed_uid}", 'Content-Type' => 'application/json') request.body = {"url"=>root_url}.to_json response = http_request( http_req , request ) if response.code.to_i == 200 self.update(:enable_notify=>true) end end end end def remove_notify if self.enable_notify root_url = Site.first.root_url rescue "" if root_url.present? uri = URI(self.remote_site_url) http_req = Net::HTTP.new(uri.host, uri.port) if self.remote_site_url.include?('https') http_req.use_ssl = true http_req.verify_mode = OpenSSL::SSL::VERIFY_NONE end http_req.open_timeout = 10 request = Net::HTTP::Post.new("/xhr/#{self.channel_key.pluralize}/feed_remove_remote/#{self.feed_uid}", 'Content-Type' => 'application/json') request.body = {"url"=>root_url}.to_json response = http_request( http_req , request ) if response.code.to_i == 200 self.update(:enable_notify=>false) end end end end end