From 522946bf30c89e528c44e51fe5d1aeac3f5985e3 Mon Sep 17 00:00:00 2001 From: bohung Date: Fri, 31 Jul 2020 14:50:17 +0800 Subject: [PATCH] Fix bugs.(include https,other feed source except announcement) Add module_name select tag at /admin/feeds/announcements page. Add crontab automatically to update feed contents evevery minutes when adding new feed. --- app/controllers/admin/feeds_controller.rb | 72 ++++++----- app/models/site_feed.rb | 39 +++++- app/models/site_feed_annc.rb | 146 +++++++++++++++------- config/locales/en.yml | 1 + config/locales/zh_tw.yml | 1 + 5 files changed, 182 insertions(+), 77 deletions(-) diff --git a/app/controllers/admin/feeds_controller.rb b/app/controllers/admin/feeds_controller.rb index 9d6b0fc..5d6482f 100644 --- a/app/controllers/admin/feeds_controller.rb +++ b/app/controllers/admin/feeds_controller.rb @@ -63,13 +63,18 @@ class Admin::FeedsController < OrbitAdminController end def announcements @filter_fields = {} - @filter_fields['source'] = SiteFeed.all.pluck(:channel_title,:remote_site_url).collect do |a,b| + @filter_fields['feed.source'] = SiteFeed.all.pluck(:channel_title,:remote_site_url).collect do |a,b| tp = (a.blank? || a[I18n.locale].blank?) ? b.gsub(/http:\/\/|https:\/\//,'').gsub(/\./,'-') : a[I18n.locale] {:title => tp,:id => tp} end - @all_feed_annc = SiteFeedAnnc.all.order(created_at: 1).to_a rescue [] - @source = params[:filters][:source] rescue [] - @keywords = params[:keywords] rescue nil + @filter_fields['feed.module_name'] = SiteFeed.all.map{|s| s.channel_key}.uniq.map do |key| + {:title => I18n.t("module_name.#{key}"),:id => key} + end + @module_name = params[:filters][:module_name] rescue [] + @module_name = SiteFeed.all.map{|s| s.channel_key}.uniq if @module_name == [] + @all_feed_annc = SiteFeedAnnc.where(:channel_key.in => @module_name).order(created_at: 1).to_a rescue [] + @source = params[:filters][:source] rescue [] + @keywords = params[:keywords] rescue nil if request.xhr? render :partial => "announcements" end @@ -90,36 +95,45 @@ class Admin::FeedsController < OrbitAdminController end def get_channel_list - url = params['url'].chomp("/") + "/feeds/channel_lists" - uri = URI.parse(url) - res = Net::HTTP.start(uri.host, uri.port,:use_ssl => uri.scheme == 'https',open_timeout: 60,read_timeout: 60) do |http| - req = Net::HTTP::Get.new(uri) - http.request(req).body rescue nil - end - data = JSON.parse(res) rescue {} - render :json => data.to_json + puts params['url'] + url = URI.decode(params['url']).chomp("/") + "/feeds/channel_lists" + puts url + uri = URI.parse(URI.decode(url).encode) + begin + res = Timeout.timeout(10){Net::HTTP.get_response(uri).body} + rescue + res = nil + puts uri.to_s + puts uri.hostname + end + data = JSON.parse(res) rescue {} + print data + render :json => data.to_json end def get_feed_list url = params['url'].chomp("/") + params[:feed_list_url] uri = URI.parse(url) - res = Net::HTTP.start(uri.host, uri.port,:use_ssl => uri.scheme == 'https',open_timeout: 60,read_timeout: 60) do |http| - req = Net::HTTP::Get.new(uri) - http.request(req).body rescue nil - end - data = JSON.parse(res) rescue {} - data_to_send = {} - data_to_send["feeds"] = [] - Array(data["feeds"]).each do |feed| - sf = SiteFeed.find_by(:feed_uid => feed["uid"]) rescue nil - if !sf.nil? - feed["subscribed"] = true - else - feed["subscribed"] = false - end - data_to_send["feeds"] << feed - end - render :json => data_to_send.to_json + begin + res = Timeout.timeout(10){Net::HTTP.get_response(uri).body} + rescue + res = nil + puts uri.to_s + puts uri.hostname + end + data = JSON.parse(res) rescue {} + data_to_send = {} + data_to_send["feeds"] = [] + Array(data["feeds"]).each do |feed| + sf = SiteFeed.find_by(:feed_uid => feed["uid"]) rescue nil + if !sf.nil? + feed["subscribed"] = true + else + feed["subscribed"] = false + end + data_to_send["feeds"] << feed + end + render :json => data_to_send.to_json end def channel_title diff --git a/app/models/site_feed.rb b/app/models/site_feed.rb index b3cdbb9..7b8e425 100644 --- a/app/models/site_feed.rb +++ b/app/models/site_feed.rb @@ -11,10 +11,21 @@ class SiteFeed field :feed_url field :feed_uid require 'feed_model/cache' + require 'fileutils' include FeedModel::Cache Category.send(:include,FeedModel::Cache) + after_create do + crontab_list = `crontab -l`.split("\n") rescue [] + site_root = Rails.root.to_s + if crontab_list.select{|s| s.include?(site_root) && s.include?("feeds_module:make_cache")} == [] + `(crontab -l ; echo "*/5 * * * * /bin/bash -l -c 'cd #{site_root} && bundle exec rake feeds_module:make_cache > /dev/null'") | crontab` + end + end before_destroy do tmp = SiteFeedAnnc.where(feed_id: self.id).first + main_directory = File.join("#{Rails.root}","public","site_feeds") + feed_directory = File.join(main_directory.to_s, self.id.to_s) + FileUtils.rm_rf(feed_directory.to_s) if !tmp.nil? tmp.destroy end @@ -24,22 +35,38 @@ class SiteFeed main_directory = File.join("#{Rails.root}","public","site_feeds") feed_directory = File.join(main_directory.to_s, self.id.to_s) if File.exists?(feed_directory) - anns = JSON.parse(File.read(File.join(feed_directory.to_s, self.feed_uid + ".json")))['announcements'] rescue [] + anns = JSON.parse(File.read(File.join(feed_directory.to_s, self.feed_uid + ".json")))[self.channel_key.pluralize] rescue [] else uri = URI(self.feed_url) - res = Net::HTTP.start(uri.host, uri.port,:use_ssl => uri.scheme == 'https',open_timeout: 20,read_timeout: 20) do |http| - req = Net::HTTP::Get.new(uri) - http.request(req).body rescue '' - end + res = get_response_body(uri) rescue '' FileUtils.mkdir_p(feed_directory) if !File.exists?(feed_directory) File.open(File.join(feed_directory.to_s,self.feed_uid + ".json"),"w") do |file| res.force_encoding("utf-8") file.write(res) end - anns = JSON.parse(res)['announcements'] rescue [] + anns = JSON.parse(res)[self.channel_key.pluralize] rescue [] end anns end + def get_response_body(uri) + res = Net::HTTP.get_response(uri) + if res.code == "302" || res.code == "301" + location = res['Location'] + cookie = res['Set-Cookie'] + headers = { + 'Cookie' => cookie, + } + if location[0] == "/" + uri = URI.parse("#{uri.scheme}://#{uri.host}#{location}") + else + uri = URI.parse(location) + end + res = Net::HTTP.get_response(uri,nil,headers) + return res.body + else + return res.body + end + end def category Category.find(self.merge_with_category) rescue nil end diff --git a/app/models/site_feed_annc.rb b/app/models/site_feed_annc.rb index 2766e90..cebc99e 100644 --- a/app/models/site_feed_annc.rb +++ b/app/models/site_feed_annc.rb @@ -20,53 +20,115 @@ class SiteFeedAnnc Array(self[:all_contents_for_feed]).collect do |v| tmp = v if hidden_annc.exclude?(v['id']) && !tmp["title_translations"][locale].blank? - tmp['statuses'] = [] - if self[:top_list].count == 0 || self[:top_list].exclude?(tmp['id']) - tmp[:is_top] = false - else - tmp[:is_top] = true + if self.channel_key == "announcement" + tmp['statuses'] = [] + if self[:top_list].count == 0 || self[:top_list].exclude?(tmp['id']) + tmp[:is_top] = false + else + tmp[:is_top] = true + tmp['statuses'] << { + "status" => I18n.t(:top), + "status-class" => "status-top" + } + end + if self[:hot_list].count == 0 || self[:top_list].exclude?(tmp['id']) + tmp[:is_hot] = false + else + tmp[:is_hot] = true + tmp['statuses'] << { + "status" => I18n.t(:hot), + "status-class" => "status-hot" + } + end + tmp["category"] = cat + tmp["source-site"] = self.remote_site_url + tmp["source-site-title"] = (self[:channel_title][locale] rescue "") + tmp["params"] = tmp["params"].to_s + "_" + self.feed_id.to_s + "h" + next if !site_source.nil? && site_source != tmp["source-site-title"] tmp['statuses'] << { - "status" => I18n.t(:top), - "status-class" => "status-top" + "status" => "#{tmp["source-site-title"]}", + "status-class" => "status-source" } - end - if self[:hot_list].count == 0 || self[:top_list].exclude?(tmp['id']) - tmp[:is_hot] = false - else - tmp[:is_hot] = true - tmp['statuses'] << { - "status" => I18n.t(:hot), - "status-class" => "status-hot" - } - end - tmp["category"] = cat - tmp["source-site"] = self.remote_site_url - tmp["source-site-title"] = self[:channel_title][locale] - tmp["params"] = tmp["params"].to_s + "_" + self.feed_id.to_s + "h" - next if !site_source.nil? && site_source != fa["source-site-title"] - tmp['statuses'] << { - "status" => "#{tmp["source-site-title"]}", - "status-class" => "status-source" - } - files = tmp["bulletin_files"].collect{|bf| { "file_url" => bf["url"], "file_title" => (fa["title_translations"][locale].blank? ? File.basename(fa["url"]) : fa["title_translations"][locale] rescue '') }} rescue [] - links = tmp["bulletin_links"].map{|link| { "link_url" => link["url"], "link_title" => (link["title_translations"][locale].blank? ? link["url"] : link["title_translations"][locale]) } } rescue [] - tmp["bulletin_links"] = links - tmp["bulletin_files"] = files - tmp["title"] = tmp["title_translations"][locale] - tmp["subtitle"] = tmp["subtitle_translations"][locale] - tmp["source-site-link"] = tmp["source-site"] - tmp["source-site"] = "#{tmp["source-site-title"]}" - if !is_widget - tmp["link_to_show"] = OrbitHelper.url_to_show(tmp["params"]) rescue '' + files = tmp["bulletin_files"].collect{|bf| { "file_url" => bf["url"], "file_title" => (bf["title_translations"][locale].blank? ? File.basename(bf["url"]) : bf["title_translations"][locale] rescue '') }} rescue [] + links = tmp["bulletin_links"].map{|link| { "link_url" => link["url"], "link_title" => (link["title_translations"][locale].blank? ? link["url"] : link["title_translations"][locale]) } } rescue [] + tmp["bulletin_links"] = links + tmp["bulletin_files"] = files + tmp["title"] = tmp["title_translations"][locale] + tmp["subtitle"] = tmp["subtitle_translations"][locale] + tmp["source-site-link"] = tmp["source-site"] + tmp["source-site"] = "#{tmp["source-site-title"]}" + if !is_widget + tmp["link_to_show"] = OrbitHelper.url_to_show(tmp["params"]) rescue '' + else + tmp["link_to_show"] = OrbitHelper.widget_item_url(tmp["params"]) rescue '' + end + tmp["target"] = "_self" + tmp["img_src"] = tmp["image"]["thumb"] || "/assets/announcement-default.jpg" + tmp["img_description"] = tmp["image_description_translations"][locale] + tmp["more"] = I18n.t(:more_plus) + tmp["view_count"] = "" else - tmp["link_to_show"] = OrbitHelper.widget_item_url(tmp["params"]) rescue '' + tmp['statuses'] = [] + if self[:top_list].count == 0 || self[:top_list].exclude?(tmp['id']) + tmp[:is_top] = false + else + tmp[:is_top] = true + tmp['statuses'] << { + "status" => I18n.t(:top), + "status-class" => "status-top" + } + end + if self[:hot_list].count == 0 || self[:top_list].exclude?(tmp['id']) + tmp[:is_hot] = false + else + tmp[:is_hot] = true + tmp['statuses'] << { + "status" => I18n.t(:hot), + "status-class" => "status-hot" + } + end + tmp["category"] = cat + tmp["source-site"] = self.remote_site_url + tmp["source-site-title"] = (self[:channel_title][locale] rescue "") + tmp["params"] = tmp["params"].to_s + "_" + self.feed_id.to_s + "h" + next if !site_source.nil? && site_source != tmp["source-site-title"] + tmp['statuses'] << { + "status" => "#{tmp["source-site-title"]}", + "status-class" => "status-source" + } + tmp["source-site-link"] = tmp["source-site"] + tmp["source-site"] = "#{tmp["source-site-title"]}" + if !is_widget + tmp["link_to_show"] = OrbitHelper.url_to_show(tmp["params"]) rescue '' + else + tmp["link_to_show"] = OrbitHelper.widget_item_url(tmp["params"]) rescue '' + end + tmp["target"] = "_self" + tmp["more"] = I18n.t(:more_plus) + tmp["view_count"] = "" + new_tmp = {} + tmp.each do |key,value| + if key.include? "_translations" + new_tmp[key.sub("_translations","")] = value[locale].to_s rescue "" + elsif key.include?("date") || key.include?("Date") + new_tmp[key] = DateTime.parse(value) rescue nil + else + if value.class == Hash + value.each do |sub_k,sub_v| + if sub_k.include? "_translations" + new_tmp[key][sub_k.sub("_translations","")] = sub_v[locale].to_s rescue "" + else + new_tmp[key][sub_k] = sub_v + end + end + else + new_tmp[key] = value + end + end + end + tmp = BSON::Document.new(new_tmp) end - tmp["target"] = "_self" - tmp["img_src"] = tmp["image"]["thumb"] || "/assets/announcement-default.jpg" - tmp["img_description"] = tmp["image_description_translations"][locale] - tmp["more"] = I18n.t(:more_plus) - tmp["view_count"] = "" else tmp = nil end diff --git a/config/locales/en.yml b/config/locales/en.yml index f576246..c96fa67 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -1,5 +1,6 @@ en: feed: + module_name: Module name new_: "Susbscribe to channel" feed: Feed all_feeds: All Feeds diff --git a/config/locales/zh_tw.yml b/config/locales/zh_tw.yml index 45eb3d5..5aca003 100644 --- a/config/locales/zh_tw.yml +++ b/config/locales/zh_tw.yml @@ -1,5 +1,6 @@ zh_tw: feed: + module_name: 模組名稱 new_: "訂閱頻道" feed: Feed all_feeds: 'Feeds 列表'