harry
/
feeds
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add new feature for rss feed

This commit is contained in:
邱博亞 2022-09-25 19:34:29 +08:00
parent e3dcba09b0
commit 86fbe38431
7 changed files with 190 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
app/assets/javascripts/new_channel.js
View File

@ -46,11 +46,16 @@
$("#channels").fadeOut(function(){ $("#channels").fadeOut(function(){
$("#channels").html(""); $("#channels").html("");
displayLoading(true,"Fetching feed list for " + channel.title); displayLoading(true,"Fetching feed list for " + channel.title);
var data_to_send = {"url" : site_url, "channel" : channel.key, "feed_list_url" : channel.url};
if (channel.feeds){
data_to_send.feeds = JSON.stringify(channel.feeds)
}
console.log(data_to_send)
$.ajax({ $.ajax({
url : "/admin/feeds/get_feed_list", url : "/admin/feeds/get_feed_list",
data : {"url" : site_url, "channel" : channel.key, "feed_list_url" : channel.url}, data : data_to_send,
dataType : "json", dataType : "json",
type : "get" type : "post"
}).done(function(data){ }).done(function(data){
$.each(data.feeds, function(index,feed){ $.each(data.feeds, function(index,feed){
var klass = (feed.subscribed ? "active" : ""); var klass = (feed.subscribed ? "active" : "");

165
app/controllers/admin/feeds_controller.rb
View File

@ -3,9 +3,9 @@ class Admin::FeedsController < OrbitAdminController
helper_method :search_all_words helper_method :search_all_words
def compute_layout def compute_layout
if action_name== 'annc_content' if action_name== 'annc_content'
false false
else else
'back_end' 'back_end'
end end
end end
def index def index
@ -24,8 +24,8 @@ class Admin::FeedsController < OrbitAdminController
@setting.update_attributes(p) @setting.update_attributes(p)
Thread.new do Thread.new do
sleep 3 sleep 3
content = "UNICORN_PID=\"`fuser tmp/pids/unicorn.sock tmp/sockets/unicorn.sock tmp/unicorn.sock` `cat tmp/pids/unicorn.pid `\" && kill -s USR2 $UNICORN_PID ; n=20; while (kill -0 $UNICORN_PID > /dev/null 2>&1) && test $n -ge 0; do printf '.' && sleep 1 && n=$(( $n - 1 )); done ; if test $n -lt 0; then kill -s TERM $UNICORN_PID; sleep 3; bundle exec unicorn_rails -c config/unicorn.rb -D -E #{Rails.env}; else kill -s QUIT $UNICORN_PID; fi" content = "UNICORN_PID=\"`fuser tmp/pids/unicorn.sock tmp/sockets/unicorn.sock tmp/unicorn.sock` `cat tmp/pids/unicorn.pid `\" && kill -s USR2 $UNICORN_PID ; n=20; while (kill -0 $UNICORN_PID > /dev/null 2>&1) && test $n -ge 0; do printf '.' && sleep 1 && n=$(( $n - 1 )); done ; if test $n -lt 0; then kill -s TERM $UNICORN_PID; sleep 3; bundle exec unicorn_rails -c config/unicorn.rb -D -E #{Rails.env}; else kill -s QUIT $UNICORN_PID; fi"
system(content) system(content)
end end
redirect_to admin_feeds_settings_path redirect_to admin_feeds_settings_path
end end
@ -108,21 +108,21 @@ class Admin::FeedsController < OrbitAdminController
tp = (a.blank? || a[I18n.locale].blank?) ? b.gsub(/http:\/\/|https:\/\//,'').gsub(/\./,'-') : a[I18n.locale] tp = (a.blank? || a[I18n.locale].blank?) ? b.gsub(/http:\/\/|https:\/\//,'').gsub(/\./,'-') : a[I18n.locale]
tp tp
end.uniq.map{|tp| {:title => tp,:id => tp}} end.uniq.map{|tp| {:title => tp,:id => tp}}
@filter_fields['feed.module_name'] = SiteFeed.all.map{|s| s.channel_key}.uniq.map do |key| @filter_fields['feed.module_name'] = SiteFeed.all.map{|s| s.channel_key}.uniq.map do |key|
{:title => I18n.t("module_name.#{key}"),:id => key} {:title => I18n.t("module_name.#{key}"),:id => key}
end end
@max_content_count = 100 @max_content_count = 100
@page_no = params[:page].to_i @page_no = params[:page].to_i
@page_no = 1 if @page_no==0 @page_no = 1 if @page_no==0
@skip_count = (@page_no-1)*@max_content_count @skip_count = (@page_no-1)*@max_content_count
@module_name = params[:filters][:module_name].to_a rescue [] @module_name = params[:filters][:module_name].to_a rescue []
@module_name = SiteFeed.all.map{|s| s.channel_key}.uniq if @module_name == [] @module_name = SiteFeed.all.map{|s| s.channel_key}.uniq if @module_name == []
@all_feed_annc = SiteFeedAnnc.where(:channel_key.in => @module_name).order(created_at: 1).to_a rescue [] @all_feed_annc = SiteFeedAnnc.where(:channel_key.in => @module_name).order(created_at: 1).to_a rescue []
@source = params[:filters][:source] rescue [] @source = params[:filters][:source] rescue []
@keywords = params[:keywords] rescue nil @keywords = params[:keywords] rescue nil
if request.xhr? if request.xhr?
render :partial => "announcements" render :partial => "announcements"
end end
end end
def get_category_list def get_category_list
app_key = params[:channel] app_key = params[:channel]
@ -140,55 +140,56 @@ class Admin::FeedsController < OrbitAdminController
end end
def get_channel_list def get_channel_list
puts params['url']
url = URI.decode(params['url']).chomp("/") + "/feeds/channel_lists" url = URI.decode(params['url']).chomp("/") + "/feeds/channel_lists"
puts url uri = URI(URI.decode(url).encode)
uri = URI.parse(URI.decode(url).encode) res_net = http_request(uri) rescue nil
begin res = res_net ? res_net.body : nil
res = Timeout.timeout(10){Net::HTTP.get_response(uri).body} data = {}
rescue if res_net && res_net['Content-Type'].split(";")[0]=="application/json"
http = Net::HTTP.new(uri.host, uri.port) data = JSON.parse(res) rescue {}
http.use_ssl = true if (uri.scheme == "https") else
http.verify_mode = OpenSSL::SSL::VERIFY_NONE uri = URI(params['url'])
res = http.get(uri.request_uri).body rescue nil body = http_request(uri).body
puts uri.to_s links = get_rss_link(params['url'], body)
puts uri.hostname if links.length > 0
end data = {channels: [{title: "Announcement",
data = JSON.parse(res) rescue {} key: "announcement",
print data url: params['url'],
render :json => data.to_json feeds: links,
app_icon: "icons-megaphone"}]}
end
end
render :json => data.to_json
end end
def get_feed_list def get_feed_list
url = params['url'].chomp("/") + params[:feed_list_url] data_to_send = {}
uri = URI.parse(url) if params['feeds']
begin data = {"feeds"=> JSON.parse(params[:feeds])}
res = Net::HTTP.start(uri.host, else
uri.port,:use_ssl => uri.scheme == 'https', url = params['url'].chomp("/") + params[:feed_list_url]
open_timeout: 10,read_timeout: 10, uri = URI.parse(url)
verify_mode: OpenSSL::SSL::VERIFY_NONE) do |http| begin
req = Net::HTTP::Get.new(uri) res = http_request(uri)
http.request(req) res = res.code=='200' ? res.body : nil
end rescue
res = res.code=='200' ? res.body : nil res = nil
rescue puts uri.to_s
res = nil puts uri.hostname
puts uri.to_s end
puts uri.hostname data = JSON.parse(res) rescue {}
end end
data = JSON.parse(res) rescue {} data_to_send["feeds"] = []
data_to_send = {} Array(data["feeds"]).each do |feed|
data_to_send["feeds"] = [] sf = SiteFeed.find_by(:feed_uid => feed["uid"]) rescue nil
Array(data["feeds"]).each do |feed| if !sf.nil?
sf = SiteFeed.find_by(:feed_uid => feed["uid"]) rescue nil feed["subscribed"] = true
if !sf.nil? else
feed["subscribed"] = true feed["subscribed"] = false
else end
feed["subscribed"] = false data_to_send["feeds"] << feed
end end
data_to_send["feeds"] << feed render :json => data_to_send.to_json
end
render :json => data_to_send.to_json
end end
def channel_title def channel_title
@ -207,7 +208,8 @@ class Admin::FeedsController < OrbitAdminController
def subscribe def subscribe
site_feed = SiteFeed.new site_feed = SiteFeed.new
site_feed.remote_site_url = params[:url].chomp("/") uri = URI(params[:url])
site_feed.remote_site_url = uri.to_s[/\A.*(?=#{uri.path}\z)/]
site_feed.merge_with_category = params[:category] site_feed.merge_with_category = params[:category]
site_feed.channel_name = params[:channel] site_feed.channel_name = params[:channel]
site_feed.channel_key = params[:channel_key] site_feed.channel_key = params[:channel_key]
@ -215,6 +217,9 @@ class Admin::FeedsController < OrbitAdminController
site_feed.feed_name_translations = params[:feed][:title_translations] site_feed.feed_name_translations = params[:feed][:title_translations]
site_feed.disabled = false site_feed.disabled = false
site_feed.feed_url = params[:feed][:url] site_feed.feed_url = params[:feed][:url]
if params[:feed][:is_rss]
site_feed.is_rss = params[:feed][:is_rss]
end
uri = URI(site_feed.remote_site_url) uri = URI(site_feed.remote_site_url)
site_host = uri.host site_host = uri.host
other_site_feeds = SiteFeed.where(:remote_site_url=>/:\/\/#{::Regexp.escape(site_host)}/) other_site_feeds = SiteFeed.where(:remote_site_url=>/:\/\/#{::Regexp.escape(site_host)}/)
@ -268,9 +273,37 @@ class Admin::FeedsController < OrbitAdminController
end end
def search_all_words(target, word) def search_all_words(target, word)
target = target.upcase target = target.upcase
words = word.upcase.split(' ') words = word.upcase.split(' ')
words.select { |value| target.include? value } == words words.select { |value| target.include? value } == words
end end
private
def http_request(uri, open_timeout=10)
Net::HTTP.start(uri.host, uri.port,
:use_ssl => uri.scheme == 'https',
open_timeout: open_timeout,read_timeout: 60,
verify_mode: OpenSSL::SSL::VERIFY_NONE) do |http|
req = Net::HTTP::Get.new(uri)
req['User-Agent'] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36'
http.request(req)
end
end
def get_rss_link(url, body)
html = Nokogiri::HTML(body)
html.search('.data_midlle_rss a').map{|v|
href = URI.join(url, v.attr('href')).to_s
uid = href.split("?n=")[-1]
title = v.text()
title_translations = I18n.available_locales.collect{|v| [v, title]}.to_h
{
url: href,
xml_url: href,
uid: uid,
title_translations: title_translations,
tags: title_translations,
is_rss: true
}
}
end
end end

92
app/models/site_feed.rb
View File

@ -18,12 +18,15 @@ class SiteFeed
field :feeds_finish_callback field :feeds_finish_callback
field :feeds_update_statuses_callback field :feeds_update_statuses_callback
field :enable_notify, type: Boolean, default: false field :enable_notify, type: Boolean, default: false
field :is_rss, type: Boolean, default: false
require 'feed_model/cache' require 'feed_model/cache'
require 'fileutils' require 'fileutils'
include FeedModel::Cache include FeedModel::Cache
Category.send(:include,FeedModel::Cache) Category.send(:include,FeedModel::Cache)
after_create do after_create do
self.add_notify if !self.is_rss
self.add_notify
end
crontab_list = `crontab -l`.split("\n") rescue [] crontab_list = `crontab -l`.split("\n") rescue []
site_root = Rails.root.to_s site_root = Rails.root.to_s
if crontab_list.select{|s| s.include?(site_root) && s.include?("feeds_module:make_cache")} == [] if crontab_list.select{|s| s.include?(site_root) && s.include?("feeds_module:make_cache")} == []
@ -31,7 +34,9 @@ class SiteFeed
end end
end end
before_destroy do before_destroy do
self.remove_notify if !self.is_rss
self.remove_notify
end
tmp = SiteFeedAnnc.where(feed_id: self.id) tmp = SiteFeedAnnc.where(feed_id: self.id)
main_directory = File.join("#{Rails.root}","public","site_feeds") main_directory = File.join("#{Rails.root}","public","site_feeds")
feed_directory = File.join(main_directory.to_s, self.id.to_s) feed_directory = File.join(main_directory.to_s, self.id.to_s)
@ -50,8 +55,6 @@ class SiteFeed
self.update_remote_site_url self.update_remote_site_url
end end
end end
else
self.sync_data_to_annc
end end
@site_feed_annc = nil @site_feed_annc = nil
end end
@ -80,6 +83,16 @@ class SiteFeed
end end
end end
def get_annc(force_refresh=false) def get_annc(force_refresh=false)
def to_I18n(title, have_key=false, key="title_translations")
value = I18n.available_locales.map{|v| [v,title]}.to_h
if have_key
{
key => value
}
else
value
end
end
main_directory = File.join("#{Rails.root}","public","site_feeds") main_directory = File.join("#{Rails.root}","public","site_feeds")
feed_directory = File.join(main_directory.to_s, self.id.to_s) feed_directory = File.join(main_directory.to_s, self.id.to_s)
if !force_refresh && File.exists?(feed_directory) if !force_refresh && File.exists?(feed_directory)
@ -87,6 +100,49 @@ class SiteFeed
else else
uri = URI(self.feed_url) uri = URI(self.feed_url)
res = get_response_body(uri) rescue '' res = get_response_body(uri) rescue ''
if self.is_rss
rss_data = Hash.from_xml(res)
category_title = rss_data['rss']['channel']['title']
category_title_trans = to_I18n(category_title, true)
res_to_json = rss_data['rss']['channel']["item"].collect do |item|
tag_titles = Array(item['類別']).collect{|title| to_I18n(title, true, "name_translations")}
id = item['link'].split("&s=")[-1]
title = Array(item['title'])[0]
text = Array(item['description'])[0]
files = []
if text.include?('附件')
files = Nokogiri::HTML(get_response_body(URI(item['link']))).search('#RelData1_liFile ol a').collect do |v|
file_title = v.attr('title')
{
"description_translations" => to_I18n(file_title),
"title_translations" => to_I18n(file_title.sub(/^\[(?:(?!\]).)*\]/,'').rpartition(".")[0]),
"url" => v.attr('href')
}
end
end
{
"show_url" => item['link'],
"org_is_top" => 0,
"id" => id,
"title_translations" => to_I18n(title),
"subtitle_translations" => {},
"text_translations" => to_I18n(text),
"postdate" => DateTime.parse(item['pubDate']).to_s,
"image_description_translations" => {},
"image" => {},
"display_img" => false,
"tags" => tag_titles,
"category" => category_title_trans,
"author" => item['author'],
"params" => "#{title}-#{id}",
"bulletin_links" => [],
"bulletin_files" => files,
"bulletin_carousel_images" => [],
"external_link" => nil
}
end
res = JSON.dump({"announcements" => res_to_json})
end
FileUtils.mkdir_p(feed_directory) if !File.exists?(feed_directory) FileUtils.mkdir_p(feed_directory) if !File.exists?(feed_directory)
File.open(File.join(feed_directory.to_s,self.feed_uid + ".json"),"w") do |file| File.open(File.join(feed_directory.to_s,self.feed_uid + ".json"),"w") do |file|
res.force_encoding("utf-8") res.force_encoding("utf-8")
@ -102,6 +158,7 @@ class SiteFeed
open_timeout: 60,read_timeout: 60, open_timeout: 60,read_timeout: 60,
verify_mode: OpenSSL::SSL::VERIFY_NONE) do |http| verify_mode: OpenSSL::SSL::VERIFY_NONE) do |http|
req = Net::HTTP::Get.new(uri) req = Net::HTTP::Get.new(uri)
req['User-Agent'] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36'
http.request(req) http.request(req)
end end
if res.code == "302" || res.code == "301" if res.code == "302" || res.code == "301"
@ -112,15 +169,22 @@ class SiteFeed
else else
uri = URI.parse(location) uri = URI.parse(location)
end end
res = Net::HTTP.start(uri.host, uri.port, begin
:use_ssl => uri.scheme == 'https', res = Net::HTTP.start(uri.host, uri.port,
open_timeout: 60,read_timeout: 60, :use_ssl => uri.scheme == 'https',
verify_mode: OpenSSL::SSL::VERIFY_NONE) do |http| open_timeout: 60,read_timeout: 60,
req = Net::HTTP::Get.new(uri) verify_mode: OpenSSL::SSL::VERIFY_NONE) do |http|
req['Cookie'] = cookie req = Net::HTTP::Get.new(uri)
http.request(req) req['Cookie'] = cookie
http.request(req)
end
res_body = res.body
rescue => e
res_body = ''
puts e
puts e.backtrace
end end
return res.body return res_body
else else
return res.body return res.body
end end
@ -128,8 +192,8 @@ class SiteFeed
def category def category
Category.find(self.merge_with_category) rescue nil Category.find(self.merge_with_category) rescue nil
end end
def channel_title_for_cache def channel_title_for_cache #empty? must for hash
!self[:channel_title].to_s.empty? ? self[:channel_title] : I18n.t("feed.source") !self[:channel_title].blank? ? self[:channel_title] : I18n.available_locales.collect{|v| [v,I18n.t("feed.source")]}.to_h
end end
def http_request(http, request) def http_request(http, request)
response = http.request(request) response = http.request(request)

4
app/models/site_feed_annc.rb
View File

@ -333,9 +333,9 @@ class SiteFeedAnnc
end end
if tmp["link_to_show"].nil? if tmp["link_to_show"].nil?
if !is_widget if !is_widget
tmp["link_to_show"] = UseSourceUrl && tmp["show_url"] ? "#{tmp["source_url"]}/#{locale}#{tmp["show_url"]}" : OrbitHelper.url_to_show(tmp["params"]) rescue '' tmp["link_to_show"] = UseSourceUrl && tmp["show_url"] ? URI.join("#{tmp["source_url"]}/#{locale}","#{tmp["show_url"]}").to_s : OrbitHelper.url_to_show(tmp["params"]) rescue ''
else else
tmp["link_to_show"] = UseSourceUrl && tmp["show_url"] ? "#{tmp["source_url"]}/#{locale}#{tmp["show_url"]}" : OrbitHelper.widget_item_url(tmp["params"]) rescue '' tmp["link_to_show"] = UseSourceUrl && tmp["show_url"] ? URI.join("#{tmp["source_url"]}/#{locale}","#{tmp["show_url"]}").to_s : OrbitHelper.widget_item_url(tmp["params"]) rescue ''
end end
end end
tmp tmp

2
config/routes.rb
View File

@ -66,7 +66,7 @@ Rails.application.routes.draw do
scope "(:locale)", locale: Regexp.new(locales.join("|")) do scope "(:locale)", locale: Regexp.new(locales.join("|")) do
namespace :admin do namespace :admin do
get "/feeds/get_channel_list", to: 'feeds#get_channel_list' get "/feeds/get_channel_list", to: 'feeds#get_channel_list'
get "/feeds/get_feed_list", to: 'feeds#get_feed_list' post "/feeds/get_feed_list", to: 'feeds#get_feed_list'
get "/feeds/get_category_list", to: 'feeds#get_category_list' get "/feeds/get_category_list", to: 'feeds#get_category_list'
post "/feeds/subscribe", to: 'feeds#subscribe' post "/feeds/subscribe", to: 'feeds#subscribe'
post "/feeds/unsubscribe", to: 'feeds#unsubscribe' post "/feeds/unsubscribe", to: 'feeds#unsubscribe'

1
lib/feed_model/cache.rb
View File

@ -16,6 +16,7 @@ module FeedModel
tmp = SiteFeedAnnc.new(feed_id: feed.id) tmp = SiteFeedAnnc.new(feed_id: feed.id)
end end
tmp.all_contents_for_feed = tmp.cache_annc(true) tmp.all_contents_for_feed = tmp.cache_annc(true)
feed.sync_data_to_annc(tmp)
tmp.save tmp.save
if feed.feeds_model && feed.feeds_finish_callback if feed.feeds_model && feed.feeds_finish_callback
feeds_model = feed.feeds_model.constantize feeds_model = feed.feeds_model.constantize

35
lib/tasks/feeds_tasks.rake
View File

@ -3,40 +3,9 @@ namespace :feeds_module do
task :make_cache,[:url] => :environment do |task,args| task :make_cache,[:url] => :environment do |task,args|
main_directory = File.join("#{Rails.root}","public","site_feeds") main_directory = File.join("#{Rails.root}","public","site_feeds")
FileUtils.mkdir_p(main_directory) if !File.exists?(main_directory) FileUtils.mkdir_p(main_directory) if !File.exists?(main_directory)
SiteFeedAnnc SiteFeedAnnc #Don't remove this, this will auto require SiteFeedAnnc
SiteFeed.enabled.each do |site_feed| SiteFeed.enabled.each do |site_feed|
feed_directory = File.join(main_directory.to_s, site_feed.id.to_s) FeedModel::Cache.recreate_annc_cache(site_feed)
FileUtils.mkdir_p(feed_directory) if !File.exists?(feed_directory)
uri = URI(site_feed.feed_url)
res_net = Net::HTTP.start(uri.host, uri.port,
:use_ssl => uri.scheme == 'https',
open_timeout: 60,read_timeout: 60,
verify_mode: OpenSSL::SSL::VERIFY_NONE) do |http|
req = Net::HTTP::Get.new(uri)
http.request(req)
end
if (res_net.code == '301' rescue true)
begin
uri = URI(res_net['location'])
res_net = Net::HTTP.start(uri.host, uri.port,
:use_ssl => uri.scheme == 'https',
open_timeout: 60,read_timeout: 60,
verify_mode: OpenSSL::SSL::VERIFY_NONE) do |http|
req = Net::HTTP::Get.new(uri)
http.request(req)
end
rescue => e
res_net = ''
puts e
puts e.backtrace
end
end
res = res_net.body rescue ''
File.open(File.join(feed_directory.to_s,site_feed.feed_uid + ".json"),"w") do |file|
res = res.force_encoding("utf-8") rescue ''
file.write(res)
end
site_feed.save
end end
end end
end end