harry
/
feeds
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

1. accelerate feed and search 

2. fix feed error
This commit is contained in:
邱博亞 2024-10-13 21:47:51 +08:00
parent 1bfaf4d6f4
commit 16963a9cb2
10 changed files with 252 additions and 219 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/controllers/admin/feeds_controller.rb
View File

@ -267,7 +267,7 @@ class Admin::FeedsController < OrbitAdminController
def force_refresh
sf = SiteFeed.find_by(:feed_uid => params[:feed_uid]) rescue nil
if !sf.nil?
FeedModel::Cache.recreate_annc_cache(sf)
Feeds::Cache.recreate_annc_cache(sf)
end
render :json => {"success" => true}.to_json
end

11
app/controllers/feeds_controller.rb
View File

@ -3,6 +3,7 @@ class FeedsController < ApplicationController
def notify_change
feed = SiteFeed.where(:feed_uid=>params[:uid]).first
feed_annc = SiteFeedAnnc.where(:feed_id=>feed.id).first
site = Site.first
if feed_annc
raw_all_contents_for_feed_translations = feed_annc.raw_all_contents_for_feed_translations
feeds_model = feed.feeds_model
@ -16,9 +17,9 @@ class FeedsController < ApplicationController
feeds_finish_callback = feed.feeds_finish_callback
category_id = feed.merge_with_category
can_create_record = feeds_model && feeds_uid_field && feeds_update_callback
locales = Site.first.in_use_locales rescue I18n.available_locales
locales = site.in_use_locales rescue I18n.available_locales
locales.map!{|l| l.to_s}
site_root_url = Site.first.root_url rescue ""
site_root_url = site.root_url rescue ""
main_directory = File.join("#{Rails.root}","public","site_feeds")
feed_directory = File.join(main_directory.to_s, feed.id.to_s)
feed_data = JSON.parse(File.read(File.join(feed_directory.to_s, feed.feed_uid + ".json")))
@ -49,7 +50,7 @@ class FeedsController < ApplicationController
insert_idx = ((a["org_is_top"] == 1) ? 0 : -1)
end
end
raw_all_contents_for_feed_translations[locale].insert(insert_idx, feed_annc.process_tmp(a,locale,trans,site_root_url))
raw_all_contents_for_feed_translations[locale].insert(insert_idx, feed_annc.process_tmp(site,a,locale,trans,site_root_url))
feed_data[channel_key_pluralize].insert(insert_idx, a)
end
if can_create_record
@ -86,7 +87,7 @@ class FeedsController < ApplicationController
has_create = false
raw_all_contents_for_feed_translations[locale].each_with_index do |aa, i|
if aa["id"] == a["id"]
raw_all_contents_for_feed_translations[locale][i] = feed_annc.process_tmp(a,locale,trans,site_root_url)
raw_all_contents_for_feed_translations[locale][i] = feed_annc.process_tmp(site,a,locale,trans,site_root_url)
feed_data[channel_key_pluralize][i] = a
has_create = true
break
@ -100,7 +101,7 @@ class FeedsController < ApplicationController
insert_idx = ((a["org_is_top"] == 1) ? 0 : -1)
end
end
raw_all_contents_for_feed_translations[locale].insert(insert_idx, feed_annc.process_tmp(a,locale,trans,site_root_url))
raw_all_contents_for_feed_translations[locale].insert(insert_idx, feed_annc.process_tmp(site,a,locale,trans,site_root_url))
feed_data[channel_key_pluralize].insert(insert_idx, a)
end
end

11
app/models/site_feed.rb
View File

@ -2,6 +2,7 @@ class SiteFeed
include Mongoid::Document
include Mongoid::Timestamps
ModuleAppSyncFields = ["feeds_model", "feeds_uid_field", "feeds_update_callback", "feeds_time_field", "feeds_finish_callback", "feeds_update_statuses_callback"]
SiteFeedAnncSyncFields = ["channel_key", "merge_with_category", "remote_site_url", "feeds_model", "feeds_uid_field", "feeds_update_callback", "feeds_time_field", "feeds_update_statuses_callback"]
field :remote_site_url
field :merge_with_category
field :channel_name
@ -14,15 +15,14 @@ class SiteFeed
field :feeds_model
field :feeds_uid_field
field :feeds_update_callback
field :feeds_time_field, type: String, default: "postdate"
field :feeds_time_field, default: "postdate"
field :feeds_finish_callback
field :feeds_update_statuses_callback
field :enable_notify, type: Boolean, default: false
field :is_rss, type: Boolean, default: false
require 'feed_model/cache'
require 'fileutils'
include FeedModel::Cache
Category.send(:include,FeedModel::Cache)
include Feeds::Cache
Category.send(:include,Feeds::Cache)
after_create do
if !self.is_rss
self.add_notify
@ -71,9 +71,8 @@ class SiteFeed
def sync_data_to_annc(site_feed_annc=nil)
category_title = self.category.title_translations rescue {}
tmp_channel_title = self.channel_title_for_cache
clone_fields =["channel_key", "merge_with_category", "remote_site_url", "feeds_model", "feeds_uid_field", "feeds_update_callback", "feeds_time_field", "feeds_update_statuses_callback"]
(site_feed_annc ? [site_feed_annc] : SiteFeedAnnc.where(:feed_id=>self.id)).each do |tmp|
clone_fields.each do |f|
SiteFeedAnncSyncFields.each do |f|
tmp.send("#{f}=", self.send(f))
end
tmp[:feed_name] = self.feed_name_translations

296
app/models/site_feed_annc.rb
View File

@ -13,10 +13,9 @@ class SiteFeedAnnc
field :remote_site_url
field :channel_title
field :feeds_model
field :feeds_model
field :feeds_uid_field
field :feeds_update_callback
field :feeds_time_field, type: String, default: "postdate"
field :feeds_time_field, default: "postdate"
field :feeds_finish_callback
field :feeds_update_statuses_callback
@ -36,7 +35,9 @@ class SiteFeedAnnc
self.site_feed_annc_details = []
@raw_all_contents_for_feed_translations.each do |locale, contents|
contents.each do |content|
self.site_feed_annc_details << SiteFeedAnncDetail.new(locale: locale, site_feed_annc: self, data: content)
self.site_feed_annc_details << SiteFeedAnncDetail.new(
locale: locale, site_feed_annc: self, data: content
)
end
end
end
@ -132,7 +133,7 @@ class SiteFeedAnnc
self.raw_all_contents_for_feed_translations = anns_translations
self.save
end
def process_tmp(a, locale=nil, trans=nil, site_root_url=nil)
def process_tmp(site, a, locale=nil, trans=nil, site_root_url=nil)
cat = self[:category_title]
locale = I18n.locale.to_s if locale.nil?
if trans.nil?
@ -148,57 +149,21 @@ class SiteFeedAnnc
end
end
if site_root_url.nil?
site_root_url = Site.first.root_url rescue ""
site_root_url = site.root_url
end
tmp = a.deep_dup
tmp[:is_hidden] = self.hidden_annc.include?(tmp['id'])
if self.channel_key == "announcement"
tmp['statuses'] = []
if self[:top_list].count == 0 || self[:top_list].exclude?(tmp['id'])
tmp[:is_top] = false
else
tmp[:is_top] = true
tmp['statuses'] << {
"status" => trans[locale]['top'],
"status-class" => "status-top"
}
end
if self[:hot_list].count == 0 || self[:top_list].exclude?(tmp['id'])
tmp[:is_hot] = false
else
tmp[:is_hot] = true
tmp['statuses'] << {
"status" => trans[locale]['hot'],
"status-class" => "status-hot"
}
end
tmp["category"] = cat
tmp["source_url"] = self.remote_site_url
tmp["source-site"] = self.remote_site_url
tmp["source-site-title"] = (self[:channel_title][locale] rescue "")
tmp["params"] = tmp["params"].to_s + "_" + self.feed_id.to_s + "h"
if !tmp["source-site-title"].blank?
tmp['statuses'] << {
"status" => "<a href=\"#{tmp["source-site"]}\" target=\"_blank\" class=\"feed-source\">#{tmp["source-site-title"]}</a>",
"status-class" => "status-source"
}
end
files = tmp["bulletin_files"].collect{|bf| { "file_url" => bf["url"], "file_title" => (bf["title_translations"][locale].blank? ? File.basename(bf["url"]) : bf["title_translations"][locale] rescue '') }} rescue []
links = tmp["bulletin_links"].map{|link| { "link_url" => link["url"], "link_title" => (link["title_translations"][locale].blank? ? link["url"] : link["title_translations"][locale]) } } rescue []
tmp["bulletin_links"] = links
tmp["bulletin_files"] = files
tmp["title"] = tmp["title_translations"][locale]
tmp["subtitle"] = tmp["subtitle_translations"][locale]
tmp["source-site-link"] = tmp["source-site"]
tmp["source-site"] = "<a href=\"#{tmp["source-site"]}\" target=\"_blank\" class=\"feed-source\">#{tmp["source-site-title"]}</a>"
tmp["link_to_show"] = !tmp["external_link"].blank? ? tmp["external_link"] : nil
tmp["target"] = (site_root_url.blank? || tmp["link_to_show"].blank?) ? '_blank' : (tmp["link_to_show"].include?(site_root_url) ? '_self' : '_blank')
tmp["img_src"] = tmp["image"]["thumb"] || "/assets/announcement-default.jpg"
tmp["img_src"] = "/assets/announcement-default.jpg" if tmp["img_src"] == ""
tmp["img_description"] = tmp["image_description_translations"][locale]
tmp["more"] = trans[locale]['more_plus']
tmp["view_count"] = ""
else
new_tmp = {}
tmp.each do |key,value|
@ -225,54 +190,74 @@ class SiteFeedAnnc
end
end
tmp = BSON::Document.new(new_tmp)
tmp['statuses'] = []
if self[:top_list].count == 0 || self[:top_list].exclude?(tmp['id'])
tmp[:is_top] = false
else
tmp[:is_top] = true
tmp['statuses'] << {
"status" => trans[locale]['top'],
"status-class" => "status-top"
}
end
if self[:hot_list].count == 0 || self[:top_list].exclude?(tmp['id'])
tmp[:is_hot] = false
else
tmp[:is_hot] = true
tmp['statuses'] << {
"status" => trans[locale]['hot'],
"status-class" => "status-hot"
}
end
tmp["category"] = cat
tmp["source_url"] = self.remote_site_url
tmp["source-site"] = self.remote_site_url
tmp["source-site-title"] = (self[:channel_title][locale] rescue "")
tmp["params"] = tmp["params"].to_s + "_" + self.feed_id.to_s + "h"
end
if !tmp["title"].blank?
tmp["title_plain_text"] = Nokogiri::HTML5.fragment(tmp["title"]).text
end
tmp["category"] = cat
tmp["target"] = (
tmp["link_to_show"].blank? || (!site_root_url.blank? && tmp["link_to_show"].include?(site_root_url))
) ? '_self' : '_blank'
tmp["params"] = tmp["params"].to_s + "_" + self.feed_id.to_s + "h"
tmp["more"] = trans[locale]['more_plus']
tmp["view_count"] = ""
tmp['statuses'] = []
if self.top_list.exclude?(tmp['id'])
tmp["is_top"] = false
else
tmp["is_top"] = true
tmp['statuses'] << {
"status" => trans[locale]['top'],
"status-class" => "status-top"
}
end
if self.hot_list.exclude?(tmp['id'])
tmp["is_hot"] = false
else
tmp["is_hot"] = true
tmp['statuses'] << {
"status" => trans[locale]['hot'],
"status-class" => "status-hot"
}
end
tmp["is_hidden"] = self.hidden_annc.include?(tmp['id'])
tmp["source_url"] = self.remote_site_url
tmp["source-site"] = self.remote_site_url
tmp["source-site-title"] = self.channel_title(locale)
if !tmp["source-site-title"].blank?
tmp['statuses'] << {
"status" => "<a href=\"#{tmp["source-site"]}\" target=\"_blank\" class=\"feed-source\">#{tmp["source-site-title"]}</a>",
"status-class" => "status-source"
}
tmp["source-site-link"] = tmp["source-site"]
tmp["source-site"] = "<a href=\"#{tmp["source-site"]}\" target=\"_blank\" class=\"feed-source\">#{tmp["source-site-title"]}</a>"
tmp["target"] = "_self"
tmp["more"] = trans[locale]['more_plus']
tmp["view_count"] = ""
end
feeds_time_field = self.feeds_time_field
feeds_time_field = 'postdate' if feeds_time_field.nil?
time_field_value = tmp[feeds_time_field]
tmp[feeds_time_field] = (time_field_value.blank? ? nil : time_field_value.to_time rescue time_field_value)
tmp[feeds_time_field] = time_field_value if tmp[feeds_time_field].nil?
tmp["source-site-link"] = tmp["source-site"]
tmp["source-site"] = "<a href=\"#{tmp["source-site"]}\" target=\"_blank\" class=\"feed-source\">#{tmp["source-site-title"]}</a>"
feeds_time_fields = Array(self.feeds_time_field)
feeds_time_fields.each do |feeds_time_field|
time_field_value = tmp[feeds_time_field]
tmp[feeds_time_field] = (time_field_value.blank? ? nil : time_field_value.to_time rescue time_field_value)
tmp[feeds_time_field] = time_field_value if tmp[feeds_time_field].nil?
end
return tmp
end
def cache_annc(force_refresh=false,locales=nil,trans=nil)
feed = SiteFeed.find(self.feed_id)
anns = feed.get_annc(force_refresh)
cat = self[:category_title]
feeds_time_field = self.feeds_time_field
site = Site.first
if locales.nil?
locales = Site.first.in_use_locales rescue I18n.available_locales
locales = site.in_use_locales rescue I18n.available_locales
end
if trans.nil?
trans = {}
@ -287,80 +272,117 @@ class SiteFeedAnnc
end
end
cache_data = {}
site_root_url = Site.first.root_url rescue ""
site_root_url = site.root_url rescue ""
locales.each do |locale|
locale = locale.to_s
data_for_locale = anns.collect do |a|
tmp = process_tmp(a, locale, trans, site_root_url)
tmp = process_tmp(site, a, locale, trans, site_root_url)
end
cache_data[locale.to_s] = data_for_locale
end
cache_data
end
def self.get_feed_cache(channel_key,merge_with_category=nil,site_source=nil,locale=I18n.locale.to_s,is_widget=false,max_len=nil)
locale = I18n.locale.to_s
max_len = ((max_len.to_i < 0 rescue true) ? 0 : max_len.to_i)
if max_len > 0
module_app = ModuleApp.where(:key=>channel_key).first
feeds_time_field = module_app.feeds_time_field rescue nil
if feeds_time_field.nil?
feeds_time_field = 'postdate'
def self.add_prefix(match_cond)
case match_cond
when Array
match_cond = match_cond.collect do |val|
self.add_prefix(val)
end
site_feed_anncs = self.where("channel_key" => channel_key)
if !merge_with_category.blank? && merge_with_category.exclude?('all')
site_feed_anncs = site_feed_anncs.where(:merge_with_category.in => merge_with_category)
end
if site_source
site_feed_anncs = site_feed_anncs.where({"channel_title.#{locale}" => site_source})
end
site_feed_annc_ids = site_feed_anncs.pluck(:id)
match_cond = {
"site_feed_annc_id" => {"$in" => site_feed_annc_ids},
"locale" => locale
}
pipeline = [
{"$match" => match_cond},
{"$project"=>{"data"=>1}},
{"$sort"=>{"data.is_hidden"=>-1,"data.is_top"=>-1,"data.#{feeds_time_field}"=>-1}},
{"$match"=>{"data.is_hidden"=>{"$ne"=>true},
"data.postdate"=>{"$lte"=>Time.now},
"data.title" => {"$gt"=>""}
}
}
]
if max_len
pipeline << {"$limit"=> max_len}
end
SiteFeedAnncDetail.collection.aggregate(pipeline).collect do |data|
tmp = data['data']
if !(tmp['category'].is_a? String)
tmp['category'] = tmp['category'][locale]
when Hash
match_cond = match_cond.inject({}) do |m, (key, val)|
if val.class == Hash || (val.class == Array && val[0].class == Hash)
val = self.add_prefix(val)
end
if @image_version != 'thumb' && !@image_version.nil?
if @image_version == 'mobile'
tmp["img_src"] = tmp["image"]["mobile"]
else
tmp["img_src"] = tmp["image"]["original"]
end
tmp["img_src"] = tmp["img_src"] || "/assets/announcement-default.jpg"
if !key.start_with?("$")
key = "data.#{key}"
end
if tmp["link_to_show"].nil?
if !is_widget
tmp["link_to_show"] = UseSourceUrl && tmp["show_url"] ? URI.join("#{tmp["source_url"]}/#{locale}",URI.encode("#{tmp["show_url"]}")).to_s : OrbitHelper.url_to_show(tmp["params"]) rescue ''
else
tmp["link_to_show"] = UseSourceUrl && tmp["show_url"] ? URI.join("#{tmp["source_url"]}/#{locale}",URI.encode("#{tmp["show_url"]}")).to_s : OrbitHelper.widget_item_url(tmp["params"]) rescue ''
end
end
tmp
m[key] = val
m
end
else
[]
end
return match_cond
end
def self.get_feed_cache(
channel_key,merge_with_category=nil,
site_source=nil,locale=I18n.locale.to_s,
is_widget=false,max_len=nil,sort_maps=nil,extra_match_cond=nil
)
locale = I18n.locale.to_s
max_len = max_len.to_i
module_app = ModuleApp.where(:key=>channel_key).first
feeds_time_field = module_app.feeds_time_field rescue nil
if feeds_time_field.nil?
feeds_time_field = 'postdate'
end
feeds_time_fields = Array(feeds_time_field)
site_feed_anncs = self.where("channel_key" => channel_key)
if !merge_with_category.blank? && merge_with_category.exclude?('all')
site_feed_anncs = site_feed_anncs.where(:merge_with_category.in => merge_with_category)
end
if site_source
site_feed_anncs = site_feed_anncs.where({"channel_title.#{locale}" => site_source})
end
site_feed_annc_ids = site_feed_anncs.pluck(:id)
match_cond = {
"site_feed_annc_id" => {"$in" => site_feed_annc_ids},
"locale" => locale
}
if extra_match_cond
match_cond = match_cond.merge(add_prefix(extra_match_cond))
end
pipeline = [
{"$match" => match_cond},
{"$project"=>{"data"=>1}}
]
pipeline_to_get_data = pipeline.clone
if sort_maps
sort_maps = sort_maps.collect do |sort_key, sort_val|
["data.#{sort_key}", sort_val == :desc ? -1 : 1]
end.to_h
pipeline_to_get_data << {"$sort"=>sort_maps}
end
if max_len > 0
pipeline_to_get_data << {"$limit"=> max_len}
end
pipeline_to_get_count = pipeline.clone
pipeline_to_get_count << {"$count" => "count"}
count_result = SiteFeedAnncDetail.collection.aggregate(pipeline_to_get_count).first
feeds_count = count_result ? count_result["count"] : 0
feeds = SiteFeedAnncDetail.collection.aggregate(pipeline_to_get_data).collect do |data|
tmp = data['data']
if !(tmp['category'].is_a? String)
tmp['category'] = tmp['category'][locale]
end
if @image_version != 'thumb' && !@image_version.nil?
if @image_version == 'mobile'
tmp["img_src"] = tmp["image"]["mobile"]
else
tmp["img_src"] = tmp["image"]["original"]
end
tmp["img_src"] = tmp["img_src"] || "/assets/announcement-default.jpg"
end
if tmp["link_to_show"].nil?
tmp["link_to_show"] = UseSourceUrl && tmp["show_url"] ?
URI.join("#{tmp["source_url"]}/#{locale}",URI.encode("#{tmp["show_url"]}")).to_s :
is_widget ? OrbitHelper.widget_item_url(tmp["params"]) : OrbitHelper.url_to_show(tmp["params"])
end
tmp
end
return feeds, feeds_count
end
# @deprecated Please use {#get_feed_cache} instead
def all_contents_for_feed(site_source=nil,locale=I18n.locale.to_s,is_widget=false)
if !site_source.nil? && site_source != self[:channel_title][locale]
return []

66
lib/feed_model/migrate.rb
View File

@ -1,66 +0,0 @@
module Migrate
def self.call
puts ['feeds migrate start']
gem_root = Feeds::Engine.root
require File.join(gem_root, 'app/models/site_feed')
require File.join(gem_root, 'app/models/site_feed_annc')
locales = Site.first.in_use_locales rescue I18n.available_locales
trans = {}
locales.each do |locale|
locale = locale.to_s
trans[locale] = {}
I18n.with_locale(locale) do
trans[locale]['top'] = I18n.t(:top)
trans[locale]['hot'] = I18n.t(:hot)
trans[locale]['more_plus'] = I18n.t("feed.more")
end
end
org_remote_site_urls = SiteFeed.all.pluck(:remote_site_url).uniq
all_remote_site_urls = org_remote_site_urls.map{|a| a.split('://')[1]}.uniq
all_remote_site_urls.each do |remote_site_url|
match_urls = org_remote_site_urls.select{|s| s.include?("://#{remote_site_url}")}
if match_urls.count == 2
SiteFeed.where(:remote_site_url.in=>match_urls).update_all(:remote_site_url=>"https://#{remote_site_url}")
SiteFeedAnnc.where(:remote_site_url.in=>match_urls).update_all(:remote_site_url=>"https://#{remote_site_url}")
end
end
SiteFeed.all.order_by(:channel_title=>-1).to_a.group_by(&:remote_site_url).each do |url, site_feeds|
site_feed = site_feeds.first
channel_title_translations = site_feed.channel_title_translations
SiteFeed.where(:id.in=>site_feeds[1..-1].map{|sf| sf.id}).update_all(:channel_title_translations => channel_title_translations)
end
SiteFeedAnnc.where(:feed_id.nin=>SiteFeed.all.pluck(:id)).destroy
SiteFeed.each do |site_feed|
site_feed.add_notify
count = SiteFeedAnnc.where(feed_id: site_feed.id).count
if count>1
SiteFeedAnnc.where(feed_id: site_feed.id).limit(count-1).destroy
end
tmp = SiteFeedAnnc.where(feed_id: site_feed.id).first
if site_feed.disabled != true
if tmp.nil?
tmp = SiteFeedAnnc.new(feed_id: site_feed.id)
end
site_feed.sync_data_to_annc(tmp)
tmp.raw_all_contents_for_feed_translations = tmp.cache_annc(false,locales,trans)
tmp.save
if site_feed.feeds_model && site_feed.feeds_finish_callback
feeds_model = site_feed.feeds_model.constantize
feeds_model.send(site_feed.feeds_finish_callback, "update_all", {"data"=>site_feed.get_annc(false),"feed_id"=>site_feed.id,"category_id"=>site_feed.merge_with_category})
end
elsif !tmp.nil?
tmp.destroy
end
end
SiteFeedAnnc.create_indexes
sync_fields = SiteFeed::ModuleAppSyncFields
ModuleApp.where(:feeds_model.ne=>nil).each do |module_app|
sync_fields_data = sync_fields.map{|f| [f, module_app[f]] }.to_h
SiteFeedAnnc.where(:channel_key=>module_app.key).update_all(sync_fields_data)
SiteFeed.where(:channel_key=>module_app.key).update_all(sync_fields_data)
end
SiteFeedAnnc.update_all({"$unset" => {"all_contents_for_feed" => ""}})
puts ['feeds migrate finished']
end
end

2
lib/feeds.rb
View File

@ -1,4 +1,6 @@
require "feeds/engine"
require "feeds/cache"
require "feeds/migrate"
module Feeds
end

2
lib/feed_model/cache.rb → lib/feeds/cache.rb
View File

@ -1,4 +1,4 @@
module FeedModel
module Feeds
module Cache
require 'active_support/concern'
extend ActiveSupport::Concern

3
lib/feeds/engine.rb
View File

@ -13,10 +13,9 @@ module Feeds
Thread.new do
begin
require File.join(gem_root, 'lib/feed_model/migrate')
Migrate.call
rescue => e
puts ['feed_routes',e]
puts ['feed_routes',e, e.backtrace]
end
end
end

76
lib/feeds/migrate.rb Normal file
View File

@ -0,0 +1,76 @@
module Feeds
module Migrate
def self.call
puts ['feeds migrate start']
gem_root = Feeds::Engine.root
require File.join(gem_root, 'app/models/site_feed')
require File.join(gem_root, 'app/models/site_feed_annc')
locales = Site.first.in_use_locales rescue I18n.available_locales
trans = {}
locales.each do |locale|
locale = locale.to_s
trans[locale] = {}
I18n.with_locale(locale) do
trans[locale]['top'] = I18n.t(:top)
trans[locale]['hot'] = I18n.t(:hot)
trans[locale]['more_plus'] = I18n.t("feed.more")
end
end
org_remote_site_urls = SiteFeed.all.pluck(:remote_site_url).uniq
all_remote_site_urls = org_remote_site_urls.map{|a| a.split('://')[1]}.uniq
all_remote_site_urls.each do |remote_site_url|
match_urls = org_remote_site_urls.select{|s| s.include?("://#{remote_site_url}")}
if match_urls.count == 2
SiteFeed.where(:remote_site_url.in=>match_urls).update_all(:remote_site_url=>"https://#{remote_site_url}")
SiteFeedAnnc.where(:remote_site_url.in=>match_urls).update_all(:remote_site_url=>"https://#{remote_site_url}")
end
end
SiteFeed.all.order_by(:channel_title=>-1).to_a.group_by(&:remote_site_url).each do |url, site_feeds|
site_feed = site_feeds.first
channel_title_translations = site_feed.channel_title_translations
SiteFeed.where(:id.in=>site_feeds[1..-1].map{|sf| sf.id}).update_all(:channel_title_translations => channel_title_translations)
end
SiteFeedAnnc.where(:feed_id.nin=>SiteFeed.all.pluck(:id)).delete
SiteFeed.all.each do |site_feed|
site_feed.add_notify
count = SiteFeedAnnc.where(feed_id: site_feed.id).count
if count>1
SiteFeedAnnc.where(feed_id: site_feed.id).limit(count-1).delete
end
tmp = SiteFeedAnnc.where(feed_id: site_feed.id).first
if site_feed.disabled != true
if tmp.nil?
tmp = SiteFeedAnnc.new(feed_id: site_feed.id)
end
site_feed.sync_data_to_annc(tmp)
tmp.raw_all_contents_for_feed_translations = tmp.cache_annc(false,locales,trans)
tmp.save
if site_feed.feeds_model && site_feed.feeds_finish_callback
feeds_model = site_feed.feeds_model.constantize
feeds_model.send(site_feed.feeds_finish_callback, "update_all", {"data"=>site_feed.get_annc(false),"feed_id"=>site_feed.id,"category_id"=>site_feed.merge_with_category})
end
elsif !tmp.nil?
tmp.destroy
end
end
SiteFeedAnncDetail.where(:site_feed_annc.nin => SiteFeedAnnc.all.pluck(:id)).delete
SiteFeedAnnc.create_indexes
self.sync_module_apps
SiteFeedAnnc.update_all({"$unset" => {"all_contents_for_feed" => ""}})
puts ['feeds migrate finished']
end
def self.sync_module_apps
sync_fields = SiteFeed::ModuleAppSyncFields
module_apps = ModuleApp.where(:feeds_model.ne=>nil)
module_apps.each do |module_app|
sync_fields_data = sync_fields.map{|f| [f, module_app[f]] }.to_h
SiteFeedAnnc.where(:channel_key=>module_app.key).update_all(sync_fields_data)
SiteFeed.where(:channel_key=>module_app.key).update_all(sync_fields_data)
end
end
end
end

2
lib/tasks/feeds_tasks.rake
View File

@ -5,7 +5,7 @@ namespace :feeds_module do
FileUtils.mkdir_p(main_directory) if !File.exist?(main_directory)
SiteFeedAnnc #Don't remove this, this will auto require SiteFeedAnnc
SiteFeed.enabled.each do |site_feed|
FeedModel::Cache.recreate_annc_cache(site_feed)
Feeds::Cache.recreate_annc_cache(site_feed)
end
end
end