harry
/
feeds
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix Feed Annc all_contents_for_feed too big to save issue

This commit is contained in:
邱博亞 2024-08-19 22:14:52 +08:00
parent fa83427442
commit 10f998d08c
8 changed files with 139 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
app/controllers/feeds_controller.rb
View File

@ -4,6 +4,7 @@ class FeedsController < ApplicationController
feed = SiteFeed.where(:feed_uid=>params[:uid]).first feed = SiteFeed.where(:feed_uid=>params[:uid]).first
feed_annc = SiteFeedAnnc.where(:feed_id=>feed.id).first feed_annc = SiteFeedAnnc.where(:feed_id=>feed.id).first
if feed_annc if feed_annc
raw_all_contents_for_feed_translations = feed_annc.raw_all_contents_for_feed_translations
feeds_model = feed.feeds_model feeds_model = feed.feeds_model
if feeds_model.present? if feeds_model.present?
feeds_model = feeds_model.constantize feeds_model = feeds_model.constantize
@ -37,20 +38,18 @@ class FeedsController < ApplicationController
a = JSON.parse(a) a = JSON.parse(a)
a["category_id"] = category_id a["category_id"] = category_id
locales.each do |locale| locales.each do |locale|
locale_sym = locale.to_sym
time_field_value = Time.parse(a[feeds_time_field]) rescue a[feeds_time_field] time_field_value = Time.parse(a[feeds_time_field]) rescue a[feeds_time_field]
if feed_annc[:all_contents_for_feed][locale_sym].index{|aa| aa["id"] == a["id"]} if raw_all_contents_for_feed_translations[locale].index{|aa| aa["id"] == a["id"]}
next next
end end
insert_idx = 0 insert_idx = 0
if time_field_value if time_field_value
insert_idx = feed_annc[:all_contents_for_feed][locale_sym].index{|aa| (aa["org_is_top"] < a["org_is_top"] rescue false) || ((aa["org_is_top"] == a["org_is_top"]) && (aa[feeds_time_field] <= time_field_value))} insert_idx = raw_all_contents_for_feed_translations[locale].index{|aa| (aa["org_is_top"] < a["org_is_top"] rescue false) || ((aa["org_is_top"] == a["org_is_top"]) && (aa[feeds_time_field] <= time_field_value))}
if insert_idx.nil? if insert_idx.nil?
insert_idx = ((a["org_is_top"] == 1) ? 0 : -1) insert_idx = ((a["org_is_top"] == 1) ? 0 : -1)
end end
end end
feed_annc.all_contents_for_feed_will_change! raw_all_contents_for_feed_translations[locale].insert(insert_idx, feed_annc.process_tmp(a,locale,trans,site_root_url))
feed_annc[:all_contents_for_feed][locale_sym].insert(insert_idx, feed_annc.process_tmp(a,locale,trans,site_root_url))
feed_data[channel_key_pluralize].insert(insert_idx, a) feed_data[channel_key_pluralize].insert(insert_idx, a)
end end
if can_create_record if can_create_record
@ -66,6 +65,7 @@ class FeedsController < ApplicationController
record.send(feeds_update_callback, a) record.send(feeds_update_callback, a)
end end
end end
feed_annc.raw_all_contents_for_feed_translations = raw_all_contents_for_feed_translations
feed_annc.instance_variable_set(:@skip_callback, true) feed_annc.instance_variable_set(:@skip_callback, true)
feed_annc.save! feed_annc.save!
elsif params[:type] == 'update' elsif params[:type] == 'update'
@ -82,13 +82,11 @@ class FeedsController < ApplicationController
a = JSON.parse(a) a = JSON.parse(a)
a["category_id"] = category_id a["category_id"] = category_id
locales.each do |locale| locales.each do |locale|
locale_sym = locale.to_sym
time_field_value = Time.parse(a[feeds_time_field]) rescue a[feeds_time_field] time_field_value = Time.parse(a[feeds_time_field]) rescue a[feeds_time_field]
has_create = false has_create = false
feed_annc[:all_contents_for_feed][locale_sym].each_with_index do |aa, i| raw_all_contents_for_feed_translations[locale].each_with_index do |aa, i|
if aa["id"] == a["id"] if aa["id"] == a["id"]
feed_annc.all_contents_for_feed_will_change! raw_all_contents_for_feed_translations[locale][i] = feed_annc.process_tmp(a,locale,trans,site_root_url)
feed_annc[:all_contents_for_feed][locale_sym][i] = feed_annc.process_tmp(a,locale,trans,site_root_url)
feed_data[channel_key_pluralize][i] = a feed_data[channel_key_pluralize][i] = a
has_create = true has_create = true
break break
@ -97,13 +95,12 @@ class FeedsController < ApplicationController
unless has_create unless has_create
insert_idx = 0 insert_idx = 0
if time_field_value if time_field_value
insert_idx = feed_annc[:all_contents_for_feed][locale_sym].index{|aa| (aa["org_is_top"] < a["org_is_top"] rescue false) || ((aa["org_is_top"] == a["org_is_top"]) && (aa[feeds_time_field] <= time_field_value))} insert_idx = raw_all_contents_for_feed_translations[locale].index{|aa| (aa["org_is_top"] < a["org_is_top"] rescue false) || ((aa["org_is_top"] == a["org_is_top"]) && (aa[feeds_time_field] <= time_field_value))}
if insert_idx.nil? if insert_idx.nil?
insert_idx = ((a["org_is_top"] == 1) ? 0 : -1) insert_idx = ((a["org_is_top"] == 1) ? 0 : -1)
end end
end end
feed_annc.all_contents_for_feed_will_change! raw_all_contents_for_feed_translations[locale].insert(insert_idx, feed_annc.process_tmp(a,locale,trans,site_root_url))
feed_annc[:all_contents_for_feed][locale_sym].insert(insert_idx, feed_annc.process_tmp(a,locale,trans,site_root_url))
feed_data[channel_key_pluralize].insert(insert_idx, a) feed_data[channel_key_pluralize].insert(insert_idx, a)
end end
end end
@ -120,15 +117,15 @@ class FeedsController < ApplicationController
record.send(feeds_update_callback, a) record.send(feeds_update_callback, a)
end end
end end
feed_annc.raw_all_contents_for_feed_translations = raw_all_contents_for_feed_translations
feed_annc.instance_variable_set(:@skip_callback, true) feed_annc.instance_variable_set(:@skip_callback, true)
feed_annc.save! feed_annc.save!
elsif params[:type] == 'destroy' elsif params[:type] == 'destroy'
locales.each do |locale| locales.each do |locale|
locale_sym = locale.to_sym raw_all_contents_for_feed_translations[locale].reject!{|a| params[:data].include?(a["id"]) }
feed_annc.all_contents_for_feed_will_change!
feed_annc[:all_contents_for_feed][locale_sym].reject!{|a| params[:data].include?(a["id"]) }
feed_data[channel_key_pluralize].reject!{|a| params[:data].include?(a["id"]) } feed_data[channel_key_pluralize].reject!{|a| params[:data].include?(a["id"]) }
end end
feed_annc.raw_all_contents_for_feed_translations = raw_all_contents_for_feed_translations
feed_annc.instance_variable_set(:@skip_callback, true) feed_annc.instance_variable_set(:@skip_callback, true)
feed_annc.save! feed_annc.save!
if can_create_record if can_create_record
@ -142,8 +139,7 @@ class FeedsController < ApplicationController
file.write(feed_data) file.write(feed_data)
end end
need_write = false need_write = false
feed_annc.all_contents_for_feed_will_change! feed_annc.raw_all_contents_for_feed_translations = feed_annc.cache_annc
feed_annc[:all_contents_for_feed] = feed_annc.cache_annc
feed_annc.save! feed_annc.save!
if feeds_finish_callback if feeds_finish_callback
feeds_model.send(feeds_finish_callback, 'update_all', {"data"=>feed.get_annc(false),"feed_id"=>feed.id,"category_id"=>feed.merge_with_category}) feeds_model.send(feeds_finish_callback, 'update_all', {"data"=>feed.get_annc(false),"feed_id"=>feed.id,"category_id"=>feed.merge_with_category})

56
app/models/site_feed_annc.rb
View File

@ -4,7 +4,6 @@ class SiteFeedAnnc
UseSourceUrl = SiteFeedSetting.first.use_source_url rescue false UseSourceUrl = SiteFeedSetting.first.use_source_url rescue false
field :top_list,type: Array,default: [] field :top_list,type: Array,default: []
field :hot_list,type: Array,default: [] field :hot_list,type: Array,default: []
field :all_contents_for_feed
field :channel_key field :channel_key
field :feed_id field :feed_id
field :feed_name field :feed_name
@ -20,28 +19,35 @@ class SiteFeedAnnc
field :feeds_time_field, type: String, default: "postdate" field :feeds_time_field, type: String, default: "postdate"
field :feeds_finish_callback field :feeds_finish_callback
field :feeds_update_statuses_callback field :feeds_update_statuses_callback
#I18n.available_locales.each do |locale|
# index({ "all_contents_for_feed.#{locale}.is_hidden"=> -1, has_many :site_feed_annc_details, :autosave => true, :dependent => :destroy
# "all_contents_for_feed.#{locale}.is_top"=> -1,
# "all_contents_for_feed.#{locale}.#{self.feeds_time_field}"=> -1}, { unique: false, background: true, name: "cache_#{locale}" })
#end
def get_annc(annc_uid) def get_annc(annc_uid)
Array(self[:all_contents_for_feed][I18n.locale.to_s]).select{|v| v['id']==annc_uid}[0] rescue {} raw_all_contents_for_feed(I18n.locale.to_s).select{|v| v['id']==annc_uid}[0] rescue {}
end end
before_save do before_save do
unless @skip_callback unless @skip_callback
if self.top_list_changed? || self.hot_list_changed? || self.hidden_annc_changed? || self.category_title_changed? if self.top_list_changed? || self.hot_list_changed? || self.hidden_annc_changed? || self.category_title_changed?
self[:all_contents_for_feed] = self.cache_annc self.raw_all_contents_for_feed_translations = self.cache_annc
end
end
if @raw_all_contents_for_feed_translations_changed
self.site_feed_annc_details = []
@raw_all_contents_for_feed_translations.each do |locale, contents|
contents.each do |content|
self.site_feed_annc_details << SiteFeedAnncDetail.new(locale: locale, site_feed_annc: self, data: content)
end
end end
end end
end end
def update_remote_site_url def update_remote_site_url
update_fields = ['source_url','source-site'] update_fields = ['source_url','source-site']
tmp_url = self.remote_site_url tmp_url = self.remote_site_url
self.all_contents_for_feed_will_change!
locales = Site.first.in_use_locales rescue I18n.available_locales locales = Site.first.in_use_locales rescue I18n.available_locales
anns_translations = self.raw_all_contents_for_feed_translations
locales.each do |l| locales.each do |l|
anns = self[:all_contents_for_feed][l.to_s] anns = anns_translations[l.to_s]
if anns if anns
(0...anns.count).each do |i| (0...anns.count).each do |i|
a = anns[i] a = anns[i]
@ -70,6 +76,7 @@ class SiteFeedAnnc
end end
end end
end end
self.raw_all_contents_for_feed_translations = anns_translations
self.save self.save
end end
def update_channel_title(update_url=false) #update_url=true will also fix remote_site_url in data def update_channel_title(update_url=false) #update_url=true will also fix remote_site_url in data
@ -78,7 +85,6 @@ class SiteFeedAnnc
update_fields = ['source_url','source-site'] update_fields = ['source_url','source-site']
end end
tmp_url = self.remote_site_url tmp_url = self.remote_site_url
self.all_contents_for_feed_will_change!
locales = Site.first.in_use_locales rescue I18n.available_locales locales = Site.first.in_use_locales rescue I18n.available_locales
trans = {} trans = {}
locales.each do |locale| locales.each do |locale|
@ -90,9 +96,10 @@ class SiteFeedAnnc
trans[locale]['more_plus'] = I18n.t("feed.more") trans[locale]['more_plus'] = I18n.t("feed.more")
end end
end end
anns_translations = self.raw_all_contents_for_feed_translations
locales.each do |locale| locales.each do |locale|
source_site_title = (self[:channel_title][locale] rescue "") source_site_title = (self[:channel_title][locale] rescue "")
anns = self[:all_contents_for_feed][locale.to_s] anns = anns_translations[locale]
if anns if anns
(0...anns.count).each do |i| (0...anns.count).each do |i|
a = anns[i] a = anns[i]
@ -122,6 +129,7 @@ class SiteFeedAnnc
end end
end end
end end
self.raw_all_contents_for_feed_translations = anns_translations
self.save self.save
end end
def process_tmp(a, locale=nil, trans=nil, site_root_url=nil) def process_tmp(a, locale=nil, trans=nil, site_root_url=nil)
@ -352,7 +360,7 @@ class SiteFeedAnnc
return [] return []
end end
time_now = Time.now time_now = Time.now
Array(self[:all_contents_for_feed][locale.to_s]).collect do |v| self.raw_all_contents_for_feed(locale.to_s).collect do |v|
tmp = v tmp = v
next if tmp["is_hidden"] || (!tmp["postdate"].nil? && tmp["postdate"]<Time.now) || tmp['title'].blank? next if tmp["is_hidden"] || (!tmp["postdate"].nil? && tmp["postdate"]<Time.now) || tmp['title'].blank?
tmp['category'] = tmp['category'][locale] tmp['category'] = tmp['category'][locale]
@ -374,6 +382,28 @@ class SiteFeedAnnc
tmp tmp
end.compact end.compact
end end
def raw_all_contents_for_feed(locale)
SiteFeedAnncDetail.where(site_feed_annc: self, locale: locale).collect{|detail|
detail.data
}
end
def raw_all_contents_for_feed_translations
(@raw_all_contents_for_feed_translations ||= SiteFeedAnncDetail.where(site_feed_annc: self).group_by{|detail|
detail.locale
}.transform_values do |details|
details.collect{|detail| detail.data}
end).deep_dup
end
def raw_all_contents_for_feed_translations=(translations)
if raw_all_contents_for_feed_translations != translations
@raw_all_contents_for_feed_translations_changed = true
@raw_all_contents_for_feed_translations = translations
end
end
def channel_title(locale=I18n.locale) def channel_title(locale=I18n.locale)
self[:channel_title][locale] rescue self[:channel_title] self[:channel_title][locale] rescue self[:channel_title]
end end

6
app/models/site_feed_annc_detail.rb Normal file
View File

@ -0,0 +1,6 @@
class SiteFeedAnncDetail
include Mongoid::Document
field :locale, type: String
belongs_to :site_feed_annc
field :data
end

2
app/views/admin/feeds/_announcements.html.erb
View File

@ -15,7 +15,7 @@
<% @all_feed_annc.each do |feed_annc| %> <% @all_feed_annc.each do |feed_annc| %>
<% feed_name = available_locales.collect{|v| feed_annc[:feed_name][v]}.join(' / ') %> <% feed_name = available_locales.collect{|v| feed_annc[:feed_name][v]}.join(' / ') %>
<% if @source.blank? || (@source.include?(feed_annc.channel_title) rescue false) || @source.include?(feed_annc.remote_site_url.to_s.gsub(/http:\/\/|https:\/\//,'').gsub(/\./,'-')) %> <% if @source.blank? || (@source.include?(feed_annc.channel_title) rescue false) || @source.include?(feed_annc.remote_site_url.to_s.gsub(/http:\/\/|https:\/\//,'').gsub(/\./,'-')) %>
<% Array(feed_annc[:all_contents_for_feed][I18n.locale.to_s]).each do |annc| %> <% feed_annc.raw_all_contents_for_feed(I18n.locale.to_s).each do |annc| %>
<% annc_title = available_locales.collect{|v| annc['title_translations'][v]}.join('/') %> <% annc_title = available_locales.collect{|v| annc['title_translations'][v]}.join('/') %>
<% annc_tags = annc['tags'].collect{|tmp| available_locales.collect{|v| tmp['name_translations'][v]}.join(' / ')} %> <% annc_tags = annc['tags'].collect{|tmp| available_locales.collect{|v| tmp['name_translations'][v]}.join(' / ')} %>
<% if @keywords.blank? || search_all_words(Nokogiri.HTML(annc_title).text()+' '+feed_name+' '+annc_tags.join(' '),@keywords) %> <% if @keywords.blank? || search_all_words(Nokogiri.HTML(annc_title).text()+' '+feed_name+' '+annc_tags.join(' '),@keywords) %>

62
config/routes.rb
View File

@ -1,68 +1,6 @@
Rails.application.routes.draw do Rails.application.routes.draw do
locales = Site.first.in_use_locales rescue I18n.available_locales locales = Site.first.in_use_locales rescue I18n.available_locales
Thread.new do
begin
if ENV['worker_num']=='0' && File.basename($0) != 'rake' && !Rails.const_defined?('Console')
trans = {}
locales.each do |locale|
locale = locale.to_s
trans[locale] = {}
I18n.with_locale(locale) do
trans[locale]['top'] = I18n.t(:top)
trans[locale]['hot'] = I18n.t(:hot)
trans[locale]['more_plus'] = I18n.t("feed.more")
end
end
org_remote_site_urls = SiteFeed.all.pluck(:remote_site_url).uniq
all_remote_site_urls = org_remote_site_urls.map{|a| a.split('://')[1]}.uniq
all_remote_site_urls.each do |remote_site_url|
match_urls = org_remote_site_urls.select{|s| s.include?("://#{remote_site_url}")}
if match_urls.count == 2
SiteFeed.where(:remote_site_url.in=>match_urls).update_all(:remote_site_url=>"https://#{remote_site_url}")
SiteFeedAnnc.where(:remote_site_url.in=>match_urls).update_all(:remote_site_url=>"https://#{remote_site_url}")
end
end
SiteFeed.all.order_by(:channel_title=>-1).to_a.group_by(&:remote_site_url).each do |url, site_feeds|
site_feed = site_feeds.first
channel_title_translations = site_feed.channel_title_translations
SiteFeed.where(:id.in=>site_feeds[1..-1].map{|sf| sf.id}).update_all(:channel_title_translations => channel_title_translations)
end
SiteFeedAnnc.where(:feed_id.nin=>SiteFeed.all.pluck(:id)).destroy
SiteFeed.each do |site_feed|
site_feed.add_notify
count = SiteFeedAnnc.where(feed_id: site_feed.id).count
if count>1
SiteFeedAnnc.where(feed_id: site_feed.id).limit(count-1).destroy
end
tmp = SiteFeedAnnc.where(feed_id: site_feed.id).first
if site_feed.disabled != true
if tmp.nil?
tmp = SiteFeedAnnc.new(feed_id: site_feed.id)
end
site_feed.sync_data_to_annc(tmp)
tmp.all_contents_for_feed = tmp.cache_annc(false,locales,trans)
tmp.save
if site_feed.feeds_model && site_feed.feeds_finish_callback
feeds_model = site_feed.feeds_model.constantize
feeds_model.send(site_feed.feeds_finish_callback, "update_all", {"data"=>site_feed.get_annc(false),"feed_id"=>site_feed.id,"category_id"=>site_feed.merge_with_category})
end
elsif !tmp.nil?
tmp.destroy
end
end
SiteFeedAnnc.create_indexes
sync_fields = SiteFeed::ModuleAppSyncFields
ModuleApp.where(:feeds_model.ne=>nil).each do |module_app|
sync_fields_data = sync_fields.map{|f| [f, module_app[f]] }.to_h
SiteFeedAnnc.where(:channel_key=>module_app.key).update_all(sync_fields_data)
SiteFeed.where(:channel_key=>module_app.key).update_all(sync_fields_data)
end
end
rescue => e
puts ['feed_routes',e]
end
end
scope "(:locale)", locale: Regexp.new(locales.join("|")) do scope "(:locale)", locale: Regexp.new(locales.join("|")) do
namespace :admin do namespace :admin do
get "/feeds/get_channel_list", to: 'feeds#get_channel_list' get "/feeds/get_channel_list", to: 'feeds#get_channel_list'

2
lib/feed_model/cache.rb
View File

@ -15,7 +15,7 @@ module FeedModel
if tmp.nil? if tmp.nil?
tmp = SiteFeedAnnc.new(feed_id: feed.id) tmp = SiteFeedAnnc.new(feed_id: feed.id)
end end
tmp.all_contents_for_feed = tmp.cache_annc(true) tmp.raw_all_contents_for_feed_translations = tmp.cache_annc(true)
feed.sync_data_to_annc(tmp) feed.sync_data_to_annc(tmp)
tmp.save tmp.save
if feed.feeds_model && feed.feeds_finish_callback if feed.feeds_model && feed.feeds_finish_callback

66
lib/feed_model/migrate.rb Normal file
View File

@ -0,0 +1,66 @@
module Migrate
def self.call
puts ['feeds migrate start']
gem_root = Feeds::Engine.root
require File.join(gem_root, 'app/models/site_feed')
require File.join(gem_root, 'app/models/site_feed_annc')
locales = Site.first.in_use_locales rescue I18n.available_locales
trans = {}
locales.each do |locale|
locale = locale.to_s
trans[locale] = {}
I18n.with_locale(locale) do
trans[locale]['top'] = I18n.t(:top)
trans[locale]['hot'] = I18n.t(:hot)
trans[locale]['more_plus'] = I18n.t("feed.more")
end
end
org_remote_site_urls = SiteFeed.all.pluck(:remote_site_url).uniq
all_remote_site_urls = org_remote_site_urls.map{|a| a.split('://')[1]}.uniq
all_remote_site_urls.each do |remote_site_url|
match_urls = org_remote_site_urls.select{|s| s.include?("://#{remote_site_url}")}
if match_urls.count == 2
SiteFeed.where(:remote_site_url.in=>match_urls).update_all(:remote_site_url=>"https://#{remote_site_url}")
SiteFeedAnnc.where(:remote_site_url.in=>match_urls).update_all(:remote_site_url=>"https://#{remote_site_url}")
end
end
SiteFeed.all.order_by(:channel_title=>-1).to_a.group_by(&:remote_site_url).each do |url, site_feeds|
site_feed = site_feeds.first
channel_title_translations = site_feed.channel_title_translations
SiteFeed.where(:id.in=>site_feeds[1..-1].map{|sf| sf.id}).update_all(:channel_title_translations => channel_title_translations)
end
SiteFeedAnnc.where(:feed_id.nin=>SiteFeed.all.pluck(:id)).destroy
SiteFeed.each do |site_feed|
site_feed.add_notify
count = SiteFeedAnnc.where(feed_id: site_feed.id).count
if count>1
SiteFeedAnnc.where(feed_id: site_feed.id).limit(count-1).destroy
end
tmp = SiteFeedAnnc.where(feed_id: site_feed.id).first
if site_feed.disabled != true
if tmp.nil?
tmp = SiteFeedAnnc.new(feed_id: site_feed.id)
end
site_feed.sync_data_to_annc(tmp)
tmp.raw_all_contents_for_feed_translations = tmp.cache_annc(false,locales,trans)
tmp.save
if site_feed.feeds_model && site_feed.feeds_finish_callback
feeds_model = site_feed.feeds_model.constantize
feeds_model.send(site_feed.feeds_finish_callback, "update_all", {"data"=>site_feed.get_annc(false),"feed_id"=>site_feed.id,"category_id"=>site_feed.merge_with_category})
end
elsif !tmp.nil?
tmp.destroy
end
end
SiteFeedAnnc.create_indexes
sync_fields = SiteFeed::ModuleAppSyncFields
ModuleApp.where(:feeds_model.ne=>nil).each do |module_app|
sync_fields_data = sync_fields.map{|f| [f, module_app[f]] }.to_h
SiteFeedAnnc.where(:channel_key=>module_app.key).update_all(sync_fields_data)
SiteFeed.where(:channel_key=>module_app.key).update_all(sync_fields_data)
end
SiteFeedAnnc.update_all({"$unset" => {"all_contents_for_feed" => ""}})
puts ['feeds migrate finished']
end
end

9
lib/feeds/engine.rb
View File

@ -10,6 +10,15 @@ module Feeds
SiteFeedSetting.create SiteFeedSetting.create
end end
end end
Thread.new do
begin
require File.join(gem_root, 'lib/feed_model/migrate')
Migrate.call
rescue => e
puts ['feed_routes',e]
end
end
end end
OrbitApp.registration "Feeds", :type => "ModuleApp" do OrbitApp.registration "Feeds", :type => "ModuleApp" do
module_label "feed.feed" module_label "feed.feed"