From 10f998d08c112af8eebf14b97318ee6b42101a6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=B1=E5=8D=9A=E4=BA=9E?= Date: Mon, 19 Aug 2024 22:14:52 +0800 Subject: [PATCH] Fix Feed Annc all_contents_for_feed too big to save issue --- app/controllers/feeds_controller.rb | 30 ++++----- app/models/site_feed_annc.rb | 56 ++++++++++++---- app/models/site_feed_annc_detail.rb | 6 ++ app/views/admin/feeds/_announcements.html.erb | 2 +- config/routes.rb | 62 ----------------- lib/feed_model/cache.rb | 2 +- lib/feed_model/migrate.rb | 66 +++++++++++++++++++ lib/feeds/engine.rb | 9 +++ 8 files changed, 139 insertions(+), 94 deletions(-) create mode 100644 app/models/site_feed_annc_detail.rb create mode 100644 lib/feed_model/migrate.rb diff --git a/app/controllers/feeds_controller.rb b/app/controllers/feeds_controller.rb index da42b02..19fe233 100644 --- a/app/controllers/feeds_controller.rb +++ b/app/controllers/feeds_controller.rb @@ -4,6 +4,7 @@ class FeedsController < ApplicationController feed = SiteFeed.where(:feed_uid=>params[:uid]).first feed_annc = SiteFeedAnnc.where(:feed_id=>feed.id).first if feed_annc + raw_all_contents_for_feed_translations = feed_annc.raw_all_contents_for_feed_translations feeds_model = feed.feeds_model if feeds_model.present? feeds_model = feeds_model.constantize @@ -37,20 +38,18 @@ class FeedsController < ApplicationController a = JSON.parse(a) a["category_id"] = category_id locales.each do |locale| - locale_sym = locale.to_sym time_field_value = Time.parse(a[feeds_time_field]) rescue a[feeds_time_field] - if feed_annc[:all_contents_for_feed][locale_sym].index{|aa| aa["id"] == a["id"]} + if raw_all_contents_for_feed_translations[locale].index{|aa| aa["id"] == a["id"]} next end insert_idx = 0 if time_field_value - insert_idx = feed_annc[:all_contents_for_feed][locale_sym].index{|aa| (aa["org_is_top"] < a["org_is_top"] rescue false) || ((aa["org_is_top"] == a["org_is_top"]) && (aa[feeds_time_field] <= time_field_value))} + insert_idx = raw_all_contents_for_feed_translations[locale].index{|aa| (aa["org_is_top"] < a["org_is_top"] rescue false) || ((aa["org_is_top"] == a["org_is_top"]) && (aa[feeds_time_field] <= time_field_value))} if insert_idx.nil? insert_idx = ((a["org_is_top"] == 1) ? 0 : -1) end end - feed_annc.all_contents_for_feed_will_change! - feed_annc[:all_contents_for_feed][locale_sym].insert(insert_idx, feed_annc.process_tmp(a,locale,trans,site_root_url)) + raw_all_contents_for_feed_translations[locale].insert(insert_idx, feed_annc.process_tmp(a,locale,trans,site_root_url)) feed_data[channel_key_pluralize].insert(insert_idx, a) end if can_create_record @@ -66,6 +65,7 @@ class FeedsController < ApplicationController record.send(feeds_update_callback, a) end end + feed_annc.raw_all_contents_for_feed_translations = raw_all_contents_for_feed_translations feed_annc.instance_variable_set(:@skip_callback, true) feed_annc.save! elsif params[:type] == 'update' @@ -82,13 +82,11 @@ class FeedsController < ApplicationController a = JSON.parse(a) a["category_id"] = category_id locales.each do |locale| - locale_sym = locale.to_sym time_field_value = Time.parse(a[feeds_time_field]) rescue a[feeds_time_field] has_create = false - feed_annc[:all_contents_for_feed][locale_sym].each_with_index do |aa, i| + raw_all_contents_for_feed_translations[locale].each_with_index do |aa, i| if aa["id"] == a["id"] - feed_annc.all_contents_for_feed_will_change! - feed_annc[:all_contents_for_feed][locale_sym][i] = feed_annc.process_tmp(a,locale,trans,site_root_url) + raw_all_contents_for_feed_translations[locale][i] = feed_annc.process_tmp(a,locale,trans,site_root_url) feed_data[channel_key_pluralize][i] = a has_create = true break @@ -97,13 +95,12 @@ class FeedsController < ApplicationController unless has_create insert_idx = 0 if time_field_value - insert_idx = feed_annc[:all_contents_for_feed][locale_sym].index{|aa| (aa["org_is_top"] < a["org_is_top"] rescue false) || ((aa["org_is_top"] == a["org_is_top"]) && (aa[feeds_time_field] <= time_field_value))} + insert_idx = raw_all_contents_for_feed_translations[locale].index{|aa| (aa["org_is_top"] < a["org_is_top"] rescue false) || ((aa["org_is_top"] == a["org_is_top"]) && (aa[feeds_time_field] <= time_field_value))} if insert_idx.nil? insert_idx = ((a["org_is_top"] == 1) ? 0 : -1) end end - feed_annc.all_contents_for_feed_will_change! - feed_annc[:all_contents_for_feed][locale_sym].insert(insert_idx, feed_annc.process_tmp(a,locale,trans,site_root_url)) + raw_all_contents_for_feed_translations[locale].insert(insert_idx, feed_annc.process_tmp(a,locale,trans,site_root_url)) feed_data[channel_key_pluralize].insert(insert_idx, a) end end @@ -120,15 +117,15 @@ class FeedsController < ApplicationController record.send(feeds_update_callback, a) end end + feed_annc.raw_all_contents_for_feed_translations = raw_all_contents_for_feed_translations feed_annc.instance_variable_set(:@skip_callback, true) feed_annc.save! elsif params[:type] == 'destroy' locales.each do |locale| - locale_sym = locale.to_sym - feed_annc.all_contents_for_feed_will_change! - feed_annc[:all_contents_for_feed][locale_sym].reject!{|a| params[:data].include?(a["id"]) } + raw_all_contents_for_feed_translations[locale].reject!{|a| params[:data].include?(a["id"]) } feed_data[channel_key_pluralize].reject!{|a| params[:data].include?(a["id"]) } end + feed_annc.raw_all_contents_for_feed_translations = raw_all_contents_for_feed_translations feed_annc.instance_variable_set(:@skip_callback, true) feed_annc.save! if can_create_record @@ -142,8 +139,7 @@ class FeedsController < ApplicationController file.write(feed_data) end need_write = false - feed_annc.all_contents_for_feed_will_change! - feed_annc[:all_contents_for_feed] = feed_annc.cache_annc + feed_annc.raw_all_contents_for_feed_translations = feed_annc.cache_annc feed_annc.save! if feeds_finish_callback feeds_model.send(feeds_finish_callback, 'update_all', {"data"=>feed.get_annc(false),"feed_id"=>feed.id,"category_id"=>feed.merge_with_category}) diff --git a/app/models/site_feed_annc.rb b/app/models/site_feed_annc.rb index f3339fe..187437e 100644 --- a/app/models/site_feed_annc.rb +++ b/app/models/site_feed_annc.rb @@ -4,7 +4,6 @@ class SiteFeedAnnc UseSourceUrl = SiteFeedSetting.first.use_source_url rescue false field :top_list,type: Array,default: [] field :hot_list,type: Array,default: [] - field :all_contents_for_feed field :channel_key field :feed_id field :feed_name @@ -20,28 +19,35 @@ class SiteFeedAnnc field :feeds_time_field, type: String, default: "postdate" field :feeds_finish_callback field :feeds_update_statuses_callback - #I18n.available_locales.each do |locale| - # index({ "all_contents_for_feed.#{locale}.is_hidden"=> -1, - # "all_contents_for_feed.#{locale}.is_top"=> -1, - # "all_contents_for_feed.#{locale}.#{self.feeds_time_field}"=> -1}, { unique: false, background: true, name: "cache_#{locale}" }) - #end + + has_many :site_feed_annc_details, :autosave => true, :dependent => :destroy + def get_annc(annc_uid) - Array(self[:all_contents_for_feed][I18n.locale.to_s]).select{|v| v['id']==annc_uid}[0] rescue {} + raw_all_contents_for_feed(I18n.locale.to_s).select{|v| v['id']==annc_uid}[0] rescue {} end + before_save do unless @skip_callback if self.top_list_changed? || self.hot_list_changed? || self.hidden_annc_changed? || self.category_title_changed? - self[:all_contents_for_feed] = self.cache_annc + self.raw_all_contents_for_feed_translations = self.cache_annc + end + end + if @raw_all_contents_for_feed_translations_changed + self.site_feed_annc_details = [] + @raw_all_contents_for_feed_translations.each do |locale, contents| + contents.each do |content| + self.site_feed_annc_details << SiteFeedAnncDetail.new(locale: locale, site_feed_annc: self, data: content) + end end end end def update_remote_site_url update_fields = ['source_url','source-site'] tmp_url = self.remote_site_url - self.all_contents_for_feed_will_change! locales = Site.first.in_use_locales rescue I18n.available_locales + anns_translations = self.raw_all_contents_for_feed_translations locales.each do |l| - anns = self[:all_contents_for_feed][l.to_s] + anns = anns_translations[l.to_s] if anns (0...anns.count).each do |i| a = anns[i] @@ -70,6 +76,7 @@ class SiteFeedAnnc end end end + self.raw_all_contents_for_feed_translations = anns_translations self.save end def update_channel_title(update_url=false) #update_url=true will also fix remote_site_url in data @@ -78,7 +85,6 @@ class SiteFeedAnnc update_fields = ['source_url','source-site'] end tmp_url = self.remote_site_url - self.all_contents_for_feed_will_change! locales = Site.first.in_use_locales rescue I18n.available_locales trans = {} locales.each do |locale| @@ -90,9 +96,10 @@ class SiteFeedAnnc trans[locale]['more_plus'] = I18n.t("feed.more") end end + anns_translations = self.raw_all_contents_for_feed_translations locales.each do |locale| source_site_title = (self[:channel_title][locale] rescue "") - anns = self[:all_contents_for_feed][locale.to_s] + anns = anns_translations[locale] if anns (0...anns.count).each do |i| a = anns[i] @@ -122,6 +129,7 @@ class SiteFeedAnnc end end end + self.raw_all_contents_for_feed_translations = anns_translations self.save end def process_tmp(a, locale=nil, trans=nil, site_root_url=nil) @@ -352,7 +360,7 @@ class SiteFeedAnnc return [] end time_now = Time.now - Array(self[:all_contents_for_feed][locale.to_s]).collect do |v| + self.raw_all_contents_for_feed(locale.to_s).collect do |v| tmp = v next if tmp["is_hidden"] || (!tmp["postdate"].nil? && tmp["postdate"] <% feed_name = available_locales.collect{|v| feed_annc[:feed_name][v]}.join(' / ') %> <% if @source.blank? || (@source.include?(feed_annc.channel_title) rescue false) || @source.include?(feed_annc.remote_site_url.to_s.gsub(/http:\/\/|https:\/\//,'').gsub(/\./,'-')) %> - <% Array(feed_annc[:all_contents_for_feed][I18n.locale.to_s]).each do |annc| %> + <% feed_annc.raw_all_contents_for_feed(I18n.locale.to_s).each do |annc| %> <% annc_title = available_locales.collect{|v| annc['title_translations'][v]}.join('/') %> <% annc_tags = annc['tags'].collect{|tmp| available_locales.collect{|v| tmp['name_translations'][v]}.join(' / ')} %> <% if @keywords.blank? || search_all_words(Nokogiri.HTML(annc_title).text()+' '+feed_name+' '+annc_tags.join(' '),@keywords) %> diff --git a/config/routes.rb b/config/routes.rb index b65402b..e3eed35 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -1,68 +1,6 @@ Rails.application.routes.draw do locales = Site.first.in_use_locales rescue I18n.available_locales - Thread.new do - begin - if ENV['worker_num']=='0' && File.basename($0) != 'rake' && !Rails.const_defined?('Console') - trans = {} - locales.each do |locale| - locale = locale.to_s - trans[locale] = {} - I18n.with_locale(locale) do - trans[locale]['top'] = I18n.t(:top) - trans[locale]['hot'] = I18n.t(:hot) - trans[locale]['more_plus'] = I18n.t("feed.more") - end - end - org_remote_site_urls = SiteFeed.all.pluck(:remote_site_url).uniq - all_remote_site_urls = org_remote_site_urls.map{|a| a.split('://')[1]}.uniq - all_remote_site_urls.each do |remote_site_url| - match_urls = org_remote_site_urls.select{|s| s.include?("://#{remote_site_url}")} - if match_urls.count == 2 - SiteFeed.where(:remote_site_url.in=>match_urls).update_all(:remote_site_url=>"https://#{remote_site_url}") - SiteFeedAnnc.where(:remote_site_url.in=>match_urls).update_all(:remote_site_url=>"https://#{remote_site_url}") - end - end - SiteFeed.all.order_by(:channel_title=>-1).to_a.group_by(&:remote_site_url).each do |url, site_feeds| - site_feed = site_feeds.first - channel_title_translations = site_feed.channel_title_translations - SiteFeed.where(:id.in=>site_feeds[1..-1].map{|sf| sf.id}).update_all(:channel_title_translations => channel_title_translations) - end - SiteFeedAnnc.where(:feed_id.nin=>SiteFeed.all.pluck(:id)).destroy - SiteFeed.each do |site_feed| - site_feed.add_notify - count = SiteFeedAnnc.where(feed_id: site_feed.id).count - if count>1 - SiteFeedAnnc.where(feed_id: site_feed.id).limit(count-1).destroy - end - tmp = SiteFeedAnnc.where(feed_id: site_feed.id).first - if site_feed.disabled != true - if tmp.nil? - tmp = SiteFeedAnnc.new(feed_id: site_feed.id) - end - site_feed.sync_data_to_annc(tmp) - tmp.all_contents_for_feed = tmp.cache_annc(false,locales,trans) - tmp.save - if site_feed.feeds_model && site_feed.feeds_finish_callback - feeds_model = site_feed.feeds_model.constantize - feeds_model.send(site_feed.feeds_finish_callback, "update_all", {"data"=>site_feed.get_annc(false),"feed_id"=>site_feed.id,"category_id"=>site_feed.merge_with_category}) - end - elsif !tmp.nil? - tmp.destroy - end - end - SiteFeedAnnc.create_indexes - sync_fields = SiteFeed::ModuleAppSyncFields - ModuleApp.where(:feeds_model.ne=>nil).each do |module_app| - sync_fields_data = sync_fields.map{|f| [f, module_app[f]] }.to_h - SiteFeedAnnc.where(:channel_key=>module_app.key).update_all(sync_fields_data) - SiteFeed.where(:channel_key=>module_app.key).update_all(sync_fields_data) - end - end - rescue => e - puts ['feed_routes',e] - end - end scope "(:locale)", locale: Regexp.new(locales.join("|")) do namespace :admin do get "/feeds/get_channel_list", to: 'feeds#get_channel_list' diff --git a/lib/feed_model/cache.rb b/lib/feed_model/cache.rb index 17d812d..0a63f0c 100644 --- a/lib/feed_model/cache.rb +++ b/lib/feed_model/cache.rb @@ -15,7 +15,7 @@ module FeedModel if tmp.nil? tmp = SiteFeedAnnc.new(feed_id: feed.id) end - tmp.all_contents_for_feed = tmp.cache_annc(true) + tmp.raw_all_contents_for_feed_translations = tmp.cache_annc(true) feed.sync_data_to_annc(tmp) tmp.save if feed.feeds_model && feed.feeds_finish_callback diff --git a/lib/feed_model/migrate.rb b/lib/feed_model/migrate.rb new file mode 100644 index 0000000..351a5c8 --- /dev/null +++ b/lib/feed_model/migrate.rb @@ -0,0 +1,66 @@ +module Migrate + def self.call + puts ['feeds migrate start'] + gem_root = Feeds::Engine.root + require File.join(gem_root, 'app/models/site_feed') + require File.join(gem_root, 'app/models/site_feed_annc') + + locales = Site.first.in_use_locales rescue I18n.available_locales + trans = {} + locales.each do |locale| + locale = locale.to_s + trans[locale] = {} + I18n.with_locale(locale) do + trans[locale]['top'] = I18n.t(:top) + trans[locale]['hot'] = I18n.t(:hot) + trans[locale]['more_plus'] = I18n.t("feed.more") + end + end + org_remote_site_urls = SiteFeed.all.pluck(:remote_site_url).uniq + all_remote_site_urls = org_remote_site_urls.map{|a| a.split('://')[1]}.uniq + all_remote_site_urls.each do |remote_site_url| + match_urls = org_remote_site_urls.select{|s| s.include?("://#{remote_site_url}")} + if match_urls.count == 2 + SiteFeed.where(:remote_site_url.in=>match_urls).update_all(:remote_site_url=>"https://#{remote_site_url}") + SiteFeedAnnc.where(:remote_site_url.in=>match_urls).update_all(:remote_site_url=>"https://#{remote_site_url}") + end + end + SiteFeed.all.order_by(:channel_title=>-1).to_a.group_by(&:remote_site_url).each do |url, site_feeds| + site_feed = site_feeds.first + channel_title_translations = site_feed.channel_title_translations + SiteFeed.where(:id.in=>site_feeds[1..-1].map{|sf| sf.id}).update_all(:channel_title_translations => channel_title_translations) + end + SiteFeedAnnc.where(:feed_id.nin=>SiteFeed.all.pluck(:id)).destroy + SiteFeed.each do |site_feed| + site_feed.add_notify + count = SiteFeedAnnc.where(feed_id: site_feed.id).count + if count>1 + SiteFeedAnnc.where(feed_id: site_feed.id).limit(count-1).destroy + end + tmp = SiteFeedAnnc.where(feed_id: site_feed.id).first + if site_feed.disabled != true + if tmp.nil? + tmp = SiteFeedAnnc.new(feed_id: site_feed.id) + end + site_feed.sync_data_to_annc(tmp) + tmp.raw_all_contents_for_feed_translations = tmp.cache_annc(false,locales,trans) + tmp.save + if site_feed.feeds_model && site_feed.feeds_finish_callback + feeds_model = site_feed.feeds_model.constantize + feeds_model.send(site_feed.feeds_finish_callback, "update_all", {"data"=>site_feed.get_annc(false),"feed_id"=>site_feed.id,"category_id"=>site_feed.merge_with_category}) + end + elsif !tmp.nil? + tmp.destroy + end + end + SiteFeedAnnc.create_indexes + sync_fields = SiteFeed::ModuleAppSyncFields + ModuleApp.where(:feeds_model.ne=>nil).each do |module_app| + sync_fields_data = sync_fields.map{|f| [f, module_app[f]] }.to_h + SiteFeedAnnc.where(:channel_key=>module_app.key).update_all(sync_fields_data) + SiteFeed.where(:channel_key=>module_app.key).update_all(sync_fields_data) + end + SiteFeedAnnc.update_all({"$unset" => {"all_contents_for_feed" => ""}}) + puts ['feeds migrate finished'] + end +end \ No newline at end of file diff --git a/lib/feeds/engine.rb b/lib/feeds/engine.rb index fba8415..0a2de02 100644 --- a/lib/feeds/engine.rb +++ b/lib/feeds/engine.rb @@ -10,6 +10,15 @@ module Feeds SiteFeedSetting.create end end + + Thread.new do + begin + require File.join(gem_root, 'lib/feed_model/migrate') + Migrate.call + rescue => e + puts ['feed_routes',e] + end + end end OrbitApp.registration "Feeds", :type => "ModuleApp" do module_label "feed.feed"