From 16963a9cb2466ddc08fcfce6b0ea59adc7fe51f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=B1=E5=8D=9A=E4=BA=9E?= Date: Sun, 13 Oct 2024 21:47:51 +0800 Subject: [PATCH] 1. accelerate feed and search 2. fix feed error --- app/controllers/admin/feeds_controller.rb | 2 +- app/controllers/feeds_controller.rb | 11 +- app/models/site_feed.rb | 11 +- app/models/site_feed_annc.rb | 296 ++++++++++++---------- lib/feed_model/migrate.rb | 66 ----- lib/feeds.rb | 2 + lib/{feed_model => feeds}/cache.rb | 2 +- lib/feeds/engine.rb | 3 +- lib/feeds/migrate.rb | 76 ++++++ lib/tasks/feeds_tasks.rake | 2 +- 10 files changed, 252 insertions(+), 219 deletions(-) delete mode 100644 lib/feed_model/migrate.rb rename lib/{feed_model => feeds}/cache.rb (98%) create mode 100644 lib/feeds/migrate.rb diff --git a/app/controllers/admin/feeds_controller.rb b/app/controllers/admin/feeds_controller.rb index b786e12..6790303 100644 --- a/app/controllers/admin/feeds_controller.rb +++ b/app/controllers/admin/feeds_controller.rb @@ -267,7 +267,7 @@ class Admin::FeedsController < OrbitAdminController def force_refresh sf = SiteFeed.find_by(:feed_uid => params[:feed_uid]) rescue nil if !sf.nil? - FeedModel::Cache.recreate_annc_cache(sf) + Feeds::Cache.recreate_annc_cache(sf) end render :json => {"success" => true}.to_json end diff --git a/app/controllers/feeds_controller.rb b/app/controllers/feeds_controller.rb index 19fe233..5bed69a 100644 --- a/app/controllers/feeds_controller.rb +++ b/app/controllers/feeds_controller.rb @@ -3,6 +3,7 @@ class FeedsController < ApplicationController def notify_change feed = SiteFeed.where(:feed_uid=>params[:uid]).first feed_annc = SiteFeedAnnc.where(:feed_id=>feed.id).first + site = Site.first if feed_annc raw_all_contents_for_feed_translations = feed_annc.raw_all_contents_for_feed_translations feeds_model = feed.feeds_model @@ -16,9 +17,9 @@ class FeedsController < ApplicationController feeds_finish_callback = feed.feeds_finish_callback category_id = feed.merge_with_category can_create_record = feeds_model && feeds_uid_field && feeds_update_callback - locales = Site.first.in_use_locales rescue I18n.available_locales + locales = site.in_use_locales rescue I18n.available_locales locales.map!{|l| l.to_s} - site_root_url = Site.first.root_url rescue "" + site_root_url = site.root_url rescue "" main_directory = File.join("#{Rails.root}","public","site_feeds") feed_directory = File.join(main_directory.to_s, feed.id.to_s) feed_data = JSON.parse(File.read(File.join(feed_directory.to_s, feed.feed_uid + ".json"))) @@ -49,7 +50,7 @@ class FeedsController < ApplicationController insert_idx = ((a["org_is_top"] == 1) ? 0 : -1) end end - raw_all_contents_for_feed_translations[locale].insert(insert_idx, feed_annc.process_tmp(a,locale,trans,site_root_url)) + raw_all_contents_for_feed_translations[locale].insert(insert_idx, feed_annc.process_tmp(site,a,locale,trans,site_root_url)) feed_data[channel_key_pluralize].insert(insert_idx, a) end if can_create_record @@ -86,7 +87,7 @@ class FeedsController < ApplicationController has_create = false raw_all_contents_for_feed_translations[locale].each_with_index do |aa, i| if aa["id"] == a["id"] - raw_all_contents_for_feed_translations[locale][i] = feed_annc.process_tmp(a,locale,trans,site_root_url) + raw_all_contents_for_feed_translations[locale][i] = feed_annc.process_tmp(site,a,locale,trans,site_root_url) feed_data[channel_key_pluralize][i] = a has_create = true break @@ -100,7 +101,7 @@ class FeedsController < ApplicationController insert_idx = ((a["org_is_top"] == 1) ? 0 : -1) end end - raw_all_contents_for_feed_translations[locale].insert(insert_idx, feed_annc.process_tmp(a,locale,trans,site_root_url)) + raw_all_contents_for_feed_translations[locale].insert(insert_idx, feed_annc.process_tmp(site,a,locale,trans,site_root_url)) feed_data[channel_key_pluralize].insert(insert_idx, a) end end diff --git a/app/models/site_feed.rb b/app/models/site_feed.rb index 0985638..84408dd 100644 --- a/app/models/site_feed.rb +++ b/app/models/site_feed.rb @@ -2,6 +2,7 @@ class SiteFeed include Mongoid::Document include Mongoid::Timestamps ModuleAppSyncFields = ["feeds_model", "feeds_uid_field", "feeds_update_callback", "feeds_time_field", "feeds_finish_callback", "feeds_update_statuses_callback"] + SiteFeedAnncSyncFields = ["channel_key", "merge_with_category", "remote_site_url", "feeds_model", "feeds_uid_field", "feeds_update_callback", "feeds_time_field", "feeds_update_statuses_callback"] field :remote_site_url field :merge_with_category field :channel_name @@ -14,15 +15,14 @@ class SiteFeed field :feeds_model field :feeds_uid_field field :feeds_update_callback - field :feeds_time_field, type: String, default: "postdate" + field :feeds_time_field, default: "postdate" field :feeds_finish_callback field :feeds_update_statuses_callback field :enable_notify, type: Boolean, default: false field :is_rss, type: Boolean, default: false - require 'feed_model/cache' require 'fileutils' - include FeedModel::Cache - Category.send(:include,FeedModel::Cache) + include Feeds::Cache + Category.send(:include,Feeds::Cache) after_create do if !self.is_rss self.add_notify @@ -71,9 +71,8 @@ class SiteFeed def sync_data_to_annc(site_feed_annc=nil) category_title = self.category.title_translations rescue {} tmp_channel_title = self.channel_title_for_cache - clone_fields =["channel_key", "merge_with_category", "remote_site_url", "feeds_model", "feeds_uid_field", "feeds_update_callback", "feeds_time_field", "feeds_update_statuses_callback"] (site_feed_annc ? [site_feed_annc] : SiteFeedAnnc.where(:feed_id=>self.id)).each do |tmp| - clone_fields.each do |f| + SiteFeedAnncSyncFields.each do |f| tmp.send("#{f}=", self.send(f)) end tmp[:feed_name] = self.feed_name_translations diff --git a/app/models/site_feed_annc.rb b/app/models/site_feed_annc.rb index 41b94c6..9626394 100644 --- a/app/models/site_feed_annc.rb +++ b/app/models/site_feed_annc.rb @@ -13,10 +13,9 @@ class SiteFeedAnnc field :remote_site_url field :channel_title field :feeds_model - field :feeds_model field :feeds_uid_field field :feeds_update_callback - field :feeds_time_field, type: String, default: "postdate" + field :feeds_time_field, default: "postdate" field :feeds_finish_callback field :feeds_update_statuses_callback @@ -36,7 +35,9 @@ class SiteFeedAnnc self.site_feed_annc_details = [] @raw_all_contents_for_feed_translations.each do |locale, contents| contents.each do |content| - self.site_feed_annc_details << SiteFeedAnncDetail.new(locale: locale, site_feed_annc: self, data: content) + self.site_feed_annc_details << SiteFeedAnncDetail.new( + locale: locale, site_feed_annc: self, data: content + ) end end end @@ -132,7 +133,7 @@ class SiteFeedAnnc self.raw_all_contents_for_feed_translations = anns_translations self.save end - def process_tmp(a, locale=nil, trans=nil, site_root_url=nil) + def process_tmp(site, a, locale=nil, trans=nil, site_root_url=nil) cat = self[:category_title] locale = I18n.locale.to_s if locale.nil? if trans.nil? @@ -148,57 +149,21 @@ class SiteFeedAnnc end end if site_root_url.nil? - site_root_url = Site.first.root_url rescue "" + site_root_url = site.root_url end tmp = a.deep_dup - tmp[:is_hidden] = self.hidden_annc.include?(tmp['id']) if self.channel_key == "announcement" - tmp['statuses'] = [] - if self[:top_list].count == 0 || self[:top_list].exclude?(tmp['id']) - tmp[:is_top] = false - else - tmp[:is_top] = true - tmp['statuses'] << { - "status" => trans[locale]['top'], - "status-class" => "status-top" - } - end - if self[:hot_list].count == 0 || self[:top_list].exclude?(tmp['id']) - tmp[:is_hot] = false - else - tmp[:is_hot] = true - tmp['statuses'] << { - "status" => trans[locale]['hot'], - "status-class" => "status-hot" - } - end - tmp["category"] = cat - tmp["source_url"] = self.remote_site_url - tmp["source-site"] = self.remote_site_url - tmp["source-site-title"] = (self[:channel_title][locale] rescue "") - tmp["params"] = tmp["params"].to_s + "_" + self.feed_id.to_s + "h" - if !tmp["source-site-title"].blank? - tmp['statuses'] << { - "status" => "#{tmp["source-site-title"]}", - "status-class" => "status-source" - } - end - files = tmp["bulletin_files"].collect{|bf| { "file_url" => bf["url"], "file_title" => (bf["title_translations"][locale].blank? ? File.basename(bf["url"]) : bf["title_translations"][locale] rescue '') }} rescue [] links = tmp["bulletin_links"].map{|link| { "link_url" => link["url"], "link_title" => (link["title_translations"][locale].blank? ? link["url"] : link["title_translations"][locale]) } } rescue [] tmp["bulletin_links"] = links tmp["bulletin_files"] = files tmp["title"] = tmp["title_translations"][locale] + tmp["subtitle"] = tmp["subtitle_translations"][locale] - tmp["source-site-link"] = tmp["source-site"] - tmp["source-site"] = "#{tmp["source-site-title"]}" tmp["link_to_show"] = !tmp["external_link"].blank? ? tmp["external_link"] : nil - tmp["target"] = (site_root_url.blank? || tmp["link_to_show"].blank?) ? '_blank' : (tmp["link_to_show"].include?(site_root_url) ? '_self' : '_blank') tmp["img_src"] = tmp["image"]["thumb"] || "/assets/announcement-default.jpg" tmp["img_src"] = "/assets/announcement-default.jpg" if tmp["img_src"] == "" tmp["img_description"] = tmp["image_description_translations"][locale] - tmp["more"] = trans[locale]['more_plus'] - tmp["view_count"] = "" else new_tmp = {} tmp.each do |key,value| @@ -225,54 +190,74 @@ class SiteFeedAnnc end end tmp = BSON::Document.new(new_tmp) - tmp['statuses'] = [] - if self[:top_list].count == 0 || self[:top_list].exclude?(tmp['id']) - tmp[:is_top] = false - else - tmp[:is_top] = true - tmp['statuses'] << { - "status" => trans[locale]['top'], - "status-class" => "status-top" - } - end - if self[:hot_list].count == 0 || self[:top_list].exclude?(tmp['id']) - tmp[:is_hot] = false - else - tmp[:is_hot] = true - tmp['statuses'] << { - "status" => trans[locale]['hot'], - "status-class" => "status-hot" - } - end - tmp["category"] = cat - tmp["source_url"] = self.remote_site_url - tmp["source-site"] = self.remote_site_url - tmp["source-site-title"] = (self[:channel_title][locale] rescue "") - tmp["params"] = tmp["params"].to_s + "_" + self.feed_id.to_s + "h" + end + + if !tmp["title"].blank? + tmp["title_plain_text"] = Nokogiri::HTML5.fragment(tmp["title"]).text + end + + tmp["category"] = cat + + tmp["target"] = ( + tmp["link_to_show"].blank? || (!site_root_url.blank? && tmp["link_to_show"].include?(site_root_url)) + ) ? '_self' : '_blank' + + tmp["params"] = tmp["params"].to_s + "_" + self.feed_id.to_s + "h" + tmp["more"] = trans[locale]['more_plus'] + tmp["view_count"] = "" + + + tmp['statuses'] = [] + if self.top_list.exclude?(tmp['id']) + tmp["is_top"] = false + else + tmp["is_top"] = true + tmp['statuses'] << { + "status" => trans[locale]['top'], + "status-class" => "status-top" + } + end + if self.hot_list.exclude?(tmp['id']) + tmp["is_hot"] = false + else + tmp["is_hot"] = true + tmp['statuses'] << { + "status" => trans[locale]['hot'], + "status-class" => "status-hot" + } + end + + tmp["is_hidden"] = self.hidden_annc.include?(tmp['id']) + + tmp["source_url"] = self.remote_site_url + tmp["source-site"] = self.remote_site_url + tmp["source-site-title"] = self.channel_title(locale) + + + if !tmp["source-site-title"].blank? tmp['statuses'] << { "status" => "#{tmp["source-site-title"]}", "status-class" => "status-source" } - tmp["source-site-link"] = tmp["source-site"] - tmp["source-site"] = "#{tmp["source-site-title"]}" - tmp["target"] = "_self" - tmp["more"] = trans[locale]['more_plus'] - tmp["view_count"] = "" end - feeds_time_field = self.feeds_time_field - feeds_time_field = 'postdate' if feeds_time_field.nil? - time_field_value = tmp[feeds_time_field] - tmp[feeds_time_field] = (time_field_value.blank? ? nil : time_field_value.to_time rescue time_field_value) - tmp[feeds_time_field] = time_field_value if tmp[feeds_time_field].nil? + + tmp["source-site-link"] = tmp["source-site"] + tmp["source-site"] = "#{tmp["source-site-title"]}" + + feeds_time_fields = Array(self.feeds_time_field) + feeds_time_fields.each do |feeds_time_field| + time_field_value = tmp[feeds_time_field] + tmp[feeds_time_field] = (time_field_value.blank? ? nil : time_field_value.to_time rescue time_field_value) + tmp[feeds_time_field] = time_field_value if tmp[feeds_time_field].nil? + end return tmp end def cache_annc(force_refresh=false,locales=nil,trans=nil) feed = SiteFeed.find(self.feed_id) anns = feed.get_annc(force_refresh) - cat = self[:category_title] - feeds_time_field = self.feeds_time_field + site = Site.first if locales.nil? - locales = Site.first.in_use_locales rescue I18n.available_locales + locales = site.in_use_locales rescue I18n.available_locales end if trans.nil? trans = {} @@ -287,80 +272,117 @@ class SiteFeedAnnc end end cache_data = {} - site_root_url = Site.first.root_url rescue "" + site_root_url = site.root_url rescue "" locales.each do |locale| locale = locale.to_s data_for_locale = anns.collect do |a| - tmp = process_tmp(a, locale, trans, site_root_url) + tmp = process_tmp(site, a, locale, trans, site_root_url) end cache_data[locale.to_s] = data_for_locale end cache_data end - def self.get_feed_cache(channel_key,merge_with_category=nil,site_source=nil,locale=I18n.locale.to_s,is_widget=false,max_len=nil) - locale = I18n.locale.to_s - max_len = ((max_len.to_i < 0 rescue true) ? 0 : max_len.to_i) - if max_len > 0 - module_app = ModuleApp.where(:key=>channel_key).first - feeds_time_field = module_app.feeds_time_field rescue nil - if feeds_time_field.nil? - feeds_time_field = 'postdate' + def self.add_prefix(match_cond) + case match_cond + when Array + match_cond = match_cond.collect do |val| + self.add_prefix(val) end - site_feed_anncs = self.where("channel_key" => channel_key) - if !merge_with_category.blank? && merge_with_category.exclude?('all') - site_feed_anncs = site_feed_anncs.where(:merge_with_category.in => merge_with_category) - end - if site_source - site_feed_anncs = site_feed_anncs.where({"channel_title.#{locale}" => site_source}) - end - site_feed_annc_ids = site_feed_anncs.pluck(:id) - match_cond = { - "site_feed_annc_id" => {"$in" => site_feed_annc_ids}, - "locale" => locale - } - - pipeline = [ - {"$match" => match_cond}, - {"$project"=>{"data"=>1}}, - {"$sort"=>{"data.is_hidden"=>-1,"data.is_top"=>-1,"data.#{feeds_time_field}"=>-1}}, - {"$match"=>{"data.is_hidden"=>{"$ne"=>true}, - "data.postdate"=>{"$lte"=>Time.now}, - "data.title" => {"$gt"=>""} - } - } - ] - if max_len - pipeline << {"$limit"=> max_len} - end - - SiteFeedAnncDetail.collection.aggregate(pipeline).collect do |data| - tmp = data['data'] - if !(tmp['category'].is_a? String) - tmp['category'] = tmp['category'][locale] + when Hash + match_cond = match_cond.inject({}) do |m, (key, val)| + if val.class == Hash || (val.class == Array && val[0].class == Hash) + val = self.add_prefix(val) end - if @image_version != 'thumb' && !@image_version.nil? - if @image_version == 'mobile' - tmp["img_src"] = tmp["image"]["mobile"] - else - tmp["img_src"] = tmp["image"]["original"] - end - tmp["img_src"] = tmp["img_src"] || "/assets/announcement-default.jpg" + if !key.start_with?("$") + key = "data.#{key}" end - if tmp["link_to_show"].nil? - if !is_widget - tmp["link_to_show"] = UseSourceUrl && tmp["show_url"] ? URI.join("#{tmp["source_url"]}/#{locale}",URI.encode("#{tmp["show_url"]}")).to_s : OrbitHelper.url_to_show(tmp["params"]) rescue '' - else - tmp["link_to_show"] = UseSourceUrl && tmp["show_url"] ? URI.join("#{tmp["source_url"]}/#{locale}",URI.encode("#{tmp["show_url"]}")).to_s : OrbitHelper.widget_item_url(tmp["params"]) rescue '' - end - end - tmp + m[key] = val + m end - else - [] end + return match_cond end + def self.get_feed_cache( + channel_key,merge_with_category=nil, + site_source=nil,locale=I18n.locale.to_s, + is_widget=false,max_len=nil,sort_maps=nil,extra_match_cond=nil + ) + locale = I18n.locale.to_s + max_len = max_len.to_i + module_app = ModuleApp.where(:key=>channel_key).first + feeds_time_field = module_app.feeds_time_field rescue nil + if feeds_time_field.nil? + feeds_time_field = 'postdate' + end + feeds_time_fields = Array(feeds_time_field) + site_feed_anncs = self.where("channel_key" => channel_key) + if !merge_with_category.blank? && merge_with_category.exclude?('all') + site_feed_anncs = site_feed_anncs.where(:merge_with_category.in => merge_with_category) + end + if site_source + site_feed_anncs = site_feed_anncs.where({"channel_title.#{locale}" => site_source}) + end + site_feed_annc_ids = site_feed_anncs.pluck(:id) + match_cond = { + "site_feed_annc_id" => {"$in" => site_feed_annc_ids}, + "locale" => locale + } + + if extra_match_cond + match_cond = match_cond.merge(add_prefix(extra_match_cond)) + end + + pipeline = [ + {"$match" => match_cond}, + {"$project"=>{"data"=>1}} + ] + + pipeline_to_get_data = pipeline.clone + + if sort_maps + sort_maps = sort_maps.collect do |sort_key, sort_val| + ["data.#{sort_key}", sort_val == :desc ? -1 : 1] + end.to_h + pipeline_to_get_data << {"$sort"=>sort_maps} + end + + if max_len > 0 + pipeline_to_get_data << {"$limit"=> max_len} + end + + pipeline_to_get_count = pipeline.clone + pipeline_to_get_count << {"$count" => "count"} + + count_result = SiteFeedAnncDetail.collection.aggregate(pipeline_to_get_count).first + feeds_count = count_result ? count_result["count"] : 0 + + + feeds = SiteFeedAnncDetail.collection.aggregate(pipeline_to_get_data).collect do |data| + tmp = data['data'] + if !(tmp['category'].is_a? String) + tmp['category'] = tmp['category'][locale] + end + if @image_version != 'thumb' && !@image_version.nil? + if @image_version == 'mobile' + tmp["img_src"] = tmp["image"]["mobile"] + else + tmp["img_src"] = tmp["image"]["original"] + end + tmp["img_src"] = tmp["img_src"] || "/assets/announcement-default.jpg" + end + if tmp["link_to_show"].nil? + tmp["link_to_show"] = UseSourceUrl && tmp["show_url"] ? + URI.join("#{tmp["source_url"]}/#{locale}",URI.encode("#{tmp["show_url"]}")).to_s : + is_widget ? OrbitHelper.widget_item_url(tmp["params"]) : OrbitHelper.url_to_show(tmp["params"]) + end + tmp + end + return feeds, feeds_count + end + + # @deprecated Please use {#get_feed_cache} instead def all_contents_for_feed(site_source=nil,locale=I18n.locale.to_s,is_widget=false) if !site_source.nil? && site_source != self[:channel_title][locale] return [] diff --git a/lib/feed_model/migrate.rb b/lib/feed_model/migrate.rb deleted file mode 100644 index 351a5c8..0000000 --- a/lib/feed_model/migrate.rb +++ /dev/null @@ -1,66 +0,0 @@ -module Migrate - def self.call - puts ['feeds migrate start'] - gem_root = Feeds::Engine.root - require File.join(gem_root, 'app/models/site_feed') - require File.join(gem_root, 'app/models/site_feed_annc') - - locales = Site.first.in_use_locales rescue I18n.available_locales - trans = {} - locales.each do |locale| - locale = locale.to_s - trans[locale] = {} - I18n.with_locale(locale) do - trans[locale]['top'] = I18n.t(:top) - trans[locale]['hot'] = I18n.t(:hot) - trans[locale]['more_plus'] = I18n.t("feed.more") - end - end - org_remote_site_urls = SiteFeed.all.pluck(:remote_site_url).uniq - all_remote_site_urls = org_remote_site_urls.map{|a| a.split('://')[1]}.uniq - all_remote_site_urls.each do |remote_site_url| - match_urls = org_remote_site_urls.select{|s| s.include?("://#{remote_site_url}")} - if match_urls.count == 2 - SiteFeed.where(:remote_site_url.in=>match_urls).update_all(:remote_site_url=>"https://#{remote_site_url}") - SiteFeedAnnc.where(:remote_site_url.in=>match_urls).update_all(:remote_site_url=>"https://#{remote_site_url}") - end - end - SiteFeed.all.order_by(:channel_title=>-1).to_a.group_by(&:remote_site_url).each do |url, site_feeds| - site_feed = site_feeds.first - channel_title_translations = site_feed.channel_title_translations - SiteFeed.where(:id.in=>site_feeds[1..-1].map{|sf| sf.id}).update_all(:channel_title_translations => channel_title_translations) - end - SiteFeedAnnc.where(:feed_id.nin=>SiteFeed.all.pluck(:id)).destroy - SiteFeed.each do |site_feed| - site_feed.add_notify - count = SiteFeedAnnc.where(feed_id: site_feed.id).count - if count>1 - SiteFeedAnnc.where(feed_id: site_feed.id).limit(count-1).destroy - end - tmp = SiteFeedAnnc.where(feed_id: site_feed.id).first - if site_feed.disabled != true - if tmp.nil? - tmp = SiteFeedAnnc.new(feed_id: site_feed.id) - end - site_feed.sync_data_to_annc(tmp) - tmp.raw_all_contents_for_feed_translations = tmp.cache_annc(false,locales,trans) - tmp.save - if site_feed.feeds_model && site_feed.feeds_finish_callback - feeds_model = site_feed.feeds_model.constantize - feeds_model.send(site_feed.feeds_finish_callback, "update_all", {"data"=>site_feed.get_annc(false),"feed_id"=>site_feed.id,"category_id"=>site_feed.merge_with_category}) - end - elsif !tmp.nil? - tmp.destroy - end - end - SiteFeedAnnc.create_indexes - sync_fields = SiteFeed::ModuleAppSyncFields - ModuleApp.where(:feeds_model.ne=>nil).each do |module_app| - sync_fields_data = sync_fields.map{|f| [f, module_app[f]] }.to_h - SiteFeedAnnc.where(:channel_key=>module_app.key).update_all(sync_fields_data) - SiteFeed.where(:channel_key=>module_app.key).update_all(sync_fields_data) - end - SiteFeedAnnc.update_all({"$unset" => {"all_contents_for_feed" => ""}}) - puts ['feeds migrate finished'] - end -end \ No newline at end of file diff --git a/lib/feeds.rb b/lib/feeds.rb index ddf12bb..02ec191 100644 --- a/lib/feeds.rb +++ b/lib/feeds.rb @@ -1,4 +1,6 @@ require "feeds/engine" +require "feeds/cache" +require "feeds/migrate" module Feeds end diff --git a/lib/feed_model/cache.rb b/lib/feeds/cache.rb similarity index 98% rename from lib/feed_model/cache.rb rename to lib/feeds/cache.rb index 0a63f0c..9a9ea23 100644 --- a/lib/feed_model/cache.rb +++ b/lib/feeds/cache.rb @@ -1,4 +1,4 @@ -module FeedModel +module Feeds module Cache require 'active_support/concern' extend ActiveSupport::Concern diff --git a/lib/feeds/engine.rb b/lib/feeds/engine.rb index 0a2de02..ef448d6 100644 --- a/lib/feeds/engine.rb +++ b/lib/feeds/engine.rb @@ -13,10 +13,9 @@ module Feeds Thread.new do begin - require File.join(gem_root, 'lib/feed_model/migrate') Migrate.call rescue => e - puts ['feed_routes',e] + puts ['feed_routes',e, e.backtrace] end end end diff --git a/lib/feeds/migrate.rb b/lib/feeds/migrate.rb new file mode 100644 index 0000000..e97bc31 --- /dev/null +++ b/lib/feeds/migrate.rb @@ -0,0 +1,76 @@ +module Feeds + module Migrate + def self.call + puts ['feeds migrate start'] + gem_root = Feeds::Engine.root + require File.join(gem_root, 'app/models/site_feed') + require File.join(gem_root, 'app/models/site_feed_annc') + + locales = Site.first.in_use_locales rescue I18n.available_locales + trans = {} + locales.each do |locale| + locale = locale.to_s + trans[locale] = {} + I18n.with_locale(locale) do + trans[locale]['top'] = I18n.t(:top) + trans[locale]['hot'] = I18n.t(:hot) + trans[locale]['more_plus'] = I18n.t("feed.more") + end + end + org_remote_site_urls = SiteFeed.all.pluck(:remote_site_url).uniq + all_remote_site_urls = org_remote_site_urls.map{|a| a.split('://')[1]}.uniq + all_remote_site_urls.each do |remote_site_url| + match_urls = org_remote_site_urls.select{|s| s.include?("://#{remote_site_url}")} + if match_urls.count == 2 + SiteFeed.where(:remote_site_url.in=>match_urls).update_all(:remote_site_url=>"https://#{remote_site_url}") + SiteFeedAnnc.where(:remote_site_url.in=>match_urls).update_all(:remote_site_url=>"https://#{remote_site_url}") + end + end + SiteFeed.all.order_by(:channel_title=>-1).to_a.group_by(&:remote_site_url).each do |url, site_feeds| + site_feed = site_feeds.first + channel_title_translations = site_feed.channel_title_translations + SiteFeed.where(:id.in=>site_feeds[1..-1].map{|sf| sf.id}).update_all(:channel_title_translations => channel_title_translations) + end + SiteFeedAnnc.where(:feed_id.nin=>SiteFeed.all.pluck(:id)).delete + SiteFeed.all.each do |site_feed| + site_feed.add_notify + count = SiteFeedAnnc.where(feed_id: site_feed.id).count + if count>1 + SiteFeedAnnc.where(feed_id: site_feed.id).limit(count-1).delete + end + tmp = SiteFeedAnnc.where(feed_id: site_feed.id).first + if site_feed.disabled != true + if tmp.nil? + tmp = SiteFeedAnnc.new(feed_id: site_feed.id) + end + site_feed.sync_data_to_annc(tmp) + tmp.raw_all_contents_for_feed_translations = tmp.cache_annc(false,locales,trans) + tmp.save + if site_feed.feeds_model && site_feed.feeds_finish_callback + feeds_model = site_feed.feeds_model.constantize + feeds_model.send(site_feed.feeds_finish_callback, "update_all", {"data"=>site_feed.get_annc(false),"feed_id"=>site_feed.id,"category_id"=>site_feed.merge_with_category}) + end + elsif !tmp.nil? + tmp.destroy + end + end + SiteFeedAnncDetail.where(:site_feed_annc.nin => SiteFeedAnnc.all.pluck(:id)).delete + SiteFeedAnnc.create_indexes + self.sync_module_apps + SiteFeedAnnc.update_all({"$unset" => {"all_contents_for_feed" => ""}}) + puts ['feeds migrate finished'] + end + + def self.sync_module_apps + sync_fields = SiteFeed::ModuleAppSyncFields + + module_apps = ModuleApp.where(:feeds_model.ne=>nil) + + module_apps.each do |module_app| + sync_fields_data = sync_fields.map{|f| [f, module_app[f]] }.to_h + SiteFeedAnnc.where(:channel_key=>module_app.key).update_all(sync_fields_data) + SiteFeed.where(:channel_key=>module_app.key).update_all(sync_fields_data) + end + end + end +end \ No newline at end of file diff --git a/lib/tasks/feeds_tasks.rake b/lib/tasks/feeds_tasks.rake index fcaed65..68ba126 100644 --- a/lib/tasks/feeds_tasks.rake +++ b/lib/tasks/feeds_tasks.rake @@ -5,7 +5,7 @@ namespace :feeds_module do FileUtils.mkdir_p(main_directory) if !File.exist?(main_directory) SiteFeedAnnc #Don't remove this, this will auto require SiteFeedAnnc SiteFeed.enabled.each do |site_feed| - FeedModel::Cache.recreate_annc_cache(site_feed) + Feeds::Cache.recreate_annc_cache(site_feed) end end end \ No newline at end of file