1. accelerate feed and search

2. fix feed error

app/controllers/admin/feeds_controller.rb

@@ -267,7 +267,7 @@ class Admin::FeedsController < OrbitAdminController
 	def force_refresh
 		sf = SiteFeed.find_by(:feed_uid => params[:feed_uid]) rescue nil
 		if !sf.nil?
-			FeedModel::Cache.recreate_annc_cache(sf)
+			Feeds::Cache.recreate_annc_cache(sf)
 		end
 		render :json => {"success" => true}.to_json
 	end

app/controllers/feeds_controller.rb

@@ -3,6 +3,7 @@ class FeedsController < ApplicationController
 	def notify_change
 		feed = SiteFeed.where(:feed_uid=>params[:uid]).first
 		feed_annc = SiteFeedAnnc.where(:feed_id=>feed.id).first
+		site = Site.first
 		if feed_annc
 			raw_all_contents_for_feed_translations = feed_annc.raw_all_contents_for_feed_translations
 			feeds_model = feed.feeds_model
@@ -16,9 +17,9 @@ class FeedsController < ApplicationController
 			feeds_finish_callback = feed.feeds_finish_callback
 			category_id = feed.merge_with_category
 			can_create_record = feeds_model && feeds_uid_field && feeds_update_callback
-			locales = Site.first.in_use_locales rescue I18n.available_locales
+			locales = site.in_use_locales rescue I18n.available_locales
 			locales.map!{|l| l.to_s}
-			site_root_url = Site.first.root_url rescue ""
+			site_root_url = site.root_url rescue ""
 			main_directory = File.join("#{Rails.root}","public","site_feeds")
 			feed_directory = File.join(main_directory.to_s, feed.id.to_s)
 			feed_data = JSON.parse(File.read(File.join(feed_directory.to_s, feed.feed_uid + ".json")))
@@ -49,7 +50,7 @@ class FeedsController < ApplicationController
 								insert_idx = ((a["org_is_top"] == 1) ? 0 : -1)
 							end
 						end
-						raw_all_contents_for_feed_translations[locale].insert(insert_idx, feed_annc.process_tmp(a,locale,trans,site_root_url))
+						raw_all_contents_for_feed_translations[locale].insert(insert_idx, feed_annc.process_tmp(site,a,locale,trans,site_root_url))
 						feed_data[channel_key_pluralize].insert(insert_idx, a)
 					end
 					if can_create_record
@@ -86,7 +87,7 @@ class FeedsController < ApplicationController
 						has_create = false
 						raw_all_contents_for_feed_translations[locale].each_with_index do |aa, i|
 							if aa["id"] == a["id"]
-								raw_all_contents_for_feed_translations[locale][i] = feed_annc.process_tmp(a,locale,trans,site_root_url)
+								raw_all_contents_for_feed_translations[locale][i] = feed_annc.process_tmp(site,a,locale,trans,site_root_url)
 								feed_data[channel_key_pluralize][i] = a
 								has_create = true
 								break
@@ -100,7 +101,7 @@ class FeedsController < ApplicationController
 									insert_idx = ((a["org_is_top"] == 1) ? 0 : -1)
 								end
 							end
-							raw_all_contents_for_feed_translations[locale].insert(insert_idx, feed_annc.process_tmp(a,locale,trans,site_root_url))
+							raw_all_contents_for_feed_translations[locale].insert(insert_idx, feed_annc.process_tmp(site,a,locale,trans,site_root_url))
 							feed_data[channel_key_pluralize].insert(insert_idx, a)
 						end
 					end

app/models/site_feed.rb

@@ -2,6 +2,7 @@ class SiteFeed
   include Mongoid::Document
   include Mongoid::Timestamps
   ModuleAppSyncFields = ["feeds_model", "feeds_uid_field", "feeds_update_callback", "feeds_time_field", "feeds_finish_callback", "feeds_update_statuses_callback"]
+  SiteFeedAnncSyncFields = ["channel_key", "merge_with_category", "remote_site_url", "feeds_model", "feeds_uid_field", "feeds_update_callback", "feeds_time_field", "feeds_update_statuses_callback"]
   field :remote_site_url
   field :merge_with_category
   field :channel_name
@@ -14,15 +15,14 @@ class SiteFeed
   field :feeds_model
   field :feeds_uid_field
   field :feeds_update_callback
-  field :feeds_time_field, type: String, default: "postdate"
+  field :feeds_time_field, default: "postdate"
   field :feeds_finish_callback
   field :feeds_update_statuses_callback
   field :enable_notify, type: Boolean, default: false
   field :is_rss, type: Boolean, default: false
-  require 'feed_model/cache'
   require 'fileutils'
-  include FeedModel::Cache
+  include Feeds::Cache
-  Category.send(:include,FeedModel::Cache)
+  Category.send(:include,Feeds::Cache)
   after_create do
     if !self.is_rss
       self.add_notify
@@ -71,9 +71,8 @@ class SiteFeed
   def sync_data_to_annc(site_feed_annc=nil)
     category_title = self.category.title_translations rescue {}
     tmp_channel_title = self.channel_title_for_cache
-    clone_fields =["channel_key", "merge_with_category", "remote_site_url", "feeds_model", "feeds_uid_field", "feeds_update_callback", "feeds_time_field", "feeds_update_statuses_callback"]
     (site_feed_annc ? [site_feed_annc] : SiteFeedAnnc.where(:feed_id=>self.id)).each do |tmp|
-      clone_fields.each do |f|
+      SiteFeedAnncSyncFields.each do |f|
         tmp.send("#{f}=", self.send(f))
       end
       tmp[:feed_name] = self.feed_name_translations

app/models/site_feed_annc.rb

@@ -13,10 +13,9 @@ class SiteFeedAnnc
   field :remote_site_url
   field :channel_title
   field :feeds_model
-  field :feeds_model
   field :feeds_uid_field
   field :feeds_update_callback
-  field :feeds_time_field, type: String, default: "postdate"
+  field :feeds_time_field, default: "postdate"
   field :feeds_finish_callback
   field :feeds_update_statuses_callback
 
@@ -36,7 +35,9 @@ class SiteFeedAnnc
       self.site_feed_annc_details = []
       @raw_all_contents_for_feed_translations.each do |locale, contents|
         contents.each do |content|
-          self.site_feed_annc_details << SiteFeedAnncDetail.new(locale: locale, site_feed_annc: self, data: content)
+          self.site_feed_annc_details << SiteFeedAnncDetail.new(
+            locale: locale, site_feed_annc: self, data: content
+          )
         end
       end
     end
@@ -132,7 +133,7 @@ class SiteFeedAnnc
     self.raw_all_contents_for_feed_translations = anns_translations
     self.save
   end
-  def process_tmp(a, locale=nil, trans=nil, site_root_url=nil)
+  def process_tmp(site, a, locale=nil, trans=nil, site_root_url=nil)
     cat = self[:category_title]
     locale = I18n.locale.to_s if locale.nil?
     if trans.nil?
@@ -148,57 +149,21 @@ class SiteFeedAnnc
       end
     end
     if site_root_url.nil?
-      site_root_url = Site.first.root_url rescue ""
+      site_root_url = site.root_url
     end
     tmp = a.deep_dup
-    tmp[:is_hidden] = self.hidden_annc.include?(tmp['id'])
     if self.channel_key == "announcement"
-      tmp['statuses'] = []
-      if self[:top_list].count == 0 || self[:top_list].exclude?(tmp['id'])
-        tmp[:is_top] = false
-      else
-        tmp[:is_top] = true
-        tmp['statuses'] << {
-          "status" => trans[locale]['top'],
-          "status-class" => "status-top"
-        }
-      end
-      if self[:hot_list].count == 0 || self[:top_list].exclude?(tmp['id'])
-        tmp[:is_hot] = false
-      else
-        tmp[:is_hot] = true
-        tmp['statuses'] << {
-          "status" => trans[locale]['hot'],
-          "status-class" => "status-hot"
-        }
-      end
-      tmp["category"] = cat
-      tmp["source_url"] = self.remote_site_url
-      tmp["source-site"] = self.remote_site_url
-      tmp["source-site-title"] = (self[:channel_title][locale] rescue "")
-      tmp["params"] = tmp["params"].to_s + "_" + self.feed_id.to_s + "h"
-      if !tmp["source-site-title"].blank?
-        tmp['statuses'] << {
-          "status" => "<a href=\"#{tmp["source-site"]}\" target=\"_blank\" class=\"feed-source\">#{tmp["source-site-title"]}</a>",
-          "status-class" => "status-source"
-        }
-      end
-
       files = tmp["bulletin_files"].collect{|bf| { "file_url" => bf["url"], "file_title" => (bf["title_translations"][locale].blank? ? File.basename(bf["url"]) : bf["title_translations"][locale] rescue '') }} rescue []
       links = tmp["bulletin_links"].map{|link|  { "link_url" => link["url"], "link_title" => (link["title_translations"][locale].blank? ? link["url"] : link["title_translations"][locale]) } } rescue []
       tmp["bulletin_links"] = links
       tmp["bulletin_files"] = files
       tmp["title"] = tmp["title_translations"][locale]
+
       tmp["subtitle"] = tmp["subtitle_translations"][locale]
-      tmp["source-site-link"] = tmp["source-site"]
-      tmp["source-site"] = "<a href=\"#{tmp["source-site"]}\" target=\"_blank\" class=\"feed-source\">#{tmp["source-site-title"]}</a>"
       tmp["link_to_show"] = !tmp["external_link"].blank? ? tmp["external_link"] : nil
-      tmp["target"] = (site_root_url.blank? || tmp["link_to_show"].blank?) ? '_blank' : (tmp["link_to_show"].include?(site_root_url) ? '_self' : '_blank')
       tmp["img_src"] = tmp["image"]["thumb"] || "/assets/announcement-default.jpg"
       tmp["img_src"] = "/assets/announcement-default.jpg" if tmp["img_src"] == ""
       tmp["img_description"] = tmp["image_description_translations"][locale]
-      tmp["more"] = trans[locale]['more_plus']
-      tmp["view_count"] = ""
     else
       new_tmp = {}
       tmp.each do |key,value|
@@ -225,54 +190,74 @@ class SiteFeedAnnc
         end
       end
       tmp = BSON::Document.new(new_tmp)
-      tmp['statuses'] = []
+    end
-      if self[:top_list].count == 0 || self[:top_list].exclude?(tmp['id'])
+
-        tmp[:is_top] = false
+    if !tmp["title"].blank?
-      else
+      tmp["title_plain_text"] = Nokogiri::HTML5.fragment(tmp["title"]).text
-        tmp[:is_top] = true
+    end
-        tmp['statuses'] << {
+
-          "status" => trans[locale]['top'],
+    tmp["category"] = cat
-          "status-class" => "status-top"
+
-        }
+    tmp["target"] = (
-      end
+      tmp["link_to_show"].blank? || (!site_root_url.blank? && tmp["link_to_show"].include?(site_root_url))
-      if self[:hot_list].count == 0 || self[:top_list].exclude?(tmp['id'])
+    ) ? '_self' : '_blank'
-        tmp[:is_hot] = false
+
-      else
+    tmp["params"] = tmp["params"].to_s + "_" + self.feed_id.to_s + "h"
-        tmp[:is_hot] = true
+    tmp["more"] = trans[locale]['more_plus']
-        tmp['statuses'] << {
+    tmp["view_count"] = ""
-          "status" => trans[locale]['hot'],
+
-          "status-class" => "status-hot"
+
-        }
+    tmp['statuses'] = []
-      end
+    if self.top_list.exclude?(tmp['id'])
-      tmp["category"] = cat
+      tmp["is_top"] = false
-      tmp["source_url"] = self.remote_site_url
+    else
-      tmp["source-site"] = self.remote_site_url
+      tmp["is_top"] = true
-      tmp["source-site-title"] = (self[:channel_title][locale] rescue "")
+      tmp['statuses'] << {
-      tmp["params"] = tmp["params"].to_s + "_" + self.feed_id.to_s + "h"
+        "status" => trans[locale]['top'],
+        "status-class" => "status-top"
+      }
+    end
+    if self.hot_list.exclude?(tmp['id'])
+      tmp["is_hot"] = false
+    else
+      tmp["is_hot"] = true
+      tmp['statuses'] << {
+        "status" => trans[locale]['hot'],
+        "status-class" => "status-hot"
+      }
+    end
+
+    tmp["is_hidden"] = self.hidden_annc.include?(tmp['id'])
+
+    tmp["source_url"] = self.remote_site_url
+    tmp["source-site"] = self.remote_site_url
+    tmp["source-site-title"] = self.channel_title(locale)
+    
+
+    if !tmp["source-site-title"].blank?
       tmp['statuses'] << {
         "status" => "<a href=\"#{tmp["source-site"]}\" target=\"_blank\" class=\"feed-source\">#{tmp["source-site-title"]}</a>",
         "status-class" => "status-source"
       }
-      tmp["source-site-link"] = tmp["source-site"]
-      tmp["source-site"] = "<a href=\"#{tmp["source-site"]}\" target=\"_blank\" class=\"feed-source\">#{tmp["source-site-title"]}</a>"
-      tmp["target"] = "_self"
-      tmp["more"] = trans[locale]['more_plus']
-      tmp["view_count"] = ""
     end
-    feeds_time_field = self.feeds_time_field
+
-    feeds_time_field = 'postdate' if feeds_time_field.nil?
+    tmp["source-site-link"] = tmp["source-site"]
-    time_field_value = tmp[feeds_time_field]
+    tmp["source-site"] = "<a href=\"#{tmp["source-site"]}\" target=\"_blank\" class=\"feed-source\">#{tmp["source-site-title"]}</a>"
-    tmp[feeds_time_field] = (time_field_value.blank? ? nil : time_field_value.to_time rescue time_field_value)
+
-    tmp[feeds_time_field] = time_field_value if tmp[feeds_time_field].nil?
+    feeds_time_fields = Array(self.feeds_time_field)
+    feeds_time_fields.each do |feeds_time_field|
+      time_field_value = tmp[feeds_time_field]
+      tmp[feeds_time_field] = (time_field_value.blank? ? nil : time_field_value.to_time rescue time_field_value)
+      tmp[feeds_time_field] = time_field_value if tmp[feeds_time_field].nil?
+    end
     return tmp
   end
   def cache_annc(force_refresh=false,locales=nil,trans=nil)
     feed = SiteFeed.find(self.feed_id)
     anns = feed.get_annc(force_refresh)
-    cat = self[:category_title]
+    site = Site.first
-    feeds_time_field = self.feeds_time_field
     if locales.nil?
-      locales = Site.first.in_use_locales rescue I18n.available_locales
+      locales = site.in_use_locales rescue I18n.available_locales
     end
     if trans.nil?
       trans = {}
@@ -287,80 +272,117 @@ class SiteFeedAnnc
       end
     end
     cache_data = {}
-    site_root_url = Site.first.root_url rescue ""
+    site_root_url = site.root_url rescue ""
     locales.each do |locale|
       locale = locale.to_s
       data_for_locale = anns.collect do |a|
-        tmp = process_tmp(a, locale, trans, site_root_url)
+        tmp = process_tmp(site, a, locale, trans, site_root_url)
       end
       cache_data[locale.to_s] = data_for_locale
     end
     cache_data
   end
 
-  def self.get_feed_cache(channel_key,merge_with_category=nil,site_source=nil,locale=I18n.locale.to_s,is_widget=false,max_len=nil)
+  def self.add_prefix(match_cond)
-    locale = I18n.locale.to_s
+    case match_cond
-    max_len = ((max_len.to_i < 0 rescue true) ? 0 : max_len.to_i)
+    when Array
-    if max_len > 0
+      match_cond = match_cond.collect do |val|
-      module_app = ModuleApp.where(:key=>channel_key).first
+        self.add_prefix(val)
-      feeds_time_field = module_app.feeds_time_field rescue nil
-      if feeds_time_field.nil?
-        feeds_time_field = 'postdate'
       end
-      site_feed_anncs = self.where("channel_key" => channel_key)
+    when Hash
-      if !merge_with_category.blank? && merge_with_category.exclude?('all')
+      match_cond = match_cond.inject({}) do |m, (key, val)|
-        site_feed_anncs = site_feed_anncs.where(:merge_with_category.in => merge_with_category)
+        if val.class == Hash || (val.class == Array && val[0].class == Hash)
-      end
+          val = self.add_prefix(val)
-      if site_source
-        site_feed_anncs = site_feed_anncs.where({"channel_title.#{locale}" => site_source})
-      end
-      site_feed_annc_ids = site_feed_anncs.pluck(:id)
-      match_cond = {
-        "site_feed_annc_id" => {"$in" => site_feed_annc_ids},
-        "locale" => locale
-      }
-
-      pipeline = [
-        {"$match" => match_cond},
-        {"$project"=>{"data"=>1}},
-        {"$sort"=>{"data.is_hidden"=>-1,"data.is_top"=>-1,"data.#{feeds_time_field}"=>-1}},
-        {"$match"=>{"data.is_hidden"=>{"$ne"=>true},
-                    "data.postdate"=>{"$lte"=>Time.now},
-                    "data.title" => {"$gt"=>""}
-                }
-        }
-      ]
-      if max_len
-        pipeline << {"$limit"=> max_len}
-      end
-
-      SiteFeedAnncDetail.collection.aggregate(pipeline).collect do |data|
-        tmp = data['data']
-        if !(tmp['category'].is_a? String)
-          tmp['category'] = tmp['category'][locale]
         end
-        if @image_version != 'thumb' && !@image_version.nil?
+        if !key.start_with?("$")
-          if @image_version == 'mobile'
+          key = "data.#{key}"
-            tmp["img_src"] = tmp["image"]["mobile"]
-          else
-            tmp["img_src"] = tmp["image"]["original"]
-          end
-          tmp["img_src"] = tmp["img_src"] || "/assets/announcement-default.jpg"
         end
-        if tmp["link_to_show"].nil?
+        m[key] = val
-          if !is_widget
+        m
-            tmp["link_to_show"] = UseSourceUrl && tmp["show_url"] ? URI.join("#{tmp["source_url"]}/#{locale}",URI.encode("#{tmp["show_url"]}")).to_s : OrbitHelper.url_to_show(tmp["params"]) rescue ''
-          else
-            tmp["link_to_show"] = UseSourceUrl && tmp["show_url"] ? URI.join("#{tmp["source_url"]}/#{locale}",URI.encode("#{tmp["show_url"]}")).to_s : OrbitHelper.widget_item_url(tmp["params"]) rescue ''
-          end
-        end
-        tmp
       end
-    else
-      []
     end
+    return match_cond
   end
 
+  def self.get_feed_cache(
+    channel_key,merge_with_category=nil,
+    site_source=nil,locale=I18n.locale.to_s,
+    is_widget=false,max_len=nil,sort_maps=nil,extra_match_cond=nil
+  )
+    locale = I18n.locale.to_s
+    max_len = max_len.to_i
+    module_app = ModuleApp.where(:key=>channel_key).first
+    feeds_time_field = module_app.feeds_time_field rescue nil
+    if feeds_time_field.nil?
+      feeds_time_field = 'postdate'
+    end
+    feeds_time_fields = Array(feeds_time_field)
+    site_feed_anncs = self.where("channel_key" => channel_key)
+    if !merge_with_category.blank? && merge_with_category.exclude?('all')
+      site_feed_anncs = site_feed_anncs.where(:merge_with_category.in => merge_with_category)
+    end
+    if site_source
+      site_feed_anncs = site_feed_anncs.where({"channel_title.#{locale}" => site_source})
+    end
+    site_feed_annc_ids = site_feed_anncs.pluck(:id)
+    match_cond = {
+      "site_feed_annc_id" => {"$in" => site_feed_annc_ids},
+      "locale" => locale
+    }
+
+    if extra_match_cond
+      match_cond = match_cond.merge(add_prefix(extra_match_cond))
+    end
+
+    pipeline = [
+      {"$match" => match_cond},
+      {"$project"=>{"data"=>1}}
+    ]
+    
+    pipeline_to_get_data = pipeline.clone
+
+    if sort_maps
+      sort_maps = sort_maps.collect do |sort_key, sort_val|
+        ["data.#{sort_key}", sort_val == :desc ? -1 : 1]
+      end.to_h
+      pipeline_to_get_data << {"$sort"=>sort_maps}
+    end
+
+    if max_len > 0
+      pipeline_to_get_data << {"$limit"=> max_len}
+    end
+
+    pipeline_to_get_count = pipeline.clone
+    pipeline_to_get_count << {"$count" => "count"}
+
+    count_result = SiteFeedAnncDetail.collection.aggregate(pipeline_to_get_count).first
+    feeds_count = count_result ? count_result["count"] : 0
+
+
+    feeds = SiteFeedAnncDetail.collection.aggregate(pipeline_to_get_data).collect do |data|
+      tmp = data['data']
+      if !(tmp['category'].is_a? String)
+        tmp['category'] = tmp['category'][locale]
+      end
+      if @image_version != 'thumb' && !@image_version.nil?
+        if @image_version == 'mobile'
+          tmp["img_src"] = tmp["image"]["mobile"]
+        else
+          tmp["img_src"] = tmp["image"]["original"]
+        end
+        tmp["img_src"] = tmp["img_src"] || "/assets/announcement-default.jpg"
+      end
+      if tmp["link_to_show"].nil?
+        tmp["link_to_show"] = UseSourceUrl && tmp["show_url"] ?
+            URI.join("#{tmp["source_url"]}/#{locale}",URI.encode("#{tmp["show_url"]}")).to_s :
+            is_widget ? OrbitHelper.widget_item_url(tmp["params"]) : OrbitHelper.url_to_show(tmp["params"])
+      end
+      tmp
+    end
+    return feeds, feeds_count
+  end
+
+  # @deprecated Please use {#get_feed_cache} instead
   def all_contents_for_feed(site_source=nil,locale=I18n.locale.to_s,is_widget=false)
     if !site_source.nil? && site_source != self[:channel_title][locale]
       return []

lib/feed_model/migrate.rb

@@ -1,66 +0,0 @@
-module Migrate
-  def self.call
-    puts ['feeds migrate start']
-    gem_root = Feeds::Engine.root
-    require File.join(gem_root, 'app/models/site_feed')
-    require File.join(gem_root, 'app/models/site_feed_annc')
-
-    locales = Site.first.in_use_locales rescue I18n.available_locales
-    trans = {}
-    locales.each do |locale|
-      locale = locale.to_s
-      trans[locale] = {}
-      I18n.with_locale(locale) do
-        trans[locale]['top'] = I18n.t(:top)
-        trans[locale]['hot'] = I18n.t(:hot)
-        trans[locale]['more_plus'] = I18n.t("feed.more")
-      end
-    end
-    org_remote_site_urls = SiteFeed.all.pluck(:remote_site_url).uniq
-    all_remote_site_urls = org_remote_site_urls.map{|a| a.split('://')[1]}.uniq
-    all_remote_site_urls.each do |remote_site_url|
-      match_urls = org_remote_site_urls.select{|s| s.include?("://#{remote_site_url}")}
-      if match_urls.count == 2
-        SiteFeed.where(:remote_site_url.in=>match_urls).update_all(:remote_site_url=>"https://#{remote_site_url}")
-        SiteFeedAnnc.where(:remote_site_url.in=>match_urls).update_all(:remote_site_url=>"https://#{remote_site_url}")
-      end
-    end
-    SiteFeed.all.order_by(:channel_title=>-1).to_a.group_by(&:remote_site_url).each do |url, site_feeds|
-      site_feed = site_feeds.first
-      channel_title_translations = site_feed.channel_title_translations
-      SiteFeed.where(:id.in=>site_feeds[1..-1].map{|sf| sf.id}).update_all(:channel_title_translations => channel_title_translations)
-    end
-    SiteFeedAnnc.where(:feed_id.nin=>SiteFeed.all.pluck(:id)).destroy
-    SiteFeed.each do |site_feed|
-      site_feed.add_notify
-      count = SiteFeedAnnc.where(feed_id: site_feed.id).count
-      if count>1
-        SiteFeedAnnc.where(feed_id: site_feed.id).limit(count-1).destroy
-      end
-      tmp = SiteFeedAnnc.where(feed_id: site_feed.id).first
-      if site_feed.disabled != true
-        if tmp.nil?
-          tmp = SiteFeedAnnc.new(feed_id: site_feed.id)
-        end
-        site_feed.sync_data_to_annc(tmp)
-        tmp.raw_all_contents_for_feed_translations = tmp.cache_annc(false,locales,trans)
-        tmp.save
-        if site_feed.feeds_model && site_feed.feeds_finish_callback
-          feeds_model = site_feed.feeds_model.constantize
-          feeds_model.send(site_feed.feeds_finish_callback, "update_all", {"data"=>site_feed.get_annc(false),"feed_id"=>site_feed.id,"category_id"=>site_feed.merge_with_category})
-        end
-      elsif !tmp.nil?
-        tmp.destroy
-      end
-    end
-    SiteFeedAnnc.create_indexes
-    sync_fields = SiteFeed::ModuleAppSyncFields
-    ModuleApp.where(:feeds_model.ne=>nil).each do |module_app|
-      sync_fields_data = sync_fields.map{|f| [f, module_app[f]] }.to_h
-      SiteFeedAnnc.where(:channel_key=>module_app.key).update_all(sync_fields_data)
-      SiteFeed.where(:channel_key=>module_app.key).update_all(sync_fields_data)
-    end
-    SiteFeedAnnc.update_all({"$unset" => {"all_contents_for_feed" => ""}})
-    puts ['feeds migrate finished']
-  end
-end

lib/feeds.rb

@@ -1,4 +1,6 @@
 require "feeds/engine"
+require "feeds/cache"
+require "feeds/migrate"
 
 module Feeds
 end

lib/feeds/cache.rb

@@ -1,4 +1,4 @@
-module FeedModel
+module Feeds
   module Cache
     require 'active_support/concern'
     extend ActiveSupport::Concern

lib/feeds/engine.rb

@@ -13,10 +13,9 @@ module Feeds
 
           Thread.new do
             begin
-              require File.join(gem_root, 'lib/feed_model/migrate')
               Migrate.call
             rescue  => e
-              puts ['feed_routes',e]
+              puts ['feed_routes',e, e.backtrace]
             end
           end
         end

lib/feeds/migrate.rb

@@ -0,0 +1,76 @@
+module Feeds
+  module Migrate
+    def self.call
+      puts ['feeds migrate start']
+      gem_root = Feeds::Engine.root
+      require File.join(gem_root, 'app/models/site_feed')
+      require File.join(gem_root, 'app/models/site_feed_annc')
+
+      locales = Site.first.in_use_locales rescue I18n.available_locales
+      trans = {}
+      locales.each do |locale|
+        locale = locale.to_s
+        trans[locale] = {}
+        I18n.with_locale(locale) do
+          trans[locale]['top'] = I18n.t(:top)
+          trans[locale]['hot'] = I18n.t(:hot)
+          trans[locale]['more_plus'] = I18n.t("feed.more")
+        end
+      end
+      org_remote_site_urls = SiteFeed.all.pluck(:remote_site_url).uniq
+      all_remote_site_urls = org_remote_site_urls.map{|a| a.split('://')[1]}.uniq
+      all_remote_site_urls.each do |remote_site_url|
+        match_urls = org_remote_site_urls.select{|s| s.include?("://#{remote_site_url}")}
+        if match_urls.count == 2
+          SiteFeed.where(:remote_site_url.in=>match_urls).update_all(:remote_site_url=>"https://#{remote_site_url}")
+          SiteFeedAnnc.where(:remote_site_url.in=>match_urls).update_all(:remote_site_url=>"https://#{remote_site_url}")
+        end
+      end
+      SiteFeed.all.order_by(:channel_title=>-1).to_a.group_by(&:remote_site_url).each do |url, site_feeds|
+        site_feed = site_feeds.first
+        channel_title_translations = site_feed.channel_title_translations
+        SiteFeed.where(:id.in=>site_feeds[1..-1].map{|sf| sf.id}).update_all(:channel_title_translations => channel_title_translations)
+      end
+      SiteFeedAnnc.where(:feed_id.nin=>SiteFeed.all.pluck(:id)).delete
+      SiteFeed.all.each do |site_feed|
+        site_feed.add_notify
+        count = SiteFeedAnnc.where(feed_id: site_feed.id).count
+        if count>1
+          SiteFeedAnnc.where(feed_id: site_feed.id).limit(count-1).delete
+        end
+        tmp = SiteFeedAnnc.where(feed_id: site_feed.id).first
+        if site_feed.disabled != true
+          if tmp.nil?
+            tmp = SiteFeedAnnc.new(feed_id: site_feed.id)
+          end
+          site_feed.sync_data_to_annc(tmp)
+          tmp.raw_all_contents_for_feed_translations = tmp.cache_annc(false,locales,trans)
+          tmp.save
+          if site_feed.feeds_model && site_feed.feeds_finish_callback
+            feeds_model = site_feed.feeds_model.constantize
+            feeds_model.send(site_feed.feeds_finish_callback, "update_all", {"data"=>site_feed.get_annc(false),"feed_id"=>site_feed.id,"category_id"=>site_feed.merge_with_category})
+          end
+        elsif !tmp.nil?
+          tmp.destroy
+        end
+      end
+      SiteFeedAnncDetail.where(:site_feed_annc.nin => SiteFeedAnnc.all.pluck(:id)).delete
+      SiteFeedAnnc.create_indexes
+      self.sync_module_apps
+      SiteFeedAnnc.update_all({"$unset" => {"all_contents_for_feed" => ""}})
+      puts ['feeds migrate finished']
+    end
+
+    def self.sync_module_apps
+      sync_fields = SiteFeed::ModuleAppSyncFields
+
+      module_apps = ModuleApp.where(:feeds_model.ne=>nil)
+
+      module_apps.each do |module_app|
+        sync_fields_data = sync_fields.map{|f| [f, module_app[f]] }.to_h
+        SiteFeedAnnc.where(:channel_key=>module_app.key).update_all(sync_fields_data)
+        SiteFeed.where(:channel_key=>module_app.key).update_all(sync_fields_data)
+      end
+    end
+  end
+end

lib/tasks/feeds_tasks.rake

@@ -5,7 +5,7 @@ namespace :feeds_module do
     FileUtils.mkdir_p(main_directory) if !File.exist?(main_directory)
     SiteFeedAnnc #Don't remove this, this will auto require SiteFeedAnnc
     SiteFeed.enabled.each do |site_feed|
-      FeedModel::Cache.recreate_annc_cache(site_feed)
+      Feeds::Cache.recreate_annc_cache(site_feed)
     end
   end 
 end