fix feed error

announcement.gemspec

@@ -45,7 +45,7 @@ if bundle_update_flag
             i = texts.rindex(s1[-1])
             texts[i] = ""
           end
-          regex_pattern = /.response-content {\n  justify-self: auto;\n}|.response-content {\n  position: static;\n}|.response-content {\n  position: relative;\n}/m  #|.response-content[^{]*{[^}]*@media[^{]*{[^{]*}[^{]*@media[^{]*{[^{]*}[^}]*}[^}]*}
+          regex_pattern = /.response-content {\n  justify-self: auto;\n}[\r\n]*|.response-content {\n  position: static;\n}[\r\n]*|.response-content {\n  position: relative;\n}[\r\n]*/m  #|.response-content[^{]*{[^}]*@media[^{]*{[^{]*}[^{]*@media[^{]*{[^{]*}[^}]*}[^}]*}
           if !texts.include? "$font-h1:"
             texts = "$font-h1: 1.5rem;\n$font-h2: 1.35rem;\n$font-h3: 1.2rem;\n$font-h4: 1.1rem;\n$font-h5: 1rem;\n$font-h6: 0.9rem;\n\n" + texts
           end
@@ -72,15 +72,17 @@ if bundle_update_flag
               f.write tp_last_text
             end
           end
-          if texts.split(regex_pattern).count >= 1
+          auto_save_array = texts.scan(regex_pattern)
-            auto_save_text = texts.scan(regex_pattern).join('')
+          if auto_save_array.count > 1
+            auto_save_text = auto_save_array.join('')
             File.open(folder+'assets/stylesheets/template/base/_autobackup.bak', 'a') do |file|
               file.puts auto_save_text
             end
+            texts = texts.gsub(regex_pattern,'')
             File.open(filename, 'w') do |file|
-              file.write (texts.split(regex_pattern).join('') + check_texts[1])
+              file.write (texts)
+            end
           end
-          else
           check_texts.each do |check_text|
             if !texts.include?(check_text)
               puts "editing #{folder}assets/stylesheets/template/base/_variables.scss"
@@ -91,7 +93,6 @@ if bundle_update_flag
             end
           end
         end
-        end
       rescue => e
         puts "not found #{folder}assets/stylesheets/template/base/_variables.scss"
       end

app/controllers/admin/announcements_controller.rb

@@ -159,7 +159,7 @@ class Admin::AnnouncementsController < OrbitAdminController
 
   def feedform
     if params[:type] == "new"
-      @announcement_feed = BulletinFeed.new
+      @announcement_feed = BulletinFeed.new(id: nil)
       render :partial => "feed_form"
     else params[:type] == "edit"
       @announcement_feed = BulletinFeed.find(params[:id])
@@ -170,7 +170,7 @@ class Admin::AnnouncementsController < OrbitAdminController
   def createfeed
     announcement_feed = BulletinFeed.new(feed_params)
     announcement_feed.save
-    create_feed_cache(nil,announcement_feed)
+    #BulletinFeed.create_feed_cache(nil,announcement_feed)
     feeds = BulletinFeed.all.asc(:created_at)
     render :partial => "feed", :collection => feeds
   end
@@ -179,7 +179,7 @@ class Admin::AnnouncementsController < OrbitAdminController
     ann_feed  = BulletinFeed.find(params[:id])
     ann_feed.update_attributes(feed_params)
     ann_feed.save
-    create_feed_cache(nil,ann_feed)
+    #BulletinFeed.create_feed_cache(nil,ann_feed)
     feeds = BulletinFeed.all.asc(:created_at)
     render :partial => "feed", :collection => feeds
   end
@@ -243,7 +243,7 @@ class Admin::AnnouncementsController < OrbitAdminController
     
     bulletin.save
     build_email(bulletin,I18n.locale)
-    create_feed_cache(bulletin)
+    #BulletinFeed.create_feed_cache(bulletin)
     redirect_to admin_announcements_path
   end
 
@@ -261,7 +261,7 @@ class Admin::AnnouncementsController < OrbitAdminController
       send_rejection_email(bulletin,I18n.locale)
     end
     bulletin.save
-    create_feed_cache(bulletin)
+    #BulletinFeed.create_feed_cache(bulletin)
     redirect_to admin_announcements_path
   end
 
@@ -320,7 +320,7 @@ class Admin::AnnouncementsController < OrbitAdminController
       bulletin.save
     end
     build_email(bulletin,I18n.locale)
-    create_feed_cache(bulletin)
+    #BulletinFeed.create_feed_cache(bulletin)
     now_bulletin_page = Bulletin.where(:title.ne => "",:is_preview.in=>[false,nil])
                               .order_by(sort).map(&:id).map.with_index.select{|v,i| v==bulletin.id}[0][1] rescue nil
     now_bulletin_page = now_bulletin_page.nil? ? 0 : ((now_bulletin_page+1).to_f/10).ceil
@@ -403,7 +403,7 @@ class Admin::AnnouncementsController < OrbitAdminController
     
     bulletin.is_preview = true
     bulletin.save
-    create_feed_cache(bulletin)
+    #BulletinFeed.create_feed_cache(bulletin)
     render :text=>page_for_bulletin(bulletin) + "?preview=true"
   end
 
@@ -508,27 +508,4 @@ class Admin::AnnouncementsController < OrbitAdminController
     end
     bps 
   end
-  def create_feed_cache(bulletin=nil,bulletin_feed=nil)
-    if !bulletin.nil?
-      BulletinFeed.where(:tag_ids.in => Array(bulletin.tag_ids).collect{|v| v.to_s}).each do |bulletin_feed|
-        uid = bulletin_feed.uid
-        uri = URI(request.protocol + request.host_with_port + "/xhr/announcements/feed/#{uid}.json")
-        Thread.new do
-          res_net = Net::HTTP.start(uri.host, uri.port,:use_ssl => uri.scheme == 'https',open_timeout: 60,read_timeout: 60) do |http|
-            req = Net::HTTP::Get.new(uri)
-            http.request(req)
-          end
-        end
-      end
-    elsif !bulletin_feed.nil?
-      uid = bulletin_feed.uid
-      uri = URI(request.protocol + request.host_with_port + "/xhr/announcements/feed/#{uid}.json")
-      Thread.new do
-        res_net = Net::HTTP.start(uri.host, uri.port,:use_ssl => uri.scheme == 'https',open_timeout: 60,read_timeout: 60) do |http|
-          req = Net::HTTP::Get.new(uri)
-          http.request(req)
-        end
-      end
-    end
-  end 
 end

app/controllers/announcement_feeds_controller.rb

@@ -3,17 +3,19 @@ class AnnouncementFeedsController < ApplicationController
 	include Admin::AnnouncementsHelper
 	def feed
 		uid = params[:uid]
-		feed_cache = BulletinFeedCache.where(uid: uid).first
+	    startdt = params[:start].blank? ? nil : params[:start]
+	    enddt = params[:end].blank? ? nil : params[:end]
+	    dt = params[:date].blank? ? nil : params[:date]
+		feed_cache = BulletinFeedCache.where(uid: uid, start: startdt, end: enddt, date: dt)
+		feed_cache.where(:invalid_date.lte => Time.now).destroy
+		count = feed_cache.count
+	    if count > 1
+	      feed_cache.limit(count-1).destroy
+	    end
+	    feed_cache = feed_cache.first
 		anns = ''
 		if feed_cache.nil?
-			BulletinFeedCache.create(uid: uid,content: '')
+			anns = BulletinFeed.where(uid: uid).first.generate_one_cache_timeout(startdt: startdt,enddt: enddt,dt: dt,base_url: request.base_url,timeout: 20)
-			Thread.new do
-				anns = get_announcements(uid).to_json
-				feed_cache = BulletinFeedCache.where(uid: uid).first
-				if !feed_cache.nil?
-					feed_cache.update_attributes(content: anns)
-				end
-			end
 		else
 			anns = feed_cache.content
 		end
@@ -61,118 +63,7 @@ class AnnouncementFeedsController < ApplicationController
 		render :json => {"feeds" => feeds}.to_json
 	end
 
-	private 
 
-	def smart_convertor(text)
-		html_string = text
-	  url = request.protocol + request.host_with_port
-	  html_string = html_string.gsub(/img.*?src="(?=\/)(.*?)|a.*?href="(?=\/)(.*?)/i){|w| w+url}
-	  html_string = html_string.gsub(/img.*?src="\.\.(?=\/)(.*?)|a.*?href="\.\.(?=\/)(.*?)/i){|w| w[0...-2]+url}
-	  return html_string
-	end
-
-	def get_announcements(uid)
-		bf = BulletinFeed.find_by(:uid => uid) rescue nil
-		startdt = params[:start]
-		enddt = params[:end]
-		dt = params[:date]
-		if !bf.nil?
-			tags = bf.tag_ids
-			categories = bf.category_ids
-			if !(categories.empty? && tags.empty?)
-				if !dt.nil?
-					dt = DateTime.parse(dt)
-					dtt = dt + 1.day
-					announcements = Bulletin.where(:postdate.gt => dt, :postdate.lt => dtt).can_display_and_sorted.is_approved
-				elsif !startdt.nil? && enddt.nil?
-					startdt = DateTime.parse(startdt)
-					enddt = DateTime.now
-					announcements = Bulletin.where(:postdate.gt => startdt, :postdate.lt => enddt).can_display_and_sorted.is_approved
-				elsif !startdt.nil? && !enddt.nil?
-					startdt = DateTime.parse(startdt)
-					enddt = DateTime.parse(enddt) + 1.day
-					announcements = Bulletin.where(:postdate.gt => startdt, :postdate.lt => enddt).can_display_and_sorted.is_approved
-				else
-					announcements = Bulletin.all.can_display_and_sorted.is_approved
-				end
-				if !tags.empty?
-					announcements = announcements.filter_by_tags(tags)
-				end
-				if !categories.empty?
-					announcements = announcements.filter_by_categories(categories)
-				end
-			else
-				announcements = []
-			end
-		end
-		all_anns = []
-		tag_names = []
-		category_titles = []
-		tag_ids = []
-		category_ids = []
-		announcements.each do |anns|
-			user = User.find(anns.create_user_id) rescue nil
-			if !user.nil?
-				author = user.member_profile && user.member_profile.name == "" ? user.user_name : user.member_profile.name
-			else
-				author = ""
-			end
-			a = {}
-			a["id"] = anns.uid
-			a["title_translations"] = anns.title_translations
-			a["subtitle_translations"] = anns.subtitle_translations
-			a["text_translations"] = {}
-			a["text_translations"]["en"] = smart_convertor(anns.text_translations["en"]) if !anns.text_translations["en"].blank?
-			a["text_translations"]["zh_tw"] = smart_convertor(anns.text_translations["zh_tw"]) if !anns.text_translations["zh_tw"].blank?
-			a["postdate"] = anns.postdate
-			a["image_description_translations"] = anns.image_description_translations
-			a["image"] = {}
-			a["display_img"] = anns.display_img
-			a["image"]["original"] = ("#{request.base_url}" + anns.image.url rescue "")
-			a["image"]["thumb"] = ("#{request.base_url}" + anns.image.thumb.url rescue "")
-			a["image"]["mobile"] = ("#{request.base_url}" + anns.image.mobile.url rescue "")
-			a["tags"] = []
-			a["category"] = {}
-			a["author"] = author
-			a["params"] = anns.to_param
-			a["subtitle_ann"] = anns.subtitle if anns.display_subtitle?
-			a["bulletin_links"] = []
-			a["bulletin_files"] = []
-			a["bulletin_carousel_images"] = anns.bulletin_carousel_images.map{|image| {"src"=>"#{request.base_url}" + image.file.url,"description"=>image.description.to_s,"description_text"=>image.description_text }}
-			anns.tags.each do |tag|
-				if !tag_ids.include?(tag.id.to_s)
-					tag_ids << tag.id.to_s
-					tag_names << {"name_translations" => tag.name_translations}
-				end
-				a["tags"] << {"name_translations" => tag.name_translations}
-			end
-			cat = anns.category
-			if (!category_ids.include?(cat.id.to_s) rescue false)
-				category_ids << cat.id.to_s
-				category_titles << {"title_translations" => cat.title_translations}
-			end
-			a["category"] = {"title_translations" => (cat.title_translations rescue {})}
-			anns.bulletin_links.each do |bl|
-				b = {}
-				b["url"] = bl.url
-				b["title_translations"] = bl.title_translations
-				a["bulletin_links"] << b
-			end
-			anns.bulletin_files.each do |bf|
-				b = {}
-				b["description_translations"] = bf.description_translations
-				b["title_translations"] = bf.title_translations
-				b["url"] = ("#{request.base_url}" + bf.file.url rescue "")
-				a["bulletin_files"] << b
-			end
-			all_anns << a
-		end
-		{
-			"announcements" => all_anns,
-			"tags" => tag_names,
-			"categories" => category_titles
-		}
-	end
 end
 
 

app/controllers/announcements_controller.rb

@@ -451,13 +451,14 @@ class AnnouncementsController < ApplicationController
     set_image_version_for_widget()
     devide_flag = (!(defined? SiteFeed).nil?)
     anns_cache.where(:invalid_date.lte => Time.now).destroy
-    if anns_cache.count > 1
+    count = anns_cache.count
-      anns_cache.skip(1).destroy
+    if count > 1
+      anns_cache.limit(count-1).destroy
     end
     if anns_cache.count == 0 || is_random
       Bulletin.remove_expired_status
       uid = OrbitHelper.params[:uid] rescue ""
-      anns_for_locale = Bulletin.where(:title.nin => ["",nil], :uid.ne => uid).is_approved.filter_cats_and_tags(cats,tags)
+      anns_for_locale = Bulletin.where(:title.nin => ["",nil], :uid.ne => uid).is_approved_and_show.filter_cats_and_tags(cats,tags)
       sorted_anns = anns_for_locale.can_display_and_sorted
       if !is_random
         sorted_anns = sorted_anns.limit(widget_data_count)

app/models/bulletin.rb

@@ -81,6 +81,7 @@ class Bulletin
   scope :open_in_future, ->{where(:is_hidden.ne=>true,:is_preview.ne => true,:postdate.gt=>Time.now).order(postdate: :asc)}
   scope :can_display_and_sorted, ->{where(:is_hidden.ne=>true,:is_preview.ne => true).any_of({:postdate.lte=>Time.now, :deadline.gte=>Time.now},{:postdate.lte=>Time.now, :deadline=>nil},{:postdate=>nil}).order(is_top: :desc,postdate: :desc,id: :desc)}
   scope :is_approved, ->{where(:approved => true)}
+  scope :is_approved_and_show, ->{where(:approved => true,:is_hidden.ne=>true,:is_preview.ne => true)}
   scope :filter_cats_and_tags, ->(cats,tags) {filter_by_widget_categories(cats,false).filter_by_tags(tags)}
   before_create :set_expire
   before_save :check_limit

app/models/bulletin_feed.rb

@@ -7,6 +7,164 @@ class BulletinFeed
 	field :tag_ids, type: Array, default: []
 	field :category_ids, type: Array, default: []
   before_save do
-    BulletinFeedCache.where(uid: self.uid).destroy
+    BulletinFeedCache.where(uid: self.uid).each do |cache|
+      cache.regenerate
+    end
+  end
+  def self.create_feed_cache(bulletin=nil,bulletin_feed=nil)
+    Thread.new do
+      if !bulletin.nil?
+        BulletinFeed.where(:tag_ids.in => Array(bulletin.tag_ids).collect{|v| v.to_s}).each do |bulletin_feed|
+          uid = bulletin_feed.uid
+          BulletinFeedCache.where(:uid => uid).each do |cache|
+            cache.regenerate
+          end
+        end
+      elsif !bulletin_feed.nil?
+        uid = bulletin_feed.uid
+        BulletinFeedCache.where(:uid => uid).each do |cache|
+          cache.regenerate
+        end
+      end
+    end
+  end
+  def generate_one_cache_timeout(startdt: nil,enddt: nil,dt: nil, base_url: nil, timeout: nil)
+    timeout = 100000 if timeout.nil?
+    begin
+      Timeout::timeout(timeout) {
+        feed_cache = nil
+        Thread.new do
+          feed_cache = self.generate_one_cache(startdt: startdt,enddt: enddt,dt: dt,base_url: base_url)
+        end
+        (1..(timeout.to_i+1)).each do
+          sleep(1)
+          break if !feed_cache.nil?
+        end
+        feed_cache.content
+      }
+    rescue=> e
+      puts [e,e.backtrace]
+      ""
+    end
+  end
+  def generate_one_cache(startdt: nil,enddt: nil,dt: nil, base_url: nil)
+    base_url = Site.first.root_url if base_url.nil?
+    uid = self.uid
+    bf = self
+    if !bf.nil?
+      tags = bf.tag_ids
+      categories = bf.category_ids
+      if !(categories.empty? && tags.empty?)
+        anns_before_filter = Bulletin.any_of([{"title.en"=>{"$nin"=>["", nil]}},{"title.zh_tw"=>{"$nin"=>["", nil]}}]).is_approved_and_show
+        if !dt.nil?
+          dt = DateTime.parse(dt)
+          dtt = dt + 1.day
+          anns_before_filter = anns_before_filter.where(:postdate.gt => dt, :postdate.lt => dtt)
+        elsif !startdt.blank? && enddt.blank?
+          startdt = DateTime.parse(startdt)
+          enddt = DateTime.now
+          anns_before_filter = anns_before_filter.where(:postdate.gt => startdt, :postdate.lt => enddt)
+        elsif !startdt.blank? && !enddt.blank?
+          startdt = DateTime.parse(startdt)
+          enddt = DateTime.parse(enddt) + 1.day
+          anns_before_filter = anns_before_filter.where(:postdate.gt => startdt, :postdate.lt => enddt)
+        end
+        anns_before_filter = anns_before_filter.can_display_and_sorted
+        if !tags.empty?
+          anns_before_filter = anns_before_filter.filter_by_tags(tags)
+        end
+        if !categories.empty?
+          anns_before_filter = anns_before_filter.filter_by_categories(categories)
+        end
+        announcements = anns_before_filter.can_display_and_sorted
+      else
+        announcements = []
+      end
+    end
+    all_anns = []
+    tag_names = []
+    category_titles = []
+    tag_ids = []
+    category_ids = []
+
+    first_postdate = anns_before_filter.open_in_future.limit(1).pluck(:postdate)[0]
+    first_deadline = nil
+    
+    announcements.each do |anns|
+      deadline = anns.deadline
+      if !deadline.blank?
+        if first_deadline.nil? || first_deadline>deadline
+          first_deadline = deadline
+        end
+      end
+      user = User.find(anns.create_user_id) rescue nil
+      if !user.nil?
+        author = user.member_profile && user.member_profile.name == "" ? user.user_name : user.member_profile.name
+      else
+        author = ""
+      end
+      a = {}
+      a["id"] = anns.uid
+      a["title_translations"] = anns.title_translations
+      a["subtitle_translations"] = anns.subtitle_translations
+      a["text_translations"] = {}
+      a["text_translations"]["en"] = self.class.smart_convertor(anns.text_translations["en"],base_url) if !anns.text_translations["en"].blank?
+      a["text_translations"]["zh_tw"] = self.class.smart_convertor(anns.text_translations["zh_tw"],base_url) if !anns.text_translations["zh_tw"].blank?
+      a["postdate"] = anns.postdate
+      a["image_description_translations"] = anns.image_description_translations
+      a["image"] = {}
+      a["display_img"] = anns.display_img
+      a["image"]["original"] = ("#{base_url}" + anns.image.url rescue "")
+      a["image"]["thumb"] = ("#{base_url}" + anns.image.thumb.url rescue "")
+      a["image"]["mobile"] = ("#{base_url}" + anns.image.mobile.url rescue "")
+      a["tags"] = []
+      a["category"] = {}
+      a["author"] = author
+      a["params"] = anns.to_param
+      a["subtitle_ann"] = anns.subtitle if anns.display_subtitle?
+      a["bulletin_links"] = []
+      a["bulletin_files"] = []
+      a["bulletin_carousel_images"] = anns.bulletin_carousel_images.map{|image| {"src"=>"#{base_url}" + image.file.url,"description"=>image.description.to_s,"description_text"=>image.description_text }}
+      anns.tags.each do |tag|
+        if !tag_ids.include?(tag.id.to_s)
+          tag_ids << tag.id.to_s
+          tag_names << {"name_translations" => tag.name_translations}
+        end
+        a["tags"] << {"name_translations" => tag.name_translations}
+      end
+      cat = anns.category
+      if (!category_ids.include?(cat.id.to_s) rescue false)
+        category_ids << cat.id.to_s
+        category_titles << {"title_translations" => cat.title_translations}
+      end
+      a["category"] = {"title_translations" => (cat.title_translations rescue {})}
+      anns.bulletin_links.each do |bl|
+        b = {}
+        b["url"] = bl.url
+        b["title_translations"] = bl.title_translations
+        a["bulletin_links"] << b
+      end
+      anns.bulletin_files.each do |bf|
+        b = {}
+        b["description_translations"] = bf.description_translations
+        b["title_translations"] = bf.title_translations
+        b["url"] = ("#{base_url}" + bf.file.url rescue "")
+        a["bulletin_files"] << b
+      end
+      all_anns << a
+    end
+    invalid_date = [first_postdate,first_deadline].compact.sort[0]
+    anns = {
+      "announcements" => all_anns,
+      "tags" => tag_names,
+      "categories" => category_titles
+    }.to_json
+    feed_cache = BulletinFeedCache.create(uid: uid,content: anns,start: startdt,end: enddt,date: dt,invalid_date: invalid_date)
+  end
+  def self.smart_convertor(text,url)
+    html_string = text
+    html_string = html_string.gsub(/img.*?src="(?=\/)(.*?)|a.*?href="(?=\/)(.*?)/i){|w| w+url}
+    html_string = html_string.gsub(/img.*?src="\.\.(?=\/)(.*?)|a.*?href="\.\.(?=\/)(.*?)/i){|w| w[0...-2]+url}
+    return html_string
   end
 end

app/models/bulletin_feed_cache.rb

@@ -4,5 +4,26 @@ class BulletinFeedCache
 
 	field :content, type: String, default: ''
 	field :uid
-
+	field :start
+	field :end
+	field :date
+  field :invalid_date, type: DateTime
+  def self.regenerate_all
+  	caches = BulletinFeedCache.all.to_a
+  	caches.each do |cache|
+  		cache.regenerate
+  	end
+  	uids = BulletinFeed.all.pluck(:uid) - caches.collect(&:uid)
+  	BulletinFeed.where(:uid.in=> uids).each do |feed|
+  		feed.generate_one_cache()
+  	end
+  end
+  def regenerate
+  	st = self.start
+  	et = self.end
+  	dt = self.date
+  	uid = self.uid
+  	BulletinFeed.where(uid: uid).first.generate_one_cache(startdt: st,enddt: et,dt: dt) rescue nil
+  	self.destroy
+  end
 end

config/routes.rb

@@ -1,5 +1,9 @@
 Rails.application.routes.draw do
-
+  if ENV['worker_num']=='0' && File.basename($0) != 'rake' && !Rails.const_defined?('Console')
+    Thread.new do
+      BulletinFeedCache.regenerate_all
+    end
+  end
   locales = Site.first.in_use_locales rescue I18n.available_locales
   
   scope "(:locale)", locale: Regexp.new(locales.join("|")) do

lib/announcement/engine.rb

@@ -71,10 +71,12 @@ module Announcement
         data[key4] = key4_attr
         data_item[key_item1] = [value_item1,value_item2,value_item3]
         data_item[key_item2] = [value2_item1,value2_item2]
+        if ENV['worker_num']=='0' && File.basename($0) != 'rake' && !Rails.const_defined?('Console')
           require File.expand_path('../../../app/models/anns_cache', __FILE__)
-        if defined? AnnsCache
+          if defined?(AnnsCache)
             AnnsCache.destroy_all
           end
+        end
       rescue  => e
         puts ['error in announcement',e]
       end

lib/bulletin_model/cache.rb

@@ -11,7 +11,11 @@ module BulletinModel
       elsif self.class == Bulletin || (self.class == Page && self.module == "announcement")
         if self.class == Bulletin
           tmp_tag_ids = (Array(self.tag_ids) + Array(self.org_tag_ids)).uniq
-          BulletinFeedCache.where(:uid.in => BulletinFeed.where(:tag_ids.in => tmp_tag_ids.collect{|v| v.to_s}).pluck(:uid)).destroy
+          Thread.new do
+            BulletinFeedCache.where(:uid.in => BulletinFeed.where(:tag_ids.in => tmp_tag_ids.collect{|v| v.to_s}).pluck(:uid)).each do |cache|
+              cache.regenerate
+            end
+          end
         end
         AnnsCache.all.destroy
       end