diff --git a/announcement.gemspec b/announcement.gemspec index bc8ee23..79a310d 100644 --- a/announcement.gemspec +++ b/announcement.gemspec @@ -45,7 +45,7 @@ if bundle_update_flag i = texts.rindex(s1[-1]) texts[i] = "" end - regex_pattern = /.response-content {\n justify-self: auto;\n}|.response-content {\n position: static;\n}|.response-content {\n position: relative;\n}/m #|.response-content[^{]*{[^}]*@media[^{]*{[^{]*}[^{]*@media[^{]*{[^{]*}[^}]*}[^}]*} + regex_pattern = /.response-content {\n justify-self: auto;\n}[\r\n]*|.response-content {\n position: static;\n}[\r\n]*|.response-content {\n position: relative;\n}[\r\n]*/m #|.response-content[^{]*{[^}]*@media[^{]*{[^{]*}[^{]*@media[^{]*{[^{]*}[^}]*}[^}]*} if !texts.include? "$font-h1:" texts = "$font-h1: 1.5rem;\n$font-h2: 1.35rem;\n$font-h3: 1.2rem;\n$font-h4: 1.1rem;\n$font-h5: 1rem;\n$font-h6: 0.9rem;\n\n" + texts end @@ -72,22 +72,23 @@ if bundle_update_flag f.write tp_last_text end end - if texts.split(regex_pattern).count >= 1 - auto_save_text = texts.scan(regex_pattern).join('') + auto_save_array = texts.scan(regex_pattern) + if auto_save_array.count > 1 + auto_save_text = auto_save_array.join('') File.open(folder+'assets/stylesheets/template/base/_autobackup.bak', 'a') do |file| file.puts auto_save_text end + texts = texts.gsub(regex_pattern,'') File.open(filename, 'w') do |file| - file.write (texts.split(regex_pattern).join('') + check_texts[1]) + file.write (texts) end - else - check_texts.each do |check_text| - if !texts.include?(check_text) - puts "editing #{folder}assets/stylesheets/template/base/_variables.scss" - texts = texts + check_text - File.open(filename, 'w') do |file| - file.write(texts) - end + end + check_texts.each do |check_text| + if !texts.include?(check_text) + puts "editing #{folder}assets/stylesheets/template/base/_variables.scss" + texts = texts + check_text + File.open(filename, 'w') do |file| + file.write(texts) end end end diff --git a/app/controllers/admin/announcements_controller.rb b/app/controllers/admin/announcements_controller.rb index a14c54c..cfaf7be 100644 --- a/app/controllers/admin/announcements_controller.rb +++ b/app/controllers/admin/announcements_controller.rb @@ -159,7 +159,7 @@ class Admin::AnnouncementsController < OrbitAdminController def feedform if params[:type] == "new" - @announcement_feed = BulletinFeed.new + @announcement_feed = BulletinFeed.new(id: nil) render :partial => "feed_form" else params[:type] == "edit" @announcement_feed = BulletinFeed.find(params[:id]) @@ -170,7 +170,7 @@ class Admin::AnnouncementsController < OrbitAdminController def createfeed announcement_feed = BulletinFeed.new(feed_params) announcement_feed.save - create_feed_cache(nil,announcement_feed) + #BulletinFeed.create_feed_cache(nil,announcement_feed) feeds = BulletinFeed.all.asc(:created_at) render :partial => "feed", :collection => feeds end @@ -179,7 +179,7 @@ class Admin::AnnouncementsController < OrbitAdminController ann_feed = BulletinFeed.find(params[:id]) ann_feed.update_attributes(feed_params) ann_feed.save - create_feed_cache(nil,ann_feed) + #BulletinFeed.create_feed_cache(nil,ann_feed) feeds = BulletinFeed.all.asc(:created_at) render :partial => "feed", :collection => feeds end @@ -243,7 +243,7 @@ class Admin::AnnouncementsController < OrbitAdminController bulletin.save build_email(bulletin,I18n.locale) - create_feed_cache(bulletin) + #BulletinFeed.create_feed_cache(bulletin) redirect_to admin_announcements_path end @@ -261,7 +261,7 @@ class Admin::AnnouncementsController < OrbitAdminController send_rejection_email(bulletin,I18n.locale) end bulletin.save - create_feed_cache(bulletin) + #BulletinFeed.create_feed_cache(bulletin) redirect_to admin_announcements_path end @@ -320,7 +320,7 @@ class Admin::AnnouncementsController < OrbitAdminController bulletin.save end build_email(bulletin,I18n.locale) - create_feed_cache(bulletin) + #BulletinFeed.create_feed_cache(bulletin) now_bulletin_page = Bulletin.where(:title.ne => "",:is_preview.in=>[false,nil]) .order_by(sort).map(&:id).map.with_index.select{|v,i| v==bulletin.id}[0][1] rescue nil now_bulletin_page = now_bulletin_page.nil? ? 0 : ((now_bulletin_page+1).to_f/10).ceil @@ -403,7 +403,7 @@ class Admin::AnnouncementsController < OrbitAdminController bulletin.is_preview = true bulletin.save - create_feed_cache(bulletin) + #BulletinFeed.create_feed_cache(bulletin) render :text=>page_for_bulletin(bulletin) + "?preview=true" end @@ -508,27 +508,4 @@ class Admin::AnnouncementsController < OrbitAdminController end bps end - def create_feed_cache(bulletin=nil,bulletin_feed=nil) - if !bulletin.nil? - BulletinFeed.where(:tag_ids.in => Array(bulletin.tag_ids).collect{|v| v.to_s}).each do |bulletin_feed| - uid = bulletin_feed.uid - uri = URI(request.protocol + request.host_with_port + "/xhr/announcements/feed/#{uid}.json") - Thread.new do - res_net = Net::HTTP.start(uri.host, uri.port,:use_ssl => uri.scheme == 'https',open_timeout: 60,read_timeout: 60) do |http| - req = Net::HTTP::Get.new(uri) - http.request(req) - end - end - end - elsif !bulletin_feed.nil? - uid = bulletin_feed.uid - uri = URI(request.protocol + request.host_with_port + "/xhr/announcements/feed/#{uid}.json") - Thread.new do - res_net = Net::HTTP.start(uri.host, uri.port,:use_ssl => uri.scheme == 'https',open_timeout: 60,read_timeout: 60) do |http| - req = Net::HTTP::Get.new(uri) - http.request(req) - end - end - end - end end diff --git a/app/controllers/announcement_feeds_controller.rb b/app/controllers/announcement_feeds_controller.rb index 3d15d4c..74d269a 100644 --- a/app/controllers/announcement_feeds_controller.rb +++ b/app/controllers/announcement_feeds_controller.rb @@ -3,17 +3,19 @@ class AnnouncementFeedsController < ApplicationController include Admin::AnnouncementsHelper def feed uid = params[:uid] - feed_cache = BulletinFeedCache.where(uid: uid).first + startdt = params[:start].blank? ? nil : params[:start] + enddt = params[:end].blank? ? nil : params[:end] + dt = params[:date].blank? ? nil : params[:date] + feed_cache = BulletinFeedCache.where(uid: uid, start: startdt, end: enddt, date: dt) + feed_cache.where(:invalid_date.lte => Time.now).destroy + count = feed_cache.count + if count > 1 + feed_cache.limit(count-1).destroy + end + feed_cache = feed_cache.first anns = '' if feed_cache.nil? - BulletinFeedCache.create(uid: uid,content: '') - Thread.new do - anns = get_announcements(uid).to_json - feed_cache = BulletinFeedCache.where(uid: uid).first - if !feed_cache.nil? - feed_cache.update_attributes(content: anns) - end - end + anns = BulletinFeed.where(uid: uid).first.generate_one_cache_timeout(startdt: startdt,enddt: enddt,dt: dt,base_url: request.base_url,timeout: 20) else anns = feed_cache.content end @@ -61,118 +63,7 @@ class AnnouncementFeedsController < ApplicationController render :json => {"feeds" => feeds}.to_json end - private - def smart_convertor(text) - html_string = text - url = request.protocol + request.host_with_port - html_string = html_string.gsub(/img.*?src="(?=\/)(.*?)|a.*?href="(?=\/)(.*?)/i){|w| w+url} - html_string = html_string.gsub(/img.*?src="\.\.(?=\/)(.*?)|a.*?href="\.\.(?=\/)(.*?)/i){|w| w[0...-2]+url} - return html_string - end - - def get_announcements(uid) - bf = BulletinFeed.find_by(:uid => uid) rescue nil - startdt = params[:start] - enddt = params[:end] - dt = params[:date] - if !bf.nil? - tags = bf.tag_ids - categories = bf.category_ids - if !(categories.empty? && tags.empty?) - if !dt.nil? - dt = DateTime.parse(dt) - dtt = dt + 1.day - announcements = Bulletin.where(:postdate.gt => dt, :postdate.lt => dtt).can_display_and_sorted.is_approved - elsif !startdt.nil? && enddt.nil? - startdt = DateTime.parse(startdt) - enddt = DateTime.now - announcements = Bulletin.where(:postdate.gt => startdt, :postdate.lt => enddt).can_display_and_sorted.is_approved - elsif !startdt.nil? && !enddt.nil? - startdt = DateTime.parse(startdt) - enddt = DateTime.parse(enddt) + 1.day - announcements = Bulletin.where(:postdate.gt => startdt, :postdate.lt => enddt).can_display_and_sorted.is_approved - else - announcements = Bulletin.all.can_display_and_sorted.is_approved - end - if !tags.empty? - announcements = announcements.filter_by_tags(tags) - end - if !categories.empty? - announcements = announcements.filter_by_categories(categories) - end - else - announcements = [] - end - end - all_anns = [] - tag_names = [] - category_titles = [] - tag_ids = [] - category_ids = [] - announcements.each do |anns| - user = User.find(anns.create_user_id) rescue nil - if !user.nil? - author = user.member_profile && user.member_profile.name == "" ? user.user_name : user.member_profile.name - else - author = "" - end - a = {} - a["id"] = anns.uid - a["title_translations"] = anns.title_translations - a["subtitle_translations"] = anns.subtitle_translations - a["text_translations"] = {} - a["text_translations"]["en"] = smart_convertor(anns.text_translations["en"]) if !anns.text_translations["en"].blank? - a["text_translations"]["zh_tw"] = smart_convertor(anns.text_translations["zh_tw"]) if !anns.text_translations["zh_tw"].blank? - a["postdate"] = anns.postdate - a["image_description_translations"] = anns.image_description_translations - a["image"] = {} - a["display_img"] = anns.display_img - a["image"]["original"] = ("#{request.base_url}" + anns.image.url rescue "") - a["image"]["thumb"] = ("#{request.base_url}" + anns.image.thumb.url rescue "") - a["image"]["mobile"] = ("#{request.base_url}" + anns.image.mobile.url rescue "") - a["tags"] = [] - a["category"] = {} - a["author"] = author - a["params"] = anns.to_param - a["subtitle_ann"] = anns.subtitle if anns.display_subtitle? - a["bulletin_links"] = [] - a["bulletin_files"] = [] - a["bulletin_carousel_images"] = anns.bulletin_carousel_images.map{|image| {"src"=>"#{request.base_url}" + image.file.url,"description"=>image.description.to_s,"description_text"=>image.description_text }} - anns.tags.each do |tag| - if !tag_ids.include?(tag.id.to_s) - tag_ids << tag.id.to_s - tag_names << {"name_translations" => tag.name_translations} - end - a["tags"] << {"name_translations" => tag.name_translations} - end - cat = anns.category - if (!category_ids.include?(cat.id.to_s) rescue false) - category_ids << cat.id.to_s - category_titles << {"title_translations" => cat.title_translations} - end - a["category"] = {"title_translations" => (cat.title_translations rescue {})} - anns.bulletin_links.each do |bl| - b = {} - b["url"] = bl.url - b["title_translations"] = bl.title_translations - a["bulletin_links"] << b - end - anns.bulletin_files.each do |bf| - b = {} - b["description_translations"] = bf.description_translations - b["title_translations"] = bf.title_translations - b["url"] = ("#{request.base_url}" + bf.file.url rescue "") - a["bulletin_files"] << b - end - all_anns << a - end - { - "announcements" => all_anns, - "tags" => tag_names, - "categories" => category_titles - } - end end diff --git a/app/controllers/announcements_controller.rb b/app/controllers/announcements_controller.rb index 069748b..b38c73a 100644 --- a/app/controllers/announcements_controller.rb +++ b/app/controllers/announcements_controller.rb @@ -451,13 +451,14 @@ class AnnouncementsController < ApplicationController set_image_version_for_widget() devide_flag = (!(defined? SiteFeed).nil?) anns_cache.where(:invalid_date.lte => Time.now).destroy - if anns_cache.count > 1 - anns_cache.skip(1).destroy + count = anns_cache.count + if count > 1 + anns_cache.limit(count-1).destroy end if anns_cache.count == 0 || is_random Bulletin.remove_expired_status uid = OrbitHelper.params[:uid] rescue "" - anns_for_locale = Bulletin.where(:title.nin => ["",nil], :uid.ne => uid).is_approved.filter_cats_and_tags(cats,tags) + anns_for_locale = Bulletin.where(:title.nin => ["",nil], :uid.ne => uid).is_approved_and_show.filter_cats_and_tags(cats,tags) sorted_anns = anns_for_locale.can_display_and_sorted if !is_random sorted_anns = sorted_anns.limit(widget_data_count) diff --git a/app/models/bulletin.rb b/app/models/bulletin.rb index 49940ab..bb3eec9 100644 --- a/app/models/bulletin.rb +++ b/app/models/bulletin.rb @@ -81,6 +81,7 @@ class Bulletin scope :open_in_future, ->{where(:is_hidden.ne=>true,:is_preview.ne => true,:postdate.gt=>Time.now).order(postdate: :asc)} scope :can_display_and_sorted, ->{where(:is_hidden.ne=>true,:is_preview.ne => true).any_of({:postdate.lte=>Time.now, :deadline.gte=>Time.now},{:postdate.lte=>Time.now, :deadline=>nil},{:postdate=>nil}).order(is_top: :desc,postdate: :desc,id: :desc)} scope :is_approved, ->{where(:approved => true)} + scope :is_approved_and_show, ->{where(:approved => true,:is_hidden.ne=>true,:is_preview.ne => true)} scope :filter_cats_and_tags, ->(cats,tags) {filter_by_widget_categories(cats,false).filter_by_tags(tags)} before_create :set_expire before_save :check_limit diff --git a/app/models/bulletin_feed.rb b/app/models/bulletin_feed.rb index 50df796..bcea409 100644 --- a/app/models/bulletin_feed.rb +++ b/app/models/bulletin_feed.rb @@ -7,6 +7,164 @@ class BulletinFeed field :tag_ids, type: Array, default: [] field :category_ids, type: Array, default: [] before_save do - BulletinFeedCache.where(uid: self.uid).destroy + BulletinFeedCache.where(uid: self.uid).each do |cache| + cache.regenerate + end + end + def self.create_feed_cache(bulletin=nil,bulletin_feed=nil) + Thread.new do + if !bulletin.nil? + BulletinFeed.where(:tag_ids.in => Array(bulletin.tag_ids).collect{|v| v.to_s}).each do |bulletin_feed| + uid = bulletin_feed.uid + BulletinFeedCache.where(:uid => uid).each do |cache| + cache.regenerate + end + end + elsif !bulletin_feed.nil? + uid = bulletin_feed.uid + BulletinFeedCache.where(:uid => uid).each do |cache| + cache.regenerate + end + end + end + end + def generate_one_cache_timeout(startdt: nil,enddt: nil,dt: nil, base_url: nil, timeout: nil) + timeout = 100000 if timeout.nil? + begin + Timeout::timeout(timeout) { + feed_cache = nil + Thread.new do + feed_cache = self.generate_one_cache(startdt: startdt,enddt: enddt,dt: dt,base_url: base_url) + end + (1..(timeout.to_i+1)).each do + sleep(1) + break if !feed_cache.nil? + end + feed_cache.content + } + rescue=> e + puts [e,e.backtrace] + "" + end + end + def generate_one_cache(startdt: nil,enddt: nil,dt: nil, base_url: nil) + base_url = Site.first.root_url if base_url.nil? + uid = self.uid + bf = self + if !bf.nil? + tags = bf.tag_ids + categories = bf.category_ids + if !(categories.empty? && tags.empty?) + anns_before_filter = Bulletin.any_of([{"title.en"=>{"$nin"=>["", nil]}},{"title.zh_tw"=>{"$nin"=>["", nil]}}]).is_approved_and_show + if !dt.nil? + dt = DateTime.parse(dt) + dtt = dt + 1.day + anns_before_filter = anns_before_filter.where(:postdate.gt => dt, :postdate.lt => dtt) + elsif !startdt.blank? && enddt.blank? + startdt = DateTime.parse(startdt) + enddt = DateTime.now + anns_before_filter = anns_before_filter.where(:postdate.gt => startdt, :postdate.lt => enddt) + elsif !startdt.blank? && !enddt.blank? + startdt = DateTime.parse(startdt) + enddt = DateTime.parse(enddt) + 1.day + anns_before_filter = anns_before_filter.where(:postdate.gt => startdt, :postdate.lt => enddt) + end + anns_before_filter = anns_before_filter.can_display_and_sorted + if !tags.empty? + anns_before_filter = anns_before_filter.filter_by_tags(tags) + end + if !categories.empty? + anns_before_filter = anns_before_filter.filter_by_categories(categories) + end + announcements = anns_before_filter.can_display_and_sorted + else + announcements = [] + end + end + all_anns = [] + tag_names = [] + category_titles = [] + tag_ids = [] + category_ids = [] + + first_postdate = anns_before_filter.open_in_future.limit(1).pluck(:postdate)[0] + first_deadline = nil + + announcements.each do |anns| + deadline = anns.deadline + if !deadline.blank? + if first_deadline.nil? || first_deadline>deadline + first_deadline = deadline + end + end + user = User.find(anns.create_user_id) rescue nil + if !user.nil? + author = user.member_profile && user.member_profile.name == "" ? user.user_name : user.member_profile.name + else + author = "" + end + a = {} + a["id"] = anns.uid + a["title_translations"] = anns.title_translations + a["subtitle_translations"] = anns.subtitle_translations + a["text_translations"] = {} + a["text_translations"]["en"] = self.class.smart_convertor(anns.text_translations["en"],base_url) if !anns.text_translations["en"].blank? + a["text_translations"]["zh_tw"] = self.class.smart_convertor(anns.text_translations["zh_tw"],base_url) if !anns.text_translations["zh_tw"].blank? + a["postdate"] = anns.postdate + a["image_description_translations"] = anns.image_description_translations + a["image"] = {} + a["display_img"] = anns.display_img + a["image"]["original"] = ("#{base_url}" + anns.image.url rescue "") + a["image"]["thumb"] = ("#{base_url}" + anns.image.thumb.url rescue "") + a["image"]["mobile"] = ("#{base_url}" + anns.image.mobile.url rescue "") + a["tags"] = [] + a["category"] = {} + a["author"] = author + a["params"] = anns.to_param + a["subtitle_ann"] = anns.subtitle if anns.display_subtitle? + a["bulletin_links"] = [] + a["bulletin_files"] = [] + a["bulletin_carousel_images"] = anns.bulletin_carousel_images.map{|image| {"src"=>"#{base_url}" + image.file.url,"description"=>image.description.to_s,"description_text"=>image.description_text }} + anns.tags.each do |tag| + if !tag_ids.include?(tag.id.to_s) + tag_ids << tag.id.to_s + tag_names << {"name_translations" => tag.name_translations} + end + a["tags"] << {"name_translations" => tag.name_translations} + end + cat = anns.category + if (!category_ids.include?(cat.id.to_s) rescue false) + category_ids << cat.id.to_s + category_titles << {"title_translations" => cat.title_translations} + end + a["category"] = {"title_translations" => (cat.title_translations rescue {})} + anns.bulletin_links.each do |bl| + b = {} + b["url"] = bl.url + b["title_translations"] = bl.title_translations + a["bulletin_links"] << b + end + anns.bulletin_files.each do |bf| + b = {} + b["description_translations"] = bf.description_translations + b["title_translations"] = bf.title_translations + b["url"] = ("#{base_url}" + bf.file.url rescue "") + a["bulletin_files"] << b + end + all_anns << a + end + invalid_date = [first_postdate,first_deadline].compact.sort[0] + anns = { + "announcements" => all_anns, + "tags" => tag_names, + "categories" => category_titles + }.to_json + feed_cache = BulletinFeedCache.create(uid: uid,content: anns,start: startdt,end: enddt,date: dt,invalid_date: invalid_date) + end + def self.smart_convertor(text,url) + html_string = text + html_string = html_string.gsub(/img.*?src="(?=\/)(.*?)|a.*?href="(?=\/)(.*?)/i){|w| w+url} + html_string = html_string.gsub(/img.*?src="\.\.(?=\/)(.*?)|a.*?href="\.\.(?=\/)(.*?)/i){|w| w[0...-2]+url} + return html_string end end \ No newline at end of file diff --git a/app/models/bulletin_feed_cache.rb b/app/models/bulletin_feed_cache.rb index 09f7e57..91672f0 100644 --- a/app/models/bulletin_feed_cache.rb +++ b/app/models/bulletin_feed_cache.rb @@ -4,5 +4,26 @@ class BulletinFeedCache field :content, type: String, default: '' field :uid - + field :start + field :end + field :date + field :invalid_date, type: DateTime + def self.regenerate_all + caches = BulletinFeedCache.all.to_a + caches.each do |cache| + cache.regenerate + end + uids = BulletinFeed.all.pluck(:uid) - caches.collect(&:uid) + BulletinFeed.where(:uid.in=> uids).each do |feed| + feed.generate_one_cache() + end + end + def regenerate + st = self.start + et = self.end + dt = self.date + uid = self.uid + BulletinFeed.where(uid: uid).first.generate_one_cache(startdt: st,enddt: et,dt: dt) rescue nil + self.destroy + end end \ No newline at end of file diff --git a/config/routes.rb b/config/routes.rb index 0a405ba..481d6db 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -1,5 +1,9 @@ Rails.application.routes.draw do - + if ENV['worker_num']=='0' && File.basename($0) != 'rake' && !Rails.const_defined?('Console') + Thread.new do + BulletinFeedCache.regenerate_all + end + end locales = Site.first.in_use_locales rescue I18n.available_locales scope "(:locale)", locale: Regexp.new(locales.join("|")) do diff --git a/lib/announcement/engine.rb b/lib/announcement/engine.rb index 23f90a7..84c3cce 100644 --- a/lib/announcement/engine.rb +++ b/lib/announcement/engine.rb @@ -71,9 +71,11 @@ module Announcement data[key4] = key4_attr data_item[key_item1] = [value_item1,value_item2,value_item3] data_item[key_item2] = [value2_item1,value2_item2] - require File.expand_path('../../../app/models/anns_cache', __FILE__) - if defined? AnnsCache - AnnsCache.destroy_all + if ENV['worker_num']=='0' && File.basename($0) != 'rake' && !Rails.const_defined?('Console') + require File.expand_path('../../../app/models/anns_cache', __FILE__) + if defined?(AnnsCache) + AnnsCache.destroy_all + end end rescue => e puts ['error in announcement',e] diff --git a/lib/bulletin_model/cache.rb b/lib/bulletin_model/cache.rb index bd8a652..7da6eb9 100644 --- a/lib/bulletin_model/cache.rb +++ b/lib/bulletin_model/cache.rb @@ -11,7 +11,11 @@ module BulletinModel elsif self.class == Bulletin || (self.class == Page && self.module == "announcement") if self.class == Bulletin tmp_tag_ids = (Array(self.tag_ids) + Array(self.org_tag_ids)).uniq - BulletinFeedCache.where(:uid.in => BulletinFeed.where(:tag_ids.in => tmp_tag_ids.collect{|v| v.to_s}).pluck(:uid)).destroy + Thread.new do + BulletinFeedCache.where(:uid.in => BulletinFeed.where(:tag_ids.in => tmp_tag_ids.collect{|v| v.to_s}).pluck(:uid)).each do |cache| + cache.regenerate + end + end end AnnsCache.all.destroy end