This commit is contained in:
rulingcom 2014-04-11 10:56:47 +08:00 committed by manson
parent 3c95f053ab
commit 37376a1050
3 changed files with 130 additions and 0 deletions

View File

@ -0,0 +1,7 @@
class GetAnnouncementFromRss
@queue = :high
def self.perform()
%x(ruby "#{Rails.root}/lib/rss_ntu_job.rb")
end
end

View File

@ -15,3 +15,9 @@ email_cron:
class: EmailCron
args:
description: EmailCron
get_announcement_from_rss:
cron: 0 0 [2,12] * * *
class: GetAnnouncementFromRss
args:
description: Loop through the announcement RSS until 24h ago

117
lib/rss_ntu_job.rb Normal file
View File

@ -0,0 +1,117 @@
# encoding: utf-8
require 'rss'
require 'mongo'
AUTHORS = %w[ 學生事務處-學生住宿服務組
-
-
-
-
-
-
-
-
-]
DB_BASE_NAME = "site_new"
all = []
continue = true
i = 1
yesterday = Time.now - 86400
while continue do
open("http://ann.cc.ntu.edu.tw/asp/rss.asp?page=#{i}") do |rss|
feed = RSS::Parser.parse(rss.read.encode('utf-8', 'big5', invalid: :replace, undef: :replace, replace: '').gsub('<pubDate>Wes,', '<pubDate>Wed,').gsub(/(encoding=\"big5\")/, 'encoding="utf-8"'))
feed.items.each do |item|
if item.pubDate > yesterday
if AUTHORS.include?(item.author)
author = item.author.strip
category = item.category.to_s.gsub(/\<(\/)*category\>/, '')
all << {title: item.title.strip, author: author, link: item.link.strip, date: item.pubDate, category: category, description: item.description.gsub("\r\n", '<br/>').strip}
end
else
continue = false
break
end
end
end
i += 1
end
# Get corresponding category_id or create a new one
def get_category_id(category, categories, coll_cat, bulletin_module_id)
if categories.keys.include? "rss_#{category}"
[categories["rss_#{category}"], categories]
else
cat = {
_type: "Category",
module_app_id: bulletin_module_id,
key: "rss_#{category}",
disable: false,
custom: false,
title: {:zh_tw => category},
created_at: Time.now,
updated_at: Time.now
}
categories["rss_#{category}"] = result = coll_cat.save(cat)
[result, categories]
end
end
# Get categories and id based on a given site number
def get_mongo_and_categories(site_number="0")
db = Mongo::Connection.new("localhost", 27017).db("#{DB_BASE_NAME}_#{site_number}")
bulletin_module_id = db["module_apps"].find(key: "announcement").first
coll_bulletin = db["bulletins"]
coll_buffer_cat = db["buffer_categories"]
coll_cat = db["categories"]
coll_bulletin_cat = db["categories"].find(module_app_id: bulletin_module_id["_id"])
categories = coll_bulletin_cat.find().to_a.inject({}) do |categories, category|
categories[category['key']] = category['_id']
categories
end
[categories, coll_bulletin, coll_cat, bulletin_module_id["_id"],coll_buffer_cat]
end
# Get categories
categories, coll_bulletin, coll_cat, bulletin_module_id, coll_buffer_cat = get_mongo_and_categories
all.each do |bul| # Loop through all the items
category_id, categories = get_category_id(bul[:category], categories, coll_cat, bulletin_module_id)
unless coll_bulletin.find_one(rss_link: bul[:link])
bulletin = { _type: "Bulletin",
postdate: bul[:date],
created_at: bul[:date],
updated_at: bul[:date],
public: true,
is_checked: true,
is_pending: false,
is_rejected: false,
category_id: category_id,
title: {:zh_tw => bul[:title]},
text: {:zh_tw => bul[:description]},
available_for_en: false,
available_for_zh_tw: true,
rss_link: bul[:link],
is_top: false,
is_hot: false,
is_hidden: false }
bs = coll_bulletin.save(bulletin)
buffer_cat = { _type: "BufferCategory",
category_id: category_id,
categorizable_type: "Bulletin",
categorizable_id: bs }
coll_buffer_cat.save(buffer_cat)
end
end