# encoding: utf-8 require 'rss' require 'mongo' SITES = { "總務處-各單位公告" => "0", "總務處-文書組" => "1", "總務處-出納組" => "2", "總務處-事務組" => "3", "總務處-保管組" => "5", "總務處-採購組" => "6", "總務處-經營管理組" => "7", "總務處-駐衛警察隊" => "9", "總務處-營繕組" => "10", "總務處-總務處" => "11", "社會科學院-社會科學院總務分處" => "4", "醫學院-醫學院總務分處" => "8" } SITE_KEYS = SITES.keys DB_BASE_NAME = "production_new" all = {} continue = true i = 1 yesterday = Time.now - 86400 while continue do open("http://ann.cc.ntu.edu.tw/asp/rss.asp?page=#{i}") do |rss| feed = RSS::Parser.parse(rss.read.encode('utf-8', 'big5', invalid: :replace, undef: :replace, replace: '').gsub('Wes,', 'Wed,').gsub(/(encoding=\"big5\")/, 'encoding="utf-8"')) feed.items.each do |item| if item.pubDate > yesterday if SITE_KEYS.include?(item.author) author = item.author.strip category = item.category.to_s.gsub(/\<(\/)*category\>/, '') if all[author] all[author][item.link.strip] = {title: item.title.strip, author: author, link: item.link.strip, date: item.pubDate, category: category, description: item.description.gsub("\r\n", '
').strip} else all[author] = {item.link.strip => {title: item.title.strip, author: author, link: item.link.strip, date: item.pubDate, category: category, description: item.description.gsub("\r\n", '
').strip}} end end else continue = false break end end end i += 1 end # Get corresponding category_id or create a new one def get_category_id(category, categories, coll_cat, bulletin_module_id) if categories.keys.include? "rss_#{category}" [categories["rss_#{category}"], categories] else cat = { _type: "Category", module_app_id: bulletin_module_id, key: "rss_#{category}", disable: false, custom: false, title: {:zh_tw => category}, created_at: Time.now, updated_at: Time.now } categories["rss_#{category}"] = result = coll_cat.save(cat) [result, categories] end end # Get categories and id based on a given site number def get_mongo_and_categories(site_number="0") db = Mongo::Connection.new("localhost", 27017).db("#{DB_BASE_NAME}_#{site_number}") bulletin_module_id = db["module_apps"].find(key: "announcement").first coll_bulletin = db["bulletins"] coll_buffer_cat = db["buffer_categories"] coll_cat = db["categories"] coll_bulletin_cat = db["categories"].find(module_app_id: bulletin_module_id["_id"]) categories = coll_bulletin_cat.find().to_a.inject({}) do |categories, category| categories[category['key']] = category['_id'] categories end [categories, coll_bulletin, coll_cat, bulletin_module_id["_id"],coll_buffer_cat] end # Get main site (總務處) categories @main_categories, @main_coll_bulletin, @main_coll_cat, @main_bulletin_module_id, @main_coll_buffer_cat = get_mongo_and_categories @copy_categories, @copy_coll_bulletin, @copy_coll_cat, @copy_bulletin_module_id, @copy_coll_buffer_cat = get_mongo_and_categories('11') all.each do |key, value| # Loop through all the authors site_number = SITES[key] categories, coll_bulletin, coll_cat, bulletin_module_id, coll_buffer_cat = get_mongo_and_categories(site_number) # Get current's site categories value.each_value do |bul| # Loop through all the items category_id, categories = get_category_id(bul[:category], categories, coll_cat, bulletin_module_id) unless coll_bulletin.find_one(rss_link: bul[:link]) bulletin = { _type: "Bulletin", postdate: bul[:date], created_at: bul[:date], updated_at: bul[:date], public: true, is_checked: true, is_pending: false, is_rejected: false, category_id: category_id, title: {:zh_tw => bul[:title]}, text: {:zh_tw => bul[:description]}, available_for_en: false, available_for_zh_tw: true, rss_link: bul[:link], is_top: false, is_hot: false, is_hidden: false } bs = coll_bulletin.save(bulletin) buffer_cat = { _type: "BufferCategory", category_id: category_id, categorizable_type: "Bulletin", categorizable_id: bs } coll_buffer_cat.save(buffer_cat) unless site_number.eql?("0") || @main_coll_bulletin.find_one(rss_link: bul[:link]) # Copy the item to the main site category_id, @main_categories = get_category_id(bul[:category], @main_categories, @main_coll_cat, @main_bulletin_module_id) main_bulletin = bulletin.clone main_bulletin['_id'] = BSON::ObjectId.new main_bulletin[:category_id] = category_id @main_coll_bulletin.save(main_bulletin) main_buffer_cat = { _type: "BufferCategory", category_id: category_id, categorizable_type: "Bulletin", categorizable_id: main_bulletin['_id'] } @main_coll_buffer_cat.save(main_buffer_cat) category_id, @copy_categories = get_category_id(bul[:category], @copy_categories, @copy_coll_cat, @copy_bulletin_module_id) copy_bulletin = bulletin.clone copy_bulletin['_id'] = BSON::ObjectId.new copy_bulletin[:category_id] = category_id @copy_coll_bulletin.save(copy_bulletin) copy_buffer_cat = { _type: "BufferCategory", category_id: category_id, categorizable_type: "Bulletin", categorizable_id: copy_bulletin['_id'] } @copy_coll_buffer_cat.save(copy_buffer_cat) end end end end