nccu data task

This commit is contained in:
Matthew K. Fu JuYuan 2012-09-24 12:18:40 +08:00
parent a32027b64b
commit 136fc68cdc
5 changed files with 6093 additions and 0 deletions

View File

@ -58,6 +58,7 @@ class Bulletin
validates :title, :at_least_one => true validates :title, :at_least_one => true
before_save :check_deadline, :update_avliable_language, :clean_tags before_save :check_deadline, :update_avliable_language, :clean_tags
before_create :update_avliable_language
before_save :fetch_dept before_save :fetch_dept
after_save :save_bulletin_links after_save :save_bulletin_links

File diff suppressed because one or more lines are too long

View File

@ -10,6 +10,38 @@ namespace :nccu_data do
new_unit.build_title :en => row[4], :zh_tw => row[3] new_unit.build_title :en => row[4], :zh_tw => row[3]
new_unit.save new_unit.save
end end
end
task :setup_0918_announcement => :environment do
def null_or_val(str)
if str == "NULL"
return nil
else
if str.class == "String"
return str.force_encoding("utf-8")
else
return str
end
end
end end
require 'csv'
Bulletin.destroy_all
to_this_category = BulletinCategory.first({conditions:{key: 'Announcement'}})
counter = 0
CSV.foreach("vendor/built_in_modules/announcement/lib/nccu_ann20120918.csv") do |row|
#bul_h1,bul_content,bul_ENh1,bul_ENcontent,bul_date,bul_deadline
data = {:title_translations=>{:zh_tw=>(null_or_val(row[0]) rescue nil),:en=>(null_or_val(row[2]) rescue nil)},
:text_translations=>{:zh_tw=>(null_or_val(row[1]) rescue nil),:en=>(null_or_val(row[3]) rescue nil)},
:postdate=>(Date.parse(row[4]) rescue nil),
:deadline=>(Date.parse(row[5]) rescue nil),
:bulletin_category_id=>to_this_category.id }
if counter >=1
Bulletin.create!(data.merge({:is_checked=> true,:is_pending=>false}))
end
counter+=1
end
Bulletin.all.each{|t| t.save}
end
end end

File diff suppressed because one or more lines are too long

View File

@ -0,0 +1,108 @@
# encoding: utf-8
require 'open-uri'
require 'tempfile'
require 'nokogiri'
require 'csv'
namespace :nccu_data do
desc "load nccu data from csv"
task :setup_0918_news => :environment do
def null_or_val(str)
# binding.pry
if str == "NULL"
return nil
else
if str.class.to_s == "String"
return str.force_encoding("utf-8")
else
return str
end
end
end
# def get_file(str)
# default_base = "http://secrt.nccu.edu.tw/newsimages/"
# path = null_or_val(str)
# file = nil
# if !path.nil?
# # binding.pry
# file = Tempfile.new(str)
# full_path = ""
# begin
# full_path = default_base+path
# opened = open(full_path)
# file.binmode
# open(url, headers) { |data| file.write data.read }
# rescue Exception => e
# case e.message
# when /404/ then puts "Str:#{str} 404"
# else
# e.to_s
# end
# return nil
# end
# binding.pry
# abort('Debug')
# end
# return file
# end
NewsBulletin.destroy_all
type={
1 =>NewsBulletinCategory.first({conditions:{key: "administration"}}),
2 =>NewsBulletinCategory.first({conditions:{key: "People"}}),
3 =>NewsBulletinCategory.first({conditions:{key: "campus_life"}}),
4 =>NewsBulletinCategory.first({conditions:{key: "academics"}}),
6 =>NewsBulletinCategory.first({conditions:{key: "alumni"}}),
8 =>NewsBulletinCategory.first({conditions:{key: "Statement"}}),
11 =>NewsBulletinCategory.first({conditions:{key: "International"}})
}
counter = 0
CSV.foreach("vendor/built_in_modules/news/lib/nccu_news20120918.csv") do |row|
# news_id,uid,type_id,news_title,news_subtitle,news_content,news_posttime,news_overtime,news_editor,news_units,news_type,news_language,news_frontshow,news_check,news_head,news_pic,news_hit
if counter >=1
csv_title = ""
csv_subtitle = ""
csv_text = ""
case row[11]
when "1" #zh_tw
csv_title = {:zh_tw=>(null_or_val(row[3]) rescue nil),:en=>nil}
csv_subtitle = {:zh_tw=>(null_or_val(row[4]) rescue nil),:en=>nil}
csv_text = {:zh_tw=>(null_or_val(row[5]) rescue nil),:en=>nil}
when "2" #en
csv_title = {:zh_tw=>nil,:en=>(null_or_val(row[3]) rescue nil)}
csv_subtitle = {:zh_tw=>nil,:en=>(null_or_val(row[4]) rescue nil)}
csv_text = {:zh_tw=>nil,:en=>(null_or_val(row[5]) rescue nil)}
end
data = {:title_translations=> csv_title,
:subtitle_translations=>csv_subtitle,
:text_translations=>csv_text,
:postdate=>(Date.parse(row[6]) rescue nil),
:deadline=>(Date.parse(row[7]) rescue nil),
:news_bulletin_category_id=> type[(null_or_val(row[2].to_i))].id,
:view_count => (row[16].to_i rescue 0)
}
news_bulletin = NewsBulletin.new(data.merge({:is_checked=> true,:is_pending=>false,:public=>true}))
# get_file(row[15])
news_bulletin.save
#p bulletin
end
counter+=1
end
NewsBulletin.all.each{|t| t.save}
end
end