added spinner and loader and moved to different thread

app/controllers/admin/universal_tables_controller.rb

@@ -122,6 +122,11 @@ end
     render :json => {"status" => running}.to_json
   end
 
+	def checkforimportthread
+    utable = UTable.find(params[:utable_id])
+    render :json => {"currentCount" => utable.current_xlsx_value, "totalCount" => utable.table_entries.count}.to_json
+  end
+
 	def export_data
 		I18n.locale = :zh_tw
 		table = UTable.find(params[:id])
@@ -135,131 +140,45 @@ end
     render :json => {"success" => true, "title" => title}.to_json
 	end
 
-def import_data_from_excel
+	def import_data_from_excel
-	site_in_use_locales = @site_in_use_locales.sort
+		site_in_use_locales = @site_in_use_locales.sort
-  workbook = RubyXL::Parser.parse(params["import_data"].tempfile)
+		workbook = RubyXL::Parser.parse(params["import_data"].tempfile)
-  response = {}
+		response = {}
-  current_locale = I18n.locale
+		current_locale = I18n.locale
-  table = UTable.find(params["universal_table_id"]) rescue nil
+		table = UTable.find(params["universal_table_id"]) rescue nil
-  if table.nil?
+		if table.nil?
-    render json: { success: false, msg: "Table not found." }.to_json and return
+			render json: { success: false, msg: "Table not found." }.to_json and return
-  end
+		end
+ 		table.set(current_xlsx_value: 0)
+		sheet = workbook[0]
+		if sheet.count > 503
+			render json: { success: false, msg: "More than 500 entries. Please split the entries in different files." }.to_json and return
+		end
 
-  sheet = workbook[0]
+		uploaded_io = params[:import_data]
-  if sheet.count > 503
+		safe_filename = uploaded_io.original_filename.gsub(/[^0-9A-Za-z.\-_]/, '_')
-    render json: { success: false, msg: "More than 500 entries. Please split the entries in different files." }.to_json and return
+		table_id = params[:universal_table_id]
-  end
+		site_locales = @site_in_use_locales.join(",") # e.g., "en,zh"
 
-  # 前三列是欄位名、key、格式描述
+		unless uploaded_io
-  column_titles = sheet[0].cells.map { |c| c&.value.to_s.strip }
+			render json: { success: false, msg: "No file uploaded." } and return
-  column_keys   = sheet[1].cells.map { |c| c&.value.to_s.strip }
+		end
-  column_types  = sheet[2].cells.map { |c| c&.value.to_s.strip }
 
-  # 準備欄位對應
+		# Save to tmp path
-  columns = column_keys.uniq.map.with_index do |key, i|
+		tmp_path = Rails.root.join('tmp', "import_#{Time.now.to_i}_#{safe_filename}")
-    tc = table.table_columns.where(key: key).first
+		File.open(tmp_path, 'wb') do |file|
-    [i, tc]
+			file.write(uploaded_io.read)
-  end.to_h
+		end
+		Rails.logger.info "rake universal_table_tasks:import[#{tmp_path},#{table_id},#{site_locales}]"
+		# Call the Rake task with file path
+		system("rake universal_table_tasks:import[#{tmp_path},#{table_id},#{site_locales}]")
 
-  sheet.each_with_index do |row, i|
+		render json: {
-    next if i < 3 || row.cells.compact.map { |c| c.value.to_s.strip }.all?(&:blank?)
+			success: true,
-    uid_val = row[0]&.value.to_s.strip rescue nil
+			totalCount: sheet.count - 3,
-    te = uid_val.present? ?
+			id: table.id.to_s
-           TableEntry.where(uid: uid_val, u_table_id: table.id).first_or_initialize :
+		}.to_json
-           TableEntry.new
+	end
-    te.u_table = table
-    skip = 0
-
-    tc_idx = 0
-
-    row.cells.each_with_index do |cell, col_idx|
-      next if skip > 0 && (skip -= 1) >= 0
-
-      val = cell&.value
-      tc = columns[tc_idx]
-      tc_idx += 1
-      next if tc.nil?
-
-      ce = te.column_entries.where(table_column_id: tc.id).first
-      ce =  ColumnEntry.new(table_column_id: tc.id) if ce.nil?
-      case tc.type
-      when "text", "editor"
-        v = {}
-        site_in_use_locales.each_with_index do |locale, offset|
-          v[locale.to_s] = row[col_idx + offset]&.value.to_s rescue ""
-        end
-        skip = site_in_use_locales.size - 1
-        if tc.type == "text"
-          ce.text_translations = v
-        else
-          ce.content_translations = v
-        end
-
-      when "integer"
-        ce.number = val.present? ? val.to_i : nil
-
-      when "image"
-        ce.remote_image_url = val if val.present?
-      when "file"
-        file_urls   = val.to_s.split(";").map(&:strip)
-        file_titles = row[col_idx + 1]&.value.to_s.split(";").map(&:strip)
-        skip = 1
-
-        ce.column_entry_files.destroy_all
-        ce.column_entry_files = []
-
-        file_urls.each_with_index do |remote_url, file_idx|
-          next if remote_url.blank?
-          file = ColumnEntryFile.new
-          file.remote_file_url = remote_url
-
-          # 處理多語言標題
-          titles = {}
-          site_in_use_locales.each do |locale|
-            titles[locale.to_s] = file_titles[file_idx] rescue file.file.file.filename
-          end
-          file.file_title_translations = titles
-          file.save!
-          ce.column_entry_files << file
-        end
-      when "date"
-        ce.date = val
-
-      when "period"
-        skip = 1
-        ce.period_from = val
-        ce.period_to = row[col_idx + 1]&.value rescue nil
-      end
-
-      ce.save!
-      te.column_entries << ce
-    end
-
-    # hashtags (倒數第2欄)
-    if row.cells.count >= 2
-      tags_text = row.cells[-2]&.value.to_s rescue ""
-      create_get_table_tags(te, tags_text.split(";"))
-    end
-
-    # related_entries (倒數第1欄)
-    if row.cells.count >= 1
-      related_uids = row.cells[-1]&.value.to_s.split(";").map(&:strip)
-      related_ids = TableEntry.where(:uid.in => related_uids).pluck(:id)
-      te.related_entries = related_ids.join(",")
-    end
-
-    te.save!
-    te.fix_have_data
-    te.uid = uid_val if uid_val.present?
-    te.save!
-  end
-
-  render json: {
-    success: true,
-    count: table.table_entries.count,
-    id: table.id.to_s
-  }.to_json
-end
 
 	def new_entry
 		uid = params[:universal_table_id].split("-").last
@@ -422,4 +341,4 @@ end
 	def is_uuid?(str)
     !!(str =~ /\A[\da-f]{24}\z/i || str =~ /\A[\da-f]{8}-([\da-f]{4}-){3}[\da-f]{12}\z/i)
   end
-end
+end

app/models/u_table.rb

@@ -11,6 +11,7 @@ class UTable
 
 	field :ordered_with_created_at, type: Boolean, default: true
 	field :created_at_order_direction, type: String, default: 'desc'
+	field :current_xlsx_value, type: Integer, default: 0
 
 	has_many :table_columns, :dependent => :destroy
 	has_many :table_entries, :dependent => :destroy

app/views/admin/universal_tables/index.html.erb

@@ -4,26 +4,7 @@
 <div id="index_table">
   <%= render 'index'%>
 </div>
-<script type="text/javascript">
+
-	$("form.import_from_excel_form").on("submit",function(){
-		var form = this;
-		if($(this).find("input[type=file]").val() != ""){
-			$(this).ajaxSubmit({
-				dataType : "json",
-				success : function(data){
-					if(data.success){
-						alert("Import successfull.")
-						$("tr#table_" + data.id + " td:eq(2)").text(data.count);
-					}else{
-						alert(data.msg);
-					}
-					form.reset();
-				}
-			})
-		}
-		return false;
-	})
-</script>
 <div id="downloadModal" data-backdrop="static" class="modal hide fade" tabindex="-1" role="dialog" aria-labelledby="downloadModalLabel" aria-hidden="true">
   <div class="modal-header">
     <h3 id="downloadModalLabel">Download</h3>
@@ -44,7 +25,72 @@
     <button class="btn" id="modal-close-btn" style="display:none;" data-dismiss="modal" aria-hidden="true">Close</button>
   </div>
 </div>
+<div id="importModal" data-backdrop="static" class="modal hide fade" tabindex="-1" role="dialog" aria-labelledby="importModalLabel" aria-hidden="true">
+  <div class="modal-header">
+    <h3 id="importModalLabel">Import</h3>
+  </div>
+  <div class="modal-body">
+    <div id="import-wait-zone" style="text-align: center;">
+    	<span class="msg">Please wait while we import your data. This may take a while.</span>
+    	<br />
+    	<img src="/assets/spin.gif" />
+      <br />
+      <div style="text-align: center;">Importing <span class="current-number">0</span> / <span class="total-number">0</span></div>
+    </div>
+  </div>
+  <div class="modal-footer">
+    <button class="btn" id="import-modal-close-btn" style="display:none;" data-dismiss="modal" aria-hidden="true">Close</button>
+  </div>
+</div>
 <script type="text/javascript" src="/assets/lib/process.manager.js"></script>
+<script type="text/javascript">
+  var importModal = $("#importModal"),
+    checkForImportThread = null,
+    importWaitZone = $("#import-wait-zone"),
+    importModalBtn = $("#import-modal-close-btn"),
+    currentNumber = importWaitZone.find(".current-number");
+    processManager = new ProcessManager();
+	$("form.import_from_excel_form").on("submit",function(){
+		var form = this;
+        tableID = $(this).find("input[name=universal_table_id]").val()
+		if($(this).find("input[type=file]").val() != ""){
+      importModal.modal("show");
+      importWaitZone.show();
+      importModalBtn.hide();
+			$(this).ajaxSubmit({
+				dataType : "json",
+				success : function(data){
+					if(data.success){
+            importWaitZone.find(".total-number").text(data.totalCount);
+            checkForThread = new Process(function(){
+              $.ajax({
+                url : "/admin/universal_tables/checkforimportthread",
+                type : "get",
+                data : {"utable_id" : tableID},
+                dataType : "json"
+              }).done(function(threadData){
+                currentNumber.text(threadData.currentCount)
+                if(threadData.currentCount >= threadData.totalCount){
+                  checkForThread.kill();
+                  importWaitZone.find(".msg").text("Imported successfully!!!");
+                  importWaitZone.find("img").hide();
+                  importModalBtn.show();
+                }
+              })
+            })
+            checkForThread.setTimeInterval(1000);
+            checkForThread.setRepeat(Process.CONSTANTS.REPEAT_INFINITE);
+            processManager.queue(checkForThread);
+					}else{
+						importWaitZone.find(".msg").text(data.msg)
+					}
+					form.reset();
+				}
+			})
+		}
+		return false;
+	})
+</script>
 <script type="text/javascript">
 
   var downloadModal = $("#downloadModal"),
@@ -52,8 +98,7 @@
     waitZone = $("#wait-zone"),
     linkZone = $("#link-zone"),
     downloadLink = $("a#download-link"),
-    modalBtn = $("#modal-close-btn"),
+    modalBtn = $("#modal-close-btn");
-    processManager = new ProcessManager();
 
   $(document).on("click", ".export-xls", function(){
     var link = $(this).attr("href"),

app/views/utable_export/export.xlsx.axlsx

@@ -100,29 +100,29 @@ wb.add_worksheet(name: "Structure") do |sheet|
         from = (column.period_from.strftime(format_str) rescue "")
         to = (column.period_to.strftime(format_str) rescue "")
         row << "#{from} ~ #{to}"
-when "file"
+      when "file"
-  file_links = []
+        file_links = []
-  file_titles = []
+        file_titles = []
-  locale = "zh_tw"
+        locale = "zh_tw"
-  if column
+        if column
-    column.column_entry_files.desc(:sort_number).each do |entry_file|
+          column.column_entry_files.desc(:sort_number).each do |entry_file|
-      next unless entry_file.choose_lang_display(locale)
+            next unless entry_file.choose_lang_display(locale)
-      file_links << (url + entry_file.get_link)
+            file_links << (url + entry_file.get_link)
 
-      title = if entry_file.respond_to?(:file_title_translations) && entry_file.file_title_translations.is_a?(Hash)
+            title = if entry_file.respond_to?(:file_title_translations) && entry_file.file_title_translations.is_a?(Hash)
-        entry_file.file_title_translations[locale]
+              entry_file.file_title_translations[locale]
-      elsif entry_file.file_title.is_a?(Hash)
+            elsif entry_file.file_title.is_a?(Hash)
-        entry_file.file_title[locale]
+              entry_file.file_title[locale]
-      else
+            else
-        entry_file.file_title
+              entry_file.file_title
-      end
+            end
 
-      title = entry_file.file.filename.to_s if title.blank?
+            title = entry_file.file.filename.to_s if title.blank?
-      file_titles << title
+            file_titles << title
-    end
+          end
-  end
+        end
-  row << file_links.join(";")
+        row << file_links.join(";")
-  row << file_titles.join(";")
+        row << file_titles.join(";")
       end
     end
 

config/routes.rb

@@ -24,6 +24,7 @@ Rails.application.routes.draw do
    	patch "/universal_tables/update_entry", to: 'universal_tables#update_entry'
     post "/universal_tables/import_data_from_excel", to: 'universal_tables#import_data_from_excel'
     get "universal_tables/checkforthread", to: "universal_tables#checkforthread"
+    get "universal_tables/checkforimportthread", to: "universal_tables#checkforimportthread"
     get "/universal_table/:id/mind_maps", to: "mind_maps#index"
     resources :universal_tables do
     	get "new_entry"

lib/tasks/universal_table_tasks.rake

@@ -2,6 +2,7 @@
 # task :universal_table do
 #   # Task goes here
 # end
+require 'rubyXL'
 namespace :universal_table_tasks do
   task :prepare_download,[:utable_id, :url] => :environment do |task,args|
     id = args.utable_id
@@ -23,4 +24,146 @@ namespace :universal_table_tasks do
     xlsx.force_encoding("utf-8")
     file.write(xlsx)
   end
+
+  desc "Import entries from XLSX file"
+  task :import, [:file_path, :table_id, :site_locales] => :environment do |t, args|
+    def create_get_table_tags(entry,new_tags=nil)
+      if new_tags.nil?
+        new_tags = params["table_tags"].split(",")
+      end
+      tags = []
+      entry.table_tags = []
+      new_tags.each do |tag|
+        if is_uuid?(tag) === false
+          tt = TableTag.new
+          tt.u_table_id = entry.u_table.id
+          tt.title = tag.downcase.strip
+          tt.save
+          entry.table_tags << tt
+        else
+          tt = TableTag.find(tag)
+          entry.table_tags << tt
+        end
+      end
+      return tags
+    end
+    def is_uuid?(str)
+      !!(str =~ /\A[\da-f]{24}\z/i || str =~ /\A[\da-f]{8}-([\da-f]{4}-){3}[\da-f]{12}\z/i)
+    end
+
+    file_path = args[:file_path]
+    table_id = args[:table_id]
+    site_locales = args[:site_locales].to_s.split(",").map(&:to_sym)
+
+    if !file_path || !File.exist?(file_path)
+      puts "Invalid or missing file: #{file_path}"
+      exit(1)
+    end
+
+    workbook = RubyXL::Parser.parse(file_path)
+    sheet = workbook[0]
+
+    if sheet.count > 503
+      puts "More than 500 entries. Please split the entries in different files."
+      exit(1)
+    end
+
+    table = UTable.find_by(id: table_id)
+    unless table
+      puts "Table not found with ID #{table_id}"
+      exit(1)
+    end
+
+    column_titles = sheet[0].cells.map { |c| c&.value.to_s.strip }
+    column_keys   = sheet[1].cells.map { |c| c&.value.to_s.strip }
+    column_types  = sheet[2].cells.map { |c| c&.value.to_s.strip }
+
+    columns = column_keys.uniq.map.with_index do |key, i|
+      tc = table.table_columns.where(key: key).first
+      [i, tc]
+    end.to_h
+
+    sheet.each_with_index do |row, i|
+      next if i < 3 || row.cells.compact.map { |c| c.value.to_s.strip }.all?(&:blank?)
+      uid_val = row[0]&.value.to_s.strip rescue nil
+
+      te = uid_val.present? ? TableEntry.where(uid: uid_val, u_table_id: table.id).first_or_initialize : TableEntry.new
+      te.u_table = table
+      skip = 0
+      tc_idx = 0
+
+      row.cells.each_with_index do |cell, col_idx|
+        next if skip > 0 && (skip -= 1) >= 0
+        val = cell&.value
+        tc = columns[tc_idx]
+        tc_idx += 1
+        next if tc.nil?
+
+        ce = te.column_entries.where(table_column_id: tc.id).first || ColumnEntry.new(table_column_id: tc.id)
+
+        case tc.type
+        when "text", "editor"
+          v = {}
+          site_locales.each_with_index do |locale, offset|
+            v[locale.to_s] = row[col_idx + offset]&.value.to_s rescue ""
+          end
+          skip = site_locales.size - 1
+          tc.type == "text" ? ce.text_translations = v : ce.content_translations = v
+
+        when "integer"
+          ce.number = val.present? ? val.to_i : nil
+
+        when "image"
+          ce.remote_image_url = val if val.present?
+
+        when "file"
+          file_urls   = val.to_s.split(";").map(&:strip)
+          file_titles = row[col_idx + 1]&.value.to_s.split(";").map(&:strip)
+          skip = 1
+          ce.column_entry_files.destroy_all
+          ce.column_entry_files = []
+
+          file_urls.each_with_index do |remote_url, file_idx|
+            next if remote_url.blank?
+            file = ColumnEntryFile.new
+            file.remote_file_url = remote_url
+            titles = {}
+            site_locales.each do |locale|
+              titles[locale.to_s] = file_titles[file_idx] rescue file.file.file.filename
+            end
+            file.file_title_translations = titles
+            file.save!
+            ce.column_entry_files << file
+          end
+
+        when "date"
+          ce.date = val
+
+        when "period"
+          skip = 1
+          ce.period_from = val
+          ce.period_to = row[col_idx + 1]&.value rescue nil
+        end
+
+        ce.save!
+        te.column_entries << ce
+      end
+
+      tags_text = row.cells[-2]&.value.to_s rescue ""
+      create_get_table_tags(te, tags_text.split(";")) if row.cells.count >= 2
+
+      related_uids = row.cells[-1]&.value.to_s.split(";").map(&:strip)
+      related_ids = TableEntry.where(:uid.in => related_uids).pluck(:id)
+      te.related_entries = related_ids.join(",") if row.cells.count >= 1
+
+      te.save!
+      te.fix_have_data
+      te.uid = uid_val if uid_val.present?
+      te.save!
+      table.inc(current_xlsx_value: 1)
+    end
+    # Cleanup
+		File.delete(file_path) if File.exist?(file_path)
+    puts "Import finished. Total entries in table: #{table.table_entries.count}"
+  end
 end