2248 make the uploading of a spreadsheet a background job

app/assets/javascripts/single_page/dynamic_table.js.erb

@@ -48,7 +48,7 @@ function sanitizeData(data) {
 
 const objectInputTemp = '<input type="hidden" name="_NAME_[]" id="inpt-_NAME_" value="" autocomplete="off" />' +
   '<select name="_NAME_[]" id="select-_NAME_" class="form-control _EXTRACLASS_" title="_TITLE_" data-role="seek-objectsinput" ' +
-  'data-tags-limit="_LIMIT?_" multiple style="background-color: coral;" data-typeahead-template="_TYPEHEAD_"' +
+  'data-tags-limit="_LIMIT?_" multiple style="background-color: coral;" data-typeahead-template="_TYPEHEAD_" ' +
   'data-typeahead-query-url="_URL_" data-allow-new-items=_ALLOW_FREE_TEXT_>_OPTIONS_</select>';
 
 const typeaheadSamplesUrl = "<%= typeahead_samples_path(linked_sample_type_id: '_LINKED_') %>";
@@ -416,6 +416,7 @@ const handleSelect = (e) => {
         this.options.callback();
       }
       if (disableLoading) disableLoading();
+      location.reload();
     },
     headers: function () {
       return this.table

app/assets/javascripts/single_page/index.js.erb

@@ -168,6 +168,7 @@ async function fetchTerms(elem, cvId) {
 async function batchCreateSample(sampleTypes, projectDefaultPolicy) {
   try {
     let data = [];
+    let sample_type_id = sampleTypes[0].sampleTypeId;
     sampleTypes.forEach((s) => {
       s.samples.forEach((sa, k) => {
         data.push(
@@ -179,7 +180,7 @@ async function batchCreateSample(sampleTypes, projectDefaultPolicy) {
     if (data.length == 0) {
       return;
     }
-    return ajaxCall("<%= batch_create_samples_path %>", "POST", { data: JSON.stringify({ data }) });
+    return ajaxCall("<%= batch_create_samples_path %>", "POST", { data: JSON.stringify({ sample_type_id, data }) });
   } catch (e) {
     console.log(e);
   }
@@ -188,16 +189,17 @@ async function batchCreateSample(sampleTypes, projectDefaultPolicy) {
 async function batchDeleteSample(sampleTypes) {
   try {
     let data = [];
+    let sample_type_id = sampleTypes[0].sampleTypeId;
     sampleTypes.forEach((s) => {
       s.samples.forEach((sa, k) => {
-        data.push(batchSampleDeleteStruct(sa.exId, sa.id));
+        data.push(batchSampleDeleteStruct(sa.exId, sa.id, s));
       });
     });
 
     if (data.length == 0) {
       return;
     }
-    return ajaxCall("<%= batch_delete_samples_path %>", "DELETE", { data: JSON.stringify({ data }) });
+    return ajaxCall("<%= batch_delete_samples_path %>", "DELETE", { data: JSON.stringify({ sample_type_id, data }) });
   } catch (e) {
     console.log(e);
   }
@@ -215,7 +217,7 @@ async function exportToExcel(tableName, studyId, assayId, sampleTypeId) {
 
   // Checks whether the dynamic table has errors
   // The excel export will be aborted as long as the dynamic table has errors
-  hasErrorcells = $j(`table[id=${tableName}]`).find('select.select2__error').size() > 0
+  const hasErrorcells = $j(`table[id=${tableName}]`).find('select.select2__error').size() > 0;
   if (hasErrorcells) {
     alert('It appears this sample table has some errors. Please correct the errors in the current sample table and try downloading again.');
     return;
@@ -273,8 +275,7 @@ async function exportToExcel(tableName, studyId, assayId, sampleTypeId) {
     }
   })
   .done( function(response){
-    downloadUrl = `<%= download_samples_excel_single_pages_path() %>?uuid=${response.uuid}`;
-    window.location.href = downloadUrl;
+    window.location.href = `<%= download_samples_excel_single_pages_path() %>?uuid=${response.uuid}`;
   })
   .fail( function(response){
     alert(`Failed to export through excel!\nStatus: ${response.status}\nError: ${JSON.stringify(response.error().statusText)}`);
@@ -287,6 +288,7 @@ async function exportToExcel(tableName, studyId, assayId, sampleTypeId) {
 async function batchUpdateSample(sampleTypes) {
   try {
     let data = [];
+    let sample_type_id = sampleTypes[0].sampleTypeId;
     sampleTypes.forEach((s) => {
       s.samples.forEach((sa, k) => {
         data.push(batchSampleUpdateStruct(sa.exId, sa.data, sa.id));
@@ -296,7 +298,7 @@ async function batchUpdateSample(sampleTypes) {
     if (data.length == 0) {
       return;
     }
-    return ajaxCall("<%= batch_update_samples_path %>", "PUT", { data: JSON.stringify({ data }) });
+    return ajaxCall("<%= batch_update_samples_path %>", "PUT", { data: JSON.stringify({ sample_type_id, data }) });
   } catch (e) {
     console.log(e);
   }
@@ -305,7 +307,7 @@ async function batchUpdateSample(sampleTypes) {
 async function handleUploadSubmit(formData){
   $j.ajax({
     type: 'POST',
-    url: "<%= upload_samples_single_pages_path %>",
+    url: "<%= preview_upload_samples_single_pages_path %>",
     data: formData,
     dataType: 'html',
     processData: false,

app/assets/stylesheets/styles.scss

@@ -803,3 +803,7 @@ div#super_tag_cloud {
   max-width: 100%;
   border: 1px solid #ddd;
 }
+
+[inert] {
+  opacity: 0.5;
+}

app/controllers/samples_controller.rb

@@ -150,62 +150,141 @@ def filter
   def batch_create
     errors = []
     results = []
-    param_converter = Seek::Api::ParameterConverter.new("samples")
-    Sample.transaction do
-      params[:data].each do |par|
-        converted_params = param_converter.convert(par)
-        sample_type = SampleType.find_by_id(converted_params.dig(:sample, :sample_type_id))
-        sample = Sample.new(sample_type: sample_type)
-        sample = update_sample_with_params(converted_params, sample)
-        if sample.save
-          results.push({ ex_id: par[:ex_id], id: sample.id })
-        else
-          errors.push({ ex_id: par[:ex_id], error: sample.errors.messages })
-        end
-      end
-      raise ActiveRecord::Rollback if errors.any?
+    sample_type_id = params.permit(:sample_type_id).dig(:sample_type_id)
+    parameters = batch_upload_sample_params(params, sample_type_id)
+    sample_type = SampleType.find(sample_type_id)
+
+    if sample_type.nil?
+      err_message = "Sample Type with ID '#{sample_type_id}' not found."
+      errors.push err_message
+      raise err_message
+    end
+
+    if sample_type.locked?
+      err_message = 'Batch upload not allowed. Sample Type is currently locked! Wait until the lock is removed and try again.'
+      errors.push err_message
+      raise err_message
     end
-    status = errors.empty? ? :ok : :unprocessable_entity
-    render json: { status: status, errors: errors, results: results }, status: :ok
+
+    if parameters.count < 100
+      batch_create_processor = Samples::SampleBatchProcessor.new(sample_type_id: sample_type_id,
+                                                                 batch_process_params: parameters,
+                                                                 user: @current_user)
+      batch_create_processor.create!
+      results.concat(batch_create_processor.results)
+      errors.concat(batch_create_processor.errors)
+      raise "The following errors occurred: #{errors.join("\n")}" unless errors.empty?
+    else
+      SamplesBatchCreateJob.perform_later(sample_type_id, parameters, @current_user, true)
+      results = ['A background job has been launched.']
+    end
+    status = :ok
+  rescue StandardError => e
+    flash[:error] = e.message
+    status = :unprocessable_entity
+  ensure
+    render json: {
+      errors: errors,
+      results: results,
+      status: status
+    },
+    status: status
   end
 
   def batch_update
     errors = []
-    param_converter = Seek::Api::ParameterConverter.new("samples")
-    Sample.transaction do
-      params[:data].each do |par|
-        begin
-          converted_params = param_converter.convert(par)
-          sample = Sample.find(par[:id])
-          raise 'shouldnt get this far without manage rights' unless sample.can_manage?
-          sample = update_sample_with_params(converted_params, sample)
-          saved = sample.save
-          errors.push({ ex_id: par[:ex_id], error: sample.errors.messages }) unless saved
-        rescue
-          errors.push({ ex_id: par[:ex_id], error: "Can not be updated." })
-        end
-      end
-      raise ActiveRecord::Rollback if errors.any?
+    results = []
+    sample_type_id = params.permit(:sample_type_id).dig(:sample_type_id)
+    parameters = batch_upload_sample_params(params, sample_type_id)
+    sample_type = SampleType.find(sample_type_id)
+
+    if sample_type.nil?
+      err_message = "Sample Type with ID '#{sample_type_id}' not found."
+      errors.push err_message
+      raise err_message
+    end
+
+    if sample_type.locked?
+      err_message = 'Batch upload not allowed. Sample Type is currently locked! Wait until the lock is removed and try again.'
+      errors.push err_message
+      raise err_message
+    end
+
+    if sample_type.batch_upload_in_progress?
+      err_message = 'Batch upload not allowed. There is already a background job in progress for this Sample Type. Please wait and try again later.'
+      errors.push err_message
+      raise err_message
+    end
+
+    if parameters.count < 100
+      batch_update_processor = Samples::SampleBatchProcessor.new(sample_type_id: sample_type_id,
+                                                                 batch_process_params: parameters,
+                                                                 user: @current_user)
+      batch_update_processor.update!
+      errors.concat(batch_update_processor.errors)
+      results.concat(batch_update_processor.results)
+      raise "The following errors occurred: #{errors.join("\n")}" unless errors.empty?
+    else
+      SamplesBatchUpdateJob.perform_later(sample_type_id, parameters, @current_user, true)
+      results = ['A background job has been launched. This Sample Type will now lock itself as long as the background job is in progress.']
     end
-    status = errors.empty? ? :ok : :unprocessable_entity
-    render json: { status: status, errors: errors }, status: :ok
+    status = :ok
+  rescue StandardError => e
+    flash[:error] = e.message
+    status = :unprocessable_entity
+  ensure
+    render json: {
+      errors: errors,
+      results: results,
+      status: status
+    },
+    status: status
   end
 
   def batch_delete
     errors = []
-    Sample.transaction do
-      params[:data].each do |par|
-        begin
-          sample = Sample.find(par[:id])
-          errors.push({ ex_id: par[:ex_id], error: "Can not be deleted." }) if !(sample.can_delete? && sample.destroy)
-        rescue 
-          errors.push({ ex_id: par[:ex_id], error: sample.errors.messages })
-        end
-      end
-      raise ActiveRecord::Rollback if errors.any?
+    results = []
+    sample_type_id = params.permit(:sample_type_id).dig(:sample_type_id)
+    parameters = batch_delete_sample_params
+    sample_type = SampleType.find(sample_type_id)
+
+    if sample_type.nil?
+      err_message = "Sample Type with ID '#{sample_type_id}' not found."
+      errors.push err_message
+      raise err_message
+    end
+
+    if sample_type.locked?
+      err_message = 'Batch upload not allowed. Sample Type is currently locked! Wait until the lock is removed and try again.'
+      errors.push err_message
+      raise err_message
+    end
+
+    if sample_type.batch_upload_in_progress?
+      err_message = 'Batch upload not allowed. There is already a background job in progress for this Sample Type. Please wait and try again later.'
+      errors.push err_message
+      raise err_message
     end
-    status = errors.empty? ? :ok : :unprocessable_entity
-    render json: { status: status, errors: errors }, status: :ok
+
+    batch_update_processor = Samples::SampleBatchProcessor.new(sample_type_id: sample_type_id,
+                                                               batch_process_params: parameters,
+                                                               user: @current_user)
+    batch_update_processor.delete!
+    errors.concat(batch_update_processor.errors)
+    results.concat(batch_update_processor.results)
+    raise "The following errors occurred: #{errors.join("\n")}" unless errors.empty?
+
+    status = :ok
+  rescue StandardError => e
+    flash[:error] = e.message
+    status = :unprocessable_entity
+  ensure
+    render json: {
+      errors: errors,
+      results: results,
+      status: status
+    },
+           status: status
   end
 
   def typeahead
@@ -283,6 +362,31 @@ def query_form
 
   private
 
+  def batch_delete_sample_params(parameters = params)
+    batch_params = []
+    parameters[:data].each do |par|
+      batch_params << par.permit(:id, :ex_id)
+    end
+    batch_params
+  end
+
+  def batch_upload_sample_params(parameters = params, sample_type_id)
+    param_converter = Seek::Api::ParameterConverter.new("samples")
+    batch_params = []
+    parameters[:data].each do |par|
+      converted_params = param_converter.convert(par)
+      ex_id = par[:ex_id]
+      sample_type = SampleType.find_by_id(sample_type_id)
+      conv_par = sample_params(sample_type, converted_params).merge(ex_id: ex_id) unless ex_id.blank?
+      if parameters[:action] == 'batch_update'
+        sample_id = par[:id]
+        conv_par.merge!(id: sample_id)
+      end
+      batch_params << conv_par
+    end
+    batch_params
+  end
+
   def sample_params(sample_type = nil, parameters = params)
 
     sample_type_param_keys = []
@@ -304,6 +408,7 @@ def sample_params(sample_type = nil, parameters = params)
                               discussion_links_attributes:[:id, :url, :label, :_destroy])
   end
 
+
   def update_sample_with_params(parameters = params, sample = @sample)
     sample.assign_attributes(sample_params(sample.sample_type, parameters))
     update_sharing_policies(sample, parameters)
@@ -352,6 +457,7 @@ def filter_linked_samples(samples, link, options, template_attribute)
       end
     end
   end
+
   def templates_enabled?
     unless Seek::Config.isa_json_compliance_enabled
       flash[:error] = 'Not available'

app/controllers/single_pages_controller.rb

@@ -10,7 +10,6 @@ class SinglePagesController < ApplicationController
   before_action :check_user_logged_in,
                 only: %i[batch_sharing_permission_preview batch_change_permission_for_selected_items]
   respond_to :html, :js
-
   def show
     @project = Project.find(params[:id])
     @folders = project_folders
@@ -118,7 +117,7 @@ def export_to_excel
     end
   end
 
-  def upload_samples
+  def preview_upload_samples
     uploaded_file = params[:file]
     project_id = params[:project_id]
     @project = Project.find(project_id)

app/jobs/samples_batch_create_job.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class SamplesBatchCreateJob < ApplicationJob
+  queue_with_priority 1
+  queue_as QueueNames::SAMPLES
+
+  def perform(sample_type_id, parameters, user, send_email)
+    processor = Samples::SampleBatchProcessor.new(sample_type_id:, batch_process_params: parameters, user:, send_email:)
+    processor.create!
+  end
+end

app/jobs/samples_batch_update_job.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+class SamplesBatchUpdateJob < TaskJob
+  queue_with_priority 1
+  queue_as QueueNames::SAMPLES
+
+  def perform(sample_type_id, parameters, user, send_email)
+    processor = Samples::SampleBatchProcessor.new(sample_type_id:, batch_process_params: parameters, user:, send_email:)
+    processor.update!
+  end
+
+  def task
+    sample_type = SampleType.find(arguments[0])
+    sample_type.sample_batch_upload_task
+  end
+end