Revert "Allow sending set of provider-specific CSV files #420 (#430)"

This reverts commit 0d4b281, reversing
changes made to c384912.

Author: dcollie2

app/jobs/file_upload_job.rb

@@ -1,7 +1,7 @@
 class FileUploadJob < ApplicationJob
   # Consider removing concurrency limits due to SolidQueue blocking issues
   # or use a more specific key to avoid blocking all jobs for a language
-  limits_concurrency to: 3, key: ->(*args) { "hard-limit" }
+  limits_concurrency to: 3, key: ->(_language_id, content_id, _content_type) { "hard-limit" }
 
   retry_on AzureFileShares::Errors::ApiError, wait: :exponentially_longer, attempts: 3
   retry_on Timeout::Error, wait: :exponentially_longer, attempts: 2
@@ -12,34 +12,34 @@ class FileUploadJob < ApplicationJob
     Rails.logger.error "Suggestion: Check provider names for invalid characters if Azure API errors"
   end
 
-  def perform(language_id, file_id, provider_id = nil, share = ENV["AZURE_STORAGE_SHARE_NAME"])
+  def perform(language_id, content_id, content_type, share = ENV["AZURE_STORAGE_SHARE_NAME"])
     @language = Language.find(language_id)
-    @share = share
-    @file_id = file_id.to_sym
     @processor = LanguageContentProcessor.new(language)
+    @share = share
 
-    send_provider_content(provider_id) if provider_id.present?
-    send_language_content if provider_id.blank?
+    send_provider_content(content_id) if content_type == "provider"
+    send_language_content(content_id.to_sym) if content_type == "file"
   end
 
   private
 
-  attr_reader :language, :file_id, :share, :processor
+  attr_reader :language, :processor, :share
 
   def send_provider_content(provider_id)
     provider = language.providers.find(provider_id)
-    file = processor.provider_files[file_id]
-    return unless provider && file
-
-    FileWorker.new(
-      share:,
-      name: file.name[provider],
-      path: file.path,
-      file: file.content[provider],
-    ).send
+    return unless provider
+
+    processor.provider_files.each do |file|
+      FileWorker.new(
+        share:,
+        name: file.name[provider],
+        path: file.path,
+        file: file.content[provider],
+      ).send
+    end
   end
 
-  def send_language_content
+  def send_language_content(file_id)
     file = processor.language_files[file_id]
     return unless file
 

app/services/csv_generator/base.rb

@@ -1,9 +1,4 @@
 class CsvGenerator::Base
-  def initialize(source, **args)
-    @source = source
-    @args = args
-  end
-
   def perform
     CSV.generate(row_sep: "\n") do |csv|
       csv << headers
@@ -12,18 +7,4 @@ def perform
       end
     end
   end
-
-  private
-
-  attr_reader :source, :args
-
-  def topics_collection
-    return source.topics if provider?
-
-    source.topics
-  end
-
-  def language = language? ? source : args.fetch(:language)
-  def language? = source.is_a?(Language)
-  def provider? = source.is_a?(Provider)
 end

app/services/csv_generator/files.rb

@@ -1,12 +1,19 @@
 class CsvGenerator::Files < CsvGenerator::Base
+  def initialize(language, **args)
+    @language = language
+    @args = args
+  end
+
   private
 
+  attr_reader :language, :args
+
   def headers
     %w[FileID TopicID FileName FileType FileSize]
   end
 
   def scope
-    topics_collection.active
+    language.topics.active
       .flat_map do |topic|
         topic.documents.map do |doc|
           [

app/services/csv_generator/tag_details.rb

@@ -1,12 +1,19 @@
 class CsvGenerator::TagDetails < CsvGenerator::Base
+  def initialize(language, **args)
+    @language = language
+    @args = args
+  end
+
   private
 
+  attr_reader :language, :args
+
   def headers
     %w[TagID Tag]
   end
 
   def scope
-    topics_collection.active.includes(:tags)
+    language.topics.active.includes(:tags)
       .flat_map { |topic| topic.tags_on(language.code.to_sym) }
       .uniq
       .map do |tag|

app/services/csv_generator/topic_authors.rb

@@ -1,11 +1,18 @@
 class CsvGenerator::TopicAuthors < CsvGenerator::Base
+  def initialize(language, **args)
+    @language = language
+    @args = args
+  end
+
   private
 
+  attr_reader :language, :args
+
   def headers
     %w[TopicID AuthorID]
   end
 
   def scope
-    topics_collection.active.map { |topic| [ topic.id, 0 ] }
+    language.topics.active.map { |topic| [ topic.id, 0 ] }
   end
 end

app/services/csv_generator/topic_tags.rb

@@ -1,12 +1,19 @@
 class CsvGenerator::TopicTags < CsvGenerator::Base
+  def initialize(language, **args)
+    @language = language
+    @args = args
+  end
+
   private
 
+  attr_reader :language, :args
+
   def headers
     %w[TopicID TagID]
   end
 
   def scope
-    topics_collection.active.includes(:tags)
+    language.topics.active.includes(:tags)
       .flat_map do |topic|
         topic.tags_on(language.code.to_sym).map do |tag|
           [

app/services/csv_generator/topics.rb

@@ -1,12 +1,19 @@
 class CsvGenerator::Topics < CsvGenerator::Base
+  def initialize(language, **args)
+    @language = language
+    @args = args
+  end
+
   private
 
+  attr_reader :language, :args
+
   def headers
     %w[TopicID TopicName TopicVolume TopicIssue TopicYear TopicMonth ContentProvider]
   end
 
   def scope
-    topics_collection.active.includes(:provider)
+    language.topics.active.includes(:provider)
       .map do |topic|
         [
           topic.id,

app/services/language_content_processor.rb

@@ -12,38 +12,13 @@ def perform
   # this is needed to avoid loading all files into memory at once
   # Field 'name' is a lambda to allow dynamic naming based on the provider
   def provider_files
-    {
-      single_provider: FileToUpload.new(
+    [
+      FileToUpload.new(
         content: ->(provider) { XmlGenerator::SingleProvider.new(provider).perform },
         name: ->(provider) { "#{language.file_storage_prefix}#{provider.name.parameterize}.xml" },
         path: "#{language.file_storage_prefix}CMES-Pi/assets/XML",
       ),
-      files: FileToUpload.new(
-        content: ->(provider) { CsvGenerator::Files.new(provider).perform },
-        name: ->(provider) { "#{language.file_storage_prefix}#{provider.name.parameterize}-file.csv" },
-        path: "#{language.file_storage_prefix}CMES-v2/assets/csv",
-      ),
-      topics: FileToUpload.new(
-        content: ->(provider) { CsvGenerator::Topics.new(provider).perform },
-        name: ->(provider) { "#{language.file_storage_prefix}#{provider.name.parameterize}-topic.csv" },
-        path: "#{language.file_storage_prefix}CMES-v2/assets/csv",
-      ),
-      tag_details: FileToUpload.new(
-        content: ->(provider) { CsvGenerator::TagDetails.new(provider, language:).perform },
-        name: ->(provider) { "#{language.file_storage_prefix}#{provider.name.parameterize}-tag.csv" },
-        path: "#{language.file_storage_prefix}CMES-v2/assets/csv",
-      ),
-      topic_tags: FileToUpload.new(
-        content: ->(provider) { CsvGenerator::TopicTags.new(provider, language:).perform },
-        name: ->(provider) { "#{language.file_storage_prefix}#{provider.name.parameterize}-topic-tag.csv" },
-        path: "#{language.file_storage_prefix}CMES-v2/assets/csv",
-      ),
-      topic_authors: FileToUpload.new(
-        content: ->(provider) { CsvGenerator::TopicAuthors.new(provider).perform },
-        name: ->(provider) { "#{language.file_storage_prefix}#{provider.name.parameterize}-topic-author.csv" },
-        path: "#{language.file_storage_prefix}CMES-v2/assets/csv",
-      ),
-    }
+    ]
   end
 
   # Field 'content' is a lambda to allow lazy evaluation
@@ -90,7 +65,7 @@ def language_files
         name: "#{language.file_storage_prefix}TopicTag.csv",
         path: "#{language.file_storage_prefix}CMES-v2/assets/csv",
       ),
-      topic_authors: FileToUpload.new(
+        topic_authors: FileToUpload.new(
         content: ->(language) { CsvGenerator::TopicAuthors.new(language).perform },
         name: "#{language.file_storage_prefix}TopicAuthor.csv",
         path: "#{language.file_storage_prefix}CMES-v2/assets/csv",
@@ -104,13 +79,11 @@ def language_files
 
   def process_language_content!
     language_files.keys.each do |file_id|
-      FileUploadJob.perform_later(language.id, file_id.to_s)
+      FileUploadJob.perform_later(language.id, file_id.to_s, "file")
     end
 
     language.providers.distinct.find_each do |provider|
-      provider_files.keys.each do |file_id|
-        FileUploadJob.perform_later(language.id, file_id.to_s, provider.id)
-      end
+      FileUploadJob.perform_later(language.id, provider.id, "provider")
     end
   end
 end

spec/jobs/file_upload_job_spec.rb

@@ -19,7 +19,7 @@
             file: file.content[language],
           )
 
-          described_class.perform_now(language.id, file_id.to_s)
+          described_class.perform_now(language.id, file_id.to_s, "file")
         end
       end
     end
@@ -30,31 +30,42 @@
       before { create(:topic, :tagged, language:, provider:) }
 
       it "processes specific file" do
-        processor.provider_files.each do |file_id, file|
-          expect(FileWorker).to receive(:new).with(
-            share: ENV["AZURE_STORAGE_SHARE_NAME"],
-            name: file.name[provider],
-            path: file.path,
-            file: file.content[provider],
-          )
+        expect(FileWorker).to receive(:new).with(
+          share: ENV["AZURE_STORAGE_SHARE_NAME"],
+          name: "#{language.file_storage_prefix}test-provider.xml",
+          path: "#{language.file_storage_prefix}CMES-Pi/assets/XML",
+          file: XmlGenerator::SingleProvider.new(provider).perform,
+        )
 
-          described_class.perform_now(language.id, file_id.to_s, provider.id)
-        end
+        described_class.perform_now(language.id, provider.id, "provider")
       end
 
       context "when provider name contains /" do
         let(:provider) { create(:provider, name: "Test/Provider") }
 
         it "replaces / with - in the file name" do
-          processor.provider_files.each do |file_id, file|
+          expect(FileWorker).to receive(:new).with(
+            share: ENV["AZURE_STORAGE_SHARE_NAME"],
+            name: "#{language.file_storage_prefix}test-provider.xml",
+            path: "#{language.file_storage_prefix}CMES-Pi/assets/XML",
+            file: XmlGenerator::SingleProvider.new(provider).perform,
+          )
+
+          described_class.perform_now(language.id, provider.id, "provider")
+        end
+
+        context "when provider name contains /" do
+          let(:provider) { create(:provider, name: "WHO/Guidelines") }
+
+          it "replaces / with - in the file name" do
             expect(FileWorker).to receive(:new).with(
               share: ENV["AZURE_STORAGE_SHARE_NAME"],
-              name: file.name[provider],
-              path: file.path,
-              file: file.content[provider],
+              name: "#{language.file_storage_prefix}who-guidelines.xml",
+              path: "#{language.file_storage_prefix}CMES-Pi/assets/XML",
+              file: XmlGenerator::SingleProvider.new(provider).perform,
             )
 
-            described_class.perform_now(language.id, file_id.to_s, provider.id)
+            described_class.perform_now(language.id, provider.id, "provider")
           end
         end
       end

spec/services/csv_generator/files_spec.rb

@@ -1,11 +1,9 @@
 require "rails_helper"
 
 RSpec.describe CsvGenerator::Files do
-  subject { described_class.new(source, **args) }
+  subject { described_class.new(language) }
 
   let(:language) { create(:language) }
-  let(:source) { language }
-  let(:args) { {} }
   let(:header) { "FileID,TopicID,FileName,FileType,FileSize\n" }
 
   it "generates empty csv" do
@@ -43,15 +41,6 @@
         expect(subject.perform).to eq(data)
       end
     end
-
-    context "when generated for provider" do
-      let(:source) { topic.provider }
-      let(:args) { { language: } }
-
-      it "generates csv with documents info" do
-        expect(subject.perform).to eq(data)
-      end
-    end
   end
 
   context "when topic exists but archived" do