Merge branch 'main' into fix/file-worker-timeout

Author: dcollie2

Gemfile

@@ -14,6 +14,7 @@ gem "importmap-rails"
 gem "jbuilder"
 gem "kamal", require: false
 gem "mission_control-jobs"
+gem "ox"
 gem "pagy"
 gem "pg", "~> 1.6"
 gem "propshaft"

Gemfile.lock

@@ -301,6 +301,8 @@ GEM
     nokogiri (1.18.9-x86_64-linux-musl)
       racc (~> 1.4)
     ostruct (0.6.3)
+    ox (2.14.23)
+      bigdecimal (>= 3.0)
     pagy (9.3.5)
     parallel (1.27.0)
     parser (3.3.9.0)
@@ -543,6 +545,7 @@ DEPENDENCIES
   kamal
   letter_opener
   mission_control-jobs
+  ox
   pagy
   pg (~> 1.6)
   propshaft

app/services/language_content_processor.rb

@@ -25,16 +25,16 @@ def provider_files
   # this is needed to avoid loading all files into memory at once
   def language_files
     {
-      all_providers: FileToUpload.new(
-        content: ->(language) { XmlGenerator::AllProviders.new(language).perform },
-        name: "#{language.file_storage_prefix}Server_XML.xml",
-        path: "#{language.file_storage_prefix}CMES-Pi/assets/XML",
-      ),
-      all_providers_recent: FileToUpload.new(
-        content: ->(language) { XmlGenerator::AllProviders.new(language, recent: true).perform },
-        name: "#{language.file_storage_prefix}New_Uploads_Server_XML.xml",
-        path: "#{language.file_storage_prefix}CMES-Pi/assets/XML",
-      ),
+      # all_providers: FileToUpload.new(
+      #   content: ->(language) { XmlGenerator::AllProviders.new(language).perform },
+      #   name: "#{language.file_storage_prefix}Server_XML.xml",
+      #   path: "#{language.file_storage_prefix}CMES-Pi/assets/XML",
+      # ),
+      # all_providers_recent: FileToUpload.new(
+      #   content: ->(language) { XmlGenerator::AllProviders.new(language, recent: true).perform },
+      #   name: "#{language.file_storage_prefix}New_Uploads_Server_XML.xml",
+      #   path: "#{language.file_storage_prefix}CMES-Pi/assets/XML",
+      # ),
       tags: FileToUpload.new(
         content: ->(language) { TextGenerator::Tags.new(language).perform },
         name: "#{language.file_storage_prefix}tags.txt",

app/services/xml_generator/all_providers.rb

@@ -8,8 +8,8 @@ def initialize(language, **args)
 
   def xml_content(xml)
     language.providers.includes(:topics)
-      .map do |provider|
-        provider_xml(xml, provider)
+      .each do |provider|
+        xml << provider_xml(provider)
       end
   end
 end

app/services/xml_generator/base.rb

@@ -1,13 +1,19 @@
 class XmlGenerator::Base
   def perform
-    builder.to_xml
+    Ox.dump(builder)
   end
 
   private
 
   def builder
-    Nokogiri::XML::Builder.new do |xml|
-      xml.cmes { xml_content(xml) }
+    Ox::Document.new.tap do |doc|
+      instruct = Ox::Instruct.new(:xml)
+      instruct[:version] = "1.0"
+      doc << instruct
+
+      xml = Ox::Element.new("cmes")
+      xml_content(xml)
+      doc << xml
     end
   end
 end

app/services/xml_generator/single_provider.rb

@@ -9,30 +9,39 @@ def initialize(provider, **args)
   attr_reader :provider, :args
 
   def xml_content(xml)
-    provider_xml(xml, provider)
+    xml << provider_xml(provider)
   end
 
-  def provider_xml(xml, provider)
-    xml.content_provider(name: provider.name) {
+  def provider_xml(provider)
+    Ox::Element.new("content_provider").tap do |xml|
+      xml[:name] = provider.name
+
       grouped_topics(provider).each do |(year, month), topics|
-        xml.topic_year(year: year) {
-          xml.topic_month(month: month) {
+        xml << Ox::Element.new("topic_year").tap do |year_element|
+          year_element[:year] = year.to_s
+          year_element << Ox::Element.new("topic_month").tap do |month_element|
+            month_element[:month] = month
             topics.each do |topic|
-              xml.title(name: topic.title) {
-                xml.topic_id topic.id
-                xml.topic_files(files: "Files") {
+              month_element << Ox::Element.new("title").tap do |title_element|
+                title_element[:name] = topic.title
+                title_element << Ox::Element.new("topic_id").tap { |id| id << topic.id.to_s }
+                title_element << Ox::Element.new("topic_files").tap do |files|
+                  files[:files] = "Files"
                   topic.documents.each_with_index do |document, index|
                     next if document.content_type == "video/mp4"
-                    xml.send("file_name_#{index + 1}", document.filename, file_size: document.byte_size)
+                    files << Ox::Element.new("file_name_#{index + 1}").tap do |file_name|
+                      file_name << document.filename.to_s
+                      file_name[:file_size] = document.byte_size
+                    end
                   end
-                }
-                xml.topic_tags topic.current_tags_list.join(", ")
-              }
+                end
+                title_element << Ox::Element.new("topic_tags").tap { |tags| tags << topic.current_tags_list.join(", ") }
+              end
             end
-          }
-        }
+          end
+        end
       end
-    }
+    end
   end
 
   def grouped_topics(prov)

config/deploy.production.yml

@@ -19,26 +19,28 @@ registry:
 # Production environment variables
 env:
   secret:
-    - RAILS_MASTER_KEY
-    - DATABASE_URL
-    - SECRET_KEY_BASE
     - AWS_ACCESS_KEY_ID
     - AWS_SECRET_ACCESS_KEY
-    - AZURE_STORAGE_ACCOUNT_NAME
     - AZURE_STORAGE_ACCOUNT_KEY
+    - AZURE_STORAGE_ACCOUNT_NAME
+    - DATABASE_URL
+    - RAILS_MASTER_KEY
+    - SECRET_KEY_BASE
     - SCOUT_KEY
   clear:
-    RAILS_ENV: production
-    AWS_DEFAULT_REGION: us-east-1
     AWS_BUCKET_NAME: skillrx-production
-    SCOUT_APP_NAME: "SkillRX Production"
+    AWS_DEFAULT_REGION: us-east-1
     AWS_ENDPOINT_URL: ""  # Empty for real AWS
     AZURE_STORAGE_SHARE_NAME: contentshare
-    AZURE_MEDIA_FILES_SYNC_DISABLED: "true"
+    DATA_IMPORT_SOURCE: azure
     LOCALSTACK_DEBUG: "0"
+    RAILS_ENV: production
+    RAILS_MAX_THREADS: 5
     S3_SKIP_SIGNATURE_VALIDATION: "0"
+    SCOUT_APP_NAME: SkillRX Production
     SOLID_QUEUE_IN_PUMA: true
-    DATA_IMPORT_SOURCE: azure
+    WEB_CONCURRENCY: 2
+
 
 aliases:
   console: app exec --interactive --reuse "bin/rails console"

config/deploy.staging.yml

@@ -19,27 +19,27 @@ registry:
 # Staging environment variables
 env:
   secret:
-    - RAILS_MASTER_KEY
-    - DATABASE_URL
-    - SECRET_KEY_BASE
     - AWS_ACCESS_KEY_ID
     - AWS_SECRET_ACCESS_KEY
-    - AZURE_STORAGE_ACCOUNT_NAME
     - AZURE_STORAGE_ACCOUNT_KEY
+    - AZURE_STORAGE_ACCOUNT_NAME
+    - DATABASE_URL
+    - RAILS_MASTER_KEY
+    - SECRET_KEY_BASE
     - SCOUT_KEY
   clear:
-    RAILS_ENV: production
-    AWS_DEFAULT_REGION: us-east-1
     AWS_BUCKET_NAME: skillrx-staging
-    SCOUT_APP_NAME: "SkillRX Staging"
-    AZURE_STORAGE_SHARE_NAME: skillrx-staging-env
+    AWS_DEFAULT_REGION: us-east-1
     AWS_ENDPOINT_URL: ""  # Empty for real AWS
+    AZURE_STORAGE_SHARE_NAME: skillrx-staging-env
+    DATA_IMPORT_SOURCE: azure
     LOCALSTACK_DEBUG: "0"
-    S3_SKIP_SIGNATURE_VALIDATION: "0"
+    RAILS_ENV: production
     RAILS_MAX_THREADS: 5
-    WEB_CONCURRENCY: 2
+    S3_SKIP_SIGNATURE_VALIDATION: "0"
+    SCOUT_APP_NAME: SkillRX Staging
     SOLID_QUEUE_IN_PUMA: true
-    DATA_IMPORT_SOURCE: azure
+    WEB_CONCURRENCY: 2
 
 aliases:
   console: app exec --interactive --reuse "bin/rails console"

db/seeds.rb

@@ -107,3 +107,21 @@
 end
 
 puts "Users created!"
+
+if ENV["SEED_MODE_HEAVY_LOAD"]
+  times = ENV["SEED_MODE_HEAVY_LOAD"].to_i
+  times.times do
+    provider = Provider.find_or_create_by!(name: Faker::Company.name, provider_type: "government")
+    times.times do
+      Topic.find_or_create_by!(
+        title: Faker::Book.title,
+        description: Faker::Lorem.paragraph,
+        language_id: Language.first.id,
+        provider_id: provider.id,
+        uid: SecureRandom.uuid,
+        published_at: Time.now - rand(1..365).days,
+        state: [ :active, :archived ].sample,
+      )
+    end
+  end
+end

spec/services/language_content_processor_spec.rb

@@ -15,13 +15,13 @@
   end
 
   it "processes content for every language" do
-    files_number = language.providers.size + 9 # 2 xml files for all provides, 2 text files for tags, 5 csv files
+    files_number = language.providers.size + 7 # 2 xml files for all provides, 2 text files for tags, 5 csv files
     subject.perform
 
     expect(FileUploadJob).to have_received(:perform_later).exactly(files_number).times
 
-    expect(FileUploadJob).to have_received(:perform_later).with(language.id, "all_providers", "file")
-    expect(FileUploadJob).to have_received(:perform_later).with(language.id, "all_providers_recent", "file")
+    # expect(FileUploadJob).to have_received(:perform_later).with(language.id, "all_providers", "file")
+    # expect(FileUploadJob).to have_received(:perform_later).with(language.id, "all_providers_recent", "file")
     expect(FileUploadJob).to have_received(:perform_later).with(language.id, "tags", "file")
     expect(FileUploadJob).to have_received(:perform_later).with(language.id, "tags_and_title", "file")
     expect(FileUploadJob).to have_received(:perform_later).with(language.id, "files", "file")