AO3-7461 Change send_data to a background job powered workflow

app/controllers/admin/admin_users_controller.rb

@@ -46,11 +46,12 @@ def bulk_search
       @users = found_users.paginate(page: params[:page] || 1)
 
       if params[:download_button]
-        header = [%w(Email Username)]
-        found = found_users.map { |u| [u.email, u.login] }
-        not_found = not_found_emails.map { |email| [email, ""] }
-        send_csv_data(header + found + not_found, "bulk_user_search_#{Time.now.strftime("%Y-%m-%d-%H%M")}.csv")
-        flash.now[:notice] = ts("Downloaded CSV")
+        queue_csv_download(
+          kind: "bulk_user_search",
+          arguments: { emails: @emails },
+          filename: "bulk_user_search_#{Time.current.strftime('%Y-%m-%d-%H%M')}.csv"
+        )
+        return
       end
       @results = {
         total: @emails.size,

app/controllers/challenge_signups_controller.rb

@@ -119,9 +119,12 @@ def index
         if privileged_collection_admin? ||
            (@collection.gift_exchange? && @challenge.user_allowed_to_see_signups?(current_user)) ||
         (@collection.prompt_meme? && @collection.user_is_maintainer?(current_user))
-          csv_data = self.send("#{@challenge.class.name.underscore}_to_csv")
-          filename = "#{@collection.name}_signups_#{Time.now.strftime('%Y-%m-%d-%H%M')}.csv"
-          send_csv_data(csv_data, filename)
+          filename = "#{@collection.name}_signups_#{Time.current.strftime('%Y-%m-%d-%H%M')}.csv"
+          queue_csv_download(
+            kind: "challenge_signups",
+            arguments: { collection_id: @collection.id },
+            filename: filename
+          )
         else
           flash[:error] = ts("You aren't allowed to see the CSV summary.")
           redirect_to collection_path(@collection) rescue redirect_to '/' and return

app/controllers/downloads_controller.rb

@@ -3,25 +3,16 @@ class DownloadsController < ApplicationController
   before_action :load_work, only: :show
   before_action :check_download_posted_status, only: :show
   before_action :check_download_visibility, only: :show
-  around_action :remove_downloads, only: :show
-
   def show
     respond_to :html, :pdf, :mobi, :epub, :azw3
-    @download = Download.new(@work, mime_type: request.format)
-    @download.generate
-
-    # Make sure we were able to generate the download.
-    unless @download.exists?
-      flash[:error] = ts("We were not able to render this work. Please try again in a little while or try another format.")
-      redirect_to work_path(@work)
-      return
-    end
-
-    # Send file synchronously so we don't delete it before we have finished
-    # sending it
-    File.open(@download.file_path, 'r') do |f|
-      send_data f.read, filename: "#{@download.file_name}.#{@download.file_type}", type: @download.mime_type
-    end
+    download = Download.new(@work, mime_type: request.format)
+    generated_download = GeneratedDownload.create!(
+      kind: "work",
+      arguments: { work_id: @work.id, format: download.file_type },
+      filename: "#{download.file_name}.#{download.file_type}"
+    )
+    GeneratedDownloadJob.perform_later(generated_download)
+    redirect_to generated_download_path(token: generated_download.token), status: :see_other
   end
 
 protected
@@ -42,14 +33,6 @@ def load_work
     @work = Work.find(params[:id])
   end
 
-  # We're currently just writing everything to tmp and feeding them through
-  # nginx so we don't want to keep the files around.
-  def remove_downloads
-    yield
-  ensure
-    @download.remove
-  end
-
   # We can't use check_visibility because this controller doesn't have access to
   # cookies on production or staging.
   def check_download_visibility

app/controllers/generated_downloads_controller.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+class GeneratedDownloadsController < ApplicationController
+  def show
+    @generated_download = GeneratedDownload.find_by!(token: params[:token])
+
+    if @generated_download.expired?
+      @generated_download.file.purge_later if @generated_download.file.attached?
+      head :gone
+    elsif @generated_download.status == "ready" && @generated_download.file.attached?
+      blob = @generated_download.file.blob
+      redirect_to blob.service.url(
+        blob.key,
+        expires_in: ActiveStorage.service_urls_expire_in,
+        filename: blob.filename,
+        content_type: blob.content_type,
+        disposition: :attachment
+      ), allow_other_host: true
+    elsif @generated_download.status == "failed"
+      render :show, status: :unprocessable_content
+    else
+      render :show, status: :accepted
+    end
+  end
+end

app/controllers/tag_wranglers_controller.rb

@@ -45,19 +45,12 @@ def report_csv
     authorize :wrangling
 
     wrangler = User.find_by!(login: params[:id])
-    wrangled_tags = Tag
-      .where(last_wrangler: wrangler)
-      .limit(ArchiveConfig.WRANGLING_REPORT_LIMIT)
-      .includes(:merger, :parents)
-    results = [%w[Name Last\ Updated Type Merger Fandoms Unwrangleable]]
-    wrangled_tags.find_each(order: :desc) do |tag|
-      merger = tag.merger&.name || ""
-      fandoms = tag.parents.filter_map { |parent| parent.name if parent.is_a?(Fandom) }
-        .join(", ")
-      results << [tag.name, tag.updated_at, tag.type, merger, fandoms, tag.unwrangleable]
-    end
     filename = "wrangled_tags_#{wrangler.login}_#{Time.now.utc.strftime('%Y-%m-%d-%H%M')}.csv"
-    send_csv_data(results, filename)
+    queue_csv_download(
+      kind: "tag_wrangler",
+      arguments: { user_id: wrangler.id },
+      filename: filename
+    )
   end
 
   def create

app/helpers/exports_helper.rb

@@ -1,17 +1,33 @@
 # frozen_string_literal: true
-module ExportsHelper
 
-  def send_csv_data(content_array, filename)
-    send_data(export_csv(content_array), filename: filename, type: :csv)
+module ExportsHelper
+  def queue_csv_download(kind:, arguments:, filename:)
+    generated_download = GeneratedDownload.create!(
+      kind: kind,
+      arguments: arguments,
+      filename: filename
+    )
+    GeneratedDownloadJob.perform_later(generated_download)
+    redirect_to generated_download_path(token: generated_download.token), status: :see_other
   end
 
   # Tab-separated CSV with utf-16le encoding (unicode) and byte order
   # mark. This seems to be the only variant Excel can get
   # automatically into proper table format. OpenOffice handles it
   # well, too.
   def export_csv(content_array)
-    csv_data = content_array.map { |x| x.to_csv(col_sep: "\t", encoding: "utf-8") }.join
-    byte_order_mark = "\uFEFF"
-    (byte_order_mark + csv_data).encode("utf-16le", "utf-8", invalid: :replace, undef: :replace, replace: "")
+    io = StringIO.new("".b)
+    ExportsHelper.write_csv(io, content_array)
+    io.string.force_encoding("utf-16le")
+  end
+
+  def self.write_csv(io, rows)
+    io.write("\uFEFF".encode("utf-16le"))
+    rows.each do |row|
+      encoded_row = row.to_csv(col_sep: "\t", encoding: "utf-8").encode(
+        "utf-16le", "utf-8", invalid: :replace, undef: :replace, replace: ""
+      )
+      io.write(encoded_row)
+    end
   end
 end

app/jobs/generated_download_job.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+class GeneratedDownloadJob < ApplicationJob
+  queue_as :utilities
+
+  def perform(generated_download)
+    return if generated_download.expired?
+
+    generated_download.update!(status: "processing", error: nil)
+
+    case generated_download.kind
+    when "work"
+      attach_work(generated_download)
+    when "challenge_signups", "tag_wrangler", "bulk_user_search"
+      attach_csv(generated_download)
+    else
+      raise ArgumentError, "Unknown generated download kind: #{generated_download.kind}"
+    end
+
+    generated_download.update!(status: "ready")
+  rescue StandardError => e
+    generated_download.update_columns(status: "failed", error: e.message, updated_at: Time.current)
+    raise
+  end
+
+  private
+
+  def attach_work(generated_download)
+    arguments = generated_download.arguments
+    download = Download.new(
+      Work.find(arguments.fetch("work_id")),
+      format: arguments.fetch("format")
+    ).generate
+    raise "Download generation failed" unless download.exists?
+
+    file_path = File.join(download.dir, File.basename(download.file_path))
+    File.open(file_path, "rb") do |file|
+      attach_file(generated_download, file, download.mime_type)
+    end
+  ensure
+    download&.remove
+  end
+
+  def attach_csv(generated_download)
+    tempfile = Tempfile.new(["generated-download", ".csv"])
+    tempfile.binmode
+    ExportsHelper.write_csv(tempfile, csv_rows(generated_download))
+    tempfile.rewind
+    attach_file(generated_download, tempfile, "text/csv; charset=utf-16le")
+  ensure
+    tempfile&.close!
+  end
+
+  def attach_file(generated_download, io, content_type)
+    blob = ActiveStorage::Blob.create_after_unfurling!(
+      io: io,
+      filename: generated_download.filename,
+      content_type: content_type,
+      identify: false
+    )
+    io.rewind
+    blob.service.upload(
+      blob.key,
+      io,
+      checksum: blob.checksum,
+      filename: blob.filename,
+      content_type: content_type,
+      disposition: :attachment
+    )
+    generated_download.file.attach(blob)
+  rescue StandardError
+    blob&.purge
+    raise
+  end
+
+  def csv_rows(generated_download)
+    arguments = generated_download.arguments
+    case generated_download.kind
+    when "tag_wrangler"
+      tag_wrangler_rows(arguments.fetch("user_id"))
+    when "bulk_user_search"
+      bulk_user_search_rows(arguments.fetch("emails"))
+    when "challenge_signups"
+      challenge_signup_rows(arguments.fetch("collection_id"))
+    end
+  end
+
+  def tag_wrangler_rows(user_id)
+    wrangler = User.find(user_id)
+    rows = [["Name", "Last Updated", "Type", "Merger", "Fandoms", "Unwrangleable"]]
+    Tag.where(last_wrangler: wrangler)
+      .limit(ArchiveConfig.WRANGLING_REPORT_LIMIT)
+      .includes(:merger, :parents)
+      .find_each(order: :desc) do |tag|
+        fandoms = tag.parents
+          .filter_map { |parent| parent.name if parent.is_a?(Fandom) }
+          .join(", ")
+        rows << [tag.name, tag.updated_at, tag.type, tag.merger&.name || "", fandoms, tag.unwrangleable]
+      end
+    rows
+  end
+
+  def bulk_user_search_rows(emails)
+    found_users, not_found_emails = User.search_multiple_by_email(emails)
+    [%w[Email Username]] +
+      found_users.map { |user| [user.email, user.login] } +
+      not_found_emails.map { |email| [email, ""] }
+  end
+
+  def challenge_signup_rows(collection_id)
+    collection = Collection.find(collection_id)
+    controller = ChallengeSignupsController.new
+    controller.instance_variable_set(:@collection, collection)
+    controller.instance_variable_set(:@challenge, collection.challenge)
+    controller.define_singleton_method(:collection_signup_url) do |requested_collection, signup|
+      Rails.application.routes.url_helpers.collection_signup_url(
+        requested_collection,
+        signup,
+        host: ArchiveConfig.APP_HOST,
+        protocol: "https"
+      )
+    end
+    controller.send("#{collection.challenge.class.name.underscore}_to_csv")
+  end
+end

app/models/generated_download.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+class GeneratedDownload < ApplicationRecord
+  EXPIRATION = 1.day
+  STATUSES = %w[pending processing ready failed].freeze
+
+  has_one_attached :file
+
+  serialize :arguments, coder: JSON
+
+  validates :token, :kind, :filename, :expires_at, presence: true
+  validates :token, uniqueness: true
+  validates :status, inclusion: { in: STATUSES }
+
+  before_validation :set_defaults, on: :create
+
+  def self.cleanup
+    where(expires_at: ...Time.current).find_each do |download|
+      download.file.purge if download.file.attached?
+      download.destroy!
+    end
+  end
+
+  def expired?
+    expires_at.past?
+  end
+
+  private
+
+  def set_defaults
+    self.token ||= SecureRandom.urlsafe_base64(32)
+    self.expires_at ||= EXPIRATION.from_now
+  end
+end

app/views/generated_downloads/show.html.erb

@@ -0,0 +1,16 @@
+<% content_for :head do %>
+  <% unless @generated_download.status == "failed" %>
+    <meta http-equiv="refresh" content="5" />
+  <% end %>
+<% end %>
+
+<h2 class="heading"><%= t(".heading") %></h2>
+
+<% if @generated_download.status == "failed" %>
+  <p class="error"><%= t(".failed") %></p>
+<% else %>
+  <p class="notice">
+    <%= t(".processing") %>
+  </p>
+  <p><%= link_to t(".check_status"), generated_download_path(token: @generated_download.token) %></p>
+<% end %>

config/locales/views/en.yml

@@ -1207,6 +1207,12 @@ en:
         current_html: For current updates on AO3 performance or downtime, please check our %{status_page_link} and follow %{bluesky_link} on Bluesky or %{tumblr_link} on Tumblr.
         status_page: status page
         tumblr: ao3org
+  generated_downloads:
+    show:
+      check_status: Check download status
+      failed: We were not able to prepare this download. Please try again.
+      heading: Preparing download
+      processing: Your file is being prepared. This page will download it automatically when it is ready.
   help:
     collectibles_add_to_collection:
       collection_names: Note that you need to use the collection's name, which gets used in the collection's URL, and not the collection's spiffy title (because different collections can have the same title). A collection's name is the equivalent of your user login. Names will be auto-completed for you if you have JavaScript turned on.

config/resque_schedule.yml

@@ -5,6 +5,13 @@ run_main_reindex_queues:
   args: main
   description: "Kick off a reindex of all main content indexing"
 
+cleanup_generated_downloads:
+  every: 1h
+  class: "GeneratedDownload"
+  queue: utilities
+  args: cleanup
+  description: "Remove expired generated downloads from storage and the database."
+
 run_background_reindex_queue:
   every: 11m
   class: "ScheduledReindexJob"

config/routes.rb

@@ -59,6 +59,7 @@
   #### DOWNLOADS ####
 
   get 'downloads/:id/:download_title.:format' => 'downloads#show', as: 'download'
+  get "generated_downloads/:token" => "generated_downloads#show", as: "generated_download"
 
   #### OPEN DOORS ####
   namespace :opendoors do