Development

app/Actions/Project/PublishProject.php

@@ -2,6 +2,7 @@
 
 namespace App\Actions\Project;
 
+use App\Jobs\ProcessMetadataExtractionBagitGenerationJob;
 use App\Models\Project;
 use App\Support\Public\PublicMoleculeAggregates;
 
@@ -21,6 +22,9 @@ public function publish($project)
         foreach ($studies as $study) {
             $study->is_public = true;
             $study->save();
+            if ($study->has_nmrium && filled($study->download_url)) {
+                ProcessMetadataExtractionBagitGenerationJob::dispatch($study->id);
+            }
             $datasets = $study->datasets;
             foreach ($datasets as $dataset) {
                 $dataset->is_public = true;

app/Actions/Study/PublishStudy.php

@@ -2,6 +2,7 @@
 
 namespace App\Actions\Study;
 
+use App\Jobs\ProcessMetadataExtractionBagitGenerationJob;
 use App\Models\Study;
 use App\Services\ChemotionRepositoryTrackerService;
 use App\Support\Public\PublicMoleculeAggregates;
@@ -25,6 +26,10 @@ public function publish($study)
             $dataset->save();
         }
 
+        if ($study->is_public && $study->has_nmrium && filled($study->download_url)) {
+            ProcessMetadataExtractionBagitGenerationJob::dispatch($study->id);
+        }
+
         PublicMoleculeAggregates::forgetPublicCatalogTotalCache();
 
         // Track publication if this is an ELN submission

app/Console/Commands/BackfillBagitArchiveLinks.php

@@ -0,0 +1,250 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Models\Study;
+use Illuminate\Console\Command;
+use Illuminate\Contracts\Filesystem\Filesystem;
+use Illuminate\Support\Facades\Log;
+use Illuminate\Support\Facades\Storage;
+use RecursiveDirectoryIterator;
+use RecursiveIteratorIterator;
+use Throwable;
+use ZipArchive;
+
+class BackfillBagitArchiveLinks extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'nmrxiv:backfill-bagit-archives
+                            {--ids= : Comma-separated study folder names (e.g. S1,S100) to process}
+                            {--limit= : Limit number of study folders to process}
+                            {--force : Regenerate the archive even if bagit_archive_link is already set}
+                            {--dry-run : Report what would happen without writing any changes}';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Zip already-generated BagIt folders on storage and backfill studies.bagit_archive_link';
+
+    private int $processed = 0;
+
+    private int $skippedHasLink = 0;
+
+    private int $skippedNotFound = 0;
+
+    private int $failed = 0;
+
+    /**
+     * Execute the console command.
+     */
+    public function handle(): int
+    {
+        $sourceDisk = Storage::disk(config('nmrxiv.spectra_parsing.storage_disk', 'local'));
+        $basePath = trim(config('nmrxiv.spectra_parsing.storage_path', 'spectra_parse'), '/');
+
+        $folders = collect($sourceDisk->directories($basePath))
+            ->map(fn (string $path) => basename($path))
+            ->filter(fn (string $name) => preg_match('/^S\d+$/i', $name) === 1)
+            ->values();
+
+        if ($ids = $this->option('ids')) {
+            $wanted = array_map(fn (string $id) => strtoupper(trim($id)), explode(',', $ids));
+            $folders = $folders->filter(fn (string $name) => in_array(strtoupper($name), $wanted, true))->values();
+        }
+
+        if ($limit = $this->option('limit')) {
+            $folders = $folders->take((int) $limit);
+        }
+
+        if ($folders->isEmpty()) {
+            $this->warn('No matching BagIt study folders found.');
+
+            return self::SUCCESS;
+        }
+
+        $this->info("Found {$folders->count()} BagIt study folders to evaluate.");
+
+        $bar = $this->output->createProgressBar($folders->count());
+        $bar->setFormat('verbose');
+
+        foreach ($folders as $folderName) {
+            $this->processFolder($sourceDisk, $basePath, $folderName);
+            $bar->advance();
+        }
+
+        $bar->finish();
+        $this->newLine(2);
+
+        $this->table(
+            ['Processed', 'Skipped (has link)', 'Skipped (study not found)', 'Failed'],
+            [[$this->processed, $this->skippedHasLink, $this->skippedNotFound, $this->failed]]
+        );
+
+        return self::SUCCESS;
+    }
+
+    /**
+     * Zip and backfill a single study's BagIt folder.
+     */
+    private function processFolder(Filesystem $sourceDisk, string $basePath, string $folderName): void
+    {
+        $identifier = (int) substr($folderName, 1);
+
+        $study = Study::where('identifier', $identifier)
+            ->where('is_public', true)
+            ->first();
+
+        if (! $study) {
+            $this->skippedNotFound++;
+            $this->line("  [skip] {$folderName}: no matching public study found");
+
+            return;
+        }
+
+        if ($study->bagit_archive_link && ! $this->option('force')) {
+            $this->skippedHasLink++;
+
+            return;
+        }
+
+        if ($this->option('dry-run')) {
+            $this->line("  [dry-run] Would archive {$folderName} for study {$study->identifier}");
+
+            return;
+        }
+
+        $remoteBagDir = "{$basePath}/{$folderName}";
+        $tempDir = storage_path('app/bagit_backfill_'.uniqid());
+        $zipPath = null;
+
+        try {
+            $this->downloadDirectory($sourceDisk, $remoteBagDir, $tempDir);
+
+            if (! file_exists($tempDir.'/bagit.txt')) {
+                throw new \RuntimeException("bagit.txt not found in {$remoteBagDir}, skipping invalid bag");
+            }
+
+            $zipPath = $this->zipDirectory($tempDir, $folderName);
+
+            $archiveKey = "archive/{$folderName}/{$folderName}.zip";
+            $archiveDisk = Storage::disk(config('filesystems.default_public', 'local'));
+            $archiveContents = file_get_contents($zipPath);
+            if ($archiveContents === false) {
+                throw new \RuntimeException("Failed to read generated zip for {$folderName}");
+            }
+
+            if (! $archiveDisk->put($archiveKey, $archiveContents, 'public')) {
+                throw new \RuntimeException("Failed to upload archive to disk for {$folderName}: {$archiveKey}");
+            }
+
+            $archiveUrl = $archiveDisk->url($archiveKey);
+
+            $study->update([
+                'bagit_archive_link' => $archiveUrl,
+                'metadata_bagit_generation_status' => 'completed',
+                'metadata_bagit_generation_logs' => array_merge((array) ($study->metadata_bagit_generation_logs ?: []), [
+                    'backfilled_at' => now()->toIso8601String(),
+                    'bagit_archive_link' => $archiveUrl,
+                    'archive_path' => $archiveKey,
+                ]),
+            ]);
+
+            $this->processed++;
+        } catch (Throwable $e) {
+            $this->failed++;
+            $this->error("  [failed] {$folderName}: {$e->getMessage()}");
+            Log::error("Backfill BagIt archive failed for {$folderName}: {$e->getMessage()}");
+        } finally {
+            if ($zipPath && file_exists($zipPath)) {
+                @unlink($zipPath);
+            }
+            $this->removeDirectory($tempDir);
+        }
+    }
+
+    /**
+     * Mirror a remote storage directory into a local temp directory.
+     */
+    private function downloadDirectory(Filesystem $disk, string $remoteDir, string $localDir): void
+    {
+        if (! is_dir($localDir)) {
+            mkdir($localDir, 0755, true);
+        }
+
+        foreach ($disk->allFiles($remoteDir) as $remoteFile) {
+            $relative = ltrim(substr($remoteFile, strlen($remoteDir)), '/');
+            $localPath = $localDir.'/'.$relative;
+            $localFileDir = dirname($localPath);
+
+            if (! is_dir($localFileDir)) {
+                mkdir($localFileDir, 0755, true);
+            }
+
+            $stream = $disk->readStream($remoteFile);
+            if ($stream === null) {
+                continue;
+            }
+
+            file_put_contents($localPath, stream_get_contents($stream));
+            fclose($stream);
+        }
+    }
+
+    /**
+     * Zip a local bag directory and return the local zip path.
+     */
+    private function zipDirectory(string $localDir, string $identifier): string
+    {
+        $zipPath = storage_path('app/bagit_backfill_'.$identifier.'_'.uniqid().'.zip');
+        $zip = new ZipArchive;
+
+        if ($zip->open($zipPath, ZipArchive::CREATE | ZipArchive::OVERWRITE) !== true) {
+            throw new \RuntimeException("Failed to create archive for {$identifier}");
+        }
+
+        $files = new RecursiveIteratorIterator(
+            new RecursiveDirectoryIterator($localDir, RecursiveDirectoryIterator::SKIP_DOTS),
+            RecursiveIteratorIterator::LEAVES_ONLY
+        );
+
+        foreach ($files as $file) {
+            if (! $file->isFile()) {
+                continue;
+            }
+
+            $relativePath = ltrim(str_replace($localDir.'/', '', $file->getPathname()), '/');
+            $zip->addFile($file->getPathname(), $relativePath);
+        }
+
+        $zip->close();
+
+        return $zipPath;
+    }
+
+    /**
+     * Recursively remove a local directory.
+     */
+    private function removeDirectory(string $directory): void
+    {
+        if (! is_dir($directory)) {
+            return;
+        }
+
+        $files = new RecursiveIteratorIterator(
+            new RecursiveDirectoryIterator($directory, RecursiveDirectoryIterator::SKIP_DOTS),
+            RecursiveIteratorIterator::CHILD_FIRST
+        );
+
+        foreach ($files as $file) {
+            $file->isDir() ? @rmdir($file->getPathname()) : @unlink($file->getPathname());
+        }
+
+        @rmdir($directory);
+    }
+}

app/Http/Resources/StudyResource.php

@@ -49,6 +49,8 @@ public function toArray($request): array
             'study_preview_urls' => $this->study_preview_urls,
             'experiment_types' => $this->study_experiment_types,
             'download_url' => $this->download_url,
+            'metadata_bagit_generation_status' => $this->metadata_bagit_generation_status,
+            'bagit_archive_link' => $this->bagit_archive_link,
             'has_nmrium' => $this->has_nmrium,
             'submitted_through' => $this->submitted_through,
             'external_id' => $this->external_id,