diff --git a/src/apps/api/views/submissions.py b/src/apps/api/views/submissions.py index afe14fb36..15268e11e 100644 --- a/src/apps/api/views/submissions.py +++ b/src/apps/api/views/submissions.py @@ -15,11 +15,10 @@ from rest_framework.viewsets import ModelViewSet from rest_framework_csv import renderers from django.core.files.base import ContentFile -from django.http import StreamingHttpResponse from profiles.models import Organization, Membership from tasks.models import Task -from api.serializers.submissions import SubmissionCreationSerializer, SubmissionSerializer, SubmissionFilesSerializer +from api.serializers.submissions import SubmissionCreationSerializer, SubmissionSerializer, SubmissionFilesSerializer, SubmissionDetailSerializer from competitions.models import Submission, SubmissionDetails, Phase, CompetitionParticipant from leaderboards.strategies import put_on_leaderboard_by_submission_rule from leaderboards.models import SubmissionScore, Column, Leaderboard @@ -349,26 +348,27 @@ def re_run_many_submissions(self, request): submission.re_run() return Response({}) - @action(detail=False, methods=['get']) + @action(detail=False, methods=('POST',)) def download_many(self, request): - """ - Download a ZIP containing several submissions. - """ - pks = request.query_params.get('pks') - if pks: - pks = json.loads(pks) # Convert JSON string to list - else: - return Response({"error": "`pks` query parameter is required"}, status=400) + pks = request.data.get('pks') + if not pks: + return Response({"error": "`pks` field is required"}, status=400) + + # pks is already parsed as a list if JSON was sent properly + if not isinstance(pks, list): + return Response({"error": "`pks` must be a list"}, status=400) # Get submissions submissions = Submission.objects.filter(pk__in=pks).select_related( "owner", - "phase__competition", - "phase__competition__created_by", - ).prefetch_related("phase__competition__collaborators") - if submissions.count() != len(pks): + "phase", + "data" + ) + + if len(list(submissions)) != len(pks): return Response({"error": "One or more submission IDs are invalid"}, status=404) + # Nicolas Homberg : should create a function for this ? # Check permissions if not request.user.is_authenticated: raise PermissionDenied("You must be logged in to download submissions") @@ -389,12 +389,17 @@ def download_many(self, request): "You do not have permission to download one or more of the requested submissions" ) - # Download - from competitions.tasks import stream_batch_download - in_memory_zip = stream_batch_download(pks) - response = StreamingHttpResponse(in_memory_zip, content_type='application/zip') - response['Content-Disposition'] = 'attachment; filename="bulk_submissions.zip"' - return response + files = [] + + for sub in submissions: + file_path = sub.data.data_file.name.split('/')[-1] + short_name = f"{sub.id}_{sub.owner}_PhaseId{sub.phase.id}_{sub.data.created_when.strftime('%Y-%m-%d:%M-%S')}_{file_path}" + # url = sub.data.data_file.url + url = SubmissionDetailSerializer(sub.data, context=self.get_serializer_context()).data['data_file'] + # url = SubmissionFilesSerializer(sub, context=self.get_serializer_context()).data['data_file'] + files.append({"name": short_name, "url": url}) + + return Response(files) @action(detail=True, methods=('GET',)) def get_details(self, request, pk): diff --git a/src/apps/competitions/tasks.py b/src/apps/competitions/tasks.py index 9b685d1d7..dc7242ff9 100644 --- a/src/apps/competitions/tasks.py +++ b/src/apps/competitions/tasks.py @@ -20,10 +20,6 @@ from django.utils.timezone import now from rest_framework.exceptions import ValidationError -from urllib.request import urlopen -from contextlib import closing -from urllib.error import ContentTooShortError - from celery_config import app from competitions.models import Submission, CompetitionCreationTaskStatus, SubmissionDetails, Competition, \ CompetitionDump, Phase @@ -280,49 +276,6 @@ def send_child_id(submission, child_id): }) -def retrieve_data(url, data=None): - with closing(urlopen(url, data)) as fp: - headers = fp.info() - - bs = 1024 * 8 - size = -1 - read = 0 - if "content-length" in headers: - size = int(headers["Content-Length"]) - - while True: - block = fp.read(bs) - if not block: - break - read += len(block) - yield(block) - - if size >= 0 and read < size: - raise ContentTooShortError( - "retrieval incomplete: got only %i out of %i bytes" - % (read, size)) - - -def zip_generator(submission_pks): - in_memory_zip = BytesIO() - with zipfile.ZipFile(in_memory_zip, 'w', zipfile.ZIP_DEFLATED) as zip_file: - for submission_id in submission_pks: - submission = Submission.objects.get(id=submission_id) - short_name = "ID_" + str(submission_id) + '_' + submission.data.data_file.name.split('/')[-1] - url = make_url_sassy(path=submission.data.data_file.name) - for block in retrieve_data(url): - zip_file.writestr(short_name, block) - - in_memory_zip.seek(0) - - return in_memory_zip - - -@app.task(queue='site-worker', soft_time_limit=60 * 60) -def stream_batch_download(submission_pks): - return zip_generator(submission_pks) - - @app.task(queue='site-worker', soft_time_limit=60) def _run_submission(submission_pk, task_pks=None, is_scoring=False): """This function is wrapped so that when we run tests we can run this function not diff --git a/src/static/js/ours/client.js b/src/static/js/ours/client.js index 410ec34cf..e4ffaa3fc 100644 --- a/src/static/js/ours/client.js +++ b/src/static/js/ours/client.js @@ -128,31 +128,13 @@ CODALAB.api = { return CODALAB.api.request('GET', `${URLS.API}submissions/${id}/get_detail_result/`) }, download_many_submissions: function (pks) { - console.log('Request bulk'); - const params = new URLSearchParams({ pks: JSON.stringify(pks) }); - const url = `${URLS.API}submissions/download_many/?${params}`; - return fetch(url, { - method: 'GET', - headers: { - 'Content-Type': 'application/json' - } - }).then(response => { - if (!response.ok) { - throw new Error('Network response was not ok ' + response.statusText); - } - return response.blob(); - }).then(blob => { - const link = document.createElement('a'); - link.href = window.URL.createObjectURL(blob); - link.download = 'bulk_submissions.zip'; - document.body.appendChild(link); - link.click(); - document.body.removeChild(link); - }).catch(error => { - console.error('Error downloading submissions:', error); - }); + return CODALAB.api.request( + 'POST', + URLS.API + "submissions/download_many/", + { pks: pks } // body is JSON by convention + ); }, - + /*--------------------------------------------------------------------- Leaderboards ---------------------------------------------------------------------*/ diff --git a/src/static/riot/competitions/detail/submission_manager.tag b/src/static/riot/competitions/detail/submission_manager.tag index 74b746cc6..39bcacb7c 100644 --- a/src/static/riot/competitions/detail/submission_manager.tag +++ b/src/static/riot/competitions/detail/submission_manager.tag @@ -1,29 +1,36 @@ -
-
+
+
+
Rerun all submissions per phase
Select a phase
-
- { phase.name } +
+ { phase.name }
-
- +
+ + Download as CSV - + - + - - + +
+
@@ -567,16 +574,230 @@ .modal('show') CODALAB.events.trigger('submission_clicked') } - + + + // self.bulk_download = function () { + // const statusBox = document.getElementById('downloadStatus'); + // const progressEl = document.getElementById('downloadProgress'); + // const textEl = document.getElementById('progressText'); + + // statusBox.style.display = "flex"; + // // statusBox.style.display = "inline"; + + + // progressEl.style.display="flex"; + // progressEl.value = 0; + // textEl.textContent = "Preparing download..."; + + // console.log("Files returned by server:", files); + + + // CODALAB.api.download_many_submissions(self.checked_submissions) + // .done( async function(files) { + // // .done( function(files) { + // console.log("Files returned by server:", files); + + // const zip = new JSZip(); + // const total = files.length; + // let completed = 0; + // const failed = []; + + // const fetchFiles = files.map(async file => { + // try { + // const response = await fetch(file.url); + + // if (!response.ok) { + // throw new Error(`HTTP ${response.status}`); + // } + + // const blob = await response.blob(); + + // zip.file(file.name.replace(/[:/\\]/g, '_'), blob); + // } catch (err) { + // console.error(`Failed to fetch ${file.name}:`, err); + // failed.push(file.name); + // } finally { + // // Update progress regardless of success/failure + // completed++; + // const percent = Math.floor((completed / total) * 100); + // progressEl.value = percent; + // textEl.textContent = `${completed} / ${total} files (${percent}%)`; + // } + // }); + + // Promise.allSettled(fetchFiles).then(() => { + // // If some files failed, include them as failed.txt inside the zip + // if (failed.length > 0) { + // const failedContent = `The following submissions failed to download:\n\n${failed.join("\n")}`; + // zip.file("failed.txt", failedContent); + // } + + + // textEl.textContent = "Generating bundle"; + // progressEl.style.display = "none"; + + // zip.generateAsync({ type: "blob" }).then(blob => { + // const link = document.createElement("a"); + // link.href = URL.createObjectURL(blob); + // link.download = "bulk_submissions.zip"; + // document.body.appendChild(link); + // link.click(); + // document.body.removeChild(link); + + // if (failed.length > 0) { + // textEl.textContent = `Download complete, but ${failed.length} failed (see failed.txt in the zip)`; + // } else { + // textEl.textContent = "Download ready!"; + // } + + // setTimeout(() => { + // statusBox.style.display = "none"; + // }, 5000); + // }); + // }); + // }) + // .fail(function(err) { + // console.error("Error downloading submissions:", err); + // textEl.textContent = "Error downloading!"; + // }); + // }; self.bulk_download = function () { - CODALAB.api.download_many_submissions(self.checked_submissions) - .catch(function (error) { - console.error('Error:', error); - }); - } + const statusBox = document.getElementById('downloadStatus'); + const progressEl = document.getElementById('downloadProgress'); + const textEl = document.getElementById('progressText'); + + statusBox.style.display = "flex"; + progressEl.style.display = "flex"; + progressEl.value = 0; + textEl.textContent = "Preparing download..."; + + // Kick the API request + const req = CODALAB.api.download_many_submissions(self.checked_submissions); + + // Common error handler + const handleError = (err) => { + console.error("Error downloading submissions:", err); + textEl.textContent = "Error downloading!"; + setTimeout(() => { statusBox.style.display = "none"; }, 5000); + }; + + // Success handler (async because we await inside) + const handleSuccess = async (resp) => { + // Normalize response -> files array + let files = resp; + if (resp && typeof resp === 'object' && !Array.isArray(resp)) { + // common shapes: { files: [...] } or { data: [...] } or direct array + if (Array.isArray(resp.files)) files = resp.files; + else if (Array.isArray(resp.data)) files = resp.data; + else if (Array.isArray(resp.results)) files = resp.results; + else if (Array.isArray(resp)) files = resp; + else { + // if jQuery passes multiple args (data, textStatus, jqXHR), pick the first arg + if (arguments && arguments[0] && Array.isArray(arguments[0])) files = arguments[0]; + else { + console.warn("Unexpected response shape from download_many_submissions:", resp); + files = []; + } + } + } + if (!Array.isArray(files) || files.length === 0) { + textEl.textContent = "No files to download"; + setTimeout(() => { statusBox.style.display = "none"; }, 3000); + return; + } + console.log("Files returned by server:", files); + + const zip = new JSZip(); + const total = files.length; + let completed = 0; + const failed = []; + + const limit = 5; + const queue = files.slice(); // clone + const running = []; + + // worker to fetch one file + async function worker(file) { + try { + const response = await fetch(file.url); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + const blob = await response.blob(); + zip.file(file.name.replace(/[:/\\]/g, "_"), blob); + } catch (err) { + console.error(`Failed to fetch ${file.name}:`, err); + failed.push(`${file.name} (${err.message || 'fetch error'})`); + } finally { + completed++; + const percent = Math.floor((completed / total) * 100); + progressEl.value = percent; + textEl.textContent = `${completed} / ${total} files (${percent}%)`; + } + } + + // run queue with limited concurrency + while (queue.length > 0) { + while (running.length < limit && queue.length > 0) { + const file = queue.shift(); + const p = worker(file).then(() => { + // remove finished promise from running + const idx = running.indexOf(p); + if (idx !== -1) running.splice(idx, 1); + }); + running.push(p); + } + // Wait for at least one running promise to finish + if (running.length > 0) { + await Promise.race(running); + } + } + await Promise.all(running); + + // Add failed.txt if necessary + if (failed.length > 0) { + const failedContent = `The following submissions failed to download:\n\n${failed.join("\n")}`; + zip.file("failed.txt", failedContent); + } + + textEl.textContent = "Generating bundle"; + progressEl.style.display = "none"; + + const blob = await zip.generateAsync({ type: "blob" }); + const link = document.createElement("a"); + link.href = URL.createObjectURL(blob); + link.download = "bulk_submissions.zip"; + document.body.appendChild(link); + link.click(); + document.body.removeChild(link); + + if (failed.length > 0) { + textEl.textContent = `Download complete, but ${failed.length} failed (see failed.txt in the zip)`; + } else { + textEl.textContent = "Download ready!"; + } + + setTimeout(() => { statusBox.style.display = "none"; }, 5000); + }; + + // Support both jQuery jqXHR (.done/.fail) and native Promise (.then/.catch) + if (req && typeof req.done === "function") { + // jQuery-style + req.done(function() { + // jQuery passes (data, textStatus, jqXHR) + // forward only the first argument to our handler + const args = Array.from(arguments); + handleSuccess(args[0]); + }).fail(handleError); + } else if (req && typeof req.then === "function") { + // native Promise + req.then(handleSuccess).catch(handleError); + } else { + console.error("download_many_submissions returned non-promise/non-jqXHR:", req); + handleError(new Error("Invalid request return type")); + } + }; self.submission_handling = function () { // console.log(self.checked_submissions) diff --git a/src/templates/base.html b/src/templates/base.html index 30584b35a..014a7c63d 100644 --- a/src/templates/base.html +++ b/src/templates/base.html @@ -256,6 +256,7 @@

CodaBench

+