diff --git a/functions/src/common/datastore.ts b/functions/src/common/datastore.ts index 66a15c732..f7612930f 100644 --- a/functions/src/common/datastore.ts +++ b/functions/src/common/datastore.ts @@ -165,6 +165,19 @@ export class Datastore { .get(); } + fetchPartialLocationsOfInterest( + surveyId: string, + jobId: string, + limit: number + ) { + return this.db_ + .collection(lois(surveyId)) + .where(l.jobId, '==', jobId) + .orderBy(FieldPath.documentId()) + .select(String(l.ownerId), String(l.source), String(l.properties)) + .limit(limit); + } + fetchMailTemplate(templateId: string) { return this.fetchDoc_(mailTemplate(templateId)); } diff --git a/functions/src/export-csv.ts b/functions/src/export-csv.ts index 33a9c7a47..fe4d94904 100644 --- a/functions/src/export-csv.ts +++ b/functions/src/export-csv.ts @@ -24,13 +24,14 @@ import { getDatastore } from './common/context'; import { DecodedIdToken } from 'firebase-admin/auth'; import { StatusCodes } from 'http-status-codes'; import { List } from 'immutable'; -import { QuerySnapshot } from 'firebase-admin/firestore'; -import { timestampToInt, toMessage } from '@ground/lib'; +import { registry, timestampToInt, toMessage } from '@ground/lib'; import { GroundProtos } from '@ground/proto'; import { toGeoJsonGeometry } from '@ground/lib'; import Pb = GroundProtos.ground.v1beta1; +const l = registry.getFieldIds(Pb.LocationOfInterest); + /** * Iterates over all LOIs and submissions in a job, joining them * into a single table written to the response as a quote CSV file. @@ -85,8 +86,31 @@ export async function exportCsvHandler( const ownerIdFilter = canViewAll ? null : userId; const tasks = job.tasks.sort((a, b) => a.index! - b.index!); - const snapshot = await db.fetchLocationsOfInterest(surveyId, jobId); - const loiProperties = createProperySetFromSnapshot(snapshot, ownerIdFilter); + + const loiProperties = new Set(); + let query = db.fetchPartialLocationsOfInterest(surveyId, jobId, 1000); + let lastVisible = null; + do { + const snapshot = await query.get(); + if (snapshot.empty) break; + await Promise.all( + snapshot.docs.map(async doc => { + const loi = doc.data(); + if ( + loi[l.source] === Pb.LocationOfInterest.Source.IMPORTED || + ownerIdFilter === null || + loi[l.ownerId] === ownerIdFilter + ) { + Object.keys(loi[l.properties] || {}).forEach(key => + loiProperties.add(key) + ); + } else return; + }) + ); + lastVisible = snapshot.docs[snapshot.docs.length - 1]; + query = query.startAfter(lastVisible); + } while (lastVisible); + const headers = getHeaders(tasks, loiProperties); res.type('text/csv'); @@ -293,23 +317,9 @@ function getFileName(jobName: string | null) { return `${fileBase}.csv`; } -function createProperySetFromSnapshot( - snapshot: QuerySnapshot, - ownerId: string | null -): Set { - const allKeys = new Set(); - snapshot.forEach(doc => { - const loi = toMessage(doc.data(), Pb.LocationOfInterest); - if (loi instanceof Error) return; - if (!isAccessibleLoi(loi, ownerId)) return; - const properties = loi.properties; - for (const key of Object.keys(properties || {})) { - allKeys.add(key); - } - }); - return allKeys; -} - +/** + * Retrieves the values of specified properties from a LocationOfInterest object. + */ function getPropertiesByName( loi: Pb.LocationOfInterest, properties: Set