diff --git a/scripts/sentinelDownload/README.md b/scripts/sentinelDownload/README.md index e6033f97b..12c758a3a 100644 --- a/scripts/sentinelDownload/README.md +++ b/scripts/sentinelDownload/README.md @@ -27,13 +27,41 @@ The script accepts command-line options via `click`: - `--start-date` _(str, default `2025-01-01`)_ - Start date in `YYYY-MM-DD` format. - `--end-date` _(str, default = today)_ - End date in `YYYY-MM-DD` format. - `--max-results` _(int, default `5`)_ - Maximum number of images to download. -- `--output-dir` _(path, default `sequentialTestRasters`)_ - Directory to save clipped files and JSON file. +- `--output-name` _(str, default `sequentialTestRasters`)_ - Base name for output. Rasters are saved under `downloads//`; ingest JSON is written as `.json` next to the script. - `--cloud-cover` _(float, default `30.0`)_ - Maximum allowed cloud cover percentage of files found. - `--size-km` _(float, default `10.0`)_ - Size of square window (in kilometers) to clip around the point. +- `--single-file` _(flag, default off)_ - Combine all downloaded frames into one multiframe GeoTIFF instead of writing separate files per date. The generated ingest JSON uses `frame_property: "frame"` for multiframe ingest. + +### `--single-file` GDAL requirement + +The `--single-file` option shells out to `gdal_translate` to append each clipped frame as a subdataset in one multiframe GeoTIFF. GDAL must be installed and `gdal_translate` must be on your `PATH`. + +If GDAL is not available, run without `--single-file` (the default writes one GeoTIFF per frame). --- ## Outputs -- **GeoTIFF files** - Clipped Sentinel-2 visual images (RGB). -- **`sample.json`** - JSON metadata describing datasets, layers, and frames, useful for ingestion into GeoDatalytics. +Files are written relative to this script directory: + +``` +scripts/sentinelDownload/ + sentinel2Download.py + .json # ingest manifest (sibling to the script) + downloads/ + / + *.tif # clipped GeoTIFF(s) +``` + +- **GeoTIFF files** - Clipped Sentinel-2 visual images (RGB). With `--single-file`, one multiframe GeoTIFF is written instead of separate per-date files. +- **`.json`** - Ingest manifest describing the project, dataset, layers, and frames. + +## Ingesting into GeoDatalytics + +Then ingest the sequential data from the project root: + +```bash +./manage.py ingest .json --replace +``` + +Use `--replace` if you have previously ingested the same project or dataset and need to refresh it. diff --git a/scripts/sentinelDownload/sentinel2Download.py b/scripts/sentinelDownload/sentinel2Download.py index c0d1ca120..dc90d6666 100644 --- a/scripts/sentinelDownload/sentinel2Download.py +++ b/scripts/sentinelDownload/sentinel2Download.py @@ -14,6 +14,9 @@ from datetime import UTC, datetime import json from pathlib import Path +import shutil +import subprocess +import tempfile import click import numpy as np @@ -26,6 +29,8 @@ # STAC API from AWS Earth Search STAC_API_URL = "https://earth-search.aws.element84.com/v1" +SCRIPT_DIR = Path(__file__).resolve().parent +DOWNLOADS_DIR = SCRIPT_DIR / "downloads" def default_end_date(): @@ -101,6 +106,46 @@ def read_cog_window_rgb(cog_url, lon, lat, size_km=10): return data, meta +def _run_checked_command(cmd: list[str]) -> None: + subprocess.run(cmd, check=True) # noqa: S603 + + +def combine_frames_to_multiframe(frame_paths, output_path): + """ + Combine single-frame GeoTIFFs into one multiframe GeoTIFF. + + Each appended page becomes a scrubbable frame when imported with + frame_property: "frame". + """ + if not frame_paths: + return + + gdal_translate = shutil.which("gdal_translate") + if gdal_translate is None: + msg = ( + "gdal_translate is required for --single-file but was not found on PATH. " + "Install GDAL or run without --single-file." + ) + raise RuntimeError(msg) + + output_path = Path(output_path) + creation_options = ["-co", "COMPRESS=LZW"] + _run_checked_command( + [gdal_translate, *creation_options, str(frame_paths[0]), str(output_path)], + ) + for frame_path in frame_paths[1:]: + _run_checked_command( + [ + gdal_translate, + *creation_options, + "-co", + "APPEND_SUBDATASET=YES", + str(frame_path), + str(output_path), + ], + ) + + @click.command() @click.option( "--lat", default=43.135763, type=float, required=True, help="Latitude of the location." @@ -130,11 +175,14 @@ def read_cog_window_rgb(cog_url, lon, lat, size_km=10): help="Maximum number of images to download.", ) @click.option( - "--output-dir", - type=click.Path(), + "--output-name", + type=str, default="sequentialTestRasters", show_default=True, - help="Directory to save the downloaded files.", + help=( + "Base name for output: writes rasters to downloads// and " + "a sibling ingest JSON named .json next to this script." + ), ) @click.option( "--cloud-cover", type=float, default=30.0, show_default=True, help="Max cloud cover percentage." @@ -146,11 +194,21 @@ def read_cog_window_rgb(cog_url, lon, lat, size_km=10): show_default=True, help="Size of square window to clip around the point in kilometers.", ) -def download_stac_sentinel( # noqa: PLR0913, PLR0915 - lat, lon, start_date, end_date, max_results, output_dir, cloud_cover, size_km +@click.option( + "--single-file", + is_flag=True, + default=False, + help=( + "Write all frames into one multiframe GeoTIFF instead of separate files. " + 'The generated output JSON will use frame_property: "frame".' + ), +) +def download_stac_sentinel( # noqa: C901, PLR0912, PLR0913, PLR0915 + lat, lon, start_date, end_date, max_results, output_name, cloud_cover, size_km, single_file ): """Download clipped Sentinel-2 L1C visual images from AWS via STAC API.""" - Path(output_dir).mkdir(parents=True, exist_ok=True) + output_dir = DOWNLOADS_DIR / output_name + output_dir.mkdir(parents=True, exist_ok=True) catalog = Client.open(STAC_API_URL) @@ -166,7 +224,7 @@ def download_stac_sentinel( # noqa: PLR0913, PLR0915 limit=max_results, ) - items = list(search.get_items()) + items = list(search.items()) if not items: click.echo("⚠️ No Sentinel-2 images found.") @@ -187,40 +245,59 @@ def download_stac_sentinel( # noqa: PLR0913, PLR0915 ) downloaded_files = [] - - for i, item in enumerate(items): - if i >= max_results: - break - date_str = item.datetime.strftime("%Y-%m-%d") - item_id = item.id - click.echo(f"[{i + 1}/{len(items)}] {item_id} from {date_str}") - - visual_asset = item.assets.get("visual") - if visual_asset: - url = visual_asset.href - filename = f"{item_id}_visual_clip_{int(size_km)}km.tif" - filepath = Path(output_dir) / filename - - click.echo(f" - Reading {size_km}km x {size_km}km window around point") - try: - data, meta = read_cog_window_rgb(url, lon, lat, size_km=size_km) - with rasterio.open(filepath, "w", **meta) as dst: - dst.write(data) - except (RasterioError, RasterioIOError) as e: - click.echo(f" - ⚠️ Failed to read or save clipped image: {e}") + downloaded_frame_paths = [] + multiframe_filename = None + + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + + for i, item in enumerate(items): + if i >= max_results: + break + date_str = item.datetime.strftime("%Y-%m-%d") + item_id = item.id + click.echo(f"[{i + 1}/{len(items)}] {item_id} from {date_str}") + + visual_asset = item.assets.get("visual") + if visual_asset: + url = visual_asset.href + filename = f"{item_id}_visual_clip_{int(size_km)}km.tif" + filepath = output_dir / filename + write_path = temp_path / filename if single_file else filepath + + click.echo(f" - Reading {size_km}km x {size_km}km window around point") + try: + data, meta = read_cog_window_rgb(url, lon, lat, size_km=size_km) + with rasterio.open(write_path, "w", **meta) as dst: + dst.write(data) + except (RasterioError, RasterioIOError) as e: + click.echo(f" - ⚠️ Failed to read or save clipped image: {e}") + else: + if single_file: + click.echo(f" - Buffered frame {len(downloaded_frame_paths)}") + downloaded_frame_paths.append(write_path) + else: + click.echo(f" - Saved clipped image to {filename}") + downloaded_files.append(filename) else: - click.echo(f" - Saved clipped image to {filename}") - downloaded_files.append(filename) - else: - click.echo(f" - ⚠️ Visual asset not available in item {item_id}") + click.echo(f" - ⚠️ Visual asset not available in item {item_id}") + + if single_file and downloaded_frame_paths: + multiframe_filename = f"sentinel_visual_clip_{int(size_km)}km_multiframe.tif" + multiframe_path = output_dir / multiframe_filename + click.echo( + f"Combining {len(downloaded_frame_paths)} frames into {multiframe_filename}..." + ) + combine_frames_to_multiframe(downloaded_frame_paths, multiframe_path) + click.echo(f" - Saved multiframe image to {multiframe_filename}") click.echo("✅ Download complete.") - # Generate dataset.json dataset_json = { "type": "Dataset", "name": "Sequential Test Rasters", "description": "Clipped Sentinel-2 images downloaded and clipped around point", "category": "imagery", + "tags": ["sentinel-2", "imagery", "sequential"], "files": [], "layers": [], } @@ -229,22 +306,40 @@ def download_stac_sentinel( # noqa: PLR0913, PLR0915 "type": "Project", "name": "Sentinel-2 Clipped Images", "datasets": ["Sequential Test Rasters"], - "default_map_center": [lat, lon], + "default_map_center": [lon, lat], "default_map_zoom": 11, } - # Add each file as its own layer - layer_frames = [] - for idx, f in enumerate(downloaded_files): - dataset_json["files"].append({"path": f"{output_dir}/{f}", "name": f"Frame {idx}"}) - layer_frames.append({"name": f"Sequential Layer {idx}", "index": idx, "data": f}) - - layer = {"name": "Sequential Test Layers", "frames": layer_frames} - dataset_json["layers"].append(layer) - - json_path = Path(output_dir, "sample.json") + if single_file and multiframe_filename: + dataset_json["files"].append( + {"path": f"{output_name}/{multiframe_filename}", "name": multiframe_filename} + ) + dataset_json["layers"].append( + { + "name": "Sequential Test Layers", + "frame_property": "frame", + "data": multiframe_filename, + } + ) + else: + layer_frames = [] + for idx, f in enumerate(downloaded_files): + dataset_json["files"].append({"path": f"{output_name}/{f}", "name": f"Frame {idx}"}) + layer_frames.append( + { + "name": f"Sequential Layer {idx}", + "index": idx, + "data": f, + } + ) + + dataset_json["layers"].append({"name": "Sequential Test Layers", "frames": layer_frames}) + + json_path = SCRIPT_DIR / f"{output_name}.json" with json_path.open("w") as jf: json.dump([project_json, dataset_json], jf, indent=4) + click.echo(f" - Wrote ingest JSON to {json_path}") + click.echo(f" - Rasters saved under {output_dir}") if __name__ == "__main__": diff --git a/uvdat/core/admin.py b/uvdat/core/admin.py index 1083f2407..de07aa538 100644 --- a/uvdat/core/admin.py +++ b/uvdat/core/admin.py @@ -19,6 +19,7 @@ NetworkNode, Project, RasterData, + RasterFramePreview, Region, SizeConfig, SizeRangeConfig, @@ -69,6 +70,12 @@ class LayerStyleAdmin(admin.ModelAdmin): list_display = ["id", "name", "layer"] +@admin.register(RasterFramePreview) +class RasterFramePreviewAdmin(admin.ModelAdmin): + list_display = ["id", "layer_style", "layer_frame", "width", "height", "status"] + list_select_related = ["layer_style", "layer_frame"] + + @admin.register(Colormap) class ColormapAdmin(admin.ModelAdmin): list_display = ["id", "name"] diff --git a/uvdat/core/frame_previews/__init__.py b/uvdat/core/frame_previews/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/uvdat/core/frame_previews/fingerprint.py b/uvdat/core/frame_previews/fingerprint.py new file mode 100644 index 000000000..b22402a69 --- /dev/null +++ b/uvdat/core/frame_previews/fingerprint.py @@ -0,0 +1,18 @@ +from __future__ import annotations + +import hashlib +import json +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from uvdat.core.models import LayerStyle + + +def _fingerprint_payload(params: dict[str, Any] | None) -> str: + return json.dumps(params or {}, sort_keys=True, default=str) + + +def style_fingerprint(layer_style: LayerStyle) -> str: + """Sha256 of ``raster_style_params``, the JSON used to render preview PNGs.""" + payload = _fingerprint_payload(layer_style.raster_style_params) + return hashlib.sha256(payload.encode()).hexdigest() diff --git a/uvdat/core/frame_previews/preview_regeneration.py b/uvdat/core/frame_previews/preview_regeneration.py new file mode 100644 index 000000000..c3b9099b9 --- /dev/null +++ b/uvdat/core/frame_previews/preview_regeneration.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +from django.utils import timezone + +from uvdat.core.frame_previews.fingerprint import style_fingerprint +from uvdat.core.models import Layer, LayerStyle, RasterFramePreview, TaskResult +from uvdat.core.models.frame_preview import PreviewStatus +from uvdat.core.models.task_result import suppress_task_notifications +from uvdat.core.tasks.frame_preview import generate_layer_style_previews + + +def style_needs_previews(layer_style: LayerStyle) -> bool: + return layer_style.layer.is_multiframe_raster() + + +def supersede_pending_preview_tasks(layer_style_id: int) -> None: + TaskResult.objects.filter( + task_type="frame_preview", + completed__isnull=True, + inputs__layer_style_id=layer_style_id, + ).update( + completed=timezone.now(), + status="Superseded by newer style save.", + ) + + +def previews_current_for_fingerprint(layer_style: LayerStyle, fingerprint: str) -> bool: + """Return whether every frame already has a complete preview for this fingerprint.""" + frames = layer_style.layer.multiframe_raster_frames() + if not frames: + return False + + previews_by_frame = { + preview.layer_frame_id: preview for preview in layer_style.frame_previews.all() + } + + return all( + (preview := previews_by_frame.get(frame.id)) + and preview.status == PreviewStatus.COMPLETE + and preview.image + and preview.style_fingerprint == fingerprint + for frame in frames + ) + + +def clear_style_preview_instance_cache(layer_style: LayerStyle) -> None: + """Drop queryset annotations and prefetches that go stale after invalidation.""" + layer_style.refresh_from_db() + layer_style.__dict__.pop("preview_status", None) + layer_style.__dict__.pop("_raster_frame_count", None) + layer_style.__dict__.pop("_complete_with_image_count", None) + if cache := getattr(layer_style, "_prefetched_objects_cache", None): + cache.pop("frame_previews", None) + + +def mark_previews_regenerating(layer_style: LayerStyle, fingerprint: str) -> list[int]: + """Upsert one preview row per multiframe frame and clear stale images.""" + frame_ids = [] + for frame in layer_style.layer.multiframe_raster_frames(): + preview, created = RasterFramePreview.objects.get_or_create( + layer_style=layer_style, + layer_frame=frame, + defaults={ + "status": PreviewStatus.CREATING, + "style_fingerprint": fingerprint, + }, + ) + preview.style_fingerprint = fingerprint + preview.status = PreviewStatus.CREATING if created else PreviewStatus.REGENERATING + + if preview.image: + preview.image.delete(save=False) + preview.image = None + preview.width = None + preview.height = None + preview.bounds = {} + preview.save() + frame_ids.append(frame.id) + return frame_ids + + +def invalidate_and_enqueue_previews( + layer_style: LayerStyle, + *, + asynchronous: bool = True, +) -> TaskResult | None: + if not style_needs_previews(layer_style): + return None + + layer_style.refresh_from_db() + fingerprint = style_fingerprint(layer_style) + if previews_current_for_fingerprint(layer_style, fingerprint): + return None + + mark_previews_regenerating(layer_style, fingerprint) + clear_style_preview_instance_cache(layer_style) + supersede_pending_preview_tasks(layer_style.id) + + result = TaskResult.objects.create( + name=f"Frame previews: {layer_style.layer.name} - {layer_style.name}", + task_type="frame_preview", + project=layer_style.project, + inputs={ + "layer_style_id": layer_style.id, + "layer_id": layer_style.layer_id, + "layer_name": layer_style.layer.name, + "fingerprint": fingerprint, + }, + ) + + if asynchronous: + generate_layer_style_previews.delay(layer_style.id, fingerprint, result.id) + else: + with suppress_task_notifications(): + generate_layer_style_previews.apply(args=(layer_style.id, fingerprint, result.id)) + return result + + +def preview_status_for_style(layer_style: LayerStyle) -> str | None: + """Aggregate per-frame preview rows into a style-level status string.""" + if not style_needs_previews(layer_style): + return None + + frames = layer_style.layer.multiframe_raster_frames() + if not frames: + return None + + previews_by_frame = { + preview.layer_frame_id: preview for preview in layer_style.frame_previews.all() + } + + if all( + (preview := previews_by_frame.get(frame.id)) + and preview.status == PreviewStatus.COMPLETE + and preview.image + for frame in frames + ): + return "ready" + + return "notready" + + +def get_layer_style_preview_status(layer_style: LayerStyle) -> str | None: + if "preview_status" in layer_style.__dict__: + return layer_style.preview_status + return preview_status_for_style(layer_style) + + +def get_layer_preview_status(layer: Layer) -> str | None: + if "preview_status" in layer.__dict__: + return layer.preview_status + if layer.default_style_id is None: + return None + return get_layer_style_preview_status(layer.default_style) diff --git a/uvdat/core/frame_previews/raster_style.py b/uvdat/core/frame_previews/raster_style.py new file mode 100644 index 000000000..fca9ee4d9 --- /dev/null +++ b/uvdat/core/frame_previews/raster_style.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from typing import Any + + +def apply_source_filters_to_style_query( + base_query: dict[str, Any], + source_filters: dict[str, Any] | None, +) -> dict[str, Any]: + query = dict(base_query) + if source_filters and "band" in source_filters: + query["band"] = source_filters["band"] + return query + + +def raster_source_filter_kwargs(source_filters: dict[str, Any] | None) -> dict[str, Any]: + """Return large_image kwargs that must not be embedded in style JSON.""" + if not source_filters or "frame" not in source_filters: + return {} + return {"frame": source_filters["frame"]} diff --git a/uvdat/core/frame_previews/types.py b/uvdat/core/frame_previews/types.py new file mode 100644 index 000000000..632f892ad --- /dev/null +++ b/uvdat/core/frame_previews/types.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from typing import TypedDict + + +class FramePreviewCorner(TypedDict): + x: float + y: float + + +class FramePreviewBounds(TypedDict, total=False): + srs: str # EPSG:4326 + xmin: float + xmax: float + ymin: float + ymax: float + ul: FramePreviewCorner + ur: FramePreviewCorner + lr: FramePreviewCorner + ll: FramePreviewCorner + + +class FramePreviewData(TypedDict): + url: str # presigned URL of the preview image + width: int + height: int + bounds: FramePreviewBounds diff --git a/uvdat/core/management/commands/ingest.py b/uvdat/core/management/commands/ingest.py index bcfe0519f..47718de66 100644 --- a/uvdat/core/management/commands/ingest.py +++ b/uvdat/core/management/commands/ingest.py @@ -17,7 +17,9 @@ import djclick as click import pooch -from uvdat.core.models import Chart, Dataset, FileItem, Project +from uvdat.core.frame_previews.preview_regeneration import invalidate_and_enqueue_previews +from uvdat.core.models import Chart, Dataset, FileItem, Layer, Project +from uvdat.core.tasks.frame_preview import ensure_default_layer_style DATA_FOLDER = Path(os.environ.get("INGEST_BIND_MOUNT_POINT", "sample_data")) DOWNLOADS_FOLDER = DATA_FOLDER / "downloads" @@ -152,6 +154,28 @@ def ingest_file(file_info, *, index=0, dataset=None, chart=None, replace=False, new_file_item.file.save(file_path, File(f)) +def generate_ingest_multiframe_previews(converted_dataset_names: set[str]) -> int: + if not converted_dataset_names: + return 0 + style_count = 0 + for project in Project.objects.filter(datasets__name__in=converted_dataset_names).distinct(): + layers = Layer.objects.filter( + dataset__name__in=converted_dataset_names, + dataset__in=project.datasets.all(), + ) + for layer in layers: + if not layer.is_multiframe_raster(): + continue + style = ensure_default_layer_style(layer, project) + click.echo( + f"\t\t Generating multiframe raster previews for style {style.name!r} " + f"(layer {layer.name!r})..." + ) + invalidate_and_enqueue_previews(style, asynchronous=False) + style_count += 1 + return style_count + + def ingest_projects(data: list[ProjectItem], *, replace=False) -> None: for project in data: click.echo(f"\t- {project['name']}") @@ -264,7 +288,8 @@ def default_conversion_process(dataset: Dataset, options: DatasetItem): def ingest_datasets( data: list[DatasetItem], json_file_path: Path, *, replace=False, skip_cache=False -) -> None: +) -> set[str]: + converted_dataset_names: set[str] = set() superuser = User.objects.filter(is_superuser=True).first() if superuser is None: raise click.ClickException("Please create at least one superuser") @@ -320,6 +345,7 @@ def ingest_datasets( f"\t\t Dataset {dataset_for_conversion.name} converted.", fg="green", ) + converted_dataset_names.add(dataset["name"]) else: click.secho( f"\t\t Dataset {dataset['name']} already exists, not importing/converting", @@ -329,6 +355,8 @@ def ingest_datasets( dataset_for_conversion.set_tags(dataset.get("tags")) dataset_for_conversion.set_owner(superuser) + return converted_dataset_names + @click.command() @click.argument("file_path") @@ -385,10 +413,16 @@ def ingest(*, file_path, replace, clear, skip_cache): elif item["type"] == "Chart": charts.append(item) click.echo("Ingesting Datasets:") - ingest_datasets(datasets, file_path, replace=replace, skip_cache=skip_cache) + converted_dataset_names = ingest_datasets( + datasets, file_path, replace=replace, skip_cache=skip_cache + ) click.echo("Ingesting Projects:") ingest_projects(projects, replace=replace) click.echo("Ingesting Charts:") ingest_charts(charts, replace=replace, skip_cache=skip_cache) + preview_count = generate_ingest_multiframe_previews(converted_dataset_names) + if preview_count: + click.echo(f"Generated multiframe raster previews for {preview_count} layer style(s).") + click.secho("Ingestion complete.", fg="green") diff --git a/uvdat/core/migrations/0025_raster_frame_preview.py b/uvdat/core/migrations/0025_raster_frame_preview.py new file mode 100644 index 000000000..0b295b7d0 --- /dev/null +++ b/uvdat/core/migrations/0025_raster_frame_preview.py @@ -0,0 +1,76 @@ +# Generated by Django 5.2.9 on 2026-06-25 +from __future__ import annotations + +from django.db import migrations, models +import django.db.models.deletion +import s3_file_field.fields + + +class Migration(migrations.Migration): + dependencies = [ + ("core", "0024_taskresult_creator"), + ] + + operations = [ + migrations.AddField( + model_name="layerstyle", + name="raster_style_params", + field=models.JSONField(blank=True, default=None, null=True), + ), + migrations.CreateModel( + name="RasterFramePreview", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ( + "status", + models.CharField( + choices=[ + ("creating", "Creating"), + ("regenerating", "Regenerating"), + ("complete", "Complete"), + ("failed", "Failed"), + ], + default="creating", + max_length=16, + ), + ), + ( + "style_fingerprint", + models.CharField(blank=True, default="", max_length=64), + ), + ("image", s3_file_field.fields.S3FileField(blank=True, null=True)), + ("width", models.PositiveIntegerField(blank=True, null=True)), + ("height", models.PositiveIntegerField(blank=True, null=True)), + ("bounds", models.JSONField(blank=True, default=dict)), + ( + "layer_frame", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="style_previews", + to="core.layerframe", + ), + ), + ( + "layer_style", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="frame_previews", + to="core.layerstyle", + ), + ), + ], + options={ + "constraints": [ + models.UniqueConstraint( + fields=("layer_style", "layer_frame"), + name="unique_layer_style_layer_frame_preview", + ) + ], + }, + ), + ] diff --git a/uvdat/core/models/__init__.py b/uvdat/core/models/__init__.py index 83ff77636..0e4672d76 100644 --- a/uvdat/core/models/__init__.py +++ b/uvdat/core/models/__init__.py @@ -6,6 +6,7 @@ from .data import RasterData, VectorData, VectorFeature from .dataset import Dataset, DatasetTag from .file_item import FileItem +from .frame_preview import RasterFramePreview from .layer import Layer, LayerFrame from .networks import Network, NetworkEdge, NetworkNode from .project import Project @@ -39,6 +40,7 @@ "NetworkNode", "Project", "RasterData", + "RasterFramePreview", "Region", "SizeConfig", "SizeRangeConfig", diff --git a/uvdat/core/models/chart.py b/uvdat/core/models/chart.py index 0cdf4adcd..46911d318 100644 --- a/uvdat/core/models/chart.py +++ b/uvdat/core/models/chart.py @@ -2,6 +2,8 @@ from django.db import models +from uvdat.core.models.task_result import suppress_task_notifications + from .project import Project from .querysets import ProjectQuerySet @@ -34,7 +36,8 @@ def spawn_conversion_task( if asynchronous: convert_chart_signature.delay() else: - convert_chart_signature.apply() + with suppress_task_notifications(): + convert_chart_signature.apply() def new_line(self): # TODO: new line diff --git a/uvdat/core/models/dataset.py b/uvdat/core/models/dataset.py index 703c059f9..77a62132f 100644 --- a/uvdat/core/models/dataset.py +++ b/uvdat/core/models/dataset.py @@ -85,6 +85,10 @@ def spawn_conversion_task( asynchronous=True, ): # Prevent circular import + from uvdat.core.models.task_result import ( # noqa: PLC0415 + TaskResult, + suppress_task_notifications, + ) from uvdat.core.tasks.dataset import convert_dataset # noqa: PLC0415 convert_dataset_signature = convert_dataset.s( @@ -95,9 +99,6 @@ def spawn_conversion_task( ) if asynchronous: - # Prevent circular import - from uvdat.core.models.task_result import TaskResult # noqa: PLC0415 - result = TaskResult.objects.create( name=f"Conversion of Dataset {self.name}", task_type="conversion", @@ -113,5 +114,6 @@ def spawn_conversion_task( convert_dataset_signature.delay(result_id=result.id) return result else: - convert_dataset_signature.apply() + with suppress_task_notifications(): + convert_dataset_signature.apply() return None diff --git a/uvdat/core/models/frame_preview.py b/uvdat/core/models/frame_preview.py new file mode 100644 index 000000000..26fc7b43c --- /dev/null +++ b/uvdat/core/models/frame_preview.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +from django.db import models +from django.dispatch import receiver +from s3_file_field import S3FileField + +from .layer import LayerFrame + + +class PreviewStatus(models.TextChoices): + CREATING = "creating", "Creating" + REGENERATING = "regenerating", "Regenerating" + COMPLETE = "complete", "Complete" + FAILED = "failed", "Failed" + + +class RasterFramePreview(models.Model): + layer_style = models.ForeignKey( + "LayerStyle", + related_name="frame_previews", + on_delete=models.CASCADE, + ) + layer_frame = models.ForeignKey( + LayerFrame, + related_name="style_previews", + on_delete=models.CASCADE, + ) + status = models.CharField( + max_length=16, + choices=PreviewStatus.choices, + default=PreviewStatus.CREATING, + ) + style_fingerprint = models.CharField(max_length=64, blank=True, default="") + image = S3FileField(blank=True, null=True) + width = models.PositiveIntegerField(null=True, blank=True) + height = models.PositiveIntegerField(null=True, blank=True) + bounds = models.JSONField(default=dict, blank=True) + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["layer_style", "layer_frame"], + name="unique_layer_style_layer_frame_preview", + ) + ] + + def __str__(self): + return f"Preview style={self.layer_style_id} frame={self.layer_frame_id} ({self.id})" + + +@receiver(models.signals.post_delete, sender=RasterFramePreview) +def delete_preview_image(sender, instance, **kwargs): + if instance.image: + instance.image.delete(save=False) diff --git a/uvdat/core/models/layer.py b/uvdat/core/models/layer.py index 88489d9b1..4c14f1066 100644 --- a/uvdat/core/models/layer.py +++ b/uvdat/core/models/layer.py @@ -1,11 +1,16 @@ from __future__ import annotations +from typing import TYPE_CHECKING + from django.db import models from .data import RasterData, VectorData from .dataset import Dataset from .querysets import ProjectQuerySet +if TYPE_CHECKING: + from uvdat.core.frame_previews.types import FramePreviewData + def default_source_filters(): return {} @@ -25,6 +30,41 @@ class Layer(models.Model): def __str__(self): return f"{self.name} ({self.id})" + def multiframe_raster_frames(self): + prefetched = getattr(self, "raster_frames", None) + if prefetched is not None: + return prefetched + return list( + self.frames.filter(raster__isnull=False).select_related("raster").order_by("index") + ) + + def is_multiframe_raster(self) -> bool: + prefetched = getattr(self, "raster_frames", None) + if prefetched is not None: + return len(prefetched) > 1 + return self.frames.filter(raster__isnull=False).count() > 1 + + def ensure_default_style(self): + """Guarantee the layer has a default style whenever any style exists. + + ``default_style`` uses ``on_delete=SET_NULL``, so removing the style a + layer pointed at (outside the viewset's reassign path) can leave the + layer with styles but no default, which hides its frame previews. Prefer + the conventional "Default" style, otherwise fall back to the oldest one. + """ + if self.default_style_id is not None: + return self.default_style + style = self.styles.filter(name="Default").first() or self.styles.order_by("id").first() + if style is not None: + self.default_style = style + self.save(update_fields=["default_style"]) + return style + + def default_multiframe_previews(self) -> list[FramePreviewData] | None: + if self.default_style_id is None: + return None + return self.default_style.multiframe_previews(layer=self) + class LayerFrame(models.Model): name = models.CharField(max_length=255, default="Layer Frame") diff --git a/uvdat/core/models/styles.py b/uvdat/core/models/styles.py index 385d1cabb..43d39866e 100644 --- a/uvdat/core/models/styles.py +++ b/uvdat/core/models/styles.py @@ -1,16 +1,22 @@ from __future__ import annotations import contextlib -from typing import Any +from typing import TYPE_CHECKING, Any from django.core.validators import MaxValueValidator, MinValueValidator from django.db import models +from uvdat.core.frame_previews.types import FramePreviewData + from .colormap import Colormap +from .frame_preview import PreviewStatus from .layer import Layer from .project import Project from .querysets import ProjectQuerySet +if TYPE_CHECKING: + from .frame_preview import RasterFramePreview + class LayerStyle(models.Model): name = models.CharField(max_length=255, default="Layer Style") @@ -26,6 +32,9 @@ class LayerStyle(models.Model): MaxValueValidator(1), ], ) + # django-large-image ``style`` query JSON for raster tiles/previews. + # Provided by the client on style create/update (same object used for tile URLs). + raster_style_params = models.JSONField(blank=True, null=True, default=None) project_filter_path = "layer__dataset__project" objects = ProjectQuerySet.as_manager() @@ -160,6 +169,7 @@ def save_style_configs(self, style_spec): # noqa: C901, PLR0912, PLR0915 filter_config.save() filter_config_ids.append(filter_config.id) FilterConfig.objects.filter(style=self).exclude(id__in=filter_config_ids).delete() + self.save(update_fields=["default_frame", "opacity"]) def repr_style_configs(self): colors = [] @@ -224,6 +234,32 @@ def repr_style_configs(self): "filters": filters, } + @staticmethod + def serialize_frame_preview(preview: RasterFramePreview) -> FramePreviewData: + return FramePreviewData( + url=preview.image.url, + width=preview.width, + height=preview.height, + bounds=preview.bounds, + ) + + def multiframe_previews(self, layer=None) -> list[FramePreviewData] | None: + layer = layer or self.layer + frames = layer.multiframe_raster_frames() + if len(frames) <= 1: + return None + + previews_by_frame_id = { + preview.layer_frame_id: preview for preview in self.frame_previews.all() + } + ordered = [previews_by_frame_id.get(frame.id) for frame in frames] + if not all( + preview is not None and preview.status == PreviewStatus.COMPLETE and preview.image + for preview in ordered + ): + return None + return [self.serialize_frame_preview(preview) for preview in ordered] + def get_default_colormap(): return Colormap.objects.filter(project__isnull=True).first() diff --git a/uvdat/core/models/task_result.py b/uvdat/core/models/task_result.py index f7f06be8a..771618e72 100644 --- a/uvdat/core/models/task_result.py +++ b/uvdat/core/models/task_result.py @@ -1,6 +1,9 @@ from __future__ import annotations +import contextlib +from contextvars import ContextVar import json +import logging from asgiref.sync import async_to_sync from channels.layers import get_channel_layer @@ -16,6 +19,24 @@ from .project import Project from .querysets import ProjectQuerySet +logger = logging.getLogger(__name__) + +# Set while running in a context (e.g. `manage.py ingest`) where TaskResult +# WebSocket notifications are meaningless: tasks run synchronously with no +# client session listening. When set, result_post_save skips the push entirely +# rather than attempting it and logging a warning. +_suppress_notifications: ContextVar[bool] = ContextVar("suppress_task_notifications", default=False) + + +@contextlib.contextmanager +def suppress_task_notifications(): + """Silence TaskResult WebSocket notifications for the current context.""" + token = _suppress_notifications.set(True) + try: + yield + finally: + _suppress_notifications.reset(token) + class TaskResult(models.Model): name = models.CharField(max_length=255) @@ -77,6 +98,12 @@ def complete(self): @receiver(post_save, sender=TaskResult) def result_post_save(sender, instance, **kwargs): + # In contexts such as `manage.py ingest`, tasks run synchronously with no + # client session listening, so the push is meaningless -- skip it silently + # (no attempt, no warning). See suppress_task_notifications(). + if _suppress_notifications.get(): + return + # Prevent circular import from uvdat.core.rest.serializers import TaskResultSerializer # noqa: PLC0415 @@ -85,6 +112,21 @@ def result_post_save(sender, instance, **kwargs): if instance.project: group_name = f"analytics_{instance.project.id}" channel_layer = get_channel_layer() - async_to_sync(channel_layer.group_send)( - group_name, {"type": "send_notification", "message": json.dumps(payload)} - ) + if channel_layer is None: + return + + # This WebSocket push streams live TaskResult updates to browser clients + # subscribed to a channel group. It is best-effort: a failed notification + # must never abort the surrounding TaskResult.save(). Failures are logged at + # WARNING so a genuinely broken channel layer in production is still visible. + try: + async_to_sync(channel_layer.group_send)( + group_name, {"type": "send_notification", "message": json.dumps(payload)} + ) + except Exception: # noqa: BLE001 - notification failures must never propagate + logger.warning( + "Failed to send TaskResult notification for %s to group %r", + instance, + group_name, + exc_info=True, + ) diff --git a/uvdat/core/rest/dataset.py b/uvdat/core/rest/dataset.py index 882c33de8..efe5ee2a7 100644 --- a/uvdat/core/rest/dataset.py +++ b/uvdat/core/rest/dataset.py @@ -5,7 +5,8 @@ from rest_framework.viewsets import ModelViewSet from uvdat.core.access_control import DatasetGuardianPermission -from uvdat.core.models import Dataset, DatasetTag, Network +from uvdat.core.models import Dataset, DatasetTag, Layer, Network +from uvdat.core.rest.querysets import layer_queryset_with_previews from uvdat.core.rest.serializers import ( DatasetSerializer, FileItemSerializer, @@ -43,7 +44,9 @@ def tags(self, request, **kwargs): @action(detail=True, methods=["get"]) def layers(self, request, **kwargs): dataset: Dataset = self.get_object() - layers = list(dataset.layers.all()) + for layer in Layer.objects.filter(dataset=dataset, default_style__isnull=True): + layer.ensure_default_style() + layers = list(layer_queryset_with_previews().filter(dataset=dataset)) serializer = LayerSerializer(layers, many=True) return Response(serializer.data, status=200) diff --git a/uvdat/core/rest/layer.py b/uvdat/core/rest/layer.py index 95e12c217..57f850e82 100644 --- a/uvdat/core/rest/layer.py +++ b/uvdat/core/rest/layer.py @@ -7,17 +7,26 @@ from rest_framework.viewsets import ModelViewSet, ReadOnlyModelViewSet from uvdat.core.models import Layer, LayerFrame, LayerStyle +from uvdat.core.rest.querysets import layer_queryset_with_previews from uvdat.core.rest.serializers import ( LayerFrameSerializer, LayerSerializer, - LayerStyleSerializer, + LayerStyleWithPreviewsSerializer, ) class LayerViewSet(ReadOnlyModelViewSet): - queryset = Layer.objects.select_related("dataset").all() serializer_class = LayerSerializer + def get_queryset(self): + return layer_queryset_with_previews() + + def retrieve(self, request, *args, **kwargs): + if Layer.objects.filter(pk=kwargs.get("pk"), default_style__isnull=True).exists(): + layer = Layer.objects.get(pk=kwargs["pk"]) + layer.ensure_default_style() + return super().retrieve(request, *args, **kwargs) + @action(detail=True, methods=["get"]) def frames(self, request, **kwargs): layer: Layer = self.get_object() @@ -33,7 +42,7 @@ class LayerFrameViewSet(ReadOnlyModelViewSet): class LayerStyleViewSet(ModelViewSet): queryset = LayerStyle.objects.all() - serializer_class = LayerStyleSerializer + serializer_class = LayerStyleWithPreviewsSerializer def get_queryset(self): qs = super().get_queryset() @@ -43,18 +52,22 @@ def get_queryset(self): layer_id = int(self.request.query_params.get("layer", -1)) if layer_id > -1: qs = qs.filter(layer=int(layer_id)) - return qs + return layer_queryset_with_previews(qs, for_layer_style=True) def create(self, request, **kwargs): is_default = request.data.pop("is_default", False) - serializer = LayerStyleSerializer(data=request.data) + serializer = self.get_serializer(data=request.data) serializer.is_valid(raise_exception=True) with transaction.atomic(): try: instance = serializer.save() except jsonschema.exceptions.ValidationError as e: return Response(e.message, status=400) - if is_default and instance.layer.default_style != instance: + # Always keep a default style: adopt the new one when the layer has none, + # so a layer with styles is never left without a default (and no previews). + if ( + is_default or instance.layer.default_style_id is None + ) and instance.layer.default_style != instance: instance.layer.default_style = instance instance.layer.save() return Response(serializer.data, status=200) @@ -62,7 +75,7 @@ def create(self, request, **kwargs): def partial_update(self, request, **kwargs): instance = self.get_object() is_default = request.data.pop("is_default", False) - serializer = LayerStyleSerializer(instance, data=request.data, partial=True) + serializer = self.get_serializer(instance, data=request.data, partial=True) serializer.is_valid(raise_exception=True) with transaction.atomic(): try: diff --git a/uvdat/core/rest/querysets.py b/uvdat/core/rest/querysets.py new file mode 100644 index 000000000..f8337c386 --- /dev/null +++ b/uvdat/core/rest/querysets.py @@ -0,0 +1,123 @@ +from __future__ import annotations + +from django.db.models import Case, CharField, Count, F, Prefetch, Q, QuerySet, Value, When + +from uvdat.core.models import Layer, LayerFrame, LayerStyle, RasterFramePreview +from uvdat.core.models.frame_preview import PreviewStatus + +RASTER_FRAMES_QUERYSET = ( + LayerFrame.objects.filter(raster__isnull=False).select_related("raster").order_by("index") +) + +FRAME_PREVIEWS_PREFETCH = Prefetch( + "frame_previews", + queryset=RasterFramePreview.objects.select_related("layer_frame"), +) + +LAYER_RASTER_FRAMES_PREFETCH = Prefetch( + "frames", + queryset=RASTER_FRAMES_QUERYSET, + to_attr="raster_frames", +) + +STYLE_LAYER_RASTER_FRAMES_PREFETCH = Prefetch( + "layer__frames", + queryset=RASTER_FRAMES_QUERYSET, + to_attr="raster_frames", +) + + +def _preview_status_case() -> Case: + return Case( + When(_raster_frame_count__lte=1, then=Value(None, output_field=CharField(null=True))), + When( + _complete_with_image_count=F("_raster_frame_count"), + then=Value("ready"), + ), + default=Value("notready"), + output_field=CharField(null=True), + ) + + +def _annotate_layer_style_preview_counts(queryset: QuerySet) -> QuerySet: + return queryset.annotate( + _raster_frame_count=Count( + "layer__frames", + filter=Q(layer__frames__raster__isnull=False), + distinct=True, + ), + _complete_with_image_count=Count( + "frame_previews", + filter=Q( + frame_previews__status=PreviewStatus.COMPLETE, + frame_previews__image__isnull=False, + ) + & ~Q(frame_previews__image=""), + distinct=True, + ), + ) + + +def _annotate_layer_preview_counts(queryset: QuerySet) -> QuerySet: + return queryset.annotate( + _raster_frame_count=Count( + "frames", + filter=Q(frames__raster__isnull=False), + distinct=True, + ), + _complete_with_image_count=Count( + "default_style__frame_previews", + filter=Q( + default_style__frame_previews__status=PreviewStatus.COMPLETE, + default_style__frame_previews__image__isnull=False, + ) + & ~Q(default_style__frame_previews__image=""), + distinct=True, + ), + ) + + +def annotate_layer_style_preview_status(queryset: QuerySet) -> QuerySet: + return _annotate_layer_style_preview_counts(queryset).annotate( + preview_status=_preview_status_case(), + ) + + +def annotate_layer_preview_status(queryset: QuerySet) -> QuerySet: + return _annotate_layer_preview_counts(queryset).annotate( + preview_status=Case( + When( + default_style_id__isnull=True, then=Value(None, output_field=CharField(null=True)) + ), + default=_preview_status_case(), + output_field=CharField(null=True), + ), + ) + + +def layer_queryset_with_previews( + queryset: QuerySet | None = None, + *, + for_layer_style: bool = False, +) -> QuerySet: + # This is for /layer-styles/?layer={layer_id} so we need to prefetch the frames for all styles + if for_layer_style: + qs = queryset if queryset is not None else LayerStyle.objects.all() + return annotate_layer_style_preview_status( + qs.select_related("layer", "layer__default_style").prefetch_related( + FRAME_PREVIEWS_PREFETCH, + STYLE_LAYER_RASTER_FRAMES_PREFETCH, + ) + ) + + qs = queryset if queryset is not None else Layer.objects.all() + # This is for /layers/{layer_id} so we need to prefetch the frames and the default style + return annotate_layer_preview_status( + qs.select_related("dataset", "default_style").prefetch_related( + LAYER_RASTER_FRAMES_PREFETCH, + Prefetch( + "default_style__frame_previews", + queryset=RasterFramePreview.objects.select_related("layer_frame"), + ), + ) + ) diff --git a/uvdat/core/rest/serializers.py b/uvdat/core/rest/serializers.py index c79ff69bb..c424a8768 100644 --- a/uvdat/core/rest/serializers.py +++ b/uvdat/core/rest/serializers.py @@ -5,6 +5,11 @@ from django.contrib.gis.serializers import geojson from rest_framework import serializers +from uvdat.core.frame_previews.preview_regeneration import ( + get_layer_preview_status, + get_layer_style_preview_status, + invalidate_and_enqueue_previews, +) from uvdat.core.models import ( Basemap, Chart, @@ -149,8 +154,15 @@ class Meta: fields = "__all__" +def _omit_null_field(data: dict, field: str) -> None: + if data.get(field) is None: + data.pop(field, None) + + class LayerStyleSerializer(serializers.ModelSerializer): is_default = serializers.SerializerMethodField("get_is_default") + # Client-computed django-large-image style JSON; write-only (used for previews). + raster_style_params = serializers.JSONField(required=False, allow_null=True, write_only=True) def get_is_default(self, obj): if obj.layer.default_style is None: @@ -166,24 +178,75 @@ def create(self, validated_data): style_spec = self.initial_data.pop("style_spec", None) instance = super().create(validated_data) instance.save_style_configs(style_spec) + invalidate_and_enqueue_previews(instance) return instance def update(self, instance, validated_data): style_spec = self.initial_data.pop("style_spec", None) instance.save_style_configs(style_spec) - return super().update(instance, validated_data) + instance = super().update(instance, validated_data) + invalidate_and_enqueue_previews(instance) + return instance class Meta: model = LayerStyle exclude = ["default_frame", "opacity"] +class LayerStyleWithPreviewsSerializer(LayerStyleSerializer): + multiframe_previews = serializers.SerializerMethodField() + preview_status = serializers.SerializerMethodField() + + def _preview_layer(self, obj): + return self.context.get("preview_layer") or obj.layer + + def get_preview_status(self, obj): + return get_layer_style_preview_status(obj) + + def get_multiframe_previews(self, obj): + if get_layer_style_preview_status(obj) != "ready": + return None + return obj.multiframe_previews(layer=self._preview_layer(obj)) + + def to_representation(self, instance): + data = super().to_representation(instance) + _omit_null_field(data, "multiframe_previews") + _omit_null_field(data, "preview_status") + return data + + class LayerSerializer(serializers.ModelSerializer): default_style = LayerStyleSerializer() + multiframe_previews = serializers.SerializerMethodField() + preview_status = serializers.SerializerMethodField() + + def get_preview_status(self, obj): + return get_layer_preview_status(obj) + + def get_multiframe_previews(self, obj): + if obj.default_style_id is None: + return None + if get_layer_preview_status(obj) != "ready": + return None + return obj.default_multiframe_previews() + + def to_representation(self, instance): + data = super().to_representation(instance) + _omit_null_field(data, "multiframe_previews") + _omit_null_field(data, "preview_status") + return data class Meta: model = Layer - fields = ["id", "name", "metadata", "dataset", "default_style"] + fields = [ + "id", + "name", + "metadata", + "dataset", + "default_style", + "multiframe_previews", + "preview_status", + ] class VectorDataSerializer(serializers.ModelSerializer): diff --git a/uvdat/core/tasks/__init__.py b/uvdat/core/tasks/__init__.py index 9e431834c..f1656a793 100644 --- a/uvdat/core/tasks/__init__.py +++ b/uvdat/core/tasks/__init__.py @@ -2,8 +2,10 @@ from .chart import convert_chart from .dataset import convert_dataset +from .frame_preview import generate_layer_style_previews __all__ = [ "convert_chart", "convert_dataset", + "generate_layer_style_previews", ] diff --git a/uvdat/core/tasks/analytics/flood_simulation.py b/uvdat/core/tasks/analytics/flood_simulation.py index 4015338cc..ebf78f117 100644 --- a/uvdat/core/tasks/analytics/flood_simulation.py +++ b/uvdat/core/tasks/analytics/flood_simulation.py @@ -220,6 +220,12 @@ def flood_simulation(result_id): ], } ) + style.raster_style_params = { + "min": 0, + "max": 2, + "palette": ["#002081", "#2AD3FF"], + } + style.save(update_fields=["raster_style_params"]) result.write_outputs( { diff --git a/uvdat/core/tasks/dataset.py b/uvdat/core/tasks/dataset.py index 2724079ca..435042786 100644 --- a/uvdat/core/tasks/dataset.py +++ b/uvdat/core/tasks/dataset.py @@ -136,7 +136,7 @@ def create_layers_and_frames(dataset, layer_options=None): # noqa: C901, PLR091 index=index, vector=vector, raster=raster, - source_filters=frame_info.get("source_filters", {"band": 1}), + source_filters=frame_info.get("source_filters", {}), ) diff --git a/uvdat/core/tasks/frame_preview.py b/uvdat/core/tasks/frame_preview.py new file mode 100644 index 000000000..7bf9a8ec2 --- /dev/null +++ b/uvdat/core/tasks/frame_preview.py @@ -0,0 +1,413 @@ +from __future__ import annotations + +from dataclasses import dataclass +import io +import json +import logging +import time +from typing import TYPE_CHECKING, Any + +from celery import shared_task +from django.core.files.base import ContentFile +from django_large_image import tilesource, utilities +from PIL import Image + +from uvdat.core.frame_previews.fingerprint import style_fingerprint +from uvdat.core.frame_previews.raster_style import ( + apply_source_filters_to_style_query, + raster_source_filter_kwargs, +) +from uvdat.core.models import ( + Layer, + LayerStyle, + Project, + RasterData, + RasterFramePreview, + TaskResult, +) +from uvdat.core.models.frame_preview import PreviewStatus + +if TYPE_CHECKING: + from uvdat.core.frame_previews.types import FramePreviewBounds + +"""Celery tasks and helpers for multiframe raster frame preview images. + +Previews are styled PNG thumbnails stored on ``RasterFramePreview`` rows. They +let the frontend show a fast full-frame image before tile loading during frame +scrubbing. Generation is keyed by a style fingerprint so rapid successive style +saves do not publish stale images from an older task run. +""" + +logger = logging.getLogger(__name__) + +# Thumbnail sizing: default to 1/8 of FRAME_PREVIEW_MAX_PX (512px), but never +# below FRAME_PREVIEW_MIN_PX when the source raster is large enough to allow it. +FRAME_PREVIEW_MAX_PX = 4096 +FRAME_PREVIEW_MIN_PX = 1024 +FRAME_PREVIEW_DEFAULT_RESOLUTION_FRACTION = 1 / 8 + +# Applied when ingest creates a default style for a new multiframe raster layer. +DEFAULT_MULTIFRAME_RASTER_STYLE_SPEC = { + "default_frame": 0, + "opacity": 1, + "colors": [{"name": "all", "visible": True, "use_feature_props": True}], + "sizes": [{"name": "all", "zoom_scaling": True, "single_size": 5}], + "filters": [], +} + + +@dataclass(frozen=True) +class _PreviewGenerationContext: + """Immutable inputs shared across all frames in one task invocation.""" + + layer_style: LayerStyle + fingerprint: str + base_style_query: dict[str, Any] + resolution_fraction: float | None = None + + @property + def layer_style_id(self) -> int: + return self.layer_style.id + + +@dataclass(frozen=True) +class _FramePreviewImage: + """PNG payload and metadata produced by ``generate_frame_preview_png``.""" + + png_bytes: bytes + width: int + height: int + bounds: FramePreviewBounds | None + + +@dataclass(frozen=True) +class _PreviewGenerationStats: + """Per-task frame counts written to ``TaskResult.outputs`` on completion.""" + + ready_count: int + failed_count: int + + +def resolve_resolution_fraction(resolution_fraction: float | None = None) -> float: + if resolution_fraction is None: + return FRAME_PREVIEW_DEFAULT_RESOLUTION_FRACTION + return float(resolution_fraction) + + +def _raster_max_dimension(metadata: dict[str, Any]) -> int: + size_x = metadata.get("sizeX") or metadata.get("width") or 0 + size_y = metadata.get("sizeY") or metadata.get("height") or 0 + return max(int(size_x), int(size_y)) + + +def resolve_preview_max_dimension( + resolution_fraction: float | None = None, + raster_max_dimension: int | None = None, +) -> int: + """Pick a thumbnail edge length, clamped to the source raster's size.""" + fraction = resolve_resolution_fraction(resolution_fraction) + fractional = round(FRAME_PREVIEW_MAX_PX * fraction) + if not raster_max_dimension or raster_max_dimension < 2: + return max(2, fractional) + # Small rasters use their native size; larger ones get at least MIN_PX. + floor = min(raster_max_dimension, FRAME_PREVIEW_MIN_PX) + return max(2, min(max(fractional, floor), raster_max_dimension)) + + +def _thumbnail_png_bytes(thumb_data: Any) -> bytes: + if isinstance(thumb_data, bytes): + return thumb_data + if isinstance(thumb_data, Image.Image): + buffer = io.BytesIO() + thumb_data.save(buffer, format="PNG") + return buffer.getvalue() + msg = f"Unsupported thumbnail data type: {type(thumb_data)!r}" + raise TypeError(msg) + + +def _preview_bounds(source) -> FramePreviewBounds | None: + bounds = tilesource.get_bounds(source, projection="EPSG:4326") + if not bounds: + return None + result: FramePreviewBounds = { + "srs": "EPSG:4326", + "xmin": bounds["xmin"], + "xmax": bounds["xmax"], + "ymin": bounds["ymin"], + "ymax": bounds["ymax"], + } + for corner in ("ul", "ur", "lr", "ll"): + corner_bounds = bounds.get(corner) + if corner_bounds: + result[corner] = {"x": corner_bounds["x"], "y": corner_bounds["y"]} + return result + + +def generate_frame_preview_png( + raster: RasterData, + source_filters: dict[str, Any] | None, + base_style_query: dict[str, Any], + resolution_fraction: float | None = None, +) -> tuple[bytes, int, int, FramePreviewBounds | None]: + """Render one frame as a styled PNG via large-image. + + Frame selection is passed through ``source_filters`` (e.g. ``{"frame": 3}``), + not embedded in the style query, so one style query can be reused for every + frame in a multiframe layer. + """ + style_query = apply_source_filters_to_style_query(base_style_query, source_filters) + style = json.dumps(style_query) if style_query else None + source_kwargs = raster_source_filter_kwargs(source_filters) + raster_path = utilities.field_file_to_local_path(raster.cloud_optimized_geotiff) + source = tilesource.get_tilesource_from_path( + raster_path, + encoding="PNG", + style=style, + ) + max_dimension = resolve_preview_max_dimension( + resolution_fraction, + _raster_max_dimension(source.getMetadata()), + ) + thumb_data, _mime_type = source.getThumbnail( + encoding="PNG", + width=max_dimension, + height=max_dimension, + **source_kwargs, + ) + png_bytes = _thumbnail_png_bytes(thumb_data) + image = Image.open(io.BytesIO(png_bytes)) + return png_bytes, image.width, image.height, _preview_bounds(source) + + +def ensure_default_layer_style(layer: Layer, project: Project) -> LayerStyle: + """Return the project's Default style for a layer, creating it on ingest if needed.""" + style = LayerStyle.objects.filter(layer=layer, project=project, name="Default").first() + if style is None: + style = LayerStyle.objects.create(name="Default", layer=layer, project=project) + style.save_style_configs(DEFAULT_MULTIFRAME_RASTER_STYLE_SPEC) + # Default ingest style has no colormap/palette; empty large-image style JSON. + style.raster_style_params = {} + style.save(update_fields=["raster_style_params"]) + if layer.default_style_id is None: + layer.default_style = style + layer.save(update_fields=["default_style"]) + return style + + +def _fingerprint_matches(layer_style: LayerStyle, fingerprint: str) -> bool: + """Return whether the style's current raster params still match the task fingerprint. + + Compares against ``raster_style_params`` in the database so a style save + after this task was enqueued aborts stale generation. + """ + return style_fingerprint(layer_style) == fingerprint + + +def _open_task_result(result_id: int | None, layer_style_id: int) -> TaskResult | None: + """Load the TaskResult for this run, or None if it was already superseded.""" + if result_id is None: + return None + + # Use filter().first() so a missing row yields None for the branching below + # (get() would raise DoesNotExist). + result = TaskResult.objects.filter(id=result_id).first() + if result is None or result.completed is not None: + logger.info( + "Skipping preview generation for style=%s; task result %s already closed", + layer_style_id, + result_id, + ) + return None + return result + + +def _save_frame_preview( + preview: RasterFramePreview, + layer_style_id: int, + frame_index: int, + image: _FramePreviewImage, +) -> None: + """Persist a generated preview and mark the row complete (API-servable).""" + preview.width = image.width + preview.height = image.height + preview.bounds = image.bounds or {} + preview.status = PreviewStatus.COMPLETE + preview.image.save( + f"frame-previews/{layer_style_id}/{frame_index}.png", + ContentFile(image.png_bytes), + save=False, + ) + preview.save() + + +def _mark_frame_preview_failed( + preview: RasterFramePreview, + layer_style: LayerStyle, + fingerprint: str, +) -> None: + """Mark a row failed only when both the row and style still match this task.""" + if preview.style_fingerprint == fingerprint and _fingerprint_matches(layer_style, fingerprint): + preview.status = PreviewStatus.FAILED + preview.save(update_fields=["status"]) + + +def _process_frame_preview(ctx: _PreviewGenerationContext, frame) -> str: + """Generate one frame preview. + + Returns ``ready``, ``failed``, ``skipped``, or ``superseded``. A superseded + outcome means a newer style save arrived and this task must stop writing. + """ + if not _fingerprint_matches(ctx.layer_style, ctx.fingerprint): + return "superseded" + + try: + preview = RasterFramePreview.objects.get( + layer_style=ctx.layer_style, + layer_frame=frame, + ) + except RasterFramePreview.DoesNotExist: + logger.warning( + "Missing preview row for style=%s frame=%s; skipping", + ctx.layer_style_id, + frame.id, + ) + return "skipped" + + # Row fingerprint is set on enqueue; skip frames already claimed by a newer save. + if preview.style_fingerprint != ctx.fingerprint: + return "skipped" + + try: + png_bytes, width, height, bounds = generate_frame_preview_png( + frame.raster, + frame.source_filters, + ctx.base_style_query, + ctx.resolution_fraction, + ) + except Exception: + logger.exception( + "Failed to generate frame preview for style=%s frame=%s", + ctx.layer_style_id, + frame.id, + ) + _mark_frame_preview_failed(preview, ctx.layer_style, ctx.fingerprint) + return "failed" + + # PNG generation is expensive; re-check before writing to storage. + if not _fingerprint_matches(ctx.layer_style, ctx.fingerprint): + return "superseded" + + _save_frame_preview( + preview, + ctx.layer_style_id, + frame.index, + _FramePreviewImage(png_bytes, width, height, bounds), + ) + return "ready" + + +def _complete_preview_task( + result: TaskResult | None, + ctx: _PreviewGenerationContext, + stats: _PreviewGenerationStats, +) -> None: + """Finalize the TaskResult, which triggers a WebSocket notification.""" + if not _fingerprint_matches(ctx.layer_style, ctx.fingerprint): + logger.info( + "Skipping task completion for style=%s; style superseded after loop", + ctx.layer_style_id, + ) + return + + if result is None: + return + + result.write_outputs( + { + "layer_style_id": ctx.layer_style_id, + "layer_id": ctx.layer_style.layer_id, + "fingerprint": ctx.fingerprint, + "ready_count": stats.ready_count, + "failed_count": stats.failed_count, + } + ) + result.complete() + + +@shared_task +def generate_layer_style_previews( + layer_style_id: int, + fingerprint: str, + result_id: int | None = None, + resolution_fraction: float | None = None, +): + """Generate styled PNG previews for every frame in a multiframe raster style. + + Enqueued by ``invalidate_and_enqueue_previews`` after a style save or ingest. + Preview rows are created upstream with ``creating``/``regenerating`` status and + cleared images; this task fills them in and sets ``complete`` or ``failed``. + + ``fingerprint`` is a sha256 of ``raster_style_params`` at enqueue time. + The task aborts whenever the live style no longer matches, so rapid + double-saves only publish previews for the latest style version. + """ + started = time.perf_counter() + layer_style = LayerStyle.objects.select_related("layer", "project").get(id=layer_style_id) + frames = layer_style.layer.multiframe_raster_frames() + if len(frames) <= 1: + return + + if not _fingerprint_matches(layer_style, fingerprint): + logger.info( + "Skipping superseded preview generation for style=%s", + layer_style_id, + ) + return + + result = _open_task_result(result_id, layer_style_id) + if result_id is not None and result is None: + return + + logger.info( + "Generating %d multiframe raster previews for style=%s layer=%r", + len(frames), + layer_style_id, + layer_style.layer.name, + ) + + ctx = _PreviewGenerationContext( + layer_style=layer_style, + fingerprint=fingerprint, + base_style_query=dict(layer_style.raster_style_params or {}), + resolution_fraction=resolution_fraction, + ) + + ready_count = 0 + failed_count = 0 + for frame in frames: + outcome = _process_frame_preview(ctx, frame) + if outcome == "superseded": + logger.info( + "Aborting preview generation for style=%s; style superseded", + layer_style_id, + ) + return + if outcome == "ready": + ready_count += 1 + elif outcome == "failed": + failed_count += 1 + + _complete_preview_task( + result, + ctx, + _PreviewGenerationStats(ready_count, failed_count), + ) + + logger.info( + "Multiframe raster previews for style=%s layer=%r: %d ready, %d failed in %.2fs", + layer_style_id, + layer_style.layer.name, + ready_count, + failed_count, + time.perf_counter() - started, + ) diff --git a/uvdat/core/tests/factories.py b/uvdat/core/tests/factories.py index d392a6293..f5de15e35 100644 --- a/uvdat/core/tests/factories.py +++ b/uvdat/core/tests/factories.py @@ -132,7 +132,7 @@ class Meta: name = factory.Faker("name") layer = factory.SubFactory(LayerFactory) - vector = factory.SubFactory(VectorDataFactory) + vector = None raster = factory.SubFactory(RasterDataFactory) diff --git a/uvdat/core/tests/test_frame_preview.py b/uvdat/core/tests/test_frame_preview.py new file mode 100644 index 000000000..170850bed --- /dev/null +++ b/uvdat/core/tests/test_frame_preview.py @@ -0,0 +1,673 @@ +from __future__ import annotations + +from django.core.files.base import ContentFile +import pytest + +from uvdat.core.frame_previews.fingerprint import _fingerprint_payload, style_fingerprint +from uvdat.core.frame_previews.preview_regeneration import ( + get_layer_style_preview_status, + invalidate_and_enqueue_previews, +) +from uvdat.core.frame_previews.raster_style import ( + apply_source_filters_to_style_query, + raster_source_filter_kwargs, +) +from uvdat.core.models import LayerStyle, RasterFramePreview +from uvdat.core.models.frame_preview import PreviewStatus +from uvdat.core.tasks.frame_preview import ( + DEFAULT_MULTIFRAME_RASTER_STYLE_SPEC, + FRAME_PREVIEW_DEFAULT_RESOLUTION_FRACTION, + FRAME_PREVIEW_MAX_PX, + FRAME_PREVIEW_MIN_PX, + resolve_preview_max_dimension, +) + + +@pytest.mark.django_db +def test_layer_style_api_stores_client_raster_style_params( + authenticated_api_client, + layer_style_factory, + layer_frame_factory, + project, + user, +): + layer_style = layer_style_factory() + layer_frame_factory(layer=layer_style.layer, index=0) + project.set_collaborators([user]) + project.datasets.set([layer_style.layer.dataset]) + raster_style_params = {"palette": "#00ff00", "min": 0, "max": 1} + + resp = authenticated_api_client.patch( + f"/api/v1/layer-styles/{layer_style.id}/", + { + "name": layer_style.name, + "layer": layer_style.layer_id, + "project": layer_style.project_id, + "style_spec": DEFAULT_MULTIFRAME_RASTER_STYLE_SPEC, + "raster_style_params": raster_style_params, + }, + format="json", + ) + assert resp.status_code == 200 + assert "raster_style_params" not in resp.json() + layer_style.refresh_from_db() + assert layer_style.raster_style_params == raster_style_params + + +def test_raster_source_filter_kwargs_extracts_frame(): + assert raster_source_filter_kwargs({"frame": 3}) == {"frame": 3} + assert raster_source_filter_kwargs({"band": 2}) == {} + assert raster_source_filter_kwargs({"frame": 1, "band": 2}) == {"frame": 1} + + +def test_apply_source_filters_to_style_query_embeds_band_not_frame(): + query = apply_source_filters_to_style_query({"palette": "#fff"}, {"frame": 3, "band": 2}) + assert query == {"palette": "#fff", "band": 2} + assert "frame" not in query + assert apply_source_filters_to_style_query({}, {"frame": 3, "band": 1}) == {"band": 1} + assert apply_source_filters_to_style_query({}, {"band": 1}) == {"band": 1} + + +@pytest.mark.parametrize( + ("resolution_fraction", "raster_max_dimension", "expected_max_px"), + [ + (None, None, round(FRAME_PREVIEW_MAX_PX * FRAME_PREVIEW_DEFAULT_RESOLUTION_FRACTION)), + (0.25, None, 1024), + (0.5, None, 2048), + (FRAME_PREVIEW_DEFAULT_RESOLUTION_FRACTION, None, 512), + (None, 4096, FRAME_PREVIEW_MIN_PX), + (0.25, 4096, FRAME_PREVIEW_MIN_PX), + (0.5, 4096, 2048), + (None, 800, 800), + (0.5, 500, 500), + (0.75, None, 3072), + ], +) +def test_resolve_preview_max_dimension( + resolution_fraction, + raster_max_dimension, + expected_max_px, +): + assert ( + resolve_preview_max_dimension(resolution_fraction, raster_max_dimension) == expected_max_px + ) + + +@pytest.mark.django_db +def test_style_fingerprint_matches_db_after_ingest_style_setup(layer_style_factory): + """Enqueue fingerprint must match what the Celery task reads from the database.""" + style = layer_style_factory() + style.raster_style_params = {"palette": "#00ff00", "min": 0, "max": 1} + style.save(update_fields=["raster_style_params"]) + + assert style_fingerprint(style) == style_fingerprint(LayerStyle.objects.get(pk=style.pk)) + + +def test_style_fingerprint_stable_key_order_and_treats_null_as_empty(): + """Fingerprint uses sorted JSON keys; null params match empty params.""" + base = {"palette": "#fff", "min": 0, "max": 1} + reordered = {"max": 1, "palette": "#fff", "min": 0} + + assert _fingerprint_payload(base) == _fingerprint_payload(reordered) + assert _fingerprint_payload(None) == _fingerprint_payload({}) + + +@pytest.mark.django_db +def test_invalidate_and_enqueue_previews_uses_db_fingerprint( + layer_style_factory, + layer_frame_factory, + mocker, +): + layer_style = layer_style_factory() + layer_frame_factory(layer=layer_style.layer, index=0) + layer_frame_factory(layer=layer_style.layer, index=1) + layer_style.save_style_configs(DEFAULT_MULTIFRAME_RASTER_STYLE_SPEC) + + delay = mocker.patch("uvdat.core.tasks.frame_preview.generate_layer_style_previews.delay") + + invalidate_and_enqueue_previews(layer_style) + + delay.assert_called_once() + _, fingerprint, _ = delay.call_args.args + assert fingerprint == style_fingerprint(LayerStyle.objects.get(pk=layer_style.pk)) + + +@pytest.mark.django_db +def test_invalidate_and_enqueue_previews_skips_when_previews_current( + layer_style_factory, + layer_frame_factory, + mocker, +): + layer_style = layer_style_factory() + frame_0 = layer_frame_factory(layer=layer_style.layer, index=0) + frame_1 = layer_frame_factory(layer=layer_style.layer, index=1) + layer_style.save_style_configs(DEFAULT_MULTIFRAME_RASTER_STYLE_SPEC) + fingerprint = style_fingerprint(layer_style) + + preview_0 = RasterFramePreview.objects.create( + layer_style=layer_style, + layer_frame=frame_0, + status=PreviewStatus.COMPLETE, + style_fingerprint=fingerprint, + width=100, + height=80, + bounds={"srs": "EPSG:4326", "xmin": -1, "xmax": 1, "ymin": -2, "ymax": 2}, + ) + preview_0.image.save("frame-0.png", ContentFile(b"png0"), save=True) + preview_1 = RasterFramePreview.objects.create( + layer_style=layer_style, + layer_frame=frame_1, + status=PreviewStatus.COMPLETE, + style_fingerprint=fingerprint, + width=120, + height=90, + bounds={"srs": "EPSG:4326", "xmin": -2, "xmax": 2, "ymin": -3, "ymax": 3}, + ) + preview_1.image.save("frame-1.png", ContentFile(b"png1"), save=True) + + delay = mocker.patch("uvdat.core.tasks.frame_preview.generate_layer_style_previews.delay") + + result = invalidate_and_enqueue_previews(layer_style) + + delay.assert_not_called() + assert result is None + assert get_layer_style_preview_status(layer_style) == "ready" + + preview_0.refresh_from_db() + preview_1.refresh_from_db() + assert preview_0.status == PreviewStatus.COMPLETE + assert preview_1.status == PreviewStatus.COMPLETE + assert preview_0.image.name + assert preview_1.image.name + assert preview_0.width == 100 + assert preview_1.width == 120 + + +@pytest.mark.django_db +def test_invalidate_and_enqueue_previews_runs_when_fingerprint_changed( + layer_style_factory, + layer_frame_factory, + mocker, +): + layer_style = layer_style_factory() + frame_0 = layer_frame_factory(layer=layer_style.layer, index=0) + frame_1 = layer_frame_factory(layer=layer_style.layer, index=1) + layer_style.save_style_configs(DEFAULT_MULTIFRAME_RASTER_STYLE_SPEC) + + preview_0 = RasterFramePreview.objects.create( + layer_style=layer_style, + layer_frame=frame_0, + status=PreviewStatus.COMPLETE, + style_fingerprint="stale-fingerprint", + width=100, + height=80, + bounds={}, + ) + preview_0.image.save("frame-0.png", ContentFile(b"png0"), save=True) + preview_1 = RasterFramePreview.objects.create( + layer_style=layer_style, + layer_frame=frame_1, + status=PreviewStatus.COMPLETE, + style_fingerprint="stale-fingerprint", + width=120, + height=90, + bounds={}, + ) + preview_1.image.save("frame-1.png", ContentFile(b"png1"), save=True) + + delay = mocker.patch("uvdat.core.tasks.frame_preview.generate_layer_style_previews.delay") + + result = invalidate_and_enqueue_previews(layer_style) + + delay.assert_called_once() + assert result is not None + + preview_0.refresh_from_db() + preview_1.refresh_from_db() + assert preview_0.status == PreviewStatus.REGENERATING + assert preview_1.status == PreviewStatus.REGENERATING + assert not preview_0.image + assert not preview_1.image + + +@pytest.mark.django_db +def test_invalidate_and_enqueue_previews_runs_synchronously( + layer_style_factory, + layer_frame_factory, + mocker, +): + layer_style = layer_style_factory() + layer_frame_factory(layer=layer_style.layer, index=0) + layer_frame_factory(layer=layer_style.layer, index=1) + layer_style.save_style_configs(DEFAULT_MULTIFRAME_RASTER_STYLE_SPEC) + + delay = mocker.patch("uvdat.core.tasks.frame_preview.generate_layer_style_previews.delay") + apply = mocker.patch("uvdat.core.tasks.frame_preview.generate_layer_style_previews.apply") + + invalidate_and_enqueue_previews(layer_style, asynchronous=False) + + apply.assert_called_once() + delay.assert_not_called() + assert apply.call_args.kwargs["args"][0] == layer_style.id + + +@pytest.mark.django_db +def test_multiframe_previews_for_style_returns_none_for_single_frame( + layer_style_factory, + layer_frame_factory, +): + layer_style = layer_style_factory() + layer_frame_factory(layer=layer_style.layer, index=0) + + assert layer_style.multiframe_previews() is None + + +@pytest.mark.django_db +def test_multiframe_previews_for_style_returns_none_when_partial( + layer_style_factory, + layer_frame_factory, +): + layer_style = layer_style_factory() + frame_0 = layer_frame_factory(layer=layer_style.layer, index=0) + layer_frame_factory(layer=layer_style.layer, index=1) + frame_2 = layer_frame_factory(layer=layer_style.layer, index=2) + + preview_0 = RasterFramePreview.objects.create( + layer_style=layer_style, + layer_frame=frame_0, + status=PreviewStatus.COMPLETE, + width=100, + height=80, + bounds={"srs": "EPSG:4326", "xmin": -1, "xmax": 1, "ymin": -2, "ymax": 2}, + ) + preview_0.image.save("frame-0.png", ContentFile(b"png0"), save=True) + preview_2 = RasterFramePreview.objects.create( + layer_style=layer_style, + layer_frame=frame_2, + status=PreviewStatus.COMPLETE, + width=200, + height=150, + bounds={"srs": "EPSG:4326", "xmin": -3, "xmax": 3, "ymin": -4, "ymax": 4}, + ) + preview_2.image.save("frame-2.png", ContentFile(b"png2"), save=True) + + assert layer_style.multiframe_previews() is None + + +@pytest.mark.django_db +def test_multiframe_previews_for_style_ordered_by_frame_index( + layer_style_factory, + layer_frame_factory, +): + layer_style = layer_style_factory() + frame_0 = layer_frame_factory(layer=layer_style.layer, index=0) + frame_1 = layer_frame_factory(layer=layer_style.layer, index=1) + frame_2 = layer_frame_factory(layer=layer_style.layer, index=2) + + preview_0 = RasterFramePreview.objects.create( + layer_style=layer_style, + layer_frame=frame_0, + status=PreviewStatus.COMPLETE, + width=100, + height=80, + bounds={"srs": "EPSG:4326", "xmin": -1, "xmax": 1, "ymin": -2, "ymax": 2}, + ) + preview_0.image.save("frame-0.png", ContentFile(b"png0"), save=True) + preview_1 = RasterFramePreview.objects.create( + layer_style=layer_style, + layer_frame=frame_1, + status=PreviewStatus.COMPLETE, + width=120, + height=90, + bounds={"srs": "EPSG:4326", "xmin": -2, "xmax": 2, "ymin": -3, "ymax": 3}, + ) + preview_1.image.save("frame-1.png", ContentFile(b"png1"), save=True) + preview_2 = RasterFramePreview.objects.create( + layer_style=layer_style, + layer_frame=frame_2, + status=PreviewStatus.COMPLETE, + width=200, + height=150, + bounds={"srs": "EPSG:4326", "xmin": -3, "xmax": 3, "ymin": -4, "ymax": 4}, + ) + preview_2.image.save("frame-2.png", ContentFile(b"png2"), save=True) + + previews = layer_style.multiframe_previews() + assert previews == [ + { + "url": preview_0.image.url, + "width": 100, + "height": 80, + "bounds": preview_0.bounds, + }, + { + "url": preview_1.image.url, + "width": 120, + "height": 90, + "bounds": preview_1.bounds, + }, + { + "url": preview_2.image.url, + "width": 200, + "height": 150, + "bounds": preview_2.bounds, + }, + ] + + +@pytest.mark.django_db +def test_preview_bounds_includes_corners( + layer_style_factory, + layer_frame_factory, +): + layer_style = layer_style_factory() + frame_0 = layer_frame_factory(layer=layer_style.layer, index=0) + frame_1 = layer_frame_factory(layer=layer_style.layer, index=1) + + preview_0 = RasterFramePreview.objects.create( + layer_style=layer_style, + layer_frame=frame_0, + status=PreviewStatus.COMPLETE, + width=100, + height=80, + bounds={ + "srs": "EPSG:4326", + "xmin": -1, + "xmax": 1, + "ymin": -2, + "ymax": 2, + "ul": {"x": -1, "y": 2}, + "ur": {"x": 1, "y": 2}, + "lr": {"x": 1, "y": -2}, + "ll": {"x": -1, "y": -2}, + }, + ) + preview_0.image.save("frame-0.png", ContentFile(b"png0"), save=True) + preview_1 = RasterFramePreview.objects.create( + layer_style=layer_style, + layer_frame=frame_1, + status=PreviewStatus.COMPLETE, + width=100, + height=80, + bounds={}, + ) + preview_1.image.save("frame-1.png", ContentFile(b"png1"), save=True) + + previews = layer_style.multiframe_previews() + assert previews[0]["bounds"]["ul"] == {"x": -1, "y": 2} + + +@pytest.mark.django_db +def test_layer_style_api_includes_multiframe_previews( + authenticated_api_client, + layer_style_factory, + layer_frame_factory, + project, + user, +): + layer_style = layer_style_factory() + project.set_collaborators([user]) + project.datasets.set([layer_style.layer.dataset]) + frame_0 = layer_frame_factory(layer=layer_style.layer, index=0) + frame_1 = layer_frame_factory(layer=layer_style.layer, index=1) + + preview_0 = RasterFramePreview.objects.create( + layer_style=layer_style, + layer_frame=frame_0, + status=PreviewStatus.COMPLETE, + width=100, + height=100, + bounds={}, + ) + preview_0.image.save("frame-0.png", ContentFile(b"png0"), save=True) + preview_1 = RasterFramePreview.objects.create( + layer_style=layer_style, + layer_frame=frame_1, + status=PreviewStatus.COMPLETE, + width=100, + height=100, + bounds={}, + ) + preview_1.image.save("frame-1.png", ContentFile(b"png1"), save=True) + + resp = authenticated_api_client.get(f"/api/v1/layer-styles/{layer_style.id}/") + assert resp.status_code == 200 + data = resp.json() + assert data["preview_status"] == "ready" + assert data["multiframe_previews"] == [ + { + "url": preview_0.image.url, + "width": 100, + "height": 100, + "bounds": {}, + }, + { + "url": preview_1.image.url, + "width": 100, + "height": 100, + "bounds": {}, + }, + ] + + +@pytest.mark.django_db +def test_layer_style_patch_reports_notready_after_preview_invalidation( + authenticated_api_client, + layer_style_factory, + layer_frame_factory, + project, + user, + mocker, +): + layer_style = layer_style_factory() + project.set_collaborators([user]) + project.datasets.set([layer_style.layer.dataset]) + frame_0 = layer_frame_factory(layer=layer_style.layer, index=0) + frame_1 = layer_frame_factory(layer=layer_style.layer, index=1) + layer_style.save_style_configs(DEFAULT_MULTIFRAME_RASTER_STYLE_SPEC) + layer_style.raster_style_params = {"palette": "#00ff00", "min": 0, "max": 1} + layer_style.save(update_fields=["raster_style_params"]) + fingerprint = style_fingerprint(layer_style) + + preview_0 = RasterFramePreview.objects.create( + layer_style=layer_style, + layer_frame=frame_0, + status=PreviewStatus.COMPLETE, + style_fingerprint=fingerprint, + width=100, + height=100, + bounds={}, + ) + preview_0.image.save("frame-0.png", ContentFile(b"png0"), save=True) + preview_1 = RasterFramePreview.objects.create( + layer_style=layer_style, + layer_frame=frame_1, + status=PreviewStatus.COMPLETE, + style_fingerprint=fingerprint, + width=100, + height=100, + bounds={}, + ) + preview_1.image.save("frame-1.png", ContentFile(b"png1"), save=True) + + mocker.patch("uvdat.core.tasks.frame_preview.generate_layer_style_previews.delay") + + resp = authenticated_api_client.patch( + f"/api/v1/layer-styles/{layer_style.id}/", + { + "name": layer_style.name, + "layer": layer_style.layer_id, + "project": layer_style.project_id, + "style_spec": DEFAULT_MULTIFRAME_RASTER_STYLE_SPEC, + "raster_style_params": {"palette": "#ff0000", "min": 0, "max": 1}, + }, + format="json", + ) + assert resp.status_code == 200 + data = resp.json() + assert data["preview_status"] == "notready" + assert "multiframe_previews" not in data + + +@pytest.mark.django_db +def test_api_omits_previews_while_not_ready( + authenticated_api_client, + layer_style_factory, + layer_frame_factory, + project, + user, +): + layer_style = layer_style_factory() + project.set_collaborators([user]) + project.datasets.set([layer_style.layer.dataset]) + frame_0 = layer_frame_factory(layer=layer_style.layer, index=0) + layer_frame_factory(layer=layer_style.layer, index=1) + + preview = RasterFramePreview.objects.create( + layer_style=layer_style, + layer_frame=frame_0, + status=PreviewStatus.COMPLETE, + width=100, + height=100, + bounds={}, + ) + preview.image.save("frame-0.png", ContentFile(b"png0"), save=True) + + resp = authenticated_api_client.get(f"/api/v1/layer-styles/{layer_style.id}/") + assert resp.status_code == 200 + data = resp.json() + assert data["preview_status"] == "notready" + assert "multiframe_previews" not in data + + +@pytest.mark.django_db +def test_layer_api_includes_multiframe_previews( + authenticated_api_client, + layer_style_factory, + layer_frame_factory, + project, + user, +): + layer_style = layer_style_factory() + layer = layer_style.layer + layer.default_style = layer_style + layer.save(update_fields=["default_style"]) + project.set_collaborators([user]) + project.datasets.set([layer.dataset]) + frame_0 = layer_frame_factory(layer=layer, index=0) + frame_1 = layer_frame_factory(layer=layer, index=1) + + preview_0 = RasterFramePreview.objects.create( + layer_style=layer_style, + layer_frame=frame_0, + status=PreviewStatus.COMPLETE, + width=100, + height=100, + bounds={}, + ) + preview_0.image.save("frame-0.png", ContentFile(b"png0"), save=True) + preview_1 = RasterFramePreview.objects.create( + layer_style=layer_style, + layer_frame=frame_1, + status=PreviewStatus.COMPLETE, + width=100, + height=100, + bounds={}, + ) + preview_1.image.save("frame-1.png", ContentFile(b"png1"), save=True) + + resp = authenticated_api_client.get(f"/api/v1/layers/{layer.id}/") + assert resp.status_code == 200 + data = resp.json() + assert data["preview_status"] == "ready" + assert data["multiframe_previews"] == [ + { + "url": preview_0.image.url, + "width": 100, + "height": 100, + "bounds": {}, + }, + { + "url": preview_1.image.url, + "width": 100, + "height": 100, + "bounds": {}, + }, + ] + assert "multiframe_previews" not in data.get("default_style", {}) + + +@pytest.mark.django_db +def test_dataset_layers_self_heal_missing_default_style( + authenticated_api_client, + layer_style_factory, + layer_frame_factory, + project, + user, +): + """A layer with styles but no default_style should still surface previews. + + ``default_style`` can be nulled (``on_delete=SET_NULL``) when the style it + pointed at is removed. The layers endpoint should adopt an existing style as + the default so its frame previews are not lost. + """ + layer_style = layer_style_factory(name="Default") + layer = layer_style.layer + layer.default_style = None + layer.save(update_fields=["default_style"]) + project.set_collaborators([user]) + project.datasets.set([layer.dataset]) + frame_0 = layer_frame_factory(layer=layer, index=0) + frame_1 = layer_frame_factory(layer=layer, index=1) + + for frame in (frame_0, frame_1): + preview = RasterFramePreview.objects.create( + layer_style=layer_style, + layer_frame=frame, + status=PreviewStatus.COMPLETE, + width=100, + height=100, + bounds={}, + ) + preview.image.save(f"frame-{frame.index}.png", ContentFile(b"png"), save=True) + + resp = authenticated_api_client.get(f"/api/v1/datasets/{layer.dataset_id}/layers/") + assert resp.status_code == 200 + layers = resp.json() + assert len(layers) == 1 + layer_data = layers[0] + assert layer_data["default_style"]["id"] == layer_style.id + assert layer_data["preview_status"] == "ready" + assert len(layer_data["multiframe_previews"]) == 2 + + layer.refresh_from_db() + assert layer.default_style_id == layer_style.id + + +@pytest.mark.django_db +def test_create_style_sets_default_when_layer_has_none( + authenticated_api_client, + layer_factory, + project, + user, +): + """Creating the first style for a layer without a default adopts it.""" + layer = layer_factory() + project.set_collaborators([user]) + project.datasets.set([layer.dataset]) + assert layer.default_style_id is None + + resp = authenticated_api_client.post( + "/api/v1/layer-styles/", + { + "name": "terrain", + "layer": layer.id, + "project": project.id, + "style_spec": DEFAULT_MULTIFRAME_RASTER_STYLE_SPEC, + }, + format="json", + ) + assert resp.status_code == 200 + + layer.refresh_from_db() + assert layer.default_style_id == resp.json()["id"] diff --git a/web/src/api/rest.ts b/web/src/api/rest.ts index 1c8019f31..e89ba8075 100644 --- a/web/src/api/rest.ts +++ b/web/src/api/rest.ts @@ -270,6 +270,10 @@ export async function getLayerStyles(layerId: number): Promise { return (await apiClient.get(`layer-styles/?layer=${layerId}`)).data.results; } +export async function getLayerStyle(styleId: number): Promise { + return (await apiClient.get(`layer-styles/${styleId}/`)).data; +} + export async function createLayerStyle(data: LayerStyle): Promise { return (await apiClient.post("layer-styles/", data)).data; } diff --git a/web/src/components/sidebars/LayerStyle.vue b/web/src/components/sidebars/LayerStyle.vue index ecd3110c2..01501a6d8 100644 --- a/web/src/components/sidebars/LayerStyle.vue +++ b/web/src/components/sidebars/LayerStyle.vue @@ -27,12 +27,14 @@ import { usePanelStore, useLayerStore, useAppStore, + useFramePreviewStore, } from "@/store"; const styleStore = useStyleStore(); const projectStore = useProjectStore(); const panelStore = usePanelStore(); const layerStore = useLayerStore(); const appStore = useAppStore(); +const framePreviewStore = useFramePreviewStore(); const emit = defineEmits(["setLayerActive"]); const props = defineProps<{ @@ -90,6 +92,40 @@ const setCurrentLayerStyle = (style: LayerStyle) => { styleStore.selectedLayerStyles[styleKey.value] = style; }; +// After a style save, the backend invalidates multiframe raster previews and +// regenerates them asynchronously. Apply the saved style (which the API returns +// with preview_status "notready" and no previews), drop any stale previews, and +// remove the on-map preview overlay so tiles are shown until regeneration +// completes and the WebSocket handler reattaches fresh previews. +const markStyleSavedAndInvalidatePreviews = (style: LayerStyle) => { + const previewStatus = + style.preview_status === "ready" && !style.multiframe_previews + ? "notready" + : (style.preview_status ?? "notready"); + const invalidatedStyle: LayerStyle = { + ...style, + preview_status: previewStatus, + multiframe_previews: undefined, + }; + setCurrentLayerStyle(invalidatedStyle); + if (invalidatedStyle.is_default) { + layerStore.selectedLayers = layerStore.selectedLayers.map((layer) => { + if ( + layer.id !== props.layer.id || + layer.copy_id !== props.layer.copy_id + ) { + return layer; + } + return { + ...layer, + preview_status: invalidatedStyle.preview_status, + multiframe_previews: undefined, + }; + }); + } + framePreviewStore.dismissPreviewForLayer(props.layer); +}; + const appliedStyleName = computed(() => { if (currentLayerStyle.value.id) return currentLayerStyle.value.name; else return undefined; @@ -187,6 +223,9 @@ function selectStyle(style: LayerStyle) { setCurrentLayerStyle(style); currentStyleSpec.value = style.style_spec; currentGroups.value = { color: undefined, size: undefined }; + // Remove any preview overlay tied to the previously selected style so the map + // only shows previews that belong to the style now in effect. + framePreviewStore.dismissPreviewForLayer(props.layer); } function fetchRasterBands() { @@ -504,9 +543,15 @@ function save() { name: newName.value || currentLayerStyle.value.name, is_default: currentLayerStyle.value.is_default, style_spec: currentStyleSpec.value, + raster_style_params: showRasterOptions.value + ? styleStore.getRasterTilesQuery( + currentStyleSpec.value, + styleStore.colormaps, + ) + : null, }).then((style) => { if (style) { - setCurrentLayerStyle(style); + markStyleSavedAndInvalidatePreviews(style); newName.value = undefined; newNameMode.value = undefined; // update other styles in case default overriden @@ -527,9 +572,15 @@ function saveAsNew() { layer: props.layer.id, project: projectStore.currentProject.id, style_spec: currentStyleSpec.value, + raster_style_params: showRasterOptions.value + ? styleStore.getRasterTilesQuery( + currentStyleSpec.value, + styleStore.colormaps, + ) + : null, }).then((style: LayerStyle) => { if (style) { - setCurrentLayerStyle(style); + markStyleSavedAndInvalidatePreviews(style); newName.value = undefined; newNameMode.value = undefined; // update other styles in case default overriden @@ -606,6 +657,15 @@ const debouncedStyleSpecUpdated = debounce(() => { watch(currentStyleSpec, debouncedStyleSpecUpdated, { deep: true }); watch(() => props.activeLayer, init); + +watch( + () => props.activeLayer === props.layer, + (isEditing) => { + styleStore.setLayerStyleEditing(props.layer, isEditing); + }, + { immediate: true }, +); + onMounted(resetCurrentStyle); diff --git a/web/src/components/sidebars/LayersPanel.vue b/web/src/components/sidebars/LayersPanel.vue index e7f8362d1..cb66a6d57 100644 --- a/web/src/components/sidebars/LayersPanel.vue +++ b/web/src/components/sidebars/LayersPanel.vue @@ -8,10 +8,11 @@ import CompareLayerStyle from "./CompareLayerStyle.vue"; import DetailView from "../DetailView.vue"; import SliderNumericInput from "../SliderNumericInput.vue"; -import { useLayerStore, useMapStore } from "@/store"; +import { useLayerStore, useMapStore, useFramePreviewStore } from "@/store"; import { useMapCompareStore } from "@/store/compare"; const layerStore = useLayerStore(); const mapStore = useMapStore(); +const framePreviewStore = useFramePreviewStore(); const compareStore = useMapCompareStore(); const isComparing = computed(() => compareStore.isComparing); const orientation = computed(() => compareStore.orientation); @@ -191,6 +192,21 @@ function setLayerActive(layer: Layer, active: boolean) { {{ element.name }} + + diff --git a/web/src/store/analysis.ts b/web/src/store/analysis.ts index 0807ab834..d3b8f0548 100644 --- a/web/src/store/analysis.ts +++ b/web/src/store/analysis.ts @@ -7,6 +7,7 @@ import type { Chart, AnalysisType, TaskResult } from "@/types"; import { defineStore } from "pinia"; import { ref, watch } from "vue"; import { useProjectStore } from "./project"; +import { useFramePreviewStore } from "./framePreview"; export const useAnalysisStore = defineStore("analysis", () => { const projectStore = useProjectStore(); @@ -50,6 +51,10 @@ export const useAnalysisStore = defineStore("analysis", () => { ws.value = new WebSocket(url); ws.value.onmessage = (event: any) => { const data = JSON.parse(JSON.parse(event.data)); + if (data.task_type === "frame_preview" && data.completed) { + // Regenerated previews are ready; reload and reattach them to layers. + useFramePreviewStore().onPreviewTaskComplete(data); + } if (currentResult.value && data.id === currentResult.value.id) { // only overwrite attributes expecting updates // overwriting the whole currentResult object will cause diff --git a/web/src/store/framePreview.ts b/web/src/store/framePreview.ts new file mode 100644 index 000000000..5482eb8af --- /dev/null +++ b/web/src/store/framePreview.ts @@ -0,0 +1,469 @@ +import { defineStore } from "pinia"; +import { ref } from "vue"; +import type { + FramePreview, + Layer, + LayerFrame, + LayerStyle, + TaskResult, +} from "@/types"; +import type { Map as MaplibreMap } from "maplibre-gl"; +import { getLayerStyle } from "@/api/rest"; +import { + fadeRasterOpacities, + hidePreviewLayer, + PREVIEW_FADE_DURATION_MS, + previewLayerId, + removeAllPreviewLayersForLayerKey, + removePreviewLayer, + removePreviewLayersExcept, + upsertPreviewLayer, + waitForRasterSourceLoaded, +} from "@/utils/framePreviewLayer"; +import { prefetchFramePreviewUrls } from "@/utils/framePreviewCache"; +import { useLayerStore, useMapStore, useStyleStore } from "."; + +function layerKey(layer: Layer) { + return `${layer.id}.${layer.copy_id}`; +} + +function orderedRasterFrames(frames: LayerFrame[]) { + return frames + .filter((frame) => frame.raster) + .toSorted((a, b) => a.index - b.index); +} + +// Previews are only safe to display once the backend reports the style as +// "ready" (every frame has a complete image). After a style save the API +// returns "notready" and omits multiframe_previews until regeneration finishes. +function previewsAreReady( + layer: Layer, + style: LayerStyle | undefined, +): boolean { + if (style?.preview_status !== undefined) { + return style.preview_status === "ready"; + } + if ( + style?.id !== undefined && + style.id === layer.default_style?.id && + layer.preview_status !== undefined + ) { + return layer.preview_status === "ready"; + } + // Backward compatibility for payloads that omit preview_status entirely. + return true; +} + +function previewsForLayer(layer: Layer, style: LayerStyle | undefined) { + if (!previewsAreReady(layer, style)) { + return undefined; + } + if (style?.multiframe_previews?.length) { + return style.multiframe_previews; + } + // Layer payloads attach previews for the default style only. Use them when the + // selected style is that default and the style object has no preview list. + if ( + style?.id !== undefined && + style.id === layer.default_style?.id && + layer.multiframe_previews?.length + ) { + return layer.multiframe_previews; + } + return undefined; +} + +function previewAtFrameIndex( + previews: (FramePreview | null)[] | undefined, + rasterFrames: LayerFrame[], + frameIndex: number, +): FramePreview | undefined { + const position = rasterFrames.findIndex( + (frame) => frame.index === frameIndex, + ); + if (position < 0) { + return undefined; + } + return previews?.[position] ?? undefined; +} + +function adjacentRasterFrames( + rasterFrames: LayerFrame[], + currentFrameIndex: number, +) { + const position = rasterFrames.findIndex( + (frame) => frame.index === currentFrameIndex, + ); + if (position < 0) { + return []; + } + return [rasterFrames[position - 1], rasterFrames[position + 1]].filter( + (frame): frame is LayerFrame => frame !== undefined, + ); +} + +export const useFramePreviewStore = defineStore("framePreview", () => { + const activePreviewByLayerKey = new Map(); + const transitionGenerationByLayerKey = new Map(); + + // Reactive set of layer keys whose preview overlay is currently visible on + // the map (i.e. the user is looking at a preview image, not the real tiles). + // Used to drive UI indicators in the layers and legend panels. + const displayingPreviewLayerKeys = ref>(new Set()); + + function markPreviewDisplayed(layerKeyValue: string) { + if (!displayingPreviewLayerKeys.value.has(layerKeyValue)) { + const next = new Set(displayingPreviewLayerKeys.value); + next.add(layerKeyValue); + displayingPreviewLayerKeys.value = next; + } + } + + function clearPreviewDisplayed(layerKeyValue: string) { + if (displayingPreviewLayerKeys.value.has(layerKeyValue)) { + const next = new Set(displayingPreviewLayerKeys.value); + next.delete(layerKeyValue); + displayingPreviewLayerKeys.value = next; + } + } + + function isDisplayingPreview(layer: Layer) { + return displayingPreviewLayerKeys.value.has(layerKey(layer)); + } + + function bumpGeneration(layerKeyValue: string) { + const next = (transitionGenerationByLayerKey.get(layerKeyValue) ?? 0) + 1; + transitionGenerationByLayerKey.set(layerKeyValue, next); + return next; + } + + function prefetchLayerPreviews(layer: Layer, style?: LayerStyle) { + const previews = previewsForLayer(layer, style); + if (!previews?.length) { + return; + } + prefetchFramePreviewUrls(previews.map((preview) => preview?.url)); + } + + async function preloadAdjacentPreviewLayers( + map: MaplibreMap, + layerKeyValue: string, + previews: (FramePreview | null)[] | undefined, + rasterFrames: LayerFrame[], + currentFrameIndex: number, + targetOpacity: number, + ) { + const adjacentFrames = adjacentRasterFrames( + rasterFrames, + currentFrameIndex, + ); + const keepFrameIndices = [ + currentFrameIndex, + ...adjacentFrames.map((frame) => frame.index), + ]; + + removePreviewLayersExcept(map, layerKeyValue, keepFrameIndices); + + const urlsToPrefetch: (string | null | undefined)[] = []; + await Promise.all( + adjacentFrames.map(async (frame) => { + const preview = previewAtFrameIndex( + previews, + rasterFrames, + frame.index, + ); + if (!preview || !frame.raster) { + return; + } + urlsToPrefetch.push(preview.url); + await upsertPreviewLayer( + map, + layerKeyValue, + frame.index, + preview, + frame.raster.metadata, + targetOpacity, + false, + ); + }), + ); + + const currentPreview = previewAtFrameIndex( + previews, + rasterFrames, + currentFrameIndex, + ); + urlsToPrefetch.push(currentPreview?.url); + prefetchFramePreviewUrls(urlsToPrefetch); + } + + function hidePreviousPreview(map: MaplibreMap, layerKeyValue: string) { + const previousFrameIndex = activePreviewByLayerKey.get(layerKeyValue); + if (previousFrameIndex !== undefined) { + hidePreviewLayer(map, layerKeyValue, previousFrameIndex); + } + } + + async function transitionToTiles( + layer: Layer, + frameIndex: number, + layerKeyValue: string, + generation: number, + targetOpacity: number, + tileLayerId: string, + tileSourceId: string, + ) { + const mapStore = useMapStore(); + const map = mapStore.getMap(); + const previewMapLayerId = previewLayerId(layerKeyValue, frameIndex); + + if (transitionGenerationByLayerKey.get(layerKeyValue) !== generation) { + return; + } + + await waitForRasterSourceLoaded(map, tileSourceId); + + if (transitionGenerationByLayerKey.get(layerKeyValue) !== generation) { + return; + } + + if (!map.getLayer(tileLayerId) || !map.getLayer(previewMapLayerId)) { + if (map.getLayer(tileLayerId)) { + map.setPaintProperty(tileLayerId, "raster-opacity", targetOpacity); + } + removePreviewLayer(map, layerKeyValue, frameIndex); + activePreviewByLayerKey.delete(layerKeyValue); + clearPreviewDisplayed(layerKeyValue); + return; + } + + const tileOpacity = + (map.getPaintProperty(tileLayerId, "raster-opacity") as number) ?? 0; + const previewOpacity = + (map.getPaintProperty(previewMapLayerId, "raster-opacity") as number) ?? + targetOpacity; + await fadeRasterOpacities( + map, + [ + { id: previewMapLayerId, from: previewOpacity, to: 0 }, + { id: tileLayerId, from: tileOpacity, to: targetOpacity }, + ], + PREVIEW_FADE_DURATION_MS, + ); + + if (transitionGenerationByLayerKey.get(layerKeyValue) !== generation) { + return; + } + + removePreviewLayer(map, layerKeyValue, frameIndex); + activePreviewByLayerKey.delete(layerKeyValue); + clearPreviewDisplayed(layerKeyValue); + } + + async function showPreviewThenTiles(layer: Layer) { + const layerStore = useLayerStore(); + const mapStore = useMapStore(); + const styleStore = useStyleStore(); + if (styleStore.isLayerStyleEditing(layer)) { + return; + } + const map = mapStore.getMap(); + const layerKeyValue = layerKey(layer); + const style = styleStore.selectedLayerStyles[layerKeyValue]; + const previews = previewsForLayer(layer, style); + + const frames = layerStore.layerFrames(layer); + const rasterFrames = orderedRasterFrames(frames); + if (rasterFrames.length <= 1) { + return; + } + + const currentFrame = rasterFrames.find( + (frame) => frame.index === layer.current_frame_index, + ); + if (!currentFrame?.raster || !layer.visible) { + return; + } + + const preview = previewAtFrameIndex( + previews, + rasterFrames, + layer.current_frame_index, + ); + + const tileSourceId = mapStore.sourceIdFromLayerFrame(layer, currentFrame); + const tileLayerId = `${tileSourceId}.raster`; + const targetOpacity = style?.style_spec?.opacity ?? 1; + + const generation = bumpGeneration(layerKeyValue); + hidePreviousPreview(map, layerKeyValue); + + if (!preview) { + clearPreviewDisplayed(layerKeyValue); + if (map.getLayer(tileLayerId)) { + map.setPaintProperty(tileLayerId, "raster-opacity", targetOpacity); + } + void preloadAdjacentPreviewLayers( + map, + layerKeyValue, + previews, + rasterFrames, + layer.current_frame_index, + targetOpacity, + ); + return; + } + + const previewMapLayerId = await upsertPreviewLayer( + map, + layerKeyValue, + layer.current_frame_index, + preview, + currentFrame.raster.metadata, + targetOpacity, + ); + if (!previewMapLayerId) { + clearPreviewDisplayed(layerKeyValue); + if (map.getLayer(tileLayerId)) { + map.setPaintProperty(tileLayerId, "raster-opacity", targetOpacity); + } + return; + } + + activePreviewByLayerKey.set(layerKeyValue, layer.current_frame_index); + markPreviewDisplayed(layerKeyValue); + + if (map.getLayer(tileLayerId)) { + map.setPaintProperty(tileLayerId, "raster-opacity", 0); + } + + void preloadAdjacentPreviewLayers( + map, + layerKeyValue, + previews, + rasterFrames, + layer.current_frame_index, + targetOpacity, + ); + + void transitionToTiles( + layer, + layer.current_frame_index, + layerKeyValue, + generation, + targetOpacity, + tileLayerId, + tileSourceId, + ); + } + + function dismissPreviewForLayer(layer: Layer) { + const mapStore = useMapStore(); + const layerStore = useLayerStore(); + const styleStore = useStyleStore(); + const map = mapStore.getMap(); + const layerKeyValue = layerKey(layer); + + bumpGeneration(layerKeyValue); + removeAllPreviewLayersForLayerKey(map, layerKeyValue); + activePreviewByLayerKey.delete(layerKeyValue); + clearPreviewDisplayed(layerKeyValue); + + const frames = layerStore.layerFrames(layer); + const currentFrame = frames.find( + (frame) => frame.index === layer.current_frame_index, + ); + if (!currentFrame?.raster) { + return; + } + + const tileLayerId = `${mapStore.sourceIdFromLayerFrame(layer, currentFrame)}.raster`; + const style = styleStore.selectedLayerStyles[layerKeyValue]; + const targetOpacity = style?.style_spec?.opacity ?? 1; + if (map.getLayer(tileLayerId)) { + map.setPaintProperty(tileLayerId, "raster-opacity", targetOpacity); + } + } + + function cleanupLayer(layer: Layer) { + const mapStore = useMapStore(); + const layerKeyValue = layerKey(layer); + transitionGenerationByLayerKey.delete(layerKeyValue); + activePreviewByLayerKey.delete(layerKeyValue); + clearPreviewDisplayed(layerKeyValue); + removeAllPreviewLayersForLayerKey(mapStore.getMap(), layerKeyValue); + } + + function clearAll() { + transitionGenerationByLayerKey.clear(); + activePreviewByLayerKey.clear(); + displayingPreviewLayerKeys.value = new Set(); + } + + // Called when a "frame_preview" TaskResult completes over the analytics + // WebSocket. Reloads the freshly generated previews and reattaches them to + // every selected layer copy that still has the regenerated style applied. + async function onPreviewTaskComplete(task: TaskResult) { + const layerStyleId = task.inputs?.layer_style_id; + const layerId = task.inputs?.layer_id; + if (layerStyleId === undefined || layerId === undefined) { + return; + } + + const layerStore = useLayerStore(); + const styleStore = useStyleStore(); + + let updatedStyle: LayerStyle; + try { + updatedStyle = await getLayerStyle(layerStyleId); + } catch { + // If the style was deleted or the fetch fails, there is nothing to attach. + return; + } + + // Keep availableLayers current so re-adding this layer picks up the new + // default-style previews. Fire-and-forget; selected copies are updated below. + void layerStore.fetchAvailableLayer(layerId).catch(() => undefined); + + layerStore.selectedLayers.forEach((layer) => { + if (layer.id !== layerId) { + return; + } + const key = layerKey(layer); + const selectedStyle = styleStore.selectedLayerStyles[key]; + // Only reattach when this copy still has the regenerated style selected; + // the user may have swapped to a different style while generation ran. + if (!selectedStyle || selectedStyle.id !== layerStyleId) { + return; + } + + styleStore.selectedLayerStyles[key] = { + ...selectedStyle, + preview_status: updatedStyle.preview_status, + multiframe_previews: updatedStyle.multiframe_previews, + }; + + // Mirror onto the layer object so the default-style fallback stays valid. + if (updatedStyle.is_default) { + layer.multiframe_previews = updatedStyle.multiframe_previews; + layer.preview_status = updatedStyle.preview_status; + } + + prefetchLayerPreviews(layer, styleStore.selectedLayerStyles[key]); + if (!styleStore.isLayerStyleEditing(layer)) { + void showPreviewThenTiles(layer); + } + }); + } + + return { + displayingPreviewLayerKeys, + isDisplayingPreview, + prefetchLayerPreviews, + showPreviewThenTiles, + dismissPreviewForLayer, + onPreviewTaskComplete, + cleanupLayer, + clearAll, + }; +}); diff --git a/web/src/store/index.ts b/web/src/store/index.ts index f78d38d5c..ea720cd28 100644 --- a/web/src/store/index.ts +++ b/web/src/store/index.ts @@ -7,6 +7,7 @@ import { useProjectStore } from "./project"; import { useStyleStore } from "./style"; import { usePanelStore } from "./panel"; import { useConversionStore } from "./conversion"; +import { useFramePreviewStore } from "./framePreview"; export { useAppStore, @@ -18,4 +19,5 @@ export { useStyleStore, usePanelStore, useConversionStore, + useFramePreviewStore, }; diff --git a/web/src/store/layer.ts b/web/src/store/layer.ts index c628e6768..f558f0e39 100644 --- a/web/src/store/layer.ts +++ b/web/src/store/layer.ts @@ -22,6 +22,7 @@ import { useStyleStore, useNetworkStore, useProjectStore, + useFramePreviewStore, } from "."; interface SourceDBObjects { @@ -43,6 +44,7 @@ export const useLayerStore = defineStore("layer", () => { const networkStore = useNetworkStore(); const styleStore = useStyleStore(); const projectStore = useProjectStore(); + const framePreviewStore = useFramePreviewStore(); /** * Return the maplibre layers associated with a Layer DB object @@ -197,6 +199,10 @@ export const useLayerStore = defineStore("layer", () => { } selectedLayers.value = [newLayer, ...selectedLayers.value]; + framePreviewStore.prefetchLayerPreviews( + newLayer, + newLayer.default_style ?? undefined, + ); } function setLayerVisibility(layers: Layer[], visible = true) { @@ -227,6 +233,10 @@ export const useLayerStore = defineStore("layer", () => { ) { styleStore.selectedLayerStyles[styleId] = { ...layer.default_style, + preview_status: layer.preview_status, + ...(layer.preview_status === "ready" && layer.multiframe_previews + ? { multiframe_previews: layer.multiframe_previews } + : {}), }; if ( styleStore.selectedLayerStyles[styleId]?.style_spec diff --git a/web/src/store/map.ts b/web/src/store/map.ts index d22204ace..21cf32926 100644 --- a/web/src/store/map.ts +++ b/web/src/store/map.ts @@ -22,7 +22,17 @@ import { Map, Popup } from "maplibre-gl"; import { getBasemaps, getRasterDataValues } from "@/api/rest"; import { baseURL } from "@/api/auth"; import proj4 from "proj4"; -import { useStyleStore, useLayerStore, useAppStore, useProjectStore } from "."; +import { + useStyleStore, + useLayerStore, + useAppStore, + useProjectStore, + useFramePreviewStore, +} from "."; +import { + isPreviewMapLayerId, + removeAllPreviewLayersForLayerKey, +} from "@/utils/framePreviewLayer"; function getLayerIsVisible(layer: MapLibreLayerWithMetadata) { // Since visibility must be 'visible' for a feature click to even be registered, @@ -324,12 +334,21 @@ export const useMapStore = defineStore("map", () => { function removeLayers(layerIds: string[]) { const map = getMap(); + const framePreviewStore = useFramePreviewStore(); + const cleanedLayerKeys = new Set(); // Must collect all source Ids so they can be removed after all layers // have been removed, since multple layers may use the same source const sourceIdsToRemove = new Set(); const updatedLayerIds: string[] = []; layerIds.forEach((id) => { + if (isPreviewMapLayerId(id)) { + return; + } + const layerKey = id.split(".").slice(0, 2).join("."); + if (layerKey.includes(".")) { + cleanedLayerKeys.add(layerKey); + } // Rasters have implicit bounds layers that also need to be removed if (id.includes(".raster.")) { updatedLayerIds.push(id.replace(".raster.", ".bounds.")); @@ -348,6 +367,17 @@ export const useMapStore = defineStore("map", () => { sourceIdsToRemove.forEach((id) => { map.removeSource(id); }); + + cleanedLayerKeys.forEach((layerKey) => { + removeAllPreviewLayersForLayerKey(map, layerKey); + const [layerId, copyId] = layerKey.split(".").map(Number); + const layer = layerStore.selectedLayers.find( + (candidate) => candidate.id === layerId && candidate.copy_id === copyId, + ); + if (layer) { + framePreviewStore.cleanupLayer(layer); + } + }); } /** @@ -524,9 +554,6 @@ export const useMapStore = defineStore("map", () => { ): Source | undefined { const map = getMap(); - const queryParams: { projection: string; style?: string } = { - projection: "epsg:3857", - }; const { layerId, layerCopyId } = parseSourceString(sourceId); const styleSpec = styleStore.selectedLayerStyles[`${layerId}.${layerCopyId}`].style_spec; @@ -540,22 +567,14 @@ export const useMapStore = defineStore("map", () => { (f: LayerFrame) => f.index === layer.current_frame_index, ); if (frame?.source_filters) { - filters = Object.entries(frame.source_filters).map(([k, v]) => ({ - filter_by: k, - list: [v], - include: true, - transparency: true, - apply: true, - })); + filters = styleStore.sourceFiltersToStyleFilters(frame.source_filters); } } - if (styleSpec) { - const styleParams = styleStore.getRasterTilesQuery( - { ...styleSpec, filters }, - styleStore.colormaps, - ); - if (styleParams) queryParams.style = JSON.stringify(styleParams); - } + const queryParams = styleStore.buildRasterTileQueryParams( + styleSpec ?? styleStore.getDefaultStyleSpec(raster, layerId), + filters, + styleStore.colormaps, + ); const query = new URLSearchParams(queryParams); rasterSourceTileURLs.value[sourceId] = `${baseURL}rasters/${raster.id}/tiles/{z}/{x}/{y}.png/?${query}`; diff --git a/web/src/store/project.ts b/web/src/store/project.ts index d68c13df8..c835c3088 100644 --- a/web/src/store/project.ts +++ b/web/src/store/project.ts @@ -20,7 +20,9 @@ import { usePanelStore, useAppStore, useStyleStore, + useFramePreviewStore, } from "."; +import { clearFramePreviewCache } from "@/utils/framePreviewCache"; export const useProjectStore = defineStore("project", () => { const networkStore = useNetworkStore(); @@ -278,6 +280,9 @@ export const useProjectStore = defineStore("project", () => { layerStore.selectedLayers = []; styleStore.selectedLayerStyles = {}; + styleStore.clearStyleEditing(); + useFramePreviewStore().clearAll(); + clearFramePreviewCache(); mapStore.clickedFeature = undefined; diff --git a/web/src/store/style.ts b/web/src/store/style.ts index e2e54fc38..b3396d934 100644 --- a/web/src/store/style.ts +++ b/web/src/store/style.ts @@ -22,6 +22,7 @@ import { useLayerStore, useProjectStore, useNetworkStore, + useFramePreviewStore, } from "."; export interface MapLayerStyleRaw { @@ -89,6 +90,51 @@ export function colormapMarkersSubsample( return markers; } +// frame/band select which slice to read; they must be flat query params, not in style JSON +const RASTER_SOURCE_FILTER_KEYS = new Set(["frame", "band"]); + +// Ingest defaults missing source_filters to { band: 1 }; that is not a real band selection. +function isDefaultBandSourceFilter( + sourceFilters: Record, +): boolean { + const keys = Object.keys(sourceFilters); + return ( + keys.length === 1 && + keys[0] === "band" && + (sourceFilters.band === 1 || sourceFilters.band === "1") + ); +} + +function sourceFiltersToStyleFilters( + sourceFilters: Record | undefined, +): StyleFilter[] { + if (!sourceFilters || !Object.keys(sourceFilters).length) return []; + if (isDefaultBandSourceFilter(sourceFilters)) return []; + return Object.entries(sourceFilters).map(([k, v]) => ({ + filter_by: k, + list: [v], + include: true, + transparency: true, + apply: true, + })); +} + +function getRasterSourceFilterParams(filters: StyleFilter[]) { + const params: Record = {}; + filters.forEach((f) => { + if ( + f.apply && + f.filter_by && + f.include && + f.list?.length === 1 && + RASTER_SOURCE_FILTER_KEYS.has(f.filter_by) + ) { + params[f.filter_by] = f.list[0]; + } + }); + return params; +} + function getRasterTilesQuery(styleSpec: StyleSpec, colormaps: Colormap[]) { let query: Record = {}; const colorSpecs = styleSpec.colors || []; @@ -126,13 +172,37 @@ function getRasterTilesQuery(styleSpec: StyleSpec, colormaps: Colormap[]) { } }); styleSpec.filters.forEach((f) => { - if (f.apply && f.filter_by && f.include && f.list?.length === 1) { + if ( + f.apply && + f.filter_by && + f.include && + f.list?.length === 1 && + !RASTER_SOURCE_FILTER_KEYS.has(f.filter_by) + ) { query[f.filter_by] = f.list[0]; } }); return query; } +function buildRasterTileQueryParams( + styleSpec: StyleSpec, + filters: StyleFilter[], + colormaps: Colormap[], +) { + const params: Record = { projection: "epsg:3857" }; + Object.entries(getRasterSourceFilterParams(filters)).forEach( + ([key, value]) => { + params[key] = String(value); + }, + ); + const styleQuery = getRasterTilesQuery({ ...styleSpec, filters }, colormaps); + if (Object.keys(styleQuery).length) { + params.style = JSON.stringify(styleQuery); + } + return params; +} + function getVectorColorPaintProperty( styleSpec: StyleSpec, groupName: string, @@ -343,11 +413,39 @@ function getVectorVisibilityPaintProperty( export const useStyleStore = defineStore("style", () => { const selectedLayerStyles = ref>({}); const colormaps = ref([]); + const editingStyleLayerKeys = ref>(new Set()); const mapStore = useMapStore(); const projectStore = useProjectStore(); const layerStore = useLayerStore(); const networkStore = useNetworkStore(); + const framePreviewStore = useFramePreviewStore(); + + function layerStyleKey(layer: Layer) { + return `${layer.id}.${layer.copy_id}`; + } + + function isLayerStyleEditing(layer: Layer) { + return editingStyleLayerKeys.value.has(layerStyleKey(layer)); + } + + function setLayerStyleEditing(layer: Layer, editing: boolean) { + const key = layerStyleKey(layer); + const next = new Set(editingStyleLayerKeys.value); + if (editing) { + next.add(key); + editingStyleLayerKeys.value = next; + framePreviewStore.dismissPreviewForLayer(layer); + return; + } + next.delete(key); + editingStyleLayerKeys.value = next; + updateLayerStyles(layer); + } + + function clearStyleEditing() { + editingStyleLayerKeys.value = new Set(); + } function getDefaultColor(layerId: number) { const color = chroma.hsl( @@ -423,6 +521,12 @@ export const useStyleStore = defineStore("style", () => { } }); networkStore.styleVisibleNetworks(); + + const hasMultiframeRaster = + frames.length > 1 && frames.some((f) => f.raster); + if (hasMultiframeRaster && !isLayerStyleEditing(layer)) { + void framePreviewStore.showPreviewThenTiles(layer); + } } type GeneratedLayerStyle = { @@ -441,13 +545,7 @@ export const useStyleStore = defineStore("style", () => { if (frame?.source_filters) { filters = [ ...filters, - ...Object.entries(frame.source_filters).map(([k, v]) => ({ - filter_by: k, - list: [v], - include: true, - transparency: true, - apply: true, - })), + ...sourceFiltersToStyleFilters(frame.source_filters), ]; } const mapLayer = map.getLayer(mapLayerId) as @@ -550,11 +648,13 @@ export const useStyleStore = defineStore("style", () => { const source = map.getSource(mapLayer.source) as RasterTileSource; const sourceURL = mapStore.rasterSourceTileURLs[mapLayer.source]; if (source && sourceURL) { - const newQueryParams: { projection: string; style?: string } = { - projection: "epsg:3857", - }; - newQueryParams.style = JSON.stringify(rasterTilesQuery); - const newQuery = new URLSearchParams(newQueryParams); + const newQuery = new URLSearchParams( + buildRasterTileQueryParams( + { ...styleSpec, filters }, + filters, + colormaps.value, + ), + ); tileURL = sourceURL.split("?")[0] + "?" + newQuery.toString(); return { paint, tileURL }; } @@ -603,12 +703,19 @@ export const useStyleStore = defineStore("style", () => { return { colormaps, selectedLayerStyles, + editingStyleLayerKeys, fetchColormaps, getRasterTilesQuery, + getRasterSourceFilterParams, + buildRasterTileQueryParams, + sourceFiltersToStyleFilters, colormapMarkersSubsample, getDefaultColor, getDefaultStyleSpec, getVectorColorPaintProperty, + isLayerStyleEditing, + setLayerStyleEditing, + clearStyleEditing, updateLayerStyles, setMapLayerStyle, returnMapLayerStyle, diff --git a/web/src/types.ts b/web/src/types.ts index ec72e09b0..4cb3d117e 100644 --- a/web/src/types.ts +++ b/web/src/types.ts @@ -28,6 +28,35 @@ export interface Dataset { n_layers?: number; } +export interface FramePreviewCorner { + x: number; + y: number; +} + +export interface FramePreviewBounds { + srs: string; + xmin: number; + xmax: number; + ymin: number; + ymax: number; + ul?: FramePreviewCorner; + ur?: FramePreviewCorner; + lr?: FramePreviewCorner; + ll?: FramePreviewCorner; +} + +export interface FramePreview { + url: string; + width: number; + height: number; + bounds: FramePreviewBounds; +} + +// Style-level preview lifecycle, derived by the API from per-frame rows. +// "ready" means every raster frame has a complete preview image; anything +// else (missing, generating, regenerating, failed) is reported as "notready". +export type PreviewStatus = "ready" | "notready"; + export interface Layer { id: number; copy_id: number; @@ -38,6 +67,8 @@ export interface Layer { visible: boolean; current_frame_index: number; default_style: LayerStyle | null; + multiframe_previews?: (FramePreview | null)[]; + preview_status?: PreviewStatus; } export interface LayerFrame { @@ -128,6 +159,10 @@ export interface LayerStyle { project?: number; is_default: boolean; style_spec?: StyleSpec; + /** Client-computed django-large-image style JSON, persisted on save for rasters. */ + raster_style_params?: Record | null; + multiframe_previews?: (FramePreview | null)[]; + preview_status?: PreviewStatus; } export interface VectorData { diff --git a/web/src/utils/framePreviewCache.ts b/web/src/utils/framePreviewCache.ts new file mode 100644 index 000000000..5598c8cd3 --- /dev/null +++ b/web/src/utils/framePreviewCache.ts @@ -0,0 +1,21 @@ +const prefetchedUrls = new Set(); + +export async function getCachedPreviewObjectUrl( + url: string, +): Promise { + return url; +} + +export function prefetchFramePreviewUrls(urls: (string | null | undefined)[]) { + urls.forEach((url) => { + if (!url || prefetchedUrls.has(url)) { + return; + } + prefetchedUrls.add(url); + void fetch(url); + }); +} + +export function clearFramePreviewCache() { + prefetchedUrls.clear(); +} diff --git a/web/src/utils/framePreviewLayer.ts b/web/src/utils/framePreviewLayer.ts new file mode 100644 index 000000000..386af43cd --- /dev/null +++ b/web/src/utils/framePreviewLayer.ts @@ -0,0 +1,299 @@ +import type { + FramePreview, + FramePreviewBounds, + FramePreviewCorner, + RasterMetadata, +} from "@/types"; +import type { Map, MapSourceDataEvent } from "maplibre-gl"; +import proj4 from "proj4"; +import { getCachedPreviewObjectUrl } from "./framePreviewCache"; + +export const PREVIEW_FADE_DURATION_MS = 400; + +const CORNER_KEYS = ["ul", "ur", "lr", "ll"] as const; +type CornerKey = (typeof CORNER_KEYS)[number]; + +function clampRasterOpacity(value: number): number { + return Math.min(1, Math.max(0, value)); +} + +export function previewSourceId(layerKey: string, frameIndex: number) { + return `${layerKey}.preview.${frameIndex}`; +} + +export function previewLayerId(layerKey: string, frameIndex: number) { + return `${previewSourceId(layerKey, frameIndex)}.raster`; +} + +export function isPreviewMapLayerId(mapLayerId: string) { + return mapLayerId.includes(".preview."); +} + +function toLngLat(srs: string, x: number, y: number): [number, number] { + if (srs && srs !== "EPSG:4326") { + return proj4(srs, "EPSG:4326", [x, y]) as [number, number]; + } + return [x, y]; +} + +function hasCornerBounds( + bounds: FramePreviewBounds, +): bounds is FramePreviewBounds & Record { + return CORNER_KEYS.every((corner) => bounds[corner] !== undefined); +} + +function cornersFromRasterMetadata( + rasterBounds: RasterMetadata["bounds"], +): FramePreviewBounds | undefined { + if (!CORNER_KEYS.every((corner) => rasterBounds[corner])) { + return undefined; + } + + const corners = Object.fromEntries( + CORNER_KEYS.map((corner) => { + const point = rasterBounds[corner]!; + const [x, y] = toLngLat(rasterBounds.srs, point.x, point.y); + return [corner, { x, y }]; + }), + ) as Record; + + const lngs = CORNER_KEYS.map((corner) => corners[corner].x); + const lats = CORNER_KEYS.map((corner) => corners[corner].y); + + return { + srs: "EPSG:4326", + xmin: Math.min(...lngs), + xmax: Math.max(...lngs), + ymin: Math.min(...lats), + ymax: Math.max(...lats), + ...corners, + }; +} + +export function resolvePreviewBounds( + preview: FramePreview, + raster?: RasterMetadata, +): FramePreviewBounds { + if (hasCornerBounds(preview.bounds)) { + return preview.bounds; + } + + if (raster?.bounds) { + const rasterCorners = cornersFromRasterMetadata(raster.bounds); + if (rasterCorners) { + return rasterCorners; + } + } + + return preview.bounds; +} + +function boundsToCoordinates( + bounds: FramePreviewBounds, +): [[number, number], [number, number], [number, number], [number, number]] { + if (hasCornerBounds(bounds)) { + return [ + toLngLat(bounds.srs, bounds.ul.x, bounds.ul.y), + toLngLat(bounds.srs, bounds.ur.x, bounds.ur.y), + toLngLat(bounds.srs, bounds.lr.x, bounds.lr.y), + toLngLat(bounds.srs, bounds.ll.x, bounds.ll.y), + ]; + } + + const { xmin, xmax, ymin, ymax, srs } = bounds; + return [ + toLngLat(srs, xmin, ymax), + toLngLat(srs, xmax, ymax), + toLngLat(srs, xmax, ymin), + toLngLat(srs, xmin, ymin), + ]; +} + +export async function upsertPreviewLayer( + map: Map, + layerKey: string, + frameIndex: number, + preview: FramePreview, + raster?: RasterMetadata, + opacity = 1, + visible = true, +) { + const sourceId = previewSourceId(layerKey, frameIndex); + const mapLayerId = previewLayerId(layerKey, frameIndex); + const objectUrl = await getCachedPreviewObjectUrl(preview.url); + if (!objectUrl) { + return undefined; + } + + const coordinates = boundsToCoordinates( + resolvePreviewBounds(preview, raster), + ); + const existingSource = map.getSource(sourceId); + if (existingSource) { + if (map.getLayer(mapLayerId)) { + map.removeLayer(mapLayerId); + } + map.removeSource(sourceId); + } + map.addSource(sourceId, { + type: "image", + url: objectUrl, + coordinates, + }); + + const previewOpacity = clampRasterOpacity(opacity); + const visibility = visible ? "visible" : "none"; + + if (!map.getLayer(mapLayerId)) { + map.addLayer({ + id: mapLayerId, + type: "raster", + source: sourceId, + layout: { + visibility, + }, + paint: { + "raster-opacity": previewOpacity, + "raster-fade-duration": 0, + }, + }); + } else { + map.setPaintProperty(mapLayerId, "raster-opacity", previewOpacity); + map.setLayoutProperty(mapLayerId, "visibility", visibility); + } + + return mapLayerId; +} + +export function hidePreviewLayer( + map: Map, + layerKey: string, + frameIndex: number, +) { + const mapLayerId = previewLayerId(layerKey, frameIndex); + if (map.getLayer(mapLayerId)) { + map.setLayoutProperty(mapLayerId, "visibility", "none"); + } +} + +export function removePreviewLayer( + map: Map, + layerKey: string, + frameIndex: number, +) { + const sourceId = previewSourceId(layerKey, frameIndex); + const mapLayerId = previewLayerId(layerKey, frameIndex); + if (map.getLayer(mapLayerId)) { + map.removeLayer(mapLayerId); + } + if (map.getSource(sourceId)) { + map.removeSource(sourceId); + } +} + +function previewFrameIndexFromLayerId( + layerKey: string, + mapLayerId: string, +): number | undefined { + const prefix = `${layerKey}.preview.`; + if (!mapLayerId.startsWith(prefix)) { + return undefined; + } + const rest = mapLayerId.slice(prefix.length); + const frameIndex = Number.parseInt(rest.split(".")[0], 10); + return Number.isNaN(frameIndex) ? undefined : frameIndex; +} + +export function removePreviewLayersExcept( + map: Map, + layerKey: string, + keepFrameIndices: number[], +) { + const keep = new Set(keepFrameIndices); + const frameIndicesToRemove = new Set(); + + map.getStyle().layers?.forEach((layer) => { + const frameIndex = previewFrameIndexFromLayerId(layerKey, layer.id); + if (frameIndex !== undefined && !keep.has(frameIndex)) { + frameIndicesToRemove.add(frameIndex); + } + }); + + frameIndicesToRemove.forEach((frameIndex) => { + removePreviewLayer(map, layerKey, frameIndex); + }); +} + +export function removeAllPreviewLayersForLayerKey(map: Map, layerKey: string) { + map.getStyle().layers?.forEach((layer) => { + if (layer.id.startsWith(`${layerKey}.preview.`)) { + map.removeLayer(layer.id); + } + }); + Object.keys(map.getStyle().sources ?? {}).forEach((sourceId) => { + if (sourceId.startsWith(`${layerKey}.preview.`)) { + map.removeSource(sourceId); + } + }); +} + +export function waitForRasterSourceLoaded( + map: Map, + sourceId: string, + timeoutMs = 10000, +): Promise { + return new Promise((resolve) => { + if (!map.getSource(sourceId)) { + resolve(); + return; + } + if (map.isSourceLoaded(sourceId)) { + resolve(); + return; + } + + const timeout = window.setTimeout(() => { + map.off("sourcedata", onSourceData); + resolve(); + }, timeoutMs); + + function onSourceData(event: MapSourceDataEvent) { + if (event.sourceId === sourceId && event.isSourceLoaded) { + window.clearTimeout(timeout); + map.off("sourcedata", onSourceData); + resolve(); + } + } + + map.on("sourcedata", onSourceData); + }); +} + +export async function fadeRasterOpacities( + map: Map, + layers: { id: string; from: number; to: number }[], + durationMs: number, +) { + const start = performance.now(); + await new Promise((resolve) => { + function step(now: number) { + const progress = Math.min(1, (now - start) / durationMs); + layers.forEach(({ id, from, to }) => { + if (map.getLayer(id)) { + const opacity = progress >= 1 ? to : from + (to - from) * progress; + map.setPaintProperty( + id, + "raster-opacity", + clampRasterOpacity(opacity), + ); + } + }); + if (progress < 1) { + requestAnimationFrame(step); + } else { + resolve(); + } + } + requestAnimationFrame(step); + }); +}