diff --git a/sample_data/boston_floods/ingest_mbta.py b/sample_data/boston_floods/ingest_mbta.py index 661345a48..d9212aa47 100644 --- a/sample_data/boston_floods/ingest_mbta.py +++ b/sample_data/boston_floods/ingest_mbta.py @@ -6,6 +6,7 @@ import requests from uvdat.core.models import NetworkNode, VectorData, VectorFeature +from uvdat.core.tasks.run_mode import TaskRunMode LINE_COLORS = { "RED": "#D31414", @@ -30,7 +31,7 @@ def convert_dataset(dataset, options): layer_options=options.get("layers"), network_options=options.get("network_options"), region_options=options.get("region_options"), - asynchronous=False, + run_mode=TaskRunMode.SYNC, ) # Post-processing diff --git a/sample_data/new_york_energy/ingest.py b/sample_data/new_york_energy/ingest.py index 99e8203f2..b09c6e3f5 100644 --- a/sample_data/new_york_energy/ingest.py +++ b/sample_data/new_york_energy/ingest.py @@ -3,6 +3,7 @@ from pathlib import Path from uvdat.core.tasks.dataset import create_layers_and_frames +from uvdat.core.tasks.run_mode import TaskRunMode from .export_networks import perform_export from .import_networks import perform_import @@ -34,5 +35,5 @@ def convert_dataset(dataset, options): layer_options=options.get("layers"), network_options=options.get("network_options"), region_options=options.get("region_options"), - asynchronous=False, + run_mode=TaskRunMode.SYNC, ) diff --git a/uvdat/core/admin.py b/uvdat/core/admin.py index 4584d4dab..ddcf59554 100644 --- a/uvdat/core/admin.py +++ b/uvdat/core/admin.py @@ -19,6 +19,7 @@ NetworkNode, Project, RasterData, + RasterFramePreview, Region, SizeConfig, SizeRangeConfig, @@ -69,6 +70,20 @@ class LayerStyleAdmin(admin.ModelAdmin): list_display = ["id", "name", "layer"] +@admin.register(RasterFramePreview) +class RasterFramePreviewAdmin(admin.ModelAdmin): + list_display = [ + "id", + "layer_frame", + "style_fingerprint", + "raster_style_params", + "width", + "height", + "status", + ] + list_select_related = ["layer_frame"] + + @admin.register(Colormap) class ColormapAdmin(admin.ModelAdmin): list_display = ["id", "name"] diff --git a/uvdat/core/frame_previews/__init__.py b/uvdat/core/frame_previews/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/uvdat/core/frame_previews/fingerprint.py b/uvdat/core/frame_previews/fingerprint.py new file mode 100644 index 000000000..c163dc29e --- /dev/null +++ b/uvdat/core/frame_previews/fingerprint.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +import hashlib +import json +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from uvdat.core.models import LayerStyle + + +def _fingerprint_payload(params: dict[str, Any] | None) -> str: + return json.dumps(params or {}, sort_keys=True, default=str) + + +def params_fingerprint(params: dict[str, Any] | None) -> str: + """Sha256 of ``raster_style_params`` JSON used to render preview PNGs.""" + payload = _fingerprint_payload(params) + return hashlib.sha256(payload.encode()).hexdigest() + + +def style_fingerprint(layer_style: LayerStyle) -> str: + """Sha256 of a style's ``raster_style_params``.""" + return params_fingerprint(layer_style.raster_style_params) diff --git a/uvdat/core/frame_previews/lookup.py b/uvdat/core/frame_previews/lookup.py new file mode 100644 index 000000000..43e95c649 --- /dev/null +++ b/uvdat/core/frame_previews/lookup.py @@ -0,0 +1,93 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from uvdat.core.frame_previews.fingerprint import params_fingerprint, style_fingerprint +from uvdat.core.frame_previews.types import FramePreviewData +from uvdat.core.models.frame_preview import PreviewStatus, RasterFramePreview + +if TYPE_CHECKING: + from uvdat.core.models import Layer, LayerFrame + + +def serialize_frame_preview(preview: RasterFramePreview) -> FramePreviewData: + return FramePreviewData( + url=preview.image.url, + width=preview.width, + height=preview.height, + bounds=preview.bounds, + ) + + +def previews_by_frame_id( + frames: list[LayerFrame], + fingerprint: str, +) -> dict[int, RasterFramePreview]: + """Return ``{frame_id: RasterFramePreview}`` for ``fingerprint``.""" + if not frames: + return {} + return { + preview.layer_frame_id: preview + for preview in RasterFramePreview.objects.filter( + layer_frame_id__in=[frame.id for frame in frames], + style_fingerprint=fingerprint, + ) + } + + +def ordered_complete_previews( + layer: Layer, + fingerprint: str, +) -> list[FramePreviewData] | None: + """Return serialized previews in frame order, or None if any frame is incomplete.""" + if not layer.is_multiframe_raster(): + return None + + frames = layer.raster_frames() + by_frame = previews_by_frame_id(frames, fingerprint) + ordered = [by_frame.get(frame.id) for frame in frames] + if not all( + preview is not None and preview.status == PreviewStatus.COMPLETE and preview.image + for preview in ordered + ): + return None + return [serialize_frame_preview(preview) for preview in ordered] + + +def preview_status_for_fingerprint( + layer: Layer, + fingerprint: str, +) -> str | None: + if not layer.is_multiframe_raster(): + return None + + frames = layer.raster_frames() + by_frame = previews_by_frame_id(frames, fingerprint) + if all( + (preview := by_frame.get(frame.id)) + and preview.status == PreviewStatus.COMPLETE + and preview.image + for frame in frames + ): + return "ready" + return "notready" + + +def previews_current_for_fingerprint(layer: Layer, fingerprint: str) -> bool: + """Return whether every frame already has a complete preview for this fingerprint.""" + return preview_status_for_fingerprint(layer, fingerprint) == "ready" + + +def layer_default_fingerprint(layer: Layer) -> str: + """Fingerprint for layer-level default previews (default style params, else ``{}``).""" + if layer.default_style_id is not None: + return style_fingerprint(layer.default_style) + return params_fingerprint({}) + + +def layer_default_multiframe_previews(layer: Layer) -> list[FramePreviewData] | None: + """Previews for the layer default fingerprint (default style params or ``{}``).""" + return ordered_complete_previews( + layer, + layer_default_fingerprint(layer), + ) diff --git a/uvdat/core/frame_previews/preview_regeneration.py b/uvdat/core/frame_previews/preview_regeneration.py new file mode 100644 index 000000000..687f52a67 --- /dev/null +++ b/uvdat/core/frame_previews/preview_regeneration.py @@ -0,0 +1,225 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from django.db import transaction + +from uvdat.core.frame_previews.fingerprint import params_fingerprint, style_fingerprint +from uvdat.core.frame_previews.lookup import ( + layer_default_fingerprint, + preview_status_for_fingerprint, + previews_current_for_fingerprint, +) +from uvdat.core.models import Layer, LayerStyle, RasterFramePreview, TaskResult +from uvdat.core.models.frame_preview import PreviewStatus +from uvdat.core.models.task_result import suppress_task_notifications + +if TYPE_CHECKING: + from uvdat.core.tasks.run_mode import TaskRunMode + + +def _coerce_run_mode(run_mode: TaskRunMode | str) -> TaskRunMode: + # Lazy: this module is imported by tasks.dataset during package init. + from uvdat.core.tasks.run_mode import TaskRunMode as _TaskRunMode # noqa: PLC0415 + + return _TaskRunMode(run_mode) + + +def pending_preview_task(*, layer_id: int, fingerprint: str) -> TaskResult | None: + """Return an in-flight preview task for this layer fingerprint, if any.""" + return ( + TaskResult.objects.filter( + task_type="frame_preview", + completed__isnull=True, + inputs__layer_id=layer_id, + inputs__fingerprint=fingerprint, + ) + .order_by("id") + .first() + ) + + +def clear_layer_preview_instance_cache(layer: Layer) -> None: + layer.__dict__.pop("preview_status", None) + layer.__dict__.pop("_raster_frame_count", None) + layer.__dict__.pop("_complete_with_image_count", None) + layer.__dict__.pop("_raster_frames", None) + if cache := getattr(layer, "_prefetched_objects_cache", None): + cache.pop("frames", None) + + +def clear_style_preview_instance_cache(layer_style: LayerStyle) -> None: + """Drop queryset annotations and prefetches that go stale after invalidation.""" + layer_style.refresh_from_db() + layer_style.__dict__.pop("preview_status", None) + layer_style.__dict__.pop("_raster_frame_count", None) + layer_style.__dict__.pop("_complete_with_image_count", None) + if cache := getattr(layer_style, "_prefetched_objects_cache", None): + cache.pop("layer", None) + clear_layer_preview_instance_cache(layer_style.layer) + + +def mark_previews_regenerating( + layer: Layer, + fingerprint: str, + raster_style_params: dict[str, Any] | None, +) -> list[int]: + """Upsert one preview row per multiframe frame for this fingerprint and clear images.""" + params = dict(raster_style_params or {}) + frame_ids = [] + for frame in layer.raster_frames(): + preview, created = RasterFramePreview.objects.get_or_create( + layer_frame=frame, + style_fingerprint=fingerprint, + defaults={ + "status": PreviewStatus.CREATING, + "raster_style_params": params, + }, + ) + preview.raster_style_params = params + preview.status = PreviewStatus.CREATING if created else PreviewStatus.REGENERATING + + if preview.image: + preview.image.delete(save=False) + preview.image = None + preview.width = None + preview.height = None + preview.bounds = {} + preview.save() + frame_ids.append(frame.id) + return frame_ids + + +def _dispatch_frame_preview_task( + result: TaskResult, + *, + fingerprint: str, + params: dict[str, Any], + task_kwargs: dict[str, Any], + run_mode: TaskRunMode | str, +) -> None: + # Lazy: preview_regeneration <- tasks.frame_preview <- tasks.__init__ <- tasks.dataset + # <- preview_regeneration when dataset imports this module at top level. + from uvdat.core.tasks.frame_preview import generate_frame_previews # noqa: PLC0415 + + layer_id = result.inputs["layer_id"] + result_id = result.id + if _coerce_run_mode(run_mode) == "async": + generate_frame_previews.delay( + layer_id, + fingerprint, + params, + result_id, + **task_kwargs, + ) + return + + with suppress_task_notifications(): + generate_frame_previews.apply( + args=(layer_id, fingerprint, params, result_id), + kwargs=task_kwargs, + ) + + +def invalidate_and_enqueue_layer_previews( + layer: Layer, + raster_style_params: dict[str, Any] | None = None, + *, + run_mode: TaskRunMode | str = "async", + project=None, + layer_style: LayerStyle | None = None, +) -> TaskResult | None: + """Invalidate/create preview rows for a params fingerprint and enqueue generation. + + If previews are already complete, or a job for this fingerprint is already + in flight, do not start another Celery task. + """ + if not layer.is_multiframe_raster(): + return None + + run_mode = _coerce_run_mode(run_mode) + params = dict(raster_style_params or {}) + fingerprint = params_fingerprint(params) + if previews_current_for_fingerprint(layer, fingerprint): + return None + + existing = pending_preview_task(layer_id=layer.id, fingerprint=fingerprint) + if existing is not None: + return existing + + style_name = layer_style.name if layer_style is not None else "default" + inputs: dict[str, Any] = { + "layer_id": layer.id, + "layer_name": layer.name, + "dataset_id": layer.dataset_id, + "fingerprint": fingerprint, + } + task_kwargs: dict[str, Any] = {} + if layer_style is not None: + inputs["layer_style_id"] = layer_style.id + task_kwargs["layer_style_id"] = layer_style.id + + with transaction.atomic(): + mark_previews_regenerating(layer, fingerprint, params) + clear_layer_preview_instance_cache(layer) + if layer_style is not None: + clear_style_preview_instance_cache(layer_style) + + result = TaskResult.objects.create( + name=f"Frame previews: {layer.name} - {style_name}", + task_type="frame_preview", + project=layer_style.project if layer_style is not None else project, + inputs=inputs, + ) + + _dispatch_frame_preview_task( + result, + fingerprint=fingerprint, + params=params, + task_kwargs=task_kwargs, + run_mode=run_mode, + ) + return result + + +def invalidate_and_enqueue_previews( + layer_style: LayerStyle, + *, + run_mode: TaskRunMode | str = "async", +) -> TaskResult | None: + """Invalidate previews for a style's current ``raster_style_params`` and enqueue.""" + if not layer_style.layer.is_multiframe_raster(): + return None + + layer_style.refresh_from_db() + return invalidate_and_enqueue_layer_previews( + layer_style.layer, + layer_style.raster_style_params, + run_mode=run_mode, + project=layer_style.project, + layer_style=layer_style, + ) + + +def preview_status_for_style(layer_style: LayerStyle) -> str | None: + if not layer_style.layer.is_multiframe_raster(): + return None + return preview_status_for_fingerprint(layer_style.layer, style_fingerprint(layer_style)) + + +def get_layer_style_preview_status(layer_style: LayerStyle) -> str | None: + if "preview_status" in layer_style.__dict__: + return layer_style.preview_status + return preview_status_for_style(layer_style) + + +def preview_status_for_layer(layer: Layer) -> str | None: + if not layer.is_multiframe_raster(): + return None + return preview_status_for_fingerprint(layer, layer_default_fingerprint(layer)) + + +def get_layer_preview_status(layer: Layer) -> str | None: + if "preview_status" in layer.__dict__: + return layer.preview_status + return preview_status_for_layer(layer) diff --git a/uvdat/core/frame_previews/raster_style.py b/uvdat/core/frame_previews/raster_style.py new file mode 100644 index 000000000..156aa2765 --- /dev/null +++ b/uvdat/core/frame_previews/raster_style.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +import json +from typing import Any + +# Thumbnail PNGs need explicit nodata handling: large_image maps masked nodata to +# black RGB unless the style includes nodata (and, for native single-band rasters, +# a palette with alpha). This mirrors styled tile transparency without changing +# stored ``raster_style_params`` fingerprints. +_NATIVE_SINGLE_BAND_PREVIEW_STYLE: dict[str, Any] = { + "nodata": "auto", + "min": "auto", + "max": "auto", + "palette": ["#00000000", "#ffffff"], +} + + +def apply_source_filters_to_style_query( + base_query: dict[str, Any], + source_filters: dict[str, Any] | None, +) -> dict[str, Any]: + query = dict(base_query) + if source_filters and "band" in source_filters: + query["band"] = source_filters["band"] + return query + + +def raster_source_filter_kwargs(source_filters: dict[str, Any] | None) -> dict[str, Any]: + """Return large_image kwargs that must not be embedded in style JSON.""" + if not source_filters or "frame" not in source_filters: + return {} + return {"frame": source_filters["frame"]} + + +def ensure_nodata_in_style_query(style_query: dict[str, Any]) -> dict[str, Any]: + """Ensure styled preview renders nodata as transparent PNG alpha, not black.""" + query = dict(style_query) + bands = query.get("bands") + if isinstance(bands, list): + query["bands"] = [ + {**band, "nodata": band.get("nodata", "auto")} if isinstance(band, dict) else band + for band in bands + ] + elif "nodata" not in query: + query["nodata"] = "auto" + return query + + +def resolve_preview_style_json( + base_style_query: dict[str, Any], + source_filters: dict[str, Any] | None, + *, + band_count: int | None, +) -> str | None: + """Build large-image style JSON for one preview frame.""" + style_query = apply_source_filters_to_style_query(base_style_query, source_filters) + if style_query: + return json.dumps(ensure_nodata_in_style_query(style_query)) + if band_count == 1: + return json.dumps(dict(_NATIVE_SINGLE_BAND_PREVIEW_STYLE)) + return None diff --git a/uvdat/core/frame_previews/types.py b/uvdat/core/frame_previews/types.py new file mode 100644 index 000000000..632f892ad --- /dev/null +++ b/uvdat/core/frame_previews/types.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from typing import TypedDict + + +class FramePreviewCorner(TypedDict): + x: float + y: float + + +class FramePreviewBounds(TypedDict, total=False): + srs: str # EPSG:4326 + xmin: float + xmax: float + ymin: float + ymax: float + ul: FramePreviewCorner + ur: FramePreviewCorner + lr: FramePreviewCorner + ll: FramePreviewCorner + + +class FramePreviewData(TypedDict): + url: str # presigned URL of the preview image + width: int + height: int + bounds: FramePreviewBounds diff --git a/uvdat/core/management/backfill_frame_previews_utils.py b/uvdat/core/management/backfill_frame_previews_utils.py new file mode 100644 index 000000000..231cd94dd --- /dev/null +++ b/uvdat/core/management/backfill_frame_previews_utils.py @@ -0,0 +1,186 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from django.db.models import Count, Q, QuerySet + +from uvdat.core.frame_previews.fingerprint import params_fingerprint +from uvdat.core.frame_previews.lookup import preview_status_for_fingerprint +from uvdat.core.frame_previews.preview_regeneration import ( + invalidate_and_enqueue_layer_previews, + pending_preview_task, +) +from uvdat.core.models import Colormap, Layer, LayerStyle, Project +from uvdat.core.rest.querysets import layer_queryset_with_previews +from uvdat.core.tasks.run_mode import TaskRunMode + +if TYPE_CHECKING: + from collections.abc import Callable + +RASTER_SOURCE_FILTER_KEYS = frozenset({"frame", "band"}) + + +def _color_query_from_spec( + color_spec: dict[str, Any], + colormaps_by_id: dict[int, Colormap], +) -> dict[str, Any]: + color_query: dict[str, Any] = {} + colormap_spec = color_spec.get("colormap") + if colormap_spec: + if colormap_spec.get("range"): + color_query["min"] = colormap_spec["range"][0] + color_query["max"] = colormap_spec["range"][1] + if colormap_spec.get("discrete"): + color_query["scheme"] = "discrete" + if colormap_spec.get("clamp") is False: + color_query["clamp"] = False + colormap = colormaps_by_id.get(colormap_spec.get("id")) + if colormap and colormap.markers: + color_query["palette"] = [marker["color"] for marker in colormap.markers] + elif color_spec.get("single_color"): + color_query["palette"] = color_spec["single_color"] + return color_query + + +def _build_raster_style_params( + style_spec: dict[str, Any], + colormaps_by_id: dict[int, Colormap], +) -> dict[str, Any]: + """Build django-large-image style JSON from style_spec (mirrors client getRasterTilesQuery).""" + query: dict[str, Any] = {} + for color_spec in style_spec.get("colors", []): + color_query = _color_query_from_spec(color_spec, colormaps_by_id) + if not color_spec.get("visible"): + continue + if color_spec.get("name") == "all": + query = color_query + elif color_query: + query.setdefault("bands", []) + color_query["band"] = color_spec["name"].replace("Band ", "") + query["bands"].append(color_query) + + for filter_spec in style_spec.get("filters", []): + if ( + filter_spec.get("include") + and filter_spec.get("filter_by") + and filter_spec.get("list") + and len(filter_spec["list"]) == 1 + and filter_spec["filter_by"] not in RASTER_SOURCE_FILTER_KEYS + ): + query[filter_spec["filter_by"]] = filter_spec["list"][0] + + return query + + +def resolve_raster_style_params(layer_style: LayerStyle) -> tuple[dict[str, Any], bool] | None: + """Return (params, reconstructed), or None when a referenced colormap is missing.""" + if layer_style.raster_style_params is not None: + return dict(layer_style.raster_style_params), False + + style_spec = layer_style.repr_style_configs() + colormap_ids = { + color_spec["colormap"]["id"] + for color_spec in style_spec.get("colors", []) + if color_spec.get("colormap", {}).get("id") + } + colormaps_by_id = { + colormap.id: colormap for colormap in Colormap.objects.filter(id__in=colormap_ids) + } + if colormap_ids - colormaps_by_id.keys(): + return None + + return _build_raster_style_params(style_spec, colormaps_by_id), True + + +def multiframe_layers_qs( + *, + layer_id: int | None = None, + dataset_id: int | None = None, + project_id: int | None = None, +) -> QuerySet: + qs = ( + layer_queryset_with_previews() + .annotate(raster_frame_count=Count("frames", filter=Q(frames__raster__isnull=False))) + .filter(raster_frame_count__gt=1) + ) + if layer_id is not None: + qs = qs.filter(id=layer_id) + if dataset_id is not None: + qs = qs.filter(dataset_id=dataset_id) + if project_id is not None: + project = Project.objects.filter(id=project_id).first() + if project is None: + return qs.none() + qs = qs.filter(dataset__in=project.datasets.all()) + return qs.order_by("id") + + +def _needs_generate(layer: Layer, params: dict[str, Any]) -> bool: + fingerprint = params_fingerprint(params) + if preview_status_for_fingerprint(layer, fingerprint) == "ready": + return False + return pending_preview_task(layer_id=layer.id, fingerprint=fingerprint) is None + + +def _collect_backfill_work( + layer: Layer, + *, + project_id: int | None, +) -> list[tuple[dict[str, Any], LayerStyle | None]]: + work: list[tuple[dict[str, Any], LayerStyle | None]] = [] + + styles = LayerStyle.objects.filter(layer=layer).order_by("id") + if project_id is not None: + styles = styles.filter(project_id=project_id) + + for layer_style in styles: + resolved = resolve_raster_style_params(layer_style) + if resolved is None: + continue + + params, reconstructed = resolved + if reconstructed: + layer_style.raster_style_params = params + layer_style.save(update_fields=["raster_style_params"]) + work.append((params, layer_style)) + + empty_fingerprint = params_fingerprint({}) + if _needs_generate(layer, {}) and not any( + params_fingerprint(params) == empty_fingerprint for params, _ in work + ): + work.insert(0, ({}, None)) + + return work + + +def _log_backfill_target( + echo: Callable[[str], None], + *, + layer_style: LayerStyle | None, +) -> None: + if layer_style is None: + echo(" native RGB (no style params)") + else: + echo(f" LayerStyle {layer_style.id} ({layer_style.name})") + + +def backfill_layer_previews( + layer: Layer, + *, + project_id: int | None = None, + echo: Callable[[str], None] | None = None, +) -> None: + if not layer.is_multiframe_raster(): + return + + for params, layer_style in _collect_backfill_work(layer, project_id=project_id): + if not _needs_generate(layer, params): + continue + if echo is not None: + _log_backfill_target(echo, layer_style=layer_style) + invalidate_and_enqueue_layer_previews( + layer, + params, + run_mode=TaskRunMode.SYNC, + layer_style=layer_style, + ) diff --git a/uvdat/core/management/commands/backfill_frame_previews.py b/uvdat/core/management/commands/backfill_frame_previews.py new file mode 100644 index 000000000..5dcd60db6 --- /dev/null +++ b/uvdat/core/management/commands/backfill_frame_previews.py @@ -0,0 +1,46 @@ +r"""Backfill multiframe raster frame previews for existing layers and styles. + +Dev reset + backfill via docker compose: + +docker compose run --rm django ./manage.py clear_frame_previews +docker compose run --rm django ./manage.py backfill_frame_previews +""" + +from __future__ import annotations + +import djclick as click + +from uvdat.core.management.backfill_frame_previews_utils import ( + backfill_layer_previews, + multiframe_layers_qs, +) + + +@click.command() +@click.option("--layer-id", type=int, default=None) +@click.option("--dataset-id", type=int, default=None) +@click.option("--project-id", type=int, default=None) +def backfill_frame_previews( + *, + layer_id: int | None, + dataset_id: int | None, + project_id: int | None, +): + """Backfill multiframe raster frame previews for existing layers and styles.""" + layers = list( + multiframe_layers_qs( + layer_id=layer_id, + dataset_id=dataset_id, + project_id=project_id, + ) + ) + + if not layers: + click.secho("No multiframe raster layers matched the filters.", fg="yellow") + return + + for layer in layers: + click.echo(f"Layer {layer.id}") + backfill_layer_previews(layer, project_id=project_id, echo=click.echo) + + click.secho("Done.", fg="green") diff --git a/uvdat/core/management/commands/clear_frame_previews.py b/uvdat/core/management/commands/clear_frame_previews.py new file mode 100644 index 000000000..9957ad807 --- /dev/null +++ b/uvdat/core/management/commands/clear_frame_previews.py @@ -0,0 +1,24 @@ +r"""Clear all raster frame previews and layer style raster params. + +Dev reset + backfill via docker compose: + +docker compose run --rm django ./manage.py clear_frame_previews +docker compose run --rm django ./manage.py backfill_frame_previews +""" + +from __future__ import annotations + +import djclick as click + +from uvdat.core.models import LayerStyle, RasterFramePreview + + +@click.command() +def clear_frame_previews(): + """Delete all frame previews and set raster_style_params to None on all layer styles.""" + preview_count, _ = RasterFramePreview.objects.all().delete() + style_count = LayerStyle.objects.update(raster_style_params=None) + + click.echo(f"Deleted {preview_count} RasterFramePreview rows") + click.echo(f"Cleared raster_style_params on {style_count} LayerStyle rows") + click.secho("Done.", fg="green") diff --git a/uvdat/core/management/commands/ingest.py b/uvdat/core/management/commands/ingest.py index 6892e53f6..e92e1f9d5 100644 --- a/uvdat/core/management/commands/ingest.py +++ b/uvdat/core/management/commands/ingest.py @@ -18,6 +18,7 @@ import pooch from uvdat.core.models import Chart, Dataset, FileItem, Project +from uvdat.core.tasks.run_mode import TaskRunMode DATA_FOLDER = Path(os.environ.get("INGEST_BIND_MOUNT_POINT", "sample_data")) DOWNLOADS_FOLDER = DATA_FOLDER / "downloads" @@ -258,7 +259,7 @@ def default_conversion_process(dataset: Dataset, options: DatasetItem): layer_options=options.get("layers"), network_options=options.get("network_options"), region_options=options.get("region_options"), - asynchronous=False, + run_mode=TaskRunMode.SYNC, ) diff --git a/uvdat/core/migrations/0028_layerstyle_raster_style_params_rasterframepreview.py b/uvdat/core/migrations/0028_layerstyle_raster_style_params_rasterframepreview.py new file mode 100644 index 000000000..6ec05416e --- /dev/null +++ b/uvdat/core/migrations/0028_layerstyle_raster_style_params_rasterframepreview.py @@ -0,0 +1,70 @@ +# Generated by Django 6.0.7 on 2026-08-26 16:41 + +from __future__ import annotations + +from django.db import migrations, models +import django.db.models.deletion +import s3_file_field.fields + + +class Migration(migrations.Migration): + dependencies = [ + ("core", "0027_project_region"), + ] + + operations = [ + migrations.AddField( + model_name="layerstyle", + name="raster_style_params", + field=models.JSONField(blank=True, default=None, null=True), + ), + migrations.CreateModel( + name="RasterFramePreview", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("style_fingerprint", models.CharField(max_length=64)), + ("raster_style_params", models.JSONField(blank=True, default=dict)), + ( + "status", + models.CharField( + choices=[ + ("creating", "Creating"), + ("regenerating", "Regenerating"), + ("complete", "Complete"), + ("failed", "Failed"), + ], + default="creating", + max_length=16, + ), + ), + ("image", s3_file_field.fields.S3FileField(blank=True, null=True)), + ("width", models.PositiveIntegerField(blank=True, null=True)), + ("height", models.PositiveIntegerField(blank=True, null=True)), + ("bounds", models.JSONField(blank=True, default=dict)), + ( + "layer_frame", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="previews", + to="core.layerframe", + ), + ), + ], + options={ + "constraints": [ + models.UniqueConstraint( + fields=("layer_frame", "style_fingerprint"), + name="unique_layer_frame_style_fingerprint_preview", + ) + ], + }, + ), + ] diff --git a/uvdat/core/models/__init__.py b/uvdat/core/models/__init__.py index 83ff77636..0e4672d76 100644 --- a/uvdat/core/models/__init__.py +++ b/uvdat/core/models/__init__.py @@ -6,6 +6,7 @@ from .data import RasterData, VectorData, VectorFeature from .dataset import Dataset, DatasetTag from .file_item import FileItem +from .frame_preview import RasterFramePreview from .layer import Layer, LayerFrame from .networks import Network, NetworkEdge, NetworkNode from .project import Project @@ -39,6 +40,7 @@ "NetworkNode", "Project", "RasterData", + "RasterFramePreview", "Region", "SizeConfig", "SizeRangeConfig", diff --git a/uvdat/core/models/chart.py b/uvdat/core/models/chart.py index 776da5817..1275ff04f 100644 --- a/uvdat/core/models/chart.py +++ b/uvdat/core/models/chart.py @@ -2,6 +2,8 @@ from django.db import models +from uvdat.core.models.task_result import suppress_task_notifications + from .project import Project from .querysets import ProjectQuerySet @@ -47,5 +49,6 @@ def spawn_conversion_task( convert_chart_signature.delay(result_id=result.id) return result else: - convert_chart_signature.apply() + with suppress_task_notifications(): + convert_chart_signature.apply() return None diff --git a/uvdat/core/models/dataset.py b/uvdat/core/models/dataset.py index 703c059f9..ac8202708 100644 --- a/uvdat/core/models/dataset.py +++ b/uvdat/core/models/dataset.py @@ -11,6 +11,8 @@ if typing.TYPE_CHECKING: from django.contrib.auth.models import User + from uvdat.core.tasks.run_mode import TaskRunMode + class DatasetTag(models.Model): tag = models.CharField(max_length=255, unique=True) @@ -82,10 +84,15 @@ def spawn_conversion_task( layer_options=None, network_options=None, region_options=None, - asynchronous=True, + run_mode: TaskRunMode | str = "async", ): # Prevent circular import + from uvdat.core.models.task_result import ( # noqa: PLC0415 + TaskResult, + suppress_task_notifications, + ) from uvdat.core.tasks.dataset import convert_dataset # noqa: PLC0415 + from uvdat.core.tasks.run_mode import TaskRunMode # noqa: PLC0415 convert_dataset_signature = convert_dataset.s( dataset_id=self.id, @@ -94,10 +101,9 @@ def spawn_conversion_task( region_options=region_options, ) - if asynchronous: - # Prevent circular import - from uvdat.core.models.task_result import TaskResult # noqa: PLC0415 + run_mode = TaskRunMode(run_mode) + if run_mode is TaskRunMode.ASYNC: result = TaskResult.objects.create( name=f"Conversion of Dataset {self.name}", task_type="conversion", @@ -112,6 +118,7 @@ def spawn_conversion_task( ) convert_dataset_signature.delay(result_id=result.id) return result - else: + + with suppress_task_notifications(): convert_dataset_signature.apply() - return None + return None diff --git a/uvdat/core/models/frame_preview.py b/uvdat/core/models/frame_preview.py new file mode 100644 index 000000000..432a72e48 --- /dev/null +++ b/uvdat/core/models/frame_preview.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +from django.db import models +from django.dispatch import receiver +from s3_file_field import S3FileField + +from .layer import LayerFrame + + +class PreviewStatus(models.TextChoices): + CREATING = "creating", "Creating" + REGENERATING = "regenerating", "Regenerating" + COMPLETE = "complete", "Complete" + FAILED = "failed", "Failed" + + +class RasterFramePreview(models.Model): + """Cache of one rendered PNG for (frame x raster_style_params fingerprint).""" + + layer_frame = models.ForeignKey( + LayerFrame, + related_name="previews", + on_delete=models.CASCADE, + ) + # sha256 of the raster_style_params JSON used to render this image + style_fingerprint = models.CharField(max_length=64) + # Params used for this render (debug / re-render without a LayerStyle). + raster_style_params = models.JSONField(default=dict, blank=True) + status = models.CharField( + max_length=16, + choices=PreviewStatus.choices, + default=PreviewStatus.CREATING, + ) + image = S3FileField(blank=True, null=True) + width = models.PositiveIntegerField(null=True, blank=True) + height = models.PositiveIntegerField(null=True, blank=True) + bounds = models.JSONField(default=dict, blank=True) + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["layer_frame", "style_fingerprint"], + name="unique_layer_frame_style_fingerprint_preview", + ) + ] + + def __str__(self): + return ( + f"Preview frame={self.layer_frame_id} " + f"fingerprint={self.style_fingerprint[:8]}… ({self.id})" + ) + + +@receiver(models.signals.post_delete, sender=RasterFramePreview) +def delete_preview_image(sender, instance, **kwargs): + if instance.image: + instance.image.delete(save=False) diff --git a/uvdat/core/models/layer.py b/uvdat/core/models/layer.py index 88489d9b1..2efaf3856 100644 --- a/uvdat/core/models/layer.py +++ b/uvdat/core/models/layer.py @@ -25,6 +25,20 @@ class Layer(models.Model): def __str__(self): return f"{self.name} ({self.id})" + def raster_frames(self): + prefetched = getattr(self, "_raster_frames", None) + if prefetched is not None: + return prefetched + return list( + self.frames.filter(raster__isnull=False).select_related("raster").order_by("index") + ) + + def is_multiframe_raster(self) -> bool: + prefetched = getattr(self, "_raster_frames", None) + if prefetched is not None: + return len(prefetched) > 1 + return self.frames.filter(raster__isnull=False).count() > 1 + class LayerFrame(models.Model): name = models.CharField(max_length=255, default="Layer Frame") diff --git a/uvdat/core/models/styles.py b/uvdat/core/models/styles.py index 6c36c3a7e..03166ca0c 100644 --- a/uvdat/core/models/styles.py +++ b/uvdat/core/models/styles.py @@ -1,16 +1,22 @@ from __future__ import annotations import contextlib -from typing import Any +from typing import TYPE_CHECKING, Any from django.core.validators import MaxValueValidator, MinValueValidator from django.db import models +from uvdat.core.frame_previews.fingerprint import style_fingerprint +from uvdat.core.frame_previews.lookup import ordered_complete_previews + from .colormap import Colormap from .layer import Layer from .project import Project from .querysets import ProjectQuerySet +if TYPE_CHECKING: + from uvdat.core.frame_previews.types import FramePreviewData + class LayerStyle(models.Model): name = models.CharField(max_length=255, default="Layer Style") @@ -26,6 +32,9 @@ class LayerStyle(models.Model): MaxValueValidator(1), ], ) + # django-large-image ``style`` query JSON for raster tiles/previews. + # Provided by the client on style create/update (same object used for tile URLs). + raster_style_params = models.JSONField(blank=True, null=True, default=None) project_filter_path = "layer__dataset__project" objects = ProjectQuerySet.as_manager() @@ -163,6 +172,7 @@ def save_style_configs(self, style_spec): # noqa: C901, PLR0912, PLR0915 filter_config.save() filter_config_ids.append(filter_config.id) FilterConfig.objects.filter(style=self).exclude(id__in=filter_config_ids).delete() + self.save(update_fields=["default_frame", "opacity"]) def repr_style_configs(self): colors = [] @@ -227,6 +237,10 @@ def repr_style_configs(self): "filters": filters, } + def multiframe_previews(self, layer=None) -> list[FramePreviewData] | None: + layer = layer or self.layer + return ordered_complete_previews(layer, style_fingerprint(self)) + def get_default_colormap(): return Colormap.objects.filter(project__isnull=True).first() diff --git a/uvdat/core/models/task_result.py b/uvdat/core/models/task_result.py index f7f06be8a..771618e72 100644 --- a/uvdat/core/models/task_result.py +++ b/uvdat/core/models/task_result.py @@ -1,6 +1,9 @@ from __future__ import annotations +import contextlib +from contextvars import ContextVar import json +import logging from asgiref.sync import async_to_sync from channels.layers import get_channel_layer @@ -16,6 +19,24 @@ from .project import Project from .querysets import ProjectQuerySet +logger = logging.getLogger(__name__) + +# Set while running in a context (e.g. `manage.py ingest`) where TaskResult +# WebSocket notifications are meaningless: tasks run synchronously with no +# client session listening. When set, result_post_save skips the push entirely +# rather than attempting it and logging a warning. +_suppress_notifications: ContextVar[bool] = ContextVar("suppress_task_notifications", default=False) + + +@contextlib.contextmanager +def suppress_task_notifications(): + """Silence TaskResult WebSocket notifications for the current context.""" + token = _suppress_notifications.set(True) + try: + yield + finally: + _suppress_notifications.reset(token) + class TaskResult(models.Model): name = models.CharField(max_length=255) @@ -77,6 +98,12 @@ def complete(self): @receiver(post_save, sender=TaskResult) def result_post_save(sender, instance, **kwargs): + # In contexts such as `manage.py ingest`, tasks run synchronously with no + # client session listening, so the push is meaningless -- skip it silently + # (no attempt, no warning). See suppress_task_notifications(). + if _suppress_notifications.get(): + return + # Prevent circular import from uvdat.core.rest.serializers import TaskResultSerializer # noqa: PLC0415 @@ -85,6 +112,21 @@ def result_post_save(sender, instance, **kwargs): if instance.project: group_name = f"analytics_{instance.project.id}" channel_layer = get_channel_layer() - async_to_sync(channel_layer.group_send)( - group_name, {"type": "send_notification", "message": json.dumps(payload)} - ) + if channel_layer is None: + return + + # This WebSocket push streams live TaskResult updates to browser clients + # subscribed to a channel group. It is best-effort: a failed notification + # must never abort the surrounding TaskResult.save(). Failures are logged at + # WARNING so a genuinely broken channel layer in production is still visible. + try: + async_to_sync(channel_layer.group_send)( + group_name, {"type": "send_notification", "message": json.dumps(payload)} + ) + except Exception: # noqa: BLE001 - notification failures must never propagate + logger.warning( + "Failed to send TaskResult notification for %s to group %r", + instance, + group_name, + exc_info=True, + ) diff --git a/uvdat/core/rest/dataset.py b/uvdat/core/rest/dataset.py index 882c33de8..5e801c13e 100644 --- a/uvdat/core/rest/dataset.py +++ b/uvdat/core/rest/dataset.py @@ -6,6 +6,7 @@ from uvdat.core.access_control import DatasetGuardianPermission from uvdat.core.models import Dataset, DatasetTag, Network +from uvdat.core.rest.querysets import layer_queryset_with_previews from uvdat.core.rest.serializers import ( DatasetSerializer, FileItemSerializer, @@ -43,7 +44,7 @@ def tags(self, request, **kwargs): @action(detail=True, methods=["get"]) def layers(self, request, **kwargs): dataset: Dataset = self.get_object() - layers = list(dataset.layers.all()) + layers = list(layer_queryset_with_previews().filter(dataset=dataset)) serializer = LayerSerializer(layers, many=True) return Response(serializer.data, status=200) diff --git a/uvdat/core/rest/layer.py b/uvdat/core/rest/layer.py index 4836f3c7f..f6546b223 100644 --- a/uvdat/core/rest/layer.py +++ b/uvdat/core/rest/layer.py @@ -6,18 +6,22 @@ from rest_framework.response import Response from rest_framework.viewsets import ModelViewSet, ReadOnlyModelViewSet +from uvdat.core.frame_previews.preview_regeneration import invalidate_and_enqueue_previews from uvdat.core.models import Layer, LayerFrame, LayerStyle, Project +from uvdat.core.rest.querysets import layer_queryset_with_previews from uvdat.core.rest.serializers import ( LayerFrameSerializer, LayerSerializer, - LayerStyleSerializer, + LayerStyleWithPreviewsSerializer, ) class LayerViewSet(ReadOnlyModelViewSet): - queryset = Layer.objects.select_related("dataset").all() serializer_class = LayerSerializer + def get_queryset(self): + return layer_queryset_with_previews() + @action(detail=True, methods=["get"]) def frames(self, request, **kwargs): layer: Layer = self.get_object() @@ -33,7 +37,7 @@ class LayerFrameViewSet(ReadOnlyModelViewSet): class LayerStyleViewSet(ModelViewSet): queryset = LayerStyle.objects.all() - serializer_class = LayerStyleSerializer + serializer_class = LayerStyleWithPreviewsSerializer def get_queryset(self): qs = super().get_queryset() @@ -43,7 +47,7 @@ def get_queryset(self): layer_id = int(self.request.query_params.get("layer", -1)) if layer_id > -1: qs = qs.filter(layer=int(layer_id)) - return qs + return layer_queryset_with_previews(qs, for_layer_style=True) def create(self, request, **kwargs): project = Project.objects.get(id=request.data.get("project")) @@ -53,7 +57,7 @@ def create(self, request, **kwargs): status=403, ) is_default = request.data.pop("is_default", False) - serializer = LayerStyleSerializer(data=request.data) + serializer = LayerStyleWithPreviewsSerializer(data=request.data) serializer.is_valid(raise_exception=True) with transaction.atomic(): try: @@ -63,12 +67,14 @@ def create(self, request, **kwargs): if is_default and instance.layer.default_style != instance: instance.layer.default_style = instance instance.layer.save() + # Enqueue after commit so a worker cannot start before style rows exist. + invalidate_and_enqueue_previews(instance) return Response(serializer.data, status=200) def partial_update(self, request, **kwargs): instance = self.get_object() is_default = request.data.pop("is_default", False) - serializer = LayerStyleSerializer(instance, data=request.data, partial=True) + serializer = LayerStyleWithPreviewsSerializer(instance, data=request.data, partial=True) serializer.is_valid(raise_exception=True) with transaction.atomic(): try: @@ -78,6 +84,7 @@ def partial_update(self, request, **kwargs): if is_default and instance.layer.default_style != instance: instance.layer.default_style = instance instance.layer.save() + invalidate_and_enqueue_previews(instance) return Response(serializer.data, status=200) def destroy(self, request, *args, **kwargs): diff --git a/uvdat/core/rest/querysets.py b/uvdat/core/rest/querysets.py new file mode 100644 index 000000000..e1516133c --- /dev/null +++ b/uvdat/core/rest/querysets.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +from django.db.models import Prefetch, QuerySet + +from uvdat.core.models import Layer, LayerFrame, LayerStyle + +RASTER_FRAMES_QUERYSET = ( + LayerFrame.objects.filter(raster__isnull=False).select_related("raster").order_by("index") +) + +LAYER_RASTER_FRAMES_PREFETCH = Prefetch( + "frames", + queryset=RASTER_FRAMES_QUERYSET, + to_attr="_raster_frames", +) + +STYLE_LAYER_RASTER_FRAMES_PREFETCH = Prefetch( + "layer__frames", + queryset=RASTER_FRAMES_QUERYSET, + to_attr="_raster_frames", +) + + +def layer_queryset_with_previews( + queryset: QuerySet | None = None, + *, + for_layer_style: bool = False, +) -> QuerySet: + # Frame list is prefetched for multiframe checks / ordering. Preview rows for a + # fingerprint are loaded in ``previews_by_frame_id`` with a filtered query. + if for_layer_style: + qs = queryset if queryset is not None else LayerStyle.objects.all() + return qs.select_related("layer", "layer__default_style").prefetch_related( + STYLE_LAYER_RASTER_FRAMES_PREFETCH, + ) + + qs = queryset if queryset is not None else Layer.objects.all() + return qs.select_related("dataset", "default_style").prefetch_related( + LAYER_RASTER_FRAMES_PREFETCH, + ) diff --git a/uvdat/core/rest/serializers.py b/uvdat/core/rest/serializers.py index a63ef1513..6a6ac5e0c 100644 --- a/uvdat/core/rest/serializers.py +++ b/uvdat/core/rest/serializers.py @@ -6,6 +6,11 @@ from rest_framework import serializers from rest_framework.exceptions import PermissionDenied +from uvdat.core.frame_previews.lookup import layer_default_multiframe_previews +from uvdat.core.frame_previews.preview_regeneration import ( + get_layer_preview_status, + get_layer_style_preview_status, +) from uvdat.core.models import ( Basemap, Chart, @@ -166,8 +171,15 @@ class Meta: fields = "__all__" +def _omit_null_field(data: dict, field: str) -> None: + if data.get(field) is None: + data.pop(field, None) + + class LayerStyleSerializer(serializers.ModelSerializer): is_default = serializers.SerializerMethodField("get_is_default") + # Client-computed django-large-image style JSON; write-only (used for previews). + raster_style_params = serializers.JSONField(required=False, allow_null=True, write_only=True) def get_is_default(self, obj): if obj.layer.default_style is None: @@ -195,12 +207,58 @@ class Meta: exclude = ["default_frame", "opacity"] +class LayerStyleWithPreviewsSerializer(LayerStyleSerializer): + multiframe_previews = serializers.SerializerMethodField() + preview_status = serializers.SerializerMethodField() + + def _preview_layer(self, obj): + return self.context.get("preview_layer") or obj.layer + + def get_preview_status(self, obj): + return get_layer_style_preview_status(obj) + + def get_multiframe_previews(self, obj): + if get_layer_style_preview_status(obj) != "ready": + return None + return obj.multiframe_previews(layer=self._preview_layer(obj)) + + def to_representation(self, instance): + data = super().to_representation(instance) + _omit_null_field(data, "multiframe_previews") + _omit_null_field(data, "preview_status") + return data + + class LayerSerializer(serializers.ModelSerializer): default_style = LayerStyleSerializer() + multiframe_previews = serializers.SerializerMethodField() + preview_status = serializers.SerializerMethodField() + + def get_preview_status(self, obj): + return get_layer_preview_status(obj) + + def get_multiframe_previews(self, obj): + if get_layer_preview_status(obj) != "ready": + return None + return layer_default_multiframe_previews(obj) + + def to_representation(self, instance): + data = super().to_representation(instance) + _omit_null_field(data, "multiframe_previews") + _omit_null_field(data, "preview_status") + return data class Meta: model = Layer - fields = ["id", "name", "metadata", "dataset", "default_style"] + fields = [ + "id", + "name", + "metadata", + "dataset", + "default_style", + "multiframe_previews", + "preview_status", + ] class VectorDataSerializer(serializers.ModelSerializer): diff --git a/uvdat/core/tasks/__init__.py b/uvdat/core/tasks/__init__.py index 9e431834c..e900fe660 100644 --- a/uvdat/core/tasks/__init__.py +++ b/uvdat/core/tasks/__init__.py @@ -2,8 +2,12 @@ from .chart import convert_chart from .dataset import convert_dataset +from .frame_preview import generate_frame_previews +from .run_mode import TaskRunMode __all__ = [ + "TaskRunMode", "convert_chart", "convert_dataset", + "generate_frame_previews", ] diff --git a/uvdat/core/tasks/analytics/flood_simulation.py b/uvdat/core/tasks/analytics/flood_simulation.py index 798715299..dad4aa495 100644 --- a/uvdat/core/tasks/analytics/flood_simulation.py +++ b/uvdat/core/tasks/analytics/flood_simulation.py @@ -8,7 +8,9 @@ from django.conf import settings from django.core.files import File +from uvdat.core.frame_previews.preview_regeneration import invalidate_and_enqueue_previews from uvdat.core.models import Chart, Colormap, Dataset, FileItem, LayerStyle, TaskResult +from uvdat.core.tasks.run_mode import TaskRunMode from .analysis_type import AnalysisTask, AnalysisType @@ -176,7 +178,7 @@ def flood_simulation(result_id): ], network_options=None, region_options=None, - asynchronous=False, + run_mode=TaskRunMode.SYNC, ) # Create a default style for new layer @@ -220,6 +222,16 @@ def flood_simulation(result_id): ], } ) + style.raster_style_params = { + "min": 0, + "max": 2, + "palette": ["#002081", "#2AD3FF"], + } + style.save(update_fields=["raster_style_params"]) + # Conversion generated empty-params previews inline (sync); regenerate for + # this style's params so layer_default_fingerprint / preview_status are + # ready before this flood TaskResult completes. + invalidate_and_enqueue_previews(style, run_mode=TaskRunMode.SYNC) result.project.datasets.add(dataset) result.write_outputs( diff --git a/uvdat/core/tasks/dataset.py b/uvdat/core/tasks/dataset.py index 5d9be14c8..bf7d2221f 100644 --- a/uvdat/core/tasks/dataset.py +++ b/uvdat/core/tasks/dataset.py @@ -4,6 +4,9 @@ from celery import shared_task +from uvdat.core.frame_previews.preview_regeneration import ( + invalidate_and_enqueue_layer_previews, +) from uvdat.core.models import ( Dataset, FileItem, @@ -13,6 +16,7 @@ TaskResult, VectorData, ) +from uvdat.core.tasks.run_mode import TaskRunMode from .conversion import convert_file_item from .data import create_vector_features @@ -20,7 +24,11 @@ from .regions import create_source_regions -def create_layers_and_frames(dataset, layer_options=None): # noqa: C901, PLR0912, PLR0915 +def create_layers_and_frames( # noqa: C901, PLR0912, PLR0915 + dataset, + layer_options=None, + task_result=None, +): Layer.objects.filter(dataset=dataset).delete() LayerFrame.objects.filter(layer__dataset=dataset).delete() vectors = VectorData.objects.filter(dataset=dataset) @@ -139,6 +147,14 @@ def create_layers_and_frames(dataset, layer_options=None): # noqa: C901, PLR091 source_filters=frame_info.get("source_filters", {}), ) + # Default empty-params previews (no Project/LayerStyle required). + for layer in Layer.objects.filter(dataset=dataset): + if layer.is_multiframe_raster(): + if task_result is not None: + task_result.write_status(f"Generating frame previews for layer {layer.name}...") + # All Dataset Preview tasks are run synchronously. + invalidate_and_enqueue_layer_previews(layer, {}, run_mode=TaskRunMode.SYNC) + @shared_task def convert_dataset( @@ -179,7 +195,11 @@ def convert_dataset( vector_data.get_summary() - create_layers_and_frames(dataset, layer_options) + create_layers_and_frames( + dataset, + layer_options, + task_result=result, + ) dataset.processing = False dataset.save() diff --git a/uvdat/core/tasks/frame_preview.py b/uvdat/core/tasks/frame_preview.py new file mode 100644 index 000000000..d2d1db7d9 --- /dev/null +++ b/uvdat/core/tasks/frame_preview.py @@ -0,0 +1,425 @@ +from __future__ import annotations + +from dataclasses import dataclass +import io +import logging +import time +from typing import TYPE_CHECKING, Any + +from celery import shared_task +from django.core.files.base import ContentFile +from django.utils import timezone +from django_large_image import tilesource, utilities +from PIL import Image + +from uvdat.core.frame_previews.fingerprint import params_fingerprint +from uvdat.core.frame_previews.raster_style import ( + raster_source_filter_kwargs, + resolve_preview_style_json, +) +from uvdat.core.models import ( + Layer, + LayerStyle, + RasterData, + RasterFramePreview, + TaskResult, +) +from uvdat.core.models.frame_preview import PreviewStatus + +if TYPE_CHECKING: + from uvdat.core.frame_previews.types import FramePreviewBounds + +"""Celery tasks and helpers for multiframe raster frame preview images. + +Previews are styled PNG thumbnails stored on ``RasterFramePreview`` rows, keyed +by ``(layer_frame, style_fingerprint)``. Enqueue skips starting a second job +when one is already in flight for the same fingerprint. +""" + +logger = logging.getLogger(__name__) + +# Thumbnail sizing: default to 1/8 of FRAME_PREVIEW_MAX_PX (512px), but never +# below FRAME_PREVIEW_MIN_PX when the source raster is large enough to allow it. +FRAME_PREVIEW_MAX_PX = 4096 +FRAME_PREVIEW_MIN_PX = 1024 +FRAME_PREVIEW_DEFAULT_RESOLUTION_FRACTION = 1 / 8 + + +@dataclass(frozen=True) +class _PreviewGenerationContext: + """Immutable inputs shared across all frames in one task invocation.""" + + layer: Layer + fingerprint: str + base_style_query: dict[str, Any] + layer_style: LayerStyle | None = None + resolution_fraction: float | None = None + task_result: TaskResult | None = None + + @property + def layer_id(self) -> int: + return self.layer.id + + +@dataclass(frozen=True) +class _FramePreviewImage: + """PNG payload and metadata produced by ``generate_frame_preview_png``.""" + + png_bytes: bytes + width: int + height: int + bounds: FramePreviewBounds | None + + +@dataclass(frozen=True) +class _PreviewGenerationStats: + """Per-task frame counts written to ``TaskResult.outputs`` on completion.""" + + ready_count: int + failed_count: int + + +def resolve_resolution_fraction(resolution_fraction: float | None = None) -> float: + if resolution_fraction is None: + return FRAME_PREVIEW_DEFAULT_RESOLUTION_FRACTION + return float(resolution_fraction) + + +def _raster_max_dimension(metadata: dict[str, Any]) -> int: + """Largest source pixel dimension from large-image metadata (sizeX/sizeY).""" + size_x = metadata.get("sizeX") or metadata.get("width") or 0 + size_y = metadata.get("sizeY") or metadata.get("height") or 0 + return max(int(size_x), int(size_y)) + + +def resolve_preview_max_dimension( + resolution_fraction: float | None = None, + raster_max_dimension: int | None = None, +) -> int: + """Pick a thumbnail edge length, clamped to the source raster's size.""" + fraction = resolve_resolution_fraction(resolution_fraction) + fractional = round(FRAME_PREVIEW_MAX_PX * fraction) + if not raster_max_dimension or raster_max_dimension < 2: + return max(2, fractional) + # Small rasters use their native size; larger ones get at least MIN_PX. + floor = min(raster_max_dimension, FRAME_PREVIEW_MIN_PX) + return max(2, min(max(fractional, floor), raster_max_dimension)) + + +def _thumbnail_png_bytes(thumb_data: Any) -> bytes: + if isinstance(thumb_data, bytes): + return thumb_data + if isinstance(thumb_data, Image.Image): + buffer = io.BytesIO() + thumb_data.save(buffer, format="PNG") + return buffer.getvalue() + msg = f"Unsupported thumbnail data type: {type(thumb_data)!r}" + raise TypeError(msg) + + +def _preview_bounds(source) -> FramePreviewBounds | None: + bounds = tilesource.get_bounds(source, projection="EPSG:4326") + if not bounds: + return None + result: FramePreviewBounds = { + "srs": "EPSG:4326", + "xmin": bounds["xmin"], + "xmax": bounds["xmax"], + "ymin": bounds["ymin"], + "ymax": bounds["ymax"], + } + for corner in ("ul", "ur", "lr", "ll"): + corner_bounds = bounds.get(corner) + if corner_bounds: + result[corner] = {"x": corner_bounds["x"], "y": corner_bounds["y"]} + return result + + +def generate_frame_preview_png( + raster: RasterData, + source_filters: dict[str, Any] | None, + base_style_query: dict[str, Any], + resolution_fraction: float | None = None, +) -> tuple[bytes, int, int, FramePreviewBounds | None]: + """Render one frame as a styled PNG via large-image. + + Frame selection is passed through ``source_filters`` (e.g. ``{"frame": 3}``), + not embedded in the style query, so one style query can be reused for every + frame in a multiframe layer. + """ + source_kwargs = raster_source_filter_kwargs(source_filters) + raster_path = utilities.field_file_to_local_path(raster.cloud_optimized_geotiff) + style = resolve_preview_style_json( + base_style_query, + source_filters, + band_count=None, + ) + if style is None: + # Single-band rasters (e.g. flood depth) need an explicit preview style with + # nodata + alpha palette—style=None thumbnails leave nodata as opaque black. + # Multi-band RGB rasters keep style=None so large-image builds the usual + # per-band default (red/green/blue) without forcing a grayscale preview style. + probe = tilesource.get_tilesource_from_path( + raster_path, + encoding="PNG", + style=None, + ) + metadata = probe.getMetadata() + band_count = metadata.get("bandCount") or probe.bandCount or 1 + style = resolve_preview_style_json( + base_style_query, + source_filters, + band_count=int(band_count), + ) + # Reopen when the probe result calls for a preview-specific style; otherwise + # reuse the probe source and avoid a second large-image open. + source = ( + tilesource.get_tilesource_from_path( + raster_path, + encoding="PNG", + style=style, + ) + if style is not None + else probe + ) + else: + source = tilesource.get_tilesource_from_path( + raster_path, + encoding="PNG", + style=style, + ) + max_dimension = resolve_preview_max_dimension( + resolution_fraction, + # Sample output dimensions from metadata so thumbnail max edge respects the + # source raster size (small rasters are not upscaled beyond native resolution). + _raster_max_dimension(source.getMetadata()), + ) + thumb_data, _mime_type = source.getThumbnail( + encoding="PNG", + width=max_dimension, + height=max_dimension, + **source_kwargs, + ) + png_bytes = _thumbnail_png_bytes(thumb_data) + image = Image.open(io.BytesIO(png_bytes)) + return png_bytes, image.width, image.height, _preview_bounds(source) + + +def _abandon_task_result(result_id: int | None, status: str) -> None: + """Close an open TaskResult without marking it successfully completed.""" + if result_id is None: + return + TaskResult.objects.filter(id=result_id, completed__isnull=True).update( + completed=timezone.now(), + status=status, + ) + + +def _open_task_result(result_id: int | None, layer_id: int) -> TaskResult | None: + """Load the TaskResult for this run, or None if it is missing/already closed.""" + if result_id is None: + return None + + result = TaskResult.objects.filter(id=result_id).first() + if result is None or result.completed is not None: + logger.info( + "Skipping preview generation for layer=%s; task result %s already closed", + layer_id, + result_id, + ) + return None + return result + + +def _save_frame_preview( + preview: RasterFramePreview, + layer_id: int, + fingerprint: str, + frame_index: int, + image: _FramePreviewImage, +) -> None: + """Persist a generated preview and mark the row complete (API-servable).""" + preview.width = image.width + preview.height = image.height + preview.bounds = image.bounds or {} + preview.status = PreviewStatus.COMPLETE + preview.image.save( + f"frame-previews/{layer_id}/{fingerprint[:16]}/{frame_index}.png", + ContentFile(image.png_bytes), + save=False, + ) + preview.save() + + +def _mark_frame_preview_failed(preview: RasterFramePreview, fingerprint: str) -> None: + if preview.style_fingerprint == fingerprint: + preview.status = PreviewStatus.FAILED + preview.save(update_fields=["status"]) + + +def _process_frame_preview(ctx: _PreviewGenerationContext, frame) -> str: + """Generate one frame preview. + + Returns ``ready``, ``failed``, or ``skipped``. + """ + try: + preview = RasterFramePreview.objects.get( + layer_frame=frame, + style_fingerprint=ctx.fingerprint, + ) + except RasterFramePreview.DoesNotExist: + logger.warning( + "Missing preview row for layer=%s fingerprint=%s frame=%s; skipping", + ctx.layer_id, + ctx.fingerprint[:8], + frame.id, + ) + return "skipped" + + try: + png_bytes, width, height, bounds = generate_frame_preview_png( + frame.raster, + frame.source_filters, + ctx.base_style_query, + ctx.resolution_fraction, + ) + except Exception: + logger.exception( + "Failed to generate frame preview for layer=%s frame=%s", + ctx.layer_id, + frame.id, + ) + _mark_frame_preview_failed(preview, ctx.fingerprint) + return "failed" + + _save_frame_preview( + preview, + ctx.layer_id, + ctx.fingerprint, + frame.index, + _FramePreviewImage(png_bytes, width, height, bounds), + ) + return "ready" + + +def _complete_preview_task( + result: TaskResult | None, + ctx: _PreviewGenerationContext, + stats: _PreviewGenerationStats, +) -> None: + """Finalize the TaskResult, which triggers a WebSocket notification.""" + if result is None: + return + + result.refresh_from_db(fields=["completed"]) + if result.completed is not None: + return + + result.outputs = { + "layer_id": ctx.layer_id, + "fingerprint": ctx.fingerprint, + "ready_count": stats.ready_count, + "failed_count": stats.failed_count, + **({"layer_style_id": ctx.layer_style.id} if ctx.layer_style is not None else {}), + } + result.save(update_fields=["outputs"]) + result.complete() + + +def _resolve_preview_style_inputs( + layer_id: int, + fingerprint: str, + base_style_query: dict[str, Any] | None, + layer_style_id: int | None, +) -> tuple[LayerStyle | None, dict[str, Any]]: + """Resolve render params from the enqueue snapshot; optionally attach a style.""" + query = dict(base_style_query or {}) + if params_fingerprint(query) != fingerprint: + logger.warning( + "base_style_query fingerprint mismatch for layer=%s; using provided query", + layer_id, + ) + + layer_style = None + if layer_style_id is not None: + layer_style = LayerStyle.objects.filter(id=layer_style_id).first() + if layer_style is None: + logger.info( + "Style %s missing for layer=%s; generating from enqueued params", + layer_style_id, + layer_id, + ) + return layer_style, query + + +@shared_task +def generate_frame_previews( # noqa: PLR0913 + layer_id: int, + fingerprint: str, + base_style_query: dict[str, Any] | None = None, + result_id: int | None = None, + layer_style_id: int | None = None, + resolution_fraction: float | None = None, +): + """Generate styled PNG previews for every frame of a multiframe raster layer. + + Rows are keyed by ``(layer_frame, fingerprint)`` and created upstream with + ``creating``/``regenerating`` status. Rendering uses the snapshotted + ``base_style_query`` from enqueue time. + """ + started = time.perf_counter() + layer = Layer.objects.get(id=layer_id) + if not layer.is_multiframe_raster(): + _abandon_task_result(result_id, "Layer is not multiframe; nothing to preview.") + return + frames = layer.raster_frames() + + layer_style, query = _resolve_preview_style_inputs( + layer_id, + fingerprint, + base_style_query, + layer_style_id, + ) + + result = _open_task_result(result_id, layer_id) + if result_id is not None and result is None: + return + + logger.info( + "Generating %d multiframe raster previews for layer=%r fingerprint=%s", + len(frames), + layer.name, + fingerprint[:8], + ) + + ctx = _PreviewGenerationContext( + layer=layer, + fingerprint=fingerprint, + base_style_query=query, + layer_style=layer_style, + resolution_fraction=resolution_fraction, + task_result=result, + ) + + ready_count = 0 + failed_count = 0 + for frame in frames: + outcome = _process_frame_preview(ctx, frame) + if outcome == "ready": + ready_count += 1 + elif outcome == "failed": + failed_count += 1 + + _complete_preview_task( + result, + ctx, + _PreviewGenerationStats(ready_count, failed_count), + ) + + logger.info( + "Multiframe raster previews for layer=%r: %d ready, %d failed in %.2fs", + layer.name, + ready_count, + failed_count, + time.perf_counter() - started, + ) diff --git a/uvdat/core/tasks/run_mode.py b/uvdat/core/tasks/run_mode.py new file mode 100644 index 000000000..9291ea8f8 --- /dev/null +++ b/uvdat/core/tasks/run_mode.py @@ -0,0 +1,10 @@ +from __future__ import annotations + +from enum import StrEnum + + +class TaskRunMode(StrEnum): + """How a Celery task should be executed (avoids boolean-trap call sites).""" + + ASYNC = "async" + SYNC = "sync" diff --git a/uvdat/core/tests/factories.py b/uvdat/core/tests/factories.py index 5b0b236ee..325d58d52 100644 --- a/uvdat/core/tests/factories.py +++ b/uvdat/core/tests/factories.py @@ -132,7 +132,7 @@ class Meta: name = factory.Faker("name") layer = factory.SubFactory(LayerFactory) - vector = factory.SubFactory(VectorDataFactory) + vector = None raster = factory.SubFactory(RasterDataFactory) diff --git a/uvdat/core/tests/test_frame_preview.py b/uvdat/core/tests/test_frame_preview.py new file mode 100644 index 000000000..f022529d9 --- /dev/null +++ b/uvdat/core/tests/test_frame_preview.py @@ -0,0 +1,836 @@ +from __future__ import annotations + +import inspect +import json + +from django.core.files.base import ContentFile +import pytest + +from uvdat.core.frame_previews.fingerprint import ( + _fingerprint_payload, + params_fingerprint, + style_fingerprint, +) +from uvdat.core.frame_previews.preview_regeneration import ( + get_layer_style_preview_status, + invalidate_and_enqueue_layer_previews, + invalidate_and_enqueue_previews, +) +from uvdat.core.frame_previews.raster_style import ( + apply_source_filters_to_style_query, + ensure_nodata_in_style_query, + raster_source_filter_kwargs, + resolve_preview_style_json, +) +from uvdat.core.models import LayerStyle, RasterFramePreview, TaskResult +from uvdat.core.models.frame_preview import PreviewStatus +from uvdat.core.tasks.analytics import flood_simulation as flood_mod +from uvdat.core.tasks.frame_preview import ( + FRAME_PREVIEW_DEFAULT_RESOLUTION_FRACTION, + FRAME_PREVIEW_MAX_PX, + FRAME_PREVIEW_MIN_PX, + generate_frame_previews, + resolve_preview_max_dimension, +) +from uvdat.core.tasks.run_mode import TaskRunMode + +# LayerStyle PATCH still requires vector style_spec (colors/sizes); unused for fingerprints. +_VECTOR_STYLE_SPEC = { + "default_frame": 0, + "opacity": 1, + "colors": [{"name": "all", "visible": True, "use_feature_props": True}], + "sizes": [{"name": "all", "zoom_scaling": True, "single_size": 5}], + "filters": [], +} + + +def _patch_preview_delay(mocker): + return mocker.patch("uvdat.core.tasks.frame_preview.generate_frame_previews.delay") + + +def _make_preview( + layer_frame, + *, + fingerprint=None, + params=None, + status=PreviewStatus.COMPLETE, + **kwargs, +): + params = dict(params or {}) + fingerprint = fingerprint if fingerprint is not None else params_fingerprint(params) + return RasterFramePreview.objects.create( + layer_frame=layer_frame, + style_fingerprint=fingerprint, + raster_style_params=params, + status=status, + **kwargs, + ) + + +@pytest.mark.django_db +def test_layer_style_api_stores_client_raster_style_params( + authenticated_api_client, + layer_style_factory, + layer_frame_factory, + project, + user, +): + layer_style = layer_style_factory() + layer_frame_factory(layer=layer_style.layer, index=0) + project.set_collaborators([user]) + project.datasets.set([layer_style.layer.dataset]) + raster_style_params = {"palette": "#00ff00", "min": 0, "max": 1} + + resp = authenticated_api_client.patch( + f"/api/v1/layer-styles/{layer_style.id}/", + { + "name": layer_style.name, + "layer": layer_style.layer_id, + "project": layer_style.project_id, + "style_spec": _VECTOR_STYLE_SPEC, + "raster_style_params": raster_style_params, + }, + format="json", + ) + assert resp.status_code == 200 + assert "raster_style_params" not in resp.json() + layer_style.refresh_from_db() + assert layer_style.raster_style_params == raster_style_params + + +def test_raster_source_filter_kwargs_extracts_frame(): + assert raster_source_filter_kwargs({"frame": 3}) == {"frame": 3} + assert raster_source_filter_kwargs({"band": 2}) == {} + assert raster_source_filter_kwargs({"frame": 1, "band": 2}) == {"frame": 1} + + +def test_apply_source_filters_to_style_query_embeds_band_not_frame(): + query = apply_source_filters_to_style_query({"palette": "#fff"}, {"frame": 3, "band": 2}) + assert query == {"palette": "#fff", "band": 2} + assert "frame" not in query + assert apply_source_filters_to_style_query({}, {"frame": 3, "band": 1}) == {"band": 1} + assert apply_source_filters_to_style_query({}, {"band": 1}) == {"band": 1} + + +def test_ensure_nodata_in_style_query_adds_auto(): + assert ensure_nodata_in_style_query({"min": 0, "max": 2, "palette": ["#000", "#fff"]}) == { + "min": 0, + "max": 2, + "palette": ["#000", "#fff"], + "nodata": "auto", + } + assert ensure_nodata_in_style_query({"bands": [{"band": 1, "min": 0}]}) == { + "bands": [{"band": 1, "min": 0, "nodata": "auto"}], + } + assert ensure_nodata_in_style_query({"nodata": -9999}) == {"nodata": -9999} + + +def test_resolve_preview_style_json_native_single_band(): + assert resolve_preview_style_json({}, None, band_count=1) is not None + assert json.loads(resolve_preview_style_json({}, None, band_count=1))["nodata"] == "auto" + assert resolve_preview_style_json({}, None, band_count=3) is None + + +def test_resolve_preview_style_json_styled_adds_nodata(): + styled = json.loads( + resolve_preview_style_json( + {"min": 0, "max": 2, "palette": ["#002081", "#2AD3FF"]}, + None, + band_count=1, + ) + ) + assert styled["nodata"] == "auto" + assert styled["min"] == 0 + + +@pytest.mark.parametrize( + ("resolution_fraction", "raster_max_dimension", "expected_max_px"), + [ + (None, None, round(FRAME_PREVIEW_MAX_PX * FRAME_PREVIEW_DEFAULT_RESOLUTION_FRACTION)), + (0.25, None, 1024), + (0.5, None, 2048), + (FRAME_PREVIEW_DEFAULT_RESOLUTION_FRACTION, None, 512), + (None, 4096, FRAME_PREVIEW_MIN_PX), + (0.25, 4096, FRAME_PREVIEW_MIN_PX), + (0.5, 4096, 2048), + (None, 800, 800), + (0.5, 500, 500), + (0.75, None, 3072), + ], +) +def test_resolve_preview_max_dimension( + resolution_fraction, + raster_max_dimension, + expected_max_px, +): + assert ( + resolve_preview_max_dimension(resolution_fraction, raster_max_dimension) == expected_max_px + ) + + +@pytest.mark.django_db +def test_style_fingerprint_matches_db_after_ingest_style_setup(layer_style_factory): + """Enqueue fingerprint must match what the Celery task reads from the database.""" + style = layer_style_factory() + style.raster_style_params = {"palette": "#00ff00", "min": 0, "max": 1} + style.save(update_fields=["raster_style_params"]) + + assert style_fingerprint(style) == style_fingerprint(LayerStyle.objects.get(pk=style.pk)) + + +def test_style_fingerprint_stable_key_order_and_treats_null_as_empty(): + """Fingerprint uses sorted JSON keys; null params match empty params.""" + base = {"palette": "#fff", "min": 0, "max": 1} + reordered = {"max": 1, "palette": "#fff", "min": 0} + + assert _fingerprint_payload(base) == _fingerprint_payload(reordered) + assert _fingerprint_payload(None) == _fingerprint_payload({}) + assert params_fingerprint(None) == params_fingerprint({}) + + +@pytest.mark.django_db +def test_invalidate_and_enqueue_previews_uses_db_fingerprint( + layer_style_factory, + layer_frame_factory, + mocker, +): + layer_style = layer_style_factory() + layer_frame_factory(layer=layer_style.layer, index=0) + layer_frame_factory(layer=layer_style.layer, index=1) + + delay = _patch_preview_delay(mocker) + + invalidate_and_enqueue_previews(layer_style) + + delay.assert_called_once() + layer_id, fingerprint, params = delay.call_args.args[:3] + assert layer_id == layer_style.layer_id + assert fingerprint == style_fingerprint(LayerStyle.objects.get(pk=layer_style.pk)) + assert params == dict(layer_style.raster_style_params or {}) + assert delay.call_args.kwargs.get("layer_style_id") == layer_style.id + assert TaskResult.objects.filter(task_type="frame_preview").count() == 1 + + +@pytest.mark.django_db +def test_invalidate_and_enqueue_previews_skips_when_previews_current( + layer_style_factory, + layer_frame_factory, + mocker, +): + layer_style = layer_style_factory() + frame_0 = layer_frame_factory(layer=layer_style.layer, index=0) + frame_1 = layer_frame_factory(layer=layer_style.layer, index=1) + fingerprint = style_fingerprint(layer_style) + + preview_0 = _make_preview( + frame_0, + fingerprint=fingerprint, + width=100, + height=80, + bounds={"srs": "EPSG:4326", "xmin": -1, "xmax": 1, "ymin": -2, "ymax": 2}, + ) + preview_0.image.save("frame-0.png", ContentFile(b"png0"), save=True) + preview_1 = _make_preview( + frame_1, + fingerprint=fingerprint, + width=120, + height=90, + bounds={"srs": "EPSG:4326", "xmin": -2, "xmax": 2, "ymin": -3, "ymax": 3}, + ) + preview_1.image.save("frame-1.png", ContentFile(b"png1"), save=True) + + delay = _patch_preview_delay(mocker) + + result = invalidate_and_enqueue_previews(layer_style) + + delay.assert_not_called() + assert result is None + assert get_layer_style_preview_status(layer_style) == "ready" + + preview_0.refresh_from_db() + preview_1.refresh_from_db() + assert preview_0.status == PreviewStatus.COMPLETE + assert preview_1.status == PreviewStatus.COMPLETE + assert preview_0.image.name + assert preview_1.image.name + assert preview_0.width == 100 + assert preview_1.width == 120 + + +@pytest.mark.django_db +def test_invalidate_and_enqueue_previews_runs_when_fingerprint_changed( + layer_style_factory, + layer_frame_factory, + mocker, +): + layer_style = layer_style_factory() + frame_0 = layer_frame_factory(layer=layer_style.layer, index=0) + frame_1 = layer_frame_factory(layer=layer_style.layer, index=1) + + preview_0 = _make_preview( + frame_0, + fingerprint="stale-fingerprint", + width=100, + height=80, + bounds={}, + ) + preview_0.image.save("frame-0.png", ContentFile(b"png0"), save=True) + preview_1 = _make_preview( + frame_1, + fingerprint="stale-fingerprint", + width=120, + height=90, + bounds={}, + ) + preview_1.image.save("frame-1.png", ContentFile(b"png1"), save=True) + + delay = _patch_preview_delay(mocker) + + result = invalidate_and_enqueue_previews(layer_style) + + delay.assert_called_once() + assert result is not None + + new_fingerprint = style_fingerprint(layer_style) + # Stale fingerprint rows are left alone; new fingerprint rows are regenerating. + preview_0.refresh_from_db() + preview_1.refresh_from_db() + assert preview_0.style_fingerprint == "stale-fingerprint" + assert preview_0.status == PreviewStatus.COMPLETE + + new_previews = list( + RasterFramePreview.objects.filter( + layer_frame__layer=layer_style.layer, + style_fingerprint=new_fingerprint, + ) + ) + assert len(new_previews) == 2 + assert all(p.status == PreviewStatus.CREATING for p in new_previews) + assert all(not p.image for p in new_previews) + + +@pytest.mark.django_db +def test_invalidate_and_enqueue_previews_runs_synchronously( + layer_style_factory, + layer_frame_factory, + mocker, +): + layer_style = layer_style_factory() + layer_frame_factory(layer=layer_style.layer, index=0) + layer_frame_factory(layer=layer_style.layer, index=1) + + delay = _patch_preview_delay(mocker) + apply = mocker.patch("uvdat.core.tasks.frame_preview.generate_frame_previews.apply") + + invalidate_and_enqueue_previews(layer_style, run_mode=TaskRunMode.SYNC) + + apply.assert_called_once() + delay.assert_not_called() + assert apply.call_args.kwargs["args"][0] == layer_style.layer_id + assert apply.call_args.kwargs["kwargs"]["layer_style_id"] == layer_style.id + + +@pytest.mark.django_db +def test_multiframe_previews_for_style_returns_none_for_single_frame( + layer_style_factory, + layer_frame_factory, +): + layer_style = layer_style_factory() + layer_frame_factory(layer=layer_style.layer, index=0) + + assert layer_style.multiframe_previews() is None + + +@pytest.mark.django_db +def test_multiframe_previews_for_style_returns_none_when_partial( + layer_style_factory, + layer_frame_factory, +): + layer_style = layer_style_factory() + frame_0 = layer_frame_factory(layer=layer_style.layer, index=0) + layer_frame_factory(layer=layer_style.layer, index=1) + frame_2 = layer_frame_factory(layer=layer_style.layer, index=2) + fingerprint = style_fingerprint(layer_style) + + preview_0 = _make_preview( + frame_0, + fingerprint=fingerprint, + width=100, + height=80, + bounds={"srs": "EPSG:4326", "xmin": -1, "xmax": 1, "ymin": -2, "ymax": 2}, + ) + preview_0.image.save("frame-0.png", ContentFile(b"png0"), save=True) + preview_2 = _make_preview( + frame_2, + fingerprint=fingerprint, + width=200, + height=150, + bounds={"srs": "EPSG:4326", "xmin": -3, "xmax": 3, "ymin": -4, "ymax": 4}, + ) + preview_2.image.save("frame-2.png", ContentFile(b"png2"), save=True) + + assert layer_style.multiframe_previews() is None + + +@pytest.mark.django_db +def test_multiframe_previews_for_style_ordered_by_frame_index( + layer_style_factory, + layer_frame_factory, +): + layer_style = layer_style_factory() + frame_0 = layer_frame_factory(layer=layer_style.layer, index=0) + frame_1 = layer_frame_factory(layer=layer_style.layer, index=1) + frame_2 = layer_frame_factory(layer=layer_style.layer, index=2) + fingerprint = style_fingerprint(layer_style) + + preview_0 = _make_preview( + frame_0, + fingerprint=fingerprint, + width=100, + height=80, + bounds={"srs": "EPSG:4326", "xmin": -1, "xmax": 1, "ymin": -2, "ymax": 2}, + ) + preview_0.image.save("frame-0.png", ContentFile(b"png0"), save=True) + preview_1 = _make_preview( + frame_1, + fingerprint=fingerprint, + width=120, + height=90, + bounds={"srs": "EPSG:4326", "xmin": -2, "xmax": 2, "ymin": -3, "ymax": 3}, + ) + preview_1.image.save("frame-1.png", ContentFile(b"png1"), save=True) + preview_2 = _make_preview( + frame_2, + fingerprint=fingerprint, + width=200, + height=150, + bounds={"srs": "EPSG:4326", "xmin": -3, "xmax": 3, "ymin": -4, "ymax": 4}, + ) + preview_2.image.save("frame-2.png", ContentFile(b"png2"), save=True) + + previews = layer_style.multiframe_previews() + assert previews == [ + { + "url": preview_0.image.url, + "width": 100, + "height": 80, + "bounds": preview_0.bounds, + }, + { + "url": preview_1.image.url, + "width": 120, + "height": 90, + "bounds": preview_1.bounds, + }, + { + "url": preview_2.image.url, + "width": 200, + "height": 150, + "bounds": preview_2.bounds, + }, + ] + + +@pytest.mark.django_db +def test_preview_bounds_includes_corners( + layer_style_factory, + layer_frame_factory, +): + layer_style = layer_style_factory() + frame_0 = layer_frame_factory(layer=layer_style.layer, index=0) + frame_1 = layer_frame_factory(layer=layer_style.layer, index=1) + fingerprint = style_fingerprint(layer_style) + + preview_0 = _make_preview( + frame_0, + fingerprint=fingerprint, + width=100, + height=80, + bounds={ + "srs": "EPSG:4326", + "xmin": -1, + "xmax": 1, + "ymin": -2, + "ymax": 2, + "ul": {"x": -1, "y": 2}, + "ur": {"x": 1, "y": 2}, + "lr": {"x": 1, "y": -2}, + "ll": {"x": -1, "y": -2}, + }, + ) + preview_0.image.save("frame-0.png", ContentFile(b"png0"), save=True) + preview_1 = _make_preview( + frame_1, + fingerprint=fingerprint, + width=100, + height=80, + bounds={}, + ) + preview_1.image.save("frame-1.png", ContentFile(b"png1"), save=True) + + previews = layer_style.multiframe_previews() + assert previews[0]["bounds"]["ul"] == {"x": -1, "y": 2} + + +@pytest.mark.django_db +def test_layer_style_api_includes_multiframe_previews( + authenticated_api_client, + layer_style_factory, + layer_frame_factory, + project, + user, +): + layer_style = layer_style_factory() + project.set_collaborators([user]) + project.datasets.set([layer_style.layer.dataset]) + frame_0 = layer_frame_factory(layer=layer_style.layer, index=0) + frame_1 = layer_frame_factory(layer=layer_style.layer, index=1) + fingerprint = style_fingerprint(layer_style) + + preview_0 = _make_preview( + frame_0, + fingerprint=fingerprint, + width=100, + height=100, + bounds={}, + ) + preview_0.image.save("frame-0.png", ContentFile(b"png0"), save=True) + preview_1 = _make_preview( + frame_1, + fingerprint=fingerprint, + width=100, + height=100, + bounds={}, + ) + preview_1.image.save("frame-1.png", ContentFile(b"png1"), save=True) + + resp = authenticated_api_client.get(f"/api/v1/layer-styles/{layer_style.id}/") + assert resp.status_code == 200 + data = resp.json() + assert data["preview_status"] == "ready" + assert data["multiframe_previews"] == [ + { + "url": preview_0.image.url, + "width": 100, + "height": 100, + "bounds": {}, + }, + { + "url": preview_1.image.url, + "width": 100, + "height": 100, + "bounds": {}, + }, + ] + + +@pytest.mark.django_db +def test_layer_style_patch_reports_notready_after_preview_invalidation( + authenticated_api_client, + layer_style_factory, + layer_frame_factory, + project, + user, + mocker, +): + layer_style = layer_style_factory() + project.set_collaborators([user]) + project.datasets.set([layer_style.layer.dataset]) + frame_0 = layer_frame_factory(layer=layer_style.layer, index=0) + frame_1 = layer_frame_factory(layer=layer_style.layer, index=1) + layer_style.raster_style_params = {"palette": "#00ff00", "min": 0, "max": 1} + layer_style.save(update_fields=["raster_style_params"]) + fingerprint = style_fingerprint(layer_style) + + preview_0 = _make_preview( + frame_0, + fingerprint=fingerprint, + params=layer_style.raster_style_params, + width=100, + height=100, + bounds={}, + ) + preview_0.image.save("frame-0.png", ContentFile(b"png0"), save=True) + preview_1 = _make_preview( + frame_1, + fingerprint=fingerprint, + params=layer_style.raster_style_params, + width=100, + height=100, + bounds={}, + ) + preview_1.image.save("frame-1.png", ContentFile(b"png1"), save=True) + + _patch_preview_delay(mocker) + + resp = authenticated_api_client.patch( + f"/api/v1/layer-styles/{layer_style.id}/", + { + "name": layer_style.name, + "layer": layer_style.layer_id, + "project": layer_style.project_id, + "style_spec": _VECTOR_STYLE_SPEC, + "raster_style_params": {"palette": "#ff0000", "min": 0, "max": 1}, + }, + format="json", + ) + assert resp.status_code == 200 + data = resp.json() + assert data["preview_status"] == "notready" + assert "multiframe_previews" not in data + + +@pytest.mark.django_db +def test_api_omits_previews_while_not_ready( + authenticated_api_client, + layer_style_factory, + layer_frame_factory, + project, + user, +): + layer_style = layer_style_factory() + project.set_collaborators([user]) + project.datasets.set([layer_style.layer.dataset]) + frame_0 = layer_frame_factory(layer=layer_style.layer, index=0) + layer_frame_factory(layer=layer_style.layer, index=1) + fingerprint = style_fingerprint(layer_style) + + preview = _make_preview( + frame_0, + fingerprint=fingerprint, + width=100, + height=100, + bounds={}, + ) + preview.image.save("frame-0.png", ContentFile(b"png0"), save=True) + + resp = authenticated_api_client.get(f"/api/v1/layer-styles/{layer_style.id}/") + assert resp.status_code == 200 + data = resp.json() + assert data["preview_status"] == "notready" + assert "multiframe_previews" not in data + + +@pytest.mark.django_db +def test_layer_api_includes_multiframe_previews( + authenticated_api_client, + layer_style_factory, + layer_frame_factory, + project, + user, +): + layer_style = layer_style_factory() + layer = layer_style.layer + layer.default_style = layer_style + layer.save(update_fields=["default_style"]) + project.set_collaborators([user]) + project.datasets.set([layer.dataset]) + frame_0 = layer_frame_factory(layer=layer, index=0) + frame_1 = layer_frame_factory(layer=layer, index=1) + fingerprint = style_fingerprint(layer_style) + + preview_0 = _make_preview( + frame_0, + fingerprint=fingerprint, + width=100, + height=100, + bounds={}, + ) + preview_0.image.save("frame-0.png", ContentFile(b"png0"), save=True) + preview_1 = _make_preview( + frame_1, + fingerprint=fingerprint, + width=100, + height=100, + bounds={}, + ) + preview_1.image.save("frame-1.png", ContentFile(b"png1"), save=True) + + resp = authenticated_api_client.get(f"/api/v1/layers/{layer.id}/") + assert resp.status_code == 200 + data = resp.json() + assert data["preview_status"] == "ready" + assert data["multiframe_previews"] == [ + { + "url": preview_0.image.url, + "width": 100, + "height": 100, + "bounds": {}, + }, + { + "url": preview_1.image.url, + "width": 100, + "height": 100, + "bounds": {}, + }, + ] + assert "multiframe_previews" not in data.get("default_style", {}) + + +@pytest.mark.django_db +def test_layer_api_serves_default_previews_without_layer_style( + authenticated_api_client, + layer_factory, + layer_frame_factory, + project, + user, +): + """Empty-params previews are available on a layer with no LayerStyle.""" + layer = layer_factory() + project.set_collaborators([user]) + project.datasets.set([layer.dataset]) + frame_0 = layer_frame_factory(layer=layer, index=0) + frame_1 = layer_frame_factory(layer=layer, index=1) + fingerprint = params_fingerprint({}) + + preview_0 = _make_preview(frame_0, fingerprint=fingerprint, width=100, height=100, bounds={}) + preview_0.image.save("frame-0.png", ContentFile(b"png0"), save=True) + preview_1 = _make_preview(frame_1, fingerprint=fingerprint, width=100, height=100, bounds={}) + preview_1.image.save("frame-1.png", ContentFile(b"png1"), save=True) + + resp = authenticated_api_client.get(f"/api/v1/layers/{layer.id}/") + assert resp.status_code == 200 + data = resp.json() + assert data["default_style"] is None + assert data["preview_status"] == "ready" + assert len(data["multiframe_previews"]) == 2 + + +@pytest.mark.django_db +def test_styles_with_identical_params_share_preview_rows( + layer_style_factory, + layer_frame_factory, +): + style_a = layer_style_factory(name="A") + style_b = layer_style_factory(name="B", layer=style_a.layer, project=style_a.project) + style_a.raster_style_params = {"palette": "#00ff00", "min": 0, "max": 1} + style_b.raster_style_params = {"palette": "#00ff00", "min": 0, "max": 1} + style_a.save(update_fields=["raster_style_params"]) + style_b.save(update_fields=["raster_style_params"]) + + frame_0 = layer_frame_factory(layer=style_a.layer, index=0) + frame_1 = layer_frame_factory(layer=style_a.layer, index=1) + fingerprint = style_fingerprint(style_a) + assert fingerprint == style_fingerprint(style_b) + + for frame in (frame_0, frame_1): + preview = _make_preview( + frame, + fingerprint=fingerprint, + params=style_a.raster_style_params, + width=50, + height=50, + bounds={}, + ) + preview.image.save(f"frame-{frame.index}.png", ContentFile(b"png"), save=True) + + assert style_a.multiframe_previews() is not None + assert style_b.multiframe_previews() is not None + assert RasterFramePreview.objects.filter(style_fingerprint=fingerprint).count() == 2 + + +@pytest.mark.django_db +def test_generate_frame_previews_continues_when_style_deleted( + layer_style_factory, + layer_frame_factory, + mocker, +): + """Deleted styles still finish from the enqueued params snapshot.""" + layer_style = layer_style_factory() + layer_frame_factory(layer=layer_style.layer, index=0) + layer_frame_factory(layer=layer_style.layer, index=1) + params = {"palette": "#00ff00", "min": 0, "max": 1} + layer_style.raster_style_params = params + layer_style.save(update_fields=["raster_style_params"]) + fingerprint = style_fingerprint(layer_style) + result = TaskResult.objects.create( + name="Frame previews", + task_type="frame_preview", + project=layer_style.project, + inputs={ + "layer_id": layer_style.layer_id, + "fingerprint": fingerprint, + "layer_style_id": layer_style.id, + }, + ) + style_id = layer_style.id + layer_id = layer_style.layer_id + for frame in layer_style.layer.raster_frames(): + _make_preview( + frame, + fingerprint=fingerprint, + params=params, + status=PreviewStatus.CREATING, + ) + layer_style.delete() + + mocker.patch( + "uvdat.core.tasks.frame_preview.generate_frame_preview_png", + return_value=(b"png", 10, 10, None), + ) + + generate_frame_previews(layer_id, fingerprint, params, result.id, layer_style_id=style_id) + + result.refresh_from_db() + assert result.completed is not None + assert result.outputs["ready_count"] == 2 + assert ( + RasterFramePreview.objects.filter( + layer_frame__layer_id=layer_id, + style_fingerprint=fingerprint, + status=PreviewStatus.COMPLETE, + ).count() + == 2 + ) + + +@pytest.mark.django_db +def test_invalidate_and_enqueue_skips_when_fingerprint_job_in_flight( + layer_style_factory, + layer_frame_factory, + mocker, +): + layer_style = layer_style_factory() + layer_frame_factory(layer=layer_style.layer, index=0) + layer_frame_factory(layer=layer_style.layer, index=1) + delay = _patch_preview_delay(mocker) + + first = invalidate_and_enqueue_previews(layer_style) + second = invalidate_and_enqueue_previews(layer_style) + + delay.assert_called_once() + assert first is not None + assert second is not None + assert first.id == second.id + assert TaskResult.objects.filter(task_type="frame_preview").count() == 1 + + +@pytest.mark.django_db +def test_invalidate_and_enqueue_starts_job_for_different_fingerprint( + layer_style_factory, + layer_frame_factory, + mocker, +): + layer_style = layer_style_factory() + layer_frame_factory(layer=layer_style.layer, index=0) + layer_frame_factory(layer=layer_style.layer, index=1) + delay = _patch_preview_delay(mocker) + + first = invalidate_and_enqueue_layer_previews(layer_style.layer, {}) + layer_style.raster_style_params = {"palette": "#00ff00", "min": 0, "max": 1} + layer_style.save(update_fields=["raster_style_params"]) + second = invalidate_and_enqueue_previews(layer_style) + + assert delay.call_count == 2 + assert first is not None + assert second is not None + assert first.id != second.id + assert first.completed is None + assert second.completed is None + + +def test_flood_simulation_enqueues_style_previews_after_params(): + """Regression: flood default style params must trigger preview generation.""" + source = inspect.getsource(flood_mod.flood_simulation) + assert "invalidate_and_enqueue_previews(style, run_mode=TaskRunMode.SYNC)" in source + assert "raster_style_params" in source diff --git a/web/src/api/rest.ts b/web/src/api/rest.ts index 395108872..3185c78c6 100644 --- a/web/src/api/rest.ts +++ b/web/src/api/rest.ts @@ -267,6 +267,10 @@ export async function getLayerStyles(layerId: number): Promise { return (await apiClient.get(`layer-styles/?layer=${layerId}`)).data.results; } +export async function getLayerStyle(styleId: number): Promise { + return (await apiClient.get(`layer-styles/${styleId}/`)).data; +} + export async function createLayerStyle(data: LayerStyle): Promise { return (await apiClient.post("layer-styles/", data)).data; } diff --git a/web/src/components/sidebars/CompareLayerStyle.vue b/web/src/components/sidebars/CompareLayerStyle.vue index 5b88935db..cedc2b200 100644 --- a/web/src/components/sidebars/CompareLayerStyle.vue +++ b/web/src/components/sidebars/CompareLayerStyle.vue @@ -31,9 +31,7 @@ const currentStyleSpecs = ref<{ B: StyleSpec | undefined; }>({ A: undefined, B: undefined }); -const styleKey = computed(() => { - return `${props.layer.id}.${props.layer.copy_id}`; -}); +const styleKey = computed(() => styleStore.layerStyleKey(props.layer)); const currentLayerStyles = computed(() => { let A = compareStore.compareLayerStyles.A[styleKey.value]; diff --git a/web/src/components/sidebars/LayerStyle.vue b/web/src/components/sidebars/LayerStyle.vue index 16c458519..e2dbeedba 100644 --- a/web/src/components/sidebars/LayerStyle.vue +++ b/web/src/components/sidebars/LayerStyle.vue @@ -27,12 +27,14 @@ import { usePanelStore, useLayerStore, useAppStore, + useFramePreviewStore, } from "@/store"; const styleStore = useStyleStore(); const projectStore = useProjectStore(); const panelStore = usePanelStore(); const layerStore = useLayerStore(); const appStore = useAppStore(); +const framePreviewStore = useFramePreviewStore(); const emit = defineEmits(["setLayerActive"]); const props = defineProps<{ @@ -72,9 +74,7 @@ const editMode = computed(() => { ); }); -const styleKey = computed(() => { - return `${props.layer.id}.${props.layer.copy_id}`; -}); +const styleKey = computed(() => styleStore.layerStyleKey(props.layer)); const currentLayerStyle = computed(() => { return styleStore.selectedLayerStyles[styleKey.value]; @@ -84,6 +84,25 @@ const setCurrentLayerStyle = (style: LayerStyle) => { styleStore.selectedLayerStyles[styleKey.value] = style; }; +// After a style save, apply the API response. When previews were invalidated, +// clear stale payloads/overlays; onPreviewTaskComplete reloads them when ready. +const markStyleSavedAndInvalidatePreviews = (style: LayerStyle) => { + const previewsStillValid = + style.preview_status === "ready" && !!style.multiframe_previews?.length; + setCurrentLayerStyle( + previewsStillValid + ? cloneDeep(style) + : { + ...style, + preview_status: style.preview_status ?? "notready", + multiframe_previews: undefined, + }, + ); + if (!previewsStillValid) { + framePreviewStore.clearPreviewsForStyleChange(props.layer, style.id); + } +}; + const appliedStyleName = computed(() => { if (currentLayerStyle.value.id) return currentLayerStyle.value.name; else return undefined; @@ -154,7 +173,7 @@ async function init() { if (currentStyleSpec.value) setAvailableGroups(); } -function applyNoneStyle() { +function applyNoneStyle(resetPreviews = false) { const styleSpec = styleStore.getDefaultStyleSpec( currentFrame.value?.raster, props.layer.id, @@ -163,16 +182,32 @@ function applyNoneStyle() { name: "None", is_default: true, style_spec: cloneDeep(styleSpec), + preview_status: props.layer.preview_status, + ...(props.layer.preview_status === "ready" && + props.layer.multiframe_previews + ? { multiframe_previews: props.layer.multiframe_previews } + : {}), }); currentStyleSpec.value = styleSpec; + if (resetPreviews) { + framePreviewStore.prepareForStylePreviewReset(props.layer); + styleStore.updateLayerStyles(props.layer); + } } -function applyStyleSelection(style: LayerStyle | undefined) { +function applyStyleSelection( + style: LayerStyle | undefined, + resetPreviews = false, +) { if (style?.id !== undefined && style.style_spec) { setCurrentLayerStyle(cloneDeep(style)); currentStyleSpec.value = cloneDeep(style.style_spec); + if (resetPreviews) { + framePreviewStore.prepareForStylePreviewReset(props.layer); + styleStore.updateLayerStyles(props.layer); + } } else { - applyNoneStyle(); + applyNoneStyle(resetPreviews); } } @@ -207,7 +242,7 @@ function resetCurrentStyle() { function selectStyle(style: LayerStyle) { if (style?.id === undefined) { - applyNoneStyle(); + applyNoneStyle(true); currentGroups.value = { color: undefined, size: undefined }; return; } @@ -217,7 +252,7 @@ function selectStyle(style: LayerStyle) { ) { return; } - applyStyleSelection(style); + applyStyleSelection(style, true); currentGroups.value = { color: undefined, size: undefined }; } @@ -390,13 +425,13 @@ function confirmDeleteColormap() { ); // update other styles in case colormap changed to default layerStore.selectedLayers.forEach((layer) => { - const layerStyleKey = `${layer.id}.${layer.copy_id}`; + const key = styleStore.layerStyleKey(layer); getLayerStyles(layer.id).then((styles) => { const updated = styles.find( - (s) => s.id === styleStore.selectedLayerStyles[layerStyleKey].id, + (s) => s.id === styleStore.selectedLayerStyles[key].id, ); if (updated) { - styleStore.selectedLayerStyles[layerStyleKey] = updated; + styleStore.selectedLayerStyles[key] = updated; if (layer.id === props.layer.id) { availableStyles.value = styles; currentStyleSpec.value = updated.style_spec; @@ -542,9 +577,15 @@ function save() { name: newName.value || currentLayerStyle.value.name, is_default: currentLayerStyle.value.is_default, style_spec: currentStyleSpec.value, + raster_style_params: showRasterOptions.value + ? styleStore.getRasterTilesQuery( + currentStyleSpec.value, + styleStore.colormaps, + ) + : null, }).then((style) => { if (style) { - setCurrentLayerStyle(style); + markStyleSavedAndInvalidatePreviews(style); newName.value = undefined; newNameMode.value = undefined; // update other styles in case default overriden @@ -571,9 +612,15 @@ function saveAsNew() { layer: props.layer.id, project: projectStore.currentProject.id, style_spec: currentStyleSpec.value, + raster_style_params: showRasterOptions.value + ? styleStore.getRasterTilesQuery( + currentStyleSpec.value, + styleStore.colormaps, + ) + : null, }).then((style: LayerStyle) => { if (style) { - setCurrentLayerStyle(style); + markStyleSavedAndInvalidatePreviews(style); newName.value = undefined; newNameMode.value = undefined; // update other styles in case default overriden @@ -588,13 +635,17 @@ function saveAsNew() { function deleteStyle() { if (!editMode.value || !currentLayerStyle.value?.id) return; - deleteLayerStyle(currentLayerStyle.value.id).then(() => { - getLayerStyles(props.layer.id).then((styles) => { - availableStyles.value = styles; - applyStyleSelection(styles.find((style) => style.is_default)); - refreshLayer(); - showDeleteConfirmation.value = false; - }); + deleteLayerStyle(currentLayerStyle.value.id).then(async () => { + const [styles] = await Promise.all([ + getLayerStyles(props.layer.id), + layerStore.fetchAvailableLayer(props.layer.id), + ]); + availableStyles.value = styles; + applyStyleSelection( + styles.find((style) => style.is_default), + true, + ); + showDeleteConfirmation.value = false; }); } @@ -625,19 +676,48 @@ watch( ); const debouncedStyleSpecUpdated = debounce(() => { - if (currentStyleSpec.value) { - styleStore.selectedLayerStyles[styleKey.value] = { - ...currentLayerStyle.value, + if (!currentStyleSpec.value) return; + + const prev = currentLayerStyle.value; + const hadPreviews = + prev?.preview_status === "ready" || !!prev?.multiframe_previews?.length; + let clearPreviews = false; + if (hadPreviews && prev?.style_spec && showRasterOptions.value) { + const prevQuery = JSON.stringify( + styleStore.getRasterTilesQuery(prev.style_spec, styleStore.colormaps), + ); + const nextQuery = JSON.stringify( + styleStore.getRasterTilesQuery( + currentStyleSpec.value, + styleStore.colormaps, + ), + ); + clearPreviews = prevQuery !== nextQuery; + } + + if (clearPreviews) { + framePreviewStore.clearPreviewsForStyleChange(props.layer, prev.id); + } else { + styleStore.patchSelectedLayerStyle(styleKey.value, { style_spec: currentStyleSpec.value, - }; - styleStore.updateLayerStyles(props.layer); - setAvailableGroups(); - unsavedChanges.value = true; + }); } + styleStore.updateLayerStyles(props.layer); + setAvailableGroups(); + unsavedChanges.value = true; }, 100); watch(currentStyleSpec, debouncedStyleSpecUpdated, { deep: true }); watch(() => props.activeLayer, init); + +watch( + () => props.activeLayer === props.layer, + (isEditing) => { + styleStore.setLayerStyleEditing(props.layer, isEditing); + }, + { immediate: true }, +); + onMounted(resetCurrentStyle); diff --git a/web/src/components/sidebars/LayersPanel.vue b/web/src/components/sidebars/LayersPanel.vue index e7f8362d1..32b371678 100644 --- a/web/src/components/sidebars/LayersPanel.vue +++ b/web/src/components/sidebars/LayersPanel.vue @@ -8,10 +8,11 @@ import CompareLayerStyle from "./CompareLayerStyle.vue"; import DetailView from "../DetailView.vue"; import SliderNumericInput from "../SliderNumericInput.vue"; -import { useLayerStore, useMapStore } from "@/store"; +import { useLayerStore, useMapStore, useFramePreviewStore } from "@/store"; import { useMapCompareStore } from "@/store/compare"; const layerStore = useLayerStore(); const mapStore = useMapStore(); +const framePreviewStore = useFramePreviewStore(); const compareStore = useMapCompareStore(); const isComparing = computed(() => compareStore.isComparing); const orientation = computed(() => compareStore.orientation); @@ -60,12 +61,16 @@ function removeLayers(layers: Layer[]) { } const debouncedUpdateFrame = debounce((layer: Layer, value: number) => { - layerStore.selectedLayers = layerStore.selectedLayers.map((l: Layer) => { - if (l.id === layer.id && l.copy_id === layer.copy_id) { - l.current_frame_index = value; - } - return l; - }); + const target = layerStore.selectedLayers.find( + (l: Layer) => l.id === layer.id && l.copy_id === layer.copy_id, + ); + if (!target || target.current_frame_index === value) { + return; + } + target.current_frame_index = value; + // Update only this layer — avoid re-running updateLayerStyles (and setTiles) + // on every other selected multiframe layer on each scrub step. + layerStore.updateLayerFrame(target); }, 10); function getLayerMaxFrames(layer: Layer) { @@ -191,6 +196,14 @@ function setLayerActive(layer: Layer, active: boolean) { {{ element.name }} + + diff --git a/web/src/store/analysis.ts b/web/src/store/analysis.ts index 12854c7b0..7cd2f9495 100644 --- a/web/src/store/analysis.ts +++ b/web/src/store/analysis.ts @@ -7,6 +7,7 @@ import { import type { Chart, AnalysisType, TaskResult } from "@/types"; import { defineStore } from "pinia"; import { ref, watch } from "vue"; +import { useFramePreviewStore } from "./framePreview"; import { useProjectStore, useMapStore } from "."; import { TerraDraw, TerraDrawPolygonMode } from "terra-draw"; import { TerraDrawMapLibreGLAdapter } from "terra-draw-maplibre-gl-adapter"; @@ -172,6 +173,10 @@ export const useAnalysisStore = defineStore("analysis", () => { ws.value = new WebSocket(url); ws.value.onmessage = (event: any) => { const data = JSON.parse(JSON.parse(event.data)); + if (data.task_type === "frame_preview" && data.completed) { + // Regenerated previews are ready; reload and reattach them to layers. + useFramePreviewStore().onPreviewTaskComplete(data); + } if (currentResult.value && data.id === currentResult.value.id) { // only overwrite attributes expecting updates // overwriting the whole currentResult object will cause diff --git a/web/src/store/conversion.ts b/web/src/store/conversion.ts index 7c8950875..4a06f6dad 100644 --- a/web/src/store/conversion.ts +++ b/web/src/store/conversion.ts @@ -1,6 +1,7 @@ import type { TaskResult } from "@/types"; import { defineStore } from "pinia"; import { ref } from "vue"; +import { useFramePreviewStore } from "./framePreview"; import { useProjectStore } from "./project"; const url = `${import.meta.env.VITE_API_ROOT}ws/conversion/`; @@ -15,7 +16,12 @@ export const useConversionStore = defineStore("conversion", () => { ws.value = new WebSocket(url); ws.value.onmessage = (event: any) => { const result = JSON.parse(JSON.parse(event.data)) as TaskResult; - const datasetId = result.inputs.dataset_id; + // Default frame-preview generation during dataset conversion has no + // project, so it arrives on this channel rather than analytics_*. + if (result.task_type === "frame_preview" && result.completed) { + useFramePreviewStore().onPreviewTaskComplete(result); + } + const datasetId = result.inputs?.dataset_id; if (datasetId !== undefined) { datasetConversionTasks.value[datasetId] = result; if (result.completed) projectStore.fetchProjectDatasets(); diff --git a/web/src/store/framePreview.ts b/web/src/store/framePreview.ts new file mode 100644 index 000000000..e854eb23d --- /dev/null +++ b/web/src/store/framePreview.ts @@ -0,0 +1,948 @@ +import { defineStore } from "pinia"; +import { ref } from "vue"; +import type { + FramePreview, + Layer, + LayerFrame, + LayerStyle, + PreviewStatus, + TaskResult, +} from "@/types"; +import type { Map as MaplibreMap } from "maplibre-gl"; +import { getLayerStyle } from "@/api/rest"; +import { + fadeRasterOpacities, + hidePreviewLayer, + isPreviewMapLayerId, + PREVIEW_FADE_DURATION_MS, + previewLayerId, + removeAllPreviewLayersForLayerKey, + removePreviewLayer, + removePreviewLayersExcept, + upsertPreviewLayer, + waitForRasterSourceLoaded, +} from "@/utils/framePreviewLayer"; +import { prefetchFramePreviewUrls } from "@/utils/framePreviewCache"; +import { useLayerStore } from "./layer"; +import { useMapStore } from "./map"; +import { useStyleStore } from "./style"; + +function orderedRasterFrames(frames: LayerFrame[]) { + return frames + .filter((frame) => frame.raster) + .toSorted((a, b) => a.index - b.index); +} + +// Previews are only safe to display once the backend reports ready (every +// frame has a complete image). After a style save the API returns "notready" +// and omits multiframe_previews until regeneration finishes. +// +// Layer payloads carry the default-fingerprint preview set (empty params from +// conversion, or the layer's default_style params). Style payloads carry +// previews for that style's params. Use layer-level data when the active style +// is the default / unset synthetic style. +function usesLayerDefaultPreviews( + layer: Layer, + style: LayerStyle | undefined, +): boolean { + if (style == null || style.id === undefined) { + return true; + } + if (layer.default_style?.id != null) { + return style.id === layer.default_style.id; + } + // No DB default style: treat is_default / synthetic default as the layer default. + return style.is_default === true; +} + +type PreviewPayload = { + preview_status?: PreviewStatus; + multiframe_previews?: (FramePreview | null)[]; +}; + +function previewsAreReady( + layer: Layer, + style: LayerStyle | undefined, +): boolean { + if (style?.preview_status !== undefined) { + return style.preview_status === "ready"; + } + if ( + usesLayerDefaultPreviews(layer, style) && + layer.preview_status !== undefined + ) { + return layer.preview_status === "ready"; + } + // Backward compatibility for payloads that omit preview_status entirely. + return true; +} + +function previewsForLayer(layer: Layer, style: LayerStyle | undefined) { + if (!previewsAreReady(layer, style)) { + return undefined; + } + if (style?.multiframe_previews?.length) { + return style.multiframe_previews; + } + if ( + usesLayerDefaultPreviews(layer, style) && + layer.multiframe_previews?.length + ) { + return layer.multiframe_previews; + } + return undefined; +} + +function previewAtFrameIndex( + previews: (FramePreview | null)[] | undefined, + rasterFrames: LayerFrame[], + frameIndex: number, +): FramePreview | undefined { + const position = rasterFrames.findIndex( + (frame) => frame.index === frameIndex, + ); + if (position < 0) { + return undefined; + } + return previews?.[position] ?? undefined; +} + +function adjacentRasterFrames( + rasterFrames: LayerFrame[], + currentFrameIndex: number, +) { + const position = rasterFrames.findIndex( + (frame) => frame.index === currentFrameIndex, + ); + if (position < 0) { + return []; + } + return [rasterFrames[position - 1], rasterFrames[position + 1]].filter( + (frame): frame is LayerFrame => frame !== undefined, + ); +} + +// Wait for scrubbing to pause before attaching/fetching raster tiles. Previews +// stay immediate so the slider feels responsive on large multiframe rasters. +const TILE_LOAD_SETTLE_MS = 1000; + +export const useFramePreviewStore = defineStore("framePreview", () => { + const layerStore = useLayerStore(); + const mapStore = useMapStore(); + const styleStore = useStyleStore(); + + // Style/layer ids with preview regeneration in flight (drives the icon). + const generatingPreviewStyleIds = ref>(new Set()); + const generatingPreviewLayerIds = ref>(new Set()); + + function markPreviewGenerating(styleId?: number, layerId?: number) { + if (styleId !== undefined) { + generatingPreviewStyleIds.value = new Set([ + ...generatingPreviewStyleIds.value, + styleId, + ]); + } + if (layerId !== undefined) { + generatingPreviewLayerIds.value = new Set([ + ...generatingPreviewLayerIds.value, + layerId, + ]); + } + } + + function markPreviewReady(styleId?: number, layerId?: number) { + if (styleId !== undefined) { + const next = new Set(generatingPreviewStyleIds.value); + next.delete(styleId); + generatingPreviewStyleIds.value = next; + } + if (layerId !== undefined) { + const next = new Set(generatingPreviewLayerIds.value); + next.delete(layerId); + generatingPreviewLayerIds.value = next; + } + } + + function attachPreviewsForLayer(layer: Layer) { + if (styleStore.isLayerStyleEditing(layer)) { + return; + } + const style = + styleStore.selectedLayerStyles[styleStore.layerStyleKey(layer)]; + if (!previewsAreReady(layer, style)) { + return; + } + prepareForStylePreviewReset(layer); + void showPreviewThenTiles(layer); + } + + function applyStylePreviewToMatchingLayers( + layerId: number, + layerStyleId: number, + payload: PreviewPayload, + ) { + layerStore.selectedLayers.forEach((layer) => { + if (layer.id !== layerId) { + return; + } + const key = styleStore.layerStyleKey(layer); + const selectedStyle = styleStore.selectedLayerStyles[key]; + if (!selectedStyle || selectedStyle.id !== layerStyleId) { + return; + } + + styleStore.patchSelectedLayerStyle(key, payload); + if ( + selectedStyle.is_default || + usesLayerDefaultPreviews(layer, selectedStyle) + ) { + layer.preview_status = payload.preview_status; + layer.multiframe_previews = payload.multiframe_previews; + } + + prefetchLayerPreviews(layer, styleStore.selectedLayerStyles[key]); + if (!styleStore.isLayerStyleEditing(layer)) { + prepareForStylePreviewReset(layer); + void showPreviewThenTiles(layer); + } + }); + } + + function applyDefaultPreviewToMatchingLayers( + layerId: number, + payload: PreviewPayload, + ) { + layerStore.selectedLayers.forEach((layer) => { + if (layer.id !== layerId) { + return; + } + layer.preview_status = payload.preview_status; + layer.multiframe_previews = payload.multiframe_previews; + + const key = styleStore.layerStyleKey(layer); + const selectedStyle = styleStore.selectedLayerStyles[key]; + if (!usesLayerDefaultPreviews(layer, selectedStyle)) { + return; + } + + styleStore.patchSelectedLayerStyle( + key, + payload, + selectedStyle ?? { name: "None", is_default: true }, + ); + prefetchLayerPreviews(layer, styleStore.selectedLayerStyles[key]); + if (!styleStore.isLayerStyleEditing(layer)) { + prepareForStylePreviewReset(layer); + void showPreviewThenTiles(layer); + } + }); + } + + const activePreviewByLayerKey = new Map(); + const transitionGenerationByLayerKey = new Map(); + const tileLoadTimerByLayerKey = new Map< + string, + ReturnType + >(); + + // Reactive set of layer keys whose preview overlay is currently visible on + // the map (i.e. the user is looking at a preview image, not the real tiles). + // Used to drive UI indicators in the layers and legend panels. + const displayingPreviewLayerKeys = ref>(new Set()); + + function markPreviewDisplayed(layerKeyValue: string) { + if (!displayingPreviewLayerKeys.value.has(layerKeyValue)) { + const next = new Set(displayingPreviewLayerKeys.value); + next.add(layerKeyValue); + displayingPreviewLayerKeys.value = next; + } + } + + function clearPreviewDisplayed(layerKeyValue: string) { + if (displayingPreviewLayerKeys.value.has(layerKeyValue)) { + const next = new Set(displayingPreviewLayerKeys.value); + next.delete(layerKeyValue); + displayingPreviewLayerKeys.value = next; + } + } + + function isDisplayingPreview(layer: Layer) { + return displayingPreviewLayerKeys.value.has( + styleStore.layerStyleKey(layer), + ); + } + + /** True while scrubbing with preview overlays — skip raster tile URL refreshes. */ + function shouldDeferRasterTileUpdates(layer: Layer) { + const layerKeyValue = styleStore.layerStyleKey(layer); + return ( + isDisplayingPreview(layer) || tileLoadTimerByLayerKey.has(layerKeyValue) + ); + } + + // True when this multiframe layer's previews are explicitly not ready + // (missing / generating / regenerating). Omitted preview_status is treated + // as ready for backward compatibility, matching previewsAreReady(). + function isGeneratingPreviews(layer: Layer) { + const rasterFrames = orderedRasterFrames(layerStore.layerFrames(layer)); + if (rasterFrames.length <= 1) { + return false; + } + const style = + styleStore.selectedLayerStyles[styleStore.layerStyleKey(layer)]; + if (style?.id != null && generatingPreviewStyleIds.value.has(style.id)) { + return true; + } + if ( + usesLayerDefaultPreviews(layer, style) && + generatingPreviewLayerIds.value.has(layer.id) + ) { + return true; + } + if (style?.preview_status !== undefined) { + return style.preview_status === "notready"; + } + if ( + usesLayerDefaultPreviews(layer, style) && + layer.preview_status !== undefined + ) { + return layer.preview_status === "notready"; + } + return false; + } + + function iconState(layer: Layer) { + if (isDisplayingPreview(layer)) { + return { + visible: true, + tooltip: + "Showing a low-resolution preview while default resolution tiles load.", + color: "primary" as const, + class: { + "preview-indicator--generating": false, + "preview-indicator--hidden": false, + }, + }; + } + if (isGeneratingPreviews(layer)) { + return { + visible: true, + tooltip: "Frame previews are being created.", + color: undefined, + class: { + "preview-indicator--generating": true, + "preview-indicator--hidden": false, + }, + }; + } + return { + visible: false, + tooltip: undefined, + color: undefined, + class: { + "preview-indicator--generating": false, + "preview-indicator--hidden": true, + }, + }; + } + + function clearTileLoadTimer(layerKeyValue: string) { + const timer = tileLoadTimerByLayerKey.get(layerKeyValue); + if (timer !== undefined) { + clearTimeout(timer); + tileLoadTimerByLayerKey.delete(layerKeyValue); + } + } + + function bumpGeneration(layerKeyValue: string) { + clearTileLoadTimer(layerKeyValue); + const next = (transitionGenerationByLayerKey.get(layerKeyValue) ?? 0) + 1; + transitionGenerationByLayerKey.set(layerKeyValue, next); + return next; + } + + function scheduleTileLoadAfterSettle( + layerKeyValue: string, + generation: number, + loadTiles: () => void, + ) { + clearTileLoadTimer(layerKeyValue); + const timer = setTimeout(() => { + tileLoadTimerByLayerKey.delete(layerKeyValue); + if (transitionGenerationByLayerKey.get(layerKeyValue) !== generation) { + return; + } + loadTiles(); + }, TILE_LOAD_SETTLE_MS); + tileLoadTimerByLayerKey.set(layerKeyValue, timer); + } + + /** + * Restore map layer z-order to match the layers panel. + * + * Walk selectedLayers bottom-to-top. For each layer, move its tile layers + * up, then its preview overlays above those tiles. Lower panel rows are + * processed first so higher rows end up on top. + */ + function reorderPreviewLayers() { + if (!mapStore.map) { + return; + } + const map = mapStore.getMap(); + const userMapLayers = mapStore.getUserMapLayers(); + + layerStore.selectedLayers.toReversed().forEach((layer) => { + const layerKeyValue = styleStore.layerStyleKey(layer); + + layerStore.layerFrames(layer).forEach((frame) => { + const sourceId = mapStore.sourceIdFromLayerFrame(layer, frame); + userMapLayers.forEach((mapLayerId) => { + if (isPreviewMapLayerId(mapLayerId)) { + return; + } + if (mapLayerId.includes(sourceId) && map.getLayer(mapLayerId)) { + map.moveLayer(mapLayerId); + } + }); + }); + + const previewPrefix = `${layerKeyValue}.preview.`; + const activeFrameIndex = activePreviewByLayerKey.get(layerKeyValue); + userMapLayers.forEach((mapLayerId) => { + if (!mapLayerId.startsWith(previewPrefix)) { + return; + } + if ( + activeFrameIndex !== undefined && + mapLayerId === previewLayerId(layerKeyValue, activeFrameIndex) + ) { + return; + } + if (map.getLayer(mapLayerId)) { + map.moveLayer(mapLayerId); + } + }); + if (activeFrameIndex !== undefined) { + const activePreviewMapLayerId = previewLayerId( + layerKeyValue, + activeFrameIndex, + ); + if (map.getLayer(activePreviewMapLayerId)) { + map.moveLayer(activePreviewMapLayerId); + } + } + }); + } + + function prefetchLayerPreviews(layer: Layer, style?: LayerStyle) { + const previews = previewsForLayer(layer, style); + if (!previews?.length) { + return; + } + prefetchFramePreviewUrls(previews.map((preview) => preview?.url)); + } + + function hasReadyPreviewForCurrentFrame(layer: Layer) { + if (styleStore.isLayerStyleEditing(layer)) { + return false; + } + const rasterFrames = orderedRasterFrames(layerStore.layerFrames(layer)); + if (rasterFrames.length <= 1) { + return false; + } + const style = + styleStore.selectedLayerStyles[styleStore.layerStyleKey(layer)]; + return !!previewAtFrameIndex( + previewsForLayer(layer, style), + rasterFrames, + layer.current_frame_index, + ); + } + + function ensureRasterTilesOnMap( + frame: LayerFrame, + tileSourceId: string, + tileLayerId: string, + opacity: number, + ) { + const map = mapStore.getMap(); + if (!map.getLayer(tileLayerId)) { + mapStore.addLayerFrameToMap(frame, tileSourceId, true, { + rasterOpacity: opacity, + }); + return; + } + map.setPaintProperty(tileLayerId, "raster-opacity", opacity); + } + + async function preloadAdjacentPreviewLayers( + map: MaplibreMap, + layerKeyValue: string, + previews: (FramePreview | null)[] | undefined, + rasterFrames: LayerFrame[], + currentFrameIndex: number, + targetOpacity: number, + ) { + const adjacentFrames = adjacentRasterFrames( + rasterFrames, + currentFrameIndex, + ); + const keepFrameIndices = [ + currentFrameIndex, + ...adjacentFrames.map((frame) => frame.index), + ]; + + removePreviewLayersExcept(map, layerKeyValue, keepFrameIndices); + + const urlsToPrefetch: (string | null | undefined)[] = []; + await Promise.all( + adjacentFrames.map(async (frame) => { + const preview = previewAtFrameIndex( + previews, + rasterFrames, + frame.index, + ); + if (!preview || !frame.raster) { + return; + } + urlsToPrefetch.push(preview.url); + await upsertPreviewLayer( + map, + layerKeyValue, + frame.index, + preview, + frame.raster.metadata, + targetOpacity, + false, + ); + }), + ); + + const currentPreview = previewAtFrameIndex( + previews, + rasterFrames, + currentFrameIndex, + ); + urlsToPrefetch.push(currentPreview?.url); + prefetchFramePreviewUrls(urlsToPrefetch); + } + + function hidePreviousPreview( + map: MaplibreMap, + layerKeyValue: string, + nextFrameIndex: number, + ) { + const previousFrameIndex = activePreviewByLayerKey.get(layerKeyValue); + if ( + previousFrameIndex !== undefined && + previousFrameIndex !== nextFrameIndex + ) { + hidePreviewLayer(map, layerKeyValue, previousFrameIndex); + } + } + + function isPreviewLayerVisible( + map: MaplibreMap, + mapLayerId: string, + ): boolean { + if (!map.getLayer(mapLayerId)) { + return false; + } + return map.getLayoutProperty(mapLayerId, "visibility") === "visible"; + } + + /** Skip preview work when this layer is already settled for the current frame. */ + function previewAlreadySettled( + map: MaplibreMap, + layerKeyValue: string, + settledFrameIndex: number, + tileLayerId: string, + previewMapLayerId: string, + targetOpacity: number, + ): boolean { + const activeFrameIndex = activePreviewByLayerKey.get(layerKeyValue); + if (activeFrameIndex === settledFrameIndex) { + return isPreviewLayerVisible(map, previewMapLayerId); + } + + if (activeFrameIndex !== undefined) { + return false; + } + + // Preview transition finished; tiles are showing at target opacity. + if (map.getLayer(previewMapLayerId)) { + return false; + } + if (!map.getLayer(tileLayerId)) { + return false; + } + const tileOpacity = + (map.getPaintProperty(tileLayerId, "raster-opacity") as number) ?? 0; + return tileOpacity >= targetOpacity - 0.01; + } + + async function transitionToTiles( + layer: Layer, + frameIndex: number, + layerKeyValue: string, + generation: number, + targetOpacity: number, + tileLayerId: string, + tileSourceId: string, + ) { + const map = mapStore.getMap(); + const previewMapLayerId = previewLayerId(layerKeyValue, frameIndex); + + if (transitionGenerationByLayerKey.get(layerKeyValue) !== generation) { + return; + } + + await waitForRasterSourceLoaded(map, tileSourceId); + + if (transitionGenerationByLayerKey.get(layerKeyValue) !== generation) { + return; + } + + if (!map.getLayer(tileLayerId) || !map.getLayer(previewMapLayerId)) { + if (map.getLayer(tileLayerId)) { + map.setPaintProperty(tileLayerId, "raster-opacity", targetOpacity); + } + removePreviewLayer(map, layerKeyValue, frameIndex); + activePreviewByLayerKey.delete(layerKeyValue); + clearPreviewDisplayed(layerKeyValue); + reorderPreviewLayers(); + return; + } + + const tileOpacity = + (map.getPaintProperty(tileLayerId, "raster-opacity") as number) ?? 0; + const previewOpacity = + (map.getPaintProperty(previewMapLayerId, "raster-opacity") as number) ?? + targetOpacity; + await fadeRasterOpacities( + map, + [ + { id: previewMapLayerId, from: previewOpacity, to: 0 }, + { id: tileLayerId, from: tileOpacity, to: targetOpacity }, + ], + PREVIEW_FADE_DURATION_MS, + ); + + if (transitionGenerationByLayerKey.get(layerKeyValue) !== generation) { + return; + } + + removePreviewLayer(map, layerKeyValue, frameIndex); + activePreviewByLayerKey.delete(layerKeyValue); + clearPreviewDisplayed(layerKeyValue); + reorderPreviewLayers(); + } + + async function showPreviewThenTiles(layer: Layer) { + if (styleStore.isLayerStyleEditing(layer)) { + return; + } + const map = mapStore.getMap(); + const layerKeyValue = styleStore.layerStyleKey(layer); + const style = styleStore.selectedLayerStyles[layerKeyValue]; + const previews = previewsForLayer(layer, style); + + const frames = layerStore.layerFrames(layer); + const rasterFrames = orderedRasterFrames(frames); + if (rasterFrames.length <= 1) { + return; + } + + const currentFrame = rasterFrames.find( + (frame) => frame.index === layer.current_frame_index, + ); + if (!currentFrame?.raster || !layer.visible) { + return; + } + + const preview = previewAtFrameIndex( + previews, + rasterFrames, + layer.current_frame_index, + ); + + const tileSourceId = mapStore.sourceIdFromLayerFrame(layer, currentFrame); + const tileLayerId = `${tileSourceId}.raster`; + const targetOpacity = style?.style_spec?.opacity ?? 1; + const settledFrameIndex = layer.current_frame_index; + const previewMapLayerId = previewLayerId(layerKeyValue, settledFrameIndex); + + if ( + previewAlreadySettled( + map, + layerKeyValue, + settledFrameIndex, + tileLayerId, + previewMapLayerId, + targetOpacity, + ) + ) { + reorderPreviewLayers(); + return; + } + + const generation = bumpGeneration(layerKeyValue); + + if (!preview) { + // Drop any leftover overlay (including adjacent preloads) so a stale + // image cannot flash while tiles catch up after a style change. + removeAllPreviewLayersForLayerKey(map, layerKeyValue); + activePreviewByLayerKey.delete(layerKeyValue); + clearPreviewDisplayed(layerKeyValue); + ensureRasterTilesOnMap( + currentFrame, + tileSourceId, + tileLayerId, + targetOpacity, + ); + return; + } + + const upsertedPreviewLayerId = await upsertPreviewLayer( + map, + layerKeyValue, + settledFrameIndex, + preview, + currentFrame.raster.metadata, + targetOpacity, + ); + if (transitionGenerationByLayerKey.get(layerKeyValue) !== generation) { + // A newer scrub already took over; don't steal its settle timer or cover + // its preview with this stale overlay. + if ( + upsertedPreviewLayerId && + layer.current_frame_index !== settledFrameIndex + ) { + hidePreviewLayer(map, layerKeyValue, settledFrameIndex); + } + return; + } + if (!upsertedPreviewLayerId) { + clearPreviewDisplayed(layerKeyValue); + ensureRasterTilesOnMap( + currentFrame, + tileSourceId, + tileLayerId, + targetOpacity, + ); + return; + } + + activePreviewByLayerKey.set(layerKeyValue, settledFrameIndex); + markPreviewDisplayed(layerKeyValue); + reorderPreviewLayers(); + hidePreviousPreview(map, layerKeyValue, settledFrameIndex); + + // Keep any already-attached tiles invisible while scrubbing; do not start + // fetching a new frame's tiles until the settle debounce fires. + if (map.getLayer(tileLayerId)) { + map.setPaintProperty(tileLayerId, "raster-opacity", 0); + } + + preloadAdjacentPreviewLayers( + map, + layerKeyValue, + previews, + rasterFrames, + settledFrameIndex, + targetOpacity, + ).then(() => reorderPreviewLayers()); + + scheduleTileLoadAfterSettle(layerKeyValue, generation, () => { + if (layer.current_frame_index !== settledFrameIndex) { + return; + } + ensureRasterTilesOnMap(currentFrame, tileSourceId, tileLayerId, 0); + reorderPreviewLayers(); + transitionToTiles( + layer, + settledFrameIndex, + layerKeyValue, + generation, + targetOpacity, + tileLayerId, + tileSourceId, + ); + }); + } + + /** + * Drop stale preview overlays and hide current-frame tiles so the next + * showPreviewThenTiles call re-attaches previews for the active style. + */ + function prepareForStylePreviewReset(layer: Layer) { + const map = mapStore.getMap(); + const layerKeyValue = styleStore.layerStyleKey(layer); + + bumpGeneration(layerKeyValue); + removeAllPreviewLayersForLayerKey(map, layerKeyValue); + activePreviewByLayerKey.delete(layerKeyValue); + clearPreviewDisplayed(layerKeyValue); + + const frames = layerStore.layerFrames(layer); + const currentFrame = frames.find( + (frame) => frame.index === layer.current_frame_index, + ); + if (currentFrame?.raster && layer.visible) { + const tileLayerId = `${mapStore.sourceIdFromLayerFrame(layer, currentFrame)}.raster`; + if (map.getLayer(tileLayerId)) { + map.setPaintProperty(tileLayerId, "raster-opacity", 0); + } + } + reorderPreviewLayers(); + } + + function dismissPreviewForLayer(layer: Layer) { + const map = mapStore.getMap(); + const layerKeyValue = styleStore.layerStyleKey(layer); + + bumpGeneration(layerKeyValue); + removeAllPreviewLayersForLayerKey(map, layerKeyValue); + activePreviewByLayerKey.delete(layerKeyValue); + clearPreviewDisplayed(layerKeyValue); + + const frames = layerStore.layerFrames(layer); + const currentFrame = frames.find( + (frame) => frame.index === layer.current_frame_index, + ); + if (!currentFrame?.raster) { + return; + } + + const tileLayerId = `${mapStore.sourceIdFromLayerFrame(layer, currentFrame)}.raster`; + const style = styleStore.selectedLayerStyles[layerKeyValue]; + const targetOpacity = style?.style_spec?.opacity ?? 1; + if (map.getLayer(tileLayerId)) { + map.setPaintProperty(tileLayerId, "raster-opacity", targetOpacity); + } + reorderPreviewLayers(); + } + + /** Clear stale preview payloads + map overlays after a style change. */ + function clearPreviewsForStyleChange(layer: Layer, styleId?: number) { + layerStore.selectedLayers.forEach((candidate) => { + if (candidate.id !== layer.id) return; + + const key = styleStore.layerStyleKey(candidate); + const selectedStyle = styleStore.selectedLayerStyles[key]; + if (!selectedStyle) return; + + const matches = + styleId !== undefined + ? selectedStyle.id === styleId + : candidate.copy_id === layer.copy_id; + if (!matches) return; + + if (styleId !== undefined) { + markPreviewGenerating(styleId); + } + if ( + selectedStyle.is_default || + usesLayerDefaultPreviews(candidate, selectedStyle) + ) { + markPreviewGenerating(undefined, candidate.id); + } + + styleStore.patchSelectedLayerStyle(key, { + preview_status: "notready", + multiframe_previews: undefined, + }); + if ( + selectedStyle.is_default || + usesLayerDefaultPreviews(candidate, selectedStyle) + ) { + candidate.preview_status = "notready"; + candidate.multiframe_previews = undefined; + } + dismissPreviewForLayer(candidate); + }); + } + + function cleanupLayer(layer: Layer) { + const layerKeyValue = styleStore.layerStyleKey(layer); + clearTileLoadTimer(layerKeyValue); + transitionGenerationByLayerKey.delete(layerKeyValue); + activePreviewByLayerKey.delete(layerKeyValue); + clearPreviewDisplayed(layerKeyValue); + removeAllPreviewLayersForLayerKey(mapStore.getMap(), layerKeyValue); + } + + function clearAll() { + tileLoadTimerByLayerKey.forEach((timer) => clearTimeout(timer)); + tileLoadTimerByLayerKey.clear(); + transitionGenerationByLayerKey.clear(); + activePreviewByLayerKey.clear(); + generatingPreviewStyleIds.value = new Set(); + generatingPreviewLayerIds.value = new Set(); + displayingPreviewLayerKeys.value = new Set(); + } + + // Called when a "frame_preview" TaskResult completes over the analytics + // WebSocket. Reloads the freshly generated previews and reattaches them to + // every selected layer copy that still has the regenerated style applied. + async function onPreviewTaskComplete(task: TaskResult) { + const layerStyleId = task.inputs?.layer_style_id as number | undefined; + const layerId = task.inputs?.layer_id as number | undefined; + if (layerId === undefined) { + return; + } + + // Conversion / dataset-default tasks have no layer_style_id; refresh the + // layer payload which carries the default-fingerprint preview set. + if (layerStyleId === undefined) { + let updatedLayer: Layer; + try { + updatedLayer = await layerStore.fetchAvailableLayer(layerId); + } catch { + return; + } + + markPreviewReady(undefined, layerId); + applyDefaultPreviewToMatchingLayers(layerId, { + preview_status: updatedLayer.preview_status, + multiframe_previews: updatedLayer.multiframe_previews, + }); + return; + } + + let updatedStyle: LayerStyle; + try { + updatedStyle = await getLayerStyle(layerStyleId); + } catch { + return; + } + + markPreviewReady(layerStyleId, layerId); + + layerStore.fetchAvailableLayer(layerId).catch(() => undefined); + + applyStylePreviewToMatchingLayers(layerId, layerStyleId, { + preview_status: updatedStyle.preview_status, + multiframe_previews: updatedStyle.multiframe_previews, + }); + } + + return { + displayingPreviewLayerKeys, + isDisplayingPreview, + shouldDeferRasterTileUpdates, + isGeneratingPreviews, + iconState, + prefetchLayerPreviews, + hasReadyPreviewForCurrentFrame, + reorderPreviewLayers, + showPreviewThenTiles, + dismissPreviewForLayer, + prepareForStylePreviewReset, + clearPreviewsForStyleChange, + onPreviewTaskComplete, + attachPreviewsForLayer, + cleanupLayer, + clearAll, + }; +}); diff --git a/web/src/store/index.ts b/web/src/store/index.ts index fb788168c..35e7f348f 100644 --- a/web/src/store/index.ts +++ b/web/src/store/index.ts @@ -7,6 +7,7 @@ import { useProjectStore } from "./project"; import { useStyleStore } from "./style"; import { usePanelStore } from "./panel"; import { useConversionStore } from "./conversion"; +import { useFramePreviewStore } from "./framePreview"; import { useTutorialStore } from "./tutorial"; export { @@ -19,5 +20,6 @@ export { useStyleStore, usePanelStore, useConversionStore, + useFramePreviewStore, useTutorialStore, }; diff --git a/web/src/store/layer.ts b/web/src/store/layer.ts index 709c0ef88..e2788b3f3 100644 --- a/web/src/store/layer.ts +++ b/web/src/store/layer.ts @@ -16,12 +16,14 @@ import { getVectorDataBounds, } from "@/api/rest"; import proj4 from "proj4"; +import { isPreviewMapLayerId } from "@/utils/framePreviewLayer"; import { useMapStore, useStyleStore, useNetworkStore, useProjectStore, + useFramePreviewStore, } from "."; interface SourceDBObjects { @@ -43,6 +45,7 @@ export const useLayerStore = defineStore("layer", () => { const networkStore = useNetworkStore(); const styleStore = useStyleStore(); const projectStore = useProjectStore(); + const framePreviewStore = useFramePreviewStore(); /** * Return the maplibre layers associated with a Layer DB object @@ -52,9 +55,7 @@ export const useLayerStore = defineStore("layer", () => { function getMapLayersFromLayerObject(layer: Layer) { return mapStore .getUserMapLayers() - .filter((layerId) => - layerId.startsWith(mapStore.uniqueLayerIdFromLayer(layer)), - ); + .filter((layerId) => layerId.startsWith(styleStore.layerStyleKey(layer))); } async function fetchAvailableLayer(layerId: number) { @@ -70,6 +71,8 @@ export const useLayerStore = defineStore("layer", () => { selectedLayers.value.forEach((selected) => { if (selected.id === layer.id) { selected.default_style = layer.default_style; + selected.multiframe_previews = layer.multiframe_previews; + selected.preview_status = layer.preview_status; } }); return layer; @@ -197,6 +200,10 @@ export const useLayerStore = defineStore("layer", () => { current_frame_index: 0, }; + framePreviewStore.prefetchLayerPreviews( + newLayer, + newLayer.default_style ?? undefined, + ); // Need to fetch the frames for this layer, if not present if (!layerFrames(layer).length) { await fetchFramesForLayer(layer.id); @@ -214,69 +221,91 @@ export const useLayerStore = defineStore("layer", () => { watch(selectedLayers, updateLayersShown); watch(framesByLayerId, updateLayersShown); - function updateLayersShown() { - if (!mapStore.map) return; // map not yet initialized - const map = mapStore.getMap(); - const userMapLayers = mapStore.getUserMapLayers(); + /** Sync one layer's frames/styles onto the map (frame scrubbing uses this alone). */ + function syncLayerOnMap(layer: Layer) { + if (!mapStore.map) return; - // reverse selected layers list for first on top - selectedLayers.value.toReversed().forEach((layer) => { - const frames = layerFrames(layer); - const multiFrame = frames.length > 1; - frames.forEach((frame) => { - const styleId = mapStore.uniqueLayerIdFromLayer(layer); - if (!styleStore.selectedLayerStyles[styleId]) { + const userMapLayers = mapStore.getUserMapLayers(); + const frames = layerFrames(layer); + const multiFrame = frames.length > 1; + frames.forEach((frame) => { + const styleId = styleStore.layerStyleKey(layer); + if (!styleStore.selectedLayerStyles[styleId]) { + if ( + layer.default_style?.style_spec && + Object.keys(layer.default_style.style_spec).length + ) { + styleStore.selectedLayerStyles[styleId] = { + ...layer.default_style, + preview_status: layer.preview_status, + ...(layer.preview_status === "ready" && layer.multiframe_previews + ? { multiframe_previews: layer.multiframe_previews } + : {}), + }; if ( - layer.default_style?.style_spec && - Object.keys(layer.default_style.style_spec).length + styleStore.selectedLayerStyles[styleId]?.style_spec + ?.default_frame !== layer.current_frame_index ) { - styleStore.selectedLayerStyles[styleId] = { - ...layer.default_style, - }; - if ( - styleStore.selectedLayerStyles[styleId]?.style_spec - ?.default_frame !== layer.current_frame_index - ) { - layer.current_frame_index = - styleStore.selectedLayerStyles[styleId].style_spec - ?.default_frame || 0; - } - } else { - const firstCurrentRaster = frames.find( - (f) => f.index == layer.current_frame_index && f.raster, - )?.raster; - styleStore.selectedLayerStyles[styleId] = { - name: "None", - is_default: true, - style_spec: styleStore.getDefaultStyleSpec( - firstCurrentRaster, - layer.id, - ), - }; + layer.current_frame_index = + styleStore.selectedLayerStyles[styleId].style_spec + ?.default_frame || 0; } + } else { + const firstCurrentRaster = frames.find( + (f) => f.index == layer.current_frame_index && f.raster, + )?.raster; + styleStore.selectedLayerStyles[styleId] = { + name: "None", + is_default: true, + style_spec: styleStore.getDefaultStyleSpec( + firstCurrentRaster, + layer.id, + ), + // Dataset conversion can leave ready default-fingerprint previews + // with no LayerStyle yet — still attach them for scrubbing. + preview_status: layer.preview_status, + ...(layer.preview_status === "ready" && layer.multiframe_previews + ? { multiframe_previews: layer.multiframe_previews } + : {}), + }; } + } - // Add current frame to map if not added yet - const sourceId = mapStore.sourceIdFromLayerFrame(layer, frame); - if ( - layer.visible && - !userMapLayers.some((mapLayerId) => mapLayerId.includes(sourceId)) && - layer.current_frame_index === frame.index - ) { - mapStore.addLayerFrameToMap(frame, sourceId, multiFrame); - } + // Add current frame to map if not added yet. Multiframe rasters with a + // ready preview skip the tile source here so the overlay can show first; + // showPreviewThenTiles adds the tiles afterward at opacity 0. + const sourceId = mapStore.sourceIdFromLayerFrame(layer, frame); + if ( + layer.visible && + !userMapLayers.some((mapLayerId) => mapLayerId.includes(sourceId)) && + layer.current_frame_index === frame.index && + !framePreviewStore.hasReadyPreviewForCurrentFrame(layer) + ) { + mapStore.addLayerFrameToMap(frame, sourceId, multiFrame); + } + }); + styleStore.updateLayerStyles(layer); + framePreviewStore.reorderPreviewLayers(); + } - userMapLayers.forEach((mapLayerId) => { - if (mapLayerId.includes(sourceId)) { - map.moveLayer(mapLayerId); // handles reordering - } - }); - }); - styleStore.updateLayerStyles(layer); + function updateLayerFrame(layer: Layer) { + syncLayerOnMap(layer); + } + + function updateLayersShown() { + if (!mapStore.map) return; // map not yet initialized + + const map = mapStore.getMap(); + const userMapLayers = mapStore.getUserMapLayers(); + + // reverse selected layers list for first on top + selectedLayers.value.toReversed().forEach((layer) => { + syncLayerOnMap(layer); }); // hide any removed layers userMapLayers.forEach((mapLayerId) => { + if (isPreviewMapLayerId(mapLayerId)) return; const { layerId, layerCopyId } = mapStore.parseLayerString(mapLayerId); if ( !selectedLayers.value.some((l) => { @@ -297,6 +326,7 @@ export const useLayerStore = defineStore("layer", () => { fetchFramesForLayer, layerFrames, updateLayersShown, + updateLayerFrame, addLayer, setLayerVisibility, getDBObjectsForSourceID, diff --git a/web/src/store/map.ts b/web/src/store/map.ts index 10d9b8d29..639b7dcf0 100644 --- a/web/src/store/map.ts +++ b/web/src/store/map.ts @@ -29,8 +29,14 @@ import { useLayerStore, useAppStore, useProjectStore, + useFramePreviewStore, useAnalysisStore, } from "."; +import { layerStyleKey } from "./style"; +import { + isPreviewMapLayerId, + removeAllPreviewLayersForLayerKey, +} from "@/utils/framePreviewLayer"; import { THEMES } from "@/themes"; function getLayerIsVisible(layer: MapLibreLayerWithMetadata) { @@ -59,17 +65,13 @@ function sourceIdFromMapLayerId(mapLayerId: string) { return mapLayerId.split(".").slice(0, -1).join("."); } -function uniqueLayerIdFromLayer(layer: Layer) { - return `${layer.id}.${layer.copy_id}`; -} - /** * Note: Rasters also have an extra `bounds` source, which allows for * interaction with the raster layer. This is not considered in this * function, as it's rarely accessed directly. */ function sourceIdFromLayerFrame(layer: Layer, frame: LayerFrame) { - const parts: (number | string)[] = [uniqueLayerIdFromLayer(layer), frame.id]; + const parts: (number | string)[] = [layerStyleKey(layer), frame.id]; if (frame.vector) { parts.push("vector"); @@ -334,12 +336,21 @@ export const useMapStore = defineStore("map", () => { function removeLayers(layerIds: string[]) { const map = getMap(); + const framePreviewStore = useFramePreviewStore(); + const cleanedLayerKeys = new Set(); // Must collect all source Ids so they can be removed after all layers // have been removed, since multple layers may use the same source const sourceIdsToRemove = new Set(); const updatedLayerIds: string[] = []; layerIds.forEach((id) => { + if (isPreviewMapLayerId(id)) { + return; + } + const layerKey = id.split(".").slice(0, 2).join("."); + if (layerKey.includes(".")) { + cleanedLayerKeys.add(layerKey); + } // Rasters have implicit bounds layers that also need to be removed if (id.includes(".raster.")) { updatedLayerIds.push(id.replace(".raster.", ".bounds.")); @@ -358,6 +369,17 @@ export const useMapStore = defineStore("map", () => { sourceIdsToRemove.forEach((id) => { map.removeSource(id); }); + + cleanedLayerKeys.forEach((layerKey) => { + removeAllPreviewLayersForLayerKey(map, layerKey); + const [layerId, copyId] = layerKey.split(".").map(Number); + const layer = layerStore.selectedLayers.find( + (candidate) => candidate.id === layerId && candidate.copy_id === copyId, + ); + if (layer) { + framePreviewStore.cleanupLayer(layer); + } + }); } /** @@ -468,6 +490,7 @@ export const useMapStore = defineStore("map", () => { tileSource: Source, boundsSource: Source, multiFrame: boolean, + rasterOpacity = 1, ) { const map = getMap(); const { layerId, layerCopyId, frameId } = parseSourceString(tileSource.id); @@ -484,6 +507,9 @@ export const useMapStore = defineStore("map", () => { type: "raster", source: tileSource.id, metadata, + paint: { + "raster-opacity": rasterOpacity, + }, }); const boundsLayerId = boundsSource.id + ".fill"; @@ -524,6 +550,7 @@ export const useMapStore = defineStore("map", () => { raster: RasterData, sourceId: string, multiFrame: boolean, + rasterOpacity = 1, ): Source | undefined { const map = getMap(); @@ -581,7 +608,12 @@ export const useMapStore = defineStore("map", () => { const boundsSource = map.getSource(boundsSourceId); if (tileSource && boundsSource) { - createRasterFeatureMapLayers(tileSource, boundsSource, multiFrame); + createRasterFeatureMapLayers( + tileSource, + boundsSource, + multiFrame, + rasterOpacity, + ); return tileSource; } } @@ -590,6 +622,7 @@ export const useMapStore = defineStore("map", () => { frame: LayerFrame, sourceId: string, multiFrame: boolean, + options?: { rasterOpacity?: number }, ) { if (getMapSources().includes(sourceId)) { return; @@ -598,7 +631,12 @@ export const useMapStore = defineStore("map", () => { if (frame.vector) { createVectorTileSource(frame.vector, sourceId, multiFrame); } else if (frame.raster) { - createRasterTileSource(frame.raster, sourceId, multiFrame); + createRasterTileSource( + frame.raster, + sourceId, + multiFrame, + options?.rasterOpacity, + ); } else { throw new Error("Layer Frame is neither raster nor vector!"); } @@ -690,7 +728,6 @@ export const useMapStore = defineStore("map", () => { parseSourceString, parseLayerString, sourceIdFromLayerFrame, - uniqueLayerIdFromLayer, getLatestLayerInstance, getUserMapLayers, setupVectorLayerClickHandlers, diff --git a/web/src/store/project.ts b/web/src/store/project.ts index b7ed95f1e..140b2212d 100644 --- a/web/src/store/project.ts +++ b/web/src/store/project.ts @@ -20,8 +20,10 @@ import { usePanelStore, useAppStore, useStyleStore, + useFramePreviewStore, useTutorialStore, } from "."; +import { clearFramePreviewCache } from "@/utils/framePreviewCache"; export const useProjectStore = defineStore("project", () => { const networkStore = useNetworkStore(); @@ -119,7 +121,7 @@ export const useProjectStore = defineStore("project", () => { ); const styleKeysToCurrentFrames = Object.fromEntries( includeLayers.map((layer) => [ - mapStore.uniqueLayerIdFromLayer(layer), + styleStore.layerStyleKey(layer), layer.current_frame_index, ]), ); @@ -250,8 +252,8 @@ export const useProjectStore = defineStore("project", () => { // Ensure correct layer order layerStore.selectedLayers = layerStore.selectedLayers.sort( (layer1, layer2) => { - const key1 = mapStore.uniqueLayerIdFromLayer(layer1); - const key2 = mapStore.uniqueLayerIdFromLayer(layer2); + const key1 = styleStore.layerStyleKey(layer1); + const key2 = styleStore.layerStyleKey(layer2); return ( viewState.selected_layer_order.indexOf(key1) - viewState.selected_layer_order.indexOf(key2) @@ -260,7 +262,7 @@ export const useProjectStore = defineStore("project", () => { ); // Ensure correct current frames layerStore.selectedLayers = layerStore.selectedLayers.map((layer) => { - const styleKey = mapStore.uniqueLayerIdFromLayer(layer); + const styleKey = styleStore.layerStyleKey(layer); if (viewState.selected_layer_current_frames[styleKey]) { layer.current_frame_index = viewState.selected_layer_current_frames[styleKey]; @@ -315,6 +317,9 @@ export const useProjectStore = defineStore("project", () => { layerStore.selectedLayers = []; styleStore.selectedLayerStyles = {}; + styleStore.clearStyleEditing(); + useFramePreviewStore().clearAll(); + clearFramePreviewCache(); mapStore.clickedFeature = undefined; diff --git a/web/src/store/style.ts b/web/src/store/style.ts index be9653a6c..156de0c11 100644 --- a/web/src/store/style.ts +++ b/web/src/store/style.ts @@ -16,12 +16,14 @@ import type { } from "@/types"; import { getProjectColormaps } from "@/api/rest"; import chroma from "chroma-js"; +import { isPreviewMapLayerId } from "@/utils/framePreviewLayer"; import { useMapStore, useLayerStore, useProjectStore, useNetworkStore, + useFramePreviewStore, } from "."; export interface MapLayerStyleRaw { @@ -58,7 +60,7 @@ function getMidMarker( }; } -export function colormapMarkersSubsample( +function colormapMarkersSubsample( colormap: Colormap, appliedColormap: AppliedColormap, n: number | undefined = undefined, @@ -347,6 +349,10 @@ function getVectorSizePaintProperty( } else return baseSize; } +export function layerStyleKey(layer: Layer) { + return `${layer.id}.${layer.copy_id}`; +} + function getVectorVisibilityPaintProperty( styleSpec: StyleSpec, groupName: string, @@ -400,11 +406,51 @@ function getVectorVisibilityPaintProperty( export const useStyleStore = defineStore("style", () => { const selectedLayerStyles = ref>({}); const colormaps = ref([]); + const editingStyleLayerKeys = ref>(new Set()); const mapStore = useMapStore(); const projectStore = useProjectStore(); const layerStore = useLayerStore(); const networkStore = useNetworkStore(); + const framePreviewStore = useFramePreviewStore(); + + function isLayerStyleEditing(layer: Layer) { + return editingStyleLayerKeys.value.has(layerStyleKey(layer)); + } + + function patchSelectedLayerStyle( + key: string, + patch: Partial, + base?: LayerStyle, + ) { + const current = selectedLayerStyles.value[key] ?? base; + if (!current) { + return; + } + selectedLayerStyles.value = { + ...selectedLayerStyles.value, + [key]: { ...current, ...patch }, + }; + } + + function setLayerStyleEditing(layer: Layer, editing: boolean) { + const key = layerStyleKey(layer); + const next = new Set(editingStyleLayerKeys.value); + if (editing) { + next.add(key); + editingStyleLayerKeys.value = next; + framePreviewStore.dismissPreviewForLayer(layer); + return; + } + next.delete(key); + editingStyleLayerKeys.value = next; + updateLayerStyles(layer); + framePreviewStore.attachPreviewsForLayer(layer); + } + + function clearStyleEditing() { + editingStyleLayerKeys.value = new Set(); + } function getDefaultColor(layerId: number) { const color = chroma.hsl( @@ -453,33 +499,53 @@ export const useStyleStore = defineStore("style", () => { (f) => f.index === layer.current_frame_index, ); if (!currentFrame) return; + + const styleKey = layerStyleKey(layer); + const currentStyleSpec: StyleSpec | undefined = + selectedLayerStyles.value[styleKey]?.style_spec; + + const deferRasterTileUpdates = + framePreviewStore.shouldDeferRasterTileUpdates(layer); + mapStore.getUserMapLayers().forEach((mapLayerId) => { + if (isPreviewMapLayerId(mapLayerId)) { + return; + } const { layerId, layerCopyId, frameId } = mapStore.parseLayerString(mapLayerId); - if (layerId === layer.id && layerCopyId === layer.copy_id) { - if (frameId === currentFrame.id) { - map.setLayoutProperty( - mapLayerId, - "visibility", - layer.visible ? "visible" : "none", - ); - const styleKey = `${layer.id}.${layer.copy_id}`; - const currentStyleSpec: StyleSpec | undefined = - selectedLayerStyles.value[styleKey].style_spec; - if (currentStyleSpec) { - setMapLayerStyle( - mapLayerId, - currentStyleSpec, - currentFrame, - currentFrame.vector, - ); - } - } else { - map.setLayoutProperty(mapLayerId, "visibility", "none"); - } + if (layerId !== layer.id || layerCopyId !== layer.copy_id) { + return; + } + + const frame = frames.find((f) => f.id === frameId); + if (!frame) { + return; + } + + const isCurrent = frameId === currentFrame.id; + map.setLayoutProperty( + mapLayerId, + "visibility", + layer.visible && isCurrent ? "visible" : "none", + ); + + // While preview overlays are driving frame scrubbing, only toggle + // visibility — setTiles on already-attached frames would fetch tiles + // for every step (and for non-current frames) in the background. + if (currentStyleSpec && !deferRasterTileUpdates) { + // Restyle every loaded frame, not only the current one. Otherwise a + // previously visited frame keeps the old tile URL and flashes the + // previous style when scrubbed to after a style change. + setMapLayerStyle(mapLayerId, currentStyleSpec, frame, frame.vector); } }); networkStore.styleVisibleNetworks(); + + const hasMultiframeRaster = + frames.length > 1 && frames.some((f) => f.raster); + if (hasMultiframeRaster && !isLayerStyleEditing(layer)) { + framePreviewStore.showPreviewThenTiles(layer); + } } type GeneratedLayerStyle = { @@ -632,9 +698,14 @@ export const useStyleStore = defineStore("style", () => { if (result.tileURL) { const mapLayer = map.getLayer(mapLayerId) as MapLibreLayerWithMetadata; const source = map.getSource(mapLayer.source) as RasterTileSource; - source?.setTiles([result.tileURL]); - if (mapLayer.source) { - mapStore.rasterSourceTileURLs[mapLayer.source] = result.tileURL; + const previousURL = mapLayer.source + ? mapStore.rasterSourceTileURLs[mapLayer.source] + : undefined; + if (source && previousURL !== result.tileURL) { + source.setTiles([result.tileURL]); + if (mapLayer.source) { + mapStore.rasterSourceTileURLs[mapLayer.source] = result.tileURL; + } } } } @@ -659,15 +730,21 @@ export const useStyleStore = defineStore("style", () => { return { colormaps, selectedLayerStyles, + editingStyleLayerKeys, fetchColormaps, getRasterTilesQuery, getRasterSourceFilterParams, buildRasterTileQueryParams, sourceFiltersToStyleFilters, colormapMarkersSubsample, + layerStyleKey, getDefaultColor, getDefaultStyleSpec, getVectorColorPaintProperty, + isLayerStyleEditing, + setLayerStyleEditing, + patchSelectedLayerStyle, + clearStyleEditing, updateLayerStyles, setMapLayerStyle, returnMapLayerStyle, diff --git a/web/src/types.ts b/web/src/types.ts index bd7700893..ee86d2832 100644 --- a/web/src/types.ts +++ b/web/src/types.ts @@ -27,6 +27,35 @@ export interface Dataset { n_layers?: number; } +export interface FramePreviewCorner { + x: number; + y: number; +} + +export interface FramePreviewBounds { + srs: string; + xmin: number; + xmax: number; + ymin: number; + ymax: number; + ul?: FramePreviewCorner; + ur?: FramePreviewCorner; + lr?: FramePreviewCorner; + ll?: FramePreviewCorner; +} + +export interface FramePreview { + url: string; + width: number; + height: number; + bounds: FramePreviewBounds; +} + +// Style-level preview lifecycle, derived by the API from per-frame rows. +// "ready" means every raster frame has a complete preview image; anything +// else (missing, generating, regenerating, failed) is reported as "notready". +export type PreviewStatus = "ready" | "notready"; + export interface Layer { id: number; copy_id: number; @@ -37,6 +66,8 @@ export interface Layer { visible: boolean; current_frame_index: number; default_style: LayerStyle | null; + multiframe_previews?: (FramePreview | null)[]; + preview_status?: PreviewStatus; } export interface LayerFrame { @@ -127,6 +158,10 @@ export interface LayerStyle { project?: number; is_default: boolean; style_spec?: StyleSpec; + /** Client-computed django-large-image style JSON, persisted on save for rasters. */ + raster_style_params?: Record | null; + multiframe_previews?: (FramePreview | null)[]; + preview_status?: PreviewStatus; } export interface VectorData { diff --git a/web/src/utils/framePreviewCache.ts b/web/src/utils/framePreviewCache.ts new file mode 100644 index 000000000..72e49dfd2 --- /dev/null +++ b/web/src/utils/framePreviewCache.ts @@ -0,0 +1,21 @@ +const prefetchedUrls = new Set(); + +export async function getCachedPreviewObjectUrl( + url: string, +): Promise { + return url; +} + +export function prefetchFramePreviewUrls(urls: (string | null | undefined)[]) { + urls.forEach((url) => { + if (!url || prefetchedUrls.has(url)) { + return; + } + prefetchedUrls.add(url); + fetch(url); + }); +} + +export function clearFramePreviewCache() { + prefetchedUrls.clear(); +} diff --git a/web/src/utils/framePreviewLayer.ts b/web/src/utils/framePreviewLayer.ts new file mode 100644 index 000000000..8515d7fc5 --- /dev/null +++ b/web/src/utils/framePreviewLayer.ts @@ -0,0 +1,315 @@ +import type { + FramePreview, + FramePreviewBounds, + FramePreviewCorner, + RasterMetadata, +} from "@/types"; +import type { Map as MaplibreMap, MapSourceDataEvent } from "maplibre-gl"; +import proj4 from "proj4"; +import { getCachedPreviewObjectUrl } from "./framePreviewCache"; + +export const PREVIEW_FADE_DURATION_MS = 400; + +const CORNER_KEYS = ["ul", "ur", "lr", "ll"] as const; + +/** Tracks the preview URL attached to each map source to avoid remove/re-add flashes. */ +const previewUrlBySourceId = new Map(); +type CornerKey = (typeof CORNER_KEYS)[number]; + +function clampRasterOpacity(value: number): number { + return Math.min(1, Math.max(0, value)); +} + +export function previewSourceId(layerKey: string, frameIndex: number) { + return `${layerKey}.preview.${frameIndex}`; +} + +export function previewLayerId(layerKey: string, frameIndex: number) { + return `${previewSourceId(layerKey, frameIndex)}.raster`; +} + +export function isPreviewMapLayerId(mapLayerId: string) { + return mapLayerId.includes(".preview."); +} + +function toLngLat(srs: string, x: number, y: number): [number, number] { + if (srs && srs !== "EPSG:4326") { + return proj4(srs, "EPSG:4326", [x, y]) as [number, number]; + } + return [x, y]; +} + +function hasCornerBounds( + bounds: FramePreviewBounds, +): bounds is FramePreviewBounds & Record { + return CORNER_KEYS.every((corner) => bounds[corner] !== undefined); +} + +function cornersFromRasterMetadata( + rasterBounds: RasterMetadata["bounds"], +): FramePreviewBounds | undefined { + if (!CORNER_KEYS.every((corner) => rasterBounds[corner])) { + return undefined; + } + + const corners = Object.fromEntries( + CORNER_KEYS.map((corner) => { + const point = rasterBounds[corner]!; + const [x, y] = toLngLat(rasterBounds.srs, point.x, point.y); + return [corner, { x, y }]; + }), + ) as Record; + + const lngs = CORNER_KEYS.map((corner) => corners[corner].x); + const lats = CORNER_KEYS.map((corner) => corners[corner].y); + + return { + srs: "EPSG:4326", + xmin: Math.min(...lngs), + xmax: Math.max(...lngs), + ymin: Math.min(...lats), + ymax: Math.max(...lats), + ...corners, + }; +} + +export function resolvePreviewBounds( + preview: FramePreview, + raster?: RasterMetadata, +): FramePreviewBounds { + if (hasCornerBounds(preview.bounds)) { + return preview.bounds; + } + + if (raster?.bounds) { + const rasterCorners = cornersFromRasterMetadata(raster.bounds); + if (rasterCorners) { + return rasterCorners; + } + } + + return preview.bounds; +} + +function boundsToCoordinates( + bounds: FramePreviewBounds, +): [[number, number], [number, number], [number, number], [number, number]] { + if (hasCornerBounds(bounds)) { + return [ + toLngLat(bounds.srs, bounds.ul.x, bounds.ul.y), + toLngLat(bounds.srs, bounds.ur.x, bounds.ur.y), + toLngLat(bounds.srs, bounds.lr.x, bounds.lr.y), + toLngLat(bounds.srs, bounds.ll.x, bounds.ll.y), + ]; + } + + const { xmin, xmax, ymin, ymax, srs } = bounds; + return [ + toLngLat(srs, xmin, ymax), + toLngLat(srs, xmax, ymax), + toLngLat(srs, xmax, ymin), + toLngLat(srs, xmin, ymin), + ]; +} + +export async function upsertPreviewLayer( + map: MaplibreMap, + layerKey: string, + frameIndex: number, + preview: FramePreview, + raster?: RasterMetadata, + opacity = 1, + visible = true, +) { + const sourceId = previewSourceId(layerKey, frameIndex); + const mapLayerId = previewLayerId(layerKey, frameIndex); + const objectUrl = await getCachedPreviewObjectUrl(preview.url); + if (!objectUrl) { + return undefined; + } + + const coordinates = boundsToCoordinates( + resolvePreviewBounds(preview, raster), + ); + const previewOpacity = clampRasterOpacity(opacity); + const visibility = visible ? "visible" : "none"; + + const existingUrl = previewUrlBySourceId.get(sourceId); + if ( + existingUrl === preview.url && + map.getSource(sourceId) && + map.getLayer(mapLayerId) + ) { + map.setPaintProperty(mapLayerId, "raster-opacity", previewOpacity); + map.setLayoutProperty(mapLayerId, "visibility", visibility); + return mapLayerId; + } + + if (map.getLayer(mapLayerId)) { + map.removeLayer(mapLayerId); + } + if (map.getSource(sourceId)) { + map.removeSource(sourceId); + } + previewUrlBySourceId.delete(sourceId); + + map.addSource(sourceId, { + type: "image", + url: objectUrl, + coordinates, + }); + previewUrlBySourceId.set(sourceId, preview.url); + + map.addLayer({ + id: mapLayerId, + type: "raster", + source: sourceId, + layout: { + visibility, + }, + paint: { + "raster-opacity": previewOpacity, + "raster-fade-duration": 0, + }, + }); + + return mapLayerId; +} + +export function hidePreviewLayer( + map: MaplibreMap, + layerKey: string, + frameIndex: number, +) { + const mapLayerId = previewLayerId(layerKey, frameIndex); + if (map.getLayer(mapLayerId)) { + map.setLayoutProperty(mapLayerId, "visibility", "none"); + } +} + +export function removePreviewLayer( + map: MaplibreMap, + layerKey: string, + frameIndex: number, +) { + const sourceId = previewSourceId(layerKey, frameIndex); + const mapLayerId = previewLayerId(layerKey, frameIndex); + if (map.getLayer(mapLayerId)) { + map.removeLayer(mapLayerId); + } + if (map.getSource(sourceId)) { + map.removeSource(sourceId); + } + previewUrlBySourceId.delete(sourceId); +} + +function previewFrameIndexFromLayerId( + layerKey: string, + mapLayerId: string, +): number | undefined { + const prefix = `${layerKey}.preview.`; + if (!mapLayerId.startsWith(prefix)) { + return undefined; + } + const rest = mapLayerId.slice(prefix.length); + const frameIndex = Number.parseInt(rest.split(".")[0], 10); + return Number.isNaN(frameIndex) ? undefined : frameIndex; +} + +export function removePreviewLayersExcept( + map: MaplibreMap, + layerKey: string, + keepFrameIndices: number[], +) { + const keep = new Set(keepFrameIndices); + const frameIndicesToRemove = new Set(); + + map.getStyle().layers?.forEach((layer) => { + const frameIndex = previewFrameIndexFromLayerId(layerKey, layer.id); + if (frameIndex !== undefined && !keep.has(frameIndex)) { + frameIndicesToRemove.add(frameIndex); + } + }); + + frameIndicesToRemove.forEach((frameIndex) => { + removePreviewLayer(map, layerKey, frameIndex); + }); +} + +export function removeAllPreviewLayersForLayerKey( + map: MaplibreMap, + layerKey: string, +) { + map.getStyle().layers?.forEach((layer) => { + if (layer.id.startsWith(`${layerKey}.preview.`)) { + map.removeLayer(layer.id); + } + }); + Object.keys(map.getStyle().sources ?? {}).forEach((sourceId) => { + if (sourceId.startsWith(`${layerKey}.preview.`)) { + map.removeSource(sourceId); + previewUrlBySourceId.delete(sourceId); + } + }); +} + +export function waitForRasterSourceLoaded( + map: MaplibreMap, + sourceId: string, + timeoutMs = 10000, +): Promise { + return new Promise((resolve) => { + if (!map.getSource(sourceId)) { + resolve(); + return; + } + if (map.isSourceLoaded(sourceId)) { + resolve(); + return; + } + + const timeout = window.setTimeout(() => { + map.off("sourcedata", onSourceData); + resolve(); + }, timeoutMs); + + function onSourceData(event: MapSourceDataEvent) { + if (event.sourceId === sourceId && event.isSourceLoaded) { + window.clearTimeout(timeout); + map.off("sourcedata", onSourceData); + resolve(); + } + } + + map.on("sourcedata", onSourceData); + }); +} + +export async function fadeRasterOpacities( + map: MaplibreMap, + layers: { id: string; from: number; to: number }[], + durationMs: number, +) { + const start = performance.now(); + await new Promise((resolve) => { + function step(now: number) { + const progress = Math.min(1, (now - start) / durationMs); + layers.forEach(({ id, from, to }) => { + if (map.getLayer(id)) { + const opacity = progress >= 1 ? to : from + (to - from) * progress; + map.setPaintProperty( + id, + "raster-opacity", + clampRasterOpacity(opacity), + ); + } + }); + if (progress < 1) { + requestAnimationFrame(step); + } else { + resolve(); + } + } + requestAnimationFrame(step); + }); +}