diff --git a/uvdat/core/management/backfill_frame_previews_utils.py b/uvdat/core/management/backfill_frame_previews_utils.py new file mode 100644 index 00000000..231cd94d --- /dev/null +++ b/uvdat/core/management/backfill_frame_previews_utils.py @@ -0,0 +1,186 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from django.db.models import Count, Q, QuerySet + +from uvdat.core.frame_previews.fingerprint import params_fingerprint +from uvdat.core.frame_previews.lookup import preview_status_for_fingerprint +from uvdat.core.frame_previews.preview_regeneration import ( + invalidate_and_enqueue_layer_previews, + pending_preview_task, +) +from uvdat.core.models import Colormap, Layer, LayerStyle, Project +from uvdat.core.rest.querysets import layer_queryset_with_previews +from uvdat.core.tasks.run_mode import TaskRunMode + +if TYPE_CHECKING: + from collections.abc import Callable + +RASTER_SOURCE_FILTER_KEYS = frozenset({"frame", "band"}) + + +def _color_query_from_spec( + color_spec: dict[str, Any], + colormaps_by_id: dict[int, Colormap], +) -> dict[str, Any]: + color_query: dict[str, Any] = {} + colormap_spec = color_spec.get("colormap") + if colormap_spec: + if colormap_spec.get("range"): + color_query["min"] = colormap_spec["range"][0] + color_query["max"] = colormap_spec["range"][1] + if colormap_spec.get("discrete"): + color_query["scheme"] = "discrete" + if colormap_spec.get("clamp") is False: + color_query["clamp"] = False + colormap = colormaps_by_id.get(colormap_spec.get("id")) + if colormap and colormap.markers: + color_query["palette"] = [marker["color"] for marker in colormap.markers] + elif color_spec.get("single_color"): + color_query["palette"] = color_spec["single_color"] + return color_query + + +def _build_raster_style_params( + style_spec: dict[str, Any], + colormaps_by_id: dict[int, Colormap], +) -> dict[str, Any]: + """Build django-large-image style JSON from style_spec (mirrors client getRasterTilesQuery).""" + query: dict[str, Any] = {} + for color_spec in style_spec.get("colors", []): + color_query = _color_query_from_spec(color_spec, colormaps_by_id) + if not color_spec.get("visible"): + continue + if color_spec.get("name") == "all": + query = color_query + elif color_query: + query.setdefault("bands", []) + color_query["band"] = color_spec["name"].replace("Band ", "") + query["bands"].append(color_query) + + for filter_spec in style_spec.get("filters", []): + if ( + filter_spec.get("include") + and filter_spec.get("filter_by") + and filter_spec.get("list") + and len(filter_spec["list"]) == 1 + and filter_spec["filter_by"] not in RASTER_SOURCE_FILTER_KEYS + ): + query[filter_spec["filter_by"]] = filter_spec["list"][0] + + return query + + +def resolve_raster_style_params(layer_style: LayerStyle) -> tuple[dict[str, Any], bool] | None: + """Return (params, reconstructed), or None when a referenced colormap is missing.""" + if layer_style.raster_style_params is not None: + return dict(layer_style.raster_style_params), False + + style_spec = layer_style.repr_style_configs() + colormap_ids = { + color_spec["colormap"]["id"] + for color_spec in style_spec.get("colors", []) + if color_spec.get("colormap", {}).get("id") + } + colormaps_by_id = { + colormap.id: colormap for colormap in Colormap.objects.filter(id__in=colormap_ids) + } + if colormap_ids - colormaps_by_id.keys(): + return None + + return _build_raster_style_params(style_spec, colormaps_by_id), True + + +def multiframe_layers_qs( + *, + layer_id: int | None = None, + dataset_id: int | None = None, + project_id: int | None = None, +) -> QuerySet: + qs = ( + layer_queryset_with_previews() + .annotate(raster_frame_count=Count("frames", filter=Q(frames__raster__isnull=False))) + .filter(raster_frame_count__gt=1) + ) + if layer_id is not None: + qs = qs.filter(id=layer_id) + if dataset_id is not None: + qs = qs.filter(dataset_id=dataset_id) + if project_id is not None: + project = Project.objects.filter(id=project_id).first() + if project is None: + return qs.none() + qs = qs.filter(dataset__in=project.datasets.all()) + return qs.order_by("id") + + +def _needs_generate(layer: Layer, params: dict[str, Any]) -> bool: + fingerprint = params_fingerprint(params) + if preview_status_for_fingerprint(layer, fingerprint) == "ready": + return False + return pending_preview_task(layer_id=layer.id, fingerprint=fingerprint) is None + + +def _collect_backfill_work( + layer: Layer, + *, + project_id: int | None, +) -> list[tuple[dict[str, Any], LayerStyle | None]]: + work: list[tuple[dict[str, Any], LayerStyle | None]] = [] + + styles = LayerStyle.objects.filter(layer=layer).order_by("id") + if project_id is not None: + styles = styles.filter(project_id=project_id) + + for layer_style in styles: + resolved = resolve_raster_style_params(layer_style) + if resolved is None: + continue + + params, reconstructed = resolved + if reconstructed: + layer_style.raster_style_params = params + layer_style.save(update_fields=["raster_style_params"]) + work.append((params, layer_style)) + + empty_fingerprint = params_fingerprint({}) + if _needs_generate(layer, {}) and not any( + params_fingerprint(params) == empty_fingerprint for params, _ in work + ): + work.insert(0, ({}, None)) + + return work + + +def _log_backfill_target( + echo: Callable[[str], None], + *, + layer_style: LayerStyle | None, +) -> None: + if layer_style is None: + echo(" native RGB (no style params)") + else: + echo(f" LayerStyle {layer_style.id} ({layer_style.name})") + + +def backfill_layer_previews( + layer: Layer, + *, + project_id: int | None = None, + echo: Callable[[str], None] | None = None, +) -> None: + if not layer.is_multiframe_raster(): + return + + for params, layer_style in _collect_backfill_work(layer, project_id=project_id): + if not _needs_generate(layer, params): + continue + if echo is not None: + _log_backfill_target(echo, layer_style=layer_style) + invalidate_and_enqueue_layer_previews( + layer, + params, + run_mode=TaskRunMode.SYNC, + layer_style=layer_style, + ) diff --git a/uvdat/core/management/commands/backfill_frame_previews.py b/uvdat/core/management/commands/backfill_frame_previews.py new file mode 100644 index 00000000..5dcd60db --- /dev/null +++ b/uvdat/core/management/commands/backfill_frame_previews.py @@ -0,0 +1,46 @@ +r"""Backfill multiframe raster frame previews for existing layers and styles. + +Dev reset + backfill via docker compose: + +docker compose run --rm django ./manage.py clear_frame_previews +docker compose run --rm django ./manage.py backfill_frame_previews +""" + +from __future__ import annotations + +import djclick as click + +from uvdat.core.management.backfill_frame_previews_utils import ( + backfill_layer_previews, + multiframe_layers_qs, +) + + +@click.command() +@click.option("--layer-id", type=int, default=None) +@click.option("--dataset-id", type=int, default=None) +@click.option("--project-id", type=int, default=None) +def backfill_frame_previews( + *, + layer_id: int | None, + dataset_id: int | None, + project_id: int | None, +): + """Backfill multiframe raster frame previews for existing layers and styles.""" + layers = list( + multiframe_layers_qs( + layer_id=layer_id, + dataset_id=dataset_id, + project_id=project_id, + ) + ) + + if not layers: + click.secho("No multiframe raster layers matched the filters.", fg="yellow") + return + + for layer in layers: + click.echo(f"Layer {layer.id}") + backfill_layer_previews(layer, project_id=project_id, echo=click.echo) + + click.secho("Done.", fg="green") diff --git a/uvdat/core/management/commands/clear_frame_previews.py b/uvdat/core/management/commands/clear_frame_previews.py new file mode 100644 index 00000000..9957ad80 --- /dev/null +++ b/uvdat/core/management/commands/clear_frame_previews.py @@ -0,0 +1,24 @@ +r"""Clear all raster frame previews and layer style raster params. + +Dev reset + backfill via docker compose: + +docker compose run --rm django ./manage.py clear_frame_previews +docker compose run --rm django ./manage.py backfill_frame_previews +""" + +from __future__ import annotations + +import djclick as click + +from uvdat.core.models import LayerStyle, RasterFramePreview + + +@click.command() +def clear_frame_previews(): + """Delete all frame previews and set raster_style_params to None on all layer styles.""" + preview_count, _ = RasterFramePreview.objects.all().delete() + style_count = LayerStyle.objects.update(raster_style_params=None) + + click.echo(f"Deleted {preview_count} RasterFramePreview rows") + click.echo(f"Cleared raster_style_params on {style_count} LayerStyle rows") + click.secho("Done.", fg="green") diff --git a/web/src/components/sidebars/LayerStyle.vue b/web/src/components/sidebars/LayerStyle.vue index cd2a58e8..1033d482 100644 --- a/web/src/components/sidebars/LayerStyle.vue +++ b/web/src/components/sidebars/LayerStyle.vue @@ -84,38 +84,23 @@ const setCurrentLayerStyle = (style: LayerStyle) => { styleStore.selectedLayerStyles[styleKey.value] = style; }; -// After a style save, the backend invalidates multiframe raster previews and -// regenerates them asynchronously. Apply the saved style (which the API returns -// with preview_status "notready" and no previews), drop any stale previews, and -// remove the on-map preview overlay so tiles are shown until regeneration -// completes and the WebSocket handler reattaches fresh previews. +// After a style save, apply the API response. When previews were invalidated, +// clear stale payloads/overlays; onPreviewTaskComplete reloads them when ready. const markStyleSavedAndInvalidatePreviews = (style: LayerStyle) => { - const previewStatus = - style.preview_status === "ready" && !style.multiframe_previews - ? "notready" - : (style.preview_status ?? "notready"); - const invalidatedStyle: LayerStyle = { - ...style, - preview_status: previewStatus, - multiframe_previews: undefined, - }; - setCurrentLayerStyle(invalidatedStyle); - if (invalidatedStyle.is_default) { - layerStore.selectedLayers = layerStore.selectedLayers.map((layer) => { - if ( - layer.id !== props.layer.id || - layer.copy_id !== props.layer.copy_id - ) { - return layer; - } - return { - ...layer, - preview_status: invalidatedStyle.preview_status, - multiframe_previews: undefined, - }; - }); + const previewsStillValid = + style.preview_status === "ready" && !!style.multiframe_previews?.length; + setCurrentLayerStyle( + previewsStillValid + ? cloneDeep(style) + : { + ...style, + preview_status: style.preview_status ?? "notready", + multiframe_previews: undefined, + }, + ); + if (!previewsStillValid) { + framePreviewStore.clearPreviewsForStyleChange(props.layer, style.id); } - framePreviewStore.dismissPreviewForLayer(props.layer); }; const appliedStyleName = computed(() => { @@ -674,15 +659,38 @@ watch( ); const debouncedStyleSpecUpdated = debounce(() => { - if (currentStyleSpec.value) { - styleStore.selectedLayerStyles[styleKey.value] = { - ...currentLayerStyle.value, - style_spec: currentStyleSpec.value, - }; - styleStore.updateLayerStyles(props.layer); - setAvailableGroups(); - unsavedChanges.value = true; + if (!currentStyleSpec.value) return; + + const prev = currentLayerStyle.value; + const hadPreviews = + prev?.preview_status === "ready" || !!prev?.multiframe_previews?.length; + let clearPreviews = false; + if (hadPreviews && prev?.style_spec && showRasterOptions.value) { + const prevQuery = JSON.stringify( + styleStore.getRasterTilesQuery(prev.style_spec, styleStore.colormaps), + ); + const nextQuery = JSON.stringify( + styleStore.getRasterTilesQuery( + currentStyleSpec.value, + styleStore.colormaps, + ), + ); + clearPreviews = prevQuery !== nextQuery; + } + + styleStore.selectedLayerStyles[styleKey.value] = { + ...prev, + style_spec: currentStyleSpec.value, + ...(clearPreviews + ? { preview_status: "notready" as const, multiframe_previews: undefined } + : {}), + }; + if (clearPreviews) { + framePreviewStore.clearPreviewsForStyleChange(props.layer, prev.id); } + styleStore.updateLayerStyles(props.layer); + setAvailableGroups(); + unsavedChanges.value = true; }, 100); watch(currentStyleSpec, debouncedStyleSpecUpdated, { deep: true }); diff --git a/web/src/components/sidebars/LayersPanel.vue b/web/src/components/sidebars/LayersPanel.vue index e0b97ba1..c6a8e8ce 100644 --- a/web/src/components/sidebars/LayersPanel.vue +++ b/web/src/components/sidebars/LayersPanel.vue @@ -193,19 +193,12 @@ function setLayerActive(layer: Layer, active: boolean) { {{ element.name }}