Add YUV420P to RGBA shader and canvas render metrics

Author: richiemcilroy

apps/desktop/src/routes/editor/Player.tsx

@@ -184,7 +184,6 @@ export function PlayerContent() {
 				await commands.stopPlayback();
 				setEditorState("playing", false);
 			} else {
-				// Ensure we seek to the current playback time before starting playback
 				await commands.seekTo(Math.floor(editorState.playbackTime * FPS));
 				await commands.startPlayback(FPS, previewResolutionBase());
 				setEditorState("playing", true);
@@ -463,12 +462,60 @@ function PreviewCanvas() {
 		createSignal<HTMLDivElement>();
 	const containerBounds = createElementBounds(canvasContainerRef);
 
+	// #region agent log
+	let canvasRenderCount = 0;
+	let totalCanvasRenderTime = 0;
+	let maxCanvasRenderTime = 0;
+	let lastCanvasMetricsLog = performance.now();
+	// #endregion
+
 	createEffect(() => {
 		const frame = latestFrame();
 		if (!frame) return;
 		if (!canvasRef) return;
-		const ctx = canvasRef.getContext("2d");
-		ctx?.putImageData(frame.data, 0, 0);
+		const ctx = canvasRef.getContext("2d", { alpha: false });
+		if (!ctx) return;
+		// #region agent log
+		const renderStart = performance.now();
+		// #endregion
+		createImageBitmap(frame.data).then((bitmap) => {
+			ctx.drawImage(bitmap, 0, 0);
+			bitmap.close();
+			// #region agent log
+			const renderTime = performance.now() - renderStart;
+			canvasRenderCount++;
+			totalCanvasRenderTime += renderTime;
+			maxCanvasRenderTime = Math.max(maxCanvasRenderTime, renderTime);
+			if (
+				performance.now() - lastCanvasMetricsLog >= 2000 &&
+				canvasRenderCount > 0
+			) {
+				const avgTime = totalCanvasRenderTime / canvasRenderCount;
+				fetch(
+					"http://127.0.0.1:7242/ingest/966647b7-72f6-4ab7-b76e-6b773ac020d7",
+					{
+						method: "POST",
+						headers: { "Content-Type": "application/json" },
+						body: JSON.stringify({
+							location: "Player.tsx:canvas_render",
+							message: "canvas render metrics",
+							data: {
+								canvasRenderCount,
+								avgRenderTimeMs: avgTime.toFixed(2),
+								maxRenderTimeMs: maxCanvasRenderTime.toFixed(2),
+								frameWidth: frame.width,
+								frameHeight: frame.data.height,
+							},
+							timestamp: Date.now(),
+							sessionId: "debug-session",
+							hypothesisId: "E",
+						}),
+					},
+				).catch(() => {});
+				lastCanvasMetricsLog = performance.now();
+			}
+			// #endregion
+		});
 	});
 
 	return (

crates/rendering/src/shaders/yuv420p_to_rgba.wgsl

@@ -0,0 +1,36 @@
+@group(0) @binding(0) var y_plane: texture_2d<f32>;
+@group(0) @binding(1) var u_plane: texture_2d<f32>;
+@group(0) @binding(2) var v_plane: texture_2d<f32>;
+@group(0) @binding(3) var output: texture_storage_2d<rgba8unorm, write>;
+
+@compute @workgroup_size(8, 8)
+fn main(@builtin(global_invocation_id) global_id: vec3<u32>) {
+    let coords = global_id.xy;
+    let dims = textureDimensions(output);
+
+    if (coords.x >= dims.x || coords.y >= dims.y) {
+        return;
+    }
+
+    let y_raw = textureLoad(y_plane, coords, 0).r;
+    let uv_coords = coords / 2;
+    let u_raw = textureLoad(u_plane, uv_coords, 0).r;
+    let v_raw = textureLoad(v_plane, uv_coords, 0).r;
+
+    let y = (y_raw - 0.0625) * 1.164;
+    let u = u_raw - 0.5;
+    let v = v_raw - 0.5;
+
+    let r = y + 1.596 * v;
+    let g = y - 0.391 * u - 0.813 * v;
+    let b = y + 2.018 * u;
+
+    let color = vec4<f32>(
+        clamp(r, 0.0, 1.0),
+        clamp(g, 0.0, 1.0),
+        clamp(b, 0.0, 1.0),
+        1.0
+    );
+
+    textureStore(output, coords, color);
+}