From 0310606cea0d138dafd0c9ded74448e41f9746b0 Mon Sep 17 00:00:00 2001 From: Kenner Miner Date: Mon, 30 Mar 2026 13:04:09 +0800 Subject: [PATCH 1/2] Fix game_view screenshots to capture UI Toolkit overlays Use ScreenCapture.CaptureScreenshotAsTexture() for game_view screenshots when include_image=true and in Play mode. This captures the final composited frame including UI Toolkit overlays, which camera.Render() misses since UI Toolkit renders at the compositor level after camera rendering. The camera-based path is still used when a specific camera is requested or when not in Play mode. Generated with [Claude Code](https://claude.ai/code) via [Happy](https://happy.engineering) Co-Authored-By: Claude Co-Authored-By: Happy --- MCPForUnity/Editor/Tools/ManageScene.cs | 80 ++++++++++++++++--- .../Runtime/Helpers/ScreenshotUtility.cs | 69 ++++++++++++++++ 2 files changed, 138 insertions(+), 11 deletions(-) diff --git a/MCPForUnity/Editor/Tools/ManageScene.cs b/MCPForUnity/Editor/Tools/ManageScene.cs index 4826ea850..770caa42b 100644 --- a/MCPForUnity/Editor/Tools/ManageScene.cs +++ b/MCPForUnity/Editor/Tools/ManageScene.cs @@ -589,26 +589,84 @@ private static object CaptureScreenshot(SceneCommand cmd) } } - // When a specific camera is requested or include_image is true, always use camera-based capture - // (synchronous, gives us bytes in memory for base64). - if (targetCamera != null || includeImage) + // When include_image is requested but no specific camera, use composited capture + // (ScreenCapture.CaptureScreenshotAsTexture) which captures UI Toolkit overlays. + // When a specific camera IS requested, use camera-based capture. + if (targetCamera != null) { + if (!Application.isBatchMode) EnsureGameView(); + + ScreenshotCaptureResult result = ScreenshotUtility.CaptureFromCameraToAssetsFolder( + targetCamera, fileName, resolvedSuperSize, ensureUniqueFileName: true, + includeImage: includeImage, maxResolution: maxResolution); + + AssetDatabase.ImportAsset(result.AssetsRelativePath, ImportAssetOptions.ForceSynchronousImport); + string message = $"Screenshot captured to '{result.AssetsRelativePath}' (camera: {targetCamera.name})."; + + var data = new Dictionary + { + { "path", result.AssetsRelativePath }, + { "fullPath", result.FullPath }, + { "superSize", result.SuperSize }, + { "isAsync", false }, + { "camera", targetCamera.name }, + { "captureSource", "game_view" }, + }; + if (includeImage && result.ImageBase64 != null) + { + data["imageBase64"] = result.ImageBase64; + data["imageWidth"] = result.ImageWidth; + data["imageHeight"] = result.ImageHeight; + } + return new SuccessResponse(message, data); + } + + if (includeImage && Application.isPlaying) + { + if (!Application.isBatchMode) EnsureGameView(); + + ScreenshotCaptureResult result = ScreenshotUtility.CaptureComposited( + fileName, resolvedSuperSize, ensureUniqueFileName: true, + includeImage: true, maxResolution: maxResolution); + + AssetDatabase.ImportAsset(result.AssetsRelativePath, ImportAssetOptions.ForceSynchronousImport); + string cameraName = Camera.main != null ? Camera.main.name : "composited"; + string message = $"Screenshot captured to '{result.AssetsRelativePath}' (camera: {cameraName})."; + + var data = new Dictionary + { + { "path", result.AssetsRelativePath }, + { "fullPath", result.FullPath }, + { "superSize", result.SuperSize }, + { "isAsync", false }, + { "camera", cameraName }, + { "captureSource", "game_view" }, + }; + if (result.ImageBase64 != null) + { + data["imageBase64"] = result.ImageBase64; + data["imageWidth"] = result.ImageWidth; + data["imageHeight"] = result.ImageHeight; + } + return new SuccessResponse(message, data); + } + + if (includeImage) + { + // Not in play mode — fall back to camera-based capture + targetCamera = Camera.main; if (targetCamera == null) { - targetCamera = Camera.main; - if (targetCamera == null) - { #if UNITY_2022_2_OR_NEWER - var allCams = UnityEngine.Object.FindObjectsByType(FindObjectsSortMode.None); + var allCams = UnityEngine.Object.FindObjectsByType(FindObjectsSortMode.None); #else - var allCams = UnityEngine.Object.FindObjectsOfType(); + var allCams = UnityEngine.Object.FindObjectsOfType(); #endif - targetCamera = allCams.Length > 0 ? allCams[0] : null; - } + targetCamera = allCams.Length > 0 ? allCams[0] : null; } if (targetCamera == null) { - return new ErrorResponse("No camera found in the scene. Add a Camera to use screenshot with camera or include_image."); + return new ErrorResponse("No camera found in the scene. Add a Camera to use screenshot with include_image outside of Play mode."); } if (!Application.isBatchMode) EnsureGameView(); diff --git a/MCPForUnity/Runtime/Helpers/ScreenshotUtility.cs b/MCPForUnity/Runtime/Helpers/ScreenshotUtility.cs index 6c32cd62b..872c12bf9 100644 --- a/MCPForUnity/Runtime/Helpers/ScreenshotUtility.cs +++ b/MCPForUnity/Runtime/Helpers/ScreenshotUtility.cs @@ -228,6 +228,75 @@ public static ScreenshotCaptureResult CaptureFromCameraToAssetsFolder( return result; } + /// + /// Captures a screenshot using ScreenCapture.CaptureScreenshotAsTexture, which captures the + /// final composited frame including UI Toolkit overlays, post-processing, etc. + /// Falls back to camera-based capture if ScreenCapture is unavailable. + /// + public static ScreenshotCaptureResult CaptureComposited( + string fileName = null, + int superSize = 1, + bool ensureUniqueFileName = true, + bool includeImage = false, + int maxResolution = 0) + { + ScreenshotCaptureResult result = PrepareCaptureResult(fileName, superSize, ensureUniqueFileName, isAsync: false); + Texture2D tex = null; + Texture2D downscaled = null; + string imageBase64 = null; + int imgW = 0, imgH = 0; + try + { + tex = ScreenCapture.CaptureScreenshotAsTexture(result.SuperSize); + if (tex == null) + { + // Fallback to camera-based if ScreenCapture fails + var cam = Camera.main; + if (cam != null) + return CaptureFromCameraToAssetsFolder(cam, fileName, superSize, ensureUniqueFileName, includeImage, maxResolution); + throw new InvalidOperationException("ScreenCapture.CaptureScreenshotAsTexture returned null and no fallback camera available."); + } + + int width = tex.width; + int height = tex.height; + + byte[] png = tex.EncodeToPNG(); + File.WriteAllBytes(result.FullPath, png); + + if (includeImage) + { + int targetMax = maxResolution > 0 ? maxResolution : 640; + if (width > targetMax || height > targetMax) + { + downscaled = DownscaleTexture(tex, targetMax); + byte[] smallPng = downscaled.EncodeToPNG(); + imageBase64 = System.Convert.ToBase64String(smallPng); + imgW = downscaled.width; + imgH = downscaled.height; + } + else + { + imageBase64 = System.Convert.ToBase64String(png); + imgW = width; + imgH = height; + } + } + } + finally + { + DestroyTexture(tex); + DestroyTexture(downscaled); + } + + if (includeImage && imageBase64 != null) + { + return new ScreenshotCaptureResult( + result.FullPath, result.AssetsRelativePath, result.SuperSize, false, + imageBase64, imgW, imgH); + } + return result; + } + /// /// Renders a camera to a Texture2D without saving to disk. Used for multi-angle captures. /// Returns the base64-encoded PNG, downscaled to fit within . From 369695cdd83b7254ca50fb2fb083df22ca838c97 Mon Sep 17 00:00:00 2001 From: Kenner Miner Date: Mon, 6 Apr 2026 11:10:46 +0800 Subject: [PATCH 2/2] fix: address screenshot PR feedback --- MCPForUnity/Editor/Tools/ManageScene.cs | 79 +++++++------------ .../Runtime/Helpers/ScreenshotUtility.cs | 11 ++- 2 files changed, 38 insertions(+), 52 deletions(-) diff --git a/MCPForUnity/Editor/Tools/ManageScene.cs b/MCPForUnity/Editor/Tools/ManageScene.cs index 770caa42b..d3ecd47c0 100644 --- a/MCPForUnity/Editor/Tools/ManageScene.cs +++ b/MCPForUnity/Editor/Tools/ManageScene.cs @@ -602,23 +602,7 @@ private static object CaptureScreenshot(SceneCommand cmd) AssetDatabase.ImportAsset(result.AssetsRelativePath, ImportAssetOptions.ForceSynchronousImport); string message = $"Screenshot captured to '{result.AssetsRelativePath}' (camera: {targetCamera.name})."; - - var data = new Dictionary - { - { "path", result.AssetsRelativePath }, - { "fullPath", result.FullPath }, - { "superSize", result.SuperSize }, - { "isAsync", false }, - { "camera", targetCamera.name }, - { "captureSource", "game_view" }, - }; - if (includeImage && result.ImageBase64 != null) - { - data["imageBase64"] = result.ImageBase64; - data["imageWidth"] = result.ImageWidth; - data["imageHeight"] = result.ImageHeight; - } - return new SuccessResponse(message, data); + return new SuccessResponse(message, BuildScreenshotResponseData(result, targetCamera.name, includeImage)); } if (includeImage && Application.isPlaying) @@ -632,23 +616,7 @@ private static object CaptureScreenshot(SceneCommand cmd) AssetDatabase.ImportAsset(result.AssetsRelativePath, ImportAssetOptions.ForceSynchronousImport); string cameraName = Camera.main != null ? Camera.main.name : "composited"; string message = $"Screenshot captured to '{result.AssetsRelativePath}' (camera: {cameraName})."; - - var data = new Dictionary - { - { "path", result.AssetsRelativePath }, - { "fullPath", result.FullPath }, - { "superSize", result.SuperSize }, - { "isAsync", false }, - { "camera", cameraName }, - { "captureSource", "game_view" }, - }; - if (result.ImageBase64 != null) - { - data["imageBase64"] = result.ImageBase64; - data["imageWidth"] = result.ImageWidth; - data["imageHeight"] = result.ImageHeight; - } - return new SuccessResponse(message, data); + return new SuccessResponse(message, BuildScreenshotResponseData(result, cameraName, includeImage: true)); } if (includeImage) @@ -677,23 +645,7 @@ private static object CaptureScreenshot(SceneCommand cmd) AssetDatabase.ImportAsset(result.AssetsRelativePath, ImportAssetOptions.ForceSynchronousImport); string message = $"Screenshot captured to '{result.AssetsRelativePath}' (camera: {targetCamera.name})."; - - var data = new Dictionary - { - { "path", result.AssetsRelativePath }, - { "fullPath", result.FullPath }, - { "superSize", result.SuperSize }, - { "isAsync", false }, - { "camera", targetCamera.name }, - { "captureSource", "game_view" }, - }; - if (includeImage && result.ImageBase64 != null) - { - data["imageBase64"] = result.ImageBase64; - data["imageWidth"] = result.ImageWidth; - data["imageHeight"] = result.ImageHeight; - } - return new SuccessResponse(message, data); + return new SuccessResponse(message, BuildScreenshotResponseData(result, targetCamera.name, includeImage)); } // Default path: use ScreenCapture API if available, camera fallback otherwise @@ -754,6 +706,31 @@ private static object CaptureScreenshot(SceneCommand cmd) } } + private static Dictionary BuildScreenshotResponseData( + ScreenshotCaptureResult result, + string cameraName, + bool includeImage) + { + var data = new Dictionary + { + { "path", result.AssetsRelativePath }, + { "fullPath", result.FullPath }, + { "superSize", result.SuperSize }, + { "isAsync", false }, + { "camera", cameraName }, + { "captureSource", "game_view" }, + }; + + if (includeImage && result.ImageBase64 != null) + { + data["imageBase64"] = result.ImageBase64; + data["imageWidth"] = result.ImageWidth; + data["imageHeight"] = result.ImageHeight; + } + + return data; + } + private static object CaptureSceneViewScreenshot( SceneCommand cmd, string fileName, diff --git a/MCPForUnity/Runtime/Helpers/ScreenshotUtility.cs b/MCPForUnity/Runtime/Helpers/ScreenshotUtility.cs index 872c12bf9..99a8ca3f3 100644 --- a/MCPForUnity/Runtime/Helpers/ScreenshotUtility.cs +++ b/MCPForUnity/Runtime/Helpers/ScreenshotUtility.cs @@ -240,6 +240,15 @@ public static ScreenshotCaptureResult CaptureComposited( bool includeImage = false, int maxResolution = 0) { + if (!IsScreenCaptureModuleAvailable) + { + var fallbackCamera = FindAvailableCamera(); + if (fallbackCamera != null) + return CaptureFromCameraToAssetsFolder(fallbackCamera, fileName, superSize, ensureUniqueFileName, includeImage, maxResolution); + + throw new InvalidOperationException("ScreenCapture module is unavailable and no fallback camera found."); + } + ScreenshotCaptureResult result = PrepareCaptureResult(fileName, superSize, ensureUniqueFileName, isAsync: false); Texture2D tex = null; Texture2D downscaled = null; @@ -251,7 +260,7 @@ public static ScreenshotCaptureResult CaptureComposited( if (tex == null) { // Fallback to camera-based if ScreenCapture fails - var cam = Camera.main; + var cam = FindAvailableCamera(); if (cam != null) return CaptureFromCameraToAssetsFolder(cam, fileName, superSize, ensureUniqueFileName, includeImage, maxResolution); throw new InvalidOperationException("ScreenCapture.CaptureScreenshotAsTexture returned null and no fallback camera available.");