feat: drags with modifier keys and multi-point paths

dist/build.js

@@ -54,6 +54,7 @@ const patches = [
     "peekaboo-preparation-message.patch",
     "peekaboo-sck-quarantine.patch",
     "peekaboo-dialog-path.patch",
+    "peekaboo-drag-path.patch",
 ].map((name) => join(native, "patches", name));
 function spawn(command, env) {
     return spawnSync(command[0], command.slice(1), { encoding: "utf8", env: env ?? process.env });

dist/client.js

@@ -440,7 +440,9 @@ function validateApplication(application) {
 function validateKey(request) {
     if (!isDesktopKey(request.key))
         throw new Error("Choose a named key or one printable character.");
-    const modifiers = request.modifiers;
+    validateModifiers(request.modifiers);
+}
+function validateModifiers(modifiers) {
     if (modifiers !== undefined && (!Array.isArray(modifiers) || modifiers.length > 4
         || new Set(modifiers).size !== modifiers.length
         || modifiers.some((modifier) => !DESKTOP_MODIFIERS.includes(modifier)))) {
@@ -560,10 +562,20 @@ function validateAction(request) {
                 || request.to_snapshot.length > 256 || request.to_snapshot === request.snapshot)) {
             throw new Error("Use another window's snapshot_id as to_snapshot, or omit it to drag within one window.");
         }
-        if (request.to_snapshot === undefined
+        if (request.path !== undefined) {
+            if (!Array.isArray(request.path) || request.path.length < 1 || request.path.length > 16) {
+                throw new Error("Give a drag path of 1 to 16 normalized screenshot points.");
+            }
+            if (request.to_snapshot !== undefined) {
+                throw new Error("A drag path stays in one window; omit path to drag into another window.");
+            }
+            request.path.forEach(validatePoint);
+        }
+        if (request.to_snapshot === undefined && request.path === undefined
             && request.from.x === request.to.x && request.from.y === request.to.y) {
             throw new Error("Choose distinct start and end points for a drag.");
         }
+        validateModifiers(request.modifiers);
         if (request.button !== undefined && !DESKTOP_BUTTONS.includes(request.button)) {
             throw new Error("Choose the left or right mouse button for a drag.");
         }

dist/protocol.d.ts

@@ -154,6 +154,8 @@ export type DesktopRequest = DesktopManagement | DesktopLaunch | DesktopOpen | D
     to_snapshot?: string;
     button?: DesktopButton;
     duration_ms?: number;
+    modifiers?: DesktopModifier[];
+    path?: DesktopPoint[];
 } | {
     op: "hover";
     snapshot: string;

dist/tools.js

@@ -544,9 +544,11 @@ export function desktopTools({ place, app }) {
         {
             name: "desktop_drag",
             op: "drag",
-            description: `Press, move in a straight line, and release inside the exact window from a fresh desktop_inspect result. Pass snapshot_id as snapshot; from/to are distinct normalized screenshot points (x from its left edge, y from its top edge, each >= 0 and < 1). button defaults to left, with right also supported. duration_ms defaults to 500 and must be 1 to 10000. The native bridge owns the whole gesture and release cleanup; no button stays held across calls. Background delivery never moves the physical pointer or switches windows. Accepted delivery does not prove a drop; verify the effect with a fresh desktop_inspect. An inactive view may ignore the gesture. If observation shows no effect, use explicit desktop_focus followed by a fresh desktop_inspect before deciding on another action. Unsupported or changed windows are refused. The snapshot is single-use; never blindly repeat an interrupted drag because part may already have run.
+            description: `Press, move in a straight line or along a path, and release inside the exact window from a fresh desktop_inspect result. Pass snapshot_id as snapshot; from/to are distinct normalized screenshot points (x from its left edge, y from its top edge, each >= 0 and < 1). Optional path is 1 to 16 more points in the same screenshot, visited in order between from and to, in straight segments that pass through each point; with a path, to may equal from to close a loop. button defaults to left, with right also supported. duration_ms defaults to 500 and must be 1 to 10000. The native bridge owns the whole gesture and release cleanup; no button stays held across calls. Background delivery never moves the physical pointer or switches windows. Accepted delivery does not prove a drop; verify the effect with a fresh desktop_inspect. An inactive view may ignore the gesture. If observation shows no effect, use explicit desktop_focus followed by a fresh desktop_inspect before deciding on another action. Unsupported or changed windows are refused. The snapshot is single-use; never blindly repeat an interrupted drag because part may already have run.
 
-To drag into another window or app, such as a file from Finder into an app, inspect both windows and pass the destination's snapshot_id as to_snapshot; from is then a point in snapshot and to a point in to_snapshot. This is foreground input: it moves the user's real pointer, and pressing in the source may bring its app forward. Both points must be visible and uncovered, so arrange the windows first if needed. duration_ms defaults to 800 for these drags. A drop between windows can't be confirmed, so the result is unknown; inspect both windows to see what happened.`,
+To drag into another window or app, such as a file from Finder into an app, inspect both windows and pass the destination's snapshot_id as to_snapshot; from is then a point in snapshot and to a point in to_snapshot. This is foreground input: it moves the user's real pointer, and pressing in the source may bring its app forward. Both points must be visible and uncovered, so arrange the windows first if needed. duration_ms defaults to 800 for these drags. A drop between windows can't be confirmed, so the result is unknown; inspect both windows to see what happened. A path can't be combined with to_snapshot.
+
+Optional modifiers (command, control, option and shift, each at most once) are held down for the whole drag, as for Option-drag to copy or Command-drag to move in Finder, or Shift to constrain. Held keys are system-wide, so a drag with modifiers is always foreground input like a drag into another window, even within one window: it moves the real pointer, holds the real keys until release, needs every point uncovered, defaults duration_ms to 800, and its result is unknown; inspect afterward to see what happened.`,
             parameters: {
                 type: "object",
                 required: ["snapshot", "from", "to"],
@@ -555,6 +557,8 @@ To drag into another window or app, such as a file from Finder into an app, insp
                     from: point,
                     to: point,
                     to_snapshot: id,
+                    path: { type: "array", items: point, minItems: 1, maxItems: 16 },
+                    modifiers,
                     button: {
                         anyOf: [
                             { type: "string", const: "left" },

docs/tools.md

@@ -54,7 +54,7 @@ A status item is pressed through Accessibility in its owner process without movi
 | --- | --- |
 | `desktop_click` | Single, double, right, middle or triple click on an element or a screenshot point |
 | `desktop_scroll` | Scroll an element or point up, down, left or right |
-| `desktop_drag` | Press, move in a straight line, and release inside one window, or into another window or app |
+| `desktop_drag` | Press, move in a straight line or along a path of up to 16 points, and release inside one window, or into another window or app; optionally hold modifier keys |
 | `desktop_hover` | Move the real pointer over an element or point, to show tooltips and hover states |
 | `desktop_press` | Press and hold an element or point for up to 10 seconds, then release in place |
 | `desktop_type` | Replace an editable field's whole value |
@@ -68,7 +68,7 @@ A status item is pressed through Accessibility in its owner process without movi
 | `desktop_dialog_answer` | Press one dialog button by its exact title, replace one dialog text field's value, or choose a path in an Open or Save panel |
 | `desktop_clipboard_write` | Replace the clipboard with text, an image, or file references |
 
-Pointer input is delivered to the exact window without moving the physical pointer, except for `desktop_hover` and a `desktop_drag` with `to_snapshot`. Hover effects follow the real pointer, so it moves the user's pointer and leaves it there. Before moving, it checks that the snapshot's window is still at the same place and frontmost at the point, and refuses otherwise. Afterward, it checks that the pointer arrived and the window is still frontmost there; if either can't be confirmed, the result is `unknown`. A drag into another window or app can't stay inside one window, so it also uses the real pointer, and pressing in the source may bring its app forward. Pass the destination's snapshot as `to_snapshot`. Both points must be uncovered at their snapshot positions, or the drag is refused before input. The pointer rests on the start point, then presses, pauses, moves, rests over the destination and releases. The result is always `unknown`; inspect both windows to see whether the drop happened. `desktop_press` holds the button through the same exact-window route as a drag, and the host always releases it, even when the call is interrupted or the window changes. When the host turns on the agent cursor, an orange arrow first glides to each click, scroll, drag, hover or press point, about a third of a second before the input is sent. It then shows a ripple for clicks, chevrons for scrolls and a trail for drags. For a press, it stays squeezed until the button is released, then ripples. The cursor fades after a few seconds idle. The cursor never takes clicks or focus and is left out of `desktop_inspect` and `desktop_screen` captures. It is decoration only: if it fails or is slow, input proceeds unchanged after at most 600 ms. Keyboard input, `desktop_type` and `desktop_select` don't show it.
+Pointer input is delivered to the exact window without moving the physical pointer, except for `desktop_hover` and a `desktop_drag` with `to_snapshot` or `modifiers`. Hover effects follow the real pointer, so it moves the user's pointer and leaves it there. Before moving, it checks that the snapshot's window is still at the same place and frontmost at the point, and refuses otherwise. Afterward, it checks that the pointer arrived and the window is still frontmost there; if either can't be confirmed, the result is `unknown`. A drag into another window or app can't stay inside one window, so it also uses the real pointer, and pressing in the source may bring its app forward. Pass the destination's snapshot as `to_snapshot`. Both points must be uncovered at their snapshot positions, or the drag is refused before input. The pointer rests on the start point, then presses, pauses, moves, rests over the destination and releases. The result is always `unknown`; inspect both windows to see whether the drop happened. A drag with `modifiers`, such as Option to copy or Command to move in Finder, or Shift to constrain, holds the real keys system-wide from before the press until after the release, so it takes the same foreground route even within one window, and its result is `unknown` too. A `path` adds up to 16 points visited in order in straight segments; it works in both routes within one window but not with `to_snapshot`. `desktop_press` holds the button through the same exact-window route as a drag, and the host always releases it, even when the call is interrupted or the window changes. When the host turns on the agent cursor, an orange arrow first glides to each click, scroll, drag, hover or press point, about a third of a second before the input is sent. It then shows a ripple for clicks, chevrons for scrolls and a trail for drags. For a press, it stays squeezed until the button is released, then ripples. The cursor fades after a few seconds idle. The cursor never takes clicks or focus and is left out of `desktop_inspect` and `desktop_screen` captures. It is decoration only: if it fails or is slow, input proceeds unchanged after at most 600 ms. Keyboard input, `desktop_type` and `desktop_select` don't show it.
 
 When the host turns on the window highlight, the window an inspection or action targets gets an orange outline and a label such as "Claude is using TextEdit". It shows from the moment the request reaches the host, moves to a new window when the target changes, and fades about four seconds after the last request. The outline follows the window as it moves or resizes and stays directly above it in the window order, so windows in front of the target also cover the outline. It hides while the window is minimized, off screen or on another Space. Like the cursor, it never takes clicks or focus and is left out of captures. Requests without one exact window, such as `desktop_screen`, menus or the clipboard, don't show it.
 
@@ -91,7 +91,7 @@ An action refused because its window was minimized after inspection reports `WIN
 ## Not supported
 
 - **Insertion outside text fields.** `desktop_insert` needs an observed focused text control with a readable selection.
-- **Confirmed drops between windows.** A drag into another window or app is always reported as `unknown`, because nothing can confirm the drop. Drags are one straight line, with no modifier keys.
+- **Confirmed drops between windows.** A drag into another window or app, or any drag with modifier keys, is always reported as `unknown`, because nothing can confirm the drop. A path can't cross into another window: a drag between windows is one straight line.
 - **Automatic retries.** By design, nothing is retried. After a `refused` or `unknown` result, observe again before deciding whether to act.
 - **Remote Macs.** The tools act on the Mac running the host app, on macOS 15 or later.
 

native/patches/README.md

@@ -386,6 +386,16 @@ reads back the name; otherwise it is `dispatchedUnverified`. Details report the
 permission set, semantic policy and result contract are listed in the upstream inventory tests, and new
 tests cover path resolution and request validation.
 
+`peekaboo-drag-path.patch` lets a drag follow a path. `ExactWindowDragRequest` and `DragOperationRequest`
+take up to 16 `waypoints`, visited in order between `from` and `to`. `DragPolyline` samples the path in
+straight segments with at least one sample per segment, shares the rest by segment length, and lands
+exactly on every waypoint, so a straight drag samples the same points as before. The exact-window drag
+keeps its sample count and dispatched unit count, and refuses waypoints outside its window or more
+segments than samples. The foreground drag follows the same samples instead of its movement profile
+when waypoints are given; its modifier keys were already supported. The bridge payload and the
+exact-window request decode a missing `waypoints` as empty. Tests in
+`ExactWindowHeldPointerLifecycleTests.swift` cover the samples, the refusals and decoding.
+
 `desktop-tools-build` resolves only `Package.resolved` versions, checks the pin and checkout revision,
 and assembles the ordered patch stack in a private Git index. A build compares the checkout
 with each complete ordered prefix, since later patches can change earlier patch contexts.