Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions contracts/fixtures/dispatch-disclosure.json
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,24 @@
"trigger": "fill admission (admitFill) refused the request before any backend call, e.g. a target with no text",
"dispatched": "no"
},
{
"id": "daemon.refusal.gesture-drag-admission",
"producer": "daemon",
"trigger": "a drag target's @ref admission refused before the gesture runtime was bound, without an earlier readiness recovery",
"dispatched": "no"
},
{
"id": "daemon.post-dispatch.gesture-drag-after-recovery",
"producer": "daemon",
"trigger": "Android readiness tapped Close app and relaunched the app, then drag ref admission refused before the gesture runtime was bound",
"dispatched": "unknown"
},
{
"id": "daemon.post-dispatch.ref-after-android-recovery",
"producer": "daemon",
"trigger": "Android blocking-dialog readiness tapped Close app and relaunched the app, then re-admission aborted the requested ref press",
"dispatched": "unknown"
},
{
"id": "daemon.refusal.selector-readiness-exhausted",
"producer": "daemon",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,16 @@ import {
} from '../../../request-dispatch-ledger.ts';
import type { BindDeviceRuntime } from '../../../request-runtime-binding.ts';
import { clearAndroidObservationFixture } from '../../../__tests__/android-observation-fixture.ts';
import { activateCompleteRefFrame, expireRefFrame } from '../../../ref-frame.ts';
import { handleInteractionCommands } from '../../index.ts';
import type { InteractionRouteInput } from '../types.ts';
import { assertAndroidPressStayedInApp } from '../interaction-android-escape.ts';
import { gestureRuntimeBindingsFixture } from './gesture-runtime-bindings.fixtures.ts';
import { contextFromFlags, makeSession } from './interaction-touch-fixtures.ts';
import {
contextFromFlags,
makeSession,
makeStaleRefSession,
} from './interaction-touch-fixtures.ts';

// contracts/fixtures/dispatch-disclosure.json, daemon and post-action guard rows: the daemon rows
// drive a real `press` through the daemon interaction handler, inside the router's dispatch seam,
Expand Down Expand Up @@ -210,6 +215,110 @@ async function swipeRefusedOnSecondRepetition(): Promise<unknown> {
throw new AppError(response.error.code, response.error.message, response.error.details);
}

/** A stale drag ref is rejected before binding any gesture operation. */
async function gestureDragRefusedBeforeDispatch(): Promise<unknown> {
const gestures = gestureRuntimeBindingsFixture();
const sessionStore = makeSessionStore();
const session = makeStaleRefSession('dispatch-disclosure-stale-drag');
expireRefFrame(session);
sessionStore.publish(session.name, session);
const response = await routeInteraction({
req: {
token: 't',
session: session.name,
command: 'gesture',
positionals: [],
flags: {},
input: {
kind: 'drag',
source: '@e1',
destination: '@e2',
sourceHoldMs: 100,
moveMs: 100,
destinationHoldMs: 100,
},
},
sessionName: session.name,
sessionStore,
contextFromFlags,
inspectFacts: gestures.inspectFacts,
bindDevice: gestures.bindDevice,
});
assert.ok(response && !response.ok, 'expected stale drag admission to fail');
assert.equal(response.error.details?.reason, 'ref_frame_expired');
assert.equal(gestures.bindDevice.mock.calls.length, 0, 'admission must precede runtime binding');
throw new AppError(response.error.code, response.error.message, response.error.details);
}

async function refRefusedAfterAndroidRecovery(command: 'press' | 'gesture'): Promise<unknown> {
const gestures = gestureRuntimeBindingsFixture();
const sessionStore = makeSessionStore();
const session = makeAndroidSession('dispatch-disclosure-recovered-drag', {
appBundleId: 'com.example.app',
});
session.snapshot = makeStaleRefSession('recovery-frame').snapshot;
activateCompleteRefFrame(session);
sessionStore.publish(session.name, session);
const tap = vi.fn(clearAndroidObservationFixture.tap);
const openApp = vi.fn(clearAndroidObservationFixture.openApp);
const androidObservation: AndroidObservationAdapter = {
...clearAndroidObservationFixture,
readBlockingDialog: async () => ({
status: 'dialog',
focus: { package: 'com.example.app', focusedWindow: 'Application Not Responding', raw: '' },
}),
readSnapshotNodes: async () => [
{
index: 0,
ref: 'e0',
type: 'Button',
label: 'Close app',
rect: { x: 10, y: 20, width: 100, height: 40 },
},
],
tap,
openApp,
};
const response = await routeInteraction({
req: {
token: 't',
session: session.name,
command,
positionals: command === 'press' ? ['@e1'] : [],
flags: {},
input: {
kind: 'drag',
source: '@e1',
destination: '@e2',
sourceHoldMs: 100,
moveMs: 100,
destinationHoldMs: 100,
},
},
sessionName: session.name,
sessionStore,
contextFromFlags,
...(command === 'gesture'
? { inspectFacts: gestures.inspectFacts, bindDevice: gestures.bindDevice }
: getRuntimeBindings()),
androidObservation,
});
assert.ok(response && !response.ok, 'expected stale drag admission to fail after recovery');
assert.equal(response.error.details?.reason, 'ref_frame_expired');
assert.equal(tap.mock.calls.length, 1);
assert.equal(openApp.mock.calls.length, 1);
assert.equal(mockTapPoint.mock.calls.length, 0);
if (command === 'gesture') {
assert.equal(gestures.bindDevice.mock.calls.length, 0);
assert.equal(
response.error.details?.warning,
'Recovered Android app ANR before gesture: closed and relaunched com.example.app.',
);
assert.match(response.error.hint ?? '', /Capture a fresh interactive snapshot/);
}
throw new AppError(response.error.code, response.error.message, response.error.details);
}

/** An Android press whose tap returns and whose post-press foreground read is refused with `no`. */
async function pressThenForegroundReadRefused(): Promise<unknown> {
const session = makeAndroidSession('dispatch-disclosure-post-dispatch', {
Expand Down Expand Up @@ -247,6 +356,10 @@ const DRIVERS: Record<string, () => Promise<unknown>> = {
'daemon.refusal.selector-readiness-exhausted': pressWhoseTargetNeverAppears,
'daemon.unclassified': () => pressAfterUnclassifiedTouchFailure(),
'daemon.series.swipe-later-repetition-refused': swipeRefusedOnSecondRepetition,
'daemon.refusal.gesture-drag-admission': gestureDragRefusedBeforeDispatch,
'daemon.post-dispatch.gesture-drag-after-recovery': () =>
refRefusedAfterAndroidRecovery('gesture'),
'daemon.post-dispatch.ref-after-android-recovery': () => refRefusedAfterAndroidRecovery('press'),
'daemon.post-dispatch.press-then-foreground-read-refused': pressThenForegroundReadRefused,
'daemon.read-only-command': () =>
press({ command: 'get', positionals: ['text', 'label="Missing"'] }),
Expand Down
44 changes: 32 additions & 12 deletions src/daemon/interaction/internal/interaction-gesture.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import {
SWIPE_REPETITION_MAX,
SWIPE_SERIES_MAX_SCHEDULED_DURATION_MS,
} from '@agent-device/contracts/scroll-gesture';
import { AppError, normalizeError } from '@agent-device/kernel/errors';
import { AppError, asAppError, normalizeError } from '@agent-device/kernel/errors';
import {
REF_GRAMMAR_HINT,
splitRefGenerationSuffix,
Expand All @@ -27,9 +27,13 @@ import {
import { resolveBoundGestureRuntime, type BoundGestureExecutor } from '../../gesture-runtime.ts';
import { isActiveProviderDevice } from '../../provider-device-admission.ts';
import { sleep } from '@agent-device/host-kit/retry';
import { ensureAndroidBlockingSystemDialogReady } from '../../android-system-dialog.ts';
import {
ensureAndroidBlockingSystemDialogReady,
type AndroidBlockingDialogReadinessResult,
} from '../../android-system-dialog.ts';
import { readRefMutationFrame } from '../../ref-frame.ts';
import type { DaemonResponse } from '../../daemon-request.ts';
import { thrownBeforeDispatch } from '../../request-dispatch-disclosure.ts';
import type { SessionState } from '../../session-state.ts';
import { assertRefMutationAdmitted } from './interaction-ref-policy.ts';
import {
Expand Down Expand Up @@ -73,17 +77,18 @@ export async function dispatchGestureViaRuntime(
params: GestureHandlerParams,
): Promise<DaemonResponse> {
params = bindInteractionSession(params);
return await dispatchGestureInteraction(params, 'gesture', async (session) =>
runGestureInteraction(params, session),
return await dispatchGestureInteraction(params, 'gesture', async (session, readiness) =>
runGestureInteraction(params, session, readiness),
);
}

async function runGestureInteraction(
params: GestureHandlerParams,
session: SessionState,
readiness: AndroidBlockingDialogReadinessResult,
): Promise<GestureInteractionResult> {
const input = readGesturePayload(params.req.input);
const gesture = prepareGestureCommandInput(input, session);
const gesture = prepareGestureCommandInput(input, session, readiness);
if (gesture.intent === 'pan' && params.req.internal?.gestureExecutionProfile) {
gesture.executionProfile = params.req.internal.gestureExecutionProfile;
}
Expand Down Expand Up @@ -207,7 +212,10 @@ function createGestureRuntime(params: GestureHandlerParams, gestures: BoundGestu
async function dispatchGestureInteraction(
params: GestureHandlerParams,
command: 'gesture' | 'swipe',
run: (session: SessionState) => Promise<GestureInteractionResult>,
run: (
session: SessionState,
readiness: AndroidBlockingDialogReadinessResult,
) => Promise<GestureInteractionResult>,
): Promise<DaemonResponse> {
const session = params.sessionStore.get(params.sessionName);
if (!session) return noActiveSessionError();
Expand All @@ -222,7 +230,7 @@ async function dispatchGestureInteraction(
phase: 'before-command',
observation: params.androidObservation,
});
const outcome = await run(session);
const outcome = await run(session, readiness);
if (isRefusal(outcome)) return outcome.refused;
if (!providerDevice) {
await ensureAndroidBlockingSystemDialogReady({
Expand Down Expand Up @@ -268,14 +276,26 @@ function resolveExecutionProfile(
function prepareGestureCommandInput(
input: GesturePayload,
session: SessionState,
readiness: AndroidBlockingDialogReadinessResult,
): GestureCommandInput {
const normalized = normalizeGestureCommandInput(input);
if (normalized.intent !== 'drag') return normalized;
return {
...normalized,
source: prepareDragTarget(normalized.source, session),
destination: prepareDragTarget(normalized.destination, session),
};
try {
return {
...normalized,
source: prepareDragTarget(normalized.source, session),
destination: prepareDragTarget(normalized.destination, session),
};
} catch (error) {
if (readiness.status === 'recovered') {
const failure = asAppError(error);
const existingWarning =
typeof failure.details?.warning === 'string' ? failure.details.warning + ' ' : '';
failure.details = { ...failure.details, warning: existingWarning + readiness.warning };
throw failure;
}
throw thrownBeforeDispatch(error);
}
}

function prepareDragTarget(target: string, session: SessionState): string {
Expand Down
Loading