Skip to content

Implement agentic recipe configuration #277

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
alexculealt merged 1 commit into from
Sep 15, 2026
Merged

Implement agentic recipe configuration #277

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 29 additions & 3 deletions client/src/components/app/catalogues/detail.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,34 @@ interface RecipeListProps {
catalogue: Catalogue;
}

function RecipeStatusLabel({ recipe }: { recipe: Recipe }) {
const jobStatusQuery = trpc.recipes.configurationJobStatus.useQuery(
{ recipeId: recipe.id },
{
enabled: recipe.status !== RecipeDetectionStatus.SUCCESS,
refetchInterval: recipe.status !== RecipeDetectionStatus.SUCCESS ? 2000 : false,
}
);

if (recipe.status === RecipeDetectionStatus.SUCCESS) {
return null;
}

if (
jobStatusQuery.data?.kind === "agentic" &&
(recipe.status === RecipeDetectionStatus.WAITING ||
recipe.status === RecipeDetectionStatus.IN_PROGRESS)
) {
return (
<Badge variant="secondary" className="ml-1">
Agentic · In progress
</Badge>
);
}

return <>— Draft</>;
}

const RecipeList = ({ catalogue }: RecipeListProps) => {
return (
<Card>
Expand All @@ -79,9 +107,7 @@ const RecipeList = ({ catalogue }: RecipeListProps) => {
<div className="flex items-center gap-2">
<span>
{recipe.name || `Recipe #${recipe.id}`}{" "}
{recipe.status == RecipeDetectionStatus.SUCCESS
? null
: "— Draft"}
<RecipeStatusLabel recipe={recipe as Recipe} />
</span>
{recipe.description && (
<TooltipProvider>
Expand Down
29 changes: 24 additions & 5 deletions client/src/components/app/recipes/create.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ import {
TooltipProvider,
TooltipTrigger,
} from "@/components/ui/tooltip";
import { HelpCircle, Search } from "lucide-react";
import { HelpCircle, LoaderIcon, Search } from "lucide-react";
import { useEffect, useRef, useState } from "react";
import {
CatalogueType,
Expand Down Expand Up @@ -162,7 +162,7 @@ const FormSchema = z.object({
manualConfig: z.boolean().default(false),
configuration: RecipeConfigurationSchema.optional(),
acknowledgedSkipRobotsTxt: z.boolean().default(false),
creationMode: z.enum(["detect", "manual", "template"]).default("detect"),
creationMode: z.enum(["detect", "agentic", "manual", "template"]).default("detect"),
});

export default function CreateRecipe() {
Expand Down Expand Up @@ -262,7 +262,7 @@ export default function CreateRecipe() {
form.setValue("manualConfig", true);
} else {
form.setValue("manualConfig", false);
if (creationMode === "detect") {
if (creationMode === "detect" || creationMode === "agentic") {
form.setValue("configuration", undefined);
}
}
Expand Down Expand Up @@ -314,6 +314,7 @@ export default function CreateRecipe() {
? data.configuration
: undefined,
acknowledgedSkipRobotsTxt: data.acknowledgedSkipRobotsTxt,
mode: creationMode === "agentic" ? "agentic" : creationMode === "detect" ? "detect" : undefined,
});
if (result.context?.message) {
toast({
Expand All @@ -324,6 +325,10 @@ export default function CreateRecipe() {
navigate(`/${catalogueId}/recipes/${result.id}`);
} catch (err) {
toast({
title:
creationMode === "agentic"
? "Could not start agentic configuration"
: "Could not create recipe",
description: (err as Error).message,
variant: "destructive",
});
Expand Down Expand Up @@ -506,7 +511,7 @@ export default function CreateRecipe() {
}
form.setValue("manualConfig", true);
setSelectedTemplateId(null);
} else if (value === "detect") {
} else if (value === "detect" || value === "agentic") {
form.setValue("manualConfig", false);
form.setValue("configuration", undefined);
setSelectedTemplateId(null);
Expand All @@ -522,13 +527,16 @@ export default function CreateRecipe() {
</FormControl>
<SelectContent>
<SelectItem value="detect">Auto-detect</SelectItem>
<SelectItem value="agentic">Agentic</SelectItem>
<SelectItem value="manual">Manual</SelectItem>
<SelectItem value="template">From template</SelectItem>
</SelectContent>
</Select>
<FormDescription>
{creationMode === "detect"
? "Automatically detect recipe configuration using AI"
: creationMode === "agentic"
? "Start agentic configuration in the background; progress appears on the recipe page"
: creationMode === "manual"
? "Manually configure the recipe"
: "Use an existing recipe as a template"}
Expand Down Expand Up @@ -744,7 +752,18 @@ export default function CreateRecipe() {

<div className="flex items-center">
<Button disabled={createRecipe.isLoading} type="submit">
Next
{createRecipe.isLoading ? (
<>
<LoaderIcon className="animate-spin mr-2 h-4 w-4" />
{creationMode === "agentic"
? "Starting agent…"
: creationMode === "detect"
? "Detecting…"
: "Saving…"}
</>
) : (
"Next"
)}
</Button>
</div>
</form>
Expand Down
90 changes: 78 additions & 12 deletions client/src/components/app/recipes/edit.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -174,8 +174,17 @@ export default function EditRecipe() {
{ id: parseInt(recipeId || "") },
{ enabled: !!recipeId }
);
const jobStatusQuery = trpc.recipes.configurationJobStatus.useQuery(
{ recipeId: parseInt(recipeId || "") },
{
enabled: !!recipeId && recipeQuery.data?.status !== RecipeDetectionStatus.SUCCESS,
refetchInterval:
recipeQuery.data?.status !== RecipeDetectionStatus.SUCCESS ? 2000 : false,
}
);
const updateRecipe = trpc.recipes.update.useMutation();
const reconfigureRecipe = trpc.recipes.reconfigure.useMutation();
const reconfigureAgenticRecipe = trpc.recipes.reconfigureAgentic.useMutation();
const setDefaultRecipe = trpc.recipes.setDefault.useMutation();
const destroyRecipe = trpc.recipes.destroy.useMutation();
const { toast } = useToast();
Expand All @@ -190,11 +199,14 @@ export default function EditRecipe() {
useEffect(() => {
const pollQuery = () => {
if (recipeQuery.data?.status == RecipeDetectionStatus.SUCCESS) {
window.clearInterval(intervalRef.current!);
intervalRef.current = null;
if (intervalRef.current) {
window.clearInterval(intervalRef.current);
intervalRef.current = null;
}
return;
}
recipeQuery.refetch();
jobStatusQuery.refetch();
};

if (!recipeQuery.data) {
Expand All @@ -204,10 +216,10 @@ export default function EditRecipe() {
if (recipeQuery.data.status != RecipeDetectionStatus.SUCCESS) {
if (!intervalRef.current) {
intervalRef.current = window.setInterval(pollQuery, 2000);
} else {
window.clearInterval(intervalRef.current);
intervalRef.current = null;
}
} else if (intervalRef.current) {
window.clearInterval(intervalRef.current);
intervalRef.current = null;
}

form.reset({
Expand All @@ -234,6 +246,9 @@ export default function EditRecipe() {
};
}, [recipeQuery.data]);

const isAgenticJob = jobStatusQuery.data?.kind === "agentic";
const agenticProgress = isAgenticJob ? jobStatusQuery.data?.progress : null;

if (!catalogueQuery.data || !recipeQuery.data) {
return null;
}
Expand All @@ -256,8 +271,23 @@ export default function EditRecipe() {
}

async function onReconfigure() {
await reconfigureRecipe.mutateAsync({ id: recipe.id });
recipeQuery.refetch();
try {
if (isAgenticJob) {
await reconfigureAgenticRecipe.mutateAsync({ id: recipe.id });
} else {
await reconfigureRecipe.mutateAsync({ id: recipe.id });
}
await recipeQuery.refetch();
await jobStatusQuery.refetch();
} catch (err) {
toast({
title: isAgenticJob
? "Could not retry agentic configuration"
: "Could not redetect configuration",
description: (err as Error).message,
variant: "destructive",
});
}
}

async function onSetDefault(e: React.MouseEvent<HTMLButtonElement>) {
Expand Down Expand Up @@ -496,6 +526,32 @@ export default function EditRecipe() {
pageSetup={recipe.configuration?.pageSetup}
/>
)}
{isAgenticJob &&
(recipe.status == RecipeDetectionStatus.WAITING ||
recipe.status == RecipeDetectionStatus.IN_PROGRESS) ? (
<div className="mt-4 grid gap-2 md:grid-cols-[1fr_250px] lg:grid-cols-2 lg:gap-4">
<Card>
<CardHeader>
<CardDescription>Agentic Configuration</CardDescription>
</CardHeader>
<CardContent className="text-sm space-y-2">
<div className="flex items-center">
<LoaderIcon className="animate-spin mr-2 w-4 h-4" />
<span>Agent is configuring this recipe…</span>
</div>
{agenticProgress?.message ? (
<p className="text-muted-foreground">
{agenticProgress.message}
{"elapsedSeconds" in agenticProgress &&
agenticProgress.elapsedSeconds != null
? ` (${agenticProgress.elapsedSeconds}s)`
: null}
</p>
) : null}
</CardContent>
</Card>
</div>
) : null}
{recipe.status == RecipeDetectionStatus.WAITING ? (
<div className="mt-4 grid gap-2 md:grid-cols-[1fr_250px] lg:grid-cols-2 lg:gap-4">
<Card>
Expand All @@ -522,8 +578,9 @@ export default function EditRecipe() {
<CardContent className="text-sm">
<div>
<p className="text-red-800 font-semibold">
CTDL xTRA failed to detect a valid configuration for
this recipe.
{isAgenticJob
? "The agentic configuration workflow failed for this recipe."
: "CTDL xTRA failed to detect a valid configuration for this recipe."}
</p>
<p className="mt-4">
You can adjust the URL, or try again.
Expand All @@ -533,14 +590,20 @@ export default function EditRecipe() {
{recipe.detectionFailureReason}
</pre>
<Button
type="button"
className="mt-8"
variant="outline"
size="sm"
onClick={onReconfigure}
disabled={reconfigureRecipe.isLoading}
disabled={
reconfigureRecipe.isLoading ||
reconfigureAgenticRecipe.isLoading
}
>
<RotateCcw className="w-4 h-4 mr-2" />
Redetect configuration
{isAgenticJob
? "Retry agentic configuration"
: "Redetect configuration"}
</Button>
</div>
</CardContent>
Expand Down Expand Up @@ -775,7 +838,9 @@ export default function EditRecipe() {
<Button disabled={true} variant={"outline"}>
<div className="flex text-sm items-center">
<LoaderIcon className="animate-spin mr-2 w-3.5" />
Detecting configuration{" "}
{isAgenticJob
? "Agent configuring recipe"
: "Detecting configuration"}{" "}
</div>
</Button>
) : (
Expand All @@ -784,6 +849,7 @@ export default function EditRecipe() {
recipeQuery.isLoading ||
updateRecipe.isLoading ||
reconfigureRecipe.isLoading ||
reconfigureAgenticRecipe.isLoading ||
recipe.status == RecipeDetectionStatus.IN_PROGRESS ||
!form.formState.isDirty
}
Expand Down
36 changes: 36 additions & 0 deletions server/src/extraction/submitAgenticRecipeDetection.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
import { PageType } from "../../../common/types";
import { findCatalogueById } from "../data/catalogues";
import { startRecipe } from "../data/recipes";
import getLogger from "../logging";
import { Queues, submitJob } from "../workers";

const logger = getLogger("extraction.submitAgenticRecipeDetection");

export async function submitAgenticRecipeDetection(
url: string,
catalogueId: number,
triggeredByUserId?: number | null
) {
const catalogue = await findCatalogueById(catalogueId);
if (!catalogue) {
throw new Error(`Catalogue not found: ${catalogueId}`);
}

// Agentic configuration runs asynchronously in the worker; avoid blocking
// recipe creation on a synchronous page fetch (proxy/network/LLM page-type detection).
const pageType = PageType.DETAIL_LINKS;
logger.info(`Creating recipe for agentic configuration`);
const result = await startRecipe(catalogueId, url, pageType);
logger.info(`Created recipe ${result.id}`);
const id = result.id;
await submitJob(
Queues.AgenticRecipeConfig,
{ recipeId: id, triggeredByUserId: triggeredByUserId ?? null },
`agenticRecipeConfig.${id}`
);
return {
id,
pageType,
message: null,
};
}
Loading
Loading