diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a766454..695ae92 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,7 +36,16 @@ jobs: bun-version: 1.3.14 - name: Install dependencies - run: bun install + run: bun ci + + - name: Typecheck + run: bun run typecheck - name: Run tests run: bun test + + - name: Fake-call every endpoint in every API snapshot + run: bun run conformance:all + + - name: Smoke-test built CLI + run: bun bin/langfuse.mjs --help diff --git a/.gitignore b/.gitignore index e7a6b96..065923d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,6 @@ node_modules dist *.tgz -bun.lock bun.lockb .env .env.local diff --git a/README.md b/README.md index 691022b..aab85b0 100644 --- a/README.md +++ b/README.md @@ -9,10 +9,13 @@ Interact with the [Langfuse](https://langfuse.com) API from the command line. ```sh # Run directly npx langfuse-cli api +# or bunx langfuse-cli api # Or install globally npm i -g langfuse-cli +# or +bun add --global langfuse-cli langfuse api ``` @@ -47,36 +50,52 @@ langfuse --public-key pk-lf-... --secret-key sk-lf-... api prompts list ## Usage ```sh -# Discover all resources -langfuse api __schema +# Discover resources naturally +langfuse api help +langfuse api help prompts +langfuse api help prompts create -# List actions for a resource -langfuse api traces --help +# Machine-readable discovery (legacy command alias) +langfuse api schema --json +langfuse api __schema --json -# List traces -langfuse api traces list --limit 10 +# List observations +langfuse api observations list --limit 10 -# Get a specific trace -langfuse api traces get +# List observations for a specific trace +langfuse api observations list --trace-id # JSON output (for piping/scripting) -langfuse api traces list --limit 5 --json +langfuse api observations list --limit 5 --json # Preview curl command -langfuse api traces list --limit 5 --curl +langfuse api observations list --limit 5 --curl # Prompts langfuse api prompts list -langfuse api prompts get --name my-prompt +langfuse api prompts get my-prompt +langfuse api prompts create --body-json '{"name":"my-prompt","type":"text","prompt":"Hello {{name}}"}' # Datasets langfuse api datasets list langfuse api dataset-items list --dataset-name my-dataset # Scores -langfuse api score-v2s get-scores --limit 20 +langfuse api scores list --limit 20 + +# Use an API snapshot compatible with an older self-hosted deployment +langfuse --api-version 3.150.0 api traces list + +# Detect the server version through /api/public/health +langfuse --api-version auto api prompts list ``` +Canonical command resources come from API paths and use concise REST actions. +OpenAPI tags and explicit route versions remain accepted aliases, for example +`scores-v3 list` for the canonical `scores list`. Verbose OpenAPI `operationId` +values remain available in `api schema --json` but are never required as CLI +commands. + ## Agent Usage The latest Langfuse skill lives in [`langfuse/skills`](https://github.com/langfuse/skills). Print the current version with: @@ -91,30 +110,43 @@ This fetches the latest skill from GitHub, so it stays up to date. Pipe it into See the full [Langfuse API Reference](https://api.reference.langfuse.com/). -## OpenAPI Patch Script +## OpenAPI conformance suite + +The version-pinned black-box suite lives in [`conformance/`](conformance/README.md). It invokes every operation through the real CLI across historical Langfuse specs. Active operations make one minimally valid mocked API call; operations marked `deprecated: true` must fail before any network request. + +```sh +bun test +bun run conformance:all +``` + +`bun test` verifies the generator, schemas, serialization, capture oracle, deprecation policy, and legacy CLI compatibility. `bun run conformance:all` builds the package and checks every operation through the native CLI using its lossless JSON input path. CI runs both. -The bundled `openapi.yml` is post-processed by `scripts/patch-openapi.ts` to flatten discriminated unions (`oneOf` with `allOf` branches) into plain objects. This is needed because specli can only generate CLI flags from flat `type: object` schemas — it doesn't handle `oneOf`/`allOf`. Without the patch, endpoints like `prompts create` produce zero flags. +## Native OpenAPI contracts -The patch runs automatically as part of `bun run build`. To fetch a fresh spec and patch it: +The CLI is implemented in TypeScript and runs natively on Bun. It has zero external runtime dependencies and never parses OpenAPI during invocation. + +Builds compile committed OpenAPI snapshots into compact versioned contracts under ignored `dist/contracts/`. Catalog entries record the exact committed hash; snapshots with explicit local annotations also record the upstream hash and modification name. Generated contracts are packaged on npm but are not committed. ```sh -# From cloud (default) -bun run refetch-openapi +# Build the Bun CLI and all versioned contracts +bun run build -# From a custom URL (e.g. local dev server) -bun run patch-openapi -- --refetch --openapi_url http://localhost:3000/generated/api/openapi.yml +# Add or refresh an immutable upstream snapshot +bun run conformance:sync -- --version -# Patch only (no fetch) -bun run patch-openapi +# Add a stable release snapshot, update metadata, and verify it +bun run conformance:add-version -- v4.11.0 ``` +`--body-json` and `--body-file` provide a lossless input path for nested objects, arrays, unions, and free-form JSON. Simple historical field flags remain supported where they were previously expressible. + ## Release ```sh bun run release ``` -This interactively selects the package version, verifies it is not already on npm, checks npm auth/registry, runs tests, rebuilds the CLI via `prepublishOnly`, checks the npm package contents with `npm pack --dry-run`, shows the post-build git status, then asks before publishing to npm. +This interactively selects the package version, verifies it is not already on npm, checks npm auth/registry, typechecks, runs both test suites, rebuilds the CLI, checks the npm package contents with `npm pack --dry-run`, shows the post-build git status, then asks before publishing to npm. To test the flow without publishing: @@ -122,7 +154,7 @@ To test the flow without publishing: bun run release -- --dry-run ``` -Dry-run still runs the full rebuild path, so it may update generated release artifacts like `dist/` and `openapi.yml`; it restores the package version before exiting. +Dry-run still runs the full reproducible build path and restores the package version before exiting. Generated `dist/` artifacts remain ignored. If you are testing local changes to the release script itself, add `--allow-dirty`. Do not use `--allow-dirty` for a real publish. diff --git a/bin/langfuse.mjs b/bin/langfuse.mjs index 4869439..709d0eb 100755 --- a/bin/langfuse.mjs +++ b/bin/langfuse.mjs @@ -1,3 +1,3 @@ -#!/usr/bin/env node +#!/usr/bin/env bun import { run } from "../dist/cli.js"; -run(process.argv); +await run(process.argv); diff --git a/bun.lock b/bun.lock new file mode 100644 index 0000000..c0a2d1a --- /dev/null +++ b/bun.lock @@ -0,0 +1,102 @@ +{ + "lockfileVersion": 1, + "configVersion": 1, + "workspaces": { + "": { + "name": "langfuse-cli", + "devDependencies": { + "@apidevtools/swagger-parser": "^12.1.0", + "@types/bun": "^1.3.14", + "ajv": "^8.17.1", + "ajv-formats": "^3.0.1", + "typescript": "^7.0.2", + "yaml": "^2.8.2", + }, + }, + }, + "packages": { + "@apidevtools/json-schema-ref-parser": ["@apidevtools/json-schema-ref-parser@14.0.1", "", { "dependencies": { "@types/json-schema": "^7.0.15", "js-yaml": "^4.1.0" } }, "sha512-Oc96zvmxx1fqoSEdUmfmvvb59/KDOnUoJ7s2t7bISyAn0XEz57LCCw8k2Y4Pf3mwKaZLMciESALORLgfe2frCw=="], + + "@apidevtools/openapi-schemas": ["@apidevtools/openapi-schemas@2.1.0", "", {}, "sha512-Zc1AlqrJlX3SlpupFGpiLi2EbteyP7fXmUOGup6/DnkRgjP9bgMM/ag+n91rsv0U1Gpz0H3VILA/o3bW7Ua6BQ=="], + + "@apidevtools/swagger-methods": ["@apidevtools/swagger-methods@3.0.2", "", {}, "sha512-QAkD5kK2b1WfjDS/UQn/qQkbwF31uqRjPTrsCs5ZG9BQGAkjwvqGFjjPqAuzac/IYzpPtRzjCP1WrTuAIjMrXg=="], + + "@apidevtools/swagger-parser": ["@apidevtools/swagger-parser@12.1.0", "", { "dependencies": { "@apidevtools/json-schema-ref-parser": "14.0.1", "@apidevtools/openapi-schemas": "^2.1.0", "@apidevtools/swagger-methods": "^3.0.2", "ajv": "^8.17.1", "ajv-draft-04": "^1.0.0", "call-me-maybe": "^1.0.2" }, "peerDependencies": { "openapi-types": ">=7" } }, "sha512-e5mJoswsnAX0jG+J09xHFYQXb/bUc5S3pLpMxUuRUA2H8T2kni3yEoyz2R3Dltw5f4A6j6rPNMpWTK+iVDFlng=="], + + "@types/bun": ["@types/bun@1.3.14", "", { "dependencies": { "bun-types": "1.3.14" } }, "sha512-h1hFqFVcvAvD9j9K7ZW7vd82aSA+rTdznZa+5bwvCwqSB1jmmfLcbIWhOLx1/+boy/xmjgCs/OMUL8hRJSmnPw=="], + + "@types/json-schema": ["@types/json-schema@7.0.15", "", {}, "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA=="], + + "@types/node": ["@types/node@26.1.1", "", { "dependencies": { "undici-types": "~8.3.0" } }, "sha512-nxAkRSVkN1Y0JC1W8ky/fTfkGsMmcrRsbx+3XoZE+rMOX71kLYTV7fLXpqud1GpbpP5TuffXFqfX7fH2GgZREw=="], + + "@typescript/typescript-aix-ppc64": ["@typescript/typescript-aix-ppc64@7.0.2", "", { "os": "aix", "cpu": "ppc64" }, "sha512-MTKKkWB7p/0E9xi1d1tHtZ5PiLkGEMIq88pK2CubZjOsLtYTLqhgIgi6zepFa+9GHZ6h05NMCkQxGKiPXMxXtQ=="], + + "@typescript/typescript-darwin-arm64": ["@typescript/typescript-darwin-arm64@7.0.2", "", { "os": "darwin", "cpu": "arm64" }, "sha512-gowzar9MwS/aRWp6f3a4KUqzRjAZjOsmGNCM6LcTgXum+dBfgsBVMN+AgvOCCbguXyick6LJhpBszxMebJ8syA=="], + + "@typescript/typescript-darwin-x64": ["@typescript/typescript-darwin-x64@7.0.2", "", { "os": "darwin", "cpu": "x64" }, "sha512-SZ9xZInqApNlNGc9s0W1VSsktYSOe9cFqNOIqmN1Gs8SmkjKZYFt017G4VwPxASInODuAdbTW7sXiFUf893RgA=="], + + "@typescript/typescript-freebsd-arm64": ["@typescript/typescript-freebsd-arm64@7.0.2", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-W5NH4y/J0plIIS5b2xvTEkU7JFxyqdMAOgf+Ilhl0vHQXKO5dZoxd+C/jEtq56c4F3wk71RB4BMRQ2XdI+bwYQ=="], + + "@typescript/typescript-freebsd-x64": ["@typescript/typescript-freebsd-x64@7.0.2", "", { "os": "freebsd", "cpu": "x64" }, "sha512-UMGDx5sTpzNw3WiPebH7l90IWfJggEd+egHt/q6p7/Cm3zqoV7VxkGXt+3DxPIw8CcmvAB0j3sVVfbhX+M4Tpw=="], + + "@typescript/typescript-linux-arm": ["@typescript/typescript-linux-arm@7.0.2", "", { "os": "linux", "cpu": "arm" }, "sha512-gffT3xPz9sR7j/YJExkyPntrI0P2EP9XbOyWzth2/Gs0RstK+90RBcO0ncXoXy/beYll1SXw846Nf2zdnEz0QQ=="], + + "@typescript/typescript-linux-arm64": ["@typescript/typescript-linux-arm64@7.0.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-Qh4eU4/y3yDjnfjjyPYihMj5/ODIlmt+Bzu17OI+fiSRDW57QmU5SiN63exPRNJPKUzcc1INa1NXdrJ+MqHjUQ=="], + + "@typescript/typescript-linux-loong64": ["@typescript/typescript-linux-loong64@7.0.2", "", { "os": "linux", "cpu": "none" }, "sha512-uEHck9i8hoAzXPiYRib1O7miOnz23SxIeVl6F4LXox+qov1K35jHcEW6VHKvZI+pyvl7fZEP4MCU5LYvIq1GuQ=="], + + "@typescript/typescript-linux-mips64el": ["@typescript/typescript-linux-mips64el@7.0.2", "", { "os": "linux", "cpu": "none" }, "sha512-R4KvAMnE43W5Qeqb0Ly56O3mWMWIAgsMyz36DCaycd5nbg/9kzm0liw3JocfRqyJY0KPmzFjbswozXyW0DnIYA=="], + + "@typescript/typescript-linux-ppc64": ["@typescript/typescript-linux-ppc64@7.0.2", "", { "os": "linux", "cpu": "ppc64" }, "sha512-DORx5b3sd/4S7eayxm4FQv+A7CrkUIGRaHiwI8oiHTAI1fAPWhF4J0vAlkC8biAlHSVVwxMQ3tjZ2/DVbnQiiA=="], + + "@typescript/typescript-linux-riscv64": ["@typescript/typescript-linux-riscv64@7.0.2", "", { "os": "linux", "cpu": "none" }, "sha512-wf0jqEDOjrPRnKwYRyyJDRo11KMbvMFrU+q4zqKyChODBzvlkbhNQfKvLxQCcwTpdDaXSHZTVuh0JoCrKCUMHQ=="], + + "@typescript/typescript-linux-s390x": ["@typescript/typescript-linux-s390x@7.0.2", "", { "os": "linux", "cpu": "s390x" }, "sha512-IkwJc3L7yhytWd/ewjyxNDfOmswCm9GWMJT/ue/dU4aZNbwZeYAetq42VyLmsmSjvoX7z74X6ZaYCtzAr0EuGw=="], + + "@typescript/typescript-linux-x64": ["@typescript/typescript-linux-x64@7.0.2", "", { "os": "linux", "cpu": "x64" }, "sha512-EYdf2cNg7rgCWJnxCdJ+F3V39O8ihb37eHAu1LK8oAFizgTQbPOK7zHHXbPt8rX24COqODXeI3sIf0fCXG7H/A=="], + + "@typescript/typescript-netbsd-arm64": ["@typescript/typescript-netbsd-arm64@7.0.2", "", { "os": "none", "cpu": "arm64" }, "sha512-+polYF4MF04aPpO5FTkHran9yUQDSXqy5GiSDKpsll5jy3l3+g9QLhpf39T+ePtefhXLOGrLl0QIjkQP6VnelA=="], + + "@typescript/typescript-netbsd-x64": ["@typescript/typescript-netbsd-x64@7.0.2", "", { "os": "none", "cpu": "x64" }, "sha512-8YIT0EHM/3dq10ZOVF/A7pc/YSMtbcecct4rWtexrnSCHOPcpC2KTLXfTCR6vDpnSiY12heNb1GiN/wu+T/FyA=="], + + "@typescript/typescript-openbsd-arm64": ["@typescript/typescript-openbsd-arm64@7.0.2", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-APT8+ClYnuYm1u9+kgGXoMj2VzWzcymwh2gNSQVySHfkRDGOTVkoWLjCmOQSaO+PoqQ57B0flRp9SA+7GnnkzQ=="], + + "@typescript/typescript-openbsd-x64": ["@typescript/typescript-openbsd-x64@7.0.2", "", { "os": "openbsd", "cpu": "x64" }, "sha512-yX7s+Q0Dln0Dt9tEzZsAjXXR/+ytBM7AlglaqyeMPxQszJ1JhlJdZ6jLA+IzldHtflX81em7lDao1xXu+aRRkg=="], + + "@typescript/typescript-sunos-x64": ["@typescript/typescript-sunos-x64@7.0.2", "", { "os": "sunos", "cpu": "x64" }, "sha512-dLJDGaLZ1D4HPQn62u1n8mBDkJREwMsAkCdkwd4Ieqw+x3TUyTsqY0YiBCtE6H6OzzgGk3iuZ3vFWRS+E8/d1g=="], + + "@typescript/typescript-win32-arm64": ["@typescript/typescript-win32-arm64@7.0.2", "", { "os": "win32", "cpu": "arm64" }, "sha512-Gyl1Vy6OsWesLzmq+EP0Fb7b4Nid5232AvcA2SFcdYreldpNtYFFofPjnt62y9hQy7VTaZp65ICJjuAQRaVcIQ=="], + + "@typescript/typescript-win32-x64": ["@typescript/typescript-win32-x64@7.0.2", "", { "os": "win32", "cpu": "x64" }, "sha512-0BQ3HkAHHlKLSp1qRvf3SUhGpGsDuhB/jgFw75guyqbxJqEaS0Cw/VFO8i2nHglJUzQCRtMMR/IBAKE3ETMC4g=="], + + "ajv": ["ajv@8.17.1", "", { "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", "json-schema-traverse": "^1.0.0", "require-from-string": "^2.0.2" } }, "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g=="], + + "ajv-draft-04": ["ajv-draft-04@1.0.0", "", { "peerDependencies": { "ajv": "^8.5.0" }, "optionalPeers": ["ajv"] }, "sha512-mv00Te6nmYbRp5DCwclxtt7yV/joXJPGS7nM+97GdxvuttCOfgI3K4U25zboyeX0O+myI8ERluxQe5wljMmVIw=="], + + "ajv-formats": ["ajv-formats@3.0.1", "", { "dependencies": { "ajv": "^8.0.0" } }, "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ=="], + + "argparse": ["argparse@2.0.1", "", {}, "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="], + + "bun-types": ["bun-types@1.3.14", "", { "dependencies": { "@types/node": "*" } }, "sha512-4N0ig0fEomHt5R0KCFWjovxow98rIoRwKolrYdCcknNwMekCXRnWEUvgu5soYV8QXtVsrUD8B95MBOZGPvr6KQ=="], + + "call-me-maybe": ["call-me-maybe@1.0.2", "", {}, "sha512-HpX65o1Hnr9HH25ojC1YGs7HCQLq0GCOibSaWER0eNpgJ/Z1MZv2mTc7+xh6WOPxbRVcmgbv4hGU+uSQ/2xFZQ=="], + + "fast-deep-equal": ["fast-deep-equal@3.1.3", "", {}, "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q=="], + + "fast-uri": ["fast-uri@3.1.0", "", {}, "sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA=="], + + "js-yaml": ["js-yaml@4.1.1", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA=="], + + "json-schema-traverse": ["json-schema-traverse@1.0.0", "", {}, "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug=="], + + "openapi-types": ["openapi-types@12.1.3", "", {}, "sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw=="], + + "require-from-string": ["require-from-string@2.0.2", "", {}, "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw=="], + + "typescript": ["typescript@7.0.2", "", { "optionalDependencies": { "@typescript/typescript-aix-ppc64": "7.0.2", "@typescript/typescript-darwin-arm64": "7.0.2", "@typescript/typescript-darwin-x64": "7.0.2", "@typescript/typescript-freebsd-arm64": "7.0.2", "@typescript/typescript-freebsd-x64": "7.0.2", "@typescript/typescript-linux-arm": "7.0.2", "@typescript/typescript-linux-arm64": "7.0.2", "@typescript/typescript-linux-loong64": "7.0.2", "@typescript/typescript-linux-mips64el": "7.0.2", "@typescript/typescript-linux-ppc64": "7.0.2", "@typescript/typescript-linux-riscv64": "7.0.2", "@typescript/typescript-linux-s390x": "7.0.2", "@typescript/typescript-linux-x64": "7.0.2", "@typescript/typescript-netbsd-arm64": "7.0.2", "@typescript/typescript-netbsd-x64": "7.0.2", "@typescript/typescript-openbsd-arm64": "7.0.2", "@typescript/typescript-openbsd-x64": "7.0.2", "@typescript/typescript-sunos-x64": "7.0.2", "@typescript/typescript-win32-arm64": "7.0.2", "@typescript/typescript-win32-x64": "7.0.2" }, "bin": { "tsc": "bin/tsc" } }, "sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA=="], + + "undici-types": ["undici-types@8.3.0", "", {}, "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ=="], + + "yaml": ["yaml@2.8.2", "", { "bin": { "yaml": "bin.mjs" } }, "sha512-mplynKqc1C2hTVYxd0PU2xQAc22TI1vShAYGksCCfxbn/dFwnHTNi1bvYsBTkhdUNtGIf5xNOg938rrSSYvS9A=="], + } +} diff --git a/conformance/README.md b/conformance/README.md new file mode 100644 index 0000000..19b0e0d --- /dev/null +++ b/conformance/README.md @@ -0,0 +1,123 @@ +# Langfuse CLI OpenAPI conformance suite + +Language-neutral, version-pinned acceptance tests for the native TypeScript CLI. The oracle does not import the runtime request builder. + +## What is tested + +The primary test invokes every operation in every cataloged spec through the real CLI. For active operations it: + +- generates one minimally valid invocation from the committed OpenAPI source +- starts a local mock HTTP server with a response generated from that operation +- runs the CLI as a subprocess against the mock host +- compares the received method, path, query, headers, authentication, and JSON body +- compares the CLI's response status, body, and exit status + +For operations marked `deprecated: true`, it instead verifies exit code 2, a helpful error on stderr, and zero network requests. + +The black-box oracle does not share request-building code with the CLI. `bun test` currently attempts all 792 operations across 9 pinned snapshots using the historical field-flag adapter. Operations that require lossless JSON bodies remain an explicit compatibility baseline; any additional failure fails the test. The native `contract-v1` adapter checks all 792 operations through `--body-json`, including pre-network rejection for deprecated operations. + +Supporting unit tests verify immutable spec hashes, valid sampling, serialization, naming, adapters, and the capture runner itself. + +OpenAPI cannot describe database setup, generated IDs, cross-request bindings, cleanup, licenses, or feature flags. Those stateful live workflows require a small reviewed scenario overlay; they are not silently invented by this generator. + +## Pinned specs + +| Langfuse | Paths | Operations | +|---|---:|---:| +| 3.0.0 | 29 | 39 | +| 3.50.0 | 44 | 68 | +| 3.100.0 | 49 | 77 | +| 3.150.0 | 55 | 86 | +| 3.176.0 | 60 | 96 | +| 3.200.0 | 61 | 98 | +| 3.212.0 | 64 | 101 | +| 3.216.0 | 69 | 113 | +| 4.10.0 | 70 | 114 | + +Each source file is downloaded by immutable commit and verified against the SHA-256 in `catalog.json`. Tests are network-free after sync. + +Some pinned specs use the JSON Schema `const` keyword while declaring OpenAPI 3.0.1. `swagger-parser` correctly reports those sources as invalid OAS 3.0 documents. The catalog records this as `oas3.0-const-keyword`; request sampling still preserves and tests the constraint with the independent JSON Schema validator. + +## Files + +```text +catalog.json immutable Git refs, commits, hashes, known source issues +policy.json implementation adapters; not API truth +specs//openapi.yml committed source snapshots +src/ compiler, serializers, adapters, capture runner +tests/ compiler, validator, and runner tests +``` + +## Commands + +```sh +# Generator, schema, serializer, capture, and compatibility tests +bun test + +# Build and check every endpoint through the lossless native CLI +bun run conformance:all + +# Re-download pinned bytes and verify their hashes +bun run conformance:sync +``` + +## CI gate + +GitHub Actions runs both commands for every pull request, merge queue entry, and push to `main`: + +```sh +bun test +bun run conformance:all +``` + +The required check name is **Test and verify OpenAPI conformance**. The interactive release script repeats the checks before building or publishing. + +## Run a focused current-CLI check + +```sh +bun run conformance:run -- \ + --version 3.212.0 \ + --adapter specli-v0 \ + --current-cli +``` + +The adapter name is retained because it describes the old field-flag grammar. The runner builds the native current source and compiles the selected committed spec into a temporary runtime contract. + +Useful filters: + +```sh +--operation prompts_create +--max 20 +--fail-fast +``` + +Failures identify current limitations inline: raw union bodies, complex body flags, response exit codes, naming mismatches, or missing version selection. + +## Run the lossless native contract + +The native adapter uses lossless JSON body input via `--body-json`: + +```sh +bun run conformance:run -- \ + --version 4.10.0 \ + --adapter contract-v1 \ + -- bun bin/langfuse.mjs --api-version 4.10.0 +``` + +The command after the second `--` is treated as an external black-box executable. + +## Add a version + +```sh +bun run conformance:add-version -- v4.11.0 +``` + +The command accepts only stable semantic release tags. It verifies the published GitHub release, resolves the tag to an immutable commit, downloads the exact OpenAPI bytes, records their SHA-256, checks both compilers, updates the catalog and this table, then runs typecheck, tests, build, and focused black-box conformance. If validation fails, it restores the catalog, spec directory, and documentation. + +Preview without writing files: + +```sh +bun run conformance:add-version -- v4.11.0 --dry-run +``` + +Review the resulting source diff and live-test added or changed endpoints before committing. Never catalog mutable `main` or `latest`. diff --git a/conformance/catalog.json b/conformance/catalog.json index 0c48b77..d78c1c0 100644 --- a/conformance/catalog.json +++ b/conformance/catalog.json @@ -53,6 +53,15 @@ "commit": "706f1bb4231ba4433c8fc101b85167471693180a", "sha256": "551e824dd11fc137557d7092596eba09ae342698c7ce63ceb86d90909afd7ee0", "knownIssues": ["oas3.0-const-keyword"] + }, + { + "version": "4.10.0", + "ref": "v4.10.0", + "commit": "ddc198b49f1439f95ecf674c68863bfb0c2bd07f", + "sha256": "bcf95d11960571222d9ee47e4a0e5d2eba667d14802cf27539e4c0ff3d9352a4", + "upstreamSha256": "fa4d21a83c49bd0bc3d3f2f751e12fbfce96f72b422267726606ff67bfacf01f", + "modifications": ["deprecated-operation-markers"], + "knownIssues": ["oas3.0-const-keyword"] } ] } diff --git a/openapi.yml b/conformance/specs/4.10.0/openapi.yml similarity index 85% rename from openapi.yml rename to conformance/specs/4.10.0/openapi.yml index 8bd6144..dd5f151 100644 --- a/openapi.yml +++ b/conformance/specs/4.10.0/openapi.yml @@ -804,7 +804,8 @@ paths: - name: objectType in: query description: >- - Filter comments by object type (trace, observation, session, prompt). + Filter comments by object type (trace, observation, session, + prompt). required: false schema: type: string @@ -1088,7 +1089,8 @@ paths: security: *ref_0 delete: description: >- - Delete a dataset item and all its run items. This action is irreversible. + Delete a dataset item and all its run items. This action is + irreversible. operationId: datasetItems_delete tags: - DatasetItems @@ -1177,6 +1179,7 @@ paths: application/json: schema: $ref: '#/components/schemas/CreateDatasetRunItemRequest' + deprecated: true get: description: List dataset run items operationId: datasetRunItems_list @@ -1240,6 +1243,7 @@ paths: application/json: schema: {} security: *ref_0 + deprecated: true /api/public/v2/datasets: get: description: Get all datasets @@ -1434,6 +1438,7 @@ paths: application/json: schema: {} security: *ref_0 + deprecated: true delete: description: Delete a dataset run and all its run items. This action is irreversible. operationId: datasets_deleteRun @@ -1483,6 +1488,7 @@ paths: application/json: schema: {} security: *ref_0 + deprecated: true /api/public/datasets/{datasetName}/runs: get: description: Get dataset runs @@ -1542,6 +1548,7 @@ paths: application/json: schema: {} security: *ref_0 + deprecated: true /api/public/experiments: get: description: |- @@ -1794,6 +1801,57 @@ paths: application/json: schema: {} security: *ref_0 + /api/public/feedback: + post: + description: >- + Submit explicit user-approved feedback about Langfuse skills, MCP tools, + CLI, docs, or public API. Do not include secrets, credentials, customer + data, trace payloads, or unrelated use-case details. + operationId: feedback_submit + tags: + - Feedback + parameters: [] + responses: + '201': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/SubmitFeedbackResponse' + '400': + description: '' + content: + application/json: + schema: {} + '401': + description: '' + content: + application/json: + schema: {} + '403': + description: '' + content: + application/json: + schema: {} + '404': + description: '' + content: + application/json: + schema: {} + '405': + description: '' + content: + application/json: + schema: {} + '409': + description: '' + security: *ref_0 + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/SubmitFeedbackRequest' /api/public/health: get: description: Check health of API and database @@ -1932,6 +1990,7 @@ paths: debugging. required: - batch + deprecated: true /api/public/metrics: get: description: >- @@ -1958,7 +2017,7 @@ paths: ```json { - "view": string, // Required. One of "traces", "observations", "scores-numeric", "scores-categorical" + "view": string, // Required. One of "traces", "observations", "scores-numeric", "scores-boolean", "scores-categorical" "dimensions": [ // Optional. Default: [] { "field": string // Field to group by, e.g. "name", "userId", "sessionId" @@ -2033,6 +2092,7 @@ paths: application/json: schema: {} security: *ref_0 + deprecated: true /api/public/observations/{observationId}: get: description: Get a observation @@ -2054,7 +2114,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/ObservationsView' + $ref: '#/components/schemas/ObservationsViewSingle' '400': description: '' content: @@ -2081,6 +2141,7 @@ paths: application/json: schema: {} security: *ref_0 + deprecated: true /api/public/observations: get: description: >- @@ -2369,52 +2430,7 @@ paths: application/json: schema: {} security: *ref_0 - /api/public/scores: - post: - description: Create a score (supports both trace and session scores) - operationId: legacy_scoreV1_create - tags: - - LegacyScoreV1 - parameters: [] - responses: - '200': - description: '' - content: - application/json: - schema: - $ref: '#/components/schemas/legacyCreateScoreResponse' - '400': - description: '' - content: - application/json: - schema: {} - '401': - description: '' - content: - application/json: - schema: {} - '403': - description: '' - content: - application/json: - schema: {} - '404': - description: '' - content: - application/json: - schema: {} - '405': - description: '' - content: - application/json: - schema: {} - security: *ref_0 - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/legacyCreateScoreRequest' + deprecated: true /api/public/scores/{scoreId}: delete: description: Delete a score (supports both trace and session scores) @@ -2753,11 +2769,14 @@ paths: ## V2 Differences - - Supports `observations`, `scores-numeric`, and `scores-categorical` - views only (traces view not supported) + - Supports `observations`, `scores-numeric`, `scores-boolean`, and + `scores-categorical` views only (traces view not supported) - Direct access to tags and release fields on observations + - Semantic-root filtering and grouping through the v2-only + `isRootObservation` dimension + - Backwards-compatible: traceName, traceRelease, traceVersion dimensions are still available on observations view @@ -2807,6 +2826,10 @@ paths: - `promptVersion` - Version of the prompt used + - `isRootObservation` - Boolean semantic-root status. `true` includes + physical roots and app roots whose SDK parent is external (so + `parentObservationId` may be non-null). + - `startTimeMonth` - Month of start_time in YYYY-MM format @@ -2888,6 +2911,24 @@ paths: - `value` - Score value (for aggregations) + ### scores-boolean + + Query boolean score data. It has the same score and parent + trace/observation dimensions as scores-numeric, plus: + + + **Dimensions:** + + - `booleanValue` - Boolean value for true/false grouping and filtering + + + **Measures:** + + - `count` - Total number of boolean scores + + - `value` - Numeric 0/1 score value; `avg` returns the true-rate + + ### scores-categorical Query categorical score data. Same dimensions as scores-numeric except @@ -2920,7 +2961,7 @@ paths: - `parentObservationId` - Use parentObservationId filter instead - **scores-numeric / scores-categorical views:** + **scores-numeric / scores-boolean / scores-categorical views:** - `id` - Use specific filters to narrow down results @@ -2959,7 +3000,7 @@ paths: ```json { - "view": string, // Required. One of "observations", "scores-numeric", "scores-categorical" + "view": string, // Required. One of "observations", "scores-numeric", "scores-boolean", "scores-categorical" "dimensions": [ // Optional. Default: [] { "field": string // Field to group by (see available dimensions above) @@ -3006,6 +3047,22 @@ paths: } ``` + + + For example, to count semantic roots (including app roots with a + non-null external parent), use a boolean filter: + + ```json + + { + "view": "observations", + "metrics": [{"measure": "count", "aggregation": "count"}], + "filters": [{"column": "isRootObservation", "operator": "=", "value": true, "type": "boolean"}], + "fromTimestamp": "2025-01-01T00:00:00.000Z", + "toTimestamp": "2025-02-01T00:00:00.000Z" + } + + ``` required: true schema: type: string @@ -3256,7 +3313,7 @@ paths: parentObservationId, type - `basic` - name, level, statusMessage, version, environment, - bookmarked, public, userId, sessionId + bookmarked, public, userId, sessionId, isRootObservation - `time` - completionStartTime, createdAt, updatedAt @@ -3354,6 +3411,13 @@ paths: schema: type: string nullable: true + - name: sessionId + in: query + description: Filter by session ID. + required: false + schema: + type: string + nullable: true - name: type in: query description: >- @@ -3381,10 +3445,30 @@ paths: nullable: true - name: parentObservationId in: query + description: >- + Filter by the physical parent observation ID. + + An empty value matches only observations without a physical parent. + Use `isRootObservation` to include observations marked as app roots + by the SDK, which may retain a non-null `parentObservationId`. required: false schema: type: string nullable: true + - name: isRootObservation + in: query + description: >- + Filter by whether an observation is a logical root. + + Root observations include observations without a physical parent and + observations marked as app roots by the SDK. + + An app-root observation may have `isRootObservation=true` and a + non-null `parentObservationId`. + required: false + schema: + type: boolean + nullable: true - name: environment in: query description: >- @@ -3489,6 +3573,10 @@ paths: - `sessionId` (string) - Session ID + - `isRootObservation` (boolean) - Whether the observation is a + logical root. Observations marked as app roots by the SDK may retain + a non-null parentObservationId. + ### Trace-Related Fields @@ -3592,6 +3680,12 @@ paths: "column": "output", "operator": "matches", "value": "needle" + }, + { + "type": "boolean", + "column": "isRootObservation", + "operator": "=", + "value": true } ] @@ -4360,7 +4454,8 @@ paths: security: *ref_0 post: description: >- - Create a new API key for a project (requires organization-scoped API key) + Create a new API key for a project (requires organization-scoped API + key) operationId: projects_createApiKey tags: - Projects @@ -4566,7 +4661,7 @@ paths: required: true schema: type: string - - name: prompt-version + - name: version in: query description: Version of the prompt to be retrieved. required: false @@ -4780,12 +4875,7 @@ paths: schema: {} security: *ref_0 post: - description: >- - Create a new version for the prompt with the given `name` - - - Example: - langfuse api prompts create --type text --name my-prompt --prompt 'Hello {{name}}' + description: Create a new version for the prompt with the given `name` operationId: prompts_create tags: - Prompts @@ -5619,6 +5709,54 @@ paths: application/json: schema: {} security: *ref_0 + /api/public/scores: + post: + description: >- + Create a score (supports trace, observation, session, and dataset run + scores) + operationId: scores_create + tags: + - Scores + parameters: [] + responses: + '200': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/CreateScoreResponse' + '400': + description: '' + content: + application/json: + schema: {} + '401': + description: '' + content: + application/json: + schema: {} + '403': + description: '' + content: + application/json: + schema: {} + '404': + description: '' + content: + application/json: + schema: {} + '405': + description: '' + content: + application/json: + schema: {} + security: *ref_0 + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/CreateScoreRequest' /api/public/v2/scores: get: description: |- @@ -5844,6 +5982,7 @@ paths: application/json: schema: {} security: *ref_0 + deprecated: true /api/public/v2/scores/{scoreId}: get: description: |- @@ -5894,6 +6033,7 @@ paths: application/json: schema: {} security: *ref_0 + deprecated: true /api/public/sessions: get: description: >- @@ -5993,6 +6133,7 @@ paths: application/json: schema: {} security: *ref_0 + deprecated: true /api/public/sessions/{sessionId}: get: description: >- @@ -6051,6 +6192,7 @@ paths: application/json: schema: {} security: *ref_0 + deprecated: true /api/public/traces/{traceId}: get: description: Get a specific trace @@ -6110,6 +6252,7 @@ paths: application/json: schema: {} security: *ref_0 + deprecated: true delete: description: Delete a specific trace operationId: trace_delete @@ -6290,7 +6433,7 @@ paths: [ { - "type": string, // Required. One of: "datetime", "string", "number", "stringOptions", "categoryOptions", "arrayOptions", "stringObject", "numberObject", "boolean", "null" + "type": string, // Required. One of: "datetime", "string", "number", "stringOptions", "categoryOptions", "arrayOptions", "stringObject", "numberObject", "booleanObject", "boolean", "null" "column": string, // Required. Column to filter on (see available columns below) "operator": string, // Required. Operator based on type: // - datetime: ">", "<", ">=", "<=" @@ -6301,10 +6444,11 @@ paths: // - number: "=", ">", "<", ">=", "<=" // - stringObject: "=", "contains", "does not contain", "starts with", "ends with" // - numberObject: "=", ">", "<", ">=", "<=" + // - booleanObject: "=", "<>" // - boolean: "=", "<>" // - null: "is null", "is not null" "value": any, // Required (except for null type). Value to compare against. Type depends on filter type - "key": string // Required only for stringObject, numberObject, and categoryOptions types when filtering on nested fields like metadata + "key": string // Required only for stringObject, numberObject, booleanObject, and categoryOptions types when filtering on nested fields like metadata or score names } ] @@ -6389,6 +6533,12 @@ paths: - `score_categories` (categoryOptions) - Categorical score values + - `score_booleans` (booleanObject) - Boolean score values. Use `key` + for the score name and a boolean `value`, e.g. `{"type": + "booleanObject", "column": "score_booleans", "key": "is_correct", + "operator": "=", "value": true}`. The `<>` operator also matches + traces without a score of that name. + ## Filter Examples @@ -6475,6 +6625,7 @@ paths: application/json: schema: {} security: *ref_0 + deprecated: true delete: description: Delete multiple traces operationId: trace_deleteMultiple @@ -6529,20 +6680,101 @@ paths: required: - traceIds /api/public/unstable/dashboard-widgets: + get: + description: |- + List dashboard widgets in the project, ordered by most recently + updated first. + + Responses may include legacy `traces` widgets created before this + API existed. New widgets cannot be created with `view: traces`. + operationId: unstable_dashboardWidgets_list + tags: + - UnstableDashboardWidgets + parameters: + - name: page + in: query + description: 1-based page number. Defaults to `1`. + required: false + schema: + type: integer + nullable: true + - name: limit + in: query + description: Maximum number of items per page. Defaults to `50`. + required: false + schema: + type: integer + nullable: true + responses: + '200': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/unstableDashboardWidgetList' + '400': + description: '' + content: + application/json: + schema: {} + '401': + description: '' + content: + application/json: + schema: {} + '403': + description: '' + content: + application/json: + schema: {} + '404': + description: '' + content: + application/json: + schema: {} + '405': + description: '' + content: + application/json: + schema: {} + '429': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/unstablePublicApiError' + '500': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/unstablePublicApiError' + security: *ref_0 post: description: >- - Create a reusable dashboard widget. + Create a dashboard widget (a standalone chart definition you place on + + any dashboard). + + + This endpoint creates the widget only; place it on a dashboard via + + `POST /dashboards/{dashboardId}/placements`. + + + Supported views are `observations`, `scores-numeric`, `scores-boolean`, + and `scores-categorical`. + The legacy `traces` view is not supported by this unstable API. - This endpoint creates the widget. It does not place the widget on a - dashboard grid, this has to be done in the UI. + Widgets are created as v2 internally. - Supported views are `observations`, `scores-numeric`, and - `scores-categorical`. + `chartConfig` is optional and defaults to the plain config for - The legacy `traces` view is not supported by this unstable API, - `minVersion` defaults to `2`; values below `2` are rejected. + `chartType`; when `chartConfig.type` is given it must match + + `chartType`. Unstable API note: @@ -6604,108 +6836,28 @@ paths: application/json: schema: $ref: '#/components/schemas/unstableCreateDashboardWidgetRequest' - /api/public/unstable/evaluation-rules: - post: - description: >- - Create an evaluation rule. - + /api/public/unstable/dashboard-widgets/{widgetId}: + get: + description: |- + Get a dashboard widget by id. - An evaluation rule defines **what** incoming data should be evaluated - and **how prompt variables should be populated** from that data. - - - Use this resource after choosing an evaluator from the evaluator - endpoints. - - - Key rules: - - - `name` must be unique within the project for public evaluation rules - - - `target` must be `observation` or `experiment` - - - `evaluator.name` + `evaluator.scope` must identify an existing - evaluator family returned by the evaluator endpoints - - - Langfuse resolves that family to its latest version before saving the - evaluation rule - - - for `target=experiment`, use dataset `id` values from `GET - /api/public/v2/datasets` when filtering by `datasetId` - - - for `llm_as_judge` evaluators, every evaluator prompt variable must be - mapped exactly once - - - for `code` evaluators, Langfuse uses the fixed code runtime mapping; - omit `mapping` in create and update requests - - - for user-provided `llm_as_judge` mappings, `expected_output` and - `experiment_item_metadata` are only valid for `target=experiment` - - - if `enabled=true`, Langfuse validates that the referenced evaluator - can currently run - - - at most 50 evaluation rules can be effectively active in one project - at the same time - - - If an evaluation rule with the same `name` already exists in the - project, the API returns `409`. - - In that case, update the existing resource with `PATCH - /api/public/unstable/evaluation-rules/{evaluationRuleId}` instead of - creating a second one. - - - If enabling this resource would exceed the 50-active limit, the API also - returns `409`. - - In that case, disable or pause another active evaluation rule before - enabling a new one. - - - Current scope: - - - evaluation rules are live-ingestion rules only - - - they do not trigger historical backfills - - - Recovery guidance: - - - `400 invalid_filter_value`: fix the filter `column` or `value` using - `details.column`, `details.invalidValues`, and `details.allowedValues` - - - `400 invalid_filter_value` with `details.column=datasetId`: call `GET - /api/public/v2/datasets`, then retry with dataset `id` values from that - response - - - `400 missing_variable_mapping`: for `llm_as_judge` evaluators, fetch - the evaluator again and make sure every variable in `variables` appears - exactly once in `mapping` - - - `400 duplicate_variable_mapping`: remove repeated mappings for the - same variable - - - `400 invalid_variable_mapping`: for `llm_as_judge`, switch to a valid - `source` for the selected `target`, or fix the variable name - - - `400 invalid_json_path`: remove or correct the `jsonPath` - - - `422 evaluator_preflight_failed`: the selected evaluator cannot run - with the resolved model configuration. Fix the evaluator/default model - setup, then retry the create request. - operationId: unstable_evaluationRules_create + The response may use `view: traces` for legacy widgets. + operationId: unstable_dashboardWidgets_get tags: - - UnstableEvaluationRules - parameters: [] + - UnstableDashboardWidgets + parameters: + - name: widgetId + in: path + required: true + schema: + type: string responses: '200': description: '' content: application/json: schema: - $ref: '#/components/schemas/unstableEvaluationRule' + $ref: '#/components/schemas/unstableDashboardWidget' '400': description: '' content: @@ -6731,18 +6883,6 @@ paths: content: application/json: schema: {} - '409': - description: '' - content: - application/json: - schema: - $ref: '#/components/schemas/unstablePublicApiError' - '422': - description: '' - content: - application/json: - schema: - $ref: '#/components/schemas/unstablePublicApiError' '429': description: '' content: @@ -6756,44 +6896,33 @@ paths: schema: $ref: '#/components/schemas/unstablePublicApiError' security: *ref_0 - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/unstableCreateEvaluationRuleRequest' - get: - description: >- - List evaluation rules in the authenticated project. + patch: + description: |- + Update a dashboard widget. + All fields are optional; at least one field is required. + Changing `chartType` without sending `chartConfig` resets the config + to the new chart type's defaults. When `chartConfig.type` is given + it must match the widget's (possibly updated) `chartType`. - Each item describes one live evaluation rule and its effective runtime - status. - operationId: unstable_evaluationRules_list + `view` cannot be changed to the legacy `traces` value. Existing + `traces` widgets may be updated on other fields. + operationId: unstable_dashboardWidgets_update tags: - - UnstableEvaluationRules + - UnstableDashboardWidgets parameters: - - name: page - in: query - description: 1-based page number. Defaults to `1`. - required: false - schema: - type: integer - nullable: true - - name: limit - in: query - description: Maximum number of items per page. Defaults to `50`. - required: false + - name: widgetId + in: path + required: true schema: - type: integer - nullable: true + type: string responses: '200': description: '' content: application/json: schema: - $ref: '#/components/schemas/unstableEvaluationRules' + $ref: '#/components/schemas/unstableDashboardWidget' '400': description: '' content: @@ -6832,22 +6961,24 @@ paths: schema: $ref: '#/components/schemas/unstablePublicApiError' security: *ref_0 - /api/public/unstable/evaluation-rules/{evaluationRuleId}: - get: - description: >- - Get one evaluation rule by its identifier. - + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/unstableUpdateDashboardWidgetRequest' + delete: + description: |- + Delete a dashboard widget. - Use this endpoint to inspect the current evaluator, target, mapping, - filters, and effective runtime status. - operationId: unstable_evaluationRules_get + The API returns `409` while the widget is still placed on a dashboard. + Remove those placements first. + operationId: unstable_dashboardWidgets_delete tags: - - UnstableEvaluationRules + - UnstableDashboardWidgets parameters: - - name: evaluationRuleId + - name: widgetId in: path - description: >- - Evaluation rule identifier returned by the evaluation rule endpoints. required: true schema: type: string @@ -6857,7 +6988,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/unstableEvaluationRule' + $ref: '#/components/schemas/unstableDeleteDashboardWidgetResponse' '400': description: '' content: @@ -6883,6 +7014,12 @@ paths: content: application/json: schema: {} + '409': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/unstablePublicApiError' '429': description: '' content: @@ -6896,74 +7033,36 @@ paths: schema: $ref: '#/components/schemas/unstablePublicApiError' security: *ref_0 - patch: - description: >- - Update an evaluation rule. - - - Typical uses: - - - enable or disable live execution - - - switch to another evaluator - - - adjust sampling - - - change filters - - - update LLM-as-judge variable mappings - - - Important behavior: - - - provide only the fields you want to change - - - if you provide `evaluator`, Langfuse resolves that evaluator family to - its latest version before saving - - - changing `target`, `filter`, or an LLM-as-judge `mapping` must still - produce a valid target-specific configuration - - - if you change `target` for an LLM-as-judge rule, also send a - compatible `filter` and `mapping` in the same request unless the - existing ones are still valid for the new target - - - for `code` evaluator rules, omit `mapping`; Langfuse stores the fixed - code runtime mapping automatically - - - if the resulting config is enabled, Langfuse re-validates that the - selected evaluator can run - - - if the update would move a non-active evaluation rule into the active - state and the project already has 50 active evaluation rules, the API - returns `409` - - - Recovery guidance: - - - if an LLM-as-judge update fails with `missing_variable_mapping` or - `invalid_variable_mapping` after changing `evaluator` or `target`, - resend the request with a complete new `mapping` - - - if the update fails with `invalid_filter_value` after changing - `target`, resend the request with a target-compatible `filter` - operationId: unstable_evaluationRules_update + /api/public/unstable/dashboards: + get: + description: |- + List dashboards in the project, ordered by most recently updated + first. + operationId: unstable_dashboards_list tags: - - UnstableEvaluationRules + - UnstableDashboards parameters: - - name: evaluationRuleId - in: path - description: Evaluation rule identifier. - required: true + - name: page + in: query + description: 1-based page number. Defaults to `1`. + required: false schema: - type: string + type: integer + nullable: true + - name: limit + in: query + description: Maximum number of items per page. Defaults to `50`. + required: false + schema: + type: integer + nullable: true responses: '200': description: '' content: application/json: schema: - $ref: '#/components/schemas/unstableEvaluationRule' + $ref: '#/components/schemas/unstableDashboardList' '400': description: '' content: @@ -6989,45 +7088,85 @@ paths: content: application/json: schema: {} - '422': + '429': description: '' content: application/json: schema: $ref: '#/components/schemas/unstablePublicApiError' - '429': + '500': description: '' content: application/json: schema: $ref: '#/components/schemas/unstablePublicApiError' - '500': + security: *ref_0 + post: + description: Create a dashboard. + operationId: unstable_dashboards_create + tags: + - UnstableDashboards + parameters: [] + responses: + '200': description: '' content: application/json: schema: - $ref: '#/components/schemas/unstablePublicApiError' - security: *ref_0 - requestBody: - required: true - content: - application/json: + $ref: '#/components/schemas/unstableDashboard' + '400': + description: '' + content: + application/json: + schema: {} + '401': + description: '' + content: + application/json: + schema: {} + '403': + description: '' + content: + application/json: + schema: {} + '404': + description: '' + content: + application/json: + schema: {} + '405': + description: '' + content: + application/json: + schema: {} + '429': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/unstablePublicApiError' + '500': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/unstablePublicApiError' + security: *ref_0 + requestBody: + required: true + content: + application/json: schema: - $ref: '#/components/schemas/unstableUpdateEvaluationRuleRequest' - delete: - description: >- - Delete an evaluation rule. - - - This removes the live-ingestion rule only. It does not delete the - referenced evaluator. - operationId: unstable_evaluationRules_delete + $ref: '#/components/schemas/unstableCreateDashboardRequest' + /api/public/unstable/dashboards/{dashboardId}: + get: + description: Get a dashboard by id. + operationId: unstable_dashboards_get tags: - - UnstableEvaluationRules + - UnstableDashboards parameters: - - name: evaluationRuleId + - name: dashboardId in: path - description: Evaluation rule identifier. required: true schema: type: string @@ -7037,7 +7176,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/unstableDeleteEvaluationRuleResponse' + $ref: '#/components/schemas/unstableDashboard' '400': description: '' content: @@ -7076,92 +7215,24 @@ paths: schema: $ref: '#/components/schemas/unstablePublicApiError' security: *ref_0 - /api/public/unstable/evaluators: - post: - description: >- - Create an evaluator in the authenticated project. - - - Use evaluators to define **how** Langfuse should score data. - - LLM-as-a-judge evaluators define a prompt, expected structured output, - and optional model configuration. - - Code evaluators define source code and a runtime language. - - - Naming behavior: - - - If this is a new evaluator name in your project, Langfuse creates - version `1`. - - - If the name already exists in your project, Langfuse creates the next - version and returns it. - - - When a new project version is created, existing evaluation rules in - that project automatically move to the newest version for that evaluator - name. - - - Recommended workflow: - - 1. Create the evaluator. - - 2. Read the returned `variables` array. - - 3. Read the returned `outputDefinition.dataType` so the client knows - whether future scores will be numeric, boolean, or categorical. - - 4. Create one or more evaluation rules that reference the returned - evaluator family using `name` and `scope`. - - - Code evaluator validation: - - - At creation, Langfuse only validates the request shape - - - The `sourceCode` itself is not executed here. It is first run - (preflight-tested against a sample observation) when you link the - evaluator to an evaluation rule, so runtime errors in the code surface - at evaluation-rule creation, not at evaluator creation. - - - Recovery guidance: - - - `422` with `code=evaluator_preflight_failed`: the evaluator cannot run - with the resolved model configuration. Add a valid explicit - `modelConfig`, or configure the project's default evaluation model, then - retry the same request. - - - `400` with `code=invalid_body`: the request shape is malformed. Use - the structured `details.issues` array to fix the specific fields and - retry. - - - `400` with `code=invalid_body` on `outputDefinition`: for - `type=llm_as_judge`, send `dataType`, `reasoning.description`, and - `score.description`. Do not send `version`; it is not part of the public - request shape. - - - If `type` is omitted, Langfuse treats the request as - `type=llm_as_judge` for backwards compatibility. New clients should send - `type` explicitly. - - - Unstable API note: - - - This surface may evolve while the underlying evaluation data model is - being redesigned. - operationId: unstable_evaluators_create + patch: + description: Update a dashboard's name, description, definition, or filters. + operationId: unstable_dashboards_update tags: - - UnstableEvaluators - parameters: [] + - UnstableDashboards + parameters: + - name: dashboardId + in: path + required: true + schema: + type: string responses: '200': description: '' content: application/json: schema: - $ref: '#/components/schemas/unstableEvaluator' + $ref: '#/components/schemas/unstableDashboard' '400': description: '' content: @@ -7187,18 +7258,6 @@ paths: content: application/json: schema: {} - '409': - description: '' - content: - application/json: - schema: - $ref: '#/components/schemas/unstablePublicApiError' - '422': - description: '' - content: - application/json: - schema: - $ref: '#/components/schemas/unstablePublicApiError' '429': description: '' content: @@ -7217,46 +7276,25 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/unstableCreateEvaluatorRequest' - get: - description: >- - List the evaluators available to the authenticated project. - - - Important behavior: - - - This endpoint returns the latest version of each available evaluator. - - - Results can include evaluators from your project and Langfuse-managed - evaluators. - - - If the same evaluator name exists in both places, both are returned as - separate items with different `scope` values. - operationId: unstable_evaluators_list + $ref: '#/components/schemas/unstableUpdateDashboardRequest' + delete: + description: Delete a dashboard. + operationId: unstable_dashboards_delete tags: - - UnstableEvaluators + - UnstableDashboards parameters: - - name: page - in: query - description: 1-based page number. Defaults to `1`. - required: false - schema: - type: integer - nullable: true - - name: limit - in: query - description: Maximum number of items per page. Defaults to `50`. - required: false + - name: dashboardId + in: path + required: true schema: - type: integer - nullable: true + type: string responses: '200': description: '' content: application/json: schema: - $ref: '#/components/schemas/unstableEvaluators' + $ref: '#/components/schemas/unstableDeleteDashboardResponse' '400': description: '' content: @@ -7295,22 +7333,25 @@ paths: schema: $ref: '#/components/schemas/unstablePublicApiError' security: *ref_0 - /api/public/unstable/evaluators/{evaluatorId}: - get: - description: >- - Get one evaluator by `id`. + /api/public/unstable/dashboards/{dashboardId}/placements: + post: + description: |- + Add a placement to a dashboard grid (see `DashboardPlacement` for + grid semantics). + `id` and the position fields are optional: when omitted, the + placement gets a server-generated id and is appended below all + existing tiles as a 6x6 tile. Returns the created placement. - Use this endpoint when you want the prompt, output definition, model - configuration, and derived variables for the evaluator you plan to use - in an evaluation rule. - operationId: unstable_evaluators_get + The referenced widget must exist in the same project or be a + Langfuse-managed widget. The API returns `409` if a placement with + the same `id` already exists on the dashboard. + operationId: unstable_dashboards_addPlacement tags: - - UnstableEvaluators + - UnstableDashboards parameters: - - name: evaluatorId + - name: dashboardId in: path - description: Evaluator identifier returned by the evaluator endpoints. required: true schema: type: string @@ -7320,7 +7361,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/unstableEvaluator' + $ref: '#/components/schemas/unstableDashboardPlacement' '400': description: '' content: @@ -7346,6 +7387,12 @@ paths: content: application/json: schema: {} + '409': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/unstablePublicApiError' '429': description: '' content: @@ -7359,30 +7406,30 @@ paths: schema: $ref: '#/components/schemas/unstablePublicApiError' security: *ref_0 - delete: - description: >- - Delete an evaluator. - - - Important behavior: - - - This deletes the evaluator including all of its stored versions; - `evaluatorId` may reference any version. - - - The API returns `409` while evaluation rules still reference the - evaluator. Delete those evaluation rules first. - - - Langfuse-managed evaluators (`scope=managed`) cannot be deleted; the - API returns `403`. - - - Scores already produced by the evaluator are not deleted. - operationId: unstable_evaluators_delete + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/unstableCreateDashboardPlacementRequest' + /api/public/unstable/dashboards/{dashboardId}/placements/{placementId}: + patch: + description: |- + Move or resize a placement. All fields are optional; at least one is + required. Omitted fields keep their current value. The placement's + content (widget/preset reference) and id cannot change — delete and + re-add the placement to swap content. Returns the updated placement. + operationId: unstable_dashboards_updatePlacement tags: - - UnstableEvaluators + - UnstableDashboards parameters: - - name: evaluatorId + - name: dashboardId + in: path + required: true + schema: + type: string + - name: placementId in: path - description: Evaluator identifier returned by the evaluator endpoints. required: true schema: type: string @@ -7392,7 +7439,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/unstableDeleteEvaluatorResponse' + $ref: '#/components/schemas/unstableDashboardPlacement' '400': description: '' content: @@ -7418,12 +7465,6 @@ paths: content: application/json: schema: {} - '409': - description: '' - content: - application/json: - schema: - $ref: '#/components/schemas/unstablePublicApiError' '429': description: '' content: @@ -7437,15 +7478,922 @@ paths: schema: $ref: '#/components/schemas/unstablePublicApiError' security: *ref_0 -components: - schemas: - AnnotationQueueStatus: - title: AnnotationQueueStatus - type: string - enum: - - PENDING - - COMPLETED - AnnotationQueueObjectType: + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/unstableUpdateDashboardPlacementRequest' + delete: + description: >- + Remove a placement from a dashboard grid without deleting the referenced + widget. + operationId: unstable_dashboards_deletePlacement + tags: + - UnstableDashboards + parameters: + - name: dashboardId + in: path + required: true + schema: + type: string + - name: placementId + in: path + required: true + schema: + type: string + responses: + '200': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/unstableDeleteDashboardPlacementResponse' + '400': + description: '' + content: + application/json: + schema: {} + '401': + description: '' + content: + application/json: + schema: {} + '403': + description: '' + content: + application/json: + schema: {} + '404': + description: '' + content: + application/json: + schema: {} + '405': + description: '' + content: + application/json: + schema: {} + '429': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/unstablePublicApiError' + '500': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/unstablePublicApiError' + security: *ref_0 + /api/public/unstable/evaluation-rules: + post: + description: >- + Create an evaluation rule. + + + An evaluation rule defines **what** incoming data should be evaluated + and **how prompt variables should be populated** from that data. + + + Use this resource after choosing an evaluator from the evaluator + endpoints. + + + Key rules: + + - `name` must be unique within the project for public evaluation rules + + - `target` must be `observation` or `experiment` + + - `evaluator.name` + `evaluator.scope` must identify an existing + evaluator family returned by the evaluator endpoints + + - Langfuse resolves that family to its latest version before saving the + evaluation rule + + - for `target=experiment`, use dataset `id` values from `GET + /api/public/v2/datasets` when filtering by `datasetId` + + - for `llm_as_judge` evaluators, every evaluator prompt variable must be + mapped exactly once + + - for `code` evaluators, Langfuse uses the fixed code runtime mapping; + omit `mapping` in create and update requests + + - for user-provided `llm_as_judge` mappings, `expected_output` and + `experiment_item_metadata` are only valid for `target=experiment` + + - if `enabled=true`, Langfuse validates that the referenced evaluator + can currently run + + - at most 50 evaluation rules can be effectively active in one project + at the same time + + + If an evaluation rule with the same `name` already exists in the + project, the API returns `409`. + + In that case, update the existing resource with `PATCH + /api/public/unstable/evaluation-rules/{evaluationRuleId}` instead of + creating a second one. + + + If enabling this resource would exceed the 50-active limit, the API also + returns `409`. + + In that case, disable or pause another active evaluation rule before + enabling a new one. + + + Current scope: + + - evaluation rules are live-ingestion rules only + + - they do not trigger historical backfills + + + Recovery guidance: + + - `400 invalid_filter_value`: fix the filter `column` or `value` using + `details.column`, `details.invalidValues`, and `details.allowedValues` + + - `400 invalid_filter_value` with `details.column=datasetId`: call `GET + /api/public/v2/datasets`, then retry with dataset `id` values from that + response + + - `400 missing_variable_mapping`: for `llm_as_judge` evaluators, fetch + the evaluator again and make sure every variable in `variables` appears + exactly once in `mapping` + + - `400 duplicate_variable_mapping`: remove repeated mappings for the + same variable + + - `400 invalid_variable_mapping`: for `llm_as_judge`, switch to a valid + `source` for the selected `target`, or fix the variable name + + - `400 invalid_json_path`: remove or correct the `jsonPath` + + - `422 evaluator_preflight_failed`: the selected evaluator cannot run + with the resolved model configuration. Fix the evaluator/default model + setup, then retry the create request. + operationId: unstable_evaluationRules_create + tags: + - UnstableEvaluationRules + parameters: [] + responses: + '200': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/unstableEvaluationRule' + '400': + description: '' + content: + application/json: + schema: {} + '401': + description: '' + content: + application/json: + schema: {} + '403': + description: '' + content: + application/json: + schema: {} + '404': + description: '' + content: + application/json: + schema: {} + '405': + description: '' + content: + application/json: + schema: {} + '409': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/unstablePublicApiError' + '422': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/unstablePublicApiError' + '429': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/unstablePublicApiError' + '500': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/unstablePublicApiError' + security: *ref_0 + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/unstableCreateEvaluationRuleRequest' + get: + description: >- + List evaluation rules in the authenticated project. + + + This includes legacy `trace` and `dataset` rules so they can be + inspected and migrated to v4 rules. Legacy rules are read-only through + this API; create, update, and delete continue to support only + `observation` and `experiment` rules. + operationId: unstable_evaluationRules_list + tags: + - UnstableEvaluationRules + parameters: + - name: page + in: query + description: 1-based page number. Defaults to `1`. + required: false + schema: + type: integer + nullable: true + - name: limit + in: query + description: Maximum number of items per page. Defaults to `50`. + required: false + schema: + type: integer + nullable: true + responses: + '200': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/unstableEvaluationRules' + '400': + description: '' + content: + application/json: + schema: {} + '401': + description: '' + content: + application/json: + schema: {} + '403': + description: '' + content: + application/json: + schema: {} + '404': + description: '' + content: + application/json: + schema: {} + '405': + description: '' + content: + application/json: + schema: {} + '429': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/unstablePublicApiError' + '500': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/unstablePublicApiError' + security: *ref_0 + /api/public/unstable/evaluation-rules/{evaluationRuleId}: + get: + description: >- + Get one evaluation rule by its identifier. + + + Use this endpoint to inspect the current evaluator, target, mapping, + filters, execution timing, and effective runtime status. Legacy `trace` + and `dataset` rules are returned for migration and are read-only through + this API. + operationId: unstable_evaluationRules_get + tags: + - UnstableEvaluationRules + parameters: + - name: evaluationRuleId + in: path + description: >- + Evaluation rule identifier returned by the evaluation rule + endpoints. + required: true + schema: + type: string + responses: + '200': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/unstableReadableEvaluationRule' + '400': + description: '' + content: + application/json: + schema: {} + '401': + description: '' + content: + application/json: + schema: {} + '403': + description: '' + content: + application/json: + schema: {} + '404': + description: '' + content: + application/json: + schema: {} + '405': + description: '' + content: + application/json: + schema: {} + '429': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/unstablePublicApiError' + '500': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/unstablePublicApiError' + security: *ref_0 + patch: + description: >- + Update an evaluation rule. + + + Typical uses: + + - enable or disable live execution + + - switch to another evaluator + + - adjust sampling + + - change filters + + - update LLM-as-judge variable mappings + + + Important behavior: + + - provide only the fields you want to change + + - if you provide `evaluator`, Langfuse resolves that evaluator family to + its latest version before saving + + - changing `target`, `filter`, or an LLM-as-judge `mapping` must still + produce a valid target-specific configuration + + - if you change `target` for an LLM-as-judge rule, also send a + compatible `filter` and `mapping` in the same request unless the + existing ones are still valid for the new target + + - for `code` evaluator rules, omit `mapping`; Langfuse stores the fixed + code runtime mapping automatically + + - if the resulting config is enabled, Langfuse re-validates that the + selected evaluator can run + + - if the update would move a non-active evaluation rule into the active + state and the project already has 50 active evaluation rules, the API + returns `409` + + + Recovery guidance: + + - if an LLM-as-judge update fails with `missing_variable_mapping` or + `invalid_variable_mapping` after changing `evaluator` or `target`, + resend the request with a complete new `mapping` + + - if the update fails with `invalid_filter_value` after changing + `target`, resend the request with a target-compatible `filter` + operationId: unstable_evaluationRules_update + tags: + - UnstableEvaluationRules + parameters: + - name: evaluationRuleId + in: path + description: Evaluation rule identifier. + required: true + schema: + type: string + responses: + '200': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/unstableEvaluationRule' + '400': + description: '' + content: + application/json: + schema: {} + '401': + description: '' + content: + application/json: + schema: {} + '403': + description: '' + content: + application/json: + schema: {} + '404': + description: '' + content: + application/json: + schema: {} + '405': + description: '' + content: + application/json: + schema: {} + '422': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/unstablePublicApiError' + '429': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/unstablePublicApiError' + '500': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/unstablePublicApiError' + security: *ref_0 + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/unstableUpdateEvaluationRuleRequest' + delete: + description: >- + Delete an evaluation rule. + + + This removes the live-ingestion rule only. It does not delete the + referenced evaluator. + operationId: unstable_evaluationRules_delete + tags: + - UnstableEvaluationRules + parameters: + - name: evaluationRuleId + in: path + description: Evaluation rule identifier. + required: true + schema: + type: string + responses: + '200': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/unstableDeleteEvaluationRuleResponse' + '400': + description: '' + content: + application/json: + schema: {} + '401': + description: '' + content: + application/json: + schema: {} + '403': + description: '' + content: + application/json: + schema: {} + '404': + description: '' + content: + application/json: + schema: {} + '405': + description: '' + content: + application/json: + schema: {} + '429': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/unstablePublicApiError' + '500': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/unstablePublicApiError' + security: *ref_0 + /api/public/unstable/evaluators: + post: + description: >- + Create an evaluator in the authenticated project. + + + Use evaluators to define **how** Langfuse should score data. + + LLM-as-a-judge evaluators define a prompt, expected structured output, + and optional model configuration. + + Code evaluators define source code and a runtime language. + + + Naming behavior: + + - If this is a new evaluator name in your project, Langfuse creates + version `1`. + + - If the name already exists in your project, Langfuse creates the next + version and returns it. + + - When a new project version is created, existing evaluation rules in + that project automatically move to the newest version for that evaluator + name. + + + Recommended workflow: + + 1. Create the evaluator. + + 2. Read the returned `variables` array. + + 3. Read the returned `outputDefinition.dataType` so the client knows + whether future scores will be numeric, boolean, or categorical. + + 4. Create one or more evaluation rules that reference the returned + evaluator family using `name` and `scope`. + + + Code evaluator validation: + + - At creation, Langfuse only validates the request shape + + - The `sourceCode` itself is not executed here. It is first run + (preflight-tested against a sample observation) when you link the + evaluator to an evaluation rule, so runtime errors in the code surface + at evaluation-rule creation, not at evaluator creation. + + + Recovery guidance: + + - `422` with `code=evaluator_preflight_failed`: the evaluator cannot run + with the resolved model configuration. Add a valid explicit + `modelConfig`, or configure the project's default evaluation model, then + retry the same request. + + - `400` with `code=invalid_body`: the request shape is malformed. Use + the structured `details.issues` array to fix the specific fields and + retry. + + - `400` with `code=invalid_body` on `outputDefinition`: for + `type=llm_as_judge`, send `dataType`, `reasoning.description`, and + `score.description`. Do not send `version`; it is not part of the public + request shape. + + - If `type` is omitted, Langfuse treats the request as + `type=llm_as_judge` for backwards compatibility. New clients should send + `type` explicitly. + + + Unstable API note: + + - This surface may evolve while the underlying evaluation data model is + being redesigned. + operationId: unstable_evaluators_create + tags: + - UnstableEvaluators + parameters: [] + responses: + '200': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/unstableEvaluator' + '400': + description: '' + content: + application/json: + schema: {} + '401': + description: '' + content: + application/json: + schema: {} + '403': + description: '' + content: + application/json: + schema: {} + '404': + description: '' + content: + application/json: + schema: {} + '405': + description: '' + content: + application/json: + schema: {} + '409': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/unstablePublicApiError' + '422': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/unstablePublicApiError' + '429': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/unstablePublicApiError' + '500': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/unstablePublicApiError' + security: *ref_0 + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/unstableCreateEvaluatorRequest' + get: + description: >- + List the evaluators available to the authenticated project. + + + Important behavior: + + - This endpoint returns the latest version of each available evaluator. + + - Results can include evaluators from your project and Langfuse-managed + evaluators. + + - If the same evaluator name exists in both places, both are returned as + separate items with different `scope` values. + operationId: unstable_evaluators_list + tags: + - UnstableEvaluators + parameters: + - name: page + in: query + description: 1-based page number. Defaults to `1`. + required: false + schema: + type: integer + nullable: true + - name: limit + in: query + description: Maximum number of items per page. Defaults to `50`. + required: false + schema: + type: integer + nullable: true + responses: + '200': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/unstableEvaluators' + '400': + description: '' + content: + application/json: + schema: {} + '401': + description: '' + content: + application/json: + schema: {} + '403': + description: '' + content: + application/json: + schema: {} + '404': + description: '' + content: + application/json: + schema: {} + '405': + description: '' + content: + application/json: + schema: {} + '429': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/unstablePublicApiError' + '500': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/unstablePublicApiError' + security: *ref_0 + /api/public/unstable/evaluators/{evaluatorId}: + get: + description: >- + Get one evaluator by `id`. + + + Use this endpoint when you want the prompt, output definition, model + configuration, and derived variables for the evaluator you plan to use + in an evaluation rule. + operationId: unstable_evaluators_get + tags: + - UnstableEvaluators + parameters: + - name: evaluatorId + in: path + description: Evaluator identifier returned by the evaluator endpoints. + required: true + schema: + type: string + responses: + '200': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/unstableEvaluator' + '400': + description: '' + content: + application/json: + schema: {} + '401': + description: '' + content: + application/json: + schema: {} + '403': + description: '' + content: + application/json: + schema: {} + '404': + description: '' + content: + application/json: + schema: {} + '405': + description: '' + content: + application/json: + schema: {} + '429': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/unstablePublicApiError' + '500': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/unstablePublicApiError' + security: *ref_0 + delete: + description: >- + Delete an evaluator. + + + Important behavior: + + - This deletes the evaluator including all of its stored versions; + `evaluatorId` may reference any version. + + - The API returns `409` while evaluation rules still reference the + evaluator. Delete those evaluation rules first. + + - Langfuse-managed evaluators (`scope=managed`) cannot be deleted; the + API returns `403`. + + - Scores already produced by the evaluator are not deleted. + operationId: unstable_evaluators_delete + tags: + - UnstableEvaluators + parameters: + - name: evaluatorId + in: path + description: Evaluator identifier returned by the evaluator endpoints. + required: true + schema: + type: string + responses: + '200': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/unstableDeleteEvaluatorResponse' + '400': + description: '' + content: + application/json: + schema: {} + '401': + description: '' + content: + application/json: + schema: {} + '403': + description: '' + content: + application/json: + schema: {} + '404': + description: '' + content: + application/json: + schema: {} + '405': + description: '' + content: + application/json: + schema: {} + '409': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/unstablePublicApiError' + '429': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/unstablePublicApiError' + '500': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/unstablePublicApiError' + security: *ref_0 +components: + schemas: + AnnotationQueueStatus: + title: AnnotationQueueStatus + type: string + enum: + - PENDING + - COMPLETED + AnnotationQueueObjectType: title: AnnotationQueueObjectType type: string enum: @@ -7631,6 +8579,13 @@ components: - JSON - CSV - JSONL + - PARQUET + description: >- + File format for exported data. `PARQUET` is a columnar binary format + encoded and compressed by the storage engine; gzip compression does not + apply to it. Note that the model-price columns (`input_price`, + `output_price`, `total_price`) are not included in Parquet observation + exports. BlobStorageIntegrationFileTypeResponse: title: BlobStorageIntegrationFileTypeResponse type: string @@ -7639,10 +8594,7 @@ components: - CSV - JSONL - PARQUET - description: >- - File type reported for an existing integration. Includes `PARQUET`, - which a project may enable through the Langfuse UI but cannot yet be set - via this API (the request `fileType` omits it). + description: File type reported for an existing integration. BlobStorageExportMode: title: BlobStorageExportMode type: string @@ -8035,7 +8987,10 @@ components: authorUserId: type: string nullable: true - description: The id of the user who created the comment. + description: >- + The id of the user who created the comment. Must be a member of the + organization that owns the project, otherwise an error will be + thrown. required: - projectId - objectType @@ -8063,6 +9018,37 @@ components: required: - data - meta + Deprecation: + title: Deprecation + type: object + description: >- + Migration signal returned by deprecated endpoints. Optional fields are + omitted when they have no value. + properties: + message: + type: string + description: >- + Human- and agent-readable summary of the deprecation and its + replacement. + replacement: + type: string + nullable: true + description: >- + The replacement endpoint, e.g. "GET /api/public/v2/observations". + Omitted when the endpoint is being removed without a direct + replacement. + docsUrl: + type: string + nullable: true + description: Link to the migration documentation (markdown), when available. + sunsetAt: + type: string + nullable: true + description: >- + ISO date after which the endpoint may stop working, when a removal + date is committed. + required: + - message Trace: title: Trace type: object @@ -8188,6 +9174,9 @@ components: items: $ref: '#/components/schemas/ScoreV1' description: List of scores + _deprecation: + $ref: '#/components/schemas/Deprecation' + nullable: true required: - htmlPath - observations @@ -8221,6 +9210,9 @@ components: type: array items: $ref: '#/components/schemas/Trace' + _deprecation: + $ref: '#/components/schemas/Deprecation' + nullable: true required: - traces allOf: @@ -8414,6 +9406,15 @@ components: - timeToFirstToken allOf: - $ref: '#/components/schemas/Observation' + ObservationsViewSingle: + title: ObservationsViewSingle + type: object + properties: + _deprecation: + $ref: '#/components/schemas/Deprecation' + nullable: true + allOf: + - $ref: '#/components/schemas/ObservationsView' ObservationV2: title: ObservationV2 type: object @@ -8445,10 +9446,22 @@ components: parentObservationId: type: string nullable: true - description: The parent observation ID + description: >- + The physical parent observation ID, if present. + + Observations marked as app roots by the SDK may retain a non-null + parent ID. type: type: string description: The type of the observation (e.g. GENERATION, SPAN, EVENT) + isRootObservation: + type: boolean + nullable: true + description: >- + Whether this observation is a logical root. + + This is true for observations without a physical parent and + observations marked as app roots by the SDK. name: type: string nullable: true @@ -8817,7 +9830,8 @@ components: type: number format: double description: >- - The numeric value of the score. Equals 1 for "True" and 0 for "False" + The numeric value of the score. Equals 1 for "True" and 0 for + "False" stringValue: type: string description: >- @@ -8856,29 +9870,56 @@ components: type: string description: The text content of the score (1-500 characters) required: - - stringValue - allOf: - - $ref: '#/components/schemas/BaseScoreV1' - ScoreV1: - title: ScoreV1 - type: object - properties: - dataType: - type: string - enum: - - NUMERIC - - CATEGORICAL - - BOOLEAN - - TEXT - value: - type: string - description: The numeric value of the score - stringValue: - type: string - description: The string representation of the score value. If no config is - linked, can be any string. Otherwise, must map to a config category - required: - - dataType + - stringValue + allOf: + - $ref: '#/components/schemas/BaseScoreV1' + ScoreV1: + title: ScoreV1 + oneOf: + - type: object + allOf: + - type: object + properties: + dataType: + type: string + enum: + - NUMERIC + - $ref: '#/components/schemas/NumericScoreV1' + required: + - dataType + - type: object + allOf: + - type: object + properties: + dataType: + type: string + enum: + - CATEGORICAL + - $ref: '#/components/schemas/CategoricalScoreV1' + required: + - dataType + - type: object + allOf: + - type: object + properties: + dataType: + type: string + enum: + - BOOLEAN + - $ref: '#/components/schemas/BooleanScoreV1' + required: + - dataType + - type: object + allOf: + - type: object + properties: + dataType: + type: string + enum: + - TEXT + - $ref: '#/components/schemas/TextScoreV1' + required: + - dataType BaseScore: title: BaseScore type: object @@ -8976,7 +10017,8 @@ components: type: number format: double description: >- - The numeric value of the score. Equals 1 for "True" and 0 for "False" + The numeric value of the score. Equals 1 for "True" and 0 for + "False" stringValue: type: string description: >- @@ -9036,25 +10078,66 @@ components: - $ref: '#/components/schemas/BaseScore' Score: title: Score - type: object + oneOf: + - type: object + allOf: + - type: object + properties: + dataType: + type: string + enum: + - NUMERIC + - $ref: '#/components/schemas/NumericScore' + required: + - dataType + - type: object + allOf: + - type: object + properties: + dataType: + type: string + enum: + - CATEGORICAL + - $ref: '#/components/schemas/CategoricalScore' + required: + - dataType + - type: object + allOf: + - type: object + properties: + dataType: + type: string + enum: + - BOOLEAN + - $ref: '#/components/schemas/BooleanScore' + required: + - dataType + - type: object + allOf: + - type: object + properties: + dataType: + type: string + enum: + - CORRECTION + - $ref: '#/components/schemas/CorrectionScore' + required: + - dataType + - type: object + allOf: + - type: object + properties: + dataType: + type: string + enum: + - TEXT + - $ref: '#/components/schemas/TextScore' + required: + - dataType properties: - dataType: - type: string - enum: - - NUMERIC - - CATEGORICAL - - BOOLEAN - - CORRECTION - - TEXT - value: - type: string - description: The numeric value of the score - stringValue: - type: string - description: The string representation of the score value. If no config is - linked, can be any string. Otherwise, must map to a config category - required: - - dataType + _deprecation: + $ref: '#/components/schemas/Deprecation' + nullable: true CreateScoreValue: title: CreateScoreValue oneOf: @@ -9323,6 +10406,9 @@ components: type: array items: $ref: '#/components/schemas/DatasetRunItem' + _deprecation: + $ref: '#/components/schemas/Deprecation' + nullable: true required: - datasetRunItems allOf: @@ -9981,6 +11067,9 @@ components: $ref: '#/components/schemas/DatasetRunItem' meta: $ref: '#/components/schemas/utilsMetaResponse' + _deprecation: + $ref: '#/components/schemas/Deprecation' + nullable: true required: - data - meta @@ -10031,6 +11120,9 @@ components: $ref: '#/components/schemas/DatasetRun' meta: $ref: '#/components/schemas/utilsMetaResponse' + _deprecation: + $ref: '#/components/schemas/Deprecation' + nullable: true required: - data - meta @@ -10211,6 +11303,61 @@ components: - experimentId - experimentName - experimentItemId + FeedbackTargetType: + title: FeedbackTargetType + type: string + enum: + - skill + - mcp-tool + - cli + - docs + - public-api + - other + SubmitFeedbackRequest: + title: SubmitFeedbackRequest + type: object + properties: + targetType: + $ref: '#/components/schemas/FeedbackTargetType' + description: Category of the thing the feedback is about. + target: + type: string + description: >- + The specific instance within targetType: the skill name, MCP tool + name, CLI command, API endpoint path, or docs page path (e.g. + 'queryMetrics', '/docs/mcp'). An identifier, not a sentence. Must be + between 1 and 200 characters. + feedback: + type: string + description: >- + Concise feedback text approved by the user. Must be between 1 and + 3000 characters. + goal: + type: string + nullable: true + description: >- + Optional user-approved goal or use case they were trying to achieve. + Must be between 1 and 1500 characters when provided. Do not include + secrets, customer data, trace payloads, or broad unrelated context. + referenceUrl: + type: string + nullable: true + description: >- + Optional HTTP(S) reference URL. Langfuse stores it as text for + triage and does not fetch it. + required: + - targetType + - target + - feedback + SubmitFeedbackResponse: + title: SubmitFeedbackResponse + type: object + properties: + id: + type: string + description: Correlation ID for the submitted feedback. + required: + - id HealthResponse: title: HealthResponse type: object @@ -10225,26 +11372,117 @@ components: - status IngestionEvent: title: IngestionEvent - type: object - properties: - type: - type: string - enum: - - trace-create - - score-create - - span-create - - span-update - - generation-create - - generation-update - - event-create - - sdk-log - - observation-create - - observation-update - body: - type: string - required: - - type - - body + oneOf: + - type: object + allOf: + - type: object + properties: + type: + type: string + enum: + - trace-create + - $ref: '#/components/schemas/TraceEvent' + required: + - type + - type: object + allOf: + - type: object + properties: + type: + type: string + enum: + - score-create + - $ref: '#/components/schemas/ScoreEvent' + required: + - type + - type: object + allOf: + - type: object + properties: + type: + type: string + enum: + - span-create + - $ref: '#/components/schemas/CreateSpanEvent' + required: + - type + - type: object + allOf: + - type: object + properties: + type: + type: string + enum: + - span-update + - $ref: '#/components/schemas/UpdateSpanEvent' + required: + - type + - type: object + allOf: + - type: object + properties: + type: + type: string + enum: + - generation-create + - $ref: '#/components/schemas/CreateGenerationEvent' + required: + - type + - type: object + allOf: + - type: object + properties: + type: + type: string + enum: + - generation-update + - $ref: '#/components/schemas/UpdateGenerationEvent' + required: + - type + - type: object + allOf: + - type: object + properties: + type: + type: string + enum: + - event-create + - $ref: '#/components/schemas/CreateEventEvent' + required: + - type + - type: object + allOf: + - type: object + properties: + type: + type: string + enum: + - sdk-log + - $ref: '#/components/schemas/SDKLogEvent' + required: + - type + - type: object + allOf: + - type: object + properties: + type: + type: string + enum: + - observation-create + - $ref: '#/components/schemas/CreateObservationEvent' + required: + - type + - type: object + allOf: + - type: object + properties: + type: + type: string + enum: + - observation-update + - $ref: '#/components/schemas/UpdateObservationEvent' + required: + - type ObservationType: title: ObservationType type: string @@ -10839,6 +12077,9 @@ components: Format varies based on the query parameters. Histograms will return an array with [lower, upper, height] tuples. + _deprecation: + $ref: '#/components/schemas/Deprecation' + nullable: true required: - data legacyObservations: @@ -10864,102 +12105,12 @@ components: $ref: '#/components/schemas/ObservationsView' meta: $ref: '#/components/schemas/utilsMetaResponse' + _deprecation: + $ref: '#/components/schemas/Deprecation' + nullable: true required: - data - meta - legacyCreateScoreRequest: - title: legacyCreateScoreRequest - type: object - properties: - id: - type: string - nullable: true - traceId: - type: string - nullable: true - sessionId: - type: string - nullable: true - observationId: - type: string - nullable: true - datasetRunId: - type: string - nullable: true - name: - type: string - value: - $ref: '#/components/schemas/CreateScoreValue' - description: >- - The value of the score. Must be passed as string for categorical and - text scores, and numeric for boolean and numeric scores. Boolean - score values must equal either 1 or 0 (true or false). Text score - values must be between 1 and 500 characters. - comment: - type: string - nullable: true - metadata: - type: object - additionalProperties: true - nullable: true - environment: - type: string - nullable: true - description: >- - The environment of the score. Can be any lowercase alphanumeric - string with hyphens and underscores that does not start with - 'langfuse'. - queueId: - type: string - nullable: true - description: >- - The annotation queue referenced by the score. Indicates if score was - initially created while processing annotation queue. - dataType: - $ref: '#/components/schemas/ScoreDataType' - nullable: true - description: >- - The data type of the score. When passing a configId this field is - inferred. Otherwise, this field must be passed or will default to - numeric. - configId: - type: string - nullable: true - description: >- - Reference a score config on a score. The unique langfuse identifier - of a score config. When passing this field, the dataType and - stringValue fields are automatically populated. - source: - $ref: '#/components/schemas/legacyCreateScoreSource' - nullable: true - description: >- - The source of the score. Defaults to API. Set to ANNOTATION to - prefill scores (e.g. from an LLM) for a human reviewer to verify in - an annotation queue. When source is ANNOTATION, a configId is - required unless dataType is CORRECTION. EVAL is reserved for - internal evaluator outputs and is not accepted on this endpoint. - required: - - name - - value - legacyCreateScoreSource: - title: legacyCreateScoreSource - type: string - enum: - - API - - ANNOTATION - description: |- - Source values accepted when creating a score via the public REST API. - EVAL is reserved for internal evaluator outputs and is intentionally not - exposed here — use commons.ScoreSource when reading scores. - legacyCreateScoreResponse: - title: legacyCreateScoreResponse - type: object - properties: - id: - type: string - description: The id of the created object in Langfuse - required: - - id LlmConnection: title: LlmConnection type: object @@ -11985,18 +13136,29 @@ components: - prompt Prompt: title: Prompt - type: object - properties: - type: - type: string - enum: - - chat - - text - prompt: - type: string - required: - - type - - prompt + oneOf: + - type: object + allOf: + - type: object + properties: + type: + type: string + enum: + - chat + - $ref: '#/components/schemas/ChatPrompt' + required: + - type + - type: object + allOf: + - type: object + properties: + type: + type: string + enum: + - text + - $ref: '#/components/schemas/TextPrompt' + required: + - type PromptType: title: PromptType type: string @@ -12551,25 +13713,54 @@ components: - id ScoreSubjectV3: title: ScoreSubjectV3 - type: object - properties: - kind: - type: string - enum: - - trace - - observation - - session - - experiment - id: - type: string - description: The trace ID. - traceId: - type: string - nullable: true - description: The parent trace ID, if available. - required: - - kind - - id + oneOf: + - type: object + allOf: + - type: object + properties: + kind: + type: string + enum: + - trace + - $ref: '#/components/schemas/ScoreSubjectTraceV3' + required: + - kind + - type: object + allOf: + - type: object + properties: + kind: + type: string + enum: + - observation + - $ref: '#/components/schemas/ScoreSubjectObservationV3' + required: + - kind + - type: object + allOf: + - type: object + properties: + kind: + type: string + enum: + - session + - $ref: '#/components/schemas/ScoreSubjectSessionV3' + required: + - kind + - type: object + allOf: + - type: object + properties: + kind: + type: string + enum: + - experiment + - $ref: '#/components/schemas/ScoreSubjectExperimentV3' + required: + - kind + description: >- + A reference to the entity this score is attached to. Discriminated by + "kind" — one of trace, observation, session, or experiment. BaseScoreV3: title: BaseScoreV3 type: object @@ -12695,26 +13886,66 @@ components: description: The correction content of the score. Empty string if not set. required: - value - allOf: - - $ref: '#/components/schemas/BaseScoreV3' - ScoreV3: - title: ScoreV3 - type: object - properties: - dataType: - type: string - enum: - - NUMERIC - - BOOLEAN - - CATEGORICAL - - TEXT - - CORRECTION - value: - type: string - description: The numeric value of the score. - required: - - dataType - - value + allOf: + - $ref: '#/components/schemas/BaseScoreV3' + ScoreV3: + title: ScoreV3 + oneOf: + - type: object + allOf: + - type: object + properties: + dataType: + type: string + enum: + - NUMERIC + - $ref: '#/components/schemas/NumericScoreV3' + required: + - dataType + - type: object + allOf: + - type: object + properties: + dataType: + type: string + enum: + - BOOLEAN + - $ref: '#/components/schemas/BooleanScoreV3' + required: + - dataType + - type: object + allOf: + - type: object + properties: + dataType: + type: string + enum: + - CATEGORICAL + - $ref: '#/components/schemas/CategoricalScoreV3' + required: + - dataType + - type: object + allOf: + - type: object + properties: + dataType: + type: string + enum: + - TEXT + - $ref: '#/components/schemas/TextScoreV3' + required: + - dataType + - type: object + allOf: + - type: object + properties: + dataType: + type: string + enum: + - CORRECTION + - $ref: '#/components/schemas/CorrectionScoreV3' + required: + - dataType GetScoresV3Meta: title: GetScoresV3Meta type: object @@ -12742,6 +13973,99 @@ components: required: - data - meta + CreateScoreRequest: + title: CreateScoreRequest + type: object + properties: + id: + type: string + nullable: true + traceId: + type: string + nullable: true + sessionId: + type: string + nullable: true + observationId: + type: string + nullable: true + datasetRunId: + type: string + nullable: true + name: + type: string + value: + $ref: '#/components/schemas/CreateScoreValue' + description: >- + The value of the score. Must be passed as string for categorical and + text scores, and numeric for boolean and numeric scores. Boolean + score values must equal either 1 or 0 (true or false). Text score + values must be between 1 and 500 characters. + comment: + type: string + nullable: true + metadata: + type: object + additionalProperties: true + nullable: true + environment: + type: string + nullable: true + description: >- + The environment of the score. Can be any lowercase alphanumeric + string with hyphens and underscores that does not start with + 'langfuse'. + queueId: + type: string + nullable: true + description: >- + The annotation queue referenced by the score. Indicates if score was + initially created while processing annotation queue. + dataType: + $ref: '#/components/schemas/ScoreDataType' + nullable: true + description: >- + The data type of the score. When passing a configId this field is + inferred. Otherwise, this field must be passed or will default to + numeric. + configId: + type: string + nullable: true + description: >- + Reference a score config on a score. The unique langfuse identifier + of a score config. When passing this field, the dataType and + stringValue fields are automatically populated. + source: + $ref: '#/components/schemas/CreateScoreSource' + nullable: true + description: >- + The source of the score. Defaults to API. Set to ANNOTATION to + prefill scores (e.g. from an LLM) for a human reviewer to verify in + an annotation queue. When source is ANNOTATION, a configId is + required unless dataType is CORRECTION. EVAL is reserved for + internal evaluator outputs and is not accepted on this endpoint. + required: + - name + - value + CreateScoreSource: + title: CreateScoreSource + type: string + enum: + - API + - ANNOTATION + description: |- + Source values accepted when creating a score via the public REST API. + EVAL is reserved for internal evaluator outputs and is intentionally not + exposed here — use commons.ScoreSource when reading scores. + CreateScoreResponse: + title: CreateScoreResponse + type: object + properties: + id: + type: string + description: The id of the created object in Langfuse + required: + - id GetScoresResponseTraceData: title: GetScoresResponseTraceData type: object @@ -12811,20 +14135,62 @@ components: - $ref: '#/components/schemas/TextScore' GetScoresResponseData: title: GetScoresResponseData - type: object - properties: - dataType: - type: string - enum: - - NUMERIC - - CATEGORICAL - - BOOLEAN - - CORRECTION - - TEXT - trace: - $ref: '#/components/schemas/GetScoresResponseTraceData' - required: - - dataType + oneOf: + - type: object + allOf: + - type: object + properties: + dataType: + type: string + enum: + - NUMERIC + - $ref: '#/components/schemas/GetScoresResponseDataNumeric' + required: + - dataType + - type: object + allOf: + - type: object + properties: + dataType: + type: string + enum: + - CATEGORICAL + - $ref: '#/components/schemas/GetScoresResponseDataCategorical' + required: + - dataType + - type: object + allOf: + - type: object + properties: + dataType: + type: string + enum: + - BOOLEAN + - $ref: '#/components/schemas/GetScoresResponseDataBoolean' + required: + - dataType + - type: object + allOf: + - type: object + properties: + dataType: + type: string + enum: + - CORRECTION + - $ref: '#/components/schemas/GetScoresResponseDataCorrection' + required: + - dataType + - type: object + allOf: + - type: object + properties: + dataType: + type: string + enum: + - TEXT + - $ref: '#/components/schemas/GetScoresResponseDataText' + required: + - dataType GetScoresResponse: title: GetScoresResponse type: object @@ -12835,6 +14201,9 @@ components: $ref: '#/components/schemas/GetScoresResponseData' meta: $ref: '#/components/schemas/utilsMetaResponse' + _deprecation: + $ref: '#/components/schemas/Deprecation' + nullable: true required: - data - meta @@ -12848,6 +14217,9 @@ components: $ref: '#/components/schemas/Session' meta: $ref: '#/components/schemas/utilsMetaResponse' + _deprecation: + $ref: '#/components/schemas/Deprecation' + nullable: true required: - data - meta @@ -12861,6 +14233,9 @@ components: $ref: '#/components/schemas/TraceWithDetails' meta: $ref: '#/components/schemas/utilsMetaResponse' + _deprecation: + $ref: '#/components/schemas/Deprecation' + nullable: true required: - data - meta @@ -12923,7 +14298,7 @@ components: - `observation` evaluates live-ingested observations such as generations, spans, and events. - It supports mapping from `input`, `output`, and `metadata`. + It supports mapping from `input`, `output`, `metadata`, and `tool_calls`. - `experiment` evaluates live experiment executions and can additionally map `expected_output` and `experiment_item_metadata`. It currently supports filtering by `datasetId`. @@ -12952,6 +14327,7 @@ components: - input - output - metadata + - tool_calls - expected_output - experiment_item_metadata description: >- @@ -12963,10 +14339,11 @@ components: Target-specific rules: - - `target=observation` supports `input`, `output`, and `metadata` + - `target=observation` supports `input`, `output`, `metadata`, and + `tool_calls` - `target=experiment` supports `input`, `output`, `metadata`, - `expected_output`, and `experiment_item_metadata` + `tool_calls`, `expected_output`, and `experiment_item_metadata` Source semantics: @@ -12978,6 +14355,11 @@ components: - `metadata`: the metadata object for the target. Combine with `jsonPath` when you need one nested field instead of the whole object. + - `tool_calls`: the tool calls recorded on the observation, as an array + of `{id, name, arguments, type, index}` objects in the order the model + emitted them. Combine with `jsonPath` (for example `$[*].name`) to + select parts of each call. + - `expected_output`: the experiment item's expected output. Only valid for `target=experiment`. @@ -13024,7 +14406,8 @@ components: model: type: string description: >- - Model identifier exposed by the provider, for example `gpt-4.1-mini`. + Model identifier exposed by the provider, for example + `gpt-4.1-mini`. required: - provider - model @@ -13060,22 +14443,59 @@ components: - description unstableEvaluatorOutputDefinition: title: unstableEvaluatorOutputDefinition - type: object - properties: - dataType: - type: string - enum: - - NUMERIC - - BOOLEAN - - CATEGORICAL - reasoning: - $ref: '#/components/schemas/unstableEvaluatorOutputFieldDefinition' - score: - type: string - required: - - dataType - - reasoning - - score + oneOf: + - type: object + allOf: + - type: object + properties: + dataType: + type: string + enum: + - NUMERIC + - $ref: >- + #/components/schemas/unstablePublicNumericEvaluatorOutputDefinition + required: + - dataType + - type: object + allOf: + - type: object + properties: + dataType: + type: string + enum: + - BOOLEAN + - $ref: >- + #/components/schemas/unstablePublicBooleanEvaluatorOutputDefinition + required: + - dataType + - type: object + allOf: + - type: object + properties: + dataType: + type: string + enum: + - CATEGORICAL + - $ref: >- + #/components/schemas/unstablePublicCategoricalEvaluatorOutputDefinition + required: + - dataType + description: >- + Structured output definition to send when creating an evaluator. + + + Agent guidance: + + - `dataType` is required. + + - Do not send `version`; that is an internal storage detail and is not + part of the public request contract. + + - For `NUMERIC` and `BOOLEAN`, provide `reasoning.description` and + `score.description`. + + - For `CATEGORICAL`, also provide `score.categories` and + `score.shouldAllowMultipleMatches`. unstablePublicNumericEvaluatorOutputDefinition: title: unstablePublicNumericEvaluatorOutputDefinition type: object @@ -13140,22 +14560,62 @@ components: - score unstablePublicEvaluatorOutputDefinition: title: unstablePublicEvaluatorOutputDefinition - type: object - properties: - dataType: - type: string - enum: - - NUMERIC - - BOOLEAN - - CATEGORICAL - reasoning: - $ref: '#/components/schemas/unstableEvaluatorOutputFieldDefinition' - score: - type: string - required: - - dataType - - reasoning - - score + oneOf: + - type: object + allOf: + - type: object + properties: + dataType: + type: string + enum: + - NUMERIC + - $ref: >- + #/components/schemas/unstablePublicNumericEvaluatorOutputDefinition + required: + - dataType + - type: object + allOf: + - type: object + properties: + dataType: + type: string + enum: + - BOOLEAN + - $ref: >- + #/components/schemas/unstablePublicBooleanEvaluatorOutputDefinition + required: + - dataType + - type: object + allOf: + - type: object + properties: + dataType: + type: string + enum: + - CATEGORICAL + - $ref: >- + #/components/schemas/unstablePublicCategoricalEvaluatorOutputDefinition + required: + - dataType + description: >- + Evaluator output definition returned by the public API. + + + This response always includes `dataType` and never includes an internal + output-definition `version`. + + Legacy stored evaluator definitions are normalized into this shape + before they are returned. + + + Use this response shape when deciding how to interpret future evaluation + scores: + + - `NUMERIC`: expect numeric score values + + - `BOOLEAN`: expect `true` / `false` + + - `CATEGORICAL`: expect one or more values from `score.categories` unstableEvaluationRuleStringFilterOperator: title: unstableEvaluationRuleStringFilterOperator type: string @@ -13442,9 +14902,9 @@ components: Quick reference: - - `target=observation`: `input`, `output`, `metadata` + - `target=observation`: `input`, `output`, `metadata`, `tool_calls` - - `target=experiment`: `input`, `output`, `metadata`, + - `target=experiment`: `input`, `output`, `metadata`, `tool_calls`, `expected_output`, `experiment_item_metadata` jsonPath: type: string @@ -13454,56 +14914,222 @@ components: is passed to the evaluator prompt. - Requirements: + Requirements: + + - Must start with `$` + + - Must be a syntactically valid JSONPath expression + + - Most useful with `source=metadata` + required: + - variable + - source + unstableEvaluationRuleFilter: + title: unstableEvaluationRuleFilter + oneOf: + - type: object + allOf: + - type: object + properties: + type: + type: string + enum: + - datetime + - $ref: '#/components/schemas/unstableDateTimeEvaluationRuleFilter' + required: + - type + - type: object + allOf: + - type: object + properties: + type: + type: string + enum: + - string + - $ref: '#/components/schemas/unstableStringEvaluationRuleFilter' + required: + - type + - type: object + allOf: + - type: object + properties: + type: + type: string + enum: + - number + - $ref: '#/components/schemas/unstableNumberEvaluationRuleFilter' + required: + - type + - type: object + allOf: + - type: object + properties: + type: + type: string + enum: + - stringOptions + - $ref: '#/components/schemas/unstableStringOptionsEvaluationRuleFilter' + required: + - type + - type: object + allOf: + - type: object + properties: + type: + type: string + enum: + - categoryOptions + - $ref: '#/components/schemas/unstableCategoryOptionsEvaluationRuleFilter' + required: + - type + - type: object + allOf: + - type: object + properties: + type: + type: string + enum: + - arrayOptions + - $ref: '#/components/schemas/unstableArrayOptionsEvaluationRuleFilter' + required: + - type + - type: object + allOf: + - type: object + properties: + type: + type: string + enum: + - stringObject + - $ref: '#/components/schemas/unstableStringObjectEvaluationRuleFilter' + required: + - type + - type: object + allOf: + - type: object + properties: + type: + type: string + enum: + - numberObject + - $ref: '#/components/schemas/unstableNumberObjectEvaluationRuleFilter' + required: + - type + - type: object + allOf: + - type: object + properties: + type: + type: string + enum: + - boolean + - $ref: '#/components/schemas/unstableBooleanEvaluationRuleFilter' + required: + - type + - type: object + allOf: + - type: object + properties: + type: + type: string + enum: + - 'null' + - $ref: '#/components/schemas/unstableNullEvaluationRuleFilter' + required: + - type + description: >- + One filter condition used to decide whether a live-ingested target + should be evaluated. + + + An evaluation rule can include zero or more filter objects. All filters + must be satisfied for the target to run. + + + How to build a valid filter object: + + - Pick the `target` first, because it changes the supported columns. + + - Pick the filter `type`. That determines which fields are required. + + - Use `key` only for object filters such as `metadata`. + + - Use the correct `value` shape for the chosen filter `type`. + + + Operator quick reference by filter `type`: + + - `string`: `"="`, `contains`, `does not contain`, `starts with`, `ends + with` + + - `number`: `"="`, `">"`, `"<"`, `">="`, `"<="` + + - `datetime`: `"="`, `">"`, `"<"`, `">="`, `"<="` + + - `stringOptions`: `any of`, `none of` + + - `arrayOptions`: `any of`, `none of`, `all of` + + - `stringObject`: same operators as `string` - - Must start with `$` + - `boolean`: `"="`, `"<>"` - - Must be a syntactically valid JSONPath expression + - `null`: `is null`, `is not null` - - Most useful with `source=metadata` - required: - - variable - - source - unstableEvaluationRuleFilter: - title: unstableEvaluationRuleFilter - type: object - properties: - type: - type: string - enum: - - datetime - - string - - number - - stringOptions - - categoryOptions - - arrayOptions - - stringObject - - numberObject - - boolean - - 'null' - column: - type: string - description: Column to filter on. - operator: - type: string - description: Comparison operator for datetime values. - value: - type: string - description: Datetime value to compare against. - key: - type: string - description: Key inside the object-valued column to filter on. - required: - - type - - column - - operator + + Supported columns by target: + + - `target=observation` + - `type`: `stringOptions`, operators `any of` / `none of`, values `GENERATION`, `SPAN`, `EVENT` + - `name`: `stringOptions`, operators `any of` / `none of` + - `environment`: `stringOptions`, operators `any of` / `none of` + - `level`: `stringOptions`, operators `any of` / `none of`, values `DEBUG`, `DEFAULT`, `WARNING`, `ERROR` + - `version`: `string` + - `traceName`: `stringOptions`, operators `any of` / `none of` + - `userId`: `string` + - `sessionId`: `string` + - `tags`: `arrayOptions`, operators `any of` / `none of` / `all of` + - `metadata`: `stringObject` with `key` + - `isRootObservation`: `boolean`, operators `=` / `<>`; true when the observation has no parent or is explicitly marked as an application root + - `parentObservationId`: `null`, operators `is null` / `is not null` + - `calledToolNames`: `arrayOptions`, operators `any of` / `none of` / `all of` + - `toolCalls`: `number` + - `target=experiment` + - `datasetId`: `stringOptions`, operators `any of` / `none of` + Use dataset `id` values from `GET /api/public/v2/datasets`, not dataset names. + + Recovery guidance: + + - `invalid_filter_value` with `details.column` but no `invalidValues`: + the selected `column` is not supported for the chosen `target` + + - `invalid_filter_value` with `details.invalidValues`: the selected + values are not allowed for that column. Replace them with one of + `details.allowedValues` when provided. + + - `invalid_filter_value` for `column=datasetId`: call `GET + /api/public/v2/datasets`, then retry with dataset `id` values from that + response. unstableDashboardWidgetView: title: unstableDashboardWidgetView type: string enum: - observations - scores-numeric + - scores-boolean + - scores-categorical + unstableDashboardWidgetViewWithLegacy: + title: unstableDashboardWidgetViewWithLegacy + type: string + enum: + - observations + - scores-numeric + - scores-boolean - scores-categorical + - traces + description: |- + Widget data view. Responses may include the legacy `traces` value for + widgets created before this API existed. unstableDashboardWidgetChartType: title: unstableDashboardWidgetChartType type: string @@ -13556,13 +15182,40 @@ components: title: unstableDashboardWidgetFilter type: object description: >- - A dashboard widget filter in Langfuse filter-state shape. + A filter in Langfuse filter-state shape. The `value` shape and the + + allowed operators depend on `type`: + + + | `type` | `value` | operators | + + |---|---|---| + | `string` | string | `=`, `contains`, `does not contain`, `starts + with`, `ends with` | - Filter shapes depend on `type`, for example string filters use a string - `value`, + | `number` | number | `=`, `>`, `<`, `>=`, `<=` | - option filters use a list of strings, and object filters include `key`. + | `datetime` | ISO datetime string | `>`, `<`, `>=`, `<=` | + + | `boolean` | boolean | `=`, `<>` | + + | `null` | `""` | `is null`, `is not null` | + + | `stringOptions` | list of strings | `any of`, `none of` | + + | `arrayOptions` | list of strings | `any of`, `none of`, `all of` | + + | `categoryOptions` | list of strings (requires `key`) | `any of`, `none + of` | + + | `stringObject` | string (requires `key`, e.g. a metadata key) | same + as `string` | + + | `numberObject` | number (requires `key`, e.g. a score name) | same as + `number` | + + | `booleanObject` | boolean (requires `key`) | `=`, `<>` | properties: column: type: string @@ -13623,6 +15276,28 @@ components: enum: - ASC - DESC + unstableDashboardWidgetChartConfigInput: + title: unstableDashboardWidgetChartConfigInput + type: object + description: |- + Input-side chart config. `type` is optional and defaults to the + widget's `chartType`; when given it must match. + properties: + type: + $ref: '#/components/schemas/unstableDashboardWidgetChartType' + nullable: true + row_limit: + type: integer + nullable: true + show_value_labels: + type: boolean + nullable: true + bins: + type: integer + nullable: true + defaultSort: + $ref: '#/components/schemas/unstableDashboardWidgetDefaultSort' + nullable: true unstableCreateDashboardWidgetRequest: title: unstableCreateDashboardWidgetRequest type: object @@ -13631,8 +15306,108 @@ components: type: string description: type: string + nullable: true + description: Defaults to an empty string. + view: + $ref: '#/components/schemas/unstableDashboardWidgetView' + dimensions: + type: array + items: + $ref: '#/components/schemas/unstableDashboardWidgetDimension' + metrics: + type: array + items: + $ref: '#/components/schemas/unstableDashboardWidgetMetric' + filters: + type: array + items: + $ref: '#/components/schemas/unstableDashboardWidgetFilter' + chartType: + $ref: '#/components/schemas/unstableDashboardWidgetChartType' + chartConfig: + $ref: '#/components/schemas/unstableDashboardWidgetChartConfigInput' + nullable: true + description: Defaults to the plain config for `chartType`. + required: + - name + - view + - dimensions + - metrics + - filters + - chartType + unstableUpdateDashboardWidgetRequest: + title: unstableUpdateDashboardWidgetRequest + type: object + properties: + name: + type: string + nullable: true + description: + type: string + nullable: true view: $ref: '#/components/schemas/unstableDashboardWidgetView' + nullable: true + dimensions: + type: array + items: + $ref: '#/components/schemas/unstableDashboardWidgetDimension' + nullable: true + metrics: + type: array + items: + $ref: '#/components/schemas/unstableDashboardWidgetMetric' + nullable: true + filters: + type: array + items: + $ref: '#/components/schemas/unstableDashboardWidgetFilter' + nullable: true + chartType: + $ref: '#/components/schemas/unstableDashboardWidgetChartType' + nullable: true + chartConfig: + $ref: '#/components/schemas/unstableDashboardWidgetChartConfigInput' + nullable: true + unstableDashboardWidgetList: + title: unstableDashboardWidgetList + type: object + properties: + data: + type: array + items: + $ref: '#/components/schemas/unstableDashboardWidget' + meta: + $ref: '#/components/schemas/utilsMetaResponse' + required: + - data + - meta + unstableDeleteDashboardWidgetResponse: + title: unstableDeleteDashboardWidgetResponse + type: object + properties: + message: + type: string + required: + - message + unstableDashboardWidget: + title: unstableDashboardWidget + type: object + properties: + id: + type: string + createdAt: + type: string + format: date-time + updatedAt: + type: string + format: date-time + name: + type: string + description: + type: string + view: + $ref: '#/components/schemas/unstableDashboardWidgetViewWithLegacy' dimensions: type: array items: @@ -13649,20 +15424,216 @@ components: $ref: '#/components/schemas/unstableDashboardWidgetChartType' chartConfig: $ref: '#/components/schemas/unstableDashboardWidgetChartConfig' - minVersion: + required: + - id + - createdAt + - updatedAt + - name + - description + - view + - dimensions + - metrics + - filters + - chartType + - chartConfig + unstableDashboardPlacement: + title: unstableDashboardPlacement + oneOf: + - type: object + allOf: + - type: object + properties: + type: + type: string + enum: + - widget + - $ref: '#/components/schemas/unstableWidgetPlacement' + required: + - type + - type: object + allOf: + - type: object + properties: + type: + type: string + enum: + - preset + - $ref: '#/components/schemas/unstablePresetPlacement' + required: + - type + description: |- + A tile on the dashboard's 12-column grid. `x`/`y` are the tile's + top-left cell (0-based; `y` grows downward), `width`/`height` its size + in cells. The UI default tile is 6x6 (half width). Overlapping tiles + are not rejected; prefer appending below existing tiles (or omit the + position on create to let the server do it). + unstableWidgetPlacement: + title: unstableWidgetPlacement + type: object + properties: + id: + type: string + widgetId: + type: string + x: + type: integer + 'y': + type: integer + width: + type: integer + height: + type: integer + required: + - id + - widgetId + - x + - 'y' + - width + - height + unstablePresetPlacement: + title: unstablePresetPlacement + type: object + properties: + id: + type: string + presetId: + type: string + x: + type: integer + 'y': + type: integer + width: + type: integer + height: + type: integer + required: + - id + - presetId + - x + - 'y' + - width + - height + unstableCreateDashboardPlacementRequest: + title: unstableCreateDashboardPlacementRequest + oneOf: + - type: object + allOf: + - type: object + properties: + type: + type: string + enum: + - widget + - $ref: '#/components/schemas/unstableCreateWidgetPlacement' + required: + - type + - type: object + allOf: + - type: object + properties: + type: + type: string + enum: + - preset + - $ref: '#/components/schemas/unstableCreatePresetPlacement' + required: + - type + unstableCreateWidgetPlacement: + title: unstableCreateWidgetPlacement + type: object + properties: + id: + type: string + nullable: true + description: Server-generated when omitted. + widgetId: + type: string + x: + type: integer + nullable: true + description: Grid column (12-column grid). Defaults to `0`. + 'y': + type: integer + nullable: true + description: Grid row. Defaults to the first row below all existing tiles. + width: + type: integer + nullable: true + description: Width in grid columns. Defaults to `6`. + height: + type: integer + nullable: true + description: Height in grid rows. Defaults to `6`. + required: + - widgetId + unstableCreatePresetPlacement: + title: unstableCreatePresetPlacement + type: object + properties: + id: + type: string + nullable: true + description: Server-generated when omitted. + presetId: + type: string + x: + type: integer + nullable: true + description: Grid column (12-column grid). Defaults to `0`. + 'y': + type: integer + nullable: true + description: Grid row. Defaults to the first row below all existing tiles. + width: + type: integer + nullable: true + description: Width in grid columns. Defaults to `6`. + height: + type: integer + nullable: true + description: Height in grid rows. Defaults to `6`. + required: + - presetId + unstableUpdateDashboardPlacementRequest: + title: unstableUpdateDashboardPlacementRequest + type: object + properties: + x: + type: integer + nullable: true + description: Grid column (12-column grid). + 'y': + type: integer + nullable: true + description: Grid row. + width: + type: integer + nullable: true + description: Width in grid columns. + height: type: integer nullable: true + description: Height in grid rows. + unstableDeleteDashboardPlacementResponse: + title: unstableDeleteDashboardPlacementResponse + type: object + properties: + message: + type: string + required: + - message + unstableDashboardDefinition: + title: unstableDashboardDefinition + type: object + properties: + widgets: + type: array + items: + $ref: '#/components/schemas/unstableDashboardPlacement' required: - - name - - description - - view - - dimensions - - metrics - - filters - - chartType - - chartConfig - unstableDashboardWidget: - title: unstableDashboardWidget + - widgets + unstableDashboard: + title: unstableDashboard type: object properties: id: @@ -13677,39 +15648,79 @@ components: type: string description: type: string - view: - $ref: '#/components/schemas/unstableDashboardWidgetView' - dimensions: - type: array - items: - $ref: '#/components/schemas/unstableDashboardWidgetDimension' - metrics: - type: array - items: - $ref: '#/components/schemas/unstableDashboardWidgetMetric' + definition: + $ref: '#/components/schemas/unstableDashboardDefinition' filters: type: array items: $ref: '#/components/schemas/unstableDashboardWidgetFilter' - chartType: - $ref: '#/components/schemas/unstableDashboardWidgetChartType' - chartConfig: - $ref: '#/components/schemas/unstableDashboardWidgetChartConfig' - minVersion: - type: integer + description: Dashboard-level filters applied to all widgets on the dashboard. required: - id - createdAt - updatedAt - name - description - - view - - dimensions - - metrics + - definition - filters - - chartType - - chartConfig - - minVersion + unstableDashboardList: + title: unstableDashboardList + type: object + properties: + data: + type: array + items: + $ref: '#/components/schemas/unstableDashboard' + meta: + $ref: '#/components/schemas/utilsMetaResponse' + required: + - data + - meta + unstableCreateDashboardRequest: + title: unstableCreateDashboardRequest + type: object + properties: + name: + type: string + description: + type: string + nullable: true + definition: + $ref: '#/components/schemas/unstableDashboardDefinition' + nullable: true + filters: + type: array + items: + $ref: '#/components/schemas/unstableDashboardWidgetFilter' + nullable: true + required: + - name + unstableUpdateDashboardRequest: + title: unstableUpdateDashboardRequest + type: object + properties: + name: + type: string + nullable: true + description: + type: string + nullable: true + definition: + $ref: '#/components/schemas/unstableDashboardDefinition' + nullable: true + filters: + type: array + items: + $ref: '#/components/schemas/unstableDashboardWidgetFilter' + nullable: true + unstableDeleteDashboardResponse: + title: unstableDeleteDashboardResponse + type: object + properties: + message: + type: string + required: + - message unstablePublicApiErrorCode: title: unstablePublicApiErrorCode type: string @@ -13903,8 +15914,8 @@ components: required: - message - code - unstableEvaluationRule: - title: unstableEvaluationRule + unstableEvaluationRuleBase: + title: unstableEvaluationRuleBase type: object description: >- Live evaluation rule for incoming data. @@ -13953,9 +15964,6 @@ components: If you create a newer project version with the same evaluator name later, existing evaluation rules are moved to it automatically. - target: - $ref: '#/components/schemas/unstableEvaluationRuleTarget' - description: Target object type that should trigger scoring. enabled: type: boolean description: Desired enabled state configured by the client. @@ -13981,20 +15989,6 @@ components: Must be greater than `0` and less than or equal to `1`. - `1` means evaluate every matching target. - `0.25` means evaluate approximately 25% of matching targets. - filter: - type: array - items: - $ref: '#/components/schemas/unstableEvaluationRuleFilter' - description: >- - List of filter conditions used to decide whether a target should be - evaluated. - mapping: - type: array - items: - $ref: '#/components/schemas/unstableEvaluationRuleMapping' - description: >- - Variable mappings used to populate evaluator runtime variables from - the live target object. createdAt: type: string format: date-time @@ -14007,16 +16001,94 @@ components: - id - name - evaluator - - target - enabled - status - pausedReason - pausedMessage - sampling - - filter - - mapping - createdAt - updatedAt + unstableEvaluationRule: + title: unstableEvaluationRule + type: object + properties: + target: + $ref: '#/components/schemas/unstableEvaluationRuleTarget' + description: Target object type that should trigger scoring. + filter: + type: array + items: + $ref: '#/components/schemas/unstableEvaluationRuleFilter' + description: >- + List of filter conditions used to decide whether a target should be + evaluated. + mapping: + type: array + items: + $ref: '#/components/schemas/unstableEvaluationRuleMapping' + description: >- + Variable mappings used to populate evaluator runtime variables from + the live target object. + required: + - target + - filter + - mapping + allOf: + - $ref: '#/components/schemas/unstableEvaluationRuleBase' + unstableLegacyEvaluationRule: + title: unstableLegacyEvaluationRule + type: object + description: >- + Legacy trace- or dataset-level evaluation rule returned by list and get + for migration. + + + This resource is read-only through the unstable public API. Its mapping + preserves the trace, dataset item, or named observation that each + evaluator variable previously read from. Its filters use the persisted + legacy filter format so migration clients can read the configuration + without losing information. + properties: + target: + $ref: '#/components/schemas/unstableLegacyEvaluationRuleTarget' + delay: + type: integer + description: Delay in milliseconds before the legacy evaluation job runs. + timeScope: + type: array + items: + $ref: '#/components/schemas/unstableEvaluationRuleTimeScope' + description: >- + Whether the legacy rule evaluates newly ingested data, existing + data, or both. + filter: + type: array + items: + $ref: '#/components/schemas/unstableEvaluationRuleFilter' + description: Stored filters used by the legacy trace or dataset rule. + mapping: + type: array + items: + $ref: '#/components/schemas/unstableLegacyEvaluationRuleMapping' + description: >- + Stored variable mappings, including the trace, dataset item, or + named observation selected for each variable. + required: + - target + - delay + - timeScope + - filter + - mapping + allOf: + - $ref: '#/components/schemas/unstableEvaluationRuleBase' + unstableReadableEvaluationRule: + title: unstableReadableEvaluationRule + oneOf: + - $ref: '#/components/schemas/unstableEvaluationRule' + - $ref: '#/components/schemas/unstableLegacyEvaluationRule' + description: >- + Evaluation rule returned by list and get, including read-only legacy + trace and dataset rules. unstableEvaluationRules: title: unstableEvaluationRules type: object @@ -14025,7 +16097,7 @@ components: data: type: array items: - $ref: '#/components/schemas/unstableEvaluationRule' + $ref: '#/components/schemas/unstableReadableEvaluationRule' description: Evaluation rules in the current page. meta: $ref: '#/components/schemas/utilsMetaResponse' @@ -14338,6 +16410,67 @@ components: type: string enum: - llm_as_judge + unstableEvaluationRuleTimeScope: + title: unstableEvaluationRuleTimeScope + type: string + enum: + - NEW + - EXISTING + unstableLegacyEvaluationRuleTarget: + title: unstableLegacyEvaluationRuleTarget + type: string + enum: + - trace + - dataset + unstableLegacyEvaluationRuleMapping: + title: unstableLegacyEvaluationRuleMapping + type: object + description: >- + Maps one evaluator variable to a trace, dataset item, or field on a + named observation in a legacy rule. + properties: + variable: + type: string + description: Evaluator prompt variable populated by this mapping. + langfuseObject: + $ref: '#/components/schemas/unstableLegacyEvaluationObject' + description: >- + Trace, dataset item, or observation type from which the value is + read. + objectName: + type: string + nullable: true + description: >- + Observation name to match, or `null` when `langfuseObject` is + `trace` or `dataset_item`. + source: + type: string + description: Stored field selected from the trace, dataset item, or observation. + jsonPath: + type: string + nullable: true + description: Optional JSONPath selector applied to the selected field. + required: + - variable + - langfuseObject + - objectName + - source + unstableLegacyEvaluationObject: + title: unstableLegacyEvaluationObject + type: string + enum: + - trace + - span + - generation + - event + - agent + - tool + - chain + - retriever + - evaluator + - embedding + - guardrail + - dataset_item unstableEvaluationRuleEvaluator: title: unstableEvaluationRuleEvaluator type: object @@ -14350,7 +16483,8 @@ components: id: type: string description: >- - Identifier of the exact evaluator version currently used by the rule. + Identifier of the exact evaluator version currently used by the + rule. name: type: string description: Evaluator family name. @@ -14377,38 +16511,59 @@ components: - message unstableEvaluator: title: unstableEvaluator - type: object - properties: - type: - type: string - enum: - - llm_as_judge - - code - prompt: - type: string - description: Prompt template used during evaluation. - outputDefinition: - $ref: '#/components/schemas/unstablePublicEvaluatorOutputDefinition' - description: >- - Structured output schema returned by this evaluator. + oneOf: + - type: object + allOf: + - type: object + properties: + type: + type: string + enum: + - llm_as_judge + - $ref: '#/components/schemas/unstableLlmAsJudgeEvaluator' + required: + - type + - type: object + allOf: + - type: object + properties: + type: + type: string + enum: + - code + - $ref: '#/components/schemas/unstableCodeEvaluator' + required: + - type + description: >- + One evaluator that can be used for scoring. - Responses always include `dataType` and omit the internal - output-definition `version`. + An evaluator describes **how** to score data. - Use `dataType` to decide how future scores should be interpreted. - modelConfig: - $ref: '#/components/schemas/unstableEvaluatorModelConfig' - description: Explicit model configuration, or `null` when the project default - evaluation model is used. - sourceCode: - type: string - description: Source code executed for each matched observation. - sourceCodeLanguage: - $ref: '#/components/schemas/unstableCodeEvaluatorSourceCodeLanguage' - description: Runtime language for `sourceCode`. - required: - - type + + It does not define **which** live objects are evaluated. That is the job + of `evaluation-rules`. + + + For agent clients, the most important fields are: + + - `type`: determines which evaluator fields are present + + - `variables`: for LLM evaluators, use these exact names when building + the evaluation-rule `mapping` array. LLM evaluators require every + variable to be mapped. Code evaluators always expose the fixed runtime + payload fields and Langfuse maps them automatically. + + + Versioning behavior: + + - `GET /evaluators` returns the latest version of each available + evaluator. + + - `GET /evaluators/{id}` can return an older version. + + - Evaluation rules always run against the latest version for the + selected evaluator name within the same source (`project` or `managed`). unstableEvaluatorBase: title: unstableEvaluatorBase type: object @@ -14519,42 +16674,41 @@ components: - meta unstableCreateEvaluatorRequest: title: unstableCreateEvaluatorRequest - type: object - properties: - type: - type: string - enum: - - llm_as_judge - - code - name: - type: string - description: Evaluator name within the authenticated project. - prompt: - type: string - description: Prompt template used by the evaluator. - outputDefinition: - $ref: '#/components/schemas/unstableEvaluatorOutputDefinition' - description: >- - Structured output schema the evaluator must return. + oneOf: + - type: object + allOf: + - type: object + properties: + type: + type: string + enum: + - llm_as_judge + - $ref: '#/components/schemas/unstableCreateLlmAsJudgeEvaluatorRequest' + required: + - type + - type: object + allOf: + - type: object + properties: + type: + type: string + enum: + - code + - $ref: '#/components/schemas/unstableCreateCodeEvaluatorRequest' + required: + - type + description: >- + Request body for creating an evaluator. - Always send `dataType`. + If the same `name` already exists in your project, Langfuse creates the + next version and returns it. - Do not send `version`; it is an internal storage detail and not part - of the public request contract. - modelConfig: - $ref: '#/components/schemas/unstableEvaluatorModelConfig' - description: Optional explicit model configuration. Omit or set to `null` to use - the project default evaluation model. - sourceCode: - type: string - description: Code executed for each matched observation. - sourceCodeLanguage: - $ref: '#/components/schemas/unstableCodeEvaluatorSourceCodeLanguage' - description: Runtime language for `sourceCode`. - required: - - type - - name + Existing evaluation rules in the same project are then moved to that new + latest version automatically. + + If `type` is omitted, Langfuse defaults it to `llm_as_judge` for + backwards compatibility. unstableCreateLlmAsJudgeEvaluatorRequest: title: unstableCreateLlmAsJudgeEvaluatorRequest type: object diff --git a/conformance/src/add-version.ts b/conformance/src/add-version.ts new file mode 100644 index 0000000..74d4f7c --- /dev/null +++ b/conformance/src/add-version.ts @@ -0,0 +1,344 @@ +import SwaggerParser from "@apidevtools/swagger-parser"; +import { mkdir, rm } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; +import { parse } from "yaml"; + +import { compileApiContract } from "../../src/contracts/compiler"; +import { + CATALOG_PATH, + CONFORMANCE_ROOT, + REPOSITORY_ROOT, + readVerifiedSpec, + sha256, +} from "./catalog"; +import { compileOpenApi } from "./openapi"; +import type { Catalog, CatalogEntry } from "./types"; + +const README_PATH = resolve(CONFORMANCE_ROOT, "README.md"); +const USER_AGENT = "langfuse-cli-conformance-suite"; + +export interface SpecSummary { + version: string; + paths: number; + operations: number; +} + +interface AddVersionOptions { + dryRun: boolean; + runChecks: boolean; +} + +function usage(): never { + process.stderr.write(`Usage: bun run conformance:add-version -- vX.Y.Z [--dry-run]\n`); + process.exit(2); +} + +export function normalizeReleaseTag(input: string): { + tag: string; + version: string; +} { + const match = /^v?(\d+)\.(\d+)\.(\d+)$/.exec(input); + if (!match) throw new Error(`Expected a stable release tag like v4.10.0, got ${input}`); + const version = `${Number(match[1])}.${Number(match[2])}.${Number(match[3])}`; + return { tag: `v${version}`, version }; +} + +function versionParts(version: string): [number, number, number] { + const normalized = normalizeReleaseTag(version).version; + return normalized.split(".").map(Number) as [number, number, number]; +} + +export function compareVersions(left: string, right: string): number { + const a = versionParts(left); + const b = versionParts(right); + for (let index = 0; index < 3; index++) { + if (a[index] !== b[index]) return a[index] - b[index]; + } + return 0; +} + +export function withCatalogEntry( + catalog: Catalog, + entry: CatalogEntry, +): Catalog { + if ( + catalog.versions.some( + (candidate) => candidate.version === entry.version || candidate.ref === entry.ref, + ) + ) { + throw new Error(`${entry.ref} is already present in the catalog`); + } + return { + ...catalog, + versions: [...catalog.versions, entry].sort((left, right) => + compareVersions(left.version, right.version), + ), + }; +} + +export function formatCatalog(catalog: Catalog): string { + return `${JSON.stringify(catalog, null, 2).replace( + /"knownIssues": \[\n\s+"([^"]+)"\n\s+\]/g, + '"knownIssues": ["$1"]', + )}\n`; +} + +export function updateConformanceReadme( + content: string, + summaries: SpecSummary[], +): string { + const total = summaries.reduce((sum, item) => sum + item.operations, 0); + let updated = replaceRequired( + content, + /currently attempts all \d+ operations across .*? snapshots using/, + `currently attempts all ${total} operations across ${summaries.length} pinned snapshots using`, + "historical adapter operation count", + ); + updated = replaceRequired( + updated, + /checks all \d+ operations through/, + `checks all ${total} operations through`, + "native adapter operation count", + ); + const heading = "| Langfuse | Paths | Operations |"; + const tableStart = updated.indexOf(heading); + if (tableStart === -1) throw new Error("Could not find pinned-spec table in conformance README"); + const tableEnd = updated.indexOf("\n\n", tableStart); + if (tableEnd === -1) throw new Error("Could not find end of pinned-spec table"); + const table = [ + heading, + "|---|---:|---:|", + ...summaries.map( + (summary) => + `| ${summary.version} | ${summary.paths} | ${summary.operations} |`, + ), + ].join("\n"); + return `${updated.slice(0, tableStart)}${table}${updated.slice(tableEnd)}`; +} + +function replaceRequired( + content: string, + pattern: RegExp, + replacement: string, + label: string, +): string { + if (!pattern.test(content)) { + throw new Error(`Could not find ${label} in conformance README`); + } + return content.replace(pattern, replacement); +} + +function githubHeaders(): HeadersInit { + const token = process.env.GITHUB_TOKEN ?? process.env.GH_TOKEN; + return { + accept: "application/vnd.github+json", + "user-agent": USER_AGENT, + ...(token ? { authorization: `Bearer ${token}` } : {}), + }; +} + +async function githubJson(path: string): Promise { + const response = await fetch(`https://api.github.com${path}`, { + headers: githubHeaders(), + }); + if (!response.ok) { + throw new Error(`GitHub ${path}: ${response.status} ${response.statusText}`); + } + return response.json(); +} + +async function resolveStableRelease(tag: string): Promise { + const release = await githubJson( + `/repos/langfuse/langfuse/releases/tags/${encodeURIComponent(tag)}`, + ); + if (release.draft || release.prerelease) { + throw new Error(`${tag} is not a stable published release`); + } + const ref = await githubJson( + `/repos/langfuse/langfuse/git/ref/tags/${encodeURIComponent(tag)}`, + ); + let object = ref.object; + for (let depth = 0; object?.type === "tag" && depth < 5; depth++) { + object = (await githubJson(`/repos/langfuse/langfuse/git/tags/${object.sha}`)) + .object; + } + if (object?.type !== "commit" || !/^[0-9a-f]{40}$/.test(object.sha)) { + throw new Error(`Could not resolve ${tag} to an immutable commit`); + } + return object.sha; +} + +async function downloadSpec(catalog: Catalog, commit: string): Promise { + const repository = catalog.repository.replace("https://github.com/", ""); + const url = `https://raw.githubusercontent.com/${repository}/${commit}/${catalog.specPath}`; + const response = await fetch(url, { headers: { "user-agent": USER_AGENT } }); + if (!response.ok) { + throw new Error(`Spec download failed: ${response.status} ${response.statusText}`); + } + return response.text(); +} + +function replaceConstWithEnum(value: any, seen = new WeakSet()): boolean { + if (!value || typeof value !== "object" || seen.has(value)) return false; + seen.add(value); + let changed = false; + if (!Array.isArray(value) && Object.hasOwn(value, "const")) { + value.enum = [value.const]; + delete value.const; + changed = true; + } + for (const child of Object.values(value)) { + changed = replaceConstWithEnum(child, seen) || changed; + } + return changed; +} + +async function knownIssues(raw: string): Promise { + const document = parse(raw, { + maxAliasCount: 100_000, + uniqueKeys: true, + }) as Record; + try { + await SwaggerParser.validate(structuredClone(document) as any); + return undefined; + } catch (originalError) { + const patched = structuredClone(document); + if (!replaceConstWithEnum(patched)) throw originalError; + await SwaggerParser.validate(patched as any); + return ["oas3.0-const-keyword"]; + } +} + +async function specSummaries( + catalog: Catalog, + newEntry: CatalogEntry, + newRaw: string, +): Promise { + return Promise.all( + catalog.versions.map(async (entry) => { + const raw = entry.version === newEntry.version + ? newRaw + : await readVerifiedSpec(entry); + const compiled = compileOpenApi(entry, raw); + return { + version: entry.version, + paths: Object.keys(compiled.document.paths ?? {}).length, + operations: compiled.manifest.operations.length, + }; + }), + ); +} + +async function run(command: string[], label: string): Promise { + process.stdout.write(`\n${label}\n`); + const child = Bun.spawn(command, { + cwd: REPOSITORY_ROOT, + stdout: "inherit", + stderr: "inherit", + }); + const code = await child.exited; + if (code !== 0) throw new Error(`${label} failed with exit code ${code}`); +} + +async function addVersion( + input: string, + options: AddVersionOptions, +): Promise { + const { tag, version } = normalizeReleaseTag(input); + const originalCatalogText = await Bun.file(CATALOG_PATH).text(); + const originalReadme = await Bun.file(README_PATH).text(); + const catalog = JSON.parse(originalCatalogText) as Catalog; + if (catalog.versions.some((entry) => entry.version === version)) { + throw new Error(`${tag} is already bundled`); + } + + process.stdout.write(`Resolving ${tag}\n`); + const commit = await resolveStableRelease(tag); + const raw = await downloadSpec(catalog, commit); + const issues = await knownIssues(raw); + const entry: CatalogEntry = { + version, + ref: tag, + commit, + sha256: await sha256(raw), + ...(issues ? { knownIssues: issues } : {}), + }; + const compiled = compileOpenApi(entry, raw); + const contract = compileApiContract(entry, raw); + if (compiled.unsupported.length > 0) { + throw new Error(`Unsupported OpenAPI features: ${compiled.unsupported.join(", ")}`); + } + if (compiled.manifest.operations.length !== contract.operations.length) { + throw new Error("Conformance and runtime compilers disagree on operation count"); + } + const updatedCatalog = withCatalogEntry(catalog, entry); + const summaries = await specSummaries(updatedCatalog, entry, raw); + const updatedReadme = updateConformanceReadme(originalReadme, summaries); + process.stdout.write( + `${tag} -> ${commit}\nSHA-256 ${entry.sha256}\n${compiled.manifest.operations.length} operations\n`, + ); + if (options.dryRun) { + process.stdout.write("Dry run complete; no files changed\n"); + return; + } + + const path = resolve(CONFORMANCE_ROOT, "specs", version, "openapi.yml"); + if (await Bun.file(path).exists()) { + throw new Error(`${path} already exists but ${tag} is not cataloged`); + } + await mkdir(dirname(path), { recursive: true }); + try { + await Bun.write(path, raw); + await Bun.write(CATALOG_PATH, formatCatalog(updatedCatalog)); + await Bun.write(README_PATH, updatedReadme); + if (options.runChecks) { + await run(["bun", "run", "typecheck"], "Typecheck"); + await run(["bun", "test"], "Test suite"); + await run(["bun", "run", "build"], "Build contracts"); + await run( + [ + "bun", + "run", + "conformance:run", + "--", + "--version", + version, + "--adapter", + "contract-v1", + "--", + "bun", + "bin/langfuse.mjs", + "--api-version", + version, + ], + `Conformance ${version}`, + ); + } + } catch (error) { + await Bun.write(CATALOG_PATH, originalCatalogText); + await Bun.write(README_PATH, originalReadme); + await rm(resolve(CONFORMANCE_ROOT, "specs", version), { + recursive: true, + force: true, + }); + throw error; + } + process.stdout.write(`\nAdded ${tag}. Review and live-test changed endpoints before commit.\n`); +} + +async function main(): Promise { + const args = process.argv.slice(2); + const input = args.find((arg) => !arg.startsWith("--")); + if (!input || args.some((arg) => ![input, "--dry-run"].includes(arg))) usage(); + await addVersion(input, { + dryRun: args.includes("--dry-run"), + runChecks: true, + }); +} + +if (import.meta.main) { + main().catch((error) => { + process.stderr.write(`${error instanceof Error ? error.stack : String(error)}\n`); + process.exitCode = 1; + }); +} diff --git a/conformance/src/all.ts b/conformance/src/all.ts new file mode 100644 index 0000000..111335b --- /dev/null +++ b/conformance/src/all.ts @@ -0,0 +1,43 @@ +import { loadCatalog } from "./catalog"; +import { generateCorpus } from "./generator"; +import { runConformance } from "./runner"; + +const build = Bun.spawn(["bun", "run", "build"], { + cwd: import.meta.dir + "/../..", + stdout: "inherit", + stderr: "inherit", +}); +if ((await build.exited) !== 0) process.exit(1); + +const catalog = await loadCatalog(); +let passed = 0; +let total = 0; +let failed = false; + +for (const entry of catalog.versions) { + const corpus = await generateCorpus(entry); + const results = await runConformance({ + entry, + manifest: corpus.compiled.manifest, + vectors: corpus.vectors, + adapter: "contract-v1", + command: ["bun", "bin/langfuse.mjs", "--api-version", entry.version], + quiet: true, + }); + const versionPassed = results.filter((result) => result.passed).length; + passed += versionPassed; + total += results.length; + process.stdout.write(`${entry.version}: ${versionPassed}/${results.length}\n`); + + for (const result of results.filter((candidate) => !candidate.passed)) { + failed = true; + process.stderr.write(`FAIL ${result.id}\n`); + for (const failure of result.failures) { + process.stderr.write(` ${failure}\n`); + } + if (result.process.stderr) process.stderr.write(result.process.stderr); + } +} + +process.stdout.write(`Total: ${passed}/${total}\n`); +if (failed) process.exit(1); diff --git a/conformance/src/catalog.ts b/conformance/src/catalog.ts index 40b5e7d..c1c36ef 100644 --- a/conformance/src/catalog.ts +++ b/conformance/src/catalog.ts @@ -60,12 +60,20 @@ export async function syncSpecs(entries?: CatalogEntry[]): Promise { } const text = await response.text(); const actual = await sha256(text); - if (actual !== entry.sha256) { + const expectedUpstream = entry.upstreamSha256 ?? entry.sha256; + if (actual !== expectedUpstream) { throw new Error( - `${entry.ref}: upstream bytes changed; expected ${entry.sha256}, got ${actual}`, + `${entry.ref}: upstream bytes changed; expected ${expectedUpstream}, got ${actual}`, ); } const path = specPath(entry); + if (entry.modifications?.length) { + await readVerifiedSpec(entry); + process.stdout.write( + `verified ${entry.ref} (${entry.modifications.join(", ")})\n`, + ); + continue; + } await mkdir(dirname(path), { recursive: true }); if (!(await Bun.file(path).exists()) || (await Bun.file(path).text()) !== text) { await Bun.write(path, text); diff --git a/conformance/src/cli.ts b/conformance/src/cli.ts index 52dfa8e..fa7d766 100644 --- a/conformance/src/cli.ts +++ b/conformance/src/cli.ts @@ -17,9 +17,9 @@ function option(args: string[], name: string): string | undefined { function usage(): never { process.stderr.write(`Usage: - bun run conformance:sync [--version 3.216.0] - bun run conformance:run --version 3.216.0 --adapter specli-v0 --current-cli [filters] - bun run conformance:run --version 3.216.0 --adapter contract-v1 [filters] -- + bun run conformance:sync [--version 4.10.0] + bun run conformance:run --version 4.10.0 --adapter specli-v0 --current-cli [filters] + bun run conformance:run --version 4.10.0 --adapter contract-v1 [filters] -- Run filters: --operation Restrict one operation diff --git a/conformance/src/generator.ts b/conformance/src/generator.ts index 6745fdf..551afea 100644 --- a/conformance/src/generator.ts +++ b/conformance/src/generator.ts @@ -1,3 +1,5 @@ +import packageJson from "../../package.json"; + import { readVerifiedSpec } from "./catalog"; import { compileOpenApi, type CompiledSpec } from "./openapi"; import { expectedRequest } from "./serialize"; @@ -45,6 +47,7 @@ function minimalInput(operation: OperationContract): SemanticInput { function requestWithAuth(operation: OperationContract, input: SemanticInput) { const request = expectedRequest(operation, input); + request.headers["user-agent"] = `langfuse-cli/${packageJson.version}`; if (operation.auth.required && operation.auth.schemes.includes("BasicAuth")) { request.headers.authorization = "Basic Y29uZm9ybWFuY2UtcHVibGljLWtleTpjb25mb3JtYW5jZS1zZWNyZXQta2V5"; diff --git a/conformance/src/naming.ts b/conformance/src/naming.ts index 5bddf61..ebeb157 100644 --- a/conformance/src/naming.ts +++ b/conformance/src/naming.ts @@ -1,13 +1,28 @@ -import type { CommandName, HttpMethod } from "./types"; +import type { CommandAlias, CommandName, HttpMethod } from "./types"; interface NamingInput { operationId?: string; method: HttpMethod; path: string; tags: string[]; + deprecated?: true; } -interface PlannedNaming extends NamingInput, CommandName {} +interface RouteName { + resource: string; + version?: string; + tail: string[]; +} + +interface PlannedName { + input: NamingInput; + route: RouteName; + baseAction: string; + resource: string; + action: string; + aliases: CommandAlias[]; + index: number; +} const IRREGULAR: Record = { person: "people", @@ -27,6 +42,7 @@ const UNCOUNTABLE = new Set([ "series", "species", ]); +const API_VERSION = /^v\d+$/i; export function kebabCase(input: string): string { return input @@ -51,132 +67,200 @@ export function pluralize(input: string): string { return `${word}s`; } -function pathArgs(path: string): string[] { - return [...path.matchAll(/\{([^}]+)\}/g)].map((match) => match[1]); +function singularize(input: string): string { + if (input.endsWith("ies")) return `${input.slice(0, -3)}y`; + if (/(ches|shes|xes|zes)$/.test(input)) return input.slice(0, -2); + if (input.endsWith("s") && !input.endsWith("ss")) return input.slice(0, -1); + return input; } -function operationIdParts(operationId = ""): { - prefix?: string; - suffix?: string; -} { - const separator = operationId.includes(".") - ? "." - : operationId.includes("__") - ? "__" - : operationId.includes("_") - ? "_" - : undefined; - if (!separator) return operationId ? { suffix: operationId } : {}; - const [prefix, ...rest] = operationId.split(separator); - return { prefix, suffix: rest.join(separator) }; +function isParameter(segment: string | undefined): boolean { + return Boolean(segment?.startsWith("{") && segment.endsWith("}")); } -function canonicalAction(input: string): string { - const action = kebabCase(input); - if (["retrieve", "read"].includes(action)) return "get"; - if (["search"].includes(action)) return "list"; - if (action === "patch") return "update"; - if (action === "remove") return "delete"; - return action; -} +function routeName(path: string): RouteName { + const all = path.split("/").filter(Boolean); + const publicIndex = all.lastIndexOf("public"); + const segments = all.slice(publicIndex === -1 ? 0 : publicIndex + 1); + let version: string | undefined; + if (API_VERSION.test(segments[0] ?? "")) version = segments.shift()!.toLowerCase(); + while (isParameter(segments[0])) segments.shift(); + if (segments.length === 0) return { resource: "api", ...(version ? { version } : {}), tail: [] }; -function inferResource(input: NamingInput): string { - const tag = input.tags[0]?.trim(); - if (tag && !["default", "defaults", "api"].includes(tag.toLowerCase())) { - return pluralize(kebabCase(tag)); + let resource = kebabCase(segments.shift()!); + if (resource === "unstable" && segments[0] && !isParameter(segments[0])) { + resource = `${resource}-${kebabCase(segments.shift()!)}`; } - const prefix = operationIdParts(input.operationId).prefix; - if (prefix) return pluralize(kebabCase(prefix)); - const segment = input.path.split("/").filter(Boolean)[0] ?? "api"; - return pluralize(kebabCase(segment.replace(/^\{.+\}$/, "") || "api")); -} - -function inferAction(input: NamingInput): string { - const suffix = operationIdParts(input.operationId).suffix; - if (suffix) { - const action = canonicalAction(suffix); - if (["get", "list", "create", "update", "delete"].includes(action)) { - return action; - } + if (resource === "otel" && API_VERSION.test(segments[0] ?? "")) { + version ??= segments.shift()!.toLowerCase(); } - const hasPathArg = pathArgs(input.path).length > 0; - if (input.method === "GET") return hasPathArg ? "get" : "list"; - if (input.method === "POST" && !hasPathArg) return "create"; - if (["PUT", "PATCH"].includes(input.method) && hasPathArg) return "update"; - if (input.method === "DELETE" && hasPathArg) return "delete"; - return kebabCase(input.method); -} - -function disambiguator(operation: PlannedNaming, index: number): string { - let name = kebabCase(operation.operationId ?? ""); - const synonyms: Record = { - get: ["get", "retrieve", "read", "list", "search"], - list: ["list", "search", "get"], - create: ["create", "post"], - update: ["update", "patch", "put"], - delete: ["delete", "remove"], - }; - for (const synonym of synonyms[operation.action] ?? [operation.action]) { - if (name.startsWith(`${synonym}-`)) { - name = name.slice(synonym.length + 1); - break; + return { resource, ...(version ? { version } : {}), tail: segments }; +} + +function operationSuffix(operationId = ""): string { + const splitAt = Math.max( + operationId.lastIndexOf("__"), + operationId.lastIndexOf("_"), + operationId.lastIndexOf("."), + ); + const suffix = splitAt === -1 ? operationId : operationId.slice(splitAt + 1); + return kebabCase(suffix).replace(/-v\d+$/i, ""); +} + +function restAction(input: NamingInput, route: RouteName, suffix: string): string { + const tail = route.tail.filter((segment) => !API_VERSION.test(segment)); + const staticTail = tail.filter((segment) => !isParameter(segment)); + const subject = kebabCase(staticTail.at(-1) ?? ""); + const singularSubject = singularize(subject); + const item = isParameter(tail.at(-1)); + + if (staticTail.length === 0) { + if (input.method === "GET") { + if (item) return "get"; + if (["health", "metrics"].includes(suffix)) return "get"; + return "list"; } + if (input.method === "POST") return "create"; + if (["PUT", "PATCH"].includes(input.method)) return "update"; + if (input.method === "DELETE") return item ? "delete" : "delete-many"; + return kebabCase(input.method); } - const singular = operation.resource.replace(/s$/, ""); - for (const resource of [operation.resource, singular]) { - if (name.startsWith(`${resource}-`)) name = name.slice(resource.length + 1); - else if (name.includes(`-${resource}-`)) { - name = name.replace(`-${resource}-`, "-"); - } else if (name.endsWith(`-${resource}`)) { - name = name.slice(0, -(resource.length + 1)); - } + + if (input.method === "GET") { + if (item) return `get-${singularSubject}`; + const suffixSubject = suffix.split("-").at(-1) ?? ""; + const explicitSingleton = suffix.startsWith("get-") && !suffixSubject.endsWith("s"); + return `${explicitSingleton ? "get" : "list"}-${subject}`; } - if (name && name !== operation.action && name !== operation.resource) { - return `${operation.action}-${name}`; + if (input.method === "POST") { + return `${suffix.startsWith("add-") ? "add" : "create"}-${singularSubject}`; } - const segments = operation.path.split("/").filter(Boolean).reverse(); - for (const segment of segments) { - if (segment.startsWith("{")) continue; - const candidate = kebabCase(segment); - if (![operation.resource, singular].includes(candidate)) { - return `${operation.action}-${candidate}`; - } + if (["PUT", "PATCH"].includes(input.method)) { + return `${suffix.startsWith("upsert-") ? "upsert" : "update"}-${singularSubject}`; + } + if (input.method === "DELETE") return `delete-${singularSubject}`; + return `${kebabCase(input.method)}-${singularSubject}`; +} + +function inferAction(input: NamingInput, route: RouteName): string { + const suffix = operationSuffix(input.operationId); + if (["batch", "submit", "upsert"].includes(suffix)) return suffix; + if (suffix === "delete-multiple") return "delete-many"; + if (suffix.startsWith("add-") || suffix.startsWith("export-")) return suffix; + if (suffix.startsWith("get-") && input.method !== "GET") return suffix; + if (suffix.endsWith("-status")) return suffix; + return restAction(input, route, suffix); +} + +function versionRank(version?: string): number { + return version ? Number(version.slice(1)) : 0; +} + +function commandKey(resource: string, action: string): string { + return `${resource}\u0000${action}`; +} + +function tagResources(input: NamingInput): string[] { + return input.tags + .map((tag) => kebabCase(tag)) + .filter((tag) => tag && !["default", "defaults", "api"].includes(tag)); +} + +function uniqueFallbackAction(plan: PlannedName, used: Set): string { + const suffix = operationSuffix(plan.input.operationId) || kebabCase(plan.input.method); + let candidate = suffix === plan.action ? `${plan.action}-${plan.index + 1}` : suffix; + let index = 2; + while (used.has(commandKey(plan.resource, candidate))) { + candidate = `${suffix}-${index++}`; } - return `${operation.action}-${index}`; + return candidate; } export function planCommandNames(inputs: NamingInput[]): CommandName[] { - const planned: PlannedNaming[] = inputs.map((input) => { - const action = inferAction(input); + const planned: PlannedName[] = inputs.map((input, index) => { + const route = routeName(input.path); + const baseAction = inferAction(input, route); + const resource = input.deprecated + ? route.version + ? `${route.resource}-${route.version}` + : `legacy-${route.resource}` + : route.resource; return { - ...input, - resource: inferResource(input), - action, - canonicalAction: action, + input, + route, + baseAction, + resource, + action: baseAction, + aliases: [], + index, }; }); - const totals = new Map(); - for (const operation of planned) { - const key = `${operation.resource}:${operation.action}`; - totals.set(key, (totals.get(key) ?? 0) + 1); + + const activeGroups = new Map(); + for (const plan of planned.filter((candidate) => !candidate.input.deprecated)) { + const key = commandKey(plan.route.resource, plan.baseAction); + const group = activeGroups.get(key) ?? []; + group.push(plan); + activeGroups.set(key, group); } - const seen = new Map(); - return planned.map((operation) => { - const key = `${operation.resource}:${operation.action}`; - if ((totals.get(key) ?? 0) === 1) { - return { - resource: operation.resource, - action: operation.action, - canonicalAction: operation.canonicalAction, - }; + for (const group of activeGroups.values()) { + group.sort((left, right) => { + const versionDifference = versionRank(right.route.version) - versionRank(left.route.version); + return versionDifference || left.index - right.index; + }); + for (const loser of group.slice(1)) { + if (loser.route.version) loser.resource = `${loser.route.resource}-${loser.route.version}`; } - const index = (seen.get(key) ?? 0) + 1; - seen.set(key, index); - return { - resource: operation.resource, - action: disambiguator(operation, index), - canonicalAction: operation.canonicalAction, - aliasOf: `${operation.resource} ${operation.canonicalAction}`, - }; - }); + } + + const usedCanonical = new Set(); + for (const plan of planned) { + let key = commandKey(plan.resource, plan.action); + if (usedCanonical.has(key)) { + plan.action = uniqueFallbackAction(plan, usedCanonical); + key = commandKey(plan.resource, plan.action); + } + usedCanonical.add(key); + } + + const claimedAliases = new Set(); + for (const plan of planned) { + const candidates: CommandAlias[] = []; + if (plan.resource !== plan.route.resource) { + candidates.push({ + resource: plan.route.resource, + action: plan.baseAction, + source: "path", + }); + } + if (plan.route.version) { + candidates.push({ + resource: `${plan.route.resource}-${plan.route.version}`, + action: plan.baseAction, + source: "version", + }); + } + for (const resource of tagResources(plan.input)) { + candidates.push({ resource, action: plan.baseAction, source: "tag" }); + } + + for (const alias of candidates) { + const key = commandKey(alias.resource, alias.action); + if ( + key === commandKey(plan.resource, plan.action) || + usedCanonical.has(key) || + claimedAliases.has(key) + ) { + continue; + } + claimedAliases.add(key); + plan.aliases.push(alias); + } + } + + return planned.map((plan) => ({ + resource: plan.resource, + action: plan.action, + ...(plan.aliases.length ? { aliases: plan.aliases } : {}), + })); } diff --git a/conformance/src/openapi.ts b/conformance/src/openapi.ts index fa1cfcc..c4fa90b 100644 --- a/conformance/src/openapi.ts +++ b/conformance/src/openapi.ts @@ -29,6 +29,7 @@ interface RawOperation { operationId: string; method: HttpMethod; path: string; + deprecated?: true; tags: string[]; auth: { required: boolean; @@ -213,6 +214,7 @@ export function compileOpenApi( operationId, method: method.toUpperCase() as HttpMethod, path, + ...(operation.deprecated === true ? { deprecated: true as const } : {}), tags: (operation.tags ?? []).map(String), auth: normalizeAuth(document, operation, unsupported), pathParameterOrder: pathParameterOrder(path), diff --git a/conformance/src/runner.ts b/conformance/src/runner.ts index 597ab9a..a22454e 100644 --- a/conformance/src/runner.ts +++ b/conformance/src/runner.ts @@ -1,4 +1,4 @@ -import { mkdir, mkdtemp, rm, symlink } from "node:fs/promises"; +import { mkdir, mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; @@ -9,6 +9,7 @@ import { } from "./adapters"; import { CaptureServer, requestDiff, sameJson } from "./capture"; import { POLICY_PATH, REPOSITORY_ROOT, readVerifiedSpec } from "./catalog"; +import { compileApiContract } from "../../src/contracts/compiler"; import type { CatalogEntry, ConformanceVector, @@ -95,11 +96,6 @@ async function currentCliCommand(entry: CatalogEntry): Promise<{ const bin = resolve(directory, "bin"); await mkdir(dist, { recursive: true }); await mkdir(bin, { recursive: true }); - await symlink( - resolve(REPOSITORY_ROOT, "node_modules"), - resolve(directory, "node_modules"), - "dir", - ); const build = Bun.spawn( [ "bun", @@ -108,7 +104,7 @@ async function currentCliCommand(entry: CatalogEntry): Promise<{ "--outfile", resolve(dist, "cli.js"), "--target", - "node", + "bun", "--format", "esm", ], @@ -125,7 +121,21 @@ async function currentCliCommand(entry: CatalogEntry): Promise<{ await rm(directory, { recursive: true, force: true }); throw new Error(`Current CLI build failed:\n${stdout}${stderr}`); } - await Bun.write(resolve(directory, "openapi.yml"), await readVerifiedSpec(entry)); + const raw = await readVerifiedSpec(entry); + const contracts = resolve(dist, "contracts"); + await mkdir(contracts, { recursive: true }); + await Bun.write( + resolve(contracts, `${entry.version}.json`), + `${JSON.stringify(compileApiContract(entry, raw))}\n`, + ); + await Bun.write( + resolve(contracts, "catalog.json"), + `${JSON.stringify({ + schemaVersion: 1, + latest: entry.version, + versions: [{ version: entry.version, sourceSha256: entry.sha256 }], + })}\n`, + ); await Bun.write( resolve(bin, "langfuse.mjs"), await Bun.file(resolve(REPOSITORY_ROOT, "bin/langfuse.mjs")).text(), @@ -168,16 +178,37 @@ export async function runConformance(options: RunOptions): Promise }); const execution = await spawn(command, args, options.timeoutMs ?? 10_000); const failures: string[] = []; - if (execution.exitCode !== 0) { - failures.push(`exit: expected zero, got ${execution.exitCode}`); - } const captured = capture.requests.slice(before); - if (captured.length !== 1) { - failures.push(`server: expected one request, got ${captured.length}`); + const operation = options.manifest.operations.find( + (candidate) => candidate.key === vector.operationKey, + ); + if (!operation) { + failures.push("manifest: operation not found"); + } else if (operation.deprecated) { + if (execution.exitCode !== 2) { + failures.push( + `exit: expected deprecated-operation exit 2, got ${execution.exitCode}`, + ); + } + if (captured.length !== 0) { + failures.push( + `server: expected no request for deprecated operation, got ${captured.length}`, + ); + } + if (!execution.stderr.includes("Cannot call deprecated API operation")) { + failures.push("stderr: expected a helpful deprecated-operation error"); + } } else { - failures.push(...requestDiff(vector.expectedRequest, captured[0])); + if (execution.exitCode !== 0) { + failures.push(`exit: expected zero, got ${execution.exitCode}`); + } + if (captured.length !== 1) { + failures.push(`server: expected one request, got ${captured.length}`); + } else { + failures.push(...requestDiff(vector.expectedRequest, captured[0])); + } } - if (execution.exitCode === 0) { + if (!operation?.deprecated && execution.exitCode === 0) { const output = parseJson(execution.stdout); if (output?.status !== vector.response.status) { failures.push( diff --git a/conformance/src/types.ts b/conformance/src/types.ts index 17cafac..1c6f4ae 100644 --- a/conformance/src/types.ts +++ b/conformance/src/types.ts @@ -19,6 +19,8 @@ export interface CatalogEntry { ref: string; commit: string; sha256: string; + upstreamSha256?: string; + modifications?: string[]; knownIssues?: string[]; } @@ -32,8 +34,13 @@ export interface Catalog { export interface CommandName { resource: string; action: string; - canonicalAction: string; - aliasOf?: string; + aliases?: CommandAlias[]; +} + +export interface CommandAlias { + resource: string; + action: string; + source: "path" | "tag" | "version"; } export interface ParameterContract { @@ -64,6 +71,7 @@ export interface OperationContract { operationId: string; method: HttpMethod; path: string; + deprecated?: true; auth: { required: boolean; schemes: string[]; diff --git a/conformance/tests/adapters.test.ts b/conformance/tests/adapters.test.ts index 5a942a1..0466d7d 100644 --- a/conformance/tests/adapters.test.ts +++ b/conformance/tests/adapters.test.ts @@ -17,7 +17,7 @@ const operation: OperationContract = { method: "POST", path: "/widgets/{id}", auth: { required: true, schemes: ["BasicAuth"] }, - command: { resource: "widgets", action: "create", canonicalAction: "create" }, + command: { resource: "widgets", action: "create" }, pathParameterOrder: ["id"], parameters: [ { diff --git a/conformance/tests/add-version.test.ts b/conformance/tests/add-version.test.ts new file mode 100644 index 0000000..600e398 --- /dev/null +++ b/conformance/tests/add-version.test.ts @@ -0,0 +1,96 @@ +import { describe, expect, test } from "bun:test"; + +import { + compareVersions, + formatCatalog, + normalizeReleaseTag, + updateConformanceReadme, + withCatalogEntry, +} from "../src/add-version"; +import type { Catalog, CatalogEntry } from "../src/types"; + +const entry = (version: string): CatalogEntry => ({ + version, + ref: `v${version}`, + commit: version.replaceAll(".", "").padEnd(40, "0"), + sha256: version.replaceAll(".", "").padEnd(64, "0"), +}); + +describe("add-version workflow", () => { + test("normalizes only stable semantic release tags", () => { + expect(normalizeReleaseTag("v4.10.0")).toEqual({ + tag: "v4.10.0", + version: "4.10.0", + }); + expect(normalizeReleaseTag("4.011.0")).toEqual({ + tag: "v4.11.0", + version: "4.11.0", + }); + expect(() => normalizeReleaseTag("v4.11.0-rc.1")).toThrow( + "Expected a stable release tag", + ); + }); + + test("inserts catalog entries in semantic order and rejects duplicates", () => { + const catalog: Catalog = { + schemaVersion: 1, + repository: "https://github.com/langfuse/langfuse", + specPath: "openapi.yml", + versions: [entry("3.216.0"), entry("4.10.0")], + }; + const updated = withCatalogEntry(catalog, entry("4.9.0")); + expect(updated.versions.map((item) => item.version)).toEqual([ + "3.216.0", + "4.9.0", + "4.10.0", + ]); + expect(compareVersions("4.10.0", "4.9.0")).toBeGreaterThan(0); + expect(() => withCatalogEntry(updated, entry("4.10.0"))).toThrow( + "already present", + ); + expect(formatCatalog({ + ...updated, + versions: [{ ...entry("4.10.0"), knownIssues: ["known-issue"] }], + })).toContain('"knownIssues": ["known-issue"]'); + }); + + test("regenerates both operation totals and the pinned-spec table", () => { + const input = `The suite currently attempts all 10 operations across 2 snapshots using X. +The native adapter checks all 10 operations through JSON. + +| Langfuse | Paths | Operations | +|---|---:|---:| +| 1.0.0 | 1 | 4 | +| 2.0.0 | 2 | 6 | + +After table. +`; + const updated = updateConformanceReadme(input, [ + { version: "1.0.0", paths: 1, operations: 4 }, + { version: "2.0.0", paths: 2, operations: 6 }, + { version: "3.0.0", paths: 3, operations: 8 }, + ]); + expect(updated).toContain("all 18 operations across 3 pinned snapshots"); + expect(updated).toContain("checks all 18 operations through"); + expect(updateConformanceReadme(input, [ + { version: "3.0.0", paths: 3, operations: 8 }, + ])).toContain("| 3.0.0 | 3 | 8 |"); + }); + + test("fails loudly when tracked README wording drifts", () => { + const input = `The suite currently attempts all 10 operations across 1 snapshots using X. +The native adapter runs every operation through JSON. + +| Langfuse | Paths | Operations | +|---|---:|---:| +| 1.0.0 | 1 | 10 | + +After table. +`; + expect(() => + updateConformanceReadme(input, [ + { version: "1.0.0", paths: 1, operations: 10 }, + ]), + ).toThrow("Could not find native adapter operation count"); + }); +}); diff --git a/conformance/tests/catalog.test.ts b/conformance/tests/catalog.test.ts index 2a14229..6c7b478 100644 --- a/conformance/tests/catalog.test.ts +++ b/conformance/tests/catalog.test.ts @@ -3,20 +3,14 @@ import SwaggerParser from "@apidevtools/swagger-parser"; import { loadCatalog, readVerifiedSpec, specPath } from "../src/catalog"; import { generateCorpus } from "../src/generator"; +import { compareVersions } from "../src/add-version"; describe("immutable OpenAPI catalog", () => { test("all pinned snapshots pass SHA-256 verification", async () => { const catalog = await loadCatalog(); - expect(catalog.versions.map((entry) => entry.version)).toEqual([ - "3.0.0", - "3.50.0", - "3.100.0", - "3.150.0", - "3.176.0", - "3.200.0", - "3.212.0", - "3.216.0", - ]); + const versions = catalog.versions.map((entry) => entry.version); + expect(new Set(versions).size).toBe(versions.length); + expect(versions).toEqual([...versions].sort(compareVersions)); for (const entry of catalog.versions) { const raw = await readVerifiedSpec(entry); expect(raw.startsWith("openapi: 3.0.")).toBe(true); @@ -34,7 +28,7 @@ describe("immutable OpenAPI catalog", () => { } }); - test("each operation produces one supported endpoint call", async () => { + test("each operation produces one supported invocation", async () => { const catalog = await loadCatalog(); for (const entry of catalog.versions) { const corpus = await generateCorpus(entry); @@ -43,6 +37,32 @@ describe("immutable OpenAPI catalog", () => { corpus.compiled.manifest.operations.length, ); expect(corpus.vectors.length).toBeGreaterThan(0); + if (entry.version === "4.10.0") { + expect( + corpus.compiled.manifest.operations + .filter((operation) => operation.deprecated) + .map((operation) => operation.operationId) + .sort(), + ).toEqual( + [ + "datasetRunItems_create", + "datasetRunItems_list", + "datasets_deleteRun", + "datasets_getRun", + "datasets_getRuns", + "ingestion_batch", + "legacy_metricsV1_metrics", + "legacy_observationsV1_get", + "legacy_observationsV1_getMany", + "scores_get-by-id", + "scores_get-many", + "sessions_get", + "sessions_list", + "trace_get", + "trace_list", + ].sort(), + ); + } } }); }); diff --git a/conformance/tests/multi-version.test.ts b/conformance/tests/multi-version.test.ts index 7f4d6c2..9257034 100644 --- a/conformance/tests/multi-version.test.ts +++ b/conformance/tests/multi-version.test.ts @@ -17,6 +17,7 @@ const CURRENT_CLI_UNSUPPORTED_OPERATIONS = new Set([ "prompts_create", "scim_createUser", "score_create", + "scores_create", "trace_deleteMultiple", "unstable_dashboardWidgets_create", "unstable_dashboards_addPlacement", @@ -26,7 +27,7 @@ const CURRENT_CLI_UNSUPPORTED_OPERATIONS = new Set([ ]); describe("multi-version black-box matrix", () => { - test("fake-calls every endpoint through the real CLI", async () => { + test("checks every endpoint through the real CLI", async () => { const catalog = await loadCatalog(); await Promise.all(catalog.versions.map(async (entry) => { const corpus = await generateCorpus(entry); @@ -42,9 +43,15 @@ describe("multi-version black-box matrix", () => { }); expect(results).toHaveLength(vectors.length); + const deprecatedOperationIds = new Set( + corpus.compiled.manifest.operations + .filter((operation) => operation.deprecated) + .map((operation) => operation.operationId), + ); const expectedFailures = vectors .filter((vector) => - CURRENT_CLI_UNSUPPORTED_OPERATIONS.has(vector.operationId ?? ""), + CURRENT_CLI_UNSUPPORTED_OPERATIONS.has(vector.operationId ?? "") && + !deprecatedOperationIds.has(vector.operationId ?? ""), ) .map((vector) => vector.id); const actualFailures = results diff --git a/conformance/tests/naming.test.ts b/conformance/tests/naming.test.ts index 9b84087..0ddb1ea 100644 --- a/conformance/tests/naming.test.ts +++ b/conformance/tests/naming.test.ts @@ -9,7 +9,7 @@ describe("stable CLI naming policy", () => { expect(pluralize("datasets")).toBe("datasets"); }); - test("uses tag resources and operationId actions", () => { + test("uses path resources and keeps tags as aliases", () => { expect( planCommandNames([ { @@ -29,32 +29,93 @@ describe("stable CLI naming policy", () => { { resource: "annotation-queues", action: "list", - canonicalAction: "list", }, { resource: "traces", action: "delete", - canonicalAction: "delete", + aliases: [{ resource: "trace", action: "delete", source: "tag" }], }, ]); }); - test("disambiguates collisions deterministically", () => { + test("uses REST semantics instead of stuttered dashboard actions", () => { const names = planCommandNames([ { - operationId: "comments_get", + operationId: "unstable_dashboards_create", + method: "POST", + path: "/api/public/unstable/dashboards", + tags: ["UnstableDashboards"], + }, + { + operationId: "unstable_dashboards_update", + method: "PATCH", + path: "/api/public/unstable/dashboards/{dashboardId}", + tags: ["UnstableDashboards"], + }, + { + operationId: "unstable_dashboards_delete", + method: "DELETE", + path: "/api/public/unstable/dashboards/{dashboardId}", + tags: ["UnstableDashboards"], + }, + { + operationId: "unstable_dashboards_addPlacement", + method: "POST", + path: "/api/public/unstable/dashboards/{dashboardId}/placements", + tags: ["UnstableDashboards"], + }, + { + operationId: "unstable_dashboards_updatePlacement", + method: "PATCH", + path: "/api/public/unstable/dashboards/{dashboardId}/placements/{placementId}", + tags: ["UnstableDashboards"], + }, + { + operationId: "unstable_dashboards_deletePlacement", + method: "DELETE", + path: "/api/public/unstable/dashboards/{dashboardId}/placements/{placementId}", + tags: ["UnstableDashboards"], + }, + ]); + expect(names.map(({ resource, action }) => `${resource} ${action}`)).toEqual([ + "unstable-dashboards create", + "unstable-dashboards update", + "unstable-dashboards delete", + "unstable-dashboards add-placement", + "unstable-dashboards update-placement", + "unstable-dashboards delete-placement", + ]); + }); + + test("prefers the latest active route and exposes path and tag aliases", () => { + const names = planCommandNames([ + { + operationId: "scores_get-many", method: "GET", - path: "/api/public/comments", - tags: ["Comments"], + path: "/api/public/v2/scores", + tags: ["Scores"], + deprecated: true, }, { - operationId: "comments_get-by-id", + operationId: "scoresV3_getManyV3", method: "GET", - path: "/api/public/comments/{commentId}", - tags: ["Comments"], + path: "/api/public/v3/scores", + tags: ["ScoresV3"], + }, + ]); + + expect(names).toEqual([ + { + resource: "scores-v2", + action: "list", + }, + { + resource: "scores", + action: "list", + aliases: [ + { resource: "scores-v3", action: "list", source: "version" }, + ], }, ]); - expect(new Set(names.map((name) => name.action)).size).toBe(2); - expect(names.every((name) => name.aliasOf === "comments get")).toBe(true); }); }); diff --git a/conformance/tests/runner.test.ts b/conformance/tests/runner.test.ts index 04fab44..0c227f0 100644 --- a/conformance/tests/runner.test.ts +++ b/conformance/tests/runner.test.ts @@ -3,6 +3,7 @@ import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; +import packageJson from "../../package.json"; import { generateVectors } from "../src/generator"; import { compileOpenApi } from "../src/openapi"; import { runConformance } from "../src/runner"; @@ -52,6 +53,7 @@ const api = args.indexOf("api"); const id = args[api + 3]; const response = await fetch( \`\${value("--host")}/widgets/\${encodeURIComponent(id)}?limit=\${value("--limit")}\`, + { headers: { "user-agent": ${JSON.stringify(`langfuse-cli/${packageJson.version}`)} } }, ); console.log(JSON.stringify({ status: response.status, body: await response.json() })); process.exit(response.ok ? 0 : 1); @@ -76,4 +78,37 @@ process.exit(response.ok ? 0 : 1); expect(results[0].failures).toEqual([]); expect(results[0].passed).toBe(true); }); + + test("expects deprecated operations to fail before making a request", async () => { + directory = await mkdtemp(join(tmpdir(), "langfuse-cli-deprecated-")); + const script = resolve(directory, "fake-cli.ts"); + await Bun.write( + script, + `console.error("Cannot call deprecated API operation"); +process.exit(2); +`, + ); + const entry = { + version: "fixture", + ref: "fixture", + commit: "0".repeat(40), + sha256: "0".repeat(64), + }; + const compiled = compileOpenApi( + entry, + raw.replace("tags: [Widgets]", "tags: [Widgets]\n deprecated: true"), + ); + const vector = generateVectors(compiled)[0]; + const results = await runConformance({ + entry, + manifest: compiled.manifest, + vectors: [vector], + adapter: "contract-v1", + command: ["bun", script], + }); + + expect(results).toHaveLength(1); + expect(results[0].failures).toEqual([]); + expect(results[0].passed).toBe(true); + }); }); diff --git a/conformance/tests/schema-validation.test.ts b/conformance/tests/schema-validation.test.ts index 7d89bc6..63a8d9e 100644 --- a/conformance/tests/schema-validation.test.ts +++ b/conformance/tests/schema-validation.test.ts @@ -58,7 +58,7 @@ function parameterValue( } describe("generated samples against original OpenAPI schemas", () => { - test("every generated endpoint call is valid against its untouched spec", async () => { + test("every generated endpoint call is valid against its committed spec", async () => { const catalog = await loadCatalog(); for (const entry of catalog.versions) { const { compiled, vectors } = await generateCorpus(entry); diff --git a/conformance/tests/serialize.test.ts b/conformance/tests/serialize.test.ts index 6b2cae7..bd26c04 100644 --- a/conformance/tests/serialize.test.ts +++ b/conformance/tests/serialize.test.ts @@ -9,7 +9,7 @@ const operation: OperationContract = { method: "GET", path: "/items/{itemId}", auth: { required: false, schemes: [] }, - command: { resource: "items", action: "get", canonicalAction: "get" }, + command: { resource: "items", action: "get" }, pathParameterOrder: ["itemId"], parameters: [ { diff --git a/package.json b/package.json index fba01be..cea6147 100644 --- a/package.json +++ b/package.json @@ -23,30 +23,29 @@ "files": [ "bin", "dist", - "openapi.yml", "README.md" ], "scripts": { "test": "bun test", + "typecheck": "tsc --noEmit", "conformance:sync": "bun conformance/src/cli.ts sync", + "conformance:add-version": "bun conformance/src/add-version.ts", "conformance:run": "bun conformance/src/cli.ts run", - "patch-openapi": "bun scripts/patch-openapi.ts", - "refetch-openapi": "bun scripts/patch-openapi.ts --refetch", - "build": "bun run refetch-openapi && bun build src/cli.ts --outdir dist --target node --format esm", + "conformance:all": "bun conformance/src/all.ts", + "build": "bun scripts/build.ts", "release": "bun scripts/release.ts", - "prepublishOnly": "rm -rf dist && bun run build" - }, - "dependencies": { - "specli": "^0.0.39" + "prepublishOnly": "bun run build" }, + "dependencies": {}, "devDependencies": { "@apidevtools/swagger-parser": "^12.1.0", "@types/bun": "^1.3.14", "ajv": "^8.17.1", "ajv-formats": "^3.0.1", + "typescript": "^7.0.2", "yaml": "^2.8.2" }, "engines": { - "node": ">=20" + "bun": ">=1.3.0" } } diff --git a/scripts/build.ts b/scripts/build.ts new file mode 100644 index 0000000..604a715 --- /dev/null +++ b/scripts/build.ts @@ -0,0 +1,54 @@ +import { mkdir, rm } from "node:fs/promises"; +import { resolve } from "node:path"; + +import { loadCatalog, readVerifiedSpec } from "../conformance/src/catalog"; +import { compileApiContract } from "../src/contracts/compiler"; +import type { ApiContractCatalog } from "../src/contracts/types"; + +const root = resolve(import.meta.dirname, ".."); +const dist = resolve(root, "dist"); +const contractsDirectory = resolve(dist, "contracts"); + +await rm(dist, { recursive: true, force: true }); +await mkdir(contractsDirectory, { recursive: true }); + +const sourceCatalog = await loadCatalog(); +const contractCatalog: ApiContractCatalog = { + schemaVersion: 1, + latest: sourceCatalog.versions.at(-1)!.version, + versions: sourceCatalog.versions.map((entry) => ({ + version: entry.version, + sourceSha256: entry.sha256, + })), +}; + +let totalOperations = 0; +for (const entry of sourceCatalog.versions) { + const raw = await readVerifiedSpec(entry); + const contract = compileApiContract(entry, raw); + totalOperations += contract.operations.length; + await Bun.write( + resolve(contractsDirectory, `${entry.version}.json`), + `${JSON.stringify(contract)}\n`, + ); +} +await Bun.write( + resolve(contractsDirectory, "catalog.json"), + `${JSON.stringify(contractCatalog)}\n`, +); + +const result = await Bun.build({ + entrypoints: [resolve(root, "src/cli.ts")], + outdir: dist, + target: "bun", + format: "esm", + minify: true, +}); +if (!result.success) { + for (const log of result.logs) process.stderr.write(`${log}\n`); + process.exit(1); +} + +process.stdout.write( + `Built native Bun CLI with ${sourceCatalog.versions.length} contracts and ${totalOperations} operations\n`, +); diff --git a/scripts/patch-openapi.ts b/scripts/patch-openapi.ts deleted file mode 100644 index ebe1774..0000000 --- a/scripts/patch-openapi.ts +++ /dev/null @@ -1,277 +0,0 @@ -/** - * Flattens discriminated unions (oneOf) in the OpenAPI spec into flat objects. - * - * specli generates CLI flags from request body schemas but can't handle oneOf/allOf yet. - * This script detects discriminated unions in components.schemas and merges their - * branches into a single flat object with unioned properties and intersected required. - * - * Uses parseDocument to preserve original YAML formatting of untouched nodes. - */ - -import { readFileSync, writeFileSync } from "fs"; -import { parseDocument, type Document } from "yaml"; -import { resolve } from "path"; -import { parseArgs } from "util"; - -const DEFAULT_OPENAPI_URL = "https://cloud.langfuse.com/generated/api/openapi.yml"; - -const { values: args } = parseArgs({ - args: process.argv.slice(2), - options: { - refetch: { type: "boolean", default: false }, - openapi_url: { type: "string", default: DEFAULT_OPENAPI_URL }, - }, -}); - -const specPath = resolve(import.meta.dirname!, "../openapi.yml"); - -if (args.refetch) { - const url = args.openapi_url!; - console.log(`Fetching spec from ${url}...`); - const res = await fetch(url); - if (!res.ok) { - console.error(`Failed to fetch: ${res.status} ${res.statusText}`); - process.exit(1); - } - writeFileSync(specPath, await res.text()); - console.log(`Wrote fresh spec to ${specPath}`); -} - -const raw = readFileSync(specPath, "utf-8"); -const doc: Document = parseDocument(raw); - -const schemas = doc.getIn(["components", "schemas"], true) as any; -if (!schemas || !schemas.items) { - console.log("No components.schemas found, nothing to patch."); - process.exit(0); -} - -// Convert to JS for analysis (easier to work with) -const schemasJS = schemas.toJSON() as Record; -let patchCount = 0; - -for (const [name, schema] of Object.entries(schemasJS)) { - if (!schema.oneOf || !Array.isArray(schema.oneOf)) continue; - - // Check if every branch matches the discriminated union pattern: - // { allOf: [{ properties: { : { enum: [val] } } }, { $ref }], required: [] } - const branches: Array<{ - discriminatorKey: string; - discriminatorValue: string; - refSchemaName: string; - }> = []; - - let isDiscriminatedUnion = true; - for (const branch of schema.oneOf) { - if (!branch.allOf || branch.allOf.length !== 2) { - isDiscriminatedUnion = false; - break; - } - - const [inline, ref] = branch.allOf; - const props = inline?.properties; - if (!props || !ref?.$ref) { - isDiscriminatedUnion = false; - break; - } - - // Find the discriminator: a property with a single-value enum - const discEntries = Object.entries(props).filter( - ([, v]) => v.type === "string" && Array.isArray(v.enum) && v.enum.length === 1, - ); - if (discEntries.length !== 1) { - isDiscriminatedUnion = false; - break; - } - - const [discKey, discSchema] = discEntries[0]; - const refName = ref.$ref.replace("#/components/schemas/", ""); - - branches.push({ - discriminatorKey: discKey, - discriminatorValue: discSchema.enum[0], - refSchemaName: refName, - }); - } - - if (!isDiscriminatedUnion || branches.length === 0) continue; - - // all branches should use the same discriminator key - const discKey = branches[0].discriminatorKey; - if (!branches.every((b) => b.discriminatorKey === discKey)) continue; - - const mergedProperties: Record = {}; - const requiredSets: Set[] = []; - - // property to discriminate - mergedProperties[discKey] = { - type: "string", - enum: branches.map((b) => b.discriminatorValue), - }; - - for (const branch of branches) { - const branchSchema = schemasJS[branch.refSchemaName]; - if (!branchSchema?.properties) continue; - - const branchRequired = new Set(branchSchema.required ?? []); - requiredSets.push(branchRequired); - - for (const [propName, propSchema] of Object.entries(branchSchema.properties)) { - if (propName === discKey) continue; // already handled - - if (!(propName in mergedProperties)) { - mergedProperties[propName] = structuredClone(propSchema); - } else { - // property exists in multiple branches — check for type conflict - const existing = mergedProperties[propName]; - if (JSON.stringify(existing) !== JSON.stringify(propSchema)) { - // conflict: fall back to string so specli still exposes the flag - mergedProperties[propName] = { - type: "string", - ...(existing.description ? { description: existing.description } : {}), - ...(existing.nullable ? { nullable: true } : {}), - }; - } - } - } - } - - // Required = intersection of all branches' required fields + discriminator - const intersectedRequired = - requiredSets.length > 0 - ? [...requiredSets[0]].filter((r) => requiredSets.every((s) => s.has(r))) - : []; - const required = [discKey, ...intersectedRequired.filter((r) => r !== discKey)]; - - // Strip nullable from properties that have no type (specli errors on these) - for (const [propName, propSchema] of Object.entries(mergedProperties)) { - if (propSchema.nullable && !propSchema.type) { - delete propSchema.nullable; - } - } - - const patched: any = { - title: schema.title ?? name, - type: "object", - properties: mergedProperties, - }; - if (required.length > 0) { - patched.required = required; - } - - // Replace the schema node in the document (preserves rest of doc formatting) - doc.setIn(["components", "schemas", name], doc.createNode(patched)); - patchCount++; - console.log( - `Patched ${name}: merged ${branches.length} branches, ${required.length} required fields`, - ); -} - -// Remove paths that shouldn't be exposed to CLI users -// const hiddenPaths = ["/api/public/traces", "/api/public/traces/{traceId}"]; -const hiddenPaths: string[] = []; - -const paths = doc.getIn(["paths"], true) as any; -if (paths?.items) { - paths.items = paths.items.filter((pair: any) => { - const pathStr = pair.key?.value; - if (hiddenPaths.includes(pathStr)) { - console.log(`Removed path: ${pathStr}`); - return false; - } - return true; - }); -} - -// Patch operation descriptions with examples -const examples: Record = { - prompts_create: [ - "Create a new version for the prompt with the given `name`", - "", - "Example:", - " langfuse api prompts create --type text --name my-prompt --prompt 'Hello {{name}}'", - ].join("\n"), -}; - -if (paths?.items) { - for (const pathPair of paths.items) { - const methods = pathPair.value; - if (!methods?.items) continue; - for (const methodPair of methods.items) { - const op = methodPair.value; - if (!op?.items) continue; - for (const field of op.items) { - if (field.key?.value === "operationId" && examples[field.value?.value]) { - for (const descField of op.items) { - if (descField.key?.value === "description") { - descField.value = doc.createNode(examples[field.value.value]); - break; - } - } - } - } - } - } -} - -// Rename query parameters that collide with specli's global flags. -// specli (via commander.js) reserves "--version" for CLI version display, -// so any OpenAPI query parameter named "version" becomes unusable. -const paramRenames: Record> = { - prompts_get: { version: "prompt-version" }, -}; - -let renameCount = 0; -if (paths?.items) { - for (const pathPair of paths.items) { - const methods = pathPair.value; - if (!methods?.items) continue; - for (const methodPair of methods.items) { - const op = methodPair.value; - if (!op?.items) continue; - - let operationId = ""; - for (const field of op.items) { - if (field.key?.value === "operationId") { - operationId = field.value?.value ?? ""; - break; - } - } - - const renames = paramRenames[operationId]; - if (!renames) continue; - - for (const field of op.items) { - if (field.key?.value !== "parameters") continue; - const params = field.value; - if (!params?.items) continue; - - for (const param of params.items) { - if (!param?.items) continue; - let nameField: any = null; - let inValue = ""; - for (const pf of param.items) { - if (pf.key?.value === "name") nameField = pf; - if (pf.key?.value === "in") inValue = pf.value?.value ?? ""; - } - const oldName = nameField?.value?.value; - if (oldName && inValue === "query" && renames[oldName]) { - nameField.value = doc.createNode(renames[oldName]); - renameCount++; - console.log( - `Renamed ${operationId} query param '${oldName}' → '${renames[oldName]}'`, - ); - } - } - } - } - } -} - -const dirty = patchCount > 0 || renameCount > 0; -if (dirty) { - writeFileSync(specPath, doc.toString({ singleQuote: true })); - console.log(`\nWrote patched spec to ${specPath} (${patchCount} schema(s), ${renameCount} param rename(s))`); -} else { - console.log("No patches needed."); -} diff --git a/scripts/release.ts b/scripts/release.ts index d8fb059..a2d5699 100644 --- a/scripts/release.ts +++ b/scripts/release.ts @@ -15,10 +15,9 @@ const exactReleaseFiles = new Set([ "LICENSE", "README.md", "bun.lock", - "openapi.yml", "package.json", ]); -const releasePathPrefixes = ["bin/", "scripts/", "src/"]; +const releasePathPrefixes = ["bin/", "conformance/", "scripts/", "src/"]; const rawArgs = process.argv.slice(2); const isDryRun = rawArgs.includes("--dry-run"); const allowDirty = rawArgs.includes("--allow-dirty"); @@ -119,8 +118,14 @@ async function runCommand( stdout: options.capture ? "pipe" : "inherit", stderr: options.capture ? "pipe" : "inherit", }); - const stdoutPromise = options.capture ? proc.stdout.text() : Promise.resolve(""); - const stderrPromise = options.capture ? proc.stderr.text() : Promise.resolve(""); + const stdoutPromise = + proc.stdout instanceof ReadableStream + ? new Response(proc.stdout).text() + : Promise.resolve(""); + const stderrPromise = + proc.stderr instanceof ReadableStream + ? new Response(proc.stderr).text() + : Promise.resolve(""); exitCode = await proc.exited; stdout = await stdoutPromise; stderr = await stderrPromise; @@ -252,7 +257,7 @@ async function printPostBuildReview(): Promise { "LICENSE", "README.md", "bin", - "openapi.yml", + "conformance", "package.json", "scripts", "src", @@ -379,8 +384,9 @@ async function main(): Promise { await writePackageJson(pkg); console.log(`Updated package.json to ${pkg.name}@${nextVersion}`); + await runCommand("bun", ["run", "typecheck"]); await runCommand("bun", ["test"]); - await runCommand("bun", ["run", "prepublishOnly"]); + await runCommand("bun", ["run", "conformance:all"]); await runCommand("npm", ["pack", "--dry-run"]); await printPostBuildReview(); @@ -401,7 +407,7 @@ async function main(): Promise { return; } - // prepublishOnly already ran above, and npm pack --dry-run showed the package + // conformance:all already built above, and npm pack --dry-run showed the package // contents. Avoid a second lifecycle run producing a different publish. publishStarted = true; await runCommand("npm", ["publish", "--ignore-scripts"], { diff --git a/src/cli.test.ts b/src/cli.test.ts index 0adde4c..28661f0 100644 --- a/src/cli.test.ts +++ b/src/cli.test.ts @@ -1,6 +1,19 @@ import { describe, expect, test } from "bun:test"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; -import { run } from "./cli"; +import { + assertOperationCallable, + operationByCommand, + parseOperationInput, + run, + runApi, + schemaOutput, + writeResult, +} from "./cli"; +import { compileApiContract } from "./contracts/compiler"; +import type { ApiOperation } from "./contracts/types"; async function captureOutput( fn: () => Promise, @@ -66,3 +79,366 @@ describe("langfuse get-skill", () => { } }); }); + +describe("operation input parsing", () => { + const promptGet: ApiOperation = { + key: "GET /api/public/v2/prompts/{promptName}", + operationId: "prompts_get", + method: "GET", + path: "/api/public/v2/prompts/{promptName}", + auth: { required: false, schemes: [] }, + command: { + resource: "prompts", + action: "get", + }, + pathParameterOrder: ["promptName"], + parameters: [ + { + location: "path", + name: "promptName", + cliName: "prompt-name", + required: true, + style: "simple", + explode: false, + kind: "string", + }, + { + location: "query", + name: "resolve", + cliName: "resolve", + required: false, + style: "form", + explode: true, + kind: "boolean", + }, + ], + }; + + test("does not consume a positional after a bare boolean flag", async () => { + const input = await parseOperationInput(promptGet, [ + "--resolve", + "my-prompt-name", + ]); + + expect(input.path.promptName).toBe("my-prompt-name"); + expect(input.query.resolve).toBe(true); + }); + + test("does not consume a positional after a negated boolean flag", async () => { + const input = await parseOperationInput(promptGet, [ + "--no-resolve", + "my-prompt-name", + ]); + + expect(input.path.promptName).toBe("my-prompt-name"); + expect(input.query.resolve).toBe(false); + }); + + test("accepts an explicit inline boolean before a positional", async () => { + const input = await parseOperationInput(promptGet, [ + "--resolve=false", + "my-prompt-name", + ]); + + expect(input.path.promptName).toBe("my-prompt-name"); + expect(input.query.resolve).toBe(false); + }); + + test("preserves the item type of array request-body flags", async () => { + const contract = compileApiContract( + { version: "test", ref: "test", sha256: "test" }, + `openapi: 3.0.3 +info: + title: Test + version: test +paths: + /models: + put: + operationId: models_put + tags: [Models] + deprecated: true + security: [] + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + customModels: + type: array + items: + type: string + responses: + "200": + description: OK +`, + ); + const operation = contract.operations[0]; + const field = operation.requestBody?.fields.find( + (candidate) => candidate.name === "customModels", + ); + + expect(operation.deprecated).toBe(true); + expect(field?.itemKind).toBe("string"); + expect( + await parseOperationInput(operation, ["--customModels", "123"]), + ).toMatchObject({ body: { customModels: ["123"] } }); + }); + + test("rejects deprecated operations with replacement guidance", () => { + const operation: ApiOperation = { + ...promptGet, + deprecated: true, + description: "**Deprecated.** Use `GET /api/public/v3/prompts` instead.", + }; + + expect(() => assertOperationCallable(operation, "4.10.0")).toThrow( + 'Cannot call deprecated API operation "prompts get"', + ); + expect(() => assertOperationCallable(operation, "4.10.0")).toThrow( + "Use `GET /api/public/v3/prompts` instead.", + ); + + const schema = schemaOutput({ + schemaVersion: 1, + apiVersion: "4.10.0", + sourceSha256: "test", + operations: [operation], + }); + expect(schema.resources[0].actions[0].deprecated).toBe(true); + }); + + test("rejects nested body flags without consuming a positional", async () => { + const operation: ApiOperation = { + ...promptGet, + key: "PATCH /api/public/widgets/{widgetId}", + operationId: "widgets_update", + method: "PATCH", + path: "/api/public/widgets/{widgetId}", + command: { + resource: "widgets", + action: "update", + }, + pathParameterOrder: ["widgetId"], + parameters: [ + { + location: "path", + name: "widgetId", + cliName: "widget-id", + required: true, + style: "simple", + explode: false, + kind: "string", + }, + ], + requestBody: { + required: true, + contentType: "application/json", + legacyFieldFlags: true, + fields: [ + { + name: "chartConfig", + required: false, + kind: "object", + }, + ], + }, + }; + + await expect( + parseOperationInput(operation, [ + "--chartConfig.show_value_labels", + "widget-123", + ]), + ).rejects.toThrow( + "Nested body option --chartConfig.show_value_labels is unsupported; pass --chartConfig with a JSON object or use --body-json", + ); + + expect( + await parseOperationInput(operation, [ + "widget-123", + "--chartConfig", + '{"show_value_labels":true}', + ]), + ).toMatchObject({ + path: { widgetId: "widget-123" }, + body: { chartConfig: { show_value_labels: true } }, + }); + }); + + test("preserves an explicit null complete body", async () => { + const operation: ApiOperation = { + ...promptGet, + key: "POST /api/public/widgets", + operationId: "widgets_create", + method: "POST", + path: "/api/public/widgets", + command: { + resource: "widgets", + action: "create", + }, + pathParameterOrder: [], + parameters: [], + requestBody: { + required: true, + contentType: "application/json", + legacyFieldFlags: false, + fields: [], + }, + }; + + expect(await parseOperationInput(operation, ["--body-json", "null"])).toEqual({ + path: {}, + query: {}, + headers: {}, + cookies: {}, + body: null, + }); + }); + + test("uses last-wins for repeated scalar body flags and appends arrays", async () => { + const operation: ApiOperation = { + ...promptGet, + key: "POST /api/public/widgets", + operationId: "widgets_create", + method: "POST", + path: "/api/public/widgets", + command: { resource: "widgets", action: "create" }, + pathParameterOrder: [], + parameters: [], + requestBody: { + required: true, + contentType: "application/json", + legacyFieldFlags: true, + fields: [ + { name: "content", required: true, kind: "string" }, + { + name: "tags", + required: false, + kind: "array", + itemKind: "string", + }, + ], + }, + }; + + expect( + await parseOperationInput(operation, [ + "--content", + "first", + "--content", + "second", + "--tags", + "one", + "--tags", + "two", + ]), + ).toMatchObject({ + body: { content: "second", tags: ["one", "two"] }, + }); + }); + + test("resolves tag and version command aliases", () => { + const operation: ApiOperation = { + ...promptGet, + operationId: "scoresV3_getManyV3", + path: "/api/public/v3/scores", + command: { + resource: "scores", + action: "list", + aliases: [ + { resource: "scores-v3", action: "list", source: "tag" }, + ], + }, + }; + const contract = { + schemaVersion: 1 as const, + apiVersion: "4.10.0", + sourceSha256: "test", + operations: [operation], + }; + + expect(operationByCommand(contract, "scores-v3", "list")).toBe(operation); + }); +}); + +describe("API version reporting", () => { + const catalog = { + schemaVersion: 1 as const, + latest: "4.10.0", + versions: [ + { version: "3.150.0", sourceSha256: "old" }, + { version: "3.216.0", sourceSha256: "new" }, + { version: "4.10.0", sourceSha256: "latest" }, + ], + }; + const config = { + host: "http://localhost:3000", + timeoutMs: 1_000, + json: false, + curl: false, + showSecrets: false, + }; + + test("versions current prints the resolved major selection", async () => { + const output = await captureOutput(() => + runApi({ ...config, apiVersion: "3" }, ["versions", "current"], catalog), + ); + + expect(output.stdout).toBe("3.216.0\n"); + expect(output.stderr).toBe(""); + }); + + test("versions current resolves auto instead of echoing it", async () => { + const originalFetch = globalThis.fetch; + globalThis.fetch = (async (): Promise => + Response.json({ version: "3.216.1" })) as typeof fetch; + try { + const output = await captureOutput(() => + runApi( + { ...config, apiVersion: "auto" }, + ["versions", "current"], + catalog, + ), + ); + expect(output.stdout).toBe("3.216.0\n"); + } finally { + globalThis.fetch = originalFetch; + } + }); + + test("versions current rejects invalid selectors", async () => { + await expect( + runApi( + { ...config, apiVersion: "bogus" }, + ["versions", "current"], + catalog, + ), + ).rejects.toThrow("Unknown API version bogus"); + }); +}); + +describe("result output", () => { + test("writes an empty file for an empty response body", async () => { + const directory = await mkdtemp(join(tmpdir(), "langfuse-cli-test-")); + const output = join(directory, "response.json"); + try { + await writeResult( + { status: 204, headers: {}, body: null, ok: true }, + { + host: "http://localhost", + timeoutMs: 1_000, + json: false, + curl: false, + showSecrets: false, + output, + }, + ); + + expect(await Bun.file(output).text()).toBe(""); + } finally { + await rm(directory, { recursive: true, force: true }); + } + }); +}); diff --git a/src/cli.ts b/src/cli.ts index 2bad7f1..456fda4 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -1,91 +1,164 @@ -import { readFileSync } from "node:fs"; +import packageJson from "../package.json"; + +import { createApiClient, renderCurl } from "./client"; +import { + loadApiContract, + loadContractCatalog, + resolveContractVersion, +} from "./contracts/loader"; +import type { + ApiBodyField, + ApiCallInput, + ApiContract, + ApiContractCatalog, + ApiOperation, + ApiParameter, + ApiResult, + JsonValue, + ValueKind, +} from "./contracts/types"; const DEFAULT_HOST = "https://cloud.langfuse.com"; -const OPENAPI_FILE_URL = new URL("../openapi.yml", import.meta.url); +const DEFAULT_TIMEOUT_MS = 30_000; const LANGFUSE_SKILL_URL = "https://raw.githubusercontent.com/langfuse/skills/main/skills/langfuse/SKILL.md"; -const GET_SKILL_FETCH_TIMEOUT_MS = 5000; -const LANGFUSE_FLAGS = new Set([ +const GET_SKILL_FETCH_TIMEOUT_MS = 5_000; +const VALUE_FLAGS = new Set([ "--public-key", "--secret-key", "--host", "--env", + "--api-version", + "--timeout", + "--output", ]); -const LANGFUSE_BOOL_FLAGS = new Set(["--refetch-api-spec"]); +const BOOLEAN_FLAGS = new Set(["--json", "--curl", "--show-secrets"]); -function loadEnvFile(filePath: string): void { - const content = readFileSync(filePath, "utf-8"); - for (const line of content.split("\n")) { +interface ParsedGlobals { + values: Record; + booleans: Set; + args: string[]; +} + +interface RuntimeConfig { + publicKey?: string; + secretKey?: string; + host: string; + apiVersion?: string; + timeoutMs: number; + json: boolean; + curl: boolean; + showSecrets: boolean; + output?: string; +} + +class CliError extends Error { + constructor(message: string, readonly exitCode = 2) { + super(message); + } +} + +function flagKey(flag: string): string { + return flag.replace(/^--/, ""); +} + +function extractGlobals(args: string[]): ParsedGlobals { + const values: Record = {}; + const booleans = new Set(); + const remaining: string[] = []; + for (let index = 0; index < args.length; index++) { + const token = args[index]; + const equals = token.indexOf("="); + const name = equals === -1 ? token : token.slice(0, equals); + if (VALUE_FLAGS.has(name)) { + const value = equals === -1 ? args[index + 1] : token.slice(equals + 1); + if (value === undefined || (equals === -1 && value.startsWith("--"))) { + throw new CliError(`${name} requires a value`); + } + values[flagKey(name)] = value; + if (equals === -1) index++; + continue; + } + if (BOOLEAN_FLAGS.has(name)) { + booleans.add(flagKey(name)); + continue; + } + remaining.push(token); + } + return { values, booleans, args: remaining }; +} + +function parseEnv(content: string): Record { + const result: Record = {}; + for (const line of content.split(/\r?\n/)) { const trimmed = line.trim(); if (!trimmed || trimmed.startsWith("#")) continue; - const eqIdx = trimmed.indexOf("="); - if (eqIdx === -1) continue; - const key = trimmed.slice(0, eqIdx).trim(); - let val = trimmed.slice(eqIdx + 1).trim(); + const separator = trimmed.indexOf("="); + if (separator === -1) continue; + const key = trimmed.slice(0, separator).trim(); + let value = trimmed.slice(separator + 1).trim(); if ( - (val.startsWith('"') && val.endsWith('"')) || - (val.startsWith("'") && val.endsWith("'")) + (value.startsWith('"') && value.endsWith('"')) || + (value.startsWith("'") && value.endsWith("'")) ) { - val = val.slice(1, -1); + value = value.slice(1, -1); } - process.env[key] = val; + result[key] = value; } + return result; } -type MainFn = ( - argv: string[], - options?: { cliName?: string; auth?: string; embeddedSpecText?: string }, -) => Promise; - -async function loadMain(): Promise { - const specliEntry = import.meta.resolve("specli"); - const cliMainUrl = new URL("cli/main.js", specliEntry); - const mod = await import(cliMainUrl.href); - return mod.main; -} - -async function getSpecText(params: { - refetch: boolean; - host: string; -}): Promise { - if (params.refetch) { - const specUrl = `${params.host}/generated/api/openapi.yml`; - return fetchText(specUrl, "spec"); +async function runtimeConfig(globals: ParsedGlobals): Promise { + const fileEnv = globals.values.env + ? parseEnv(await Bun.file(globals.values.env).text()) + : {}; + const env = { ...process.env, ...fileEnv }; + const timeoutMs = Number(globals.values.timeout ?? DEFAULT_TIMEOUT_MS); + if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) { + throw new CliError("--timeout must be a positive number of milliseconds"); } - - // Use bundled spec - return readFileSync(OPENAPI_FILE_URL, "utf-8"); + return { + publicKey: globals.values["public-key"] ?? env.LANGFUSE_PUBLIC_KEY, + secretKey: globals.values["secret-key"] ?? env.LANGFUSE_SECRET_KEY, + host: ( + globals.values.host ?? + env.LANGFUSE_BASE_URL ?? + env.LANGFUSE_HOST ?? + DEFAULT_HOST + ).replace(/\/+$/, ""), + apiVersion: globals.values["api-version"] ?? env.LANGFUSE_API_VERSION, + timeoutMs, + json: globals.booleans.has("json"), + curl: globals.booleans.has("curl"), + showSecrets: globals.booleans.has("show-secrets"), + output: globals.values.output, + }; } async function fetchText( url: string, label: string, - options?: { timeoutMs?: number }, + timeoutMs?: number, ): Promise { - const resp = await fetch(url, { - signal: - typeof options?.timeoutMs === "number" - ? AbortSignal.timeout(options.timeoutMs) - : undefined, + const response = await fetch(url, { + signal: timeoutMs ? AbortSignal.timeout(timeoutMs) : undefined, }); - if (!resp.ok) { + if (!response.ok) { throw new Error( - `Failed to fetch ${label} from ${url}: ${resp.status} ${resp.statusText}`, + `Failed to fetch ${label} from ${url}: ${response.status} ${response.statusText}`, ); } - return resp.text(); + return response.text(); } -async function getSkillText(): Promise { - return fetchText(LANGFUSE_SKILL_URL, "skill", { - timeoutMs: GET_SKILL_FETCH_TIMEOUT_MS, - }); -} - -function printGetSkillFetchError(err: unknown): void { - const reason = err instanceof Error ? err.message : String(err); - - process.stderr.write(`Failed to fetch the latest Langfuse skill from GitHub. +async function getSkill(): Promise { + try { + process.stdout.write( + await fetchText(LANGFUSE_SKILL_URL, "skill", GET_SKILL_FETCH_TIMEOUT_MS), + ); + } catch (error) { + const reason = error instanceof Error ? error.message : String(error); + process.stderr.write(`Failed to fetch the latest Langfuse skill from GitHub. This environment may block direct GitHub access. Download the skill manually from: @@ -95,68 +168,12 @@ Then add the downloaded SKILL.md to your agent context manually. Original error: ${reason} `); -} - -export async function run(argv: string[]): Promise { - const extracted: Record = {}; - const boolFlags: Record = {}; - const passthrough: string[] = [argv[0], argv[1]]; - - let i = 2; - while (i < argv.length) { - if (LANGFUSE_FLAGS.has(argv[i]) && i + 1 < argv.length) { - const key = argv[i].replace(/^--/, ""); - extracted[key] = argv[i + 1]; - i += 2; - } else if (LANGFUSE_BOOL_FLAGS.has(argv[i])) { - const key = argv[i].replace(/^--/, ""); - boolFlags[key] = true; - i++; - } else { - passthrough.push(argv[i]); - i++; - } - } - - if (extracted["env"]) { - loadEnvFile(extracted["env"]); + process.exitCode = 1; } - - const publicKey = - extracted["public-key"] ?? process.env.LANGFUSE_PUBLIC_KEY; - const secretKey = - extracted["secret-key"] ?? process.env.LANGFUSE_SECRET_KEY; - const host = ( - extracted["host"] ?? - process.env.LANGFUSE_BASE_URL ?? - process.env.LANGFUSE_HOST ?? - DEFAULT_HOST - ).replace(/\/$/, ""); - - // First positional arg determines the subcommand - const subcommand = passthrough[2]; - - if (subcommand === "api") { - passthrough.splice(2, 1); - return runApi({ passthrough, boolFlags, publicKey, secretKey, host }); - } - - if (subcommand === "get-skill") { - try { - process.stdout.write(await getSkillText()); - } catch (err) { - printGetSkillFetchError(err); - process.exitCode = 1; - } - return; - } - - // Show help for anything else (no args, --help, -h, unknown command) - printHelp(); } function printHelp(): void { - console.log(`langfuse-cli — Interact with Langfuse from the command line + process.stdout.write(`langfuse-cli — Interact with Langfuse from the command line Usage: langfuse [options] @@ -167,100 +184,625 @@ Commands: Options: --public-key Langfuse public key (or LANGFUSE_PUBLIC_KEY) --secret-key Langfuse secret key (or LANGFUSE_SECRET_KEY) - --host Langfuse host (or LANGFUSE_HOST/LANGFUSE_BASE_URL, default: ${DEFAULT_HOST}) - --env Load env vars from file - --refetch-api-spec Fetch latest API spec instead of bundled + --host Langfuse host (default: ${DEFAULT_HOST}) + --env Load env vars from a file + --api-version Exact/major version, latest, or auto + --timeout Request timeout (default: ${DEFAULT_TIMEOUT_MS}) + -h, --help Show help + --version Show CLI version Examples: - langfuse api __schema List all available resources - langfuse api --help Show actions for a resource - langfuse api traces list --limit 10 List traces - langfuse api prompts list List prompts - langfuse api scores create --name quality \\ - --traceId --value 0.9 Create a score - langfuse api datasets create --name my-dataset Create a dataset`); + langfuse api help + langfuse api prompts list + langfuse api prompts create --body-json '{"name":"my-prompt","type":"text","prompt":"Hello"}' + langfuse api observations list --limit 20 +`); } -function printApiHelp(resources: string[]): void { - const sorted = [...resources].sort(); - console.log(`Usage: langfuse api [options] +interface CommandBinding { + operation: ApiOperation; + action: string; + alias: boolean; +} -Langfuse API Resources: -${sorted.map((r) => ` ${r}`).join("\n")} +function canonicalResourceMap(contract: ApiContract): Map { + const resources = new Map(); + for (const operation of contract.operations) { + const existing = resources.get(operation.command.resource) ?? []; + existing.push(operation); + resources.set(operation.command.resource, existing); + } + for (const operations of resources.values()) { + operations.sort((left, right) => + left.command.action.localeCompare(right.command.action), + ); + } + return resources; +} -Commands: - __schema Show API spec metadata - --help Show actions for a resource - --help Show options for an action +function resourceMap(contract: ApiContract): Map { + const resources = new Map(); + const add = (resource: string, binding: CommandBinding) => { + const existing = resources.get(resource) ?? []; + existing.push(binding); + resources.set(resource, existing); + }; + for (const operation of contract.operations) { + add(operation.command.resource, { + operation, + action: operation.command.action, + alias: false, + }); + for (const alias of operation.command.aliases ?? []) { + add(alias.resource, { operation, action: alias.action, alias: true }); + } + } + for (const bindings of resources.values()) { + bindings.sort((left, right) => left.action.localeCompare(right.action)); + } + return resources; +} + +function printApiHelp(contract: ApiContract): void { + const resources = [...canonicalResourceMap(contract)].sort(([left], [right]) => + left.localeCompare(right), + ); + process.stdout.write(`Usage: langfuse api [options] + +API snapshot: ${contract.apiVersion} + +Resources: +${resources + .map( + ([resource, operations]) => + ` ${resource}${operations.some((operation) => operation.deprecated) ? " [contains deprecated actions]" : ""}`, + ) + .join("\n")} + +Discovery: + api help [resource] [action] + api schema --json Machine-readable command schema + api __schema --json Legacy command alias + api versions list Bundled historical snapshots + Path commands are canonical; OpenAPI tag and route-version aliases also work + +Action options: + --body-json Lossless JSON request body + --body-file Read JSON body from file or stdin + --json Stable JSON response envelope + --curl Print curl without executing +`); +} +function printResourceHelp(contract: ApiContract, resource: string): void { + const bindings = resourceMap(contract).get(resource); + if (!bindings) throw new CliError(`Unknown API resource: ${resource}`); + process.stdout.write(`Usage: langfuse api ${resource} [options] + +Actions: +${bindings + .map( + (binding) => { + const label = `${binding.action}${binding.alias ? " [alias]" : ""}${binding.operation.deprecated ? " [deprecated]" : ""}`; + return ` ${label.padEnd(43)} ${binding.operation.method} ${binding.operation.path}`; + }, + ) + .join("\n")} +`); +} + +function explicitDeprecationNote(operation: ApiOperation): string | undefined { + const description = operation.description?.trim(); + if (!description || !/^(?:\*\*)?deprecated\b/i.test(description)) { + return undefined; + } + return description + .split(/\n\s*\n/, 1)[0] + .replace(/^\*\*Deprecated\.\*\*\s*/i, "") + .replace(/^Deprecated\.?\s*/i, "") + .replace(/\s*\n\s*/g, " ") + .trim(); +} + +export function assertOperationCallable( + operation: ApiOperation, + apiVersion: string, +): void { + if (!operation.deprecated) return; + const note = explicitDeprecationNote(operation); + throw new CliError( + `Cannot call deprecated API operation "${operation.command.resource} ${operation.command.action}" (${operation.method} ${operation.path}) in API ${apiVersion}.` + + (note ? ` ${note}` : " No replacement is declared in its OpenAPI description.") + + ` Use "langfuse api help ${operation.command.resource}" or "langfuse api schema --json" to find supported operations.`, + ); +} + +function kindLabel(kind: ValueKind): string { + return kind === "array" ? "value (repeatable)" : kind; +} + +function flagUsage(name: string, kind: ValueKind): string { + return kind === "boolean" + ? `--${name}[=true|false] / --no-${name}` + : `--${name} <${kindLabel(kind)}>`; +} + +function printOperationHelp(operation: ApiOperation): void { + const positionals = operation.pathParameterOrder + .map((name) => `<${name}>`) + .join(" "); + const lines: string[] = []; + for (const parameter of operation.parameters) { + if (parameter.location === "path") continue; + lines.push( + ` ${flagUsage(parameter.cliName, parameter.kind)}${parameter.required ? " (required)" : ""}`, + ); + } + if (operation.requestBody?.legacyFieldFlags) { + for (const field of operation.requestBody.fields) { + lines.push( + ` ${flagUsage(field.name, field.kind)}${field.required ? " (required)" : ""}`, + ); + } + } + if (operation.requestBody) { + lines.push(" --body-json Lossless JSON body"); + lines.push(" --body-file JSON body from file or stdin"); + } + process.stdout.write(`Usage: langfuse api ${operation.command.resource} ${operation.command.action}${positionals ? ` ${positionals}` : ""} [options] + +${operation.summary ?? operation.operationId} +${operation.description ? `\n${operation.description}\n` : ""} +${ + operation.deprecated + ? `\nDEPRECATED\nThis operation is discoverable but cannot be called by this CLI.${explicitDeprecationNote(operation) ? ` ${explicitDeprecationNote(operation)}` : ""}\n` + : "" +} Options: - --json Output as JSON - --curl Preview curl command without executing - -h, --help Show help +${lines.length ? lines.join("\n") : " (no operation-specific options)"} + --json JSON response envelope + --curl Print curl without executing +`); +} + +export function operationByCommand( + contract: ApiContract, + resource: string, + action: string, +): ApiOperation { + const operation = contract.operations.find( + (candidate) => { + if ( + candidate.command.resource === resource && + candidate.command.action === action + ) { + return true; + } + return candidate.command.aliases?.some( + (alias) => alias.resource === resource && alias.action === action, + ); + }, + ); + if (!operation) { + if (!resourceMap(contract).has(resource)) { + throw new CliError(`Unknown API resource: ${resource}`); + } + throw new CliError(`Unknown action ${resource} ${action}`); + } + return operation; +} + +function parseJsonValue(value: string, kind?: ValueKind): JsonValue { + if (kind === "string") return value; + if (kind === "boolean") { + if (value === "true") return true; + if (value === "false") return false; + throw new CliError(`Expected boolean, got ${value}`); + } + if (kind === "number") { + const number = Number(value); + if (!Number.isFinite(number)) throw new CliError(`Expected number, got ${value}`); + return number; + } + if (kind === "object" || kind === "array" || kind === "null") { + let parsed: JsonValue; + try { + parsed = JSON.parse(value) as JsonValue; + } catch { + throw new CliError(`Expected ${kind} as JSON, got ${value}`); + } + if ( + (kind === "object" && + (parsed === null || typeof parsed !== "object" || Array.isArray(parsed))) || + (kind === "array" && !Array.isArray(parsed)) || + (kind === "null" && parsed !== null) + ) { + throw new CliError(`Expected ${kind} as JSON, got ${value}`); + } + return parsed; + } + try { + return JSON.parse(value) as JsonValue; + } catch { + return value; + } +} -Workflow: - 1) langfuse api __schema - 2) langfuse api --help - 3) langfuse api --help - 4) langfuse api [options]`); -} - -async function getResources(specText: string): Promise { - // Run specli's __schema --json to get the canonical resource list - const main = await loadMain(); - const chunks: string[] = []; - const origWrite = process.stdout.write.bind(process.stdout); - process.stdout.write = (chunk: any) => { - chunks.push(String(chunk)); - return true; +function addParameterValue( + input: ApiCallInput, + parameter: ApiParameter, + raw: string | undefined, +): void { + const target = + parameter.location === "path" + ? input.path + : parameter.location === "query" + ? input.query + : parameter.location === "header" + ? input.headers + : input.cookies; + if (raw === undefined && parameter.kind !== "boolean") { + throw new CliError(`--${parameter.cliName} requires a value`); + } + const parsed = parseJsonValue(raw ?? "true", parameter.itemKind ?? parameter.kind); + if (parameter.kind === "array") { + const existing = target[parameter.name]; + if (Array.isArray(existing)) existing.push(parsed); + else target[parameter.name] = [parsed]; + } else { + target[parameter.name] = parsed; + } +} + +function setBodyValue( + body: Record, + raw: string | undefined, + field: ApiBodyField, +): void { + const kind = field.kind === "array" ? field.itemKind : field.kind; + const parsed = parseJsonValue(raw ?? "true", kind); + const existing = body[field.name]; + if (field.kind === "array") { + if (Array.isArray(parsed)) body[field.name] = parsed; + else if (Array.isArray(existing)) existing.push(parsed); + else body[field.name] = [parsed]; + } else { + body[field.name] = parsed; + } +} + +function splitOption(token: string): { name: string; inline?: string; negated: boolean } { + const separator = token.indexOf("="); + const rawName = separator === -1 ? token.slice(2) : token.slice(2, separator); + return { + name: rawName.startsWith("no-") ? rawName.slice(3) : rawName, + ...(separator === -1 ? {} : { inline: token.slice(separator + 1) }), + negated: rawName.startsWith("no-"), }; +} + +async function readBodyFile(path: string): Promise { + const text = path === "-" ? await Bun.stdin.text() : await Bun.file(path).text(); try { - await main(["node", "langfuse", "__schema", "--json"], { - cliName: "langfuse api", - auth: "BasicAuth", - embeddedSpecText: specText, - }); - } finally { - process.stdout.write = origWrite; + return JSON.parse(text) as JsonValue; + } catch (error) { + throw new CliError( + `Invalid JSON in ${path === "-" ? "stdin" : path}: ${error instanceof Error ? error.message : String(error)}`, + ); } - const output = JSON.parse(chunks.join("")); - return (output.data?.resources ?? []).map((r: any) => r.name); } -async function runApi(params: { - passthrough: string[]; - boolFlags: Record; - publicKey: string | undefined; - secretKey: string | undefined; - host: string; -}): Promise { - const { passthrough, boolFlags, publicKey, secretKey, host } = params; +export async function parseOperationInput( + operation: ApiOperation, + tokens: string[], +): Promise { + const input: ApiCallInput = { + path: {}, + query: {}, + headers: {}, + cookies: {}, + }; + const parameterByFlag = new Map(); + for (const parameter of operation.parameters) { + if (parameter.location !== "path") { + parameterByFlag.set(parameter.cliName, parameter); + } + } + if (operation.operationId === "prompts_get") { + const version = operation.parameters.find( + (parameter) => parameter.location === "query" && parameter.name === "version", + ); + if (version) parameterByFlag.set("prompt-version", version); + } + const positionals: string[] = []; + let fieldBody: Record | undefined; + let completeBody: JsonValue | undefined; + for (let index = 0; index < tokens.length; index++) { + const token = tokens[index]; + if (!token.startsWith("--")) { + positionals.push(token); + continue; + } + const option = splitOption(token); + const parameter = parameterByFlag.get(option.name); + const bodyField = operation.requestBody?.legacyFieldFlags + ? operation.requestBody.fields.find( + (candidate) => candidate.name === option.name.split(".")[0], + ) + : undefined; + if (bodyField && option.name.includes(".")) { + throw new CliError( + `Nested body option --${option.name} is unsupported; pass --${bodyField.name} with a JSON object or use --body-json`, + ); + } + const isBoolean = + parameter?.kind === "boolean" || bodyField?.kind === "boolean"; + let raw = option.inline; + if ( + raw === undefined && + !isBoolean && + tokens[index + 1] !== undefined && + !tokens[index + 1].startsWith("--") + ) { + raw = tokens[++index]; + } + if (option.name === "body-json") { + if (raw === undefined) throw new CliError("--body-json requires a value"); + try { + completeBody = JSON.parse(raw) as JsonValue; + } catch (error) { + throw new CliError( + `Invalid --body-json: ${error instanceof Error ? error.message : String(error)}`, + ); + } + continue; + } + if (option.name === "body-file") { + if (raw === undefined) throw new CliError("--body-file requires a path or -"); + completeBody = await readBodyFile(raw); + continue; + } + if (parameter) { + if (option.negated && parameter.kind !== "boolean") { + throw new CliError(`--no-${option.name} is only valid for boolean options`); + } + addParameterValue(input, parameter, option.negated ? "false" : raw); + continue; + } + if (!operation.requestBody) { + throw new CliError(`Unknown option --${option.name}`); + } + if (!operation.requestBody.legacyFieldFlags) { + throw new CliError( + `${operation.operationId} requires --body-json or --body-file for request bodies`, + ); + } + const field = bodyField; + if (!field) throw new CliError(`Unknown option --${option.name}`); + if (option.negated && field.kind !== "boolean") { + throw new CliError(`--no-${option.name} is only valid for boolean options`); + } + if (raw === undefined && field.kind !== "boolean") { + throw new CliError(`--${option.name} requires a value`); + } + fieldBody ??= {}; + setBodyValue(fieldBody, option.negated ? "false" : raw, field); + } + if (completeBody !== undefined && fieldBody !== undefined) { + throw new CliError("Do not mix --body-json/--body-file with body field flags"); + } + if (positionals.length !== operation.pathParameterOrder.length) { + throw new CliError( + `${operation.operationId} expects ${operation.pathParameterOrder.length} path argument(s), got ${positionals.length}`, + ); + } + for (let index = 0; index < operation.pathParameterOrder.length; index++) { + const name = operation.pathParameterOrder[index]; + const parameter = operation.parameters.find( + (candidate) => candidate.location === "path" && candidate.name === name, + ); + if (!parameter) throw new CliError(`Missing path parameter contract: ${name}`); + input.path[name] = parseJsonValue(positionals[index], parameter.kind); + } + for (const parameter of operation.parameters) { + const target = + parameter.location === "path" + ? input.path + : parameter.location === "query" + ? input.query + : parameter.location === "header" + ? input.headers + : input.cookies; + if (parameter.required && target[parameter.name] === undefined) { + throw new CliError(`Missing required option --${parameter.cliName}`); + } + } + let body = completeBody !== undefined ? completeBody : fieldBody; + if (completeBody === undefined && operation.requestBody?.legacyFieldFlags) { + const missing = operation.requestBody.fields + .filter((field) => field.required && fieldBody?.[field.name] === undefined) + .map((field) => `--${field.name}`); + if (missing.length > 0) { + throw new CliError(`Missing required body option(s): ${missing.join(", ")}`); + } + if (body === undefined && operation.requestBody.required) body = {}; + } + if (operation.requestBody?.required && body === undefined) { + throw new CliError(`${operation.operationId} requires a request body`); + } + if (body !== undefined) input.body = body; + return input; +} - const specText = await getSpecText({ - refetch: boolFlags["refetch-api-spec"] ?? false, - host, - }); +export function schemaOutput(contract: ApiContract) { + return { + schemaVersion: 1, + apiVersion: contract.apiVersion, + sourceSha256: contract.sourceSha256, + resources: [...canonicalResourceMap(contract)].map(([name, operations]) => ({ + name, + actions: operations.map((operation) => ({ + name: operation.command.action, + aliases: operation.command.aliases ?? [], + operationId: operation.operationId, + method: operation.method, + path: operation.path, + deprecated: Boolean(operation.deprecated), + auth: operation.auth, + pathParameterOrder: operation.pathParameterOrder, + parameters: operation.parameters, + ...(operation.requestBody + ? { requestBody: operation.requestBody } + : {}), + ...(operation.summary ? { summary: operation.summary } : {}), + ...(operation.description + ? { description: operation.description } + : {}), + })), + })), + }; +} + +export async function writeResult( + result: ApiResult, + config: RuntimeConfig, +): Promise { + if (config.output) { + const content = + result.body === null + ? "" + : typeof result.body === "string" + ? result.body + : JSON.stringify(result.body, null, 2); + await Bun.write(config.output, content ?? ""); + } else if (config.json) { + process.stdout.write( + `${JSON.stringify({ status: result.status, headers: result.headers, body: result.body })}\n`, + ); + } else if (typeof result.body === "string") { + process.stdout.write(result.body.endsWith("\n") ? result.body : `${result.body}\n`); + } else if (result.body !== null) { + process.stdout.write(`${JSON.stringify(result.body, null, 2)}\n`); + } + if (!result.ok) process.exitCode = 1; +} - // Intercept help: no args, --help, or -h - const args = passthrough.slice(2); +export async function runApi( + config: RuntimeConfig, + args: string[], + providedCatalog?: ApiContractCatalog, +): Promise { + const catalog = providedCatalog ?? (await loadContractCatalog()); + if (args[0] === "versions") { + const action = args[1] ?? "list"; + if (action === "list") { + process.stdout.write( + `${catalog.versions.map((entry) => entry.version).join("\n")}\n`, + ); + return; + } + if (action === "current") { + const resolved = await resolveContractVersion({ + requested: config.apiVersion, + host: config.host, + timeoutMs: config.timeoutMs, + catalog, + }); + process.stdout.write(`${resolved.version}\n`); + return; + } + if (action === "detect") { + const resolved = await resolveContractVersion({ + requested: "auto", + host: config.host, + timeoutMs: config.timeoutMs, + catalog, + }); + process.stdout.write( + `${resolved.detected} -> ${resolved.version}\n`, + ); + return; + } + throw new CliError(`Unknown versions action: ${action}`); + } + const resolved = await resolveContractVersion({ + requested: config.apiVersion, + host: config.host, + timeoutMs: config.timeoutMs, + catalog, + }); + const contract = await loadApiContract(resolved.version); if ( args.length === 0 || - (args.length === 1 && (args[0] === "--help" || args[0] === "-h")) + (args[0] === "help" && args.length === 1) || + args[0] === "--help" || + args[0] === "-h" ) { - printApiHelp(await getResources(specText)); + printApiHelp(contract); return; } - - const specliArgv = [...passthrough]; - const inject: string[] = ["--server", host]; - if (publicKey) inject.push("--username", publicKey); - if (secretKey) inject.push("--password", secretKey); - specliArgv.splice(2, 0, ...inject); - - const main = await loadMain(); - await main(specliArgv, { - cliName: "langfuse api", - auth: "BasicAuth", - embeddedSpecText: specText, + if (["schema", "__schema", "__spec"].includes(args[0])) { + const schema = schemaOutput(contract); + if (config.json) process.stdout.write(`${JSON.stringify(schema)}\n`); + else printApiHelp(contract); + return; + } + if (args[0] === "help") { + if (!args[1]) printApiHelp(contract); + else if (!args[2]) printResourceHelp(contract, args[1]); + else printOperationHelp(operationByCommand(contract, args[1], args[2])); + return; + } + const resource = args[0]; + if (!args[1] || args[1] === "help" || args[1] === "--help" || args[1] === "-h") { + printResourceHelp(contract, resource); + return; + } + const operation = operationByCommand(contract, resource, args[1]); + if (args[2] === "help" || args[2] === "--help" || args[2] === "-h") { + printOperationHelp(operation); + return; + } + assertOperationCallable(operation, contract.apiVersion); + const input = await parseOperationInput(operation, args.slice(2)); + const client = createApiClient({ + host: config.host, + publicKey: config.publicKey, + secretKey: config.secretKey, + timeoutMs: config.timeoutMs, }); + if (config.curl) { + process.stdout.write( + `${renderCurl(client.prepare(operation, input), { showSecrets: config.showSecrets })}\n`, + ); + return; + } + await writeResult(await client.call(operation, input), config); +} + +export async function run(argv: string[]): Promise { + try { + const globals = extractGlobals(argv.slice(2)); + const [command, ...args] = globals.args; + if (command === "--version") { + process.stdout.write(`${packageJson.version}\n`); + return; + } + if (!command || command === "--help" || command === "-h") { + printHelp(); + return; + } + if (command === "get-skill") { + await getSkill(); + return; + } + if (command !== "api") { + throw new CliError(`Unknown command: ${command}`); + } + await runApi(await runtimeConfig(globals), args); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + process.stderr.write(`${message}\n`); + process.exitCode = error instanceof CliError ? error.exitCode : 1; + } } diff --git a/src/client.test.ts b/src/client.test.ts new file mode 100644 index 0000000..10bea09 --- /dev/null +++ b/src/client.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, test } from "bun:test"; + +import packageJson from "../package.json"; +import { prepareRequest } from "./client"; +import type { ApiOperation } from "./contracts/types"; + +const operation: ApiOperation = { + key: "GET /api/public/health", + operationId: "health_get", + method: "GET", + path: "/api/public/health", + auth: { required: false, schemes: [] }, + command: { + resource: "health", + action: "get", + }, + pathParameterOrder: [], + parameters: [], +}; + +describe("API client", () => { + test("identifies requests with the CLI package version", () => { + const request = prepareRequest( + { host: "https://cloud.langfuse.com", timeoutMs: 1_000 }, + operation, + { path: {}, query: {}, headers: {}, cookies: {} }, + ); + + expect(request.headers.get("user-agent")).toBe( + `langfuse-cli/${packageJson.version}`, + ); + }); +}); diff --git a/src/client.ts b/src/client.ts new file mode 100644 index 0000000..3daa5e1 --- /dev/null +++ b/src/client.ts @@ -0,0 +1,196 @@ +import packageJson from "../package.json"; + +import type { + ApiCallInput, + ApiClientConfig, + ApiOperation, + ApiResult, + JsonValue, +} from "./contracts/types"; + +export interface PreparedRequest { + url: URL; + method: string; + headers: Headers; + body?: string; +} + +const USER_AGENT = `langfuse-cli/${packageJson.version}`; + +function primitive(value: JsonValue): string { + if (value === null) return ""; + if (typeof value === "object") return JSON.stringify(value); + return String(value); +} + +function pathValue(value: JsonValue, style: string, explode: boolean): string { + if (style !== "simple") throw new Error(`Unsupported path style: ${style}`); + if (Array.isArray(value)) return value.map(primitive).join(","); + if (value && typeof value === "object") { + const entries = Object.entries(value); + return explode + ? entries.map(([key, item]) => `${key}=${primitive(item)}`).join(",") + : entries.flatMap(([key, item]) => [key, primitive(item)]).join(","); + } + return primitive(value); +} + +function queryValues( + name: string, + value: JsonValue, + style: string, + explode: boolean, +): Array<[string, string]> { + if (style !== "form") throw new Error(`Unsupported query style: ${style}`); + if (Array.isArray(value)) { + return explode + ? value.map((item) => [name, primitive(item)]) + : [[name, value.map(primitive).join(",")]]; + } + if (value && typeof value === "object") { + const entries = Object.entries(value); + return explode + ? entries.map(([key, item]) => [key, primitive(item)]) + : [[name, entries.flatMap(([key, item]) => [key, primitive(item)]).join(",")]]; + } + return [[name, primitive(value)]]; +} + +function encodePathComponent(value: string): string { + return encodeURIComponent(value); +} + +export function prepareRequest( + config: ApiClientConfig, + operation: ApiOperation, + input: ApiCallInput, +): PreparedRequest { + let pathname = operation.path; + const headers = new Headers(); + const cookies: string[] = []; + const query: Array<[string, string]> = []; + for (const parameter of operation.parameters) { + const source = + parameter.location === "path" + ? input.path + : parameter.location === "query" + ? input.query + : parameter.location === "header" + ? input.headers + : input.cookies; + const value = source[parameter.name]; + if (value === undefined) continue; + if (parameter.location === "path") { + pathname = pathname.replace( + `{${parameter.name}}`, + encodePathComponent(pathValue(value, parameter.style, parameter.explode)), + ); + } else if (parameter.location === "query") { + query.push( + ...queryValues( + parameter.name, + value, + parameter.style, + parameter.explode, + ), + ); + } else if (parameter.location === "header") { + headers.set(parameter.name, primitive(value)); + } else { + cookies.push(`${parameter.name}=${primitive(value)}`); + } + } + if (cookies.length > 0) headers.set("cookie", cookies.join("; ")); + if (operation.auth.required && operation.auth.schemes.includes("BasicAuth")) { + if (!config.publicKey || !config.secretKey) { + throw new Error( + "This operation requires LANGFUSE_PUBLIC_KEY and LANGFUSE_SECRET_KEY", + ); + } + headers.set( + "authorization", + `Basic ${Buffer.from(`${config.publicKey}:${config.secretKey}`).toString("base64")}`, + ); + } + let body: string | undefined; + if (input.body !== undefined) { + if (!operation.requestBody) { + throw new Error(`${operation.operationId} does not accept a request body`); + } + headers.set("content-type", operation.requestBody.contentType); + body = JSON.stringify(input.body); + } + headers.set("accept", "application/json"); + headers.set("user-agent", USER_AGENT); + const host = config.host.endsWith("/") ? config.host : `${config.host}/`; + const url = new URL(pathname.replace(/^\//, ""), host); + for (const [name, value] of query) url.searchParams.append(name, value); + return { + url, + method: operation.method, + headers, + ...(body !== undefined ? { body } : {}), + }; +} + +async function responseBody(response: Response): Promise { + if (response.status === 204 || response.status === 205) return null; + const text = await response.text(); + if (!text) return null; + const contentType = response.headers.get("content-type") ?? ""; + if (contentType.includes("json") || contentType.includes("+json")) { + try { + return JSON.parse(text) as JsonValue; + } catch { + return text; + } + } + return text; +} + +export function createApiClient(config: ApiClientConfig) { + return { + prepare(operation: ApiOperation, input: ApiCallInput): PreparedRequest { + return prepareRequest(config, operation, input); + }, + + async call(operation: ApiOperation, input: ApiCallInput): Promise { + const prepared = prepareRequest(config, operation, input); + const response = await fetch(prepared.url, { + method: prepared.method, + headers: prepared.headers, + body: prepared.body, + signal: AbortSignal.timeout(config.timeoutMs), + }); + return { + status: response.status, + headers: Object.fromEntries(response.headers.entries()), + body: await responseBody(response), + ok: response.ok, + }; + }, + }; +} + +function shellQuote(value: string): string { + if (/^[A-Za-z0-9_./:@%+=,-]+$/.test(value)) return value; + return `'${value.replaceAll("'", `'"'"'`)}'`; +} + +export function renderCurl( + prepared: PreparedRequest, + options: { showSecrets: boolean }, +): string { + const parts = ["curl", "--request", prepared.method, shellQuote(prepared.url.href)]; + for (const [name, value] of prepared.headers.entries()) { + const rendered = + name.toLowerCase() === "authorization" && !options.showSecrets + ? "Basic " + : value; + parts.push("--header", shellQuote(`${name}: ${rendered}`)); + } + if (prepared.body !== undefined) { + parts.push("--data", shellQuote(prepared.body)); + } + return parts.join(" "); +} diff --git a/src/contracts/compiler.ts b/src/contracts/compiler.ts new file mode 100644 index 0000000..e301738 --- /dev/null +++ b/src/contracts/compiler.ts @@ -0,0 +1,337 @@ +import { parse } from "yaml"; + +import { kebabCase, planCommandNames } from "../../conformance/src/naming"; +import type { + ApiBodyField, + ApiContract, + ApiOperation, + ApiParameter, + HttpMethod, + ValueKind, +} from "./types"; + +interface ContractSource { + version: string; + ref: string; + sha256: string; +} + +const HTTP_METHODS = [ + "get", + "post", + "put", + "patch", + "delete", + "options", + "head", + "trace", +] as const; + +const LEGACY_FIELD_FLAGS_UNSUPPORTED = new Set([ + "annotationQueues_createQueue", + "datasetItems_create", + "datasetRunItems_create", + "datasets_create", + "ingestion_batch", + "legacy_scoreV1_create", + "models_create", + "opentelemetry_exportTraces", + "promptVersion_update", + "prompts_create", + "scim_createUser", + "score_create", + "scores_create", + "trace_deleteMultiple", + "unstable_dashboardWidgets_create", + "unstable_dashboards_addPlacement", + "unstable_dashboards_create", + "unstable_evaluationRules_create", + "unstable_evaluators_create", +]); + +function resolveLocalRef( + document: Record, + value: Record, +): Record { + let current = value; + const seen = new Set(); + while (current?.$ref) { + const ref = String(current.$ref); + if (!ref.startsWith("#/")) { + throw new Error(`External OpenAPI references are unsupported: ${ref}`); + } + if (seen.has(ref)) throw new Error(`Circular OpenAPI reference: ${ref}`); + seen.add(ref); + current = ref + .slice(2) + .split("/") + .map((segment) => segment.replaceAll("~1", "/").replaceAll("~0", "~")) + .reduce((node, segment) => node?.[segment], document); + if (!current) throw new Error(`Unresolved OpenAPI reference: ${ref}`); + } + return current; +} + +function schemaKind( + document: Record, + rawSchema: Record = {}, +): ValueKind { + const schema = resolveLocalRef(document, rawSchema); + const type = Array.isArray(schema.type) + ? schema.type.find((candidate: string) => candidate !== "null") + : schema.type; + if (type === "integer" || type === "number") return "number"; + if (type === "boolean") return "boolean"; + if (type === "array") return "array"; + if (type === "object" || schema.properties || schema.additionalProperties) { + return "object"; + } + if (type === "null") return "null"; + if (schema.allOf || schema.oneOf || schema.anyOf) { + const branches = schema.allOf ?? schema.oneOf ?? schema.anyOf; + const kinds = new Set( + branches.map((branch: Record) => schemaKind(document, branch)), + ); + return kinds.size === 1 ? [...kinds][0] : "object"; + } + return "string"; +} + +function parameterDefaults(location: string): { style: string; explode: boolean } { + if (location === "query" || location === "cookie") { + return { style: "form", explode: true }; + } + return { style: "simple", explode: false }; +} + +function mergeParameters( + document: Record, + pathParameters: any[] = [], + operationParameters: any[] = [], +): ApiParameter[] { + const merged = new Map(); + for (const raw of [...pathParameters, ...operationParameters]) { + const parameter = resolveLocalRef(document, raw); + const location = parameter.in; + if (!parameter.name || !["path", "query", "header", "cookie"].includes(location)) { + continue; + } + const schema = resolveLocalRef(document, parameter.schema ?? { type: "string" }); + const defaults = parameterDefaults(location); + const kind = schemaKind(document, schema); + merged.set(`${location}:${parameter.name}`, { + location, + name: String(parameter.name), + cliName: kebabCase(String(parameter.name)), + required: location === "path" || Boolean(parameter.required), + style: parameter.style ?? defaults.style, + explode: parameter.explode ?? defaults.explode, + kind, + ...(kind === "array" + ? { itemKind: schemaKind(document, schema.items ?? { type: "string" }) } + : {}), + }); + } + return [...merged.values()].sort((left, right) => { + if (left.location !== right.location) { + return left.location.localeCompare(right.location); + } + return left.name.localeCompare(right.name); + }); +} + +function collectBodyFields( + document: Record, + rawSchema: Record, +): ApiBodyField[] { + const fields = new Map(); + const visit = (raw: Record, inheritedRequired = new Set()) => { + const schema = resolveLocalRef(document, raw); + const required = new Set([ + ...inheritedRequired, + ...(schema.required ?? []), + ]); + for (const branch of [ + ...(schema.allOf ?? []), + ...(schema.oneOf ?? []), + ...(schema.anyOf ?? []), + ]) { + visit(branch, required); + } + for (const [name, rawProperty] of Object.entries>( + schema.properties ?? {}, + )) { + const property = resolveLocalRef(document, rawProperty); + const existing = fields.get(name); + const kind = existing?.kind ?? schemaKind(document, property); + fields.set(name, { + name, + required: Boolean(existing?.required || required.has(name)), + kind, + ...(kind === "array" + ? { + itemKind: + existing?.itemKind ?? + schemaKind(document, property.items ?? { type: "string" }), + } + : {}), + ...(property.description + ? { description: String(property.description) } + : existing?.description + ? { description: existing.description } + : {}), + }); + } + }; + visit(rawSchema); + return [...fields.values()].sort((left, right) => + left.name.localeCompare(right.name), + ); +} + +function normalizeAuth( + document: Record, + operation: Record, +): ApiOperation["auth"] { + const requirements = operation.security ?? document.security ?? []; + const schemes = [ + ...new Set( + requirements.flatMap((requirement: Record) => + Object.keys(requirement ?? {}), + ), + ), + ].sort(); + for (const name of schemes) { + const scheme = resolveLocalRef( + document, + document.components?.securitySchemes?.[name] ?? {}, + ); + if (scheme.type !== "http" || scheme.scheme !== "basic") { + throw new Error(`Unsupported authentication scheme: ${name}`); + } + } + return { + required: + requirements.length > 0 && + !requirements.some( + (requirement: Record) => + Object.keys(requirement ?? {}).length === 0, + ), + schemes, + }; +} + +function normalizeRequestBody( + document: Record, + operationId: string, + raw: Record | undefined, +): ApiOperation["requestBody"] { + if (!raw) return undefined; + const body = resolveLocalRef(document, raw); + const contentTypes = Object.keys(body.content ?? {}); + const contentType = + contentTypes.find((candidate) => candidate === "application/json") ?? + contentTypes.find((candidate) => candidate.includes("+json")); + if (!contentType) { + throw new Error( + `${operationId}: only JSON request bodies are supported (${contentTypes.join(", ")})`, + ); + } + const rawSchema = body.content?.[contentType]?.schema; + if (!rawSchema) throw new Error(`${operationId}: request body has no schema`); + return { + required: Boolean(body.required), + contentType, + legacyFieldFlags: !LEGACY_FIELD_FLAGS_UNSUPPORTED.has(operationId), + fields: collectBodyFields(document, rawSchema), + }; +} + +export function compileApiContract( + source: ContractSource, + raw: string, +): ApiContract { + const document = parse(raw, { + maxAliasCount: 100_000, + uniqueKeys: true, + }) as Record; + if (!String(document.openapi ?? "").startsWith("3.0.")) { + throw new Error(`${source.ref}: expected OpenAPI 3.0.x`); + } + const pending: Array<{ + key: string; + operationId: string; + method: HttpMethod; + path: string; + deprecated?: true; + tags: string[]; + auth: ApiOperation["auth"]; + pathParameterOrder: string[]; + parameters: ApiParameter[]; + requestBody?: ApiOperation["requestBody"]; + summary?: string; + description?: string; + }> = []; + for (const [path, rawPathItem] of Object.entries>( + document.paths ?? {}, + )) { + const pathItem = resolveLocalRef(document, rawPathItem); + for (const method of HTTP_METHODS) { + if (!pathItem[method]) continue; + const operation = resolveLocalRef(document, pathItem[method]); + if (operation.callbacks) { + throw new Error(`${method.toUpperCase()} ${path}: callbacks are unsupported`); + } + const operationId = String( + operation.operationId ?? `${method.toUpperCase()}:${path}`, + ); + pending.push({ + key: `${method.toUpperCase()} ${path}`, + operationId, + method: method.toUpperCase() as HttpMethod, + path, + ...(operation.deprecated === true ? { deprecated: true as const } : {}), + tags: (operation.tags ?? []).map(String), + auth: normalizeAuth(document, operation), + pathParameterOrder: [...path.matchAll(/\{([^}]+)\}/g)].map( + (match) => match[1], + ), + parameters: mergeParameters( + document, + pathItem.parameters, + operation.parameters, + ), + requestBody: normalizeRequestBody( + document, + operationId, + operation.requestBody, + ), + ...(operation.summary ? { summary: String(operation.summary) } : {}), + ...(operation.description + ? { description: String(operation.description) } + : {}), + }); + } + } + pending.sort((left, right) => { + if (left.path !== right.path) return left.path.localeCompare(right.path); + return left.method.localeCompare(right.method); + }); + const names = planCommandNames(pending); + const operations: ApiOperation[] = pending.map( + ({ tags: _tags, ...operation }, index) => ({ + ...operation, + command: names[index], + }), + ); + const operationIds = new Set(operations.map((operation) => operation.operationId)); + if (operationIds.size !== operations.length) { + throw new Error(`${source.ref}: duplicate operationId`); + } + return { + schemaVersion: 1, + apiVersion: source.version, + sourceSha256: source.sha256, + operations, + }; +} diff --git a/src/contracts/loader.test.ts b/src/contracts/loader.test.ts new file mode 100644 index 0000000..724ae47 --- /dev/null +++ b/src/contracts/loader.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, test } from "bun:test"; + +import { resolveContractVersion } from "./loader"; +import type { ApiContractCatalog } from "./types"; + +const catalog: ApiContractCatalog = { + schemaVersion: 1, + latest: "4.10.0", + versions: [ + { version: "3.216.0", sourceSha256: "3-latest" }, + { version: "4.10.0", sourceSha256: "4-latest" }, + { version: "3.0.0", sourceSha256: "3-oldest" }, + { version: "3.150.0", sourceSha256: "3-middle" }, + ], +}; + +function resolve(requested: string) { + return resolveContractVersion({ + requested, + host: "http://localhost:3000", + timeoutMs: 1_000, + catalog, + }); +} + +describe("API contract version resolution", () => { + test.each(["3", "v3", "3.x", "v3.x"])( + "resolves major selector %s to the latest bundled v3 contract", + async (requested) => { + expect((await resolve(requested)).version).toBe("3.216.0"); + }, + ); + + test("resolves another major independently", async () => { + expect((await resolve("4")).version).toBe("4.10.0"); + }); + + test("keeps exact selection exact", async () => { + expect((await resolve("3.150.0")).version).toBe("3.150.0"); + }); + + test("reports unavailable major selectors", async () => { + await expect(resolve("5")).rejects.toThrow( + "No bundled API contract for major version 5. Available majors: 3, 4", + ); + }); +}); diff --git a/src/contracts/loader.ts b/src/contracts/loader.ts new file mode 100644 index 0000000..7855970 --- /dev/null +++ b/src/contracts/loader.ts @@ -0,0 +1,128 @@ +import type { + ApiContract, + ApiContractCatalog, + ApiContractCatalogEntry, +} from "./types"; + +const CATALOG_URL = new URL("./contracts/catalog.json", import.meta.url); + +function parseVersion(version: string): [number, number, number] | undefined { + const match = /^v?(\d+)\.(\d+)\.(\d+)(?:[-+].*)?$/.exec(version); + if (!match) return undefined; + return [Number(match[1]), Number(match[2]), Number(match[3])]; +} + +function compareVersion(left: string, right: string): number { + const a = parseVersion(left); + const b = parseVersion(right); + if (!a || !b) return left.localeCompare(right); + for (let index = 0; index < 3; index++) { + if (a[index] !== b[index]) return a[index] - b[index]; + } + return 0; +} + +function requestedMajor(version: string): number | undefined { + const match = /^v?(\d+)(?:\.x)?$/i.exec(version); + return match ? Number(match[1]) : undefined; +} + +function latestMajorEntry( + entries: ApiContractCatalogEntry[], + major: number, +): ApiContractCatalogEntry | undefined { + return [...entries] + .filter((entry) => parseVersion(entry.version)?.[0] === major) + .sort((left, right) => compareVersion(right.version, left.version))[0]; +} + +export async function loadContractCatalog(): Promise { + const catalog = (await Bun.file(CATALOG_URL).json()) as ApiContractCatalog; + if (catalog.schemaVersion !== 1 || !Array.isArray(catalog.versions)) { + throw new Error("Invalid bundled API contract catalog"); + } + return catalog; +} + +async function detectServerVersion(host: string, timeoutMs: number): Promise { + const response = await fetch(`${host}/api/public/health`, { + signal: AbortSignal.timeout(timeoutMs), + }); + if (!response.ok) { + throw new Error(`API version detection failed: HTTP ${response.status}`); + } + const body = (await response.json()) as { version?: unknown }; + if (typeof body.version !== "string" || !parseVersion(body.version)) { + throw new Error("API version detection returned no semantic version"); + } + return body.version.replace(/^v/, ""); +} + +function compatibleEntry( + entries: ApiContractCatalogEntry[], + serverVersion: string, +): ApiContractCatalogEntry | undefined { + const target = parseVersion(serverVersion); + if (!target) return undefined; + return [...entries] + .filter((entry) => { + const version = parseVersion(entry.version); + return version?.[0] === target[0] && compareVersion(entry.version, serverVersion) <= 0; + }) + .sort((left, right) => compareVersion(right.version, left.version))[0]; +} + +export async function resolveContractVersion(params: { + requested?: string; + host: string; + timeoutMs: number; + catalog?: ApiContractCatalog; +}): Promise<{ catalog: ApiContractCatalog; version: string; detected?: string }> { + const catalog = params.catalog ?? (await loadContractCatalog()); + const requested = params.requested ?? "latest"; + if (requested === "latest") { + return { catalog, version: catalog.latest }; + } + if (requested === "auto") { + const detected = await detectServerVersion(params.host, params.timeoutMs); + const exact = catalog.versions.find((entry) => entry.version === detected); + const compatible = exact ?? compatibleEntry(catalog.versions, detected); + if (!compatible) { + throw new Error( + `No bundled API contract is compatible with detected server ${detected}`, + ); + } + return { catalog, version: compatible.version, detected }; + } + const exact = catalog.versions.find((entry) => entry.version === requested); + if (exact) return { catalog, version: exact.version }; + const major = requestedMajor(requested); + if (major !== undefined) { + const latestInMajor = latestMajorEntry(catalog.versions, major); + if (latestInMajor) return { catalog, version: latestInMajor.version }; + const availableMajors = [ + ...new Set( + catalog.versions + .map((entry) => parseVersion(entry.version)?.[0]) + .filter((value): value is number => value !== undefined), + ), + ].sort((left, right) => left - right); + throw new Error( + `No bundled API contract for major version ${major}. Available majors: ${availableMajors.join(", ")}`, + ); + } + throw new Error( + `Unknown API version ${requested}. Available: ${catalog.versions + .map((entry) => entry.version) + .join(", ")}`, + ); +} + +export async function loadApiContract(version: string): Promise { + const url = new URL(`./contracts/${encodeURIComponent(version)}.json`, import.meta.url); + const contract = (await Bun.file(url).json()) as ApiContract; + if (contract.schemaVersion !== 1 || contract.apiVersion !== version) { + throw new Error(`Invalid bundled API contract for ${version}`); + } + return contract; +} diff --git a/src/contracts/types.ts b/src/contracts/types.ts new file mode 100644 index 0000000..3d79344 --- /dev/null +++ b/src/contracts/types.ts @@ -0,0 +1,119 @@ +export type JsonPrimitive = string | number | boolean | null; +export type JsonValue = + | JsonPrimitive + | JsonValue[] + | { [key: string]: JsonValue }; + +export type HttpMethod = + | "GET" + | "POST" + | "PUT" + | "PATCH" + | "DELETE" + | "OPTIONS" + | "HEAD" + | "TRACE"; + +export type ValueKind = + | "string" + | "number" + | "boolean" + | "array" + | "object" + | "null"; + +export interface CommandName { + resource: string; + action: string; + aliases?: CommandAlias[]; +} + +export interface CommandAlias { + resource: string; + action: string; + source: "path" | "tag" | "version"; +} + +export interface ApiParameter { + location: "path" | "query" | "header" | "cookie"; + name: string; + cliName: string; + required: boolean; + style: string; + explode: boolean; + kind: ValueKind; + itemKind?: ValueKind; +} + +export interface ApiBodyField { + name: string; + required: boolean; + kind: ValueKind; + itemKind?: ValueKind; + description?: string; +} + +export interface ApiRequestBody { + required: boolean; + contentType: string; + legacyFieldFlags: boolean; + fields: ApiBodyField[]; +} + +export interface ApiOperation { + key: string; + operationId: string; + method: HttpMethod; + path: string; + deprecated?: true; + auth: { + required: boolean; + schemes: string[]; + }; + command: CommandName; + pathParameterOrder: string[]; + parameters: ApiParameter[]; + requestBody?: ApiRequestBody; + summary?: string; + description?: string; +} + +export interface ApiContract { + schemaVersion: 1; + apiVersion: string; + sourceSha256: string; + operations: ApiOperation[]; +} + +export interface ApiContractCatalogEntry { + version: string; + sourceSha256: string; +} + +export interface ApiContractCatalog { + schemaVersion: 1; + latest: string; + versions: ApiContractCatalogEntry[]; +} + +export interface ApiCallInput { + path: Record; + query: Record; + headers: Record; + cookies: Record; + body?: JsonValue; +} + +export interface ApiClientConfig { + host: string; + publicKey?: string; + secretKey?: string; + timeoutMs: number; +} + +export interface ApiResult { + status: number; + headers: Record; + body: JsonValue | string | null; + ok: boolean; +} diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..0d2a1ef --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "Preserve", + "moduleResolution": "Bundler", + "resolveJsonModule": true, + "strict": true, + "noEmit": true, + "types": ["bun"], + "skipLibCheck": true + }, + "include": [ + "src/**/*.ts", + "scripts/**/*.ts", + "conformance/src/**/*.ts" + ], + "exclude": ["**/*.test.ts"] +}