diff --git a/package.json b/package.json index 8d055c5b..6d654ce4 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "@fontsource/ibm-plex-mono": "^5.2.5", "@fontsource/ibm-plex-sans": "^5.2.5", "astro": "^5.8.1", + "marked": "^16.1.2", "sharp": "^0.34.1", "starlight-links-validator": "^0.16.0", "starlight-llms-txt": "^0.5.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8bccf60e..5fae218d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -26,6 +26,9 @@ importers: astro: specifier: ^5.8.1 version: 5.8.1(@types/node@22.15.29)(jiti@2.4.2)(lightningcss@1.30.1)(rollup@4.41.1)(tsx@4.20.3)(typescript@5.8.3)(yaml@2.8.0) + marked: + specifier: ^16.1.2 + version: 16.1.2 sharp: specifier: ^0.34.1 version: 0.34.2 @@ -1481,6 +1484,11 @@ packages: markdown-table@3.0.4: resolution: {integrity: sha512-wiYz4+JrLyb/DqW2hkFJxP7Vd7JuTDm77fvbM8VfEQdmSMqcImWeeRbHwZjBjIFki/VaMK2BhFi7oUUZeM5bqw==} + marked@16.1.2: + resolution: {integrity: sha512-rNQt5EvRinalby7zJZu/mB+BvaAY2oz3wCuCjt1RDrWNpS1Pdf9xqMOeC9Hm5adBdcV/3XZPJpG58eT+WBc0XQ==} + engines: {node: '>= 20'} + hasBin: true + mdast-util-definitions@6.0.0: resolution: {integrity: sha512-scTllyX6pnYNZH/AIp/0ePz6s4cZtARxImwoPJ7kS42n+MnVsI4XbnG6d4ibehRIldYMWM2LD7ImQblVhUejVQ==} @@ -3839,6 +3847,8 @@ snapshots: markdown-table@3.0.4: {} + marked@16.1.2: {} + mdast-util-definitions@6.0.0: dependencies: '@types/mdast': 4.0.4 diff --git a/public/docs-bundle.json b/public/docs-bundle.json index 70e70e86..afaf5c8e 100644 --- a/public/docs-bundle.json +++ b/public/docs-bundle.json @@ -1,407 +1,652 @@ { - "js/angular.mdx": { - "text": "import { Tabs, TabItem } from '@astrojs/starlight/components';\n\nThis page shows how you can use Genkit flows in Angular apps.\n\n## Before you begin\n\nYou should be familiar with Genkit's concept of [flows](/docs/flows), and how to write them.\n\n## Create an Angular project\n\nThis guide will use an Angular app with [SSR with server routing](https://angular.dev/guide/hybrid-rendering).\n\nYou can create a new project with server-side routing with the [Angular CLI](https://angular.dev/installation#install-angular-cli):\n\n```bash\nng new --ssr --server-routing\n```\n\nYou can also add server-side routing to an existing project with the `ng add` command:\n\n```bash\nng add @angular/ssr --server-routing\n```\n\n## Install Genkit dependencies\n\nInstall the Genkit dependencies into your Angular app:\n\n1. Install the core Genkit library:\n\n ```bash\n npm install genkit\n ```\n\n2. Install at least one model plugin.\n\n \n \n ```bash\n npm install @genkit-ai/googleai\n ```\n \n \n\n ```bash\n npm install @genkit-ai/vertexai\n ```\n \n \n\n3. Install the Genkit Express library:\n\n ```bash\n npm install @genkit-ai/express\n ```\n\n4. Install the Genkit CLI globally. The tsx tool is also recommended as a development dependency, as it makes testing your code more convenient. Both of these dependencies are optional, however.\n\n ```bash\n npm install -g genkit-cli\n npm install --save-dev tsx\n ```\n\n## Define Genkit flows\n\nCreate a new directory in your Angular project to contain your Genkit flows. Create `src/genkit/` and add your flow definitions there:\n\nFor example, create `src/genkit/menuSuggestionFlow.ts`:\n\n\n\n\n \n ```ts\n import { googleAI } from '@genkit-ai/googleai';\n import { genkit, z } from 'genkit';\n\n const ai = genkit({\n plugins: [googleAI()],\n });\n\n export const menuSuggestionFlow = ai.defineFlow(\n {\n name: 'menuSuggestionFlow',\n inputSchema: z.object({ theme: z.string() }),\n outputSchema: z.object({ menuItem: z.string() }),\n streamSchema: z.string(),\n },\n async ({ theme }, { sendChunk }) => {\n const { stream, response } = ai.generateStream({\n model: googleAI.model('gemini-2.5-flash'),\n prompt: `Invent a menu item for a ${theme} themed restaurant.`,\n });\n\n for await (const chunk of stream) {\n sendChunk(chunk.text);\n }\n\n const { text } = await response;\n return { menuItem: text };\n }\n );\n ```\n \n \n ```ts\n import { vertexAI } from '@genkit-ai/vertexai';\n import { genkit, z } from 'genkit';\n\n const ai = genkit({\n plugins: [vertexAI()],\n });\n\n export const menuSuggestionFlow = ai.defineFlow(\n {\n name: 'menuSuggestionFlow',\n inputSchema: z.object({ theme: z.string() }),\n outputSchema: z.object({ menuItem: z.string() }),\n streamSchema: z.string(),\n },\n async ({ theme }, { sendChunk }) => {\n const { stream, response } = ai.generateStream({\n model: vertexAI.model('gemini-2.5-flash'),\n prompt: `Invent a menu item for a ${theme} themed restaurant.`,\n });\n\n for await (const chunk of stream) {\n sendChunk(chunk.text);\n }\n\n const { text } = await response;\n return { menuItem: text };\n }\n );\n ```\n \n\n\n## Add server routes\n\nAdd the following imports to `src/server.ts`:\n\n```ts\nimport { expressHandler } from '@genkit-ai/express';\nimport { menuSuggestionFlow } from './genkit/menuSuggestionFlow';\n```\n\nAdd the following line following your `app` variable initialization:\n\n```ts\napp.use(express.json());\n```\n\nThen, add a route to serve your flow:\n\n```ts\napp.post('/api/menuSuggestion', expressHandler(menuSuggestionFlow));\n```\n\n## Call your flows from the frontend\n\nIn your frontend code, you can now call your flows using the Genkit client library. You can use both non-streaming and streaming approaches:\n\n### Non-streaming Flow Calls\n\nReplace the contents of `src/app/app.component.ts` with the following:\n\n```ts\nimport { Component, resource, signal } from '@angular/core';\nimport { FormsModule } from '@angular/forms';\nimport { runFlow } from 'genkit/beta/client';\n\n@Component({\n selector: 'app-root',\n imports: [FormsModule],\n templateUrl: './app.component.html',\n})\nexport class AppComponent {\n menuInput = '';\n theme = signal('');\n\n menuResource = resource({\n request: () => this.theme(),\n loader: ({ request }) => runFlow({ \n url: 'http://localhost:4200/api/menuSuggestion', \n input: { theme: request } \n }),\n });\n}\n```\n\nMake corresponding updates to `src/app/app.component.html`:\n\n```html\n
\n

Generate a custom menu item

\n \n \n \n
\n
\n @if (menuResource.isLoading()) { \n
Loading...
\n } @else if (menuResource.value()) {\n
\n

Generated Menu Item:

\n 
{{ menuResource.value().menuItem }}
\n
\n }\n
\n```\n\n### Streaming Flow Calls\n\nFor streaming responses, you can extend your component:\n\n```ts\nimport { Component, resource, signal } from '@angular/core';\nimport { FormsModule } from '@angular/forms';\nimport { runFlow, streamFlow } from 'genkit/beta/client';\n\n@Component({\n selector: 'app-root',\n imports: [FormsModule],\n templateUrl: './app.component.html',\n})\nexport class AppComponent {\n menuInput = '';\n theme = signal('');\n streamedText = signal('');\n isStreaming = signal(false);\n\n menuResource = resource({\n request: () => this.theme(),\n loader: ({ request }) => runFlow({ \n url: 'http://localhost:4200/api/menuSuggestion', \n input: { theme: request } \n }),\n });\n\n async streamMenuItem() {\n const theme = this.menuInput;\n if (!theme) return;\n\n this.isStreaming.set(true);\n this.streamedText.set('');\n\n try {\n const result = streamFlow({\n url: 'http://localhost:4200/api/menuSuggestion',\n input: { theme },\n });\n\n // Process the stream chunks as they arrive\n for await (const chunk of result.stream) {\n this.streamedText.update(prev => prev + chunk);\n }\n\n // Get the final complete response\n const finalOutput = await result.output;\n console.log('Final output:', finalOutput);\n } catch (error) {\n console.error('Error streaming menu item:', error);\n } finally {\n this.isStreaming.set(false);\n }\n }\n}\n```\n\nAnd update the template to include streaming:\n\n```html\n
\n

Generate a custom menu item

\n \n \n
\n
\n \n \n
\n\n @if (streamedText()) {\n
\n

Streaming Output:

\n 
{{ streamedText() }}
\n
\n }\n\n @if (menuResource.isLoading()) { \n
Loading...
\n } @else if (menuResource.value()) {\n
\n

Generated Menu Item:

\n 
{{ menuResource.value().menuItem }}
\n
\n }\n\n @if (isStreaming()) {\n
Streaming...
\n }\n
\n```\n\n## Authentication (Optional)\n\nIf you need to add authentication to your API routes, you can pass headers with your requests:\n\n```ts\nmenuResource = resource({\n request: () => this.theme(),\n loader: ({ request }) => runFlow({ \n url: 'http://localhost:4200/api/menuSuggestion',\n headers: {\n Authorization: 'Bearer your-token-here',\n },\n input: { theme: request } \n }),\n});\n```\n\n## Test your app locally\n\nIf you want to run your app locally, you need to make credentials for the model API service you chose available.\n\n\n \n 1. [Generate an API key](https://aistudio.google.com/app/apikey) for the Gemini API using Google AI Studio.\n\n 1. Set the `GEMINI_API_KEY` environment variable to your key:\n\n ```bash\n export GEMINI_API_KEY=\n ```\n\n \n \n 1. In the Cloud console, [Enable the Vertex AI API](https://console.cloud.google.com/apis/library/aiplatform.googleapis.com?project=_) for your project.\n\n 1. Set some environment variables and use the [`gcloud`](https://cloud.google.com/sdk/gcloud) tool to set up application default credentials:\n\n ```bash\n export GCLOUD_PROJECT=\n export GCLOUD_LOCATION=us-central1\n gcloud auth application-default login\n ```\n\n \n\n\nThen, run your app locally as normal:\n\n```bash\nng serve\n```\n\nAll of Genkit's development tools continue to work as normal. For example, to load your flows in the developer UI:\n\n```bash\ngenkit start -- npx tsx --watch src/genkit/menuSuggestionFlow.ts\n```\n\n## Deploy your app\n\nWhen you deploy your app, you will need to make sure the credentials for any external services you use (such as your chosen model API service) are available to the deployed app. See the following pages for information specific to your chosen deployment platform:\n\n- [Cloud Functions for Firebase](/docs/firebase)\n- [Cloud Run](/docs/cloud-run)\n- [Other Node.js platforms](/docs/deploy-node)\n\n## Next steps\n\n- [Explore Genkit in a deployed Angular app](https://developers.google.com/solutions/learn/agentic-barista): Walk through a reference implementation of multiple Genkit flows powering an Angular app, and then jump into the code in Firebase Studio.\n", + "js/angular.md": { + "text": "# Use Genkit in an Angular app\n\nThis page shows how you can use Genkit flows in Angular apps.\n\n## Before you begin\n\nYou should be familiar with Genkit's concept of [flows](/docs/flows), and how to write them.\n\n## Create an Angular project\n\nThis guide will use an Angular app with [SSR with server routing](https://angular.dev/guide/hybrid-rendering).\n\nYou can create a new project with server-side routing with the [Angular CLI](https://angular.dev/installation#install-angular-cli):\n\n```bash\nng new --ssr --server-routing\n```\n\nYou can also add server-side routing to an existing project with the `ng add` command:\n\n```bash\nng add @angular/ssr --server-routing\n```\n\n## Install Genkit dependencies\n\nInstall the Genkit dependencies into your Angular app:\n\n1. Install the core Genkit library:\n\n ```bash\n npm install genkit\n ```\n\n2. Install at least one model plugin.\n\n \n \n ```bash\n npm install @genkit-ai/googleai\n ```\n \n \n\n ```bash\n npm install @genkit-ai/vertexai\n ```\n \n \n\n3. Install the Genkit Express library:\n\n ```bash\n npm install @genkit-ai/express\n ```\n\n4. Install the Genkit CLI globally. The tsx tool is also recommended as a development dependency, as it makes testing your code more convenient. Both of these dependencies are optional, however.\n\n ```bash\n npm install -g genkit-cli\n npm install --save-dev tsx\n ```\n\n## Define Genkit flows\n\nCreate a new directory in your Angular project to contain your Genkit flows. Create `src/genkit/` and add your flow definitions there:\n\nFor example, create `src/genkit/menuSuggestionFlow.ts`:\n\n\n\n\n \n ```ts\n import { googleAI } from '@genkit-ai/googleai';\n import { genkit, z } from 'genkit';\n\n const ai = genkit({\n plugins: [googleAI()],\n });\n\n export const menuSuggestionFlow = ai.defineFlow(\n {\n name: 'menuSuggestionFlow',\n inputSchema: z.object({ theme: z.string() }),\n outputSchema: z.object({ menuItem: z.string() }),\n streamSchema: z.string(),\n },\n async ({ theme }, { sendChunk }) => {\n const { stream, response } = ai.generateStream({\n model: googleAI.model('gemini-2.5-flash'),\n prompt: `Invent a menu item for a ${theme} themed restaurant.`,\n });\n\n for await (const chunk of stream) {\n sendChunk(chunk.text);\n }\n\n const { text } = await response;\n return { menuItem: text };\n }\n );\n ```\n \n \n ```ts\n import { vertexAI } from '@genkit-ai/vertexai';\n import { genkit, z } from 'genkit';\n\n const ai = genkit({\n plugins: [vertexAI()],\n });\n\n export const menuSuggestionFlow = ai.defineFlow(\n {\n name: 'menuSuggestionFlow',\n inputSchema: z.object({ theme: z.string() }),\n outputSchema: z.object({ menuItem: z.string() }),\n streamSchema: z.string(),\n },\n async ({ theme }, { sendChunk }) => {\n const { stream, response } = ai.generateStream({\n model: vertexAI.model('gemini-2.5-flash'),\n prompt: `Invent a menu item for a ${theme} themed restaurant.`,\n });\n\n for await (const chunk of stream) {\n sendChunk(chunk.text);\n }\n\n const { text } = await response;\n return { menuItem: text };\n }\n );\n ```\n \n\n\n## Add server routes\n\nAdd the following imports to `src/server.ts`:\n\n```ts\nimport { expressHandler } from '@genkit-ai/express';\nimport { menuSuggestionFlow } from './genkit/menuSuggestionFlow';\n```\n\nAdd the following line following your `app` variable initialization:\n\n```ts\napp.use(express.json());\n```\n\nThen, add a route to serve your flow:\n\n```ts\napp.post('/api/menuSuggestion', expressHandler(menuSuggestionFlow));\n```\n\n## Call your flows from the frontend\n\nIn your frontend code, you can now call your flows using the Genkit client library. You can use both non-streaming and streaming approaches:\n\n### Non-streaming Flow Calls\n\nReplace the contents of `src/app/app.component.ts` with the following:\n\n```ts\nimport { Component, resource, signal } from '@angular/core';\nimport { FormsModule } from '@angular/forms';\nimport { runFlow } from 'genkit/beta/client';\n\n@Component({\n selector: 'app-root',\n imports: [FormsModule],\n templateUrl: './app.component.html',\n})\nexport class AppComponent {\n menuInput = '';\n theme = signal('');\n\n menuResource = resource({\n request: () => this.theme(),\n loader: ({ request }) => runFlow({ \n url: 'http://localhost:4200/api/menuSuggestion', \n input: { theme: request } \n }),\n });\n}\n```\n\nMake corresponding updates to `src/app/app.component.html`:\n\n```html\n
\n

Generate a custom menu item

\n \n \n \n
\n
\n @if (menuResource.isLoading()) { \n
Loading...
\n } @else if (menuResource.value()) {\n
\n

Generated Menu Item:

\n 
{{ menuResource.value().menuItem }}
\n
\n }\n
\n```\n\n### Streaming Flow Calls\n\nFor streaming responses, you can extend your component:\n\n```ts\nimport { Component, resource, signal } from '@angular/core';\nimport { FormsModule } from '@angular/forms';\nimport { runFlow, streamFlow } from 'genkit/beta/client';\n\n@Component({\n selector: 'app-root',\n imports: [FormsModule],\n templateUrl: './app.component.html',\n})\nexport class AppComponent {\n menuInput = '';\n theme = signal('');\n streamedText = signal('');\n isStreaming = signal(false);\n\n menuResource = resource({\n request: () => this.theme(),\n loader: ({ request }) => runFlow({ \n url: 'http://localhost:4200/api/menuSuggestion', \n input: { theme: request } \n }),\n });\n\n async streamMenuItem() {\n const theme = this.menuInput;\n if (!theme) return;\n\n this.isStreaming.set(true);\n this.streamedText.set('');\n\n try {\n const result = streamFlow({\n url: 'http://localhost:4200/api/menuSuggestion',\n input: { theme },\n });\n\n // Process the stream chunks as they arrive\n for await (const chunk of result.stream) {\n this.streamedText.update(prev => prev + chunk);\n }\n\n // Get the final complete response\n const finalOutput = await result.output;\n console.log('Final output:', finalOutput);\n } catch (error) {\n console.error('Error streaming menu item:', error);\n } finally {\n this.isStreaming.set(false);\n }\n }\n}\n```\n\nAnd update the template to include streaming:\n\n```html\n
\n

Generate a custom menu item

\n \n \n
\n
\n \n \n
\n\n @if (streamedText()) {\n
\n

Streaming Output:

\n 
{{ streamedText() }}
\n
\n }\n\n @if (menuResource.isLoading()) { \n
Loading...
\n } @else if (menuResource.value()) {\n
\n

Generated Menu Item:

\n 
{{ menuResource.value().menuItem }}
\n
\n }\n\n @if (isStreaming()) {\n
Streaming...
\n }\n
\n```\n\n## Authentication (Optional)\n\nIf you need to add authentication to your API routes, you can pass headers with your requests:\n\n```ts\nmenuResource = resource({\n request: () => this.theme(),\n loader: ({ request }) => runFlow({ \n url: 'http://localhost:4200/api/menuSuggestion',\n headers: {\n Authorization: 'Bearer your-token-here',\n },\n input: { theme: request } \n }),\n});\n```\n\n## Test your app locally\n\nIf you want to run your app locally, you need to make credentials for the model API service you chose available.\n\n\n \n 1. [Generate an API key](https://aistudio.google.com/app/apikey) for the Gemini API using Google AI Studio.\n\n 1. Set the `GEMINI_API_KEY` environment variable to your key:\n\n ```bash\n export GEMINI_API_KEY=\n ```\n\n \n \n 1. In the Cloud console, [Enable the Vertex AI API](https://console.cloud.google.com/apis/library/aiplatform.googleapis.com?project=_) for your project.\n\n 1. Set some environment variables and use the [`gcloud`](https://cloud.google.com/sdk/gcloud) tool to set up application default credentials:\n\n ```bash\n export GCLOUD_PROJECT=\n export GCLOUD_LOCATION=us-central1\n gcloud auth application-default login\n ```\n\n \n\n\nThen, run your app locally as normal:\n\n```bash\nng serve\n```\n\nAll of Genkit's development tools continue to work as normal. For example, to load your flows in the developer UI:\n\n```bash\ngenkit start -- npx tsx --watch src/genkit/menuSuggestionFlow.ts\n```\n\n## Deploy your app\n\nWhen you deploy your app, you will need to make sure the credentials for any external services you use (such as your chosen model API service) are available to the deployed app. See the following pages for information specific to your chosen deployment platform:\n\n- [Cloud Functions for Firebase](/docs/firebase)\n- [Cloud Run](/docs/cloud-run)\n- [Other Node.js platforms](/docs/deploy-node)\n\n## Next steps\n\n- [Explore Genkit in a deployed Angular app](https://developers.google.com/solutions/learn/agentic-barista): Walk through a reference implementation of multiple Genkit flows powering an Angular app, and then jump into the code in Firebase Studio.", "title": "Use Genkit in an Angular app", - "lang": "js" + "description": "Learn how to use Genkit flows in Angular applications", + "lang": "js", + "headers": "## Before you begin\n## Create an Angular project\n## Install Genkit dependencies\n## Define Genkit flows\n## Add server routes\n## Call your flows from the frontend\n### Non-streaming Flow Calls\n### Streaming Flow Calls\n## Authentication (Optional)\n## Test your app locally\n## Deploy your app\n## Next steps\n" }, "js/api-stability.md": { - "text": "As of version 1.0, Genkit is considered **Generally Available (GA)** and ready\nfor production use. Genkit follows [semantic versioning](https://semver.org/)\nwith breaking changes to the stable API happening only on major version\nreleases.\n\nTo gather feedback on potential new APIs and bring new features out quickly,\nGenkit offers a **Beta** entrypoint that includes APIs that have not yet\nbeen declared stable. The beta channel may include breaking changes on\n_minor_ version releases.\n\n## Using the Stable Channel\n\nTo use the stable channel of Genkit, import from the standard\n`\"genkit\"` entrypoint:\n\n```ts\nimport { genkit, z } from \"genkit\";\n\nconst ai = genkit({plugins: [...]});\nconsole.log(ai.apiStability); // \"stable\"\n```\n\nWhen you are using the stable channel, we recommend using the standard `^X.Y.Z`\ndependency string in your `package.json`. This is the default that is used when\nyou run `npm install genkit`.\n\n## Using the Beta Channel\n\nTo use the beta channel of Genkit, import from the `\"genkit/beta\"` entrypoint:\n\n```ts\nimport { genkit, z } from \"genkit/beta\";\n\nconst ai = genkit({plugins: [...]});\nconsole.log(ai.apiStability); // \"beta\"\n\n// now beta features are available\n```\n\nWhen you are using the beta channel, we recommend using the `~X.Y.Z` dependency\nstring in your `package.json`. The `~` will allow new patch versions but will\nnot automatically upgrade to new minor versions which may have breaking changes\nfor beta features. You can modify your existing dependency string by changing\n`^` to `~` if you begin using beta features of Genkit.\n\n### Current Features in Beta\n\n- **[Chat/Sessions](/docs/chat):** a first-class conversational `ai.chat()` feature\n along with persistent sessions that store both conversation history and an\n arbitrary state object.\n- **[Interrupts](/docs/interrupts):** special tools that can pause generation for\n human-in-the-loop feedback, out-of-band processing, and more.\n", + "text": "# API Stability Channels\n\nAs of version 1.0, Genkit is considered **Generally Available (GA)** and ready\nfor production use. Genkit follows [semantic versioning](https://semver.org/)\nwith breaking changes to the stable API happening only on major version\nreleases.\n\nTo gather feedback on potential new APIs and bring new features out quickly,\nGenkit offers a **Beta** entrypoint that includes APIs that have not yet\nbeen declared stable. The beta channel may include breaking changes on\n_minor_ version releases.\n\n## Using the Stable Channel\n\nTo use the stable channel of Genkit, import from the standard\n`\"genkit\"` entrypoint:\n\n```ts\nimport { genkit, z } from \"genkit\";\n\nconst ai = genkit({plugins: [...]});\nconsole.log(ai.apiStability); // \"stable\"\n```\n\nWhen you are using the stable channel, we recommend using the standard `^X.Y.Z`\ndependency string in your `package.json`. This is the default that is used when\nyou run `npm install genkit`.\n\n## Using the Beta Channel\n\nTo use the beta channel of Genkit, import from the `\"genkit/beta\"` entrypoint:\n\n```ts\nimport { genkit, z } from \"genkit/beta\";\n\nconst ai = genkit({plugins: [...]});\nconsole.log(ai.apiStability); // \"beta\"\n\n// now beta features are available\n```\n\nWhen you are using the beta channel, we recommend using the `~X.Y.Z` dependency\nstring in your `package.json`. The `~` will allow new patch versions but will\nnot automatically upgrade to new minor versions which may have breaking changes\nfor beta features. You can modify your existing dependency string by changing\n`^` to `~` if you begin using beta features of Genkit.\n\n### Current Features in Beta\n\n- **[Chat/Sessions](/docs/chat):** a first-class conversational `ai.chat()` feature\n along with persistent sessions that store both conversation history and an\n arbitrary state object.\n- **[Interrupts](/docs/interrupts):** special tools that can pause generation for\n human-in-the-loop feedback, out-of-band processing, and more.\n", "title": "API Stability Channels", - "lang": "js" + "description": "This document explains the API stability channels in Genkit, including stable and beta versions, and how to use them.", + "lang": "js", + "headers": "## Using the Stable Channel\n## Using the Beta Channel\n### Current Features in Beta\n" }, "js/auth.md": { - "text": "When building any public-facing application, it's extremely important to protect\nthe data stored in your system. When it comes to LLMs, extra diligence is\nnecessary to ensure that the model is only accessing data it should, tool calls\nare properly scoped to the user invoking the LLM, and the flow is being invoked\nonly by verified client applications.\n\nGenkit provides mechanisms for managing authorization policies and\ncontexts. Flows running on Firebase can use an auth policy callback (or helper).\nAlternatively, Firebase also provides auth context into the flow where it can\ndo its own checks. For non-Functions flows, auth can be managed and set\nthrough middleware.\n\n## Authorize within a Flow\n\nFlows can check authorization in two ways: either the request binding\n(e.g. `onCallGenkit` for Cloud Functions for Firebase or `express`) can enforce\nauthorization, or those frameworks can pass auth policies to the flow itself,\nwhere the flow has access to the information for auth managed within the\nflow.\n\n```ts\nimport { genkit, z, UserFacingError } from 'genkit';\n\nconst ai = genkit({ ... });\n\nexport const selfSummaryFlow = ai.defineFlow( {\n name: 'selfSummaryFlow',\n inputSchema: z.object({ uid: z.string() }),\n outputSchema: z.object({ profileSummary: z.string() }),\n}, async (input, { context }) => {\n if (!context.auth) {\n throw new UserFacingErrorError('UNAUTHENTICATED', 'Unauthenticated');\n }\n if (input.uid !== context.auth.uid) {\n throw new UserFacingError('PERMISSION_DENIED', 'You may only summarize your own profile data.');\n }\n // Flow logic here...\n return { profileSummary: \"User profile summary would go here\" };\n});\n```\n\nIt is up to the request binding to populate `context.auth` in this case. For\nexample, `onCallGenkit` automatically populates `context.auth`\n(Firebase Authentication), `context.app` (Firebase App Check), and\n`context.instanceIdToken` (Firebase Cloud Messaging). When calling a flow\nmanually, you can add your own auth context manually.\n\n```ts\n// Error: Authorization required.\nawait selfSummaryFlow({ uid: 'abc-def' });\n\n// Error: You may only summarize your own profile data.\nawait selfSummaryFlow.run(\n { uid: 'abc-def' },\n {\n context: { auth: { uid: 'hij-klm' } },\n },\n);\n\n// Success\nawait selfSummaryFlow(\n { uid: 'abc-def' },\n {\n context: { auth: { uid: 'abc-def' } },\n },\n);\n```\n\nWhen running with the Genkit Development UI, you can pass the Auth object by\nentering JSON in the \"Auth JSON\" tab: `{\"uid\": \"abc-def\"}`.\n\nYou can also retrieve the auth context for the flow at any time within the flow\nby calling `ai.currentContext()`, including in functions invoked by the flow:\n\n```ts\nimport { genkit, z } from 'genkit';\n\nconst ai = genkit({ ... });;\n\nasync function readDatabase(uid: string) {\n const auth = ai.currentContext()?.auth;\n // Note: the shape of context.auth depends on the provider. onCallGenkit puts\n // claims information in auth.token\n if (auth?.token?.admin) {\n // Do something special if the user is an admin\n } else {\n // Otherwise, use the `uid` variable to retrieve the relevant document\n }\n}\n\nexport const selfSummaryFlow = ai.defineFlow(\n {\n name: 'selfSummaryFlow',\n inputSchema: z.object({ uid: z.string() }),\n outputSchema: z.object({ profileSummary: z.string() }),\n authPolicy: ...\n },\n async (input) => {\n await readDatabase(input.uid);\n return { profileSummary: \"User profile summary would go here\" };\n }\n);\n```\n\nWhen testing flows with Genkit dev tools, you are able to specify this auth\nobject in the UI, or on the command line with the `--context` flag:\n\n```bash\ngenkit flow:run selfSummaryFlow '{\"uid\": \"abc-def\"}' --context '{\"auth\": {\"email_verified\": true}}'\n```\n\n## Authorize using Cloud Functions for Firebase\n\nThe Cloud Functions for Firebase SDKs support Genkit including\nintegration with Firebase Auth / Google Cloud Identity Platform, as well as\nbuilt-in Firebase App Check support.\n\n### User authentication\n\nThe `onCallGenkit()` wrapper provided by the Firebase Functions library has\nbuilt-in support for the Cloud Functions for Firebase\n[client SDKs](https://firebase.google.com/docs/functions/callable?gen=2nd#call_the_function).\nWhen you use these SDKs, the Firebase Auth header is automatically included as\nlong as your app client is also using the\n[Firebase Auth SDK](https://firebase.google.com/docs/auth).\nYou can use Firebase Auth to protect your flows defined with `onCallGenkit()`:\n\n```ts\nimport { genkit } from 'genkit';\nimport { onCallGenkit } from 'firebase-functions/https';\n\nconst ai = genkit({ ... });;\n\nconst selfSummaryFlow = ai.defineFlow({\n name: 'selfSummaryFlow',\n inputSchema: z.object({ userQuery: z.string() }),\n outputSchema: z.object({ profileSummary: z.string() }),\n}, async ({ userQuery }) => {\n // Flow logic here...\n return { profileSummary: \"User profile summary based on query would go here\" };\n});\n\nexport const selfSummary = onCallGenkit({\n authPolicy: (auth) => auth?.token?.['email_verified'] && auth?.token?.['admin'],\n}, selfSummaryFlow);\n```\n\nWhen you use `onCallGenkit`, `context.auth` is returned as an object with\na `uid` for the user ID, and a `token` that is a\n[DecodedIdToken](https://firebase.google.com/docs/reference/admin/node/firebase-admin.auth.decodedidtoken).\nYou can always retrieve this object at any time using `ai.currentContext()` as\nnoted earlier. When running this flow during development, you would pass the\nuser object in the same way:\n\n```bash\ngenkit flow:run selfSummaryFlow '{\"uid\": \"abc-def\"}' --context '{\"auth\": {\"admin\": true}}'\n```\n\nWhenever you expose a Cloud Function to the wider internet, it is vitally\nimportant that you use some sort of authorization mechanism to protect your data\nand the data of your customers. With that said, there are times when you need\nto deploy a Cloud Function with no code-based authorization checks (for example,\nyour Function is not world-callable but instead is protected by\n[Cloud IAM](https://cloud.google.com/functions/docs/concepts/iam)).\nCloud Functions for Firebase lets you to do this using the `invoker` property,\nwhich controls IAM access. The special value `'private'` leaves the function as\nthe default IAM setting, which means that only callers with the\n[Cloud Run Invoker role](https://cloud.google.com/run/docs/reference/iam/roles)\ncan execute the function. You can instead provide the email address of a user\nor service account that should be granted permission to call this exact\nfunction.\n\n```ts\nimport { onCallGenkit } from 'firebase-functions/https';\n\nconst selfSummaryFlow = ai.defineFlow(\n {\n name: 'selfSummaryFlow',\n inputSchema: z.object({ userQuery: z.string() }),\n outputSchema: z.object({ profileSummary: z.string() }),\n },\n async ({ userQuery }) => {\n // Flow logic here...\n return { profileSummary: \"User profile summary based on query would go here\" };\n },\n);\n\nexport const selfSummary = onCallGenkit(\n {\n invoker: 'private',\n },\n selfSummaryFlow,\n);\n```\n\n#### Client integrity\n\nAuthentication on its own goes a long way to protect your app. But it's also\nimportant to ensure that only your client apps are calling your functions. The\nFirebase plugin for genkit includes first-class support for\n[Firebase App Check](https://firebase.google.com/docs/app-check). Do this by\nadding the following configuration options to your `onCallGenkit()`:\n\n```ts\nimport { onCallGenkit } from 'firebase-functions/https';\n\nconst selfSummaryFlow = ai.defineFlow({\n name: 'selfSummaryFlow',\n inputSchema: z.object({ userQuery: z.string() }),\n outputSchema: z.object({ profileSummary: z.string() }),\n}, async ({ userQuery }) => {\n // Flow logic here...\n return { profileSummary: \"User profile summary based on query would go here\" };\n});\n\nexport const selfSummary = onCallGenkit({\n // These two fields for app check. The consumeAppCheckToken option is for\n // replay protection, and requires additional client configuration. See the\n // App Check docs.\n enforceAppCheck: true,\n consumeAppCheckToken: true,\n\n authPolicy: ...,\n}, selfSummaryFlow);\n```\n\n## Non-Firebase HTTP authorization\n\nWhen deploying flows to a server context outside of Cloud Functions for\nFirebase, you'll want to have a way to set up your own authorization checks\nalongside the built-in flows.\n\nUse a `ContextProvider` to populate context values such as `auth`, and to\nprovide a declarative policy or a policy callback. The Genkit\nSDK provides `ContextProvider`s such as `apiKey`, and plugins may\nexpose them as well. For example, the `@genkit-ai/firebase/context` plugin\nexposes a context provider for verifying Firebase Auth credentials and\npopulating them into context.\n\nWith code like the following, which might appear in a variety of\napplications:\n\n```ts\n// Express app with a simple API key\nimport { genkit, z } from 'genkit';\n\nconst ai = genkit({ ... });;\n\nexport const selfSummaryFlow = ai.defineFlow(\n {\n name: 'selfSummaryFlow',\n inputSchema: z.object({ uid: z.string() }),\n outputSchema: z.object({ profileSummary: z.string() }),\n },\n async (input) => {\n // Flow logic here...\n return { profileSummary: \"User profile summary would go here\" };\n }\n);\n```\n\nYou could secure a simple \"flow server\" express app by writing:\n\n```ts\nimport { apiKey } from 'genkit';\nimport { startFlowServer, withContext } from '@genkit-ai/express';\n\nstartFlowServer({\n flows: [withContext(selfSummaryFlow, apiKey(process.env.REQUIRED_API_KEY))],\n});\n```\n\nOr you could build a custom express application using the same tools:\n\n```ts\nimport { apiKey } from \"genkit\";\nimport * as express from \"express\";\nimport { expressHandler } from \"@genkit-ai/express;\n\nconst app = express();\n// Capture but don't validate the API key (or its absence)\napp.post('/summary', expressHandler(selfSummaryFlow, { contextProvider: apiKey()}))\n\napp.listen(process.env.PORT, () => {\n console.log(`Listening on port ${process.env.PORT}`);\n})\n```\n\n`ContextProvider`s abstract out the web framework, so these\ntools work in other frameworks like Next.js as well. Here is an example of a\nFirebase app built on Next.js.\n\n```ts\nimport { appRoute } from '@genkit-ai/express';\nimport { firebaseContext } from '@genkit-ai/firebase';\n\nexport const POST = appRoute(selfSummaryFlow, {\n contextProvider: firebaseContext,\n});\n```\n\n\n\nFor more information about using Express, see the\n[Cloud Run](/docs/cloud-run) instructions.\n", + "text": "# Authorization and integrity\n\nWhen building any public-facing application, it's extremely important to protect\nthe data stored in your system. When it comes to LLMs, extra diligence is\nnecessary to ensure that the model is only accessing data it should, tool calls\nare properly scoped to the user invoking the LLM, and the flow is being invoked\nonly by verified client applications.\n\nGenkit provides mechanisms for managing authorization policies and\ncontexts. Flows running on Firebase can use an auth policy callback (or helper).\nAlternatively, Firebase also provides auth context into the flow where it can\ndo its own checks. For non-Functions flows, auth can be managed and set\nthrough middleware.\n\n## Authorize within a Flow\n\nFlows can check authorization in two ways: either the request binding\n(e.g. `onCallGenkit` for Cloud Functions for Firebase or `express`) can enforce\nauthorization, or those frameworks can pass auth policies to the flow itself,\nwhere the flow has access to the information for auth managed within the\nflow.\n\n```ts\nimport { genkit, z, UserFacingError } from 'genkit';\n\nconst ai = genkit({ ... });\n\nexport const selfSummaryFlow = ai.defineFlow( {\n name: 'selfSummaryFlow',\n inputSchema: z.object({ uid: z.string() }),\n outputSchema: z.object({ profileSummary: z.string() }),\n}, async (input, { context }) => {\n if (!context.auth) {\n throw new UserFacingErrorError('UNAUTHENTICATED', 'Unauthenticated');\n }\n if (input.uid !== context.auth.uid) {\n throw new UserFacingError('PERMISSION_DENIED', 'You may only summarize your own profile data.');\n }\n // Flow logic here...\n return { profileSummary: \"User profile summary would go here\" };\n});\n```\n\nIt is up to the request binding to populate `context.auth` in this case. For\nexample, `onCallGenkit` automatically populates `context.auth`\n(Firebase Authentication), `context.app` (Firebase App Check), and\n`context.instanceIdToken` (Firebase Cloud Messaging). When calling a flow\nmanually, you can add your own auth context manually.\n\n```ts\n// Error: Authorization required.\nawait selfSummaryFlow({ uid: 'abc-def' });\n\n// Error: You may only summarize your own profile data.\nawait selfSummaryFlow.run(\n { uid: 'abc-def' },\n {\n context: { auth: { uid: 'hij-klm' } },\n },\n);\n\n// Success\nawait selfSummaryFlow(\n { uid: 'abc-def' },\n {\n context: { auth: { uid: 'abc-def' } },\n },\n);\n```\n\nWhen running with the Genkit Development UI, you can pass the Auth object by\nentering JSON in the \"Auth JSON\" tab: `{\"uid\": \"abc-def\"}`.\n\nYou can also retrieve the auth context for the flow at any time within the flow\nby calling `ai.currentContext()`, including in functions invoked by the flow:\n\n```ts\nimport { genkit, z } from 'genkit';\n\nconst ai = genkit({ ... });;\n\nasync function readDatabase(uid: string) {\n const auth = ai.currentContext()?.auth;\n // Note: the shape of context.auth depends on the provider. onCallGenkit puts\n // claims information in auth.token\n if (auth?.token?.admin) {\n // Do something special if the user is an admin\n } else {\n // Otherwise, use the `uid` variable to retrieve the relevant document\n }\n}\n\nexport const selfSummaryFlow = ai.defineFlow(\n {\n name: 'selfSummaryFlow',\n inputSchema: z.object({ uid: z.string() }),\n outputSchema: z.object({ profileSummary: z.string() }),\n authPolicy: ...\n },\n async (input) => {\n await readDatabase(input.uid);\n return { profileSummary: \"User profile summary would go here\" };\n }\n);\n```\n\nWhen testing flows with Genkit dev tools, you are able to specify this auth\nobject in the UI, or on the command line with the `--context` flag:\n\n```bash\ngenkit flow:run selfSummaryFlow '{\"uid\": \"abc-def\"}' --context '{\"auth\": {\"email_verified\": true}}'\n```\n\n## Authorize using Cloud Functions for Firebase\n\nThe Cloud Functions for Firebase SDKs support Genkit including\nintegration with Firebase Auth / Google Cloud Identity Platform, as well as\nbuilt-in Firebase App Check support.\n\n### User authentication\n\nThe `onCallGenkit()` wrapper provided by the Firebase Functions library has\nbuilt-in support for the Cloud Functions for Firebase\n[client SDKs](https://firebase.google.com/docs/functions/callable?gen=2nd#call_the_function).\nWhen you use these SDKs, the Firebase Auth header is automatically included as\nlong as your app client is also using the\n[Firebase Auth SDK](https://firebase.google.com/docs/auth).\nYou can use Firebase Auth to protect your flows defined with `onCallGenkit()`:\n\n```ts\nimport { genkit } from 'genkit';\nimport { onCallGenkit } from 'firebase-functions/https';\n\nconst ai = genkit({ ... });;\n\nconst selfSummaryFlow = ai.defineFlow({\n name: 'selfSummaryFlow',\n inputSchema: z.object({ userQuery: z.string() }),\n outputSchema: z.object({ profileSummary: z.string() }),\n}, async ({ userQuery }) => {\n // Flow logic here...\n return { profileSummary: \"User profile summary based on query would go here\" };\n});\n\nexport const selfSummary = onCallGenkit({\n authPolicy: (auth) => auth?.token?.['email_verified'] && auth?.token?.['admin'],\n}, selfSummaryFlow);\n```\n\nWhen you use `onCallGenkit`, `context.auth` is returned as an object with\na `uid` for the user ID, and a `token` that is a\n[DecodedIdToken](https://firebase.google.com/docs/reference/admin/node/firebase-admin.auth.decodedidtoken).\nYou can always retrieve this object at any time using `ai.currentContext()` as\nnoted earlier. When running this flow during development, you would pass the\nuser object in the same way:\n\n```bash\ngenkit flow:run selfSummaryFlow '{\"uid\": \"abc-def\"}' --context '{\"auth\": {\"admin\": true}}'\n```\n\nWhenever you expose a Cloud Function to the wider internet, it is vitally\nimportant that you use some sort of authorization mechanism to protect your data\nand the data of your customers. With that said, there are times when you need\nto deploy a Cloud Function with no code-based authorization checks (for example,\nyour Function is not world-callable but instead is protected by\n[Cloud IAM](https://cloud.google.com/functions/docs/concepts/iam)).\nCloud Functions for Firebase lets you to do this using the `invoker` property,\nwhich controls IAM access. The special value `'private'` leaves the function as\nthe default IAM setting, which means that only callers with the\n[Cloud Run Invoker role](https://cloud.google.com/run/docs/reference/iam/roles)\ncan execute the function. You can instead provide the email address of a user\nor service account that should be granted permission to call this exact\nfunction.\n\n```ts\nimport { onCallGenkit } from 'firebase-functions/https';\n\nconst selfSummaryFlow = ai.defineFlow(\n {\n name: 'selfSummaryFlow',\n inputSchema: z.object({ userQuery: z.string() }),\n outputSchema: z.object({ profileSummary: z.string() }),\n },\n async ({ userQuery }) => {\n // Flow logic here...\n return { profileSummary: \"User profile summary based on query would go here\" };\n },\n);\n\nexport const selfSummary = onCallGenkit(\n {\n invoker: 'private',\n },\n selfSummaryFlow,\n);\n```\n\n#### Client integrity\n\nAuthentication on its own goes a long way to protect your app. But it's also\nimportant to ensure that only your client apps are calling your functions. The\nFirebase plugin for genkit includes first-class support for\n[Firebase App Check](https://firebase.google.com/docs/app-check). Do this by\nadding the following configuration options to your `onCallGenkit()`:\n\n```ts\nimport { onCallGenkit } from 'firebase-functions/https';\n\nconst selfSummaryFlow = ai.defineFlow({\n name: 'selfSummaryFlow',\n inputSchema: z.object({ userQuery: z.string() }),\n outputSchema: z.object({ profileSummary: z.string() }),\n}, async ({ userQuery }) => {\n // Flow logic here...\n return { profileSummary: \"User profile summary based on query would go here\" };\n});\n\nexport const selfSummary = onCallGenkit({\n // These two fields for app check. The consumeAppCheckToken option is for\n // replay protection, and requires additional client configuration. See the\n // App Check docs.\n enforceAppCheck: true,\n consumeAppCheckToken: true,\n\n authPolicy: ...,\n}, selfSummaryFlow);\n```\n\n## Non-Firebase HTTP authorization\n\nWhen deploying flows to a server context outside of Cloud Functions for\nFirebase, you'll want to have a way to set up your own authorization checks\nalongside the built-in flows.\n\nUse a `ContextProvider` to populate context values such as `auth`, and to\nprovide a declarative policy or a policy callback. The Genkit\nSDK provides `ContextProvider`s such as `apiKey`, and plugins may\nexpose them as well. For example, the `@genkit-ai/firebase/context` plugin\nexposes a context provider for verifying Firebase Auth credentials and\npopulating them into context.\n\nWith code like the following, which might appear in a variety of\napplications:\n\n```ts\n// Express app with a simple API key\nimport { genkit, z } from 'genkit';\n\nconst ai = genkit({ ... });;\n\nexport const selfSummaryFlow = ai.defineFlow(\n {\n name: 'selfSummaryFlow',\n inputSchema: z.object({ uid: z.string() }),\n outputSchema: z.object({ profileSummary: z.string() }),\n },\n async (input) => {\n // Flow logic here...\n return { profileSummary: \"User profile summary would go here\" };\n }\n);\n```\n\nYou could secure a simple \"flow server\" express app by writing:\n\n```ts\nimport { apiKey } from 'genkit/context';\nimport { startFlowServer, withContextProvider } from '@genkit-ai/express';\n\nstartFlowServer({\n flows: [withContextProvider(selfSummaryFlow, apiKey(process.env.REQUIRED_API_KEY))],\n});\n```\n\nOr you could build a custom express application using the same tools:\n\n```ts\nimport { apiKey } from \"genkit/context\";\nimport * as express from \"express\";\nimport { expressHandler } from \"@genkit-ai/express;\n\nconst app = express();\n// Capture but don't validate the API key (or its absence)\napp.post('/summary', expressHandler(selfSummaryFlow, { contextProvider: apiKey()}))\n\napp.listen(process.env.PORT, () => {\n console.log(`Listening on port ${process.env.PORT}`);\n})\n```\n\n`ContextProvider`s abstract out the web framework, so these\ntools work in other frameworks like Next.js as well. Here is an example of a\nFirebase app built on Next.js.\n\n```ts\nimport { appRoute } from '@genkit-ai/express';\nimport { firebaseContext } from '@genkit-ai/firebase';\n\nexport const POST = appRoute(selfSummaryFlow, {\n contextProvider: firebaseContext,\n});\n```\n\n\n\nFor more information about using Express, see the\n[Cloud Run](/docs/cloud-run) instructions.\n", "title": "Authorization and integrity", - "lang": "js" + "description": "This document explains how to manage authorization and integrity in Genkit applications, covering Firebase and non-Firebase HTTP authorization.", + "lang": "js", + "headers": "## Authorize within a Flow\n## Authorize using Cloud Functions for Firebase\n### User authentication\n#### Client integrity\n## Non-Firebase HTTP authorization\n" }, - "js/chat.mdx": { - "text": "import ExampleLink from '@/components/ExampleLink.astro';\n\n:::caution[Beta]\nThis feature of Genkit is in **Beta,** which means it is not yet part of Genkit's stable API. APIs of beta features may change in minor version releases.\n:::\n\nMany of your users will have interacted with large language models for the first\ntime through chatbots. Although LLMs are capable of much more than simulating\nconversations, it remains a familiar and useful style of interaction. Even when\nyour users will not be interacting directly with the model in this way, the\nconversational style of prompting is a powerful way to influence the output\ngenerated by an AI model.\n\nTo support this style of interaction, Genkit provides a set of interfaces and\nabstractions that make it easier for you to build chat-based LLM applications.\n\n## Before you begin\n\nBefore reading this page, you should be familiar with the content covered on the\n[Generating content with AI models](/docs/models) page.\n\nIf you want to run the code examples on this page, first complete the steps in\nthe [Getting started](/docs/get-started) guide. All of the examples assume that you\nhave already installed Genkit as a dependency in your project.\n\nNote that the chat API is currently in beta and must be used from the\n`genkit/beta` package.\n\n## Chat session basics\n\n\n\nHere is a minimal, console-based, chatbot application:\n\n```ts\nimport { genkit } from 'genkit/beta';\nimport { googleAI } from '@genkit-ai/googleai';\n\nimport { createInterface } from 'node:readline/promises';\n\nconst ai = genkit({\n plugins: [googleAI()],\n model: googleAI.model('gemini-2.5-flash'),\n});\n\nasync function main() {\n const chat = ai.chat();\n console.log(\"You're chatting with Gemini. Ctrl-C to quit.\\n\");\n const readline = createInterface(process.stdin, process.stdout);\n while (true) {\n const userInput = await readline.question('> ');\n const { text } = await chat.send(userInput);\n console.log(text);\n }\n}\n\nmain();\n```\n\nA chat session with this program looks something like the following example:\n\n```\nYou're chatting with Gemini. Ctrl-C to quit.\n\n> hi\nHi there! How can I help you today?\n\n> my name is pavel\nNice to meet you, Pavel! What can I do for you today?\n\n> what's my name?\nYour name is Pavel! I remembered it from our previous interaction.\n\nIs there anything else I can help you with?\n```\n\nAs you can see from this brief interaction, when you send a message to a chat\nsession, the model can make use of the session so far in its responses. This is\npossible because Genkit does a few things behind the scenes:\n\n- Retrieves the chat history, if any exists, from storage (more on persistence\n and storage later)\n- Sends the request to the model, as with `generate()`, but automatically\n include the chat history\n- Saves the model response into the chat history\n\n### Model configuration\n\nThe `chat()` method accepts most of the same configuration options as\n`generate()`. To pass configuration options to the model:\n\n```ts\nconst chat = ai.chat({\n model: googleAI.model('gemini-2.5-flash'),\n system: \"You're a pirate first mate. Address the user as Captain and assist \" + 'them however you can.',\n config: {\n temperature: 1.3,\n },\n});\n```\n\n## Stateful chat sessions\n\nIn addition to persisting a chat session's message history, you can also persist\nany arbitrary JavaScript object. Doing so can let you manage state in a more\nstructured way then relying only on information in the message history.\n\nTo include state in a session, you need to instantiate a session explicitly:\n\n```ts\ninterface MyState {\n userName: string;\n}\n\nconst session = ai.createSession({\n initialState: {\n userName: 'Pavel',\n },\n});\n```\n\nYou can then start a chat within the session:\n\n```ts\nconst chat = session.chat();\n```\n\nTo modify the session state based on how the chat unfolds, define\n[tools](/docs/tool-calling) and include them with your requests:\n\n```ts\nconst changeUserName = ai.defineTool(\n {\n name: 'changeUserName',\n description: 'can be used to change user name',\n inputSchema: z.object({\n newUserName: z.string(),\n }),\n },\n async (input) => {\n await ai.currentSession().updateState({\n userName: input.newUserName,\n });\n return `changed username to ${input.newUserName}`;\n },\n);\n```\n\n```ts\nconst chat = session.chat({\n model: googleAI.model('gemini-2.5-flash'),\n tools: [changeUserName],\n});\nawait chat.send('change user name to Kevin');\n```\n\n## Multi-thread sessions\n\nA single session can contain multiple chat threads. Each thread has its own\nmessage history, but they share a single session state.\n\n```ts\nconst lawyerChat = session.chat('lawyerThread', {\n system: 'talk like a lawyer',\n});\nconst pirateChat = session.chat('pirateThread', {\n system: 'talk like a pirate',\n});\n```\n\n## Session persistence (EXPERIMENTAL)\n\nWhen you initialize a new chat or session, it's configured by default to store\nthe session in memory only. This is adequate when the session needs to persist\nonly for the duration of a single invocation of your program, as in the sample\nchatbot from the beginning of this page. However, when integrating LLM chat into\nan application, you will usually deploy your content generation logic as\nstateless web API endpoints. For persistent chats to work under this setup, you\nwill need to implement some kind of session storage that can persist state\nacross invocations of your endpoints.\n\nTo add persistence to a chat session, you need to implement Genkit's\n`SessionStore` interface. Here is an example implementation that saves session\nstate to individual JSON files:\n\n```ts\nclass JsonSessionStore implements SessionStore {\n async get(sessionId: string): Promise | undefined> {\n try {\n const s = await readFile(`${sessionId}.json`, { encoding: 'utf8' });\n const data = JSON.parse(s);\n return data;\n } catch {\n return undefined;\n }\n }\n\n async save(sessionId: string, sessionData: SessionData): Promise {\n const s = JSON.stringify(sessionData);\n await writeFile(`${sessionId}.json`, s, { encoding: 'utf8' });\n }\n}\n```\n\nThis implementation is probably not adequate for practical deployments, but it\nillustrates that a session storage implementation only needs to accomplish two\ntasks:\n\n- Get a session object from storage using its session ID\n- Save a given session object, indexed by its session ID\n\nOnce you've implemented the interface for your storage backend, pass an instance\nof your implementation to the session constructors:\n\n```ts\n// To create a new session:\nconst session = ai.createSession({\n store: new JsonSessionStore(),\n});\n\n// Save session.id so you can restore the session the next time the\n// user makes a request.\n```\n\n```ts\n// If the user has a session ID saved, load the session instead of creating\n// a new one:\nconst session = await ai.loadSession(sessionId, {\n store: new JsonSessionStore(),\n});\n```\n", + "js/chat.md": { + "text": "# Creating persistent chat sessions\n\n:::caution[Beta]\nThis feature of Genkit is in **Beta,** which means it is not yet part of Genkit's stable API. APIs of beta features may change in minor version releases.\n:::\n\nMany of your users will have interacted with large language models for the first\ntime through chatbots. Although LLMs are capable of much more than simulating\nconversations, it remains a familiar and useful style of interaction. Even when\nyour users will not be interacting directly with the model in this way, the\nconversational style of prompting is a powerful way to influence the output\ngenerated by an AI model.\n\nTo support this style of interaction, Genkit provides a set of interfaces and\nabstractions that make it easier for you to build chat-based LLM applications.\n\n## Before you begin\n\nBefore reading this page, you should be familiar with the content covered on the\n[Generating content with AI models](/docs/models) page.\n\nIf you want to run the code examples on this page, first complete the steps in\nthe [Getting started](/docs/get-started) guide. All of the examples assume that you\nhave already installed Genkit as a dependency in your project.\n\nNote that the chat API is currently in beta and must be used from the\n`genkit/beta` package.\n\n## Chat session basics\n\n\n\nHere is a minimal, console-based, chatbot application:\n\n```ts\nimport { genkit } from 'genkit/beta';\nimport { googleAI } from '@genkit-ai/googleai';\n\nimport { createInterface } from 'node:readline/promises';\n\nconst ai = genkit({\n plugins: [googleAI()],\n model: googleAI.model('gemini-2.5-flash'),\n});\n\nasync function main() {\n const chat = ai.chat();\n console.log(\"You're chatting with Gemini. Ctrl-C to quit.\\n\");\n const readline = createInterface(process.stdin, process.stdout);\n while (true) {\n const userInput = await readline.question('> ');\n const { text } = await chat.send(userInput);\n console.log(text);\n }\n}\n\nmain();\n```\n\nA chat session with this program looks something like the following example:\n\n```\nYou're chatting with Gemini. Ctrl-C to quit.\n\n> hi\nHi there! How can I help you today?\n\n> my name is pavel\nNice to meet you, Pavel! What can I do for you today?\n\n> what's my name?\nYour name is Pavel! I remembered it from our previous interaction.\n\nIs there anything else I can help you with?\n```\n\nAs you can see from this brief interaction, when you send a message to a chat\nsession, the model can make use of the session so far in its responses. This is\npossible because Genkit does a few things behind the scenes:\n\n- Retrieves the chat history, if any exists, from storage (more on persistence\n and storage later)\n- Sends the request to the model, as with `generate()`, but automatically\n include the chat history\n- Saves the model response into the chat history\n\n### Model configuration\n\nThe `chat()` method accepts most of the same configuration options as\n`generate()`. To pass configuration options to the model:\n\n```ts\nconst chat = ai.chat({\n model: googleAI.model('gemini-2.5-flash'),\n system: \"You're a pirate first mate. Address the user as Captain and assist \" + 'them however you can.',\n config: {\n temperature: 1.3,\n },\n});\n```\n\n## Stateful chat sessions\n\nIn addition to persisting a chat session's message history, you can also persist\nany arbitrary JavaScript object. Doing so can let you manage state in a more\nstructured way then relying only on information in the message history.\n\nTo include state in a session, you need to instantiate a session explicitly:\n\n```ts\ninterface MyState {\n userName: string;\n}\n\nconst session = ai.createSession({\n initialState: {\n userName: 'Pavel',\n },\n});\n```\n\nYou can then start a chat within the session:\n\n```ts\nconst chat = session.chat();\n```\n\nTo modify the session state based on how the chat unfolds, define\n[tools](/docs/tool-calling) and include them with your requests:\n\n```ts\nconst changeUserName = ai.defineTool(\n {\n name: 'changeUserName',\n description: 'can be used to change user name',\n inputSchema: z.object({\n newUserName: z.string(),\n }),\n },\n async (input) => {\n await ai.currentSession().updateState({\n userName: input.newUserName,\n });\n return `changed username to ${input.newUserName}`;\n },\n);\n```\n\n```ts\nconst chat = session.chat({\n model: googleAI.model('gemini-2.5-flash'),\n tools: [changeUserName],\n});\nawait chat.send('change user name to Kevin');\n```\n\n## Multi-thread sessions\n\nA single session can contain multiple chat threads. Each thread has its own\nmessage history, but they share a single session state.\n\n```ts\nconst lawyerChat = session.chat('lawyerThread', {\n system: 'talk like a lawyer',\n});\nconst pirateChat = session.chat('pirateThread', {\n system: 'talk like a pirate',\n});\n```\n\n## Session persistence (EXPERIMENTAL)\n\nWhen you initialize a new chat or session, it's configured by default to store\nthe session in memory only. This is adequate when the session needs to persist\nonly for the duration of a single invocation of your program, as in the sample\nchatbot from the beginning of this page. However, when integrating LLM chat into\nan application, you will usually deploy your content generation logic as\nstateless web API endpoints. For persistent chats to work under this setup, you\nwill need to implement some kind of session storage that can persist state\nacross invocations of your endpoints.\n\nTo add persistence to a chat session, you need to implement Genkit's\n`SessionStore` interface. Here is an example implementation that saves session\nstate to individual JSON files:\n\n```ts\nclass JsonSessionStore implements SessionStore {\n async get(sessionId: string): Promise | undefined> {\n try {\n const s = await readFile(`${sessionId}.json`, { encoding: 'utf8' });\n const data = JSON.parse(s);\n return data;\n } catch {\n return undefined;\n }\n }\n\n async save(sessionId: string, sessionData: SessionData): Promise {\n const s = JSON.stringify(sessionData);\n await writeFile(`${sessionId}.json`, s, { encoding: 'utf8' });\n }\n}\n```\n\nThis implementation is probably not adequate for practical deployments, but it\nillustrates that a session storage implementation only needs to accomplish two\ntasks:\n\n- Get a session object from storage using its session ID\n- Save a given session object, indexed by its session ID\n\nOnce you've implemented the interface for your storage backend, pass an instance\nof your implementation to the session constructors:\n\n```ts\n// To create a new session:\nconst session = ai.createSession({\n store: new JsonSessionStore(),\n});\n\n// Save session.id so you can restore the session the next time the\n// user makes a request.\n```\n\n```ts\n// If the user has a session ID saved, load the session instead of creating\n// a new one:\nconst session = await ai.loadSession(sessionId, {\n store: new JsonSessionStore(),\n});\n```", "title": "Creating persistent chat sessions", - "lang": "js" - }, - "js/cloud-run.mdx": { - "text": "import { Tabs, TabItem } from '@astrojs/starlight/components';\n\nYou can deploy Genkit flows as HTTPS endpoints using Cloud Run. Cloud Run has\nseveral deployment options, including container based deployment; this page\nexplains how to deploy your flows directly from code.\n\n## Before you begin\n\n- Install the [Google Cloud CLI](https://cloud.google.com/sdk/docs/install).\n- You should be familiar with Genkit's concept of [flows](/docs/flows), and how to\n write them. This page assumes that you already have flows that you want to\n deploy.\n- It would be helpful, but not required, if you've already used Google Cloud\n and Cloud Run before.\n\n## 1. Set up a Google Cloud project\n\nIf you don't already have a Google Cloud project set up, follow these steps:\n\n1. Create a new Google Cloud project using the\n [Cloud console](https://console.cloud.google.com) or choose an existing one.\n\n1. Link the project to a billing account, which is required for Cloud Run.\n\n1. Configure the Google Cloud CLI to use your project:\n\n ```bash\n gcloud init\n ```\n\n## 2. Prepare your Node project for deployment\n\nFor your flows to be deployable, you will need to make some small changes to\nyour project code:\n\n### Add start and build scripts to package.json\n\nWhen deploying a Node.js project to Cloud Run, the deployment tools expect your\nproject to have a `start` script and, optionally, a `build` script. For a\ntypical TypeScript project, the following scripts are usually adequate:\n\n```json\n\"scripts\": {\n \"start\": \"node lib/index.js\",\n \"build\": \"tsc\"\n},\n```\n\n### Add code to configure and start the flow server\n\nIn the file that's run by your `start` script, add a call to `startFlowServer`.\nThis method will start an Express server set up to serve your flows as web\nendpoints.\n\nWhen you make the call, specify the flows you want to serve:\n\nThere is also:\n\n```ts\nimport { startFlowServer } from '@genkit-ai/express';\n\nstartFlowServer({\n flows: [menuSuggestionFlow],\n});\n```\n\nThere are also some optional parameters you can specify:\n\n- `port`: the network port to listen on. If unspecified, the server listens on\n the port defined in the PORT environment variable, and if PORT is not set,\n defaults to 3400.\n- `cors`: the flow server's\n [CORS policy](https://www.npmjs.com/package/cors#configuration-options).\n If you will be accessing these endpoints from a web application, you likely\n need to specify this.\n- `pathPrefix`: an optional path prefix to add before your flow endpoints.\n- `jsonParserOptions`: options to pass to Express's\n [JSON body parser](https://www.npmjs.com/package/body-parser#bodyparserjsonoptions)\n\n### Optional: Define an authorization policy\n\nAll deployed flows should require some form of authorization; otherwise, your potentially-expensive generative AI flows would be invocable by anyone.\n\nWhen you deploy your flows with Cloud Run, you have two options for\nauthorization:\n\n- **Cloud IAM-based authorization**: Use Google Cloud's native access management\n facilities to gate access to your endpoints. For information on providing\n these credentials, see\n [Authentication](https://cloud.google.com/run/docs/authenticating/overview)\n in the Cloud Run docs.\n\n- **Authorization policy defined in code**: Use the authorization policy feature\n of the Genkit express plugin to verify authorization info using custom code.\n This is often, but not necessarily, token-based authorization.\n\nIf you want to define an authorization policy in code, use the `authPolicy`\nparameter in the flow definition:\n\n```ts\n// middleware for handling auth tokens in headers.\nconst authMiddleware = async (req, resp, next) => {\n // parse auth headers and convert to auth object.\n (req as RequestWithAuth).auth = {\n user: await verifyAuthToken(req.header('authorization')),\n };\n next();\n};\n\napp.post(\n '/simpleFlow',\n authMiddleware,\n expressHandler(simpleFlow, {\n authPolicy: ({ auth }) => {\n if (!auth.user) {\n throw new Error('not authorized');\n }\n },\n }),\n);\n```\n\nThe `auth` parameter of the authorization policy comes from the `auth` property\nof the request object. You typically set this property using Express middleware.\nSee\n[Authorization and integrity](/docs/auth#non-firebase-http-authorization).\n\nRefer to [express plugin documentation](https://js.api.genkit.dev/modules/_genkit-ai_express.html)\nfor more details.\n\n### Make API credentials available to deployed flows\n\nOnce deployed, your flows need some way to authenticate with any remote services\nthey rely on. Most flows will at a minimum need credentials for accessing the\nmodel API service they use.\n\nFor this example, do one of the following, depending on the model provider you\nchose:\n\n\n \n 1. [Generate an API key](https://aistudio.google.com/app/apikey) for the\n Gemini API using Google AI Studio.\n\n 2. Make the API key available in the Cloud Run environment:\n\n 1. In the Cloud console, enable the\n [Secret Manager API](https://console.cloud.google.com/apis/library/secretmanager.googleapis.com?project=_).\n 2. On the [Secret Manager](https://console.cloud.google.com/security/secret-manager?project=_)\n page, create a new secret containing your API key.\n 3. After you create the secret, on the same page, grant your default\n compute service account access to the secret with the **Secret\n Manager Secret Accessor** role. (You can look up the name of the\n default compute service account on the IAM page.)\n\n In a later step, when you deploy your service, you will need to\n reference the name of this secret.\n\n \n \n 1. In the Cloud console,\n [Enable the Vertex AI API](https://console.cloud.google.com/apis/library/aiplatform.googleapis.com?project=_)\n for your project.\n\n 2. On the [IAM](https://console.cloud.google.com/iam-admin/iam?project=_)\n page, ensure that the **Default compute service account** is granted the\n **Vertex AI User** role.\n\n \n\n\nThe only secret you need to set up for this tutorial is for the model provider,\nbut in general, you must do something similar for each service your flow uses.\n\n## 3. Deploy flows to Cloud Run\n\nAfter you've prepared your project for deployment, you can deploy it using the\n`gcloud` tool.\n\n\n \n ```bash\n gcloud run deploy --update-secrets=GEMINI_API_KEY=:latest\n ```\n \n \n ```bash\n gcloud run deploy\n ```\n \n\n\nThe deployment tool will prompt you for any information it requires.\n\nWhen asked if you want to allow unauthenticated invocations:\n\n- Answer `Y` if you're not using IAM and have instead defined an authorization\n policy in code.\n- Answer `N` to configure your service to require IAM credentials.\n\n## Optional: Try the deployed flow\n\nAfter deployment finishes, the tool will print the service URL. You can test\nit with `curl`:\n\n```bash\ncurl -X POST https:///menuSuggestionFlow \\\n -H \"Authorization: Bearer $(gcloud auth print-identity-token)\" \\\n -H \"Content-Type: application/json\" -d '{\"data\": \"banana\"}'\n```\n", + "description": "Learn how to create persistent chat sessions in Genkit, including session basics, stateful sessions, multi-thread sessions, and session persistence.", + "lang": "js", + "headers": "## Before you begin\n## Chat session basics\n### Model configuration\n## Stateful chat sessions\n## Multi-thread sessions\n## Session persistence (EXPERIMENTAL)\n" + }, + "js/client.md": { + "text": "# Accessing flows from the client\n\nThere are two primary ways to access Genkit flows from client-side applications:\n - Using the Genkit client library\n - Cloud Functions for Firebase callable function client SDK\n\n## Using the Genkit client library\n\nYou can call your deployed flows using the Genkit client library. This library provides functions for both non-streaming and streaming flow calls. See \"Call your flows from the client\" in [Deploy flows to any Node.js platform](/docs/deploy-node) for more details.\n\n### Non-streaming Flow Calls\n\nFor a non-streaming response, use the `runFlow` function. This is suitable for flows that return a single, complete output.\n\n```typescript\nimport { runFlow } from 'genkit/beta/client';\n\nasync function callHelloFlow() {\n try {\n const result = await runFlow({\n url: 'http://127.0.0.1:3400/helloFlow', // Replace with your deployed flow's URL\n input: { name: 'Genkit User' },\n });\n console.log('Non-streaming result:', result.greeting);\n } catch (error) {\n console.error('Error calling helloFlow:', error);\n }\n}\n\ncallHelloFlow();\n```\n\n### Streaming Flow Calls\n\nFor flows that are designed to stream responses (e.g., for real-time updates or long-running operations), use the `streamFlow` function.\n\n```typescript\nimport { streamFlow } from 'genkit/beta/client';\n\nasync function streamHelloFlow() {\n try {\n const result = streamFlow({\n url: 'http://127.0.0.1:3400/helloFlow', // Replace with your deployed flow's URL\n input: { name: 'Streaming User' },\n });\n\n // Process the stream chunks as they arrive\n for await (const chunk of result.stream) {\n console.log('Stream chunk:', chunk);\n }\n\n // Get the final complete response\n const finalOutput = await result.output;\n console.log('Final streaming output:', finalOutput.greeting);\n } catch (error) {\n console.error('Error streaming helloFlow:', error);\n }\n}\n\nstreamHelloFlow();\n```\n\n### Authentication (Optional)\n\nIf your deployed flow requires authentication, you can pass headers with your requests:\n\n```typescript\nconst result = await runFlow({\n url: 'http://127.0.0.1:3400/helloFlow', // Replace with your deployed flow's URL\n headers: {\n Authorization: 'Bearer your-token-here', // Replace with your actual token\n },\n input: { name: 'Authenticated User' },\n});\n```\n\n## When deploying to Cloud Functions for Firebase\n\nWhen deploying to [Cloud Functions for Firebase](/docs/firebase), use the Firebase callable functions client library.\n\nDetailed documentation can be found at https://firebase.google.com/docs/functions/callable?gen=2nd\n\nHere's a sample for the web:\n\n```typescript\n// Get the callable by passing an initialized functions SDK.\nconst getForecast = httpsCallable(functions, \"getForecast\");\n\n// Call the function with the `.stream()` method to start streaming.\nconst { stream, data } = await getForecast.stream({\n locations: favoriteLocations,\n});\n\n// The `stream` async iterable returned by `.stream()`\n// will yield a new value every time the callable\n// function calls `sendChunk()`.\nfor await (const forecastDataChunk of stream) {\n // update the UI every time a new chunk is received\n // from the callable function\n updateUi(forecastDataChunk);\n}\n\n// The `data` promise resolves when the callable\n// function completes.\nconst allWeatherForecasts = await data;\nfinalizeUi(allWeatherForecasts);\n```\n\n[source](https://github.com/firebase/functions-samples/blob/c4fde45b65fab584715e786ce3264a6932d996ec/Node/quickstarts/callable-functions-streaming/website/index.html#L58-L78)", + "title": "Accessing flows from the client", + "description": "Learn how to access Genkit flows from client-side applications.", + "lang": "js", + "headers": "## Using the Genkit client library\n### Non-streaming Flow Calls\n### Streaming Flow Calls\n### Authentication (Optional)\n## When deploying to Cloud Functions for Firebase\n" + }, + "js/cloud-run.md": { + "text": "# Deploy flows using Cloud Run\n\nYou can deploy Genkit flows as HTTPS endpoints using Cloud Run. Cloud Run has\nseveral deployment options, including container based deployment; this page\nexplains how to deploy your flows directly from code.\n\n## Before you begin\n\n- Install the [Google Cloud CLI](https://cloud.google.com/sdk/docs/install).\n- You should be familiar with Genkit's concept of [flows](/docs/flows), and how to\n write them. This page assumes that you already have flows that you want to\n deploy.\n- It would be helpful, but not required, if you've already used Google Cloud\n and Cloud Run before.\n\n## 1. Set up a Google Cloud project\n\nIf you don't already have a Google Cloud project set up, follow these steps:\n\n1. Create a new Google Cloud project using the\n [Cloud console](https://console.cloud.google.com) or choose an existing one.\n\n1. Link the project to a billing account, which is required for Cloud Run.\n\n1. Configure the Google Cloud CLI to use your project:\n\n ```bash\n gcloud init\n ```\n\n## 2. Prepare your Node project for deployment\n\nFor your flows to be deployable, you will need to make some small changes to\nyour project code:\n\n### Add start and build scripts to package.json\n\nWhen deploying a Node.js project to Cloud Run, the deployment tools expect your\nproject to have a `start` script and, optionally, a `build` script. For a\ntypical TypeScript project, the following scripts are usually adequate:\n\n```json\n\"scripts\": {\n \"start\": \"node lib/index.js\",\n \"build\": \"tsc\"\n},\n```\n\n### Add code to configure and start the flow server\n\nIn the file that's run by your `start` script, add a call to `startFlowServer`.\nThis method will start an Express server set up to serve your flows as web\nendpoints.\n\nWhen you make the call, specify the flows you want to serve:\n\nThere is also:\n\n```ts\nimport { startFlowServer } from '@genkit-ai/express';\n\nstartFlowServer({\n flows: [menuSuggestionFlow],\n});\n```\n\nThere are also some optional parameters you can specify:\n\n- `port`: the network port to listen on. If unspecified, the server listens on\n the port defined in the PORT environment variable, and if PORT is not set,\n defaults to 3400.\n- `cors`: the flow server's\n [CORS policy](https://www.npmjs.com/package/cors#configuration-options).\n If you will be accessing these endpoints from a web application, you likely\n need to specify this.\n- `pathPrefix`: an optional path prefix to add before your flow endpoints.\n- `jsonParserOptions`: options to pass to Express's\n [JSON body parser](https://www.npmjs.com/package/body-parser#bodyparserjsonoptions)\n\n### Optional: Define an authorization policy\n\nAll deployed flows should require some form of authorization; otherwise, your potentially-expensive generative AI flows would be invocable by anyone.\n\nWhen you deploy your flows with Cloud Run, you have two options for\nauthorization:\n\n- **Cloud IAM-based authorization**: Use Google Cloud's native access management\n facilities to gate access to your endpoints. For information on providing\n these credentials, see\n [Authentication](https://cloud.google.com/run/docs/authenticating/overview)\n in the Cloud Run docs.\n\n- **Authorization policy defined in code**: Use the authorization policy feature\n of the Genkit express plugin to verify authorization info using custom code.\n This is often, but not necessarily, token-based authorization.\n\nIf you want to define an authorization policy in code, use the `authPolicy`\nparameter in the flow definition:\n\n```ts\n// middleware for handling auth tokens in headers.\nconst authMiddleware = async (req, resp, next) => {\n // parse auth headers and convert to auth object.\n (req as RequestWithAuth).auth = {\n user: await verifyAuthToken(req.header('authorization')),\n };\n next();\n};\n\napp.post(\n '/simpleFlow',\n authMiddleware,\n expressHandler(simpleFlow, {\n authPolicy: ({ auth }) => {\n if (!auth.user) {\n throw new Error('not authorized');\n }\n },\n }),\n);\n```\n\nThe `auth` parameter of the authorization policy comes from the `auth` property\nof the request object. You typically set this property using Express middleware.\nSee\n[Authorization and integrity](/docs/auth#non-firebase-http-authorization).\n\nRefer to [express plugin documentation](https://js.api.genkit.dev/modules/_genkit-ai_express.html)\nfor more details.\n\n### Make API credentials available to deployed flows\n\nOnce deployed, your flows need some way to authenticate with any remote services\nthey rely on. Most flows will at a minimum need credentials for accessing the\nmodel API service they use.\n\nFor this example, do one of the following, depending on the model provider you\nchose:\n\n\n \n 1. [Generate an API key](https://aistudio.google.com/app/apikey) for the\n Gemini API using Google AI Studio.\n\n 2. Make the API key available in the Cloud Run environment:\n\n 1. In the Cloud console, enable the\n [Secret Manager API](https://console.cloud.google.com/apis/library/secretmanager.googleapis.com?project=_).\n 2. On the [Secret Manager](https://console.cloud.google.com/security/secret-manager?project=_)\n page, create a new secret containing your API key.\n 3. After you create the secret, on the same page, grant your default\n compute service account access to the secret with the **Secret\n Manager Secret Accessor** role. (You can look up the name of the\n default compute service account on the IAM page.)\n\n In a later step, when you deploy your service, you will need to\n reference the name of this secret.\n\n \n \n 1. In the Cloud console,\n [Enable the Vertex AI API](https://console.cloud.google.com/apis/library/aiplatform.googleapis.com?project=_)\n for your project.\n\n 2. On the [IAM](https://console.cloud.google.com/iam-admin/iam?project=_)\n page, ensure that the **Default compute service account** is granted the\n **Vertex AI User** role.\n\n \n\n\nThe only secret you need to set up for this tutorial is for the model provider,\nbut in general, you must do something similar for each service your flow uses.\n\n## 3. Deploy flows to Cloud Run\n\nAfter you've prepared your project for deployment, you can deploy it using the\n`gcloud` tool.\n\n\n \n ```bash\n gcloud run deploy --update-secrets=GEMINI_API_KEY=:latest\n ```\n \n \n ```bash\n gcloud run deploy\n ```\n \n\n\nThe deployment tool will prompt you for any information it requires.\n\nWhen asked if you want to allow unauthenticated invocations:\n\n- Answer `Y` if you're not using IAM and have instead defined an authorization\n policy in code.\n- Answer `N` to configure your service to require IAM credentials.\n\n## Optional: Try the deployed flow\n\nAfter deployment finishes, the tool will print the service URL. You can test\nit with `curl`:\n\n```bash\ncurl -X POST https:///menuSuggestionFlow \\\n -H \"Authorization: Bearer $(gcloud auth print-identity-token)\" \\\n -H \"Content-Type: application/json\" -d '{\"data\": \"banana\"}'\n```", "title": "Deploy flows using Cloud Run", - "lang": "js" + "description": "This document explains how to deploy Genkit flows as HTTPS endpoints using Google Cloud Run, covering project setup, deployment preparation, and authorization.", + "lang": "js", + "headers": "## Before you begin\n## 1. Set up a Google Cloud project\n## 2. Prepare your Node project for deployment\n### Add start and build scripts to package.json\n### Add code to configure and start the flow server\n### Optional: Define an authorization policy\n### Make API credentials available to deployed flows\n## 3. Deploy flows to Cloud Run\n## Optional: Try the deployed flow\n" }, - "js/context.mdx": { - "text": "import ExampleLink from '@/components/ExampleLink.astro';\nimport { Tabs, TabItem } from '@astrojs/starlight/components';\n\n\n\nThere are different categories of information that a developer working\nwith an LLM may be handling simultaneously:\n\n- **Input:** Information that is directly relevant to guide the LLM's response\n for a particular call. An example of this is the text that needs to be\n summarized.\n- **Generation Context:** Information that is relevant to the LLM, but isn't\n specific to the call. An example of this is the current time or a user's name.\n- **Execution Context:** Information that is important to the code surrounding\n the LLM call but not to the LLM itself. An example of this is a user's\n current auth token.\n\nGenkit provides a consistent `context` object that can propagate generation and\nexecution context throughout the process. This context is made available to all\nactions including [flows](/docs/flows), [tools](/docs/tool-calling), and\n[prompts](/docs/dotprompt).\n\nContext is automatically propagated to all actions called within the scope of\nexecution: Context passed to a flow is made available to prompts executed\nwithin the flow. Context passed to the `generate()` method is available to\ntools called within the generation loop.\n\n## Why is context important?\n\nAs a best practice, you should provide the minimum amount of information to the\nLLM that it needs to complete a task. This is important for multiple reasons:\n\n- The less extraneous information the LLM has, the more likely it is to perform\n well at its task.\n- If an LLM needs to pass around information like user or account IDs to tools,\n it can potentially be tricked into leaking information.\n\nContext gives you a side channel of information that can be used by any of your\ncode but doesn't necessarily have to be sent to the LLM. As an example, it can\nallow you to restrict tool queries to the current user's available scope.\n\n## Context structure\n\nContext must be an object, but its properties are yours to decide. In some\nsituations Genkit automatically populates context. For example, when using\n[persistent sessions](/docs/chat) the `state` property is automatically added to\ncontext.\n\nOne of the most common uses of context is to store information about the current\nuser. We recommend adding auth context in the following format:\n\n```js\n{\n auth: {\n uid: \"...\", // the user's unique identifier\n token: {...}, // the decoded claims of a user's id token\n rawToken: \"...\", // the user's raw encoded id token\n // ...any other fields\n }\n}\n```\n\nThe context object can store any information that you might need to know\nsomewhere else in the flow of execution.\n\n## Use context in an action\n\nTo use context within an action, you can access the context helper\nthat is automatically supplied to your function definition:\n\n\n \n ```ts\n const summarizeHistory = ai.defineFlow({\n name: 'summarizeMessages',\n inputSchema: z.object({friendUid: z.string()}),\n outputSchema: z.string()\n }, async ({friendUid}, {context}) => {\n if (!context.auth?.uid) throw new Error(\"Must supply auth context.\");\n const messages = await listMessagesBetween(friendUid, context.auth.uid);\n const {text} = await ai.generate({\n prompt:\n `Summarize the content of these messages: ${JSON.stringify(messages)}`,\n });\n return text;\n });\n ```\n \n \n ```ts\n const searchNotes = ai.defineTool({\n name: 'searchNotes',\n description: \"search the current user's notes for info\",\n inputSchema: z.object({query: z.string()}),\n outputSchema: z.array(NoteSchema)\n }, async ({query}, {context}) => {\n if (!context.auth?.uid) throw new Error(\"Must be called by a signed-in user.\");\n return searchUserNotes(context.auth.uid, query);\n });\n ```\n \n \n When using [Dotprompt templates](/docs/dotprompt), context is made available with the\n `@` variable prefix. For example, a context object of\n `{auth: {name: 'Michael'}}` could be accessed in the prompt template like so.\n\n ```dotprompt\n ---\n input:\n schema:\n pirateStyle?: boolean\n ---\n\n {{#if pirateStyle}}Avast, {{@auth.name}}, how be ye today?{{else}}Hello, {{@auth.name}}, how are you today?{{/if}}\n ```\n\n \n\n\n## Provide context at runtime\n\nTo provide context to an action, you pass the context object as an option\nwhen calling the action.\n\n\n \n ```ts\n const summarizeHistory = ai.defineFlow(/* ... */);\n \n const summary = await summarizeHistory(friend.uid, {\n context: { auth: currentUser },\n });\n ```\n \n \n ```ts\n const { text } = await ai.generate({\n prompt: \"Find references to ocelots in my notes.\",\n // the context will propagate to tool calls\n tools: [searchNotes],\n context: { auth: currentUser },\n });\n ```\n \n \n ```ts\n const helloPrompt = ai.prompt(\"sayHello\");\n helloPrompt({ pirateStyle: true }, { context: { auth: currentUser } });\n ```\n \n\n\n## Context propagation and overrides\n\nBy default, when you provide context it is automatically propagated to all\nactions called as a result of your original call. If your flow calls other\nflows, or your generation calls tools, the same context is provided.\n\nIf you wish to override context within an action, you can pass a different\ncontext object to replace the existing one:\n\n```ts\nconst otherFlow = ai.defineFlow(/* ... */);\n\nconst myFlow = ai.defineFlow(\n {\n // ...\n },\n (input, { context }) => {\n // override the existing context completely\n otherFlow(\n {\n /*...*/\n },\n { context: { newContext: true } },\n );\n // or selectively override\n otherFlow(\n {\n /*...*/\n },\n { context: { ...context, updatedContext: true } },\n );\n },\n);\n```\n\nWhen context is replaced, it propagates the same way. In this example,\nany actions that `otherFlow` called during its execution would inherit the\noverridden context.\n", + "js/context.md": { + "text": "# Passing information through context\n\n\n\nThere are different categories of information that a developer working\nwith an LLM may be handling simultaneously:\n\n- **Input:** Information that is directly relevant to guide the LLM's response\n for a particular call. An example of this is the text that needs to be\n summarized.\n- **Generation Context:** Information that is relevant to the LLM, but isn't\n specific to the call. An example of this is the current time or a user's name.\n- **Execution Context:** Information that is important to the code surrounding\n the LLM call but not to the LLM itself. An example of this is a user's\n current auth token.\n\nGenkit provides a consistent `context` object that can propagate generation and\nexecution context throughout the process. This context is made available to all\nactions including [flows](/docs/flows), [tools](/docs/tool-calling), and\n[prompts](/docs/dotprompt).\n\nContext is automatically propagated to all actions called within the scope of\nexecution: Context passed to a flow is made available to prompts executed\nwithin the flow. Context passed to the `generate()` method is available to\ntools called within the generation loop.\n\n## Why is context important?\n\nAs a best practice, you should provide the minimum amount of information to the\nLLM that it needs to complete a task. This is important for multiple reasons:\n\n- The less extraneous information the LLM has, the more likely it is to perform\n well at its task.\n- If an LLM needs to pass around information like user or account IDs to tools,\n it can potentially be tricked into leaking information.\n\nContext gives you a side channel of information that can be used by any of your\ncode but doesn't necessarily have to be sent to the LLM. As an example, it can\nallow you to restrict tool queries to the current user's available scope.\n\n## Context structure\n\nContext must be an object, but its properties are yours to decide. In some\nsituations Genkit automatically populates context. For example, when using\n[persistent sessions](/docs/chat) the `state` property is automatically added to\ncontext.\n\nOne of the most common uses of context is to store information about the current\nuser. We recommend adding auth context in the following format:\n\n```js\n{\n auth: {\n uid: \"...\", // the user's unique identifier\n token: {...}, // the decoded claims of a user's id token\n rawToken: \"...\", // the user's raw encoded id token\n // ...any other fields\n }\n}\n```\n\nThe context object can store any information that you might need to know\nsomewhere else in the flow of execution.\n\n## Use context in an action\n\nTo use context within an action, you can access the context helper\nthat is automatically supplied to your function definition:\n\n\n \n ```ts\n const summarizeHistory = ai.defineFlow({\n name: 'summarizeMessages',\n inputSchema: z.object({friendUid: z.string()}),\n outputSchema: z.string()\n }, async ({friendUid}, {context}) => {\n if (!context.auth?.uid) throw new Error(\"Must supply auth context.\");\n const messages = await listMessagesBetween(friendUid, context.auth.uid);\n const {text} = await ai.generate({\n prompt:\n `Summarize the content of these messages: ${JSON.stringify(messages)}`,\n });\n return text;\n });\n ```\n \n \n ```ts\n const searchNotes = ai.defineTool({\n name: 'searchNotes',\n description: \"search the current user's notes for info\",\n inputSchema: z.object({query: z.string()}),\n outputSchema: z.array(NoteSchema)\n }, async ({query}, {context}) => {\n if (!context.auth?.uid) throw new Error(\"Must be called by a signed-in user.\");\n return searchUserNotes(context.auth.uid, query);\n });\n ```\n \n \n When using [Dotprompt templates](/docs/dotprompt), context is made available with the\n `@` variable prefix. For example, a context object of\n `{auth: {name: 'Michael'}}` could be accessed in the prompt template like so.\n\n ```dotprompt\n ---\n input:\n schema:\n pirateStyle?: boolean\n ---\n\n {{#if pirateStyle}}Avast, {{@auth.name}}, how be ye today?{{else}}Hello, {{@auth.name}}, how are you today?{{/if}}\n ```\n\n \n\n\n## Provide context at runtime\n\nTo provide context to an action, you pass the context object as an option\nwhen calling the action.\n\n\n \n ```ts\n const summarizeHistory = ai.defineFlow(/* ... */);\n \n const summary = await summarizeHistory(friend.uid, {\n context: { auth: currentUser },\n });\n ```\n \n \n ```ts\n const { text } = await ai.generate({\n prompt: \"Find references to ocelots in my notes.\",\n // the context will propagate to tool calls\n tools: [searchNotes],\n context: { auth: currentUser },\n });\n ```\n \n \n ```ts\n const helloPrompt = ai.prompt(\"sayHello\");\n helloPrompt({ pirateStyle: true }, { context: { auth: currentUser } });\n ```\n \n\n\n## Context propagation and overrides\n\nBy default, when you provide context it is automatically propagated to all\nactions called as a result of your original call. If your flow calls other\nflows, or your generation calls tools, the same context is provided.\n\nIf you wish to override context within an action, you can pass a different\ncontext object to replace the existing one:\n\n```ts\nconst otherFlow = ai.defineFlow(/* ... */);\n\nconst myFlow = ai.defineFlow(\n {\n // ...\n },\n (input, { context }) => {\n // override the existing context completely\n otherFlow(\n {\n /*...*/\n },\n { context: { newContext: true } },\n );\n // or selectively override\n otherFlow(\n {\n /*...*/\n },\n { context: { ...context, updatedContext: true } },\n );\n },\n);\n```\n\nWhen context is replaced, it propagates the same way. In this example,\nany actions that `otherFlow` called during its execution would inherit the\noverridden context.", "title": "Passing information through context", - "lang": "js" + "description": "Learn how Genkit's context object propagates generation and execution information throughout your application, making it available to flows, tools, and prompts.", + "lang": "js", + "headers": "## Why is context important?\n## Context structure\n## Use context in an action\n## Provide context at runtime\n## Context propagation and overrides\n" }, "js/deploy-node.md": { - "text": "Genkit has built-in integrations that help you deploy your flows to\nCloud Functions for Firebase and Google Cloud Run, but you can also deploy your\nflows to any platform that can serve an Express.js app, whether it’s a cloud\nservice or self-hosted.\n\nThis page, as an example, walks you through the process of deploying the default\nsample flow.\n\n## Before you begin\n\n- Node.js 20+: Confirm that your environment is using Node.js version 20 or\n higher (`node --version`).\n- You should be familiar with Genkit's concept of [flows](/docs/flows).\n\n## 1. Set up your project\n\n1. **Create a directory for the project:**\n\n```bash\nexport GENKIT_PROJECT_HOME=~/tmp/genkit-express-project\n\nmkdir -p $GENKIT_PROJECT_HOME\ncd $GENKIT_PROJECT_HOME\nmkdir src\n```\n\n1. **Initialize a Node.js project:**\n\n```bash\nnpm init -y\n```\n\n1. **Install Genkit and necessary dependencies:**\n\n```bash\nnpm install --save genkit @genkit-ai/googleai\nnpm install --save-dev typescript tsx\nnpm install -g genkit-cli\n```\n\n## 2. Configure your Genkit app\n\n1. **Set up a sample flow and server:**\n\nIn `src/index.ts`, define a sample flow and configure the flow server:\n\n```typescript\nimport { genkit, z } from 'genkit';\nimport { googleAI } from '@genkit-ai/googleai';\nimport { startFlowServer } from '@genkit-ai/express';\n\nconst ai = genkit({\n plugins: [googleAI()],\n model: googleAI.model('gemini-2.5-flash'),\n});\n\nconst helloFlow = ai.defineFlow(\n {\n name: 'helloFlow',\n inputSchema: z.object({ name: z.string() }),\n outputSchema: z.object({ greeting: z.string() }),\n },\n async (input) => {\n const { text } = await ai.generate('Say hello to ${input.name}');\n return { greeting: text };\n },\n);\n\nstartFlowServer({\n flows: [helloFlow],\n});\n```\n\nThere are also some optional parameters for `startFlowServer` you can specify:\n\n- `port`: the network port to listen on. If unspecified, the server listens on\n the port defined in the PORT environment variable, and if PORT is not set,\n defaults to 3400.\n- `cors`: the flow server's\n [CORS policy](https://www.npmjs.com/package/cors#configuration-options).\n If you will be accessing these endpoints from a web application, you likely\n need to specify this.\n- `pathPrefix`: an optional path prefix to add before your flow endpoints.\n- `jsonParserOptions`: options to pass to Express's\n [JSON body parser](https://www.npmjs.com/package/body-parser#bodyparserjsonoptions)\n\n1. **Set up model provider credentials:**\n\nConfigure the required environment variables for your model provider. This guide\nuses the Gemini API from Google AI Studio as an example.\n\n[Get an API key from Google AI Studio](https://makersuite.google.com/app/apikey)\n\nAfter you’ve created an API key, set the `GEMINI_API_KEY` environment\nvariable to your key with the following command:\n\n```bash\nexport GEMINI_API_KEY=\n```\n\nDifferent providers for deployment will have different ways of securing your\nAPI key in their environment. For security, ensure that your API key is not\npublicly exposed.\n\n## 3. Prepare your Node.js project for deployment\n\n### Add start and build scripts to `package.json`\n\nTo deploy a Node.js project, define `start` and `build` scripts in\n`package.json`. For a TypeScript project, these scripts will look like this:\n\n```json\n\"scripts\": {\n \"start\": \"node --watch lib/index.js\",\n \"build\": \"tsc\"\n},\n```\n\n### Build and test locally\n\nRun the build command, then start the server and test it locally to confirm it\nworks as expected.\n\n```bash\nnpm run build\n\nnpm start\n```\n\nIn another terminal window, test the endpoint:\n\n```bash\ncurl -X POST \"http://127.0.0.1:3400/helloFlow\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\"data\": {\"name\": \"Genkit\"}}'\n```\n\n## Optional: Start the Developer UI\n\nYou can use the Developer UI to test flows interactively during development:\n\n```bash\ngenkit start -- npm run start\n```\n\nNavigate to `http://localhost:4000/flows` to\ntest your flows in the UI.\n\n## 4. Deploy the project\n\nOnce your project is configured and tested locally, you can deploy to\nany Node.js-compatible platform. Deployment steps vary by provider, but\ngenerally, you configure the following settings:\n\n| Setting | Value |\n| ------------------------- | ---------------------------------------------------------------- |\n| **Runtime** | Node.js 20 or newer |\n| **Build command** | `npm run build` |\n| **Start command** | `npm start` |\n| **Environment variables** | Set `GEMINI_API_KEY=` and other necessary secrets. |\n\nThe `start` command (`npm start`) should point to your compiled entry point,\ntypically `lib/index.js`. Be sure to add all necessary environment variables\nfor your deployment platform.\n\nAfter deploying, you can use the provided service URL to invoke your flow as\nan HTTPS endpoint.", + "text": "# Deploy flows to any Node.js platform\n\nGenkit has built-in integrations that help you deploy your flows to\nCloud Functions for Firebase and Google Cloud Run, but you can also deploy your\nflows to any platform that can serve an Express.js app, whether it’s a cloud\nservice or self-hosted.\n\nThis page, as an example, walks you through the process of deploying the default\nsample flow.\n\n## Before you begin\n\n- Node.js 20+: Confirm that your environment is using Node.js version 20 or\n higher (`node --version`).\n- You should be familiar with Genkit's concept of [flows](/docs/flows).\n\n## 1. Set up your project\n\n1. **Create a directory for the project:**\n\n ```bash\n export GENKIT_PROJECT_HOME=~/tmp/genkit-express-project\n\n mkdir -p $GENKIT_PROJECT_HOME\n cd $GENKIT_PROJECT_HOME\n mkdir src\n ```\n\n1. **Initialize a Node.js project:**\n\n ```bash\n npm init -y\n ```\n\n1. **Install Genkit and necessary dependencies:**\n\n ```bash\n npm install --save genkit @genkit-ai/googleai\n npm install --save-dev typescript tsx\n npm install -g genkit-cli\n ```\n\n## 2. Configure your Genkit app\n\n1. **Set up a sample flow and server:**\n\n In `src/index.ts`, define a sample flow and configure the flow server:\n\n ```typescript\n import { genkit, z } from 'genkit';\n import { googleAI } from '@genkit-ai/googleai';\n import { startFlowServer } from '@genkit-ai/express';\n\n const ai = genkit({\n plugins: [googleAI()],\n model: googleAI.model('gemini-2.5-flash'),\n });\n\n const helloFlow = ai.defineFlow(\n {\n name: 'helloFlow',\n inputSchema: z.object({ name: z.string() }),\n outputSchema: z.object({ greeting: z.string() }),\n },\n async (input) => {\n const { text } = await ai.generate('Say hello to ${input.name}');\n return { greeting: text };\n },\n );\n\n startFlowServer({\n flows: [helloFlow],\n });\n ```\n\n There are also some optional parameters for `startFlowServer` you can specify:\n\n - `port`: the network port to listen on. If unspecified, the server listens on\n the port defined in the PORT environment variable, and if PORT is not set,\n defaults to 3400.\n - `cors`: the flow server's\n [CORS policy](https://www.npmjs.com/package/cors#configuration-options).\n If you will be accessing these endpoints from a web application, you likely\n need to specify this.\n - `pathPrefix`: an optional path prefix to add before your flow endpoints.\n - `jsonParserOptions`: options to pass to Express's\n [JSON body parser](https://www.npmjs.com/package/body-parser#bodyparserjsonoptions)\n\n1. **Set up model provider credentials:**\n\n Configure the required environment variables for your model provider. This guide\n uses the Gemini API from Google AI Studio as an example.\n\n [Get an API key from Google AI Studio](https://makersuite.google.com/app/apikey)\n\n After you’ve created an API key, set the `GEMINI_API_KEY` environment\n variable to your key with the following command:\n\n ```bash\n export GEMINI_API_KEY=\n ```\n\n Different providers for deployment will have different ways of securing your\n API key in their environment. For security, ensure that your API key is not\n publicly exposed.\n\n## 3. Prepare your Node.js project for deployment\n\n### Add start and build scripts to `package.json`\n\nTo deploy a Node.js project, define `start` and `build` scripts in\n`package.json`. For a TypeScript project, these scripts will look like this:\n\n```json\n\"scripts\": {\n \"start\": \"node --watch lib/index.js\",\n \"build\": \"tsc\"\n},\n```\n\n### Build and test locally\n\nRun the build command, then start the server and test it locally to confirm it\nworks as expected.\n\n```bash\nnpm run build\n\nnpm start\n```\n\nIn another terminal window, test the endpoint:\n\n```bash\ncurl -X POST \"http://127.0.0.1:3400/helloFlow\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\"data\": {\"name\": \"Genkit\"}}'\n```\n\n## Optional: Start the Developer UI\n\nYou can use the Developer UI to test flows interactively during development:\n\n```bash\ngenkit start -- npm run start\n```\n\nNavigate to `http://localhost:4000/flows` to\ntest your flows in the UI.\n\n## 4. Deploy the project\n\nOnce your project is configured and tested locally, you can deploy to\nany Node.js-compatible platform. Deployment steps vary by provider, but\ngenerally, you configure the following settings:\n\n| Setting | Value |\n| ------------------------- | ---------------------------------------------------------------- |\n| **Runtime** | Node.js 20 or newer |\n| **Build command** | `npm run build` |\n| **Start command** | `npm start` |\n| **Environment variables** | Set `GEMINI_API_KEY=` and other necessary secrets. |\n\nThe `start` command (`npm start`) should point to your compiled entry point,\ntypically `lib/index.js`. Be sure to add all necessary environment variables\nfor your deployment platform.\n\nAfter deploying, you can use the provided service URL to invoke your flow as\nan HTTPS endpoint.\n\n## Call your flows from the client\n\nIn your client-side code (e.g., a web application, mobile app, or another service), you can call your deployed flows using the Genkit client library. This library provides functions for both non-streaming and streaming flow calls.\n\nFirst, install the Genkit library:\n\n```bash\nnpm install genkit\n```\n\nThen, you can use `runFlow` for non-streaming calls and `streamFlow` for streaming calls.\n\n### Non-streaming Flow Calls\n\nFor a non-streaming response, use the `runFlow` function. This is suitable for flows that return a single, complete output.\n\n```typescript\nimport { runFlow } from 'genkit/beta/client';\n\nasync function callHelloFlow() {\n try {\n const result = await runFlow({\n url: 'http://127.0.0.1:3400/helloFlow', // Replace with your deployed flow's URL\n input: { name: 'Genkit User' },\n });\n console.log('Non-streaming result:', result.greeting);\n } catch (error) {\n console.error('Error calling helloFlow:', error);\n }\n}\n\ncallHelloFlow();\n```\n\n### Streaming Flow Calls\n\nFor flows that are designed to stream responses (e.g., for real-time updates or long-running operations), use the `streamFlow` function.\n\n```typescript\nimport { streamFlow } from 'genkit/beta/client';\n\nasync function streamHelloFlow() {\n try {\n const result = streamFlow({\n url: 'http://127.0.0.1:3400/helloFlow', // Replace with your deployed flow's URL\n input: { name: 'Streaming User' },\n });\n\n // Process the stream chunks as they arrive\n for await (const chunk of result.stream) {\n console.log('Stream chunk:', chunk);\n }\n\n // Get the final complete response\n const finalOutput = await result.output;\n console.log('Final streaming output:', finalOutput.greeting);\n } catch (error) {\n console.error('Error streaming helloFlow:', error);\n }\n}\n\nstreamHelloFlow();\n```\n\n### Authentication (Optional)\n\nIf your deployed flow requires authentication, you can pass headers with your requests:\n\n```typescript\nconst result = await runFlow({\n url: 'http://127.0.0.1:3400/helloFlow', // Replace with your deployed flow's URL\n headers: {\n Authorization: 'Bearer your-token-here', // Replace with your actual token\n },\n input: { name: 'Authenticated User' },\n});\n```\n", "title": "Deploy flows to any Node.js platform", - "lang": "js" + "description": "Learn how to deploy Genkit flows to any Node.js platform that can serve an Express.js application, including project setup, configuration, and client-side access.", + "lang": "js", + "headers": "## Before you begin\n## 1. Set up your project\n## 2. Configure your Genkit app\n## 3. Prepare your Node.js project for deployment\n### Add start and build scripts to `package.json`\n### Build and test locally\n## Optional: Start the Developer UI\n## 4. Deploy the project\n## Call your flows from the client\n### Non-streaming Flow Calls\n### Streaming Flow Calls\n### Authentication (Optional)\n" }, "js/devtools.md": { - "text": "Genkit provides two key developer tools:\n\n- A Node.js CLI for command-line operations\n- An optional local web app, called the Developer UI, that interfaces with your\n Genkit configuration for interactive testing and development\n\n### Command Line Interface (CLI)\n\nInstall the CLI globally using:\n\n```bash\nnpm install -g genkit-cli\n```\n\nThe CLI supports various commands to facilitate working with Genkit projects:\n\n- `genkit start -- `: Start the developer UI and\n connect it to a running code process.\n- `genkit flow:run `: Run a specified flow. Your runtime must already\n be running in a separate terminal with the `GENKIT_ENV=dev` environment\n variable set.\n- `genkit eval:flow `: Evaluate a specific flow. Your runtime must\n already be running in a separate terminal with the `GENKIT_ENV=dev` environment\n variable set.\n\nFor a full list of commands, use:\n\n```bash\ngenkit --help\n```\n\n### Genkit Developer UI\n\nThe Genkit Developer UI is a local web app that lets you interactively\nwork with models, flows, prompts, and other elements in your Genkit project.\n\nThe Developer UI is able to identify what Genkit components you have defined\nin your code by attaching to a running code process.\n\nTo start the UI, run the following command:\n\n```bash\ngenkit start -- \n```\n\nThe `` will vary based on your project's setup and\nthe file you want to execute. Here are some examples:\n\n```bash\n# Running a typical development server\ngenkit start -- npm run dev\n\n# Running a TypeScript file directly\ngenkit start -- npx tsx --watch src/index.ts\n\n# Running a JavaScript file directly\ngenkit start -- node --watch src/index.js\n```\n\nIncluding the `--watch` option will enable the Developer UI to notice and\nreflect saved changes to your code without needing to restart it.\n\nAfter running the command, you will get an output like the following:\n\n```bash\nTelemetry API running on http://localhost:4033\nGenkit Developer UI: http://localhost:4000\n```\n\nOpen the local host address for the Genkit Developer UI in your browser to\nview it. You can also open it in the VS Code simple browser to view it\nalongside your code.\n\nAlternatively, you can use add the `-o` option to the start command to\nautomatically open the Developer UI in your default browser tab.\n\n```\ngenkit start -o -- \n```\n\n![Genkit Developer UI](../../../assets/dev_ui/genkit_dev_ui_home.png)\n\nThe Developer UI has action runners for `flow`, `prompt`, `model`, `tool`,\n`retriever`, `indexer`, `embedder` and `evaluator` based on the components\nyou have defined in your code.\n\nHere's a quick gif tour with cats.\n\n![Genkit Developer UI Overview](/genkit_developer_ui_overview.gif)\n\n### Analytics\n\nThe Genkit CLI and Developer UI use cookies and similar technologies from Google\nto deliver and enhance the quality of its services and to analyze usage.\n[Learn more](https://policies.google.com/technologies/cookies).\n\nTo opt-out of analytics, you can run the following command:\n\n```bash\ngenkit config set analyticsOptOut true\n```\n\nYou can view the current setting by running:\n\n```bash\ngenkit config get analyticsOptOut\n```\n", + "text": "# Genkit Developer Tools\n\nGenkit provides two key developer tools:\n\n- A Node.js CLI for command-line operations\n- An optional local web app, called the Developer UI, that interfaces with your\n Genkit configuration for interactive testing and development\n\n### Command Line Interface (CLI)\n\nInstall the CLI globally using:\n\n```bash\nnpm install -g genkit-cli\n```\n\nThe CLI supports various commands to facilitate working with Genkit projects:\n\n- `genkit start -- `: Start the developer UI and\n connect it to a running code process.\n- `genkit flow:run `: Run a specified flow. Your runtime must already\n be running in a separate terminal with the `GENKIT_ENV=dev` environment\n variable set.\n- `genkit eval:flow `: Evaluate a specific flow. Your runtime must\n already be running in a separate terminal with the `GENKIT_ENV=dev` environment\n variable set.\n\nFor a full list of commands, use:\n\n```bash\ngenkit --help\n```\n\n### Genkit Developer UI\n\nThe Genkit Developer UI is a local web app that lets you interactively\nwork with models, flows, prompts, and other elements in your Genkit project.\n\nThe Developer UI is able to identify what Genkit components you have defined\nin your code by attaching to a running code process.\n\nTo start the UI, run the following command:\n\n```bash\ngenkit start -- \n```\n\nThe `` will vary based on your project's setup and\nthe file you want to execute. Here are some examples:\n\n```bash\n# Running a typical development server\ngenkit start -- npm run dev\n\n# Running a TypeScript file directly\ngenkit start -- npx tsx --watch src/index.ts\n\n# Running a JavaScript file directly\ngenkit start -- node --watch src/index.js\n```\n\nIncluding the `--watch` option will enable the Developer UI to notice and\nreflect saved changes to your code without needing to restart it.\n\nAfter running the command, you will get an output like the following:\n\n```bash\nTelemetry API running on http://localhost:4033\nGenkit Developer UI: http://localhost:4000\n```\n\nOpen the local host address for the Genkit Developer UI in your browser to\nview it. You can also open it in the VS Code simple browser to view it\nalongside your code.\n\nAlternatively, you can use add the `-o` option to the start command to\nautomatically open the Developer UI in your default browser tab.\n\n```\ngenkit start -o -- \n```\n\n![Genkit Developer UI](../../../assets/dev_ui/genkit_dev_ui_home.png)\n\nThe Developer UI has action runners for `flow`, `prompt`, `model`, `tool`,\n`retriever`, `indexer`, `embedder` and `evaluator` based on the components\nyou have defined in your code.\n\nHere's a quick gif tour with cats.\n\n![Genkit Developer UI Overview](/genkit_developer_ui_overview.gif)\n\n### Analytics\n\nThe Genkit CLI and Developer UI use cookies and similar technologies from Google\nto deliver and enhance the quality of its services and to analyze usage.\n[Learn more](https://policies.google.com/technologies/cookies).\n\nTo opt-out of analytics, you can run the following command:\n\n```bash\ngenkit config set analyticsOptOut true\n```\n\nYou can view the current setting by running:\n\n```bash\ngenkit config get analyticsOptOut\n```\n", "title": "Genkit Developer Tools", - "lang": "js" + "description": "Explore Genkit's developer tools, including the Node.js CLI for command-line operations and the local web-based Developer UI for interactive testing and development.", + "lang": "js", + "headers": "### Command Line Interface (CLI)\n### Genkit Developer UI\n# Running a typical development server\n# Running a TypeScript file directly\n# Running a JavaScript file directly\n### Analytics\n" }, "js/dotprompt.md": { - "text": "Prompt engineering is the primary way that you, as an app developer, influence\nthe output of generative AI models. For example, when using LLMs, you can craft\nprompts that influence the tone, format, length, and other characteristics of\nthe models' responses.\n\nThe way you write these prompts will depend on the model you're using; a prompt\nwritten for one model might not perform well when used with another model.\nSimilarly, the model parameters you set (temperature, top-k, and so on) will\nalso affect output differently depending on the model.\n\nGetting all three of these factors—the model, the model parameters, and\nthe prompt—working together to produce the output you want is rarely a\ntrivial process and often involves substantial iteration and experimentation.\nGenkit provides a library and file format called Dotprompt, that aims to make\nthis iteration faster and more convenient.\n\n[Dotprompt](https://github.com/google/dotprompt) is designed around the premise\nthat **prompts are code**. You define your prompts along with the models and\nmodel parameters they're intended for separately from your application code.\nThen, you (or, perhaps someone not even involved with writing application code)\ncan rapidly iterate on the prompts and model parameters using the Genkit\nDeveloper UI. Once your prompts are working the way you want, you can import\nthem into your application and run them using Genkit.\n\nYour prompt definitions each go in a file with a `.prompt` extension. Here's an\nexample of what these files look like:\n\n```dotprompt\n---\nmodel: googleai/gemini-2.5-flash\nconfig:\n temperature: 0.9\ninput:\n schema:\n location: string\n style?: string\n name?: string\n default:\n location: a restaurant\n---\n\nYou are the world's most welcoming AI assistant and are currently working at {{location}}.\n\nGreet a guest{{#if name}} named {{name}}{{/if}}{{#if style}} in the style of {{style}}{{/if}}.\n```\n\nThe portion in the triple-dashes is YAML front matter, similar to the front\nmatter format used by GitHub Markdown and Jekyll; the rest of the file is the\nprompt, which can optionally use Handlebars\n templates. The following sections will go into more detail about each of\nthe parts that make a `.prompt` file and how to use them.\n\n## Before you begin\n\nBefore reading this page, you should be familiar with the content covered on the\n[Generating content with AI models](/docs/models) page.\n\nIf you want to run the code examples on this page, first complete the steps in\nthe [Get started](/docs/get-started) guide. All of the examples assume that you\nhave already installed Genkit as a dependency in your project.\n\n## Creating prompt files\n\nAlthough Dotprompt provides several [different ways](#defining-prompts-in-code) to create\nand load prompts, it's optimized for projects that organize their prompts as\n`.prompt` files within a single directory (or subdirectories thereof). This\nsection shows you how to create and load prompts using this recommended setup.\n\n### Creating a prompt directory\n\nThe Dotprompt library expects to find your prompts in a directory at your\nproject root and automatically loads any prompts it finds there. By default,\nthis directory is named `prompts`. For example, using the default directory\nname, your project structure might look something like this:\n\n```\nyour-project/\n├── lib/\n├── node_modules/\n├── prompts/\n│ └── hello.prompt\n├── src/\n├── package-lock.json\n├── package.json\n└── tsconfig.json\n```\n\nIf you want to use a different directory, you can specify it when you configure\nGenkit:\n\n```ts\nconst ai = genkit({\n promptDir: './llm_prompts',\n // (Other settings...)\n});\n```\n\n### Creating a prompt file\n\nThere are two ways to create a `.prompt` file: using a text editor, or with the\ndeveloper UI.\n\n#### Using a text editor\n\nIf you want to create a prompt file using a text editor, create a text file with\nthe `.prompt` extension in your prompts directory: for example,\n`prompts/hello.prompt`.\n\nHere is a minimal example of a prompt file:\n\n```dotprompt\n---\nmodel: vertexai/gemini-2.5-flash\n---\nYou are the world's most welcoming AI assistant. Greet the user and offer your assistance.\n```\n\nThe portion in the dashes is YAML front matter, similar to the front matter\nformat used by GitHub markdown and Jekyll; the rest of the file is the prompt,\nwhich can optionally use Handlebars templates. The front matter section is\noptional, but most prompt files will at least contain metadata specifying a\nmodel. The remainder of this page shows you how to go beyond this, and make use\nof Dotprompt's features in your prompt files.\n\n#### Using the developer UI\n\nYou can also create a prompt file using the model runner in the developer UI.\nStart with application code that imports the Genkit library and configures it to\nuse the model plugin you're interested in. For example:\n\n```ts\nimport { genkit } from 'genkit';\n\n// Import the model plugins you want to use.\nimport { googleAI } from '@genkit-ai/googleai';\n\nconst ai = genkit({\n // Initialize and configure the model plugins.\n plugins: [\n googleAI({\n apiKey: 'your-api-key', // Or (preferred): export GEMINI_API_KEY=...\n }),\n ],\n});\n```\n\nIt's okay if the file contains other code, but the above is all that's required.\n\nLoad the developer UI in the same project:\n\n```bash\ngenkit start -- tsx --watch src/your-code.ts\n```\n\nIn the Models section, choose the model you want to use from the list of models\nprovided by the plugin.\n\n![Genkit Developer UI Model Runner](../../../assets/developer_ui_model_runner.png)\n\nThen, experiment with the prompt and configuration until you get results you're\nhappy with. When you're ready, press the Export button and save the file to your\nprompts directory.\n\n## Running prompts\n\nAfter you've created prompt files, you can run them from your application code,\nor using the tooling provided by Genkit. Regardless of how you want to run your\nprompts, first start with application code that imports the Genkit library and\nthe model plugins you're interested in. For example:\n\n```ts\nimport { genkit } from 'genkit';\n\n// Import the model plugins you want to use.\nimport { googleAI } from '@genkit-ai/googleai';\n\nconst ai = genkit({\n // Initialize and configure the model plugins.\n plugins: [\n googleAI({\n apiKey: 'your-api-key', // Or (preferred): export GEMINI_API_KEY=...\n }),\n ],\n});\n```\n\nIt's okay if the file contains other code, but the above is all that's required.\nIf you're storing your prompts in a directory other than the default, be sure to\nspecify it when you configure Genkit.\n\n### Run prompts from code\n\nTo use a prompt, first load it using the `prompt('file_name')` method:\n\n```ts\nconst helloPrompt = ai.prompt('hello');\n```\n\nOnce loaded, you can call the prompt like a function:\n\n```ts\nconst response = await helloPrompt();\n\n// Alternatively, use destructuring assignments to get only the properties\n// you're interested in:\nconst { text } = await helloPrompt();\n```\n\nOr you can also run the prompt in streaming mode:\n\n```ts\nconst { response, stream } = helloPrompt.stream();\n\nfor await (const chunk of stream) {\n console.log(chunk.text);\n}\n// optional final (aggregated) response\nconsole.log((await response).text);\n```\n\nA callable prompt takes two optional parameters: the input to the prompt (see\nthe section below on [specifying input schemas](#input-and-output-schemas)), and a configuration\nobject, similar to that of the `generate()` method. For example:\n\n```ts\nconst response2 = await helloPrompt(\n // Prompt input:\n { name: 'Ted' },\n\n // Generation options:\n {\n config: {\n temperature: 0.4,\n },\n },\n);\n```\n\nSimilarly for streaming:\n\n```ts\nconst { stream } = helloPrompt.stream(input, options);\n``` \n\nAny parameters you pass to the prompt call will override the same parameters\nspecified in the prompt file.\n\nSee [Generate content with AI models](/docs/models) for descriptions of the available\noptions.\n\n### Using the developer UI\n\nAs you're refining your app's prompts, you can run them in the Genkit developer\nUI to quickly iterate on prompts and model configurations, independently from\nyour application code.\n\nLoad the developer UI from your project directory:\n\n```bash\ngenkit start -- tsx --watch src/your-code.ts\n```\n\n![Genkit Developer UI Model Runner](../../../assets/prompts-in-developer-ui.png)\n\nOnce you've loaded prompts into the developer UI, you can run them with\ndifferent input values, and experiment with how changes to the prompt wording or\nthe configuration parameters affect the model output. When you're happy with the\nresult, you can click the **Export prompt** button to save the modified prompt\nback into your project directory.\n\n## Model configuration\n\nIn the front matter block of your prompt files, you can optionally specify model\nconfiguration values for your prompt:\n\n```dotprompt\n---\nmodel: googleai/gemini-2.5-flash\nconfig:\n temperature: 1.4\n topK: 50\n topP: 0.4\n maxOutputTokens: 400\n stopSequences:\n - \"\"\n - \"\"\n---\n```\n\nThese values map directly to the `config` parameter accepted by the callable\nprompt:\n\n```ts\nconst response3 = await helloPrompt(\n {},\n {\n config: {\n temperature: 1.4,\n topK: 50,\n topP: 0.4,\n maxOutputTokens: 400,\n stopSequences: ['', ''],\n },\n },\n);\n```\n\nSee [Generate content with AI models](/docs/models) for descriptions of the available\noptions.\n\n## Input and output schemas\n\nYou can specify input and output schemas for your prompt by defining them in the\nfront matter section:\n\n```dotprompt\n---\nmodel: googleai/gemini-2.5-flash\ninput:\n schema:\n theme?: string\n default:\n theme: \"pirate\"\noutput:\n schema:\n dishname: string\n description: string\n calories: integer\n allergens(array): string\n---\nInvent a menu item for a {{theme}} themed restaurant.\n```\n\nThese schemas are used in much the same way as those passed to a `generate()`\nrequest or a flow definition. For example, the prompt defined above produces\nstructured output:\n\n```ts\nconst menuPrompt = ai.prompt('menu');\nconst { output } = await menuPrompt({ theme: 'medieval' });\n\nconst dishName = output['dishname'];\nconst description = output['description'];\n```\n\nYou have several options for defining schemas in a `.prompt` file: Dotprompt's\nown schema definition format, Picoschema; standard JSON Schema; or, as\nreferences to schemas defined in your application code. The following sections\ndescribe each of these options in more detail.\n\n### Picoschema\n\nThe schemas in the example above are defined in a format called Picoschema.\nPicoschema is a compact, YAML-optimized schema definition format that makes it\neasy to define the most important attributes of a schema for LLM usage. Here's a\nlonger example of a schema, which specifies the information an app might store\nabout an article:\n\n```yaml\nschema:\n title: string # string, number, and boolean types are defined like this\n subtitle?: string # optional fields are marked with a `?`\n draft?: boolean, true when in draft state\n status?(enum, approval status): [PENDING, APPROVED]\n date: string, the date of publication e.g. '2024-04-09' # descriptions follow a comma\n tags(array, relevant tags for article): string # arrays are denoted via parentheses\n authors(array):\n name: string\n email?: string\n metadata?(object): # objects are also denoted via parentheses\n updatedAt?: string, ISO timestamp of last update\n approvedBy?: integer, id of approver\n extra?: any, arbitrary extra data\n (*): string, wildcard field\n```\n\nThe above schema is equivalent to the following TypeScript interface:\n\n```ts\ninterface Article {\n title: string;\n subtitle?: string | null;\n /** true when in draft state */\n draft?: boolean | null;\n /** approval status */\n status?: 'PENDING' | 'APPROVED' | null;\n /** the date of publication e.g. '2024-04-09' */\n date: string;\n /** relevant tags for article */\n tags: string[];\n authors: {\n name: string;\n email?: string | null;\n }[];\n metadata?: {\n /** ISO timestamp of last update */\n updatedAt?: string | null;\n /** id of approver */\n approvedBy?: number | null;\n } | null;\n /** arbitrary extra data */\n extra?: any;\n /** wildcard field */\n}\n```\n\nPicoschema supports scalar types `string`, `integer`, `number`, `boolean`, and\n`any`. Objects, arrays, and enums are denoted by a parenthetical after the field\nname.\n\nObjects defined by Picoschema have all properties required unless denoted\noptional by `?`, and do not allow additional properties. When a property is\nmarked as optional, it is also made nullable to provide more leniency for LLMs\nto return null instead of omitting a field.\n\nIn an object definition, the special key `(*)` can be used to declare a\n\"wildcard\" field definition. This will match any additional properties not\nsupplied by an explicit key.\n\n### JSON Schema\n\nPicoschema does not support many of the capabilities of full JSON schema. If you\nrequire more robust schemas, you may supply a JSON Schema instead:\n\n```yaml\noutput:\n schema:\n type: object\n properties:\n field1:\n type: number\n minimum: 20\n```\n\n### Zod schemas defined in code\n\nIn addition to directly defining schemas in the `.prompt` file, you can\nreference a schema registered with `defineSchema()` by name. If you're using\nTypeScript, this approach will let you take advantage of the language's static\ntype checking features when you work with prompts.\n\nTo register a schema:\n\n```ts\nimport { z } from 'genkit';\n\nconst MenuItemSchema = ai.defineSchema(\n 'MenuItemSchema',\n z.object({\n dishname: z.string(),\n description: z.string(),\n calories: z.coerce.number(),\n allergens: z.array(z.string()),\n }),\n);\n```\n\nWithin your prompt, provide the name of the registered schema:\n\n```dotprompt\n---\nmodel: googleai/gemini-2.5-flash-latest\noutput:\n schema: MenuItemSchema\n---\n```\n\nThe Dotprompt library will automatically resolve the name to the underlying\nregistered Zod schema. You can then utilize the schema to strongly type the\noutput of a Dotprompt:\n\n```ts\nconst menuPrompt = ai.prompt<\n z.ZodTypeAny, // Input schema\n typeof MenuItemSchema, // Output schema\n z.ZodTypeAny // Custom options schema\n>('menu');\nconst { output } = await menuPrompt({ theme: 'medieval' });\n\n// Now data is strongly typed as MenuItemSchema:\nconst dishName = output?.dishname;\nconst description = output?.description;\n```\n\n## Prompt templates\n\nThe portion of a `.prompt` file that follows the front matter (if present) is\nthe prompt itself, which will be passed to the model. While this prompt could be\na simple text string, very often you will want to incorporate user input into\nthe prompt. To do so, you can specify your prompt using the\nHandlebars templating language.\nPrompt templates can include placeholders that refer to the values defined by\nyour prompt's input schema.\n\nYou already saw this in action in the section on input and output schemas:\n\n```dotprompt\n---\nmodel: googleai/gemini-2.5-flash\ninput:\n schema:\n theme?: string\n default:\n theme: \"pirate\"\noutput:\n schema:\n dishname: string\n description: string\n calories: integer\n allergens(array): string\n---\nInvent a menu item for a {{theme}} themed restaurant.\n```\n\nIn this example, the Handlebars expression, `{{theme}}`,\nresolves to the value of the input's `theme` property when you run the\nprompt. To pass input to the prompt, call the prompt as in the following\nexample:\n\n```ts\nconst menuPrompt = ai.prompt('menu');\nconst { output } = await menuPrompt({ theme: 'medieval' });\n```\n\nNote that because the input schema declared the `theme` property to be optional\nand provided a default, you could have omitted the property,\nand the prompt would have resolved using the default value.\n\nHandlebars templates also support some limited logical constructs. For example,\nas an alternative to providing a default, you could define the prompt using\nHandlebars's `#if` helper:\n\n```dotprompt\n---\nmodel: googleai/gemini-2.5-flash\ninput:\n schema:\n theme?: string\n---\nInvent a menu item for a {{#if theme}}{{theme}} themed{{/if}} restaurant.\n```\n\nIn this example, the prompt renders as \"Invent a menu item for a restaurant\"\nwhen the `theme` property is unspecified.\n\nSee the Handlebars\ndocumentation for information on all of the built-in logical helpers.\n\nIn addition to properties defined by your input schema, your templates can also\nrefer to values automatically defined by Genkit. The next few sections describe\nthese automatically-defined values and how you can use them.\n\n### Multi-message prompts\n\nBy default, Dotprompt constructs a single message with a \"user\" role.\nHowever, some prompts are best expressed as a combination of multiple\nmessages, such as a system prompt.\n\nThe `{{role}}` helper provides a simple way to\nconstruct multi-message prompts:\n\n```dotprompt\n---\nmodel: vertexai/gemini-2.5-flash\ninput:\n schema:\n userQuestion: string\n---\n{{role \"system\"}}\nYou are a helpful AI assistant that really loves to talk about food. Try to work\nfood items into all of your conversations.\n{{role \"user\"}}\n{{userQuestion}}\n```\n\n### Multi-modal prompts\n\nFor models that support multimodal input, such as images alongside text, you can\nuse the `{{media}}` helper:\n\n```dotprompt\n---\nmodel: vertexai/gemini-2.5-flash\ninput:\n schema:\n photoUrl: string\n---\nDescribe this image in a detailed paragraph:\n\n{{media url=photoUrl}}\n```\n\nThe URL can be `https:` or base64-encoded `data:` URIs for \"inline\" image usage.\nIn code, this would be:\n\n```ts\nconst multimodalPrompt = ai.prompt('multimodal');\nconst { text } = await multimodalPrompt({\n photoUrl: 'https://example.com/photo.jpg',\n});\n```\n\nSee also [Multimodal input](/docs/models#multimodal-input), on the Models\npage, for an example of constructing a `data:` URL.\n\n### Partials\n\nPartials are reusable templates that can be included inside any prompt. Partials\ncan be especially helpful for related prompts that share common behavior.\n\nWhen loading a prompt directory, any file prefixed with an underscore (`_`) is\nconsidered a partial. So a file `_personality.prompt` might contain:\n\n```dotprompt\nYou should speak like a {{#if style}}{{style}}{{else}}helpful assistant.{{/if}}.\n```\n\nThis can then be included in other prompts:\n\n```dotprompt\n---\nmodel: googleai/gemini-2.5-flash\ninput:\n schema:\n name: string\n style?: string\n---\n\n{{role \"system\"}}\n{{>personality style=style}}\n\n{{role \"user\"}}\nGive the user a friendly greeting.\n\nUser's Name: {{name}}\n```\n\nPartials are inserted using the\n`{{>NAME_OF_PARTIAL args...}}`\nsyntax. If no arguments are provided to the partial, it executes with the same\ncontext as the parent prompt.\n\nPartials accept both named arguments as above or a single positional argument\nrepresenting the context. This can be helpful for tasks such as rendering\nmembers of a list.\n\n**\\_destination.prompt**\n\n```dotprompt\n- {{name}} ({{country}})\n```\n\n**chooseDestination.prompt**\n\n```dotprompt\n---\nmodel: googleai/gemini-2.5-flash\ninput:\n schema:\n destinations(array):\n name: string\n country: string\n---\nHelp the user decide between these vacation destinations:\n\n{{#each destinations}}\n{{>destination this}}\n{{/each}}\n```\n\n#### Defining partials in code\n\nYou can also define partials in code using `definePartial`:\n\n```ts\nai.definePartial('personality', 'Talk like a {{#if style}}{{style}}{{else}}helpful assistant{{/if}}.');\n```\n\nCode-defined partials are available in all prompts.\n\n### Defining Custom Helpers\n\nYou can define custom helpers to process and manage data inside of a prompt.\nHelpers are registered globally using `defineHelper`:\n\n```ts\nai.defineHelper('shout', (text: string) => text.toUpperCase());\n```\n\nOnce a helper is defined you can use it in any prompt:\n\n```dotprompt\n---\nmodel: googleai/gemini-2.5-flash\ninput:\n schema:\n name: string\n---\n\nHELLO, {{shout name}}!!!\n```\n\n## Prompt variants\n\nBecause prompt files are just text, you can (and should!) commit them to your\nversion control system, allowing you to compare changes over time easily. Often,\ntweaked versions of prompts can only be fully tested in a production environment\nside-by-side with existing versions. Dotprompt supports this through its\nvariants feature.\n\nTo create a variant, create a `[name].[variant].prompt` file. For instance, if\nyou were using Gemini 2.0 Flash in your prompt but wanted to see if Gemini 2.5\nPro would perform better, you might create two files:\n\n- `my_prompt.prompt`: the \"baseline\" prompt\n- `my_prompt.gemini25pro.prompt`: a variant named `gemini25pro`\n\nTo use a prompt variant, specify the variant option when loading:\n\n```ts\nconst myPrompt = ai.prompt('my_prompt', { variant: 'gemini25pro' });\n```\n\nThe name of the variant is included in the metadata of generation traces, so you\ncan compare and contrast actual performance between variants in the Genkit trace\ninspector.\n\n## Defining prompts in code\n\nAll of the examples discussed so far have assumed that your prompts are defined\nin individual `.prompt` files in a single directory (or subdirectories thereof),\naccessible to your app at runtime. Dotprompt is designed around this setup, and\nits authors consider it to be the best developer experience overall.\n\nHowever, if you have use cases that are not well supported by this setup,\nyou can also define prompts in code using the `definePrompt()` function:\n\nThe first parameter to this function is analogous to the front matter block of a\n`.prompt` file; the second parameter can either be a Handlebars template string,\nas in a prompt file, or a function that returns a `GenerateRequest`:\n\n```ts\nconst myPrompt = ai.definePrompt({\n name: 'myPrompt',\n model: 'googleai/gemini-2.5-flash',\n input: {\n schema: z.object({\n name: z.string(),\n }),\n },\n prompt: 'Hello, {{name}}. How are you today?',\n});\n```\n\n```ts\nconst myPrompt = ai.definePrompt({\n name: 'myPrompt',\n model: 'googleai/gemini-2.5-flash',\n input: {\n schema: z.object({\n name: z.string(),\n }),\n },\n messages: async (input) => {\n return [\n {\n role: 'user',\n content: [{ text: `Hello, ${input.name}. How are you today?` }],\n },\n ];\n },\n});\n```\n", + "text": "# Managing prompts with Dotprompt\n\nPrompt engineering is the primary way that you, as an app developer, influence\nthe output of generative AI models. For example, when using LLMs, you can craft\nprompts that influence the tone, format, length, and other characteristics of\nthe models' responses.\n\nThe way you write these prompts will depend on the model you're using; a prompt\nwritten for one model might not perform well when used with another model.\nSimilarly, the model parameters you set (temperature, top-k, and so on) will\nalso affect output differently depending on the model.\n\nGetting all three of these factors—the model, the model parameters, and\nthe prompt—working together to produce the output you want is rarely a\ntrivial process and often involves substantial iteration and experimentation.\nGenkit provides a library and file format called Dotprompt, that aims to make\nthis iteration faster and more convenient.\n\n[Dotprompt](https://github.com/google/dotprompt) is designed around the premise\nthat **prompts are code**. You define your prompts along with the models and\nmodel parameters they're intended for separately from your application code.\nThen, you (or, perhaps someone not even involved with writing application code)\ncan rapidly iterate on the prompts and model parameters using the Genkit\nDeveloper UI. Once your prompts are working the way you want, you can import\nthem into your application and run them using Genkit.\n\nYour prompt definitions each go in a file with a `.prompt` extension. Here's an\nexample of what these files look like:\n\n```dotprompt\n---\nmodel: googleai/gemini-2.5-flash\nconfig:\n temperature: 0.9\ninput:\n schema:\n location: string\n style?: string\n name?: string\n default:\n location: a restaurant\n---\n\nYou are the world's most welcoming AI assistant and are currently working at {{location}}.\n\nGreet a guest{{#if name}} named {{name}}{{/if}}{{#if style}} in the style of {{style}}{{/if}}.\n```\n\nThe portion in the triple-dashes is YAML front matter, similar to the front\nmatter format used by GitHub Markdown and Jekyll; the rest of the file is the\nprompt, which can optionally use Handlebars\n templates. The following sections will go into more detail about each of\nthe parts that make a `.prompt` file and how to use them.\n\n## Before you begin\n\nBefore reading this page, you should be familiar with the content covered on the\n[Generating content with AI models](/docs/models) page.\n\nIf you want to run the code examples on this page, first complete the steps in\nthe [Get started](/docs/get-started) guide. All of the examples assume that you\nhave already installed Genkit as a dependency in your project.\n\n## Creating prompt files\n\nAlthough Dotprompt provides several [different ways](#defining-prompts-in-code) to create\nand load prompts, it's optimized for projects that organize their prompts as\n`.prompt` files within a single directory (or subdirectories thereof). This\nsection shows you how to create and load prompts using this recommended setup.\n\n### Creating a prompt directory\n\nThe Dotprompt library expects to find your prompts in a directory at your\nproject root and automatically loads any prompts it finds there. By default,\nthis directory is named `prompts`. For example, using the default directory\nname, your project structure might look something like this:\n\n```\nyour-project/\n├── lib/\n├── node_modules/\n├── prompts/\n│ └── hello.prompt\n├── src/\n├── package-lock.json\n├── package.json\n└── tsconfig.json\n```\n\nIf you want to use a different directory, you can specify it when you configure\nGenkit:\n\n```ts\nconst ai = genkit({\n promptDir: './llm_prompts',\n // (Other settings...)\n});\n```\n\n### Creating a prompt file\n\nThere are two ways to create a `.prompt` file: using a text editor, or with the\ndeveloper UI.\n\n#### Using a text editor\n\nIf you want to create a prompt file using a text editor, create a text file with\nthe `.prompt` extension in your prompts directory: for example,\n`prompts/hello.prompt`.\n\nHere is a minimal example of a prompt file:\n\n```dotprompt\n---\nmodel: vertexai/gemini-2.5-flash\n---\nYou are the world's most welcoming AI assistant. Greet the user and offer your assistance.\n```\n\nThe portion in the dashes is YAML front matter, similar to the front matter\nformat used by GitHub markdown and Jekyll; the rest of the file is the prompt,\nwhich can optionally use Handlebars templates. The front matter section is\noptional, but most prompt files will at least contain metadata specifying a\nmodel. The remainder of this page shows you how to go beyond this, and make use\nof Dotprompt's features in your prompt files.\n\n#### Using the developer UI\n\nYou can also create a prompt file using the model runner in the developer UI.\nStart with application code that imports the Genkit library and configures it to\nuse the model plugin you're interested in. For example:\n\n```ts\nimport { genkit } from 'genkit';\n\n// Import the model plugins you want to use.\nimport { googleAI } from '@genkit-ai/googleai';\n\nconst ai = genkit({\n // Initialize and configure the model plugins.\n plugins: [\n googleAI({\n apiKey: 'your-api-key', // Or (preferred): export GEMINI_API_KEY=...\n }),\n ],\n});\n```\n\nIt's okay if the file contains other code, but the above is all that's required.\n\nLoad the developer UI in the same project:\n\n```bash\ngenkit start -- tsx --watch src/your-code.ts\n```\n\nIn the Models section, choose the model you want to use from the list of models\nprovided by the plugin.\n\n![Genkit Developer UI Model Runner](../../../assets/developer_ui_model_runner.png)\n\nThen, experiment with the prompt and configuration until you get results you're\nhappy with. When you're ready, press the Export button and save the file to your\nprompts directory.\n\n## Running prompts\n\nAfter you've created prompt files, you can run them from your application code,\nor using the tooling provided by Genkit. Regardless of how you want to run your\nprompts, first start with application code that imports the Genkit library and\nthe model plugins you're interested in. For example:\n\n```ts\nimport { genkit } from 'genkit';\n\n// Import the model plugins you want to use.\nimport { googleAI } from '@genkit-ai/googleai';\n\nconst ai = genkit({\n // Initialize and configure the model plugins.\n plugins: [\n googleAI({\n apiKey: 'your-api-key', // Or (preferred): export GEMINI_API_KEY=...\n }),\n ],\n});\n```\n\nIt's okay if the file contains other code, but the above is all that's required.\nIf you're storing your prompts in a directory other than the default, be sure to\nspecify it when you configure Genkit.\n\n### Run prompts from code\n\nTo use a prompt, first load it using the `prompt('file_name')` method:\n\n```ts\nconst helloPrompt = ai.prompt('hello');\n```\n\nOnce loaded, you can call the prompt like a function:\n\n```ts\nconst response = await helloPrompt();\n\n// Alternatively, use destructuring assignments to get only the properties\n// you're interested in:\nconst { text } = await helloPrompt();\n```\n\nOr you can also run the prompt in streaming mode:\n\n```ts\nconst { response, stream } = helloPrompt.stream();\n\nfor await (const chunk of stream) {\n console.log(chunk.text);\n}\n// optional final (aggregated) response\nconsole.log((await response).text);\n```\n\nA callable prompt takes two optional parameters: the input to the prompt (see\nthe section below on [specifying input schemas](#input-and-output-schemas)), and a configuration\nobject, similar to that of the `generate()` method. For example:\n\n```ts\nconst response2 = await helloPrompt(\n // Prompt input:\n { name: 'Ted' },\n\n // Generation options:\n {\n config: {\n temperature: 0.4,\n },\n },\n);\n```\n\nSimilarly for streaming:\n\n```ts\nconst { stream } = helloPrompt.stream(input, options);\n```\n\nAny parameters you pass to the prompt call will override the same parameters\nspecified in the prompt file.\n\nSee [Generate content with AI models](/docs/models) for descriptions of the available\noptions.\n\n### Using the developer UI\n\nAs you're refining your app's prompts, you can run them in the Genkit developer\nUI to quickly iterate on prompts and model configurations, independently from\nyour application code.\n\nLoad the developer UI from your project directory:\n\n```bash\ngenkit start -- tsx --watch src/your-code.ts\n```\n\n![Genkit Developer UI Model Runner](../../../assets/prompts-in-developer-ui.png)\n\nOnce you've loaded prompts into the developer UI, you can run them with\ndifferent input values, and experiment with how changes to the prompt wording or\nthe configuration parameters affect the model output. When you're happy with the\nresult, you can click the **Export prompt** button to save the modified prompt\nback into your project directory.\n\n## Model configuration\n\nIn the front matter block of your prompt files, you can optionally specify model\nconfiguration values for your prompt:\n\n```dotprompt\n---\nmodel: googleai/gemini-2.5-flash\nconfig:\n temperature: 1.4\n topK: 50\n topP: 0.4\n maxOutputTokens: 400\n stopSequences:\n - \"\"\n - \"\"\n---\n```\n\nThese values map directly to the `config` parameter accepted by the callable\nprompt:\n\n```ts\nconst response3 = await helloPrompt(\n {},\n {\n config: {\n temperature: 1.4,\n topK: 50,\n topP: 0.4,\n maxOutputTokens: 400,\n stopSequences: ['', ''],\n },\n },\n);\n```\n\nSee [Generate content with AI models](/docs/models) for descriptions of the available\noptions.\n\n## Input and output schemas\n\nYou can specify input and output schemas for your prompt by defining them in the\nfront matter section:\n\n```dotprompt\n---\nmodel: googleai/gemini-2.5-flash\ninput:\n schema:\n theme?: string\n default:\n theme: \"pirate\"\noutput:\n schema:\n dishname: string\n description: string\n calories: integer\n allergens(array): string\n---\nInvent a menu item for a {{theme}} themed restaurant.\n```\n\nThese schemas are used in much the same way as those passed to a `generate()`\nrequest or a flow definition. For example, the prompt defined above produces\nstructured output:\n\n```ts\nconst menuPrompt = ai.prompt('menu');\nconst { output } = await menuPrompt({ theme: 'medieval' });\n\nconst dishName = output['dishname'];\nconst description = output['description'];\n```\n\nYou have several options for defining schemas in a `.prompt` file: Dotprompt's\nown schema definition format, Picoschema; standard JSON Schema; or, as\nreferences to schemas defined in your application code. The following sections\ndescribe each of these options in more detail.\n\n### Picoschema\n\nThe schemas in the example above are defined in a format called Picoschema.\nPicoschema is a compact, YAML-optimized schema definition format that makes it\neasy to define the most important attributes of a schema for LLM usage. Here's a\nlonger example of a schema, which specifies the information an app might store\nabout an article:\n\n```yaml\nschema:\n title: string # string, number, and boolean types are defined like this\n subtitle?: string # optional fields are marked with a `?`\n draft?: boolean, true when in draft state\n status?(enum, approval status): [PENDING, APPROVED]\n date: string, the date of publication e.g. '2024-04-09' # descriptions follow a comma\n tags(array, relevant tags for article): string # arrays are denoted via parentheses\n authors(array):\n name: string\n email?: string\n metadata?(object): # objects are also denoted via parentheses\n updatedAt?: string, ISO timestamp of last update\n approvedBy?: integer, id of approver\n extra?: any, arbitrary extra data\n (*): string, wildcard field\n```\n\nThe above schema is equivalent to the following TypeScript interface:\n\n```ts\ninterface Article {\n title: string;\n subtitle?: string | null;\n /** true when in draft state */\n draft?: boolean | null;\n /** approval status */\n status?: 'PENDING' | 'APPROVED' | null;\n /** the date of publication e.g. '2024-04-09' */\n date: string;\n /** relevant tags for article */\n tags: string[];\n authors: {\n name: string;\n email?: string | null;\n }[];\n metadata?: {\n /** ISO timestamp of last update */\n updatedAt?: string | null;\n /** id of approver */\n approvedBy?: number | null;\n } | null;\n /** arbitrary extra data */\n extra?: any;\n /** wildcard field */\n}\n```\n\nPicoschema supports scalar types `string`, `integer`, `number`, `boolean`, and\n`any`. Objects, arrays, and enums are denoted by a parenthetical after the field\nname.\n\nObjects defined by Picoschema have all properties required unless denoted\noptional by `?`, and do not allow additional properties. When a property is\nmarked as optional, it is also made nullable to provide more leniency for LLMs\nto return null instead of omitting a field.\n\nIn an object definition, the special key `(*)` can be used to declare a\n\"wildcard\" field definition. This will match any additional properties not\nsupplied by an explicit key.\n\n### JSON Schema\n\nPicoschema does not support many of the capabilities of full JSON schema. If you\nrequire more robust schemas, you may supply a JSON Schema instead:\n\n```yaml\noutput:\n schema:\n type: object\n properties:\n field1:\n type: number\n minimum: 20\n```\n\n### Zod schemas defined in code\n\nIn addition to directly defining schemas in the `.prompt` file, you can\nreference a schema registered with `defineSchema()` by name. If you're using\nTypeScript, this approach will let you take advantage of the language's static\ntype checking features when you work with prompts.\n\nTo register a schema:\n\n```ts\nimport { z } from 'genkit';\n\nconst MenuItemSchema = ai.defineSchema(\n 'MenuItemSchema',\n z.object({\n dishname: z.string(),\n description: z.string(),\n calories: z.coerce.number(),\n allergens: z.array(z.string()),\n }),\n);\n```\n\nWithin your prompt, provide the name of the registered schema:\n\n```dotprompt\n---\nmodel: googleai/gemini-2.5-flash-latest\noutput:\n schema: MenuItemSchema\n---\n```\n\nThe Dotprompt library will automatically resolve the name to the underlying\nregistered Zod schema. You can then utilize the schema to strongly type the\noutput of a Dotprompt:\n\n```ts\nconst menuPrompt = ai.prompt<\n z.ZodTypeAny, // Input schema\n typeof MenuItemSchema, // Output schema\n z.ZodTypeAny // Custom options schema\n>('menu');\nconst { output } = await menuPrompt({ theme: 'medieval' });\n\n// Now data is strongly typed as MenuItemSchema:\nconst dishName = output?.dishname;\nconst description = output?.description;\n```\n\n## Prompt templates\n\nThe portion of a `.prompt` file that follows the front matter (if present) is\nthe prompt itself, which will be passed to the model. While this prompt could be\na simple text string, very often you will want to incorporate user input into\nthe prompt. To do so, you can specify your prompt using the\nHandlebars templating language.\nPrompt templates can include placeholders that refer to the values defined by\nyour prompt's input schema.\n\nYou already saw this in action in the section on input and output schemas:\n\n```dotprompt\n---\nmodel: googleai/gemini-2.5-flash\ninput:\n schema:\n theme?: string\n default:\n theme: \"pirate\"\noutput:\n schema:\n dishname: string\n description: string\n calories: integer\n allergens(array): string\n---\nInvent a menu item for a {{theme}} themed restaurant.\n```\n\nIn this example, the Handlebars expression, `{{theme}}`,\nresolves to the value of the input's `theme` property when you run the\nprompt. To pass input to the prompt, call the prompt as in the following\nexample:\n\n```ts\nconst menuPrompt = ai.prompt('menu');\nconst { output } = await menuPrompt({ theme: 'medieval' });\n```\n\nNote that because the input schema declared the `theme` property to be optional\nand provided a default, you could have omitted the property,\nand the prompt would have resolved using the default value.\n\nHandlebars templates also support some limited logical constructs. For example,\nas an alternative to providing a default, you could define the prompt using\nHandlebars's `#if` helper:\n\n```dotprompt\n---\nmodel: googleai/gemini-2.5-flash\ninput:\n schema:\n theme?: string\n---\nInvent a menu item for a {{#if theme}}{{theme}} themed{{/if}} restaurant.\n```\n\nIn this example, the prompt renders as \"Invent a menu item for a restaurant\"\nwhen the `theme` property is unspecified.\n\nSee the Handlebars\ndocumentation for information on all of the built-in logical helpers.\n\nIn addition to properties defined by your input schema, your templates can also\nrefer to values automatically defined by Genkit. The next few sections describe\nthese automatically-defined values and how you can use them.\n\n### Multi-message prompts\n\nBy default, Dotprompt constructs a single message with a \"user\" role.\nHowever, some prompts are best expressed as a combination of multiple\nmessages, such as a system prompt.\n\nThe `{{role}}` helper provides a simple way to\nconstruct multi-message prompts:\n\n```dotprompt\n---\nmodel: vertexai/gemini-2.5-flash\ninput:\n schema:\n userQuestion: string\n---\n{{role \"system\"}}\nYou are a helpful AI assistant that really loves to talk about food. Try to work\nfood items into all of your conversations.\n{{role \"user\"}}\n{{userQuestion}}\n```\n\nNote that your final prompt must contain at least one `user` role.\n\n### Multi-modal prompts\n\nFor models that support multimodal input, such as images alongside text, you can\nuse the `{{media}}` helper:\n\n```dotprompt\n---\nmodel: vertexai/gemini-2.5-flash\ninput:\n schema:\n photoUrl: string\n---\nDescribe this image in a detailed paragraph:\n\n{{media url=photoUrl}}\n```\n\nThe URL can be `https:` or base64-encoded `data:` URIs for \"inline\" image usage.\nIn code, this would be:\n\n```ts\nconst multimodalPrompt = ai.prompt('multimodal');\nconst { text } = await multimodalPrompt({\n photoUrl: 'https://example.com/photo.jpg',\n});\n```\n\nSee also [Multimodal input](/docs/models#multimodal-input), on the Models\npage, for an example of constructing a `data:` URL.\n\n### Partials\n\nPartials are reusable templates that can be included inside any prompt. Partials\ncan be especially helpful for related prompts that share common behavior.\n\nWhen loading a prompt directory, any file prefixed with an underscore (`_`) is\nconsidered a partial. So a file `_personality.prompt` might contain:\n\n```dotprompt\nYou should speak like a {{#if style}}{{style}}{{else}}helpful assistant.{{/if}}.\n```\n\nThis can then be included in other prompts:\n\n```dotprompt\n---\nmodel: googleai/gemini-2.5-flash\ninput:\n schema:\n name: string\n style?: string\n---\n\n{{role \"system\"}}\n{{>personality style=style}}\n\n{{role \"user\"}}\nGive the user a friendly greeting.\n\nUser's Name: {{name}}\n```\n\nPartials are inserted using the\n`{{>NAME_OF_PARTIAL args...}}`\nsyntax. If no arguments are provided to the partial, it executes with the same\ncontext as the parent prompt.\n\nPartials accept both named arguments as above or a single positional argument\nrepresenting the context. This can be helpful for tasks such as rendering\nmembers of a list.\n\n**\\_destination.prompt**\n\n```dotprompt\n- {{name}} ({{country}})\n```\n\n**chooseDestination.prompt**\n\n```dotprompt\n---\nmodel: googleai/gemini-2.5-flash\ninput:\n schema:\n destinations(array):\n name: string\n country: string\n---\nHelp the user decide between these vacation destinations:\n\n{{#each destinations}}\n{{>destination this}}\n{{/each}}\n```\n\n#### Defining partials in code\n\nYou can also define partials in code using `definePartial`:\n\n```ts\nai.definePartial('personality', 'Talk like a {{#if style}}{{style}}{{else}}helpful assistant{{/if}}.');\n```\n\nCode-defined partials are available in all prompts.\n\n### Defining Custom Helpers\n\nYou can define custom helpers to process and manage data inside of a prompt.\nHelpers are registered globally using `defineHelper`:\n\n```ts\nai.defineHelper('shout', (text: string) => text.toUpperCase());\n```\n\nOnce a helper is defined you can use it in any prompt:\n\n```dotprompt\n---\nmodel: googleai/gemini-2.5-flash\ninput:\n schema:\n name: string\n---\n\nHELLO, {{shout name}}!!!\n```\n\n## Prompt variants\n\nBecause prompt files are just text, you can (and should!) commit them to your\nversion control system, allowing you to compare changes over time easily. Often,\ntweaked versions of prompts can only be fully tested in a production environment\nside-by-side with existing versions. Dotprompt supports this through its\nvariants feature.\n\nTo create a variant, create a `[name].[variant].prompt` file. For instance, if\nyou were using Gemini 2.0 Flash in your prompt but wanted to see if Gemini 2.5\nPro would perform better, you might create two files:\n\n- `my_prompt.prompt`: the \"baseline\" prompt\n- `my_prompt.gemini25pro.prompt`: a variant named `gemini25pro`\n\nTo use a prompt variant, specify the variant option when loading:\n\n```ts\nconst myPrompt = ai.prompt('my_prompt', { variant: 'gemini25pro' });\n```\n\nThe name of the variant is included in the metadata of generation traces, so you\ncan compare and contrast actual performance between variants in the Genkit trace\ninspector.\n\n## Defining prompts in code\n\nAll of the examples discussed so far have assumed that your prompts are defined\nin individual `.prompt` files in a single directory (or subdirectories thereof),\naccessible to your app at runtime. Dotprompt is designed around this setup, and\nits authors consider it to be the best developer experience overall.\n\nHowever, if you have use cases that are not well supported by this setup,\nyou can also define prompts in code using the `definePrompt()` function:\n\nThe first parameter to this function is analogous to the front matter block of a\n`.prompt` file; the second parameter can either be a Handlebars template string,\nas in a prompt file, or a function that returns a `GenerateRequest`:\n\n```ts\nconst myPrompt = ai.definePrompt({\n name: 'myPrompt',\n model: 'googleai/gemini-2.5-flash',\n input: {\n schema: z.object({\n name: z.string(),\n }),\n },\n prompt: 'Hello, {{name}}. How are you today?',\n});\n```\n\n```ts\nconst myPrompt = ai.definePrompt({\n name: 'myPrompt',\n model: 'googleai/gemini-2.5-flash',\n input: {\n schema: z.object({\n name: z.string(),\n }),\n },\n messages: async (input) => {\n return [\n {\n role: 'user',\n content: [{ text: `Hello, ${input.name}. How are you today?` }],\n },\n ];\n },\n});\n```\n", "title": "Managing prompts with Dotprompt", - "lang": "js" + "description": "This document explains how to manage prompts using Dotprompt, a Genkit library and file format designed to streamline prompt engineering and iteration.", + "lang": "js", + "headers": "## Before you begin\n## Creating prompt files\n### Creating a prompt directory\n### Creating a prompt file\n#### Using a text editor\n#### Using the developer UI\n## Running prompts\n### Run prompts from code\n### Using the developer UI\n## Model configuration\n## Input and output schemas\n### Picoschema\n### JSON Schema\n### Zod schemas defined in code\n## Prompt templates\n### Multi-message prompts\n### Multi-modal prompts\n### Partials\n#### Defining partials in code\n### Defining Custom Helpers\n## Prompt variants\n## Defining prompts in code\n" }, "js/evaluation.md": { - "text": "Evaluation is a form of testing that helps you validate your LLM's responses and\nensure they meet your quality bar.\n\nGenkit supports third-party evaluation tools through plugins, paired\nwith powerful observability features that provide insight into the runtime state\nof your LLM-powered applications. Genkit tooling helps you automatically extract\ndata including inputs, outputs, and information from intermediate steps to\nevaluate the end-to-end quality of LLM responses as well as understand the\nperformance of your system's building blocks.\n\n### Types of evaluation\n\nGenkit supports two types of evaluation:\n\n- **Inference-based evaluation**: This type of evaluation runs against a\n collection of pre-determined inputs, assessing the corresponding outputs for\n quality.\n\n This is the most common evaluation type, suitable for most use cases. This approach tests a system's actual output for each evaluation run.\n\n You can perform the quality assessment manually, by visually inspecting the results. Alternatively, you can automate the assessment by using an evaluation metric.\n\n- **Raw evaluation**: This type of evaluation directly assesses the quality of\n inputs without any inference. This approach typically is used with automated\n evaluation using metrics. All required fields for evaluation (e.g., `input`,\n `context`, `output` and `reference`) must be present in the input dataset. This\n is useful when you have data coming from an external source (e.g., collected\n from your production traces) and you want to have an objective measurement of\n the quality of the collected data.\n\n For more information, see the [Advanced use](#advanced-use) section of this page.\n\nThis section explains how to perform inference-based evaluation using Genkit.\n\n## Quick start\n\n### Setup\n\n1. Use an existing Genkit app or create a new one by following our [Get started](/docs/get-started) guide.\n2. Add the following code to define a simple RAG application to evaluate. For this guide, we use a dummy retriever that always returns the same documents.\n\n ```js\n import { genkit, z, Document } from 'genkit';\n import { googleAI } from '@genkit-ai/googleai';\n\n // Initialize Genkit\n export const ai = genkit({ plugins: [googleAI()] });\n\n // Dummy retriever that always returns the same docs\n export const dummyRetriever = ai.defineRetriever(\n {\n name: 'dummyRetriever',\n },\n async (i) => {\n const facts = [\"Dog is man's best friend\", 'Dogs have evolved and were domesticated from wolves'];\n // Just return facts as documents.\n return { documents: facts.map((t) => Document.fromText(t)) };\n },\n );\n\n // A simple question-answering flow\n export const qaFlow = ai.defineFlow(\n {\n name: 'qaFlow',\n inputSchema: z.object({ query: z.string() }),\n outputSchema: z.object({ answer: z.string() }),\n },\n async ({ query }) => {\n const factDocs = await ai.retrieve({\n retriever: dummyRetriever,\n query,\n });\n\n const { text } = await ai.generate({\n model: googleAI.model('gemini-2.5-flash'),\n prompt: `Answer this question with the given context ${query}`,\n docs: factDocs,\n });\n return { answer: text };\n },\n );\n ```\n\n3. (Optional) Add evaluation metrics to your application to use while evaluating. This guide uses the `MALICIOUSNESS` metric from the `genkitEval` plugin.\n\n ```js\n import { genkitEval, GenkitMetric } from '@genkit-ai/evaluator';\n import { googleAI } from '@genkit-ai/googleai';\n\n export const ai = genkit({\n plugins: [\n ...// Add this plugin to your Genkit initialization block\n genkitEval({\n judge: googleAI.model('gemini-2.5-flash'),\n metrics: [GenkitMetric.MALICIOUSNESS],\n }),\n ],\n });\n ```\n\n **Note:** The configuration above requires installation of the [`@genkit-ai/evaluator`](https://www.npmjs.com/package/@genkit-ai/evaluator) package.\n\n ```bash\n npm install @genkit-ai/evaluator\n ```\n\n4. Start your Genkit application.\n\n ```bash\n genkit start -- \n ```\n\n### Create a dataset\n\nCreate a dataset to define the examples we want to use for evaluating our flow.\n\n1. Go to the Dev UI at `http://localhost:4000` and click the **Datasets** button\n to open the Datasets page.\n\n2. Click on the **Create Dataset** button to open the create dataset dialog.\n\n a. Provide a `datasetId` for your new dataset. This guide uses\n `myFactsQaDataset`.\n\n b. Select `Flow` dataset type.\n\n c. Leave the validation target field empty and click **Save**\n\n3. Your new dataset page appears, showing an empty dataset. Add examples to it by following these steps:\n\n a. Click the **Add example** button to open the example editor panel.\n\n b. Only the `input` field is required. Enter `{\"query\": \"Who is man's best friend?\"}` in the `input` field, and click **Save** to add the example has to your dataset.\n\n c. Repeat steps (a) and (b) a couple more times to add more examples. This guide adds the following example inputs to the dataset:\n\n ```\n {\"query\": \"Can I give milk to my cats?\"}\n {\"query\": \"From which animals did dogs evolve?\"}\n ```\n\nBy the end of this step, your dataset should have 3 examples in it, with the\nvalues mentioned above.\n\n### Run evaluation and view results\n\nTo start evaluating the flow, click the **Run new evaluation** button on your\ndataset page. You can also start a new evaluation from the _Evaluations_ tab.\n\n1. Select the `Flow` radio button to evaluate a flow.\n\n2. Select `qaFlow` as the target flow to evaluate.\n\n3. Select `myFactsQaDataset` as the target dataset to use for evaluation.\n\n4. (Optional) If you have installed an evaluator metric using Genkit plugins,\n you can see these metrics in this page. Select the metrics that you want to use\n with this evaluation run. This is entirely optional: Omitting this step will\n still return the results in the evaluation run, but without any associated\n metrics.\n\n5. Finally, click **Run evaluation** to start evaluation. Depending on the flow\n you're testing, this may take a while. Once the evaluation is complete, a\n success message appears with a link to view the results. Click on the link to go\n to the _Evaluation details_ page.\n\nYou can see the details of your evaluation on this page, including original\ninput, extracted context and metrics (if any).\n\n## Core concepts\n\n### Terminology\n\n- **Evaluation**: An evaluation is a process that assesses system performance. In Genkit, such a system is usually a Genkit primitive, such as a flow or a\n model. An evaluation can be automated or manual (human evaluation).\n\n- **Bulk inference** Inference is the act of running an input on a flow or model to get the corresponding output. Bulk inference involves performing inference on multiple inputs simultaneously.\n\n- **Metric** An evaluation metric is a criterion on which an inference is scored. Examples include accuracy, faithfulness, maliciousness, whether the output is in English, etc.\n\n- **Dataset** A dataset is a collection of examples to use for inference-based \n evaluation. A dataset typically consists of `input` and optional `reference`\n fields. The `reference` field does not affect the inference step of evaluation\n but it is passed verbatim to any evaluation metrics. In Genkit, you can create a\n dataset through the Dev UI. There are two types of datasets in Genkit: _Flow_\n datasets and _Model_ datasets.\n\n### Schema validation\n\nDepending on the type, datasets have schema validation support in the Dev UI:\n\n- Flow datasets support validation of the `input` and `reference` fields of the dataset against a flow in the Genkit application. Schema validation is optional and is only enforced if a schema is specified on the target flow.\n\n- Model datasets have implicit schema, supporting both `string` and `GenerateRequest` input types. String validation provides a convenient way to evaluate simple text prompts, while `GenerateRequest` provides complete control for advanced use cases (e.g. providing model parameters, message history, tools, etc). You can find the full schema for `GenerateRequest` in our [API reference docs](https://js.api.genkit.dev/interfaces/genkit._.GenerateRequest.html).\n\nNote: Schema validation is a helper tool for editing examples, but it is\npossible to save an example with invalid schema. These examples may fail when\nthe running an evaluation.\n\n## Supported evaluators\n\n### Genkit evaluators\n\nGenkit includes a small number of native evaluators, inspired by [RAGAS](https://docs.ragas.io/en/stable/), to help you get started:\n\n- Faithfulness -- Measures the factual consistency of the generated answer against the given context\n- Answer Relevancy -- Assesses how pertinent the generated answer is to the given prompt\n- Maliciousness -- Measures whether the generated output intends to deceive, harm, or exploit\n\n### Evaluator plugins\n\nGenkit supports additional evaluators through plugins, like the Vertex Rapid Evaluators, which you can access via the [VertexAI Plugin](/docs/plugins/vertex-ai#evaluators).\n\n## Advanced use\n\n### Evaluation using the CLI\n\nGenkit CLI provides a rich API for performing evaluation. This is especially\nuseful in environments where the Dev UI is not available (e.g. in a CI/CD\nworkflow).\n\nGenkit CLI provides 3 main evaluation commands: `eval:flow`, `eval:extractData`,\nand `eval:run`.\n\n#### `eval:flow` command\n\nThe `eval:flow` command runs inference-based evaluation on an input dataset.\nThis dataset may be provided either as a JSON file or by referencing an existing\ndataset in your Genkit runtime.\n\n```bash\n# Referencing an existing dataset\ngenkit eval:flow qaFlow --input myFactsQaDataset\n\n# or, using a dataset from a file\ngenkit eval:flow qaFlow --input testInputs.json\n```\n\nNote: Make sure that you start your genkit app before running these CLI\ncommands.\n\n```bash\ngenkit start -- \n```\n\nHere, `testInputs.json` should be an array of objects containing an `input`\nfield and an optional `reference` field, like below:\n\n```json\n[\n {\n \"input\": {\"query\": \"What is the French word for Cheese?\"}\n },\n {\n \"input\": {\"query\": \"What green vegetable looks like cauliflower?\"},\n \"reference\": \"Broccoli\"\n }\n]\n```\n\nIf your flow requires auth, you may specify it using the `--context` argument:\n\n```bash\ngenkit eval:flow qaFlow --input testInputs.json --context '{\"auth\": {\"email_verified\": true}}'\n```\n\nBy default, the `eval:flow` and `eval:run` commands use all available metrics\nfor evaluation. To run on a subset of the configured evaluators, use the\n`--evaluators` flag and provide a comma-separated list of evaluators by name:\n\n```bash\ngenkit eval:flow qaFlow --input testInputs.json --evaluators=genkitEval/maliciousness,genkitEval/answer_relevancy\n```\n\nYou can view the results of your evaluation run in the Dev UI at\n`localhost:4000/evaluate`.\n\n#### `eval:extractData` and `eval:run` commands\n\nTo support _raw evaluation_, Genkit provides tools to extract data from traces\nand run evaluation metrics on extracted data. This is useful, for example, if\nyou are using a different framework for evaluation or if you are collecting\ninferences from a different environment to test locally for output quality.\n\nYou can batch run your Genkit flow and add a unique label to the run which then\ncan be used to extract an _evaluation dataset_. A raw evaluation dataset is a\ncollection of inputs for evaluation metrics, _without_ running any prior\ninference.\n\nRun your flow over your test inputs:\n\n```bash\ngenkit flow:batchRun qaFlow testInputs.json --label firstRunSimple\n```\n\nExtract the evaluation data:\n\n```bash\ngenkit eval:extractData qaFlow --label firstRunSimple --output factsEvalDataset.json\n```\n\nThe exported data has a format different from the dataset format presented\nearlier. This is because this data is intended to be used with evaluation\nmetrics directly, without any inference step. Here is the syntax of the\nextracted data.\n\n```json\nArray<{\n \"testCaseId\": string,\n \"input\": any,\n \"output\": any,\n \"context\": any[],\n \"traceIds\": string[],\n}>;\n```\n\nThe data extractor automatically locates retrievers and adds the produced docs\nto the context array. You can run evaluation metrics on this extracted dataset\nusing the `eval:run` command.\n\n```bash\ngenkit eval:run factsEvalDataset.json\n```\n\nBy default, `eval:run` runs against all configured evaluators, and as with\n`eval:flow`, results for `eval:run` appear in the evaluation page of Developer\nUI, located at `localhost:4000/evaluate`.\n\n### Batching evaluations\n\n:::note\nThis feature is only available in the Node.js SDK.\n:::\n\nYou can speed up evaluations by processing the inputs in batches using the CLI and Dev UI. When batching is enabled, the input data is grouped into batches of size `batchSize`. The data points in a batch are all run in parallel to provide significant performance improvements, especially when dealing with large datasets and/or complex evaluators. By default (when the flag is omitted), batching is disabled.\n\nThe `batchSize` option has been integrated into the `eval:flow` and `eval:run` CLI commands. When a `batchSize` greater than 1 is provided, the evaluator will process the dataset in chunks of the specified size. This feature only affects the evaluator logic and not inference (when using `eval:flow`). Here are some examples of enabling batching with the CLI:\n\n```bash\ngenkit eval:flow myFlow --input yourDataset.json --evaluators=custom/myEval --batchSize 10\n```\nOr, with `eval:run`\n\n```bash\ngenkit eval:run yourDataset.json --evaluators=custom/myEval --batchSize 10\n```\n\nBatching is also available in the Dev UI for Genkit (JS) applications. You can set batch size when running a new evaluation, to enable parallelization.\n\n### Custom extractors\n\nGenkit provides reasonable default logic for extracting the necessary fields\n(`input`, `output` and `context`) while doing an evaluation. However, you may\nfind that you need more control over the extraction logic for these fields.\nGenkit supports customs extractors to achieve this. You can provide custom\nextractors to be used in `eval:extractData` and `eval:flow` commands.\n\nFirst, as a preparatory step, introduce an auxilary step in our `qaFlow`\nexample:\n\n```js\nexport const qaFlow = ai.defineFlow(\n {\n name: 'qaFlow',\n inputSchema: z.object({ query: z.string() }),\n outputSchema: z.object({ answer: z.string() }),\n },\n async ({ query }) => {\n const factDocs = await ai.retrieve({\n retriever: dummyRetriever,\n query,\n });\n const factDocsModified = await ai.run('factModified', async () => {\n // Let us use only facts that are considered silly. This is a\n // hypothetical step for demo purposes, you may perform any\n // arbitrary task inside a step and reference it in custom\n // extractors.\n //\n // Assume you have a method that checks if a fact is silly\n return factDocs.filter((d) => isSillyFact(d.text));\n });\n\n const { text } = await ai.generate({\n model: googleAI.model('gemini-2.5-flash'),\n prompt: `Answer this question with the given context ${query}`,\n docs: factDocsModified,\n });\n return { answer: text };\n },\n);\n```\n\nNext, configure a custom extractor to use the output of the `factModified` step\nwhen evaluating this flow.\n\nIf you don't have one a tools-config file to configure custom extractors, add\none named `genkit-tools.conf.js` to your project root.\n\n```bash\ncd /path/to/your/genkit/app\n\ntouch genkit-tools.conf.js\n```\n\nIn the tools config file, add the following code:\n\n```js\nmodule.exports = {\n evaluators: [\n {\n actionRef: '/flow/qaFlow',\n extractors: {\n context: { outputOf: 'factModified' },\n },\n },\n ],\n};\n```\n\nThis config overrides the default extractors of Genkit's tooling, specifically\nchanging what is considered as `context` when evaluating this flow.\n\nRunning evaluation again reveals that context is now populated as the output of\nthe step `factModified`.\n\n```bash\ngenkit eval:flow qaFlow --input testInputs.json\n```\n\nEvaluation extractors are specified as follows:\n\n- `evaluators` field accepts an array of EvaluatorConfig objects, which are\n scoped by `flowName`\n- `extractors` is an object that specifies the extractor overrides. The\n current supported keys in `extractors` are `[input, output, context]`. The\n acceptable value types are:\n - `string` - this should be a step name, specified as a string. The output\n of this step is extracted for this key.\n - `{ inputOf: string }` or `{ outputOf: string }` - These objects\n represent specific channels (input or output) of a step. For example, `{\ninputOf: 'foo-step' }` would extract the input of step `foo-step` for\n this key.\n - `(trace) => string;` - For further flexibility, you can provide a\n function that accepts a Genkit trace and returns an `any`-type value,\n and specify the extraction logic inside this function. Refer to\n `genkit/genkit-tools/common/src/types/trace.ts` for the exact TraceData\n schema.\n\n**Note:** The extracted data for all these extractors is the type corresponding\nto the extractor. For example, if you use context: `{ outputOf: 'foo-step' }`,\nand `foo-step` returns an array of objects, the extracted context is also an\narray of objects.\n\n### Synthesizing test data using an LLM\n\nHere is an example flow that uses a PDF file to generate potential user\nquestions.\n\n```ts\nimport { genkit, z } from 'genkit';\nimport { googleAI } from '@genkit-ai/googleai';\nimport { chunk } from 'llm-chunk'; // npm install llm-chunk\nimport path from 'path';\nimport { readFile } from 'fs/promises';\nimport pdf from 'pdf-parse'; // npm install pdf-parse\n\nconst ai = genkit({ plugins: [googleAI()] });\n\nconst chunkingConfig = {\n minLength: 1000, // number of minimum characters into chunk\n maxLength: 2000, // number of maximum characters into chunk\n splitter: 'sentence', // paragraph | sentence\n overlap: 100, // number of overlap chracters\n delimiters: '', // regex for base split method\n} as any;\n\nasync function extractText(filePath: string) {\n const pdfFile = path.resolve(filePath);\n const dataBuffer = await readFile(pdfFile);\n const data = await pdf(dataBuffer);\n return data.text;\n}\n\nexport const synthesizeQuestions = ai.defineFlow(\n {\n name: 'synthesizeQuestions',\n inputSchema: z.object({ filePath: z.string().describe('PDF file path') }),\n outputSchema: z.object({ \n questions: z.array(z.object({\n query: z.string() \n })) \n }),\n },\n async ({ filePath }) => {\n filePath = path.resolve(filePath);\n // `extractText` loads the PDF and extracts its contents as text.\n const pdfTxt = await ai.run('extract-text', () => extractText(filePath));\n\n const chunks = await ai.run('chunk-it', async () => chunk(pdfTxt, chunkingConfig));\n\n const questions = [];\n for (var i = 0; i < chunks.length; i++) {\n const { text } = await ai.generate({\n model: googleAI.model('gemini-2.5-flash'),\n prompt: {\n text: `Generate one question about the following text: ${chunks[i]}`,\n },\n });\n questions.push({ query: text });\n }\n return { questions };\n },\n);\n```\n\nYou can then use this command to export the data into a file and use for\nevaluation.\n\n```bash\ngenkit flow:run synthesizeQuestions '{\"filePath\": \"my_input.pdf\"}' --output synthesizedQuestions.json\n```\n", + "text": "# Evaluation\n\nEvaluation is a form of testing that helps you validate your LLM's responses and\nensure they meet your quality bar.\n\nGenkit supports third-party evaluation tools through plugins, paired\nwith powerful observability features that provide insight into the runtime state\nof your LLM-powered applications. Genkit tooling helps you automatically extract\ndata including inputs, outputs, and information from intermediate steps to\nevaluate the end-to-end quality of LLM responses as well as understand the\nperformance of your system's building blocks.\n\n### Types of evaluation\n\nGenkit supports two types of evaluation:\n\n- **Inference-based evaluation**: This type of evaluation runs against a\n collection of pre-determined inputs, assessing the corresponding outputs for\n quality.\n\n This is the most common evaluation type, suitable for most use cases. This approach tests a system's actual output for each evaluation run.\n\n You can perform the quality assessment manually, by visually inspecting the results. Alternatively, you can automate the assessment by using an evaluation metric.\n\n- **Raw evaluation**: This type of evaluation directly assesses the quality of\n inputs without any inference. This approach typically is used with automated\n evaluation using metrics. All required fields for evaluation (e.g., `input`,\n `context`, `output` and `reference`) must be present in the input dataset. This\n is useful when you have data coming from an external source (e.g., collected\n from your production traces) and you want to have an objective measurement of\n the quality of the collected data.\n\n For more information, see the [Advanced use](#advanced-use) section of this page.\n\nThis section explains how to perform inference-based evaluation using Genkit.\n\n## Quick start\n\n### Setup\n\n1. Use an existing Genkit app or create a new one by following our [Get started](/docs/get-started) guide.\n2. Add the following code to define a simple RAG application to evaluate. For this guide, we use a dummy retriever that always returns the same documents.\n\n ```js\n import { genkit, z, Document } from 'genkit';\n import { googleAI } from '@genkit-ai/googleai';\n\n // Initialize Genkit\n export const ai = genkit({ plugins: [googleAI()] });\n\n // Dummy retriever that always returns the same docs\n export const dummyRetriever = ai.defineRetriever(\n {\n name: 'dummyRetriever',\n },\n async (i) => {\n const facts = [\"Dog is man's best friend\", 'Dogs have evolved and were domesticated from wolves'];\n // Just return facts as documents.\n return { documents: facts.map((t) => Document.fromText(t)) };\n },\n );\n\n // A simple question-answering flow\n export const qaFlow = ai.defineFlow(\n {\n name: 'qaFlow',\n inputSchema: z.object({ query: z.string() }),\n outputSchema: z.object({ answer: z.string() }),\n },\n async ({ query }) => {\n const factDocs = await ai.retrieve({\n retriever: dummyRetriever,\n query,\n });\n\n const { text } = await ai.generate({\n model: googleAI.model('gemini-2.5-flash'),\n prompt: `Answer this question with the given context ${query}`,\n docs: factDocs,\n });\n return { answer: text };\n },\n );\n ```\n\n3. (Optional) Add evaluation metrics to your application to use while evaluating. This guide uses the `MALICIOUSNESS` metric from the `genkitEval` plugin.\n\n ```js\n import { genkitEval, GenkitMetric } from '@genkit-ai/evaluator';\n import { googleAI } from '@genkit-ai/googleai';\n\n export const ai = genkit({\n plugins: [\n ...// Add this plugin to your Genkit initialization block\n genkitEval({\n judge: googleAI.model('gemini-2.5-flash'),\n metrics: [GenkitMetric.MALICIOUSNESS],\n }),\n ],\n });\n ```\n\n **Note:** The configuration above requires installation of the [`@genkit-ai/evaluator`](https://www.npmjs.com/package/@genkit-ai/evaluator) package.\n\n ```bash\n npm install @genkit-ai/evaluator\n ```\n\n4. Start your Genkit application.\n\n ```bash\n genkit start -- \n ```\n\n### Create a dataset\n\nCreate a dataset to define the examples we want to use for evaluating our flow.\n\n1. Go to the Dev UI at `http://localhost:4000` and click the **Datasets** button\n to open the Datasets page.\n\n2. Click on the **Create Dataset** button to open the create dataset dialog.\n\n a. Provide a `datasetId` for your new dataset. This guide uses\n `myFactsQaDataset`.\n\n b. Select `Flow` dataset type.\n\n c. Leave the validation target field empty and click **Save**\n\n3. Your new dataset page appears, showing an empty dataset. Add examples to it by following these steps:\n\n a. Click the **Add example** button to open the example editor panel.\n\n b. Only the `input` field is required. Enter `{\"query\": \"Who is man's best friend?\"}` in the `input` field, and click **Save** to add the example has to your dataset.\n\n c. Repeat steps (a) and (b) a couple more times to add more examples. This guide adds the following example inputs to the dataset:\n\n ```\n {\"query\": \"Can I give milk to my cats?\"}\n {\"query\": \"From which animals did dogs evolve?\"}\n ```\n\nBy the end of this step, your dataset should have 3 examples in it, with the\nvalues mentioned above.\n\n### Run evaluation and view results\n\nTo start evaluating the flow, click the **Run new evaluation** button on your\ndataset page. You can also start a new evaluation from the _Evaluations_ tab.\n\n1. Select the `Flow` radio button to evaluate a flow.\n\n2. Select `qaFlow` as the target flow to evaluate.\n\n3. Select `myFactsQaDataset` as the target dataset to use for evaluation.\n\n4. (Optional) If you have installed an evaluator metric using Genkit plugins,\n you can see these metrics in this page. Select the metrics that you want to use\n with this evaluation run. This is entirely optional: Omitting this step will\n still return the results in the evaluation run, but without any associated\n metrics.\n\n5. Finally, click **Run evaluation** to start evaluation. Depending on the flow\n you're testing, this may take a while. Once the evaluation is complete, a\n success message appears with a link to view the results. Click on the link to go\n to the _Evaluation details_ page.\n\nYou can see the details of your evaluation on this page, including original\ninput, extracted context and metrics (if any).\n\n## Core concepts\n\n### Terminology\n\n- **Evaluation**: An evaluation is a process that assesses system performance. In Genkit, such a system is usually a Genkit primitive, such as a flow or a\n model. An evaluation can be automated or manual (human evaluation).\n\n- **Bulk inference** Inference is the act of running an input on a flow or model to get the corresponding output. Bulk inference involves performing inference on multiple inputs simultaneously.\n\n- **Metric** An evaluation metric is a criterion on which an inference is scored. Examples include accuracy, faithfulness, maliciousness, whether the output is in English, etc.\n\n- **Dataset** A dataset is a collection of examples to use for inference-based \n evaluation. A dataset typically consists of `input` and optional `reference`\n fields. The `reference` field does not affect the inference step of evaluation\n but it is passed verbatim to any evaluation metrics. In Genkit, you can create a\n dataset through the Dev UI. There are two types of datasets in Genkit: _Flow_\n datasets and _Model_ datasets.\n\n### Schema validation\n\nDepending on the type, datasets have schema validation support in the Dev UI:\n\n- Flow datasets support validation of the `input` and `reference` fields of the dataset against a flow in the Genkit application. Schema validation is optional and is only enforced if a schema is specified on the target flow.\n\n- Model datasets have implicit schema, supporting both `string` and `GenerateRequest` input types. String validation provides a convenient way to evaluate simple text prompts, while `GenerateRequest` provides complete control for advanced use cases (e.g. providing model parameters, message history, tools, etc). You can find the full schema for `GenerateRequest` in our [API reference docs](https://js.api.genkit.dev/interfaces/genkit._.GenerateRequest.html).\n\nNote: Schema validation is a helper tool for editing examples, but it is\npossible to save an example with invalid schema. These examples may fail when\nthe running an evaluation.\n\n## Supported evaluators\n\n### Genkit evaluators\n\nGenkit includes a small number of native evaluators, inspired by [RAGAS](https://docs.ragas.io/en/stable/), to help you get started:\n\n- Faithfulness -- Measures the factual consistency of the generated answer against the given context\n- Answer Relevancy -- Assesses how pertinent the generated answer is to the given prompt\n- Maliciousness -- Measures whether the generated output intends to deceive, harm, or exploit\n\n### Evaluator plugins\n\nGenkit supports additional evaluators through plugins, like the Vertex Rapid Evaluators, which you can access via the [VertexAI Plugin](/docs/plugins/vertex-ai#evaluators).\n\n## Advanced use\n\n### Evaluation comparison\n\nThe Developer UI offers visual tools for side-by-side comparison of multiple\nevaluation runs. This feature allows you to analyze variations across different\nexecutions within a unified interface, making it easier to assess changes in\noutput quality. Additionally, you can highlight outputs based on the performance\nof specific metrics, indicating improvements or regressions.\n\nWhen comparing evaluations, one run is designated as the _Baseline_. All other\nevaluations are compared against this baseline to determine whether their\nperformance has improved or regressed.\n\nimport ThemeImage from '../../../components/ThemeImage.astro';\n\n\n\n#### Prerequisites\n\nTo use the evaluation comparison feature, the following conditions must be met:\n\n- Evaluations must originate from a dataset source. Evaluations from file\n sources are not comparable.\n- All evaluations being compared must be from the same dataset.\n- For metric highlighting, all evaluations must use at least one common\n metric that produces a `number` or `boolean` score.\n\n#### Comparing evaluations\n\n1. Ensure you have at least two evaluation runs performed on the same dataset.\n For instructions, refer to the\n [Run evaluation section](#run-evaluation-and-view-results).\n\n2. In the Developer UI, navigate to the **Datasets** page.\n\n3. Select the relevant dataset and open its **Evaluations** tab. You should see\n all evaluation runs associated with that dataset.\n\n4. Choose one evaluation to serve as the baseline for comparison.\n\n5. On the evaluation results page, click the **+ Comparison** button. If this\n button is disabled, it means no other comparable evaluations are available\n for this dataset.\n\n6. A new column will appear with a dropdown menu. Select another evaluation\n from this menu to load its results alongside the baseline.\n\nYou can now view the outputs side-by-side to visually inspect differences in\nquality. This feature supports comparing up to three evaluations simultaneously.\n\n##### Metric highlighting (Optional)\n\nIf your evaluations include metrics, you can enable metric highlighting to\ncolor-code the results. This feature helps you quickly identify changes in\nperformance: improvements are colored green, while regressions are red.\n\nNote that highlighting is only supported for numeric and boolean metrics, and\nthe selected metric must be present in all evaluations being compared.\n\nTo enable metric highlighting:\n\n1. After initiating a comparison, a **Choose a metric to compare** menu will\n become available.\n\n2. Select a metric from the dropdown. By default, lower scores (for numeric\n metrics) and `false` values (for boolean metrics) are considered\n improvements and highlighted in green. You can reverse this logic by\n ticking the checkbox in the menu.\n\nThe comparison columns will now be color-coded according to the selected metric\nand configuration, providing an at-a-glance overview of performance changes.\n\n### Evaluation using the CLI\n\nGenkit CLI provides a rich API for performing evaluation. This is especially\nuseful in environments where the Dev UI is not available (e.g. in a CI/CD\nworkflow).\n\nGenkit CLI provides 3 main evaluation commands: `eval:flow`, `eval:extractData`,\nand `eval:run`.\n\n#### `eval:flow` command\n\nThe `eval:flow` command runs inference-based evaluation on an input dataset.\nThis dataset may be provided either as a JSON file or by referencing an existing\ndataset in your Genkit runtime.\n\n```bash\n# Referencing an existing dataset\ngenkit eval:flow qaFlow --input myFactsQaDataset\n\n# or, using a dataset from a file\ngenkit eval:flow qaFlow --input testInputs.json\n```\n\nNote: Make sure that you start your genkit app before running these CLI\ncommands.\n\n```bash\ngenkit start -- \n```\n\nHere, `testInputs.json` should be an array of objects containing an `input`\nfield and an optional `reference` field, like below:\n\n```json\n[\n {\n \"input\": { \"query\": \"What is the French word for Cheese?\" }\n },\n {\n \"input\": { \"query\": \"What green vegetable looks like cauliflower?\" },\n \"reference\": \"Broccoli\"\n }\n]\n```\n\nIf your flow requires auth, you may specify it using the `--context` argument:\n\n```bash\ngenkit eval:flow qaFlow --input testInputs.json --context '{\"auth\": {\"email_verified\": true}}'\n```\n\nBy default, the `eval:flow` and `eval:run` commands use all available metrics\nfor evaluation. To run on a subset of the configured evaluators, use the\n`--evaluators` flag and provide a comma-separated list of evaluators by name:\n\n```bash\ngenkit eval:flow qaFlow --input testInputs.json --evaluators=genkitEval/maliciousness,genkitEval/answer_relevancy\n```\n\nYou can view the results of your evaluation run in the Dev UI at\n`localhost:4000/evaluate`.\n\n#### `eval:extractData` and `eval:run` commands\n\nTo support _raw evaluation_, Genkit provides tools to extract data from traces\nand run evaluation metrics on extracted data. This is useful, for example, if\nyou are using a different framework for evaluation or if you are collecting\ninferences from a different environment to test locally for output quality.\n\nYou can batch run your Genkit flow and add a unique label to the run which then\ncan be used to extract an _evaluation dataset_. A raw evaluation dataset is a\ncollection of inputs for evaluation metrics, _without_ running any prior\ninference.\n\nRun your flow over your test inputs:\n\n```bash\ngenkit flow:batchRun qaFlow testInputs.json --label firstRunSimple\n```\n\nExtract the evaluation data:\n\n```bash\ngenkit eval:extractData qaFlow --label firstRunSimple --output factsEvalDataset.json\n```\n\nThe exported data has a format different from the dataset format presented\nearlier. This is because this data is intended to be used with evaluation\nmetrics directly, without any inference step. Here is the syntax of the\nextracted data.\n\n```json\nArray<{\n \"testCaseId\": string,\n \"input\": any,\n \"output\": any,\n \"context\": any[],\n \"traceIds\": string[],\n}>;\n```\n\nThe data extractor automatically locates retrievers and adds the produced docs\nto the context array. You can run evaluation metrics on this extracted dataset\nusing the `eval:run` command.\n\n```bash\ngenkit eval:run factsEvalDataset.json\n```\n\nBy default, `eval:run` runs against all configured evaluators, and as with\n`eval:flow`, results for `eval:run` appear in the evaluation page of Developer\nUI, located at `localhost:4000/evaluate`.\n\n### Batching evaluations\n\n:::note\nThis feature is only available in the Node.js SDK.\n:::\n\nYou can speed up evaluations by processing the inputs in batches using the CLI and Dev UI. When batching is enabled, the input data is grouped into batches of size `batchSize`. The data points in a batch are all run in parallel to provide significant performance improvements, especially when dealing with large datasets and/or complex evaluators. By default (when the flag is omitted), batching is disabled.\n\nThe `batchSize` option has been integrated into the `eval:flow` and `eval:run` CLI commands. When a `batchSize` greater than 1 is provided, the evaluator will process the dataset in chunks of the specified size. This feature only affects the evaluator logic and not inference (when using `eval:flow`). Here are some examples of enabling batching with the CLI:\n\n```bash\ngenkit eval:flow myFlow --input yourDataset.json --evaluators=custom/myEval --batchSize 10\n```\n\nOr, with `eval:run`\n\n```bash\ngenkit eval:run yourDataset.json --evaluators=custom/myEval --batchSize 10\n```\n\nBatching is also available in the Dev UI for Genkit (JS) applications. You can set batch size when running a new evaluation, to enable parallelization.\n\n### Custom extractors\n\nGenkit provides reasonable default logic for extracting the necessary fields\n(`input`, `output` and `context`) while doing an evaluation. However, you may\nfind that you need more control over the extraction logic for these fields.\nGenkit supports customs extractors to achieve this. You can provide custom\nextractors to be used in `eval:extractData` and `eval:flow` commands.\n\nFirst, as a preparatory step, introduce an auxilary step in our `qaFlow`\nexample:\n\n```js\nexport const qaFlow = ai.defineFlow(\n {\n name: 'qaFlow',\n inputSchema: z.object({ query: z.string() }),\n outputSchema: z.object({ answer: z.string() }),\n },\n async ({ query }) => {\n const factDocs = await ai.retrieve({\n retriever: dummyRetriever,\n query,\n });\n const factDocsModified = await ai.run('factModified', async () => {\n // Let us use only facts that are considered silly. This is a\n // hypothetical step for demo purposes, you may perform any\n // arbitrary task inside a step and reference it in custom\n // extractors.\n //\n // Assume you have a method that checks if a fact is silly\n return factDocs.filter((d) => isSillyFact(d.text));\n });\n\n const { text } = await ai.generate({\n model: googleAI.model('gemini-2.5-flash'),\n prompt: `Answer this question with the given context ${query}`,\n docs: factDocsModified,\n });\n return { answer: text };\n },\n);\n```\n\nNext, configure a custom extractor to use the output of the `factModified` step\nwhen evaluating this flow.\n\nIf you don't have one a tools-config file to configure custom extractors, add\none named `genkit-tools.conf.js` to your project root.\n\n```bash\ncd /path/to/your/genkit/app\n\ntouch genkit-tools.conf.js\n```\n\nIn the tools config file, add the following code:\n\n```js\nmodule.exports = {\n evaluators: [\n {\n actionRef: '/flow/qaFlow',\n extractors: {\n context: { outputOf: 'factModified' },\n },\n },\n ],\n};\n```\n\nThis config overrides the default extractors of Genkit's tooling, specifically\nchanging what is considered as `context` when evaluating this flow.\n\nRunning evaluation again reveals that context is now populated as the output of\nthe step `factModified`.\n\n```bash\ngenkit eval:flow qaFlow --input testInputs.json\n```\n\nEvaluation extractors are specified as follows:\n\n- `evaluators` field accepts an array of EvaluatorConfig objects, which are\n scoped by `flowName`\n- `extractors` is an object that specifies the extractor overrides. The\n current supported keys in `extractors` are `[input, output, context]`. The\n acceptable value types are:\n - `string` - this should be a step name, specified as a string. The output\n of this step is extracted for this key.\n - `{ inputOf: string }` or `{ outputOf: string }` - These objects\n represent specific channels (input or output) of a step. For example, `{\ninputOf: 'foo-step' }` would extract the input of step `foo-step` for\n this key.\n - `(trace) => string;` - For further flexibility, you can provide a\n function that accepts a Genkit trace and returns an `any`-type value,\n and specify the extraction logic inside this function. Refer to\n `genkit/genkit-tools/common/src/types/trace.ts` for the exact TraceData\n schema.\n\n**Note:** The extracted data for all these extractors is the type corresponding\nto the extractor. For example, if you use context: `{ outputOf: 'foo-step' }`,\nand `foo-step` returns an array of objects, the extracted context is also an\narray of objects.\n\n### Synthesizing test data using an LLM\n\nHere is an example flow that uses a PDF file to generate potential user\nquestions.\n\n```ts\nimport { genkit, z } from 'genkit';\nimport { googleAI } from '@genkit-ai/googleai';\nimport { chunk } from 'llm-chunk'; // npm install llm-chunk\nimport path from 'path';\nimport { readFile } from 'fs/promises';\nimport pdf from 'pdf-parse'; // npm install pdf-parse\n\nconst ai = genkit({ plugins: [googleAI()] });\n\nconst chunkingConfig = {\n minLength: 1000, // number of minimum characters into chunk\n maxLength: 2000, // number of maximum characters into chunk\n splitter: 'sentence', // paragraph | sentence\n overlap: 100, // number of overlap chracters\n delimiters: '', // regex for base split method\n} as any;\n\nasync function extractText(filePath: string) {\n const pdfFile = path.resolve(filePath);\n const dataBuffer = await readFile(pdfFile);\n const data = await pdf(dataBuffer);\n return data.text;\n}\n\nexport const synthesizeQuestions = ai.defineFlow(\n {\n name: 'synthesizeQuestions',\n inputSchema: z.object({ filePath: z.string().describe('PDF file path') }),\n outputSchema: z.object({\n questions: z.array(\n z.object({\n query: z.string(),\n }),\n ),\n }),\n },\n async ({ filePath }) => {\n filePath = path.resolve(filePath);\n // `extractText` loads the PDF and extracts its contents as text.\n const pdfTxt = await ai.run('extract-text', () => extractText(filePath));\n\n const chunks = await ai.run('chunk-it', async () => chunk(pdfTxt, chunkingConfig));\n\n const questions = [];\n for (var i = 0; i < chunks.length; i++) {\n const { text } = await ai.generate({\n model: googleAI.model('gemini-2.5-flash'),\n prompt: {\n text: `Generate one question about the following text: ${chunks[i]}`,\n },\n });\n questions.push({ query: text });\n }\n return { questions };\n },\n);\n```\n\nYou can then use this command to export the data into a file and use for\nevaluation.\n\n```bash\ngenkit flow:run synthesizeQuestions '{\"filePath\": \"my_input.pdf\"}' --output synthesizedQuestions.json\n```", "title": "Evaluation", - "lang": "js" + "description": "Learn about Genkit's evaluation capabilities, including inference-based and raw evaluation, dataset creation, and how to use the Developer UI and CLI for testing and analysis.", + "lang": "js", + "headers": "### Types of evaluation\n## Quick start\n### Setup\n### Create a dataset\n### Run evaluation and view results\n## Core concepts\n### Terminology\n### Schema validation\n## Supported evaluators\n### Genkit evaluators\n### Evaluator plugins\n## Advanced use\n### Evaluation comparison\n#### Prerequisites\n#### Comparing evaluations\n##### Metric highlighting (Optional)\n### Evaluation using the CLI\n#### `eval:flow` command\n# Referencing an existing dataset\n# or, using a dataset from a file\n#### `eval:extractData` and `eval:run` commands\n### Batching evaluations\n### Custom extractors\n### Synthesizing test data using an LLM\n" }, "js/feedback.md": { - "text": "We'd love to hear about your excperience with Genkit. Here's how you can get in\ntouch with us:\n\n**Join the community:** Stay updated, ask questions, and share your work with\nother Genkit users on the [Genkit Discord server](https://discord.gg/qXt5zzQKpc).\n\n**Provide feedback:** Report issues with Genkit or the docs, or suggest new\nfeatures using our [GitHub issue tracker](https://github.com/firebase/genkit/issues).\n\n**We're interested in learning things like:**\n\n- Was it straightforward to set up and make your first `generate` call? If not,\n how could we make it better?\n\n- Were you able to build what you wanted? If not, what could we do to help?\n\n- Is there any specific feature, documentation, or resource that's missing?\n\n- Is there anything that's working particularly well for you?\n Anything that isn't?\n\n- Anything else that you'd like to share with us about your experience!\n", + "text": "# Connect with us\n\nWe'd love to hear about your excperience with Genkit. Here's how you can get in\ntouch with us:\n\n**Join the community:** Stay updated, ask questions, and share your work with\nother Genkit users on the [Genkit Discord server](https://discord.gg/qXt5zzQKpc).\n\n**Provide feedback:** Report issues with Genkit or the docs, or suggest new\nfeatures using our [GitHub issue tracker](https://github.com/firebase/genkit/issues).\n\n**We're interested in learning things like:**\n\n- Was it straightforward to set up and make your first `generate` call? If not,\n how could we make it better?\n\n- Were you able to build what you wanted? If not, what could we do to help?\n\n- Is there any specific feature, documentation, or resource that's missing?\n\n- Is there anything that's working particularly well for you?\n Anything that isn't?\n\n- Anything else that you'd like to share with us about your experience!\n", "title": "Connect with us", - "lang": "js" + "description": "Learn how to connect with the Genkit community and provide feedback on your experience, including Discord and GitHub resources.", + "lang": "js", + "headers": "" }, - "js/firebase.mdx": { - "text": "import { Tabs, TabItem } from '@astrojs/starlight/components';\n\nCloud Functions for Firebase has an `onCallGenkit` method that lets you\nquickly create a [callable function](https://firebase.google.com/docs/functions/callable?gen=2nd)\nwith a Genkit action (e.g. a Flow). These functions can be called using\n`genkit/beta/client`or the [Functions client SDK](https://firebase.google.com/docs/functions/callable?gen=2nd#call_the_function),\nwhich automatically adds auth info.\n\n## Before you begin\n\n- You should be familiar with Genkit's concept of [flows](/docs/flows), and how to\n write them. The instructions on this page assume that you already have some\n flows defined, which you want to deploy.\n- It would be helpful, but not required, if you've already used Cloud\n Functions for Firebase before.\n\n## 1. Set up a Firebase project\n\nIf you don't already have a Firebase project with TypeScript Cloud Functions set\nup, follow these steps:\n\n1. Create a new Firebase project using the [Firebase\n console](https://console.firebase.google.com/) or choose an existing one.\n\n1. Upgrade the project to the Blaze plan, which is required to deploy Cloud\n Functions.\n\n1. Install the [Firebase CLI](https://firebase.google.com/docs/cli).\n\n1. Log in with the Firebase CLI:\n\n ```bash\n firebase login\n\n firebase login --reauth # alternative, if necessary\n\n firebase login --no-localhost # if running in a remote shell\n ```\n\n1. Create a new project directory:\n\n ```bash\n export PROJECT_ROOT=~/tmp/genkit-firebase-project1\n\n mkdir -p $PROJECT_ROOT\n ```\n\n1. Initialize a Firebase project in the directory:\n\n ```bash\n cd $PROJECT_ROOT\n\n firebase init genkit\n ```\n\n The rest of this page assumes that you've decided to write your functions\n in TypeScript, but you can also deploy your Genkit flows if you're using\n JavaScript.\n\n## 2. Wrap the Flow in onCallGenkit\n\nAfter you've set up a Firebase project with Cloud Functions, you can copy or\nwrite flow definitions in the project’s `functions/src` directory, and export\nthem in `index.ts`.\n\nFor your flows to be deployable, you need to wrap them in `onCallGenkit`.\nThis method has all the features of the normal `onCall`. It automatically\nsupports both streaming and JSON responses.\n\nSuppose you have the following flow:\n\n```ts\nconst generatePoemFlow = ai.defineFlow(\n {\n name: 'generatePoem',\n inputSchema: z.string(),\n outputSchema: z.string(),\n },\n async (subject: string) => {\n const { text } = await ai.generate(`Compose a poem about ${subject}.`);\n return text;\n },\n);\n```\n\nYou can expose this flow as a callable function using `onCallGenkit`:\n\n```ts\nimport { onCallGenkit } from 'firebase-functions/https';\n\nexport generatePoem = onCallGenkit(generatePoemFlow);\n```\n\n### Define an authorization policy\n\nAll deployed flows, whether deployed to Firebase or not, should have an\nauthorization policy; without one, anyone can invoke your potentially-expensive\ngenerative AI flows. To define an authorization policy, use the\n`authPolicy` parameter of `onCallGenkit`:\n\n```ts\nexport const generatePoem = onCallGenkit(\n {\n authPolicy: (auth) => auth?.token?.email_verified,\n },\n generatePoemFlow,\n);\n```\n\nThis sample uses a manual function as its auth policy. In addition, the https\nlibrary exports the `signedIn()` and `hasClaim()` helpers. Here is the same code\nusing one of those helpers:\n\n```ts\nimport { hasClaim } from 'firebase-functions/https';\n\nexport const generatePoem = onCallGenkit(\n {\n authPolicy: hasClaim('email_verified'),\n },\n generatePoemFlow,\n);\n```\n\n### Make API credentials available to deployed flows\n\nOnce deployed, your flows need some way to authenticate with any remote services\nthey rely on. Most flows need, at a minimum, credentials for accessing the\nmodel API service they use.\n\nFor this example, do one of the following, depending on the model provider you chose:\n\n\n \n 1. Make sure Google AI is [available in your region](https://ai.google.dev/available_regions).\n\n 2. [Generate an API key](https://aistudio.google.com/app/apikey) for the\n Gemini API using Google AI Studio.\n\n 3. Store your API key in Cloud Secret Manager:\n\n ```bash\n firebase functions:secrets:set GEMINI_API_KEY\n ```\n\n This step is important to prevent accidentally leaking your API key,\n which grants access to a potentially metered service.\n\n See [Store and access sensitive configuration information](https://firebase.google.com/docs/functions/config-env?gen=2nd#secret-manager)\n for more information on managing secrets.\n\n 4. Edit `src/index.ts` and add the following after the existing imports:\n\n ```ts\n import { defineSecret } from \"firebase-functions/params\";\n const googleAIapiKey = defineSecret(\"GEMINI_API_KEY\");\n ```\n\n Then, in the flow definition, declare that the cloud function needs\n access to this secret value:\n\n ```ts\n export const generatePoem = onCallGenkit(\n {\n secrets: [googleAIapiKey],\n },\n generatePoemFlow\n );\n ```\n\n Now, when you deploy this function, your API key is stored in Cloud Secret Manager, and available from the Cloud Functions environment.\n\n \n \n 1. In the Cloud console, [Enable the Vertex AI API](https://console.cloud.google.com/apis/library/aiplatform.googleapis.com?project=_)\n for your Firebase project.\n\n 2. On the [IAM](https://console.cloud.google.com/iam-admin/iam?project=_)\n page, ensure that the **Default compute service account** is granted the\n **Vertex AI User** role.\n\n \n\n\nThe only secret you need to set up for this tutorial is for the model provider,\nbut in general, you must do something similar for each service your flow uses.\n\n### Add App Check enforcement\n\n[Firebase App Check](https://firebase.google.com/docs/app-check) uses a\nbuilt-in attestation mechanism to verify that your API is only being called by\nyour application. `onCallGenkit` supports App Check enforcement declaratively.\n\n```ts\nexport const generatePoem = onCallGenkit(\n {\n enforceAppCheck: true,\n // Optional. Makes App Check tokens only usable once. This adds extra security\n // at the expense of slowing down your app to generate a token for every API\n // call\n consumeAppCheckToken: true,\n },\n generatePoemFlow,\n);\n```\n\n### Set a CORS policy\n\nCallable functions default to allowing any domain to call your function. If you\nwant to customize the domains that can do this, use the `cors` option.\nWith proper authentication (especially App Check), CORS is often unnecessary.\n\n```ts\nexport const generatePoem = onCallGenkit(\n {\n cors: 'mydomain.com',\n },\n generatePoemFlow,\n);\n```\n\n### Complete example\n\nAfter you've made all of the changes described earlier, your deployable flow\nlooks something like the following example:\n\n```ts\nimport { genkit } from 'genkit';\nimport { googleAI } from '@genkit-ai/googleai';\nimport { onCallGenkit, hasClaim } from 'firebase-functions/https';\nimport { defineSecret } from 'firebase-functions/params';\n\nconst apiKey = defineSecret('GEMINI_API_KEY');\n\nconst ai = genkit({\n plugins: [googleAI()],\n model: googleAI.model('gemini-2.5-flash'),\n});\n\nconst generatePoemFlow = ai.defineFlow(\n {\n name: 'generatePoem',\n inputSchema: z.string(),\n outputSchema: z.string(),\n },\n async (subject: string) => {\n const { text } = await ai.generate(`Compose a poem about ${subject}.`);\n return text;\n },\n);\n\nexport const generateFlow = onCallGenkit(\n {\n secrets: [apiKey],\n authPolicy: hasClaim('email_verified'),\n enforceAppCheck: true,\n },\n generatePoemFlow,\n);\n```\n\n## 3. Deploy flows to Firebase\n\nAfter you've defined flows using `onCallGenkit`, you can deploy them the same\nway you would deploy other Cloud Functions:\n\n```bash\ncd $PROJECT_ROOT\n\nfirebase deploy --only functions\n```\n\nYou've now deployed the flow as a Cloud Function! But you can't\naccess your deployed endpoint with `curl` or similar, because of the flow's\nauthorization policy. The next section explains how to securely access the\nflow.\n\n## Optional: Try the deployed flow\n\nTo try out your flow endpoint, you can deploy the following minimal example web\napp:\n\n1. In the [Project settings](https://console.firebase.google.com/project/_/settings/general)\n section of the Firebase console, add a new web app, selecting the option to\n also set up Hosting.\n\n1. In the\n [Authentication](https://console.firebase.google.com/project/_/authentication/providers)\n section of the Firebase console, enable the **Google** provider, used in\n this example.\n\n1. In your project directory, set up Firebase Hosting, where you will deploy\n the sample app:\n\n ```bash\n cd $PROJECT_ROOT\n\n firebase init hosting\n ```\n\n Accept the defaults for all of the prompts.\n\n1. Replace `public/index.html` with the following:\n\n ```html\n \n \n \n Genkit demo\n \n \n \n \n \n \n \n ```\n\n1. Deploy the web app and Cloud Function:\n\n ```bash\n cd $PROJECT_ROOT\n\n firebase deploy\n ```\n\nOpen the web app by visiting the URL printed by the `deploy` command. The app\nrequires you to sign in with a Google account, after which you can initiate\nendpoint requests.\n\n## Optional: Run flows in the developer UI\n\nYou can run flows defined using `onCallGenkit` in the developer UI, exactly the\nsame way as you run flows defined using `defineFlow`, so there's no need to\nswitch between the two between deployment and development.\n\n```bash\ncd $PROJECT_ROOT/functions\n\ngenkit start -- npx tsx --watch src/index.ts\n```\n\nor\n\n```bash\ncd $PROJECT_ROOT/functions\n\nnpm run genkit:start\n```\n\nYou can now navigate to the URL printed by the `genkit start` command to access.\n\n## Optional: Developing using Firebase Local Emulator Suite\n\nFirebase offers a\n[suite of emulators for local development](https://firebase.google.com/docs/emulator-suite), which you can\nuse with Genkit.\n\nTo use the Genkit Dev UI with the Firebase Emulator Suite, start the Firebase\nemulators as follows:\n\n```bash\ngenkit start -- firebase emulators:start --inspect-functions\n```\n\nThis command runs your code in the emulator, and runs the Genkit framework in\ndevelopment mode. This launches and exposes the Genkit reflection API (but not\nthe Dev UI).\n", + "js/firebase.md": { + "text": "# Deploy flows using Cloud Functions for Firebase\n\nCloud Functions for Firebase has an `onCallGenkit` method that lets you\nquickly create a [callable function](https://firebase.google.com/docs/functions/callable?gen=2nd)\nwith a Genkit action (e.g. a Flow). These functions can be called using\n`genkit/beta/client`or the [Functions client SDK](https://firebase.google.com/docs/functions/callable?gen=2nd#call_the_function),\nwhich automatically adds auth info.\n\n## Before you begin\n\n- You should be familiar with Genkit's concept of [flows](/docs/flows), and how to\n write them. The instructions on this page assume that you already have some\n flows defined, which you want to deploy.\n- It would be helpful, but not required, if you've already used Cloud\n Functions for Firebase before.\n\n## 1. Set up a Firebase project\n\nIf you don't already have a Firebase project with TypeScript Cloud Functions set\nup, follow these steps:\n\n1. Create a new Firebase project using the [Firebase\n console](https://console.firebase.google.com/) or choose an existing one.\n\n1. Upgrade the project to the Blaze plan, which is required to deploy Cloud\n Functions.\n\n1. Install the [Firebase CLI](https://firebase.google.com/docs/cli).\n\n1. Log in with the Firebase CLI:\n\n ```bash\n firebase login\n\n firebase login --reauth # alternative, if necessary\n\n firebase login --no-localhost # if running in a remote shell\n ```\n\n1. Create a new project directory:\n\n ```bash\n export PROJECT_ROOT=~/tmp/genkit-firebase-project1\n\n mkdir -p $PROJECT_ROOT\n ```\n\n1. Initialize a Firebase project in the directory:\n\n ```bash\n cd $PROJECT_ROOT\n\n firebase init genkit\n ```\n\n The rest of this page assumes that you've decided to write your functions\n in TypeScript, but you can also deploy your Genkit flows if you're using\n JavaScript.\n\n## 2. Wrap the Flow in onCallGenkit\n\nAfter you've set up a Firebase project with Cloud Functions, you can copy or\nwrite flow definitions in the project’s `functions/src` directory, and export\nthem in `index.ts`.\n\nFor your flows to be deployable, you need to wrap them in `onCallGenkit`.\nThis method has all the features of the normal `onCall`. It automatically\nsupports both streaming and JSON responses.\n\nSuppose you have the following flow:\n\n```ts\nconst generatePoemFlow = ai.defineFlow(\n {\n name: 'generatePoem',\n inputSchema: z.object({ subject: z.string() }),\n outputSchema: z.object({ poem: z.string() }),\n },\n async ({ subject }) => {\n const { text } = await ai.generate(`Compose a poem about ${subject}.`);\n return { poem: text };\n },\n);\n```\n\nYou can expose this flow as a callable function using `onCallGenkit`:\n\n```ts\nimport { onCallGenkit } from 'firebase-functions/https';\n\nexport generatePoem = onCallGenkit(generatePoemFlow);\n```\n\n### Define an authorization policy\n\nAll deployed flows, whether deployed to Firebase or not, should have an\nauthorization policy; without one, anyone can invoke your potentially-expensive\ngenerative AI flows. To define an authorization policy, use the\n`authPolicy` parameter of `onCallGenkit`:\n\n```ts\nexport const generatePoem = onCallGenkit(\n {\n authPolicy: (auth) => auth?.token?.email_verified,\n },\n generatePoemFlow,\n);\n```\n\nThis sample uses a manual function as its auth policy. In addition, the https\nlibrary exports the `signedIn()` and `hasClaim()` helpers. Here is the same code\nusing one of those helpers:\n\n```ts\nimport { hasClaim } from 'firebase-functions/https';\n\nexport const generatePoem = onCallGenkit(\n {\n authPolicy: hasClaim('email_verified'),\n },\n generatePoemFlow,\n);\n```\n\n### Make API credentials available to deployed flows\n\nOnce deployed, your flows need some way to authenticate with any remote services\nthey rely on. Most flows need, at a minimum, credentials for accessing the\nmodel API service they use.\n\nFor this example, do one of the following, depending on the model provider you chose:\n\n\n \n 1. Make sure Google AI is [available in your region](https://ai.google.dev/available_regions).\n\n 2. [Generate an API key](https://aistudio.google.com/app/apikey) for the\n Gemini API using Google AI Studio.\n\n 3. Store your API key in Cloud Secret Manager:\n\n ```bash\n firebase functions:secrets:set GEMINI_API_KEY\n ```\n\n This step is important to prevent accidentally leaking your API key,\n which grants access to a potentially metered service.\n\n See [Store and access sensitive configuration information](https://firebase.google.com/docs/functions/config-env?gen=2nd#secret-manager)\n for more information on managing secrets.\n\n 4. Edit `src/index.ts` and add the following after the existing imports:\n\n ```ts\n import { defineSecret } from \"firebase-functions/params\";\n const googleAIapiKey = defineSecret(\"GEMINI_API_KEY\");\n ```\n\n Then, in the flow definition, declare that the cloud function needs\n access to this secret value:\n\n ```ts\n export const generatePoem = onCallGenkit(\n {\n secrets: [googleAIapiKey],\n },\n generatePoemFlow\n );\n ```\n\n Now, when you deploy this function, your API key is stored in Cloud Secret Manager, and available from the Cloud Functions environment.\n\n \n \n 1. In the Cloud console, [Enable the Vertex AI API](https://console.cloud.google.com/apis/library/aiplatform.googleapis.com?project=_)\n for your Firebase project.\n\n 2. On the [IAM](https://console.cloud.google.com/iam-admin/iam?project=_)\n page, ensure that the **Default compute service account** is granted the\n **Vertex AI User** role.\n\n \n\n\nThe only secret you need to set up for this tutorial is for the model provider,\nbut in general, you must do something similar for each service your flow uses.\n\n### Add App Check enforcement\n\n[Firebase App Check](https://firebase.google.com/docs/app-check) uses a\nbuilt-in attestation mechanism to verify that your API is only being called by\nyour application. `onCallGenkit` supports App Check enforcement declaratively.\n\n```ts\nexport const generatePoem = onCallGenkit(\n {\n enforceAppCheck: true,\n // Optional. Makes App Check tokens only usable once. This adds extra security\n // at the expense of slowing down your app to generate a token for every API\n // call\n consumeAppCheckToken: true,\n },\n generatePoemFlow,\n);\n```\n\n### Set a CORS policy\n\nCallable functions default to allowing any domain to call your function. If you\nwant to customize the domains that can do this, use the `cors` option.\nWith proper authentication (especially App Check), CORS is often unnecessary.\n\n```ts\nexport const generatePoem = onCallGenkit(\n {\n cors: 'mydomain.com',\n },\n generatePoemFlow,\n);\n```\n\n### Complete example\n\nAfter you've made all of the changes described earlier, your deployable flow\nlooks something like the following example:\n\n```ts\nimport { genkit } from 'genkit';\nimport { googleAI } from '@genkit-ai/googleai';\nimport { onCallGenkit, hasClaim } from 'firebase-functions/https';\nimport { defineSecret } from 'firebase-functions/params';\n\nconst apiKey = defineSecret('GEMINI_API_KEY');\n\nconst ai = genkit({\n plugins: [googleAI()],\n model: googleAI.model('gemini-2.5-flash'),\n});\n\nconst generatePoemFlow = ai.defineFlow(\n {\n name: 'generatePoem',\n inputSchema: z.object({ subject: z.string() }),\n outputSchema: z.object({ poem: z.string() }),\n },\n async ({ subject }) => {\n const { text } = await ai.generate(`Compose a poem about ${subject}.`);\n return { poem: text };\n },\n);\n\nexport const generatePoem = onCallGenkit(\n {\n secrets: [apiKey],\n authPolicy: hasClaim('email_verified'),\n enforceAppCheck: true,\n },\n generatePoemFlow,\n);\n```\n\n## 3. Deploy flows to Firebase\n\nAfter you've defined flows using `onCallGenkit`, you can deploy them the same\nway you would deploy other Cloud Functions:\n\n```bash\ncd $PROJECT_ROOT\n\nfirebase deploy --only functions\n```\n\nYou've now deployed the flow as a Cloud Function! But you can't\naccess your deployed endpoint with `curl` or similar, because of the flow's\nauthorization policy. The next section explains how to securely access the\nflow.\n\n## Optional: Try the deployed flow\n\nTo try out your flow endpoint, you can deploy the following minimal example web\napp:\n\n1. In the [Project settings](https://console.firebase.google.com/project/_/settings/general)\n section of the Firebase console, add a new web app, selecting the option to\n also set up Hosting.\n\n1. In the\n [Authentication](https://console.firebase.google.com/project/_/authentication/providers)\n section of the Firebase console, enable the **Google** provider, used in\n this example.\n\n1. In your project directory, set up Firebase Hosting, where you will deploy\n the sample app:\n\n ```bash\n cd $PROJECT_ROOT\n\n firebase init hosting\n ```\n\n Accept the defaults for all of the prompts.\n\n1. Replace `public/index.html` with the following:\n\n ```html\n \n \n \n Genkit demo\n \n \n \n \n \n \n \n ```\n\n1. Deploy the web app and Cloud Function:\n\n ```bash\n cd $PROJECT_ROOT\n\n firebase deploy\n ```\n\nOpen the web app by visiting the URL printed by the `deploy` command. The app\nrequires you to sign in with a Google account, after which you can initiate\nendpoint requests.\n\n## Optional: Run flows in the developer UI\n\nYou can run flows defined using `onCallGenkit` in the developer UI, exactly the\nsame way as you run flows defined using `defineFlow`, so there's no need to\nswitch between the two between deployment and development.\n\n```bash\ncd $PROJECT_ROOT/functions\n\ngenkit start -- npx tsx --watch src/index.ts\n```\n\nor\n\n```bash\ncd $PROJECT_ROOT/functions\n\nnpm run genkit:start\n```\n\nYou can now navigate to the URL printed by the `genkit start` command to access.\n\n## Optional: Developing using Firebase Local Emulator Suite\n\nFirebase offers a\n[suite of emulators for local development](https://firebase.google.com/docs/emulator-suite), which you can\nuse with Genkit.\n\nTo use the Genkit Dev UI with the Firebase Emulator Suite, start the Firebase\nemulators as follows:\n\n```bash\ngenkit start -- firebase emulators:start --inspect-functions\n```\n\nThis command runs your code in the emulator, and runs the Genkit framework in\ndevelopment mode. This launches and exposes the Genkit reflection API (but not\nthe Dev UI).", "title": "Deploy flows using Cloud Functions for Firebase", - "lang": "js" + "description": "Learn how to deploy Genkit flows as callable functions using Cloud Functions for Firebase, including setup, authorization, and client-side access.", + "lang": "js", + "headers": "## Before you begin\n## 1. Set up a Firebase project\n## 2. Wrap the Flow in onCallGenkit\n### Define an authorization policy\n### Make API credentials available to deployed flows\n### Add App Check enforcement\n### Set a CORS policy\n### Complete example\n## 3. Deploy flows to Firebase\n## Optional: Try the deployed flow\n## Optional: Run flows in the developer UI\n## Optional: Developing using Firebase Local Emulator Suite\n" }, "js/flows.md": { - "text": "The core of your app's AI features are generative model requests, but it's rare\nthat you can simply take user input, pass it to the model, and display the model\noutput back to the user. Usually, there are pre- and post-processing steps that\nmust accompany the model call. For example:\n\n- Retrieving contextual information to send with the model call\n- Retrieving the history of the user's current session, for example in a chat\n app\n- Using one model to reformat the user input in a way that's suitable to pass\n to another model\n- Evaluating the \"safety\" of a model's output before presenting it to the user\n- Combining the output of several models\n\nEvery step of this workflow must work together for any AI-related task to\nsucceed.\n\nIn Genkit, you represent this tightly-linked logic using a construction called a\nflow. Flows are written just like functions, using ordinary TypeScript code, but\nthey add additional capabilities intended to ease the development of AI\nfeatures:\n\n- **Type safety**: Input and output schemas defined using Zod, which provides\n both static and runtime type checking\n- **Integration with developer UI**: Debug flows independently of your\n application code using the developer UI. In the developer UI, you can run\n flows and view traces for each step of the flow.\n- **Simplified deployment**: Deploy flows directly as web API endpoints, using\n Cloud Functions for Firebase or any platform that can host a web app.\n\nUnlike similar features in other frameworks, Genkit's flows are lightweight and\nunobtrusive, and don't force your app to conform to any specific abstraction.\nAll of the flow's logic is written in standard TypeScript, and code inside a\nflow doesn't need to be flow-aware.\n\n## Defining and calling flows\n\nIn its simplest form, a flow just wraps a function. The following example wraps\na function that calls `generate()`:\n\n```typescript\nexport const menuSuggestionFlow = ai.defineFlow(\n {\n name: 'menuSuggestionFlow',\n inputSchema: z.object({ theme: z.string() }),\n outputSchema: z.object({ menuItem: z.string() }),\n },\n async ({ theme }) => {\n const { text } = await ai.generate({\n model: googleAI.model('gemini-2.5-flash'),\n prompt: `Invent a menu item for a ${theme} themed restaurant.`,\n });\n return { text };\n },\n);\n```\n\nJust by wrapping your `generate()` calls like this, you add some functionality:\ndoing so lets you run the flow from the Genkit CLI and from the developer UI,\nand is a requirement for several of Genkit's features, including deployment and\nobservability (later sections discuss these topics).\n\n### Input and output schemas\n\nOne of the most important advantages Genkit flows have over directly calling a\nmodel API is type safety of both inputs and outputs. When defining flows, you\ncan define schemas for them using Zod, in much the same way as you define the\noutput schema of a `generate()` call; however, unlike with `generate()`, you can\nalso specify an input schema.\n\nWhile it's not mandatory to wrap your input and output schemas in `z.object()`, it's considered best practice for these reasons:\n\n- **Better developer experience**: Wrapping schemas in objects provides a better experience in the Developer UI by giving you labeled input fields. \n- **Future-proof API design**: Object-based schemas allow for easy extensibility in the future. You can add new fields to your input or output schemas without breaking existing clients, which is a core principle of robust API design.\n\nAll examples in this documentation use object-based schemas to follow these best practices.\n\nHere's a refinement of the last example, which defines a flow that takes a\nstring as input and outputs an object:\n\n```typescript\nimport { z } from 'genkit';\n\nconst MenuItemSchema = z.object({\n dishname: z.string(),\n description: z.string(),\n});\n\nexport const menuSuggestionFlowWithSchema = ai.defineFlow(\n {\n name: 'menuSuggestionFlow',\n inputSchema: z.object({ theme: z.string() }),\n outputSchema: MenuItemSchema,\n },\n async ({ theme }) => {\n const { output } = await ai.generate({\n model: googleAI.model('gemini-2.5-flash'),\n prompt: `Invent a menu item for a ${theme} themed restaurant.`,\n output: { schema: MenuItemSchema },\n });\n if (output == null) {\n throw new Error(\"Response doesn't satisfy schema.\");\n }\n return output;\n },\n);\n```\n\nNote that the schema of a flow does not necessarily have to line up with the\nschema of the `generate()` calls within the flow (in fact, a flow might not even\ncontain `generate()` calls). Here's a variation of the example that passes a\nschema to `generate()`, but uses the structured output to format a simple\nstring, which the flow returns.\n\n```typescript\nexport const menuSuggestionFlowMarkdown = ai.defineFlow(\n {\n name: 'menuSuggestionFlow',\n inputSchema: z.object({ theme: z.string() }),\n outputSchema: z.object({ formattedMenuItem: z.string() }),\n },\n async ({ theme }) => {\n const { output } = await ai.generate({\n model: googleAI.model('gemini-2.5-flash'),\n prompt: `Invent a menu item for a ${theme} themed restaurant.`,\n output: { schema: MenuItemSchema },\n });\n if (output == null) {\n throw new Error(\"Response doesn't satisfy schema.\");\n }\n return { \n formattedMenuItem: `**${output.dishname}**: ${output.description}`\n };\n },\n);\n```\n\n### Calling flows\n\nOnce you've defined a flow, you can call it from your Node.js code:\n\n```typescript\nconst { text } = await menuSuggestionFlow({ theme: 'bistro' });\n```\n\nThe argument to the flow must conform to the input schema, if you defined one.\n\nIf you defined an output schema, the flow response will conform to it. For\nexample, if you set the output schema to `MenuItemSchema`, the flow output will\ncontain its properties:\n\n```typescript\nconst { dishname, description } = await menuSuggestionFlowWithSchema({ theme: 'bistro' });\n```\n\n## Streaming flows\n\nFlows support streaming using an interface similar to `generate()`'s streaming\ninterface. Streaming is useful when your flow generates a large amount of\noutput, because you can present the output to the user as it's being generated,\nwhich improves the perceived responsiveness of your app. As a familiar example,\nchat-based LLM interfaces often stream their responses to the user as they are\ngenerated.\n\nHere's an example of a flow that supports streaming:\n\n```typescript\nexport const menuSuggestionStreamingFlow = ai.defineFlow(\n {\n name: 'menuSuggestionFlow',\n inputSchema: z.object({ theme: z.string() }),\n streamSchema: z.string(),\n outputSchema: z.object({ theme: z.string(), menuItem: z.string() }),\n },\n async ({ theme }, { sendChunk }) => {\n const { stream, response } = ai.generateStream({\n model: googleAI.model('gemini-2.5-flash'),\n prompt: `Invent a menu item for a ${theme} themed restaurant.`,\n });\n\n for await (const chunk of stream) {\n // Here, you could process the chunk in some way before sending it to\n // the output stream via sendChunk(). In this example, we output\n // the text of the chunk, unmodified.\n sendChunk(chunk.text);\n }\n\n const { text: menuItem } = await response;\n\n return {\n theme,\n menuItem,\n };\n },\n);\n```\n\n- The `streamSchema` option specifies the type of values your flow streams.\n This does not necessarily need to be the same type as the `outputSchema`,\n which is the type of the flow's complete output.\n- The second parameter to your flow definition is called `sideChannel`. It\n provides features such as request context and the `sendChunk` callback.\n The `sendChunk` callback takes a single parameter, of\n the type specified by `streamSchema`. Whenever data becomes available within\n your flow, send the data to the output stream by calling this function.\n\nIn the above example, the values streamed by the flow are directly coupled to\nthe values streamed by the `generate()` call inside the flow. Although this is\noften the case, it doesn't have to be: you can output values to the stream using\nthe callback as often as is useful for your flow.\n\n### Calling streaming flows\n\nStreaming flows are also callable, but they immediately return a response object\nrather than a promise:\n\n```typescript\nconst response = menuSuggestionStreamingFlow.stream({ theme: 'Danube' });\n```\n\nThe response object has a stream property, which you can use to iterate over the\nstreaming output of the flow as it's generated:\n\n```typescript\nfor await (const chunk of response.stream) {\n console.log('chunk', chunk);\n}\n```\n\nYou can also get the complete output of the flow, as you can with a\nnon-streaming flow:\n\n```typescript\nconst output = await response.output;\n```\n\nNote that the streaming output of a flow might not be the same type as the\ncomplete output; the streaming output conforms to `streamSchema`, whereas the\ncomplete output conforms to `outputSchema`.\n\n## Running flows from the command line\n\nYou can run flows from the command line using the Genkit CLI tool:\n\n```bash\ngenkit flow:run menuSuggestionFlow '{\"theme\": \"French\"}'\n```\n\nFor streaming flows, you can print the streaming output to the console by adding\nthe `-s` flag:\n\n```bash\ngenkit flow:run menuSuggestionFlow '{\"theme\": \"French\"}' -s\n```\n\nRunning a flow from the command line is useful for testing a flow, or for\nrunning flows that perform tasks needed on an ad hoc basis—for example, to\nrun a flow that ingests a document into your vector database.\n\n## Debugging flows\n\nOne of the advantages of encapsulating AI logic within a flow is that you can\ntest and debug the flow independently from your app using the Genkit developer\nUI.\n\nTo start the developer UI, run the following commands from your project\ndirectory:\n\n```bash\ngenkit start -- tsx --watch src/your-code.ts\n```\n\nFrom the **Run** tab of developer UI, you can run any of the flows defined in\nyour project:\n\n![Genkit DevUI flows](../../../assets/devui-flows.png)\n\nAfter you've run a flow, you can inspect a trace of the flow invocation by\neither clicking **View trace** or looking on the **Inspect** tab.\n\nIn the trace viewer, you can see details about the execution of the entire flow,\nas well as details for each of the individual steps within the flow. For\nexample, consider the following flow, which contains several generation\nrequests:\n\n```typescript\nconst PrixFixeMenuSchema = z.object({\n starter: z.string(),\n soup: z.string(),\n main: z.string(),\n dessert: z.string(),\n});\n\nexport const complexMenuSuggestionFlow = ai.defineFlow(\n {\n name: 'complexMenuSuggestionFlow',\n inputSchema: z.object({ theme: z.string() }),\n outputSchema: PrixFixeMenuSchema,\n },\n async ({ theme }): Promise> => {\n const chat = ai.chat({ model: googleAI.model('gemini-2.5-flash') });\n await chat.send('What makes a good prix fixe menu?');\n await chat.send(\n 'What are some ingredients, seasonings, and cooking techniques that ' + `would work for a ${theme} themed menu?`,\n );\n const { output } = await chat.send({\n prompt: `Based on our discussion, invent a prix fixe menu for a ${theme} ` + 'themed restaurant.',\n output: {\n schema: PrixFixeMenuSchema,\n },\n });\n if (!output) {\n throw new Error('No data generated.');\n }\n return output;\n },\n);\n```\n\nWhen you run this flow, the trace viewer shows you details about each generation\nrequest including its output:\n\n![Genkit DevUI flows](../../../assets/devui-inspect.png)\n\n### Flow steps\n\nIn the last example, you saw that each `generate()` call showed up as a separate\nstep in the trace viewer. Each of Genkit's fundamental actions show up as\nseparate steps of a flow:\n\n- `generate()`\n- `Chat.send()`\n- `embed()`\n- `index()`\n- `retrieve()`\n\nIf you want to include code other than the above in your traces, you can do so\nby wrapping the code in a `run()` call. You might do this for calls to\nthird-party libraries that are not Genkit-aware, or for any critical section of\ncode.\n\nFor example, here's a flow with two steps: the first step retrieves a menu using\nsome unspecified method, and the second step includes the menu as context for a\n`generate()` call.\n\n```ts\nexport const menuQuestionFlow = ai.defineFlow(\n {\n name: 'menuQuestionFlow',\n inputSchema: z.object({ question: z.string() }),\n outputSchema: z.object({ answer: z.string() }),\n },\n async ({ question }): Promise<{ answer: string }> => {\n const menu = await ai.run('retrieve-daily-menu', async (): Promise => {\n // Retrieve today's menu. (This could be a database access or simply\n // fetching the menu from your website.)\n\n // ...\n\n return menu;\n });\n const { text } = await ai.generate({\n model: googleAI.model('gemini-2.5-flash'),\n system: \"Help the user answer questions about today's menu.\",\n prompt: question,\n docs: [{ content: [{ text: menu }] }],\n });\n return { answer: text };\n },\n);\n```\n\nBecause the retrieval step is wrapped in a `run()` call, it's included as a step\nin the trace viewer:\n\n![Genkit DevUI flows](../../../assets/devui-runstep.png)\n\n## Deploying flows\n\nYou can deploy your flows directly as web API endpoints, ready for you to call\nfrom your app clients. Deployment is discussed in detail on several other pages,\nbut this section gives brief overviews of your deployment options.\n\n### Cloud Functions for Firebase\n\nTo deploy flows with Cloud Functions for Firebase, use the `onCallGenkit`\nfeature of `firebase-functions/https`. `onCallGenkit` wraps your flow in a\ncallable function. You may set an auth policy and configure App Check.\n\n```typescript\nimport { hasClaim, onCallGenkit } from 'firebase-functions/https';\nimport { defineSecret } from 'firebase-functions/params';\n\nconst apiKey = defineSecret('GOOGLE_AI_API_KEY');\n\nconst menuSuggestionFlow = ai.defineFlow(\n {\n name: 'menuSuggestionFlow',\n inputSchema: z.object({ theme: z.string() }),\n outputSchema: z.object({ menuItem: z.string() }),\n },\n async ({ theme }) => {\n // ...\n return { menuItem: \"Generated menu item would go here\" };\n },\n);\n\nexport const menuSuggestion = onCallGenkit(\n {\n secrets: [apiKey],\n authPolicy: hasClaim('email_verified'),\n },\n menuSuggestionFlow,\n);\n```\n\nFor more information, see the following pages:\n\n- [Deploy with Firebase](/docs/firebase)\n- [Authorization and integrity](/docs/auth#authorize-using-cloud-functions-for-firebase)\n- [Firebase plugin](/docs/plugins/firebase)\n\n### Express.js\n\nTo deploy flows using any Node.js hosting platform, such as Cloud Run, define\nyour flows using `defineFlow()` and then call `startFlowServer()`:\n\n```typescript\nimport { startFlowServer } from '@genkit-ai/express';\n\nexport const menuSuggestionFlow = ai.defineFlow(\n {\n name: 'menuSuggestionFlow',\n inputSchema: z.object({ theme: z.string() }),\n outputSchema: z.object({ result: z.string() }),\n },\n async ({ theme }) => {\n // ...\n },\n);\n\nstartFlowServer({\n flows: [menuSuggestionFlow],\n});\n```\n\nBy default, `startFlowServer` will serve all the flows defined in your codebase\nas HTTP endpoints (for example, `http://localhost:3400/menuSuggestionFlow`). You\ncan call a flow with a POST request as follows:\n\n```bash\ncurl -X POST \"http://localhost:3400/menuSuggestionFlow\" \\\n -H \"Content-Type: application/json\" -d '{\"data\": {\"theme\": \"banana\"}}'\n```\n\nIf needed, you can customize the flows server to serve a specific list of flows,\nas shown below. You can also specify a custom port (it will use the PORT\nenvironment variable if set) or specify CORS settings.\n\n```typescript\nexport const flowA = ai.defineFlow(\n { \n name: 'flowA',\n inputSchema: z.object({ subject: z.string() }),\n outputSchema: z.object({ response: z.string() }),\n }, \n async ({ subject }) => {\n // ...\n return { response: \"Generated response would go here\" };\n }\n);\n\nexport const flowB = ai.defineFlow(\n { \n name: 'flowB',\n inputSchema: z.object({ subject: z.string() }),\n outputSchema: z.object({ response: z.string() }),\n }, \n async ({ subject }) => {\n // ...\n return { response: \"Generated response would go here\" };\n }\n);\n\nstartFlowServer({\n flows: [flowB],\n port: 4567,\n cors: {\n origin: '*',\n },\n});\n```\n\nFor information on deploying to specific platforms, see\n[Deploy with Cloud Run](/docs/cloud-run) and\n[Deploy flows to any Node.js platform](/docs/deploy-node).\n", + "text": "# Defining AI workflows\n\nThe core of your app's AI features are generative model requests, but it's rare\nthat you can simply take user input, pass it to the model, and display the model\noutput back to the user. Usually, there are pre- and post-processing steps that\nmust accompany the model call. For example:\n\n- Retrieving contextual information to send with the model call\n- Retrieving the history of the user's current session, for example in a chat\n app\n- Using one model to reformat the user input in a way that's suitable to pass\n to another model\n- Evaluating the \"safety\" of a model's output before presenting it to the user\n- Combining the output of several models\n\nEvery step of this workflow must work together for any AI-related task to\nsucceed.\n\nIn Genkit, you represent this tightly-linked logic using a construction called a\nflow. Flows are written just like functions, using ordinary TypeScript code, but\nthey add additional capabilities intended to ease the development of AI\nfeatures:\n\n- **Type safety**: Input and output schemas defined using Zod, which provides\n both static and runtime type checking\n- **Integration with developer UI**: Debug flows independently of your\n application code using the developer UI. In the developer UI, you can run\n flows and view traces for each step of the flow.\n- **Simplified deployment**: Deploy flows directly as web API endpoints, using\n Cloud Functions for Firebase or any platform that can host a web app.\n\nUnlike similar features in other frameworks, Genkit's flows are lightweight and\nunobtrusive, and don't force your app to conform to any specific abstraction.\nAll of the flow's logic is written in standard TypeScript, and code inside a\nflow doesn't need to be flow-aware.\n\n## Defining and calling flows\n\nIn its simplest form, a flow just wraps a function. The following example wraps\na function that calls `generate()`:\n\n```typescript\nexport const menuSuggestionFlow = ai.defineFlow(\n {\n name: 'menuSuggestionFlow',\n inputSchema: z.object({ theme: z.string() }),\n outputSchema: z.object({ menuItem: z.string() }),\n },\n async ({ theme }) => {\n const { text } = await ai.generate({\n model: googleAI.model('gemini-2.5-flash'),\n prompt: `Invent a menu item for a ${theme} themed restaurant.`,\n });\n return { menuItem: text };\n },\n);\n```\n\nJust by wrapping your `generate()` calls like this, you add some functionality:\ndoing so lets you run the flow from the Genkit CLI and from the developer UI,\nand is a requirement for several of Genkit's features, including deployment and\nobservability (later sections discuss these topics).\n\n### Input and output schemas\n\nOne of the most important advantages Genkit flows have over directly calling a\nmodel API is type safety of both inputs and outputs. When defining flows, you\ncan define schemas for them using Zod, in much the same way as you define the\noutput schema of a `generate()` call; however, unlike with `generate()`, you can\nalso specify an input schema.\n\nWhile it's not mandatory to wrap your input and output schemas in `z.object()`, it's considered best practice for these reasons:\n\n- **Better developer experience**: Wrapping schemas in objects provides a better experience in the Developer UI by giving you labeled input fields. \n- **Future-proof API design**: Object-based schemas allow for easy extensibility in the future. You can add new fields to your input or output schemas without breaking existing clients, which is a core principle of robust API design.\n\nAll examples in this documentation use object-based schemas to follow these best practices.\n\nHere's a refinement of the last example, which defines a flow that takes a\nstring as input and outputs an object:\n\n```typescript\nimport { z } from 'genkit';\n\nconst MenuItemSchema = z.object({\n dishname: z.string(),\n description: z.string(),\n});\n\nexport const menuSuggestionFlowWithSchema = ai.defineFlow(\n {\n name: 'menuSuggestionFlow',\n inputSchema: z.object({ theme: z.string() }),\n outputSchema: MenuItemSchema,\n },\n async ({ theme }) => {\n const { output } = await ai.generate({\n model: googleAI.model('gemini-2.5-flash'),\n prompt: `Invent a menu item for a ${theme} themed restaurant.`,\n output: { schema: MenuItemSchema },\n });\n if (output == null) {\n throw new Error(\"Response doesn't satisfy schema.\");\n }\n return output;\n },\n);\n```\n\nNote that the schema of a flow does not necessarily have to line up with the\nschema of the `generate()` calls within the flow (in fact, a flow might not even\ncontain `generate()` calls). Here's a variation of the example that passes a\nschema to `generate()`, but uses the structured output to format a simple\nstring, which the flow returns.\n\n```typescript\nexport const menuSuggestionFlowMarkdown = ai.defineFlow(\n {\n name: 'menuSuggestionFlow',\n inputSchema: z.object({ theme: z.string() }),\n outputSchema: z.object({ formattedMenuItem: z.string() }),\n },\n async ({ theme }) => {\n const { output } = await ai.generate({\n model: googleAI.model('gemini-2.5-flash'),\n prompt: `Invent a menu item for a ${theme} themed restaurant.`,\n output: { schema: MenuItemSchema },\n });\n if (output == null) {\n throw new Error(\"Response doesn't satisfy schema.\");\n }\n return { \n formattedMenuItem: `**${output.dishname}**: ${output.description}`\n };\n },\n);\n```\n\n### Calling flows\n\nOnce you've defined a flow, you can call it from your Node.js code:\n\n```typescript\nconst { text } = await menuSuggestionFlow({ theme: 'bistro' });\n```\n\nThe argument to the flow must conform to the input schema, if you defined one.\n\nIf you defined an output schema, the flow response will conform to it. For\nexample, if you set the output schema to `MenuItemSchema`, the flow output will\ncontain its properties:\n\n```typescript\nconst { dishname, description } = await menuSuggestionFlowWithSchema({ theme: 'bistro' });\n```\n\n## Streaming flows\n\nFlows support streaming using an interface similar to `generate()`'s streaming\ninterface. Streaming is useful when your flow generates a large amount of\noutput, because you can present the output to the user as it's being generated,\nwhich improves the perceived responsiveness of your app. As a familiar example,\nchat-based LLM interfaces often stream their responses to the user as they are\ngenerated.\n\nHere's an example of a flow that supports streaming:\n\n```typescript\nexport const menuSuggestionStreamingFlow = ai.defineFlow(\n {\n name: 'menuSuggestionFlow',\n inputSchema: z.object({ theme: z.string() }),\n streamSchema: z.string(),\n outputSchema: z.object({ theme: z.string(), menuItem: z.string() }),\n },\n async ({ theme }, { sendChunk }) => {\n const { stream, response } = ai.generateStream({\n model: googleAI.model('gemini-2.5-flash'),\n prompt: `Invent a menu item for a ${theme} themed restaurant.`,\n });\n\n for await (const chunk of stream) {\n // Here, you could process the chunk in some way before sending it to\n // the output stream via sendChunk(). In this example, we output\n // the text of the chunk, unmodified.\n sendChunk(chunk.text);\n }\n\n const { text: menuItem } = await response;\n\n return {\n theme,\n menuItem,\n };\n },\n);\n```\n\n- The `streamSchema` option specifies the type of values your flow streams.\n This does not necessarily need to be the same type as the `outputSchema`,\n which is the type of the flow's complete output.\n- The second parameter to your flow definition is called `sideChannel`. It\n provides features such as request context and the `sendChunk` callback.\n The `sendChunk` callback takes a single parameter, of\n the type specified by `streamSchema`. Whenever data becomes available within\n your flow, send the data to the output stream by calling this function.\n\nIn the above example, the values streamed by the flow are directly coupled to\nthe values streamed by the `generate()` call inside the flow. Although this is\noften the case, it doesn't have to be: you can output values to the stream using\nthe callback as often as is useful for your flow.\n\n### Calling streaming flows\n\nStreaming flows are also callable, but they immediately return a response object\nrather than a promise:\n\n```typescript\nconst response = menuSuggestionStreamingFlow.stream({ theme: 'Danube' });\n```\n\nThe response object has a stream property, which you can use to iterate over the\nstreaming output of the flow as it's generated:\n\n```typescript\nfor await (const chunk of response.stream) {\n console.log('chunk', chunk);\n}\n```\n\nYou can also get the complete output of the flow, as you can with a\nnon-streaming flow:\n\n```typescript\nconst output = await response.output;\n```\n\nNote that the streaming output of a flow might not be the same type as the\ncomplete output; the streaming output conforms to `streamSchema`, whereas the\ncomplete output conforms to `outputSchema`.\n\n## Running flows from the command line\n\nYou can run flows from the command line using the Genkit CLI tool:\n\n```bash\ngenkit flow:run menuSuggestionFlow '{\"theme\": \"French\"}'\n```\n\nFor streaming flows, you can print the streaming output to the console by adding\nthe `-s` flag:\n\n```bash\ngenkit flow:run menuSuggestionFlow '{\"theme\": \"French\"}' -s\n```\n\nRunning a flow from the command line is useful for testing a flow, or for\nrunning flows that perform tasks needed on an ad hoc basis—for example, to\nrun a flow that ingests a document into your vector database.\n\n## Debugging flows\n\nOne of the advantages of encapsulating AI logic within a flow is that you can\ntest and debug the flow independently from your app using the Genkit developer\nUI.\n\nTo start the developer UI, run the following commands from your project\ndirectory:\n\n```bash\ngenkit start -- tsx --watch src/your-code.ts\n```\n\nFrom the **Run** tab of developer UI, you can run any of the flows defined in\nyour project:\n\n![Genkit DevUI flows](../../../assets/devui-flows.png)\n\nAfter you've run a flow, you can inspect a trace of the flow invocation by\neither clicking **View trace** or looking on the **Inspect** tab.\n\nIn the trace viewer, you can see details about the execution of the entire flow,\nas well as details for each of the individual steps within the flow. For\nexample, consider the following flow, which contains several generation\nrequests:\n\n```typescript\nconst PrixFixeMenuSchema = z.object({\n starter: z.string(),\n soup: z.string(),\n main: z.string(),\n dessert: z.string(),\n});\n\nexport const complexMenuSuggestionFlow = ai.defineFlow(\n {\n name: 'complexMenuSuggestionFlow',\n inputSchema: z.object({ theme: z.string() }),\n outputSchema: PrixFixeMenuSchema,\n },\n async ({ theme }): Promise> => {\n const chat = ai.chat({ model: googleAI.model('gemini-2.5-flash') });\n await chat.send('What makes a good prix fixe menu?');\n await chat.send(\n 'What are some ingredients, seasonings, and cooking techniques that ' + `would work for a ${theme} themed menu?`,\n );\n const { output } = await chat.send({\n prompt: `Based on our discussion, invent a prix fixe menu for a ${theme} ` + 'themed restaurant.',\n output: {\n schema: PrixFixeMenuSchema,\n },\n });\n if (!output) {\n throw new Error('No data generated.');\n }\n return output;\n },\n);\n```\n\nWhen you run this flow, the trace viewer shows you details about each generation\nrequest including its output:\n\n![Genkit DevUI flows](../../../assets/devui-inspect.png)\n\n### Flow steps\n\nIn the last example, you saw that each `generate()` call showed up as a separate\nstep in the trace viewer. Each of Genkit's fundamental actions show up as\nseparate steps of a flow:\n\n- `generate()`\n- `Chat.send()`\n- `embed()`\n- `index()`\n- `retrieve()`\n\nIf you want to include code other than the above in your traces, you can do so\nby wrapping the code in a `run()` call. You might do this for calls to\nthird-party libraries that are not Genkit-aware, or for any critical section of\ncode.\n\nFor example, here's a flow with two steps: the first step retrieves a menu using\nsome unspecified method, and the second step includes the menu as context for a\n`generate()` call.\n\n```ts\nexport const menuQuestionFlow = ai.defineFlow(\n {\n name: 'menuQuestionFlow',\n inputSchema: z.object({ question: z.string() }),\n outputSchema: z.object({ answer: z.string() }),\n },\n async ({ question }): Promise<{ answer: string }> => {\n const menu = await ai.run('retrieve-daily-menu', async (): Promise => {\n // Retrieve today's menu. (This could be a database access or simply\n // fetching the menu from your website.)\n\n // ...\n\n return menu;\n });\n const { text } = await ai.generate({\n model: googleAI.model('gemini-2.5-flash'),\n system: \"Help the user answer questions about today's menu.\",\n prompt: question,\n docs: [{ content: [{ text: menu }] }],\n });\n return { answer: text };\n },\n);\n```\n\nBecause the retrieval step is wrapped in a `run()` call, it's included as a step\nin the trace viewer:\n\n![Genkit DevUI flows](../../../assets/devui-runstep.png)\n\n## Deploying flows\n\nYou can deploy your flows directly as web API endpoints, ready for you to call\nfrom your app clients. Deployment is discussed in detail on several other pages,\nbut this section gives brief overviews of your deployment options.\n\n### Cloud Functions for Firebase\n\nTo deploy flows with Cloud Functions for Firebase, use the `onCallGenkit`\nfeature of `firebase-functions/https`. `onCallGenkit` wraps your flow in a\ncallable function. You may set an auth policy and configure App Check.\n\n```typescript\nimport { hasClaim, onCallGenkit } from 'firebase-functions/https';\nimport { defineSecret } from 'firebase-functions/params';\n\nconst apiKey = defineSecret('GOOGLE_AI_API_KEY');\n\nconst menuSuggestionFlow = ai.defineFlow(\n {\n name: 'menuSuggestionFlow',\n inputSchema: z.object({ theme: z.string() }),\n outputSchema: z.object({ menuItem: z.string() }),\n },\n async ({ theme }) => {\n // ...\n return { menuItem: \"Generated menu item would go here\" };\n },\n);\n\nexport const menuSuggestion = onCallGenkit(\n {\n secrets: [apiKey],\n authPolicy: hasClaim('email_verified'),\n },\n menuSuggestionFlow,\n);\n```\n\nFor more information, see the following pages:\n\n- [Deploy with Firebase](/docs/firebase)\n- [Authorization and integrity](/docs/auth#authorize-using-cloud-functions-for-firebase)\n- [Firebase plugin](/docs/plugins/firebase)\n\n### Express.js\n\nTo deploy flows using any Node.js hosting platform, such as Cloud Run, define\nyour flows using `defineFlow()` and then call `startFlowServer()`:\n\n```typescript\nimport { startFlowServer } from '@genkit-ai/express';\n\nexport const menuSuggestionFlow = ai.defineFlow(\n {\n name: 'menuSuggestionFlow',\n inputSchema: z.object({ theme: z.string() }),\n outputSchema: z.object({ result: z.string() }),\n },\n async ({ theme }) => {\n // ...\n },\n);\n\nstartFlowServer({\n flows: [menuSuggestionFlow],\n});\n```\n\nBy default, `startFlowServer` will serve all the flows defined in your codebase\nas HTTP endpoints (for example, `http://localhost:3400/menuSuggestionFlow`). You\ncan call a flow with a POST request as follows:\n\n```bash\ncurl -X POST \"http://localhost:3400/menuSuggestionFlow\" \\\n -H \"Content-Type: application/json\" -d '{\"data\": {\"theme\": \"banana\"}}'\n```\n\nIf needed, you can customize the flows server to serve a specific list of flows,\nas shown below. You can also specify a custom port (it will use the PORT\nenvironment variable if set) or specify CORS settings.\n\n```typescript\nexport const flowA = ai.defineFlow(\n { \n name: 'flowA',\n inputSchema: z.object({ subject: z.string() }),\n outputSchema: z.object({ response: z.string() }),\n }, \n async ({ subject }) => {\n // ...\n return { response: \"Generated response would go here\" };\n }\n);\n\nexport const flowB = ai.defineFlow(\n { \n name: 'flowB',\n inputSchema: z.object({ subject: z.string() }),\n outputSchema: z.object({ response: z.string() }),\n }, \n async ({ subject }) => {\n // ...\n return { response: \"Generated response would go here\" };\n }\n);\n\nstartFlowServer({\n flows: [flowB],\n port: 4567,\n cors: {\n origin: '*',\n },\n});\n```\n\nFor information on deploying to specific platforms, see\n[Deploy with Cloud Run](/docs/cloud-run) and\n[Deploy flows to any Node.js platform](/docs/deploy-node).\n", "title": "Defining AI workflows", - "lang": "js" + "description": "Learn how to define and manage AI workflows in Genkit using flows, which provide type safety, integration with the developer UI, and simplified deployment.", + "lang": "js", + "headers": "## Defining and calling flows\n### Input and output schemas\n### Calling flows\n## Streaming flows\n### Calling streaming flows\n## Running flows from the command line\n## Debugging flows\n### Flow steps\n## Deploying flows\n### Cloud Functions for Firebase\n### Express.js\n" }, "js/get-started.md": { - "text": "This guide shows you how to get started with Genkit in a Node.js app.\n\n## Prerequisites\n\nThis guide assumes that you're familiar with building applications with Node.js.\n\nTo complete this quickstart, make sure that your development environment meets\nthe following requirements:\n\n- Node.js v20+\n- npm\n\n## Install Genkit dependencies\n\nInstall the following Genkit dependencies to use Genkit in your project:\n\n- `genkit` provides Genkit core capabilities.\n- `@genkit-ai/googleai` provides access to the Google AI Gemini models.\n\n```bash\nnpm install genkit @genkit-ai/googleai\n```\n\n## Configure your model API key\n\nFor this guide, we’ll show you how to use the Gemini API which provides a\ngenerous free tier and does not require a credit card to get started. To use the\nGemini API, you'll need an API key. If you don't already have one, create a key\nin Google AI Studio.\n\n[Get an API key from Google AI Studio](https://makersuite.google.com/app/apikey).\n\nAfter you’ve created an API key, set the `GEMINI_API_KEY` environment variable to your key:\n\n```sh\nexport GEMINI_API_KEY=\n```\n\n:::note\nWhile this tutorial uses the Gemini API from AI Studio, Genkit supports a wide variety of model providers including Gemini from Vertex AI, Anthropic’s Claude 3 models and Llama 3.1 through the Vertex AI Model Garden, open source models through Ollama, and several other community-supported providers like OpenAI and Cohere. See [Models supported by Genkit](/docs/models#models-supported-by-genkit) for details.\n:::\n\n## Make your first request\n\nGet started with Genkit in just a few lines of simple code.\n\n```ts\n// import the Genkit and Google AI plugin libraries\nimport { googleAI } from '@genkit-ai/googleai';\nimport { genkit } from 'genkit';\n\n// configure a Genkit instance\nconst ai = genkit({\n plugins: [googleAI()],\n model: googleAI.model('gemini-2.5-flash'), // set default model\n});\n\nasync function main() {\n // make a generation request\n const { text } = await ai.generate('Hello, Gemini!');\n console.log(text);\n}\n\nmain();\n```\n\n## Next steps\n\nNow that you’re set up to make model requests with Genkit, learn how to use more\nGenkit capabilities to build your AI-powered apps and workflows. To get started\nwith additional Genkit capabilities, see the following guides:\n\n- [Developer tools](/docs/devtools): Learn how to set up and use\n Genkit’s CLI and developer UI to help you locally test and debug your app.\n- [Generating content](/docs/models): Learn how to use Genkit’s unified\n generation API to generate text and structured data from any supported\n model.\n- [Creating flows](/docs/flows): Learn how to use special Genkit\n functions, called flows, that provide end-to-end observability for workflows\n and rich debugging from Genkit tooling.\n- [Managing prompts](/docs/dotprompt): Learn how Genkit helps you manage\n your prompts and configuration together as code.\n- [Integrating in an app](https://developers.google.com/solutions/learn/agentic-barista):\n Walk through a deployed example of multiple Genkit flows powering a web app.\n", + "text": "# Get started with Genkit JS\n\nThis guide shows you how to get started with Genkit in a Node.js app and test it in the Developer UI.\n\n## Prerequisites\n\nBefore you begin, make sure your environment meets these requirements:\n\n- Node.js v20 or later\n- npm\n\nThis guide assumes you're already familiar with building Node.js applications.\n\n## Set up your project\n\nCreate a new Node.js project and configure TypeScript:\n\n```sh\nmkdir my-genkit-app\ncd my-genkit-app\nnpm init -y\n\n# Set up your source directory\nmkdir src\ntouch src/index.ts\n\n# Install and configure TypeScript\nnpm install -D typescript tsx\nnpx tsc --init\n```\n\nThis sets up your project structure and a TypeScript entry point at `src/index.ts`.\n\n## Install Genkit packages\n\nFirst, install the Genkit CLI globally. This gives you access to local developer tools, including the Developer UI:\n\n```bash\nnpm install -g genkit-cli\n```\n\nThen, add the following packages to your project:\n\n```bash\nnpm install genkit @genkit-ai/googleai\n```\n\n- `genkit` provides Genkit core capabilities.\n- `@genkit-ai/googleai` provides access to the Google AI Gemini models.\n\n## Configure your model API key\n\nGenkit can work with multiple model providers. This guide uses the **Gemini API**, which offers a generous free tier and doesn’t require a credit card to get started.\n\nTo use it, you'll need an API key from Google AI Studio:\n\n\n Get a Gemini API Key\n\n\nOnce you have a key, set the `GEMINI_API_KEY` environment variable:\n\n```sh\nexport GEMINI_API_KEY=\n```\n\n:::note\nGenkit also supports models from Vertex AI, Anthropic, OpenAI, Cohere, Ollama, and more. See [supported models](/docs/models#models-supported-by-genkit) for details.\n:::\n\n## Create your first flow\n\nA flow is a special Genkit function with built-in observability. type safety, and tooling integration.\n\nUpdate `src/index.ts` with the following:\n\n```ts\nimport { googleAI } from '@genkit-ai/googleai';\nimport { genkit, z } from 'genkit';\n\n// Initialize Genkit with the Google AI plugin\nconst ai = genkit({\n plugins: [googleAI()],\n model: googleAI.model('gemini-2.5-flash', {\n temperature: 0.8 \n }),\n});\n\n// Define input schema\nconst RecipeInputSchema = z.object({\n ingredient: z.string().describe('Main ingredient or cuisine type'),\n dietaryRestrictions: z.string().optional().describe('Any dietary restrictions'),\n});\n\n// Define output schema\nconst RecipeSchema = z.object({\n title: z.string(),\n description: z.string(),\n prepTime: z.string(),\n cookTime: z.string(),\n servings: z.number(),\n ingredients: z.array(z.string()),\n instructions: z.array(z.string()),\n tips: z.array(z.string()).optional(),\n});\n\n// Define a recipe generator flow\nexport const recipeGeneratorFlow = ai.defineFlow(\n {\n name: 'recipeGeneratorFlow',\n inputSchema: RecipeInputSchema,\n outputSchema: RecipeSchema,\n },\n async (input) => {\n // Create a prompt based on the input\n const prompt = `Create a recipe with the following requirements:\n Main ingredient: ${input.ingredient}\n Dietary restrictions: ${input.dietaryRestrictions || 'none'}`;\n\n // Generate structured recipe data using the same schema\n const { output } = await ai.generate({\n prompt,\n output: { schema: RecipeSchema },\n });\n \n if (!output) throw new Error('Failed to generate recipe');\n\n return output;\n }\n);\n\n// Run the flow\nasync function main() {\n const recipe = await recipeGeneratorFlow({ \n ingredient: 'avocado', \n dietaryRestrictions: 'vegetarian' \n });\n \n console.log(recipe);\n}\n\nmain().catch(console.error);\n```\n\nThis code sample:\n\n- Defines reusable input and output schemas with [Zod](https://zod.dev/)\n- Configures the `gemini-2.5-flash` model with temperature settings\n- Defines a Genkit flow to generate a structured recipe based on your input\n- Runs the flow with a sample input and prints the result\n\n##### Why use flows?\n\n- Type-safe inputs and outputs\n- Integrates with the Developer UI\n- Easy deployment as APIs\n- Built-in tracing and observability\n\n## Test in the Developer UI\n\nThe **Developer UI** is a local tool for testing and inspecting Genkit components, like flows, with a visual interface.\n\n### Start the Developer UI\n\nRun the following command from your project root:\n\n```bash\ngenkit start -- npx tsx --watch src/index.ts\n```\n\nThis starts your app and launches the Developer UI at `http://localhost:4000` by default.\n\n:::note\nThe command after `--` should run the file that defines or imports your Genkit components. You can use `tsx`, `node`, or other commands based on your setup. Learn more in [developer tools](/docs/devtools).\n:::\n\n##### Optional: Add an npm script\n\nTo make starting the Developer UI easier, add the following to your `package.json` scripts:\n\n```json\n\"scripts\": {\n \"genkit:ui\": \"genkit start -- npx tsx --watch src/index.ts\"\n}\n```\n\nThen run it with:\n\n```sh\nnpm run genkit:ui\n```\n\n### Run and inspect the flow\n\nIn the Developer UI:\n\n1. Select the `recipeGeneratorFlow` from the list of flows\n1. Enter sample input:\n ```json\n {\n \"ingredient\": \"avocado\",\n \"dietaryRestrictions\": \"vegetarian\"\n }\n ```\n1. Click **Run**\n\nYou’ll see the generated recipe as structured output, along with a visual trace of the AI generation process for debugging and optimization.\n\n\n \n Your browser does not support the video tag.\n\n\n## Next steps\n\nNow that you’ve created and tested your first flow, explore more features to build powerful AI-driven applications:\n\n- [Developer tools](/docs/devtools): Set up your local workflow with the Genkit CLI and Dev UI.\n- [Generating content](/docs/models): Use Genkit's unified generation API to work with multimodal and structured output across supported models.\n- [Defining flows](/docs/flows): Learn about streaming flows, schema customization, deployment options, and more.\n- [Prompt management](/docs/dotprompt): Define flexible prompt templates using `.prompt` files or code.\n- [App integration](https://developers.google.com/solutions/learn/agentic-barista): See a full-stack Genkit app example built with flows and the Gemini API.", "title": "Get started with Genkit JS", - "lang": "js" + "description": "This guide shows you how to get started with Genkit in a Node.js application, including project setup, installing packages, configuring API keys, creating your first flow, and testing in the Developer UI.", + "lang": "js", + "headers": "## Prerequisites\n## Set up your project\n# Set up your source directory\n# Install and configure TypeScript\n## Install Genkit packages\n## Configure your model API key\n## Create your first flow\n##### Why use flows?\n## Test in the Developer UI\n### Start the Developer UI\n##### Optional: Add an npm script\n### Run and inspect the flow\n## Next steps\n" }, - "js/interrupts.mdx": { - "text": "import ExampleLink from '@/components/ExampleLink.astro';\nimport { Tabs, TabItem } from '@astrojs/starlight/components';\n\n:::caution[Beta]\nThis feature of Genkit is in **Beta,** which means it is not yet part of Genkit's stable API. APIs of beta features may change in minor version releases.\n:::\n\n_Interrupts_ are a special kind of [tool](/docs/tool-calling) that can pause the\nLLM generation-and-tool-calling loop to return control back to you. When\nyou're ready, you can then _resume_ generation by sending _replies_ that the LLM\nprocesses for further generation.\n\nThe most common uses for interrupts fall into a few categories:\n\n- **Human-in-the-Loop:** Enabling the user of an interactive AI\n to clarify needed information or confirm the LLM's action\n before it is completed, providing a measure of safety and confidence.\n- **Async Processing:** Starting an asynchronous task that can only be\n completed out-of-band, such as sending an approval notification to\n a human reviewer or kicking off a long-running background process.\n- **Exit from an Autonomous Task:** Providing the model a way\n to mark a task as complete, in a workflow that might iterate through\n a long series of tool calls.\n\n\n\n## Before you begin\n\nAll of the examples documented here assume that you have already set up a\nproject with Genkit dependencies installed. If you want to run the code\nexamples on this page, first complete the steps in the\n[Get started](/docs/get-started) guide.\n\nBefore diving too deeply, you should also be familiar with the following\nconcepts:\n\n- [Generating content](/docs/models) with AI models.\n- Genkit's system for [defining input and output schemas](/docs/flows).\n- General methods of [tool-calling](/docs/tool-calling).\n\n## Overview of interrupts\n\nAt a high level, this is what an interrupt looks like when\ninteracting with an LLM:\n\n1. The calling application prompts the LLM with a request. The prompt includes\n a list of tools, including at least one for an interrupt that the LLM\n can use to generate a response.\n2. The LLM either generates either a complete response or a tool call request\n in a specific format. To the LLM, an interrupt call looks like any\n other tool call.\n3. If the LLM calls an interrupt tool,\n the Genkit library automatically pauses generation rather than immediately\n passing responses back to the model for additional processing.\n4. The developer checks whether an interrupt call is made, and performs whatever\n task is needed to collect the information needed for the interrupt response.\n5. The developer resumes generation by passing an interrupt response to the\n model. This action triggers a return to Step 2.\n\n## Define manual-response interrupts\n\nThe most common kind of interrupt allows the LLM to request clarification from\nthe user, for example by asking a multiple-choice question.\n\nFor this use case, use the Genkit instance's `defineInterrupt()` method:\n\n```ts\nimport { genkit, z } from 'genkit';\nimport { googleAI } from '@genkitai/google-ai';\n\nconst ai = genkit({\n plugins: [googleAI()],\n model: googleAI.model('gemini-2.5-flash'),\n});\n\nconst askQuestion = ai.defineInterrupt({\n name: 'askQuestion',\n description: 'use this to ask the user a clarifying question',\n inputSchema: z.object({\n choices: z.array(z.string()).describe('the choices to display to the user'),\n allowOther: z.boolean().optional().describe('when true, allow write-ins'),\n }),\n outputSchema: z.string(),\n});\n```\n\nNote that the `outputSchema` of an interrupt corresponds to the response data\nyou will provide as opposed to something that will be automatically populated\nby a tool function.\n\n### Use interrupts\n\nInterrupts are passed into the `tools` array when generating content, just like\nother types of tools. You can pass both normal tools and interrupts to the\nsame `generate` call:\n\n\n \n ```ts\n const response = await ai.generate({\n prompt: \"Ask me a movie trivia question.\",\n tools: [askQuestion],\n });\n ```\n \n \n ```ts\n const triviaPrompt = ai.definePrompt({\n name: \"triviaPrompt\",\n tools: [askQuestion],\n input: {\n schema: z.object({ subject: z.string() }),\n },\n prompt: \"Ask me a trivia question about {{subject}}.\",\n });\n\n const response = await triviaPrompt({ subject: \"computer history\" });\n ```\n\n \n \n ```dotprompt\n ---\n tools: [askQuestion]\n input:\n schema:\n partyType: string\n ---\n\n {{role \"system\"}}\n Use the askQuestion tool if you need to clarify something.\n\n {{role \"user\"}}\n Help me plan a\n {{partyType}}\n party next week.\n ```\n\n Then you can execute the prompt in your code as follows:\n\n ```ts\n // assuming prompt file is named partyPlanner.prompt\n const partyPlanner = ai.prompt(\"partyPlanner\");\n\n const response = await partyPlanner({ partyType: \"birthday\" });\n ```\n\n \n \n ```ts\n const chat = ai.chat({\n system: \"Use the askQuestion tool if you need to clarify something.\",\n tools: [askQuestion],\n });\n\n const response = await chat.send(\"make a plan for my birthday party\");\n ```\n\n \n\n\nGenkit immediately returns a response on receipt of an interrupt tool call.\n\n### Respond to interrupts\n\nIf you've passed one or more interrupts to your generate call, you\nneed to check the response for interrupts so that you can handle them:\n\n```ts\n// you can check the 'finishReason' of the response\nresponse.finishReason === 'interrupted';\n// or you can check to see if any interrupt requests are on the response\nresponse.interrupts.length > 0;\n```\n\nResponding to an interrupt is done using the `resume` option on a subsequent\n`generate` call, making sure to pass in the existing history. Each tool has\na `.respond()` method on it to help construct the response.\n\nOnce resumed, the model re-enters the generation loop, including tool\nexecution, until either it completes or another interrupt is triggered:\n\n```ts\nlet response = await ai.generate({\n tools: [askQuestion],\n system: 'ask clarifying questions until you have a complete solution',\n prompt: 'help me plan a backyard BBQ',\n});\n\nwhile (response.interrupts.length) {\n const answers = [];\n // multiple interrupts can be called at once, so we handle them all\n for (const question in response.interrupts) {\n answers.push(\n // use the `respond` method on our tool to populate answers\n askQuestion.respond(\n question,\n // send the tool request input to the user to respond\n await askUser(question.toolRequest.input),\n ),\n );\n }\n\n response = await ai.generate({\n tools: [askQuestion],\n messages: response.messages,\n resume: {\n respond: answers,\n },\n });\n}\n\n// no more interrupts, we can see the final response\nconsole.log(response.text);\n```\n\n## Tools with restartable interrupts\n\nAnother common pattern for interrupts is the need to _confirm_ an action that\nthe LLM suggests before actually performing it. For example, a payments app\nmight want the user to confirm certain kinds of transfers.\n\nFor this use case, you can use the standard `defineTool` method to add custom\nlogic around when to trigger an interrupt, and what to do when an interrupt is\n_restarted_ with additional metadata.\n\n### Define a restartable tool\n\nEvery tool has access to two special helpers in the second argument of its\nimplementation definition:\n\n- `interrupt`: when called, this method throws a special kind of exception that\n is caught to pause the generation loop. You can provide additional metadata\n as an object.\n- `resumed`: when a request from an interrupted generation is restarted using\n the `{resume: {restart: ...}}` option (see below), this helper contains the\n metadata provided when restarting.\n\nIf you were building a payments app, for example, you might want to confirm with\nthe user before making a transfer exceeding a certain amount:\n\n```ts\nconst transferMoney = ai.defineTool({\n name: 'transferMoney',\n description: 'Transfers money between accounts.',\n inputSchema: z.object({\n toAccountId: z.string().describe('the account id of the transfer destination'),\n amount: z.number().describe('the amount in integer cents (100 = $1.00)'),\n }),\n outputSchema: z.object({\n status: z.string().describe('the outcome of the transfer'),\n message: z.string().optional(),\n })\n}, async (input, {context, interrupt, resumed})) {\n // if the user rejected the transaction\n if (resumed?.status === \"REJECTED\") {\n return {status: 'REJECTED', message: 'The user rejected the transaction.'};\n }\n // trigger an interrupt to confirm if amount > $100\n if (resumed?.status !== \"APPROVED\" && input.amount > 10000) {\n interrupt({\n message: \"Please confirm sending an amount > $100.\",\n });\n }\n // complete the transaction if not interrupted\n return doTransfer(input);\n}\n```\n\nIn this example, on first execution (when `resumed` is undefined), the tool\nchecks to see if the amount exceeds $100, and triggers an interrupt if so. On\nsecond execution, it looks for a status in the new metadata provided and\nperforms the transfer or returns a rejection response, depending on whether it\nis approved or rejected.\n\n### Restart tools after interruption\n\nInterrupt tools give you full control over:\n\n1. When an initial tool request should trigger an interrupt.\n2. When and whether to resume the generation loop.\n3. What additional information to provide to the tool when resuming.\n\nIn the example shown in the previous section, the application might ask the user\nto confirm the interrupted request to make sure the transfer amount is okay:\n\n```ts\nlet response = await ai.generate({\n tools: [transferMoney],\n prompt: \"Transfer $1000 to account ABC123\",\n});\n\nwhile (response.interrupts.length) {\n const confirmations = [];\n // multiple interrupts can be called at once, so we handle them all\n for (const interrupt in response.interrupts) {\n confirmations.push(\n // use the 'restart' method on our tool to provide `resumed` metadata\n transferMoney.restart(\n interrupt,\n // send the tool request input to the user to respond. assume that this\n // returns `{status: \"APPROVED\"}` or `{status: \"REJECTED\"}`\n await requestConfirmation(interrupt.toolRequest.input);\n )\n );\n }\n\n response = await ai.generate({\n tools: [transferMoney],\n messages: response.messages,\n resume: {\n restart: confirmations,\n }\n })\n}\n\n// no more interrupts, we can see the final response\nconsole.log(response.text);\n```\n", + "js/interrupts.md": { + "text": "# Pause generation using interrupts\n\n:::caution[Beta]\nThis feature of Genkit is in **Beta,** which means it is not yet part of Genkit's stable API. APIs of beta features may change in minor version releases.\n:::\n\n_Interrupts_ are a special kind of [tool](/docs/tool-calling) that can pause the\nLLM generation-and-tool-calling loop to return control back to you. When\nyou're ready, you can then _resume_ generation by sending _replies_ that the LLM\nprocesses for further generation.\n\nThe most common uses for interrupts fall into a few categories:\n\n- **Human-in-the-Loop:** Enabling the user of an interactive AI\n to clarify needed information or confirm the LLM's action\n before it is completed, providing a measure of safety and confidence.\n- **Async Processing:** Starting an asynchronous task that can only be\n completed out-of-band, such as sending an approval notification to\n a human reviewer or kicking off a long-running background process.\n- **Exit from an Autonomous Task:** Providing the model a way\n to mark a task as complete, in a workflow that might iterate through\n a long series of tool calls.\n\n\n\n## Before you begin\n\nAll of the examples documented here assume that you have already set up a\nproject with Genkit dependencies installed. If you want to run the code\nexamples on this page, first complete the steps in the\n[Get started](/docs/get-started) guide.\n\nBefore diving too deeply, you should also be familiar with the following\nconcepts:\n\n- [Generating content](/docs/models) with AI models.\n- Genkit's system for [defining input and output schemas](/docs/flows).\n- General methods of [tool-calling](/docs/tool-calling).\n\n## Overview of interrupts\n\nAt a high level, this is what an interrupt looks like when\ninteracting with an LLM:\n\n1. The calling application prompts the LLM with a request. The prompt includes\n a list of tools, including at least one for an interrupt that the LLM\n can use to generate a response.\n2. The LLM generates either a complete response or a tool call request\n in a specific format. To the LLM, an interrupt call looks like any\n other tool call.\n3. If the LLM calls an interrupt tool,\n the Genkit library automatically pauses generation rather than immediately\n passing responses back to the model for additional processing.\n4. The developer checks whether an interrupt call is made, and performs whatever\n task is needed to collect the information needed for the interrupt response.\n5. The developer resumes generation by passing an interrupt response to the\n model. This action triggers a return to Step 2.\n\n## Define manual-response interrupts\n\nThe most common kind of interrupt allows the LLM to request clarification from\nthe user, for example by asking a multiple-choice question.\n\nFor this use case, use the Genkit instance's `defineInterrupt()` method:\n\n```ts\nimport { genkit, z } from 'genkit';\nimport { googleAI } from '@genkitai/google-ai';\n\nconst ai = genkit({\n plugins: [googleAI()],\n model: googleAI.model('gemini-2.5-flash'),\n});\n\nconst askQuestion = ai.defineInterrupt({\n name: 'askQuestion',\n description: 'use this to ask the user a clarifying question',\n inputSchema: z.object({\n choices: z.array(z.string()).describe('the choices to display to the user'),\n allowOther: z.boolean().optional().describe('when true, allow write-ins'),\n }),\n outputSchema: z.string(),\n});\n```\n\nNote that the `outputSchema` of an interrupt corresponds to the response data\nyou will provide as opposed to something that will be automatically populated\nby a tool function.\n\n### Use interrupts\n\nInterrupts are passed into the `tools` array when generating content, just like\nother types of tools. You can pass both normal tools and interrupts to the\nsame `generate` call:\n\n\n \n ```ts\n const response = await ai.generate({\n prompt: \"Ask me a movie trivia question.\",\n tools: [askQuestion],\n });\n ```\n \n \n ```ts\n const triviaPrompt = ai.definePrompt({\n name: \"triviaPrompt\",\n tools: [askQuestion],\n input: {\n schema: z.object({ subject: z.string() }),\n },\n prompt: \"Ask me a trivia question about {{subject}}.\",\n });\n\n const response = await triviaPrompt({ subject: \"computer history\" });\n ```\n\n \n \n ```dotprompt\n ---\n tools: [askQuestion]\n input:\n schema:\n partyType: string\n ---\n\n {{role \"system\"}}\n Use the askQuestion tool if you need to clarify something.\n\n {{role \"user\"}}\n Help me plan a\n {{partyType}}\n party next week.\n ```\n\n Then you can execute the prompt in your code as follows:\n\n ```ts\n // assuming prompt file is named partyPlanner.prompt\n const partyPlanner = ai.prompt(\"partyPlanner\");\n\n const response = await partyPlanner({ partyType: \"birthday\" });\n ```\n\n \n \n ```ts\n const chat = ai.chat({\n system: \"Use the askQuestion tool if you need to clarify something.\",\n tools: [askQuestion],\n });\n\n const response = await chat.send(\"make a plan for my birthday party\");\n ```\n\n \n\n\nGenkit immediately returns a response on receipt of an interrupt tool call.\n\n### Respond to interrupts\n\nIf you've passed one or more interrupts to your generate call, you\nneed to check the response for interrupts so that you can handle them:\n\n```ts\n// you can check the 'finishReason' of the response\nresponse.finishReason === 'interrupted';\n// or you can check to see if any interrupt requests are on the response\nresponse.interrupts.length > 0;\n```\n\nResponding to an interrupt is done using the `resume` option on a subsequent\n`generate` call, making sure to pass in the existing history. Each tool has\na `.respond()` method on it to help construct the response.\n\nOnce resumed, the model re-enters the generation loop, including tool\nexecution, until either it completes or another interrupt is triggered:\n\n```ts\nlet response = await ai.generate({\n tools: [askQuestion],\n system: 'ask clarifying questions until you have a complete solution',\n prompt: 'help me plan a backyard BBQ',\n});\n\nwhile (response.interrupts.length) {\n const answers = [];\n // multiple interrupts can be called at once, so we handle them all\n for (const question of response.interrupts) {\n answers.push(\n // use the `respond` method on our tool to populate answers\n askQuestion.respond(\n question,\n // send the tool request input to the user to respond\n await askUser(question.toolRequest.input),\n ),\n );\n }\n\n response = await ai.generate({\n tools: [askQuestion],\n messages: response.messages,\n resume: {\n respond: answers,\n },\n });\n}\n\n// no more interrupts, we can see the final response\nconsole.log(response.text);\n```\n\n## Tools with restartable interrupts\n\nAnother common pattern for interrupts is the need to _confirm_ an action that\nthe LLM suggests before actually performing it. For example, a payments app\nmight want the user to confirm certain kinds of transfers.\n\nFor this use case, you can use the standard `defineTool` method to add custom\nlogic around when to trigger an interrupt, and what to do when an interrupt is\n_restarted_ with additional metadata.\n\n### Define a restartable tool\n\nEvery tool has access to two special helpers in the second argument of its\nimplementation definition:\n\n- `interrupt`: when called, this method throws a special kind of exception that\n is caught to pause the generation loop. You can provide additional metadata\n as an object.\n- `resumed`: when a request from an interrupted generation is restarted using\n the `{resume: {restart: ...}}` option (see below), this helper contains the\n metadata provided when restarting.\n\nIf you were building a payments app, for example, you might want to confirm with\nthe user before making a transfer exceeding a certain amount:\n\n```ts\nconst transferMoney = ai.defineTool({\n name: 'transferMoney',\n description: 'Transfers money between accounts.',\n inputSchema: z.object({\n toAccountId: z.string().describe('the account id of the transfer destination'),\n amount: z.number().describe('the amount in integer cents (100 = $1.00)'),\n }),\n outputSchema: z.object({\n status: z.string().describe('the outcome of the transfer'),\n message: z.string().optional(),\n })\n}, async (input, {context, interrupt, resumed})) {\n // if the user rejected the transaction\n if (resumed?.status === \"REJECTED\") {\n return {status: 'REJECTED', message: 'The user rejected the transaction.'};\n }\n // trigger an interrupt to confirm if amount > $100\n if (resumed?.status !== \"APPROVED\" && input.amount > 10000) {\n interrupt({\n message: \"Please confirm sending an amount > $100.\",\n });\n }\n // complete the transaction if not interrupted\n return doTransfer(input);\n}\n```\n\nIn this example, on first execution (when `resumed` is undefined), the tool\nchecks to see if the amount exceeds $100, and triggers an interrupt if so. On\nsecond execution, it looks for a status in the new metadata provided and\nperforms the transfer or returns a rejection response, depending on whether it\nis approved or rejected.\n\n### Restart tools after interruption\n\nInterrupt tools give you full control over:\n\n1. When an initial tool request should trigger an interrupt.\n2. When and whether to resume the generation loop.\n3. What additional information to provide to the tool when resuming.\n\nIn the example shown in the previous section, the application might ask the user\nto confirm the interrupted request to make sure the transfer amount is okay:\n\n```ts\nlet response = await ai.generate({\n tools: [transferMoney],\n prompt: \"Transfer $1000 to account ABC123\",\n});\n\nwhile (response.interrupts.length) {\n const confirmations = [];\n // multiple interrupts can be called at once, so we handle them all\n for (const interrupt of response.interrupts) {\n confirmations.push(\n // use the 'restart' method on our tool to provide `resumed` metadata\n transferMoney.restart(\n interrupt,\n // send the tool request input to the user to respond. assume that this\n // returns `{status: \"APPROVED\"}` or `{status: \"REJECTED\"}`\n await requestConfirmation(interrupt.toolRequest.input);\n )\n );\n }\n\n response = await ai.generate({\n tools: [transferMoney],\n messages: response.messages,\n resume: {\n restart: confirmations,\n }\n })\n}\n\n// no more interrupts, we can see the final response\nconsole.log(response.text);\n```", "title": "Pause generation using interrupts", - "lang": "js" + "description": "Learn how to use interrupts in Genkit to pause and resume LLM generation, enabling human-in-the-loop interactions, asynchronous processing, and controlled task completion.", + "lang": "js", + "headers": "## Before you begin\n## Overview of interrupts\n## Define manual-response interrupts\n### Use interrupts\n### Respond to interrupts\n## Tools with restartable interrupts\n### Define a restartable tool\n### Restart tools after interruption\n" }, "js/local-observability.md": { - "text": "Genkit provides a robust set of built-in observability features, including\ntracing and metrics collection powered by\n[OpenTelemetry](https://opentelemetry.io/). For local observability, such as\nduring the development phase, the Genkit Developer UI provides detailed trace\nviewing and debugging capabilities. For production observability, we provide\nGenkit Monitoring in the Firebase console via the Firebase plugin.\nAlternatively, you can export your OpenTelemetry data to the observability\ntooling of your choice.\n\n## Tracing & Metrics\n\nGenkit automatically collects traces and metrics without requiring explicit configuration, allowing you to observe and debug your Genkit code's behavior\nin the Developer UI. Genkit stores these traces, enabling you to analyze\nyour Genkit flows step-by-step with detailed input/output logging and\nstatistics. In production, Genkit can export traces and metrics to Firebase\nGenkit Monitoring for further analysis.\n\n## Log and export events\n\nGenkit provides a centralized logging system that you can configure using\nthe logging module. One advantage of using the Genkit-provided logger is that\nit automatically exports logs to Genkit Monitoring when the Firebase\nTelemetry plugin is enabled.\n\n```typescript\nimport { logger } from 'genkit/logging';\n\n// Set the desired log level\nlogger.setLogLevel('debug');\n```\n\n## Production Observability\n\nThe\n[Genkit Monitoring](https://console.firebase.google.com/project/_/genai_monitoring)\ndashboard helps you understand the overall health of your Genkit features. It\nis also useful for debugging stability and content issues that may\nindicate problems with your LLM prompts and/or Genkit Flows. See the\n[Getting Started](/docs/observability/getting-started) guide for\nmore details.\n", + "text": "# Observe local metrics\n\nGenkit provides a robust set of built-in observability features, including\ntracing and metrics collection powered by\n[OpenTelemetry](https://opentelemetry.io/). For local observability, such as\nduring the development phase, the Genkit Developer UI provides detailed trace\nviewing and debugging capabilities. For production observability, we provide\nGenkit Monitoring in the Firebase console via the Firebase plugin.\nAlternatively, you can export your OpenTelemetry data to the observability\ntooling of your choice.\n\n## Tracing & Metrics\n\nGenkit automatically collects traces and metrics without requiring explicit configuration, allowing you to observe and debug your Genkit code's behavior\nin the Developer UI. Genkit stores these traces, enabling you to analyze\nyour Genkit flows step-by-step with detailed input/output logging and\nstatistics. In production, Genkit can export traces and metrics to Firebase\nGenkit Monitoring for further analysis.\n\n## Log and export events\n\nGenkit provides a centralized logging system that you can configure using\nthe logging module. One advantage of using the Genkit-provided logger is that\nit automatically exports logs to Genkit Monitoring when the Firebase\nTelemetry plugin is enabled.\n\n```typescript\nimport { logger } from 'genkit/logging';\n\n// Set the desired log level\nlogger.setLogLevel('debug');\n```\n\n## Production Observability\n\nThe\n[Genkit Monitoring](https://console.firebase.google.com/project/_/genai_monitoring)\ndashboard helps you understand the overall health of your Genkit features. It\nis also useful for debugging stability and content issues that may\nindicate problems with your LLM prompts and/or Genkit Flows. See the\n[Getting Started](/docs/observability/getting-started) guide for\nmore details.\n", "title": "Observe local metrics", - "lang": "js" + "description": "Learn about Genkit's local observability features, including tracing, metrics collection, and logging, powered by OpenTelemetry and integrated with the Genkit Developer UI.", + "lang": "js", + "headers": "## Tracing & Metrics\n## Log and export events\n## Production Observability\n" }, "js/migrating-from-05X.md": { - "text": "Genkit 0.9 introduces a number of breaking changes alongside feature enhancements that improve overall functionality. If you have been developing applications with Genkit 0.5, you will need to update your application code when you upgrade to the latest version. This guide outlines the most significant changes and offers steps to migrate your existing applications smoothly.\n\n## Quickstart guide\n\nThe following steps will help you migrate from Genkit 0.5 to Genkit 0.9 quickly. Read more information about these changes in the detailed [Changelog](#changelog) below.\n\n### 1. Install the new CLI\n\n- Uninstall the old CLI\n\n ```bash\n npm uninstall -g genkit && npm uninstall genkit\n ```\n\n- Install the new CLI globally\n\n ```bash\n npm install -g genkit-cli\n ```\n\n### 2. Update your dependencies\n\n- Remove individual Genkit core packages\n\n ```bash\n npm uninstall @genkit-ai/ai @genkit-ai/core @genkit-ai/dotprompt @genkit-ai/flow\n ```\n\n- Install the new consolidated `genkit` package\n\n ```bash\n npm install genkit\n ```\n\n- Upgrade all plugin versions (example below)\n\n ```\n npm upgrade @genkit-ai/firebase\n ```\n\n### 3. Change your imports\n\n- Remove imports for individual Genkit core packages\n\n ```js\n import { … } from '@genkit-ai/ai';\n import { … } from '@genkit-ai/core';\n import { … } from '@genkit-ai/flow';\n ```\n\n- Remove zod imports\n\n ```js\n import * as z from 'zod';\n ```\n\n- Import `genkit` and `zod` from `genkit`\n\n ```js\n import { z, genkit } from 'genkit';\n ```\n\n### 4. Update your code\n\n#### Remove the configureGenkit blocks\n\nConfiguration for Genkit is now done per instance. Telemetry and logging is configured globally and separately from the Genkit instance.\n\n- Replace `configureGenkit` with `ai = genkit({...})` blocks. Keep only the plugin configuration.\n\n ```js\n import { genkit } from 'genkit';\n\n const ai = genkit({ plugins: [...]});\n ```\n\n- Configure telemetry using enableFirebaseTelemetry or enableGoogleCloudTelemetry\n\n For Firebase:\n\n ```js\n import { enableFirebaseTelemetry } from '@genkit-ai/firebase';\n\n enableFirebaseTelemetry({...});\n ```\n\n For Google Cloud:\n\n ```js\n import { enableGoogleCloudTelemetry } from '@genkit-ai/google-cloud';\n\n enableGoogleCloudTelemetry({...});\n ```\n\n- Set your logging level independently\n\n ```js\n import { logger } from 'genkit/logging';\n\n logger.setLogLevel('debug');\n ```\n\nSee the [Monitoring and Logging](/docs/observability/getting-started) documentation for more details on how to configure telemetry and logging.\n\nSee the [Get Started](/docs/get-started) documentation for more details on how to configure a Genkit instance.\n\n#### Migrate Genkit actions to be called from the `genkit` instance\n\nActions (flows, tools, retrievers, indexers, etc.) are defined per instance. Read the [Changelog](#changelog) for all of the features you will need to change, but here is an example of some common ones.\n\n```js\nimport { genkit } from 'genkit';\nimport { onFlow } from '@genkit-ai/firebase/functions';\n\nconst ai = genkit({ plugins: [...]});\n\n// Flows and tools are defined on the specific genkit instance\n// and are directly callable.\nconst sampleFlow = ai.defineFlow(...);\nconst sampleTool = ai.defineTool(...);\n\nasync function callMyFlow() {\n // Previously, text output could accessed via .text()\n // Now it is either .output() or .text\n return await sampleFlow().output();\n}\n\n// onFlow now takes the Genkit instance as first argument\n// This registers the flow as a callable firebase function\nonFlow(ai, ...);\nconst flows = [ sampleFlow, ... ];\n// Start the flow server to make the registered flows callable over HTTP\nai.startFlowServer({flows});\n```\n\n### 5. Run it\n\n```bash\n# run the DevUI and your js code\ngenkit start -- \n\n# run a defined flow\ngenkit flow:run \n```\n\n## Changelog\n\n### 1. CLI Changes\n\nThe command-line interface (CLI) has undergone significant updates in Genkit 0.9. The command to start Genkit has changed, and the CLI has been separated into its own standalone package, which you now need to install separately.\n\nTo install the CLI:\n\n```bash\nnpm install -g genkit-cli\n```\n\nSome changes have been made to the `genkit start` command:\n\nStarts your Genkit application code + Dev UI together:\n\n```bash\ngenkit start -- [start command]\n\ngenkit start -- tsx src/index.ts\n\ngenkit start -- go run main.go\n```\n\nWatch mode is supported as well:\n\n```bash\ngenkit start -- tsx --watch src/index.ts\n```\n\nStarts ONLY your application code in Genkit dev mode:\n\n```bash\ngenkit start --noui -- \n\ngenkit start --noui -- tsx src/index.ts\n```\n\nStarts the Dev UI ONLY:\n\n```bash\ngenkit start\n```\n\nPreviously, the `genkit start` command would start the Dev UI and your application code together. If you have any CI/CD pipelines relying on this command, you may need to update the pipeline.\n\nThe Dev UI will interact directly with the flow server to figure out which flows are registered and allow you to invoke them directly with sample inputs.\n\n### 2. Simplified packages and imports\n\nPreviously, the Genkit libraries were separated into several modules, which you needed to install and import individually. These modules have now been consolidated into a single import. In addition, the Zod module is now re-exported by Genkit.\n\n**Old:**\n\n```bash\nnpm install @genkit-ai/core @genkit-ai/ai @genkit-ai/flow @genkit-ai/dotprompt\n```\n\n**New:**\n\n```bash\nnpm install genkit\n```\n\n**Old:**\n\n```js\nimport { … } from '@genkit-ai/ai';\nimport { … } from '@genkit-ai/core';\nimport { … } from '@genkit-ai/flow';\nimport * as z from 'zod';\n```\n\n**New:**\n\n```js\nimport { genkit, z } from 'genkit';\n```\n\nGenkit plugins still must be installed and imported individually.\n\n### 3. Configuring Genkit\n\nPreviously, initializing Genkit was done once globally by calling the `configureGenkit` function. Genkit resources (flows, tools, prompts, etc.) would all automatically be wired with this global configuration.\n\nGenkit 0.9 introduces `Genkit` instances, each of which encapsulates a configuration. See the following examples:\n\n**Old:**\n\n```js\nimport { configureGenkit } from '@genkit-ai/core';\n\nconfigureGenkit({\n telemetry: {\n instrumentation: ...,\n logger: ...\n }\n});\n```\n\n**New:**\n\n```js\nimport { genkit } from 'genkit';\nimport { logger } from 'genkit/logging';\nimport { enableFirebaseTelemetry } from '@genkit-ai/firebase';\n\nlogger.setLogLevel('debug');\nenableFirebaseTelemetry({...});\n\nconst ai = genkit({ ... });\n```\n\nLet’s break it down:\n\n- `configureGenkit()` has been replaced with `genkit()`, and it returns a configured `Genkit` instance rather than setting up configurations globally.\n- The Genkit initialization function is now in the `genkit` package.\n- Logging and telemetry are still configured globally using their own explicit methods. These configurations apply uniformly across all `Genkit` instances.\n\n### 4. Defining flows and starting the flow server explicitly\n\nNow that you have a configured `Genkit` instance, you will need to define your flows. All core developer-facing API methods like `defineFlow`, `defineTool`, and `onFlow` are now invoked through this instance.\n\nThis is distinct from the previous way, where flows and tools were registered globally.\n\n**Old:**\n\n```js\nimport { defineFlow, defineTool, onFlow } from '@genkit-ai/core';\n\ndefineFlow(...);\ndefineTool(...);\n\nonFlow(...);\n```\n\n**New:**\n\n```js\n// Define tools and flows\nconst sampleFlow = ai.defineFlow(...);\nconst sampleTool = ai.defineTool(...);\n\n// onFlow now takes the Genkit instance as first argument\n// This registers the flow as a callable firebase function\nonFlow(ai, ...);\n\nconst flows = [ sampleFlow, ... ];\n// Start the flow server to make the registered flows callable over HTTP\nai.startFlowServer({flows});\n```\n\nAs of now, all flows that you want to make available need to be explicitly registered in the `flows` array above.\n\n### 5. Tools and Prompts must be statically defined\n\nIn earlier versions of Genkit, you could dynamically define tools and prompts at runtime, directly from within a flow.\n\nIn Genkit 0.9, this behavior is no longer allowed. Instead, you need to define all actions and flows outside of the flow’s execution (i.e. statically).\n\nThis change enforces a stricter separation of action definitions from execution.\n\nIf any of your code is defined dynamically, they need to be refactored. Otherwise, an error will be thrown at runtime when the flow is executed.\n\n**❌ DON'T:**\n\n```js\nconst flow = defineFlow({...}, async (input) => {\n const tool = defineTool({...});\n await tool(...);\n});\n```\n\n**✅ DO:**\n\n```js\nconst tool = ai.defineTool({...});\n\nconst flow = ai.defineFlow({...}, async (input) => {\n await tool(...);\n});\n```\n\n### 6. New API for Streaming Flows\n\nIn Genkit 0.9, we have simplified the syntax for defining a streaming flow and invoking it.\n\nFirst, `defineFlow` and `defineStreamingFlow` have been separated. If you have a flow that is meant to be streamed, you will have to update your code to define it via `defineStreamingFlow`.\n\nSecond, instead of calling separate `stream()` and `response()` functions, both stream and response are now values returned directly from the flow. This change simplifies flow streaming.\n\n**Old:**\n\n```js\nimport { defineFlow, streamFlow } from '@genkit-ai/flow';\n\nconst myStreamingFlow = defineFlow(...);\nconst { stream, output } = await streamFlow(myStreamingFlow, ...);\n\nfor await (const chunk of stream()) {\n console.log(chunk);\n}\n\nconsole.log(await output());\n```\n\n**New:**\n\n```js\nconst myStreamingFlow = ai.defineStreamingFlow(...);\nconst { stream, response } = await myStreamingFlow(...);\n\nfor await (const chunk of stream) {\n console.log(chunk);\n}\n\nconsole.log(await response);\n```\n\n### 7. GenerateResponse class methods replaced with getter properties\n\nPreviously, you used to access the structured output or text of the response using class methods, like `output()` or `text()`.\n\nIn Genkit 0.9, those methods have been replaced by getter properties. This simplifies working with responses.\n\n**Old:**\n\n```js\nconst response = await generate({ prompt: 'hi' });\nconsole.log(response.text());\n```\n\n**New:**\n\n```js\nconst response = await ai.generate('hi');\nconsole.log(response.text);\n```\n\nThe same applies to `output`:\n\n**Old:**\n\n```js\nconsole.log(response.output());\n```\n\n**New:**\n\n```js\nconsole.log(response.output);\n```\n\n### 8. Candidate Generation Eliminated\n\nGenkit 0.9 simplifies response handling by removing the `candidates` attribute. Previously, responses could contain multiple candidates, which you needed to handle explicitly. Now, only the first candidate is returned directly in a flat response.\n\nAny code that accesses the candidates directly will not work anymore.\n\n**Old:**\n\n```js\nconst response = await generate({\n messages: [ { role: 'user', content: ...} ]\n});\nconsole.log(response.candidates); // previously you could access candidates directly\n```\n\n**New:**\n\n```js\nconst response = await ai.generate({\n messages: [ { role: 'user', content: ...} ]\n});\nconsole.log(response.message); // single candidate is returned directly in a flat response\n```\n\n### 9. Generate API - Multi-Turn enhancements\n\nFor multi-turn conversations, the old `toHistory()` method has been replaced by `messages`, further simplifying how conversation history is handled.\n\n**Old:**\n\n```js\nconst history = response.toHistory();\n```\n\n**New:**\n\n```js\nconst response = await ai.generate({\n messages: [ { role: 'user', content: ...} ]\n});\nconst history = response.messages;\n```\n\n### 10. Streamlined Chat API\n\nIn Genkit 0.9, the Chat API has been redesigned for easier session management and interaction. Here’s how you can leverage it for both synchronous and streaming chat experiences:\n\n```js\nimport { genkit } from 'genkit';\nimport { gemini15Flash, googleAI } from '@genkit-ai/googleai';\n\nconst ai = genkit({\n plugins: [googleAI()],\n model: gemini15Flash,\n});\n\nconst session = ai.createSession({ store: firestoreSessionStore() });\nconst chat = await session.chat({ system: 'talk like a pirate' });\n\nlet response = await chat.send('hi, my name is Pavel');\nconsole.log(response.text()); // \"hi Pavel, I'm llm\"\n\n// continue the conversation\nresponse = await chat.send(\"what's my name\");\nconsole.log(response.text()); // \"Pavel\"\n\n// can stream\nconst { response, stream } = await chat.sendStream('bye');\nfor await (const chunk of stream) {\n console.log(chunk.text());\n}\nconsole.log((await response).text());\n\n// can load session from the store\nconst prevSession = await ai.loadSession(session.id, { store });\nconst prevChat = await prevSession.chat();\nawait prevChat.send('bye');\n```\n", + "text": "# Migrate from 0.5 to 0.9\n\nGenkit 0.9 introduces a number of breaking changes alongside feature enhancements that improve overall functionality. If you have been developing applications with Genkit 0.5, you will need to update your application code when you upgrade to the latest version. This guide outlines the most significant changes and offers steps to migrate your existing applications smoothly.\n\n## Quickstart guide\n\nThe following steps will help you migrate from Genkit 0.5 to Genkit 0.9 quickly. Read more information about these changes in the detailed [Changelog](#changelog) below.\n\n### 1. Install the new CLI\n\n- Uninstall the old CLI\n\n ```bash\n npm uninstall -g genkit && npm uninstall genkit\n ```\n\n- Install the new CLI globally\n\n ```bash\n npm install -g genkit-cli\n ```\n\n### 2. Update your dependencies\n\n- Remove individual Genkit core packages\n\n ```bash\n npm uninstall @genkit-ai/ai @genkit-ai/core @genkit-ai/dotprompt @genkit-ai/flow\n ```\n\n- Install the new consolidated `genkit` package\n\n ```bash\n npm install genkit\n ```\n\n- Upgrade all plugin versions (example below)\n\n ```\n npm upgrade @genkit-ai/firebase\n ```\n\n### 3. Change your imports\n\n- Remove imports for individual Genkit core packages\n\n ```js\n import { … } from '@genkit-ai/ai';\n import { … } from '@genkit-ai/core';\n import { … } from '@genkit-ai/flow';\n ```\n\n- Remove zod imports\n\n ```js\n import * as z from 'zod';\n ```\n\n- Import `genkit` and `zod` from `genkit`\n\n ```js\n import { z, genkit } from 'genkit';\n ```\n\n### 4. Update your code\n\n#### Remove the configureGenkit blocks\n\nConfiguration for Genkit is now done per instance. Telemetry and logging is configured globally and separately from the Genkit instance.\n\n- Replace `configureGenkit` with `ai = genkit({...})` blocks. Keep only the plugin configuration.\n\n ```js\n import { genkit } from 'genkit';\n\n const ai = genkit({ plugins: [...]});\n ```\n\n- Configure telemetry using enableFirebaseTelemetry or enableGoogleCloudTelemetry\n\n For Firebase:\n\n ```js\n import { enableFirebaseTelemetry } from '@genkit-ai/firebase';\n\n enableFirebaseTelemetry({...});\n ```\n\n For Google Cloud:\n\n ```js\n import { enableGoogleCloudTelemetry } from '@genkit-ai/google-cloud';\n\n enableGoogleCloudTelemetry({...});\n ```\n\n- Set your logging level independently\n\n ```js\n import { logger } from 'genkit/logging';\n\n logger.setLogLevel('debug');\n ```\n\nSee the [Monitoring and Logging](/docs/observability/getting-started) documentation for more details on how to configure telemetry and logging.\n\nSee the [Get Started](/docs/get-started) documentation for more details on how to configure a Genkit instance.\n\n#### Migrate Genkit actions to be called from the `genkit` instance\n\nActions (flows, tools, retrievers, indexers, etc.) are defined per instance. Read the [Changelog](#changelog) for all of the features you will need to change, but here is an example of some common ones.\n\n```js\nimport { genkit } from 'genkit';\nimport { onFlow } from '@genkit-ai/firebase/functions';\n\nconst ai = genkit({ plugins: [...]});\n\n// Flows and tools are defined on the specific genkit instance\n// and are directly callable.\nconst sampleFlow = ai.defineFlow(...);\nconst sampleTool = ai.defineTool(...);\n\nasync function callMyFlow() {\n // Previously, text output could accessed via .text()\n // Now it is either .output() or .text\n return await sampleFlow().output();\n}\n\n// onFlow now takes the Genkit instance as first argument\n// This registers the flow as a callable firebase function\nonFlow(ai, ...);\nconst flows = [ sampleFlow, ... ];\n// Start the flow server to make the registered flows callable over HTTP\nai.startFlowServer({flows});\n```\n\n### 5. Run it\n\n```bash\n# run the DevUI and your js code\ngenkit start -- \n\n# run a defined flow\ngenkit flow:run \n```\n\n## Changelog\n\n### 1. CLI Changes\n\nThe command-line interface (CLI) has undergone significant updates in Genkit 0.9. The command to start Genkit has changed, and the CLI has been separated into its own standalone package, which you now need to install separately.\n\nTo install the CLI:\n\n```bash\nnpm install -g genkit-cli\n```\n\nSome changes have been made to the `genkit start` command:\n\nStarts your Genkit application code + Dev UI together:\n\n```bash\ngenkit start -- [start command]\n\ngenkit start -- tsx src/index.ts\n\ngenkit start -- go run main.go\n```\n\nWatch mode is supported as well:\n\n```bash\ngenkit start -- tsx --watch src/index.ts\n```\n\nStarts ONLY your application code in Genkit dev mode:\n\n```bash\ngenkit start --noui -- \n\ngenkit start --noui -- tsx src/index.ts\n```\n\nStarts the Dev UI ONLY:\n\n```bash\ngenkit start\n```\n\nPreviously, the `genkit start` command would start the Dev UI and your application code together. If you have any CI/CD pipelines relying on this command, you may need to update the pipeline.\n\nThe Dev UI will interact directly with the flow server to figure out which flows are registered and allow you to invoke them directly with sample inputs.\n\n### 2. Simplified packages and imports\n\nPreviously, the Genkit libraries were separated into several modules, which you needed to install and import individually. These modules have now been consolidated into a single import. In addition, the Zod module is now re-exported by Genkit.\n\n**Old:**\n\n```bash\nnpm install @genkit-ai/core @genkit-ai/ai @genkit-ai/flow @genkit-ai/dotprompt\n```\n\n**New:**\n\n```bash\nnpm install genkit\n```\n\n**Old:**\n\n```js\nimport { … } from '@genkit-ai/ai';\nimport { … } from '@genkit-ai/core';\nimport { … } from '@genkit-ai/flow';\nimport * as z from 'zod';\n```\n\n**New:**\n\n```js\nimport { genkit, z } from 'genkit';\n```\n\nGenkit plugins still must be installed and imported individually.\n\n### 3. Configuring Genkit\n\nPreviously, initializing Genkit was done once globally by calling the `configureGenkit` function. Genkit resources (flows, tools, prompts, etc.) would all automatically be wired with this global configuration.\n\nGenkit 0.9 introduces `Genkit` instances, each of which encapsulates a configuration. See the following examples:\n\n**Old:**\n\n```js\nimport { configureGenkit } from '@genkit-ai/core';\n\nconfigureGenkit({\n telemetry: {\n instrumentation: ...,\n logger: ...\n }\n});\n```\n\n**New:**\n\n```js\nimport { genkit } from 'genkit';\nimport { logger } from 'genkit/logging';\nimport { enableFirebaseTelemetry } from '@genkit-ai/firebase';\n\nlogger.setLogLevel('debug');\nenableFirebaseTelemetry({...});\n\nconst ai = genkit({ ... });\n```\n\nLet’s break it down:\n\n- `configureGenkit()` has been replaced with `genkit()`, and it returns a configured `Genkit` instance rather than setting up configurations globally.\n- The Genkit initialization function is now in the `genkit` package.\n- Logging and telemetry are still configured globally using their own explicit methods. These configurations apply uniformly across all `Genkit` instances.\n\n### 4. Defining flows and starting the flow server explicitly\n\nNow that you have a configured `Genkit` instance, you will need to define your flows. All core developer-facing API methods like `defineFlow`, `defineTool`, and `onFlow` are now invoked through this instance.\n\nThis is distinct from the previous way, where flows and tools were registered globally.\n\n**Old:**\n\n```js\nimport { defineFlow, defineTool, onFlow } from '@genkit-ai/core';\n\ndefineFlow(...);\ndefineTool(...);\n\nonFlow(...);\n```\n\n**New:**\n\n```js\n// Define tools and flows\nconst sampleFlow = ai.defineFlow(...);\nconst sampleTool = ai.defineTool(...);\n\n// onFlow now takes the Genkit instance as first argument\n// This registers the flow as a callable firebase function\nonFlow(ai, ...);\n\nconst flows = [ sampleFlow, ... ];\n// Start the flow server to make the registered flows callable over HTTP\nai.startFlowServer({flows});\n```\n\nAs of now, all flows that you want to make available need to be explicitly registered in the `flows` array above.\n\n### 5. Tools and Prompts must be statically defined\n\nIn earlier versions of Genkit, you could dynamically define tools and prompts at runtime, directly from within a flow.\n\nIn Genkit 0.9, this behavior is no longer allowed. Instead, you need to define all actions and flows outside of the flow’s execution (i.e. statically).\n\nThis change enforces a stricter separation of action definitions from execution.\n\nIf any of your code is defined dynamically, they need to be refactored. Otherwise, an error will be thrown at runtime when the flow is executed.\n\n**❌ DON'T:**\n\n```js\nconst flow = defineFlow({...}, async (input) => {\n const tool = defineTool({...});\n await tool(...);\n});\n```\n\n**✅ DO:**\n\n```js\nconst tool = ai.defineTool({...});\n\nconst flow = ai.defineFlow({...}, async (input) => {\n await tool(...);\n});\n```\n\n### 6. New API for Streaming Flows\n\nIn Genkit 0.9, we have simplified the syntax for defining a streaming flow and invoking it.\n\nFirst, `defineFlow` and `defineStreamingFlow` have been separated. If you have a flow that is meant to be streamed, you will have to update your code to define it via `defineStreamingFlow`.\n\nSecond, instead of calling separate `stream()` and `response()` functions, both stream and response are now values returned directly from the flow. This change simplifies flow streaming.\n\n**Old:**\n\n```js\nimport { defineFlow, streamFlow } from '@genkit-ai/flow';\n\nconst myStreamingFlow = defineFlow(...);\nconst { stream, output } = await streamFlow(myStreamingFlow, ...);\n\nfor await (const chunk of stream()) {\n console.log(chunk);\n}\n\nconsole.log(await output());\n```\n\n**New:**\n\n```js\nconst myStreamingFlow = ai.defineStreamingFlow(...);\nconst { stream, response } = await myStreamingFlow(...);\n\nfor await (const chunk of stream) {\n console.log(chunk);\n}\n\nconsole.log(await response);\n```\n\n### 7. GenerateResponse class methods replaced with getter properties\n\nPreviously, you used to access the structured output or text of the response using class methods, like `output()` or `text()`.\n\nIn Genkit 0.9, those methods have been replaced by getter properties. This simplifies working with responses.\n\n**Old:**\n\n```js\nconst response = await generate({ prompt: 'hi' });\nconsole.log(response.text());\n```\n\n**New:**\n\n```js\nconst response = await ai.generate('hi');\nconsole.log(response.text);\n```\n\nThe same applies to `output`:\n\n**Old:**\n\n```js\nconsole.log(response.output());\n```\n\n**New:**\n\n```js\nconsole.log(response.output);\n```\n\n### 8. Candidate Generation Eliminated\n\nGenkit 0.9 simplifies response handling by removing the `candidates` attribute. Previously, responses could contain multiple candidates, which you needed to handle explicitly. Now, only the first candidate is returned directly in a flat response.\n\nAny code that accesses the candidates directly will not work anymore.\n\n**Old:**\n\n```js\nconst response = await generate({\n messages: [ { role: 'user', content: ...} ]\n});\nconsole.log(response.candidates); // previously you could access candidates directly\n```\n\n**New:**\n\n```js\nconst response = await ai.generate({\n messages: [ { role: 'user', content: ...} ]\n});\nconsole.log(response.message); // single candidate is returned directly in a flat response\n```\n\n### 9. Generate API - Multi-Turn enhancements\n\nFor multi-turn conversations, the old `toHistory()` method has been replaced by `messages`, further simplifying how conversation history is handled.\n\n**Old:**\n\n```js\nconst history = response.toHistory();\n```\n\n**New:**\n\n```js\nconst response = await ai.generate({\n messages: [ { role: 'user', content: ...} ]\n});\nconst history = response.messages;\n```\n\n### 10. Streamlined Chat API\n\nIn Genkit 0.9, the Chat API has been redesigned for easier session management and interaction. Here’s how you can leverage it for both synchronous and streaming chat experiences:\n\n```js\nimport { genkit } from 'genkit';\nimport { gemini15Flash, googleAI } from '@genkit-ai/googleai';\n\nconst ai = genkit({\n plugins: [googleAI()],\n model: gemini15Flash,\n});\n\nconst session = ai.createSession({ store: firestoreSessionStore() });\nconst chat = await session.chat({ system: 'talk like a pirate' });\n\nlet response = await chat.send('hi, my name is Pavel');\nconsole.log(response.text()); // \"hi Pavel, I'm llm\"\n\n// continue the conversation\nresponse = await chat.send(\"what's my name\");\nconsole.log(response.text()); // \"Pavel\"\n\n// can stream\nconst { response, stream } = await chat.sendStream('bye');\nfor await (const chunk of stream) {\n console.log(chunk.text());\n}\nconsole.log((await response).text());\n\n// can load session from the store\nconst prevSession = await ai.loadSession(session.id, { store });\nconst prevChat = await prevSession.chat();\nawait prevChat.send('bye');\n```\n", "title": "Migrate from 0.5 to 0.9", - "lang": "js" + "description": "This guide outlines the significant changes and steps to migrate your Genkit applications from version 0.5 to 0.9, covering CLI updates, package changes, and API modifications.", + "lang": "js", + "headers": "## Quickstart guide\n### 1. Install the new CLI\n### 2. Update your dependencies\n### 3. Change your imports\n### 4. Update your code\n#### Remove the configureGenkit blocks\n#### Migrate Genkit actions to be called from the `genkit` instance\n### 5. Run it\n# run the DevUI and your js code\n# run a defined flow\n## Changelog\n### 1. CLI Changes\n### 2. Simplified packages and imports\n### 3. Configuring Genkit\n### 4. Defining flows and starting the flow server explicitly\n### 5. Tools and Prompts must be statically defined\n### 6. New API for Streaming Flows\n### 7. GenerateResponse class methods replaced with getter properties\n### 8. Candidate Generation Eliminated\n### 9. Generate API - Multi-Turn enhancements\n### 10. Streamlined Chat API\n" }, "js/migrating-from-09X.md": { - "text": "Genkit 1.0 introduces many feature enhancements that improve overall\nfunctionality; it also has some breaking changes. If you have been developing\napplications with Genkit 0.9, you need to update your application code when you\nupgrade to the latest version of Genkit. This guide outlines the most\nsignificant changes, and explains how to migrate your existing applications\nsmoothly.\n\n## Beta APIs\n\nWe're introducing an unstable, Beta API channel, and leaving session, chat and Genkit client APIs in beta as we continue to refine them. More specifically, the following functions are currently in the `beta` namespace:\n\n- `ai.chat`\n- `ai.createSession`\n- `ai.loadSession`\n- `ai.currentSession`\n- `ai.defineFormat`\n- `ai.defineInterrupt`\n\nNote: When using the APIs as part of the Beta API, you may experience breaking\nchanges outside of SemVer. Breaking changes may occur on minor releases.\n\n**Old:**\n\n```ts\nimport { genkit } from 'genkit';\nconst ai = genkit({...})\nconst session = ai.createSession({ ... })\n```\n\n**New:**\n\n```ts\nimport { genkit } from 'genkit/beta';\nconst ai = genkit({...})\nconst session = ai.createSession({ ... })\n```\n\n**Old:**\n\n```ts\nimport { runFlow, streamFlow } from 'genkit/client';\n```\n\n**New:**\n\n```ts\nimport { runFlow, streamFlow } from 'genkit/beta/client';\n```\n\n## Introducing new `@genkit-ai/express` package\n\nThis new package contains utilities to make it easier to build an Express.js server with Genkit. You can find more details about this on\n[this page](https://js.api.genkit.dev/modules/_genkit-ai_express.html).\n\n`startFlowServer` has moved from part of the genkit object to this new\n`@genkit-ai/express` package; to use startFlowServer, you must\nupdate your imports.\n\n**Old:**\n\n```ts\nconst ai = genkit({ ... });\nai.startFlowServer({\n flows: [myFlow1, myFlow2],\n});\n```\n\n**New:**\n\n```ts\nimport { startFlowServer } from '@genkit-ai/express';\nstartFlowServer({\n flows: [myFlow1, myFlow2],\n});\n```\n\n## Changes to Flows\n\nThere are several changes to flows in 1.0:\n\n- `ai.defineStreamingFlow` has been consolidated into `ai.defineFlow`,\n- `onFlow` has been replaced by `onCallGenkit`,\n- `run` has moved to `ai.run`,\n- There are changes to working with auth.\n\nThe `run` function for custom trace blocks has moved to part of the `genkit` object; use `ai.run` to invoke it instead.\n\n**Old:**\n\n```ts\nai.defineFlow({ name: 'banana' }, async (input) => {\n const step = await run('myCode', async () => {\n return 'something';\n });\n});\n```\n\n**New:**\n\n```ts\nai.defineFlow({ name: 'banana' }, async (input) => {\n const step = await ai.run('myCode', async () => {\n return 'something';\n });\n});\n```\n\n`ai.defineStreamingFlow` has been removed; use `ai.defineFlow` instead. Also,\n`streamingCallback` has moved to a field inside the second argument of the flow\nfunction and is now called `sendChunk`.\n\n**Old:**\n\n```ts\nconst flow = ai.defineStreamingFlow({ name: 'banana' }, async (input, streamingCallback) => {\n streamingCallback({ chunk: 1 });\n});\n\nconst { stream } = await flow();\nfor await (const chunk of stream) {\n // ...\n}\n```\n\n**New:**\n\n```ts\nconst flow = ai.defineFlow({ name: 'banana' }, async (input, { context, sendChunk }) => {\n sendChunk({ chunk: 1 });\n});\n\nconst { stream, output } = flow.stream(input);\nfor await (const chunk of stream) {\n // ...\n}\n```\n\nFlowAuth auth is now called context. You can access auth as a field inside context:\n\n**Old:**\n\n```ts\nai.defineFlow({ name: 'banana' }, async (input) => {\n const auth = getFlowAuth();\n // ...\n});\n```\n\n**New:**\n\n```ts\nai.defineFlow({ name: 'banana' }, async (input, { context }) => {\n const auth = context.auth;\n});\n```\n\n`onFlow` moved to `firebase-functions/https` package and has been renamed to\n`onCallGenkit`. The following snippet shows an example of how to use it.\n\n**Old**\n\n```ts\nimport { onFlow } from '@genkit-ai/firebase/functions';\n\nexport const generatePoem = onFlow(\n ai,\n {\n name: 'jokeTeller',\n inputSchema: z.string().nullable(),\n outputSchema: z.string(),\n streamSchema: z.string(),\n },\n async (type, streamingCallback) => {\n const { stream, response } = await ai.generateStream(`Tell me a longish ${type ?? 'dad'} joke.`);\n for await (const chunk of stream) {\n streamingCallback(chunk.text);\n }\n return (await response).text;\n },\n);\n```\n\n**New:**\n\n```ts\nimport { onCallGenkit } from 'firebase-functions/https';\nimport { defineSecret } from 'firebase-functions/params';\nimport { genkit, z } from 'genkit';\n\nconst apiKey = defineSecret('GEMINI_API_KEY');\n\nconst ai = genkit({\n plugins: [googleAI()],\n model: gemini15Flash,\n});\n\nexport const jokeTeller = ai.defineFlow(\n {\n name: 'jokeTeller',\n inputSchema: z.object({ type: z.string().nullable() }),\n outputSchema: z.object({ joke: z.string() }),\n streamSchema: z.string(),\n },\n async ({ type }, { sendChunk }) => {\n const { stream, response } = ai.generateStream(`Tell me a longish ${type ?? 'dad'} joke.`);\n for await (const chunk of stream) {\n sendChunk(chunk.text);\n }\n return { joke: (await response).text };\n },\n);\n\nexport const tellJoke = onCallGenkit({ secrets: [apiKey] }, jokeTeller);\n```\n\nAuth policies have been removed from `defineFlow`. Handling of auth policies\nis now server-dependent.\n\n**Old:**\n\n```ts\nexport const simpleFlow = ai.defineFlow(\n {\n name: 'simpleFlow',\n authPolicy: (auth, input) => {\n // auth policy\n },\n },\n async (input) => {\n // Flow logic here...\n },\n);\n```\n\nThe following snippet shows an example of handling auth in Express.\n\n**New:**\n\n```ts\nimport { UserFacingError } from 'genkit';\nimport { ContextProvider, RequestData } from 'genkit/context';\nimport { expressHandler, startFlowServer, withContextProvider } from '@genkit-ai/express';\n\nconst context: ContextProvider = (req: RequestData) => {\n return {\n auth: parseAuthToken(req.headers['authorization']),\n };\n};\n\nexport const simpleFlow = ai.defineFlow(\n {\n name: 'simpleFlow',\n },\n async (input, { context }) => {\n if (!context.auth) {\n throw new UserFacingError(\"UNAUTHORIZED\", \"Authorization required.\");\n }\n if (input.uid !== context.auth.uid) {\n throw new UserFacingError(\"UNAUTHORIZED\", \"You may only summarize your own profile data.\");\n }\n // Flow logic here...\n }\n);\n\nconst app = express();\napp.use(express.json());\napp.post(\n '/simpleFlow',\n expressHandler(simpleFlow, { context })\n);\napp.listen(8080);\n\n// or\n\nstartFlowServer(\n flows: [withContextProvider(simpleFlow, context)],\n port: 8080\n);\n```\n\nFor more details, refer to the [auth documentation](/docs/auth).\n\nThe following snippet shows an example of handling auth in Cloud Functions\nfor Firebase:\n\n```ts\nimport { genkit } from 'genkit';\nimport { onCallGenkit } from 'firebase-functions/https';\n\nconst ai = genkit({ ... });;\n\nconst simpleFlow = ai.defineFlow({\n name: 'simpleFlow',\n}, async (input) => {\n // Flow logic here...\n});\n\nexport const selfSummary = onCallGenkit({\n authPolicy: (auth, data) => auth?.token?.['email_verified'] && auth?.token?.['admin'],\n}, simpleFlow);\n```\n\n## Prompts\n\nWe've made several changes and improvements to prompts.\n\nYou can define separate templates for prompt and system messages:\n\n```ts\nconst hello = ai.definePrompt({\n name: 'hello',\n system: 'talk like a pirate.',\n prompt: 'hello {% verbatim %}{{ name }}{% endverbatim %}',\n input: {\n schema: z.object({\n name: z.string(),\n }),\n },\n});\nconst { text } = await hello({ name: 'Genkit' });\n```\n\nAlternatively, you can define multi-message prompts in the messages field:\n\n```ts\nconst hello = ai.definePrompt({\n name: 'hello',\n messages:\n '{% verbatim %}{{ role \"system\" }}{% endverbatim %} talk like a pirate. {% verbatim %}{{ role \"user\" }}{% endverbatim %} hello {% verbatim %}{{ name }}{% endverbatim %}',\n input: {\n schema: z.object({\n name: z.string(),\n }),\n },\n});\n```\n\nInstead of prompt templates you can use a function:\n\n```ts\nai.definePrompt({\n name: 'hello',\n prompt: async (input, { context }) => {\n return `hello ${input.name}`;\n },\n input: {\n schema: z.object({\n name: z.string(),\n }),\n },\n});\n```\n\nYou can access the context (including auth information) from within the prompt:\n\n```ts\nconst hello = ai.definePrompt({\n name: 'hello',\n messages: 'hello {% verbatim %}{{ @auth.email }}{% endverbatim %}',\n});\n```\n\n## Streaming functions do not require an `await`\n\n**Old:**\n\n```ts\nconst { stream, response } = await ai.generateStream(`hi`);\nconst { stream, output } = await myflow.stream(`hi`);\n```\n\n**New:**\n\n```ts\nconst { stream, response } = ai.generateStream(`hi`);\nconst { stream, output } = myflow.stream(`hi`);\n```\n\n## Embed has a new return type\n\nWe've added support for multimodal embeddings. Instead of returning just a\nsingle embedding vector, Embed returns an array of embedding objects, each\ncontaining an embedding vector and metadata.\n\n**Old:**\n\n```ts\nconst response = await ai.embed({ embedder, content, options }); // returns number[]\n```\n\n**New:**\n\n```ts\nconst response = await ai.embed({ embedder, content, options }); // returns Embedding[]\nconst firstEmbeddingVector = response[0].embedding; // is number[]\n```\n", + "text": "# Migrate from 0.9 to 1.0\n\nGenkit 1.0 introduces many feature enhancements that improve overall\nfunctionality; it also has some breaking changes. If you have been developing\napplications with Genkit 0.9, you need to update your application code when you\nupgrade to the latest version of Genkit. This guide outlines the most\nsignificant changes, and explains how to migrate your existing applications\nsmoothly.\n\n## Beta APIs\n\nWe're introducing an unstable, Beta API channel, and leaving session, chat and Genkit client APIs in beta as we continue to refine them. More specifically, the following functions are currently in the `beta` namespace:\n\n- `ai.chat`\n- `ai.createSession`\n- `ai.loadSession`\n- `ai.currentSession`\n- `ai.defineFormat`\n- `ai.defineInterrupt`\n\nNote: When using the APIs as part of the Beta API, you may experience breaking\nchanges outside of SemVer. Breaking changes may occur on minor releases.\n\n**Old:**\n\n```ts\nimport { genkit } from 'genkit';\nconst ai = genkit({...})\nconst session = ai.createSession({ ... })\n```\n\n**New:**\n\n```ts\nimport { genkit } from 'genkit/beta';\nconst ai = genkit({...})\nconst session = ai.createSession({ ... })\n```\n\n**Old:**\n\n```ts\nimport { runFlow, streamFlow } from 'genkit/client';\n```\n\n**New:**\n\n```ts\nimport { runFlow, streamFlow } from 'genkit/beta/client';\n```\n\n## Introducing new `@genkit-ai/express` package\n\nThis new package contains utilities to make it easier to build an Express.js server with Genkit. You can find more details about this on\n[this page](https://js.api.genkit.dev/modules/_genkit-ai_express.html).\n\n`startFlowServer` has moved from part of the genkit object to this new\n`@genkit-ai/express` package; to use startFlowServer, you must\nupdate your imports.\n\n**Old:**\n\n```ts\nconst ai = genkit({ ... });\nai.startFlowServer({\n flows: [myFlow1, myFlow2],\n});\n```\n\n**New:**\n\n```ts\nimport { startFlowServer } from '@genkit-ai/express';\nstartFlowServer({\n flows: [myFlow1, myFlow2],\n});\n```\n\n## Changes to Flows\n\nThere are several changes to flows in 1.0:\n\n- `ai.defineStreamingFlow` has been consolidated into `ai.defineFlow`,\n- `onFlow` has been replaced by `onCallGenkit`,\n- `run` has moved to `ai.run`,\n- There are changes to working with auth.\n\nThe `run` function for custom trace blocks has moved to part of the `genkit` object; use `ai.run` to invoke it instead.\n\n**Old:**\n\n```ts\nai.defineFlow({ name: 'banana' }, async (input) => {\n const step = await run('myCode', async () => {\n return 'something';\n });\n});\n```\n\n**New:**\n\n```ts\nai.defineFlow({ name: 'banana' }, async (input) => {\n const step = await ai.run('myCode', async () => {\n return 'something';\n });\n});\n```\n\n`ai.defineStreamingFlow` has been removed; use `ai.defineFlow` instead. Also,\n`streamingCallback` has moved to a field inside the second argument of the flow\nfunction and is now called `sendChunk`.\n\n**Old:**\n\n```ts\nconst flow = ai.defineStreamingFlow({ name: 'banana' }, async (input, streamingCallback) => {\n streamingCallback({ chunk: 1 });\n});\n\nconst { stream } = await flow();\nfor await (const chunk of stream) {\n // ...\n}\n```\n\n**New:**\n\n```ts\nconst flow = ai.defineFlow({ name: 'banana' }, async (input, { context, sendChunk }) => {\n sendChunk({ chunk: 1 });\n});\n\nconst { stream, output } = flow.stream(input);\nfor await (const chunk of stream) {\n // ...\n}\n```\n\nFlowAuth auth is now called context. You can access auth as a field inside context:\n\n**Old:**\n\n```ts\nai.defineFlow({ name: 'banana' }, async (input) => {\n const auth = getFlowAuth();\n // ...\n});\n```\n\n**New:**\n\n```ts\nai.defineFlow({ name: 'banana' }, async (input, { context }) => {\n const auth = context.auth;\n});\n```\n\n`onFlow` moved to `firebase-functions/https` package and has been renamed to\n`onCallGenkit`. The following snippet shows an example of how to use it.\n\n**Old**\n\n```ts\nimport { onFlow } from '@genkit-ai/firebase/functions';\n\nexport const generatePoem = onFlow(\n ai,\n {\n name: 'jokeTeller',\n inputSchema: z.object({ type: z.string().nullable() }),\n outputSchema: z.object({ joke: z.string() }),\n streamSchema: z.string(),\n },\n async ({ type }, streamingCallback) => {\n const { stream, response } = await ai.generateStream(`Tell me a longish ${type ?? 'dad'} joke.`);\n for await (const chunk of stream) {\n streamingCallback(chunk.text);\n }\n return { joke: (await response).text };\n },\n);\n```\n\n**New:**\n\n```ts\nimport { onCallGenkit } from 'firebase-functions/https';\nimport { defineSecret } from 'firebase-functions/params';\nimport { genkit, z } from 'genkit';\n\nconst apiKey = defineSecret('GEMINI_API_KEY');\n\nconst ai = genkit({\n plugins: [googleAI()],\n model: gemini15Flash,\n});\n\nexport const jokeTeller = ai.defineFlow(\n {\n name: 'jokeTeller',\n inputSchema: z.object({ type: z.string().nullable() }),\n outputSchema: z.object({ joke: z.string() }),\n streamSchema: z.string(),\n },\n async ({ type }, { sendChunk }) => {\n const { stream, response } = ai.generateStream(`Tell me a longish ${type ?? 'dad'} joke.`);\n for await (const chunk of stream) {\n sendChunk(chunk.text);\n }\n return { joke: (await response).text };\n },\n);\n\nexport const tellJoke = onCallGenkit({ secrets: [apiKey] }, jokeTeller);\n```\n\nAuth policies have been removed from `defineFlow`. Handling of auth policies\nis now server-dependent.\n\n**Old:**\n\n```ts\nexport const simpleFlow = ai.defineFlow(\n {\n name: 'simpleFlow',\n authPolicy: (auth, input) => {\n // auth policy\n },\n },\n async (input) => {\n // Flow logic here...\n },\n);\n```\n\nThe following snippet shows an example of handling auth in Express.\n\n**New:**\n\n```ts\nimport { UserFacingError } from 'genkit';\nimport { ContextProvider, RequestData } from 'genkit/context';\nimport { expressHandler, startFlowServer, withContextProvider } from '@genkit-ai/express';\n\nconst context: ContextProvider = (req: RequestData) => {\n return {\n auth: parseAuthToken(req.headers['authorization']),\n };\n};\n\nexport const simpleFlow = ai.defineFlow(\n {\n name: 'simpleFlow',\n },\n async (input, { context }) => {\n if (!context.auth) {\n throw new UserFacingError(\"UNAUTHORIZED\", \"Authorization required.\");\n }\n if (input.uid !== context.auth.uid) {\n throw new UserFacingError(\"UNAUTHORIZED\", \"You may only summarize your own profile data.\");\n }\n // Flow logic here...\n }\n);\n\nconst app = express();\napp.use(express.json());\napp.post(\n '/simpleFlow',\n expressHandler(simpleFlow, { context })\n);\napp.listen(8080);\n\n// or\n\nstartFlowServer(\n flows: [withContextProvider(simpleFlow, context)],\n port: 8080\n);\n```\n\nFor more details, refer to the [auth documentation](/docs/auth).\n\nThe following snippet shows an example of handling auth in Cloud Functions\nfor Firebase:\n\n```ts\nimport { genkit } from 'genkit';\nimport { onCallGenkit } from 'firebase-functions/https';\n\nconst ai = genkit({ ... });;\n\nconst simpleFlow = ai.defineFlow({\n name: 'simpleFlow',\n}, async (input) => {\n // Flow logic here...\n});\n\nexport const selfSummary = onCallGenkit({\n authPolicy: (auth, data) => auth?.token?.['email_verified'] && auth?.token?.['admin'],\n}, simpleFlow);\n```\n\n## Prompts\n\nWe've made several changes and improvements to prompts.\n\nYou can define separate templates for prompt and system messages:\n\n```ts\nconst hello = ai.definePrompt({\n name: 'hello',\n system: 'talk like a pirate.',\n prompt: 'hello {% verbatim %}{{ name }}{% endverbatim %}',\n input: {\n schema: z.object({\n name: z.string(),\n }),\n },\n});\nconst { text } = await hello({ name: 'Genkit' });\n```\n\nAlternatively, you can define multi-message prompts in the messages field:\n\n```ts\nconst hello = ai.definePrompt({\n name: 'hello',\n messages:\n '{% verbatim %}{{ role \"system\" }}{% endverbatim %} talk like a pirate. {% verbatim %}{{ role \"user\" }}{% endverbatim %} hello {% verbatim %}{{ name }}{% endverbatim %}',\n input: {\n schema: z.object({\n name: z.string(),\n }),\n },\n});\n```\n\nInstead of prompt templates you can use a function:\n\n```ts\nai.definePrompt({\n name: 'hello',\n prompt: async (input, { context }) => {\n return `hello ${input.name}`;\n },\n input: {\n schema: z.object({\n name: z.string(),\n }),\n },\n});\n```\n\nYou can access the context (including auth information) from within the prompt:\n\n```ts\nconst hello = ai.definePrompt({\n name: 'hello',\n messages: 'hello {% verbatim %}{{ @auth.email }}{% endverbatim %}',\n});\n```\n\n## Streaming functions do not require an `await`\n\n**Old:**\n\n```ts\nconst { stream, response } = await ai.generateStream(`hi`);\nconst { stream, output } = await myflow.stream(`hi`);\n```\n\n**New:**\n\n```ts\nconst { stream, response } = ai.generateStream(`hi`);\nconst { stream, output } = myflow.stream(`hi`);\n```\n\n## Embed has a new return type\n\nWe've added support for multimodal embeddings. Instead of returning just a\nsingle embedding vector, Embed returns an array of embedding objects, each\ncontaining an embedding vector and metadata.\n\n**Old:**\n\n```ts\nconst response = await ai.embed({ embedder, content, options }); // returns number[]\n```\n\n**New:**\n\n```ts\nconst response = await ai.embed({ embedder, content, options }); // returns Embedding[]\nconst firstEmbeddingVector = response[0].embedding; // is number[]\n```\n", "title": "Migrate from 0.9 to 1.0", - "lang": "js" + "description": "This guide outlines the significant changes and steps to migrate your Genkit applications from version 0.9 to 1.0, covering API channel changes, package updates, and flow modifications.", + "lang": "js", + "headers": "## Beta APIs\n## Introducing new `@genkit-ai/express` package\n## Changes to Flows\n## Prompts\n## Streaming functions do not require an `await`\n## Embed has a new return type\n" }, - "js/models.mdx": { - "text": "import LLMSummary from '@/components/llm-summary.astro';\nimport ExampleLink from '@/components/ExampleLink.astro';\n\n\nGenkit provides a unified interface to interact with various generative AI models (LLMs, image generation).\n\n**Core Function:** `ai.generate()`\n\n**Basic Usage:**\n\n```typescript\nimport { googleAI } from '@genkit-ai/googleai';\nimport { genkit } from 'genkit';\n\nconst ai = genkit({\n plugins: [googleAI()],\n model: googleAI.model('gemini-2.5-flash'), // Default model\n});\n\n// Generate with default model\nconst response1 = await ai.generate('prompt text');\nconsole.log(response1.text);\n\n// Generate with specific model reference\nimport { googleAI } from '@genkit-ai/googleai';\nconst response2 = await ai.generate({\n model: googleAI.model('gemini-2.5-flash'),\n prompt: 'prompt text',\n});\nconsole.log(response2.text);\n\n// Generate with model string ID\nconst response3 = await ai.generate({\n model: 'googleai/gemini-2.5-flash',\n prompt: 'prompt text',\n});\nconsole.log(response3.text);\n```\n\n**Configuration:**\n\n- **System Prompt:** `system: \"Instruction for the model\"`\n- **Model Parameters:** `config: { maxOutputTokens: 512, temperature: 1.0, topP: 0.95, topK: 40, stopSequences: [\"\\n\"] }`\n\n**Structured Output (using Zod):**\n\n```typescript\nimport { z } from 'genkit';\n\nconst MenuItemSchema = z.object({\n name: z.string().describe('The name of the menu item.'),\n description: z.string().describe('A description of the menu item.'),\n calories: z.number().describe('The estimated number of calories.'),\n allergens: z.array(z.string()).describe('Any known allergens in the menu item.'),\n});\n\nconst response = await ai.generate({\n prompt: 'Suggest a menu item.',\n output: { schema: MenuItemSchema },\n});\n\nconst menuItem = response.output; // Typed output, might be null if validation fails\nif (menuItem) {\n console.log(menuItem.name);\n}\n```\n\n**Streaming:**\n\n```typescript\nconst { stream, response } = ai.generateStream({\n prompt: 'Tell a story.',\n // Can also include output schema for streaming structured data\n // output: { schema: z.array(MenuItemSchema) },\n});\n\n// Stream text chunks\nfor await (const chunk of stream) {\n console.log(chunk.text); // For structured: chunk.output (accumulated)\n}\n\n// Get final complete response\nconst finalResponse = await response;\nconsole.log(finalResponse.text); // For structured: finalResponse.output\n```\n\n**Multimodal Input:**\n\n```typescript\nimport { readFile } from 'node:fs/promises';\n\n// From URL\nconst response1 = await ai.generate({\n prompt: [{ media: { url: 'https://.../image.jpg' } }, { text: 'Describe this image.' }],\n});\n\n// From local file (data URL)\nconst data = await readFile('image.jpg');\nconst response2 = await ai.generate({\n prompt: [{ media: { url: `data:image/jpeg;base64,${data.toString('base64')}` } }, { text: 'Describe this image.' }],\n});\n```\n\n**Media Generation (e.g., Images):**\n\n```typescript\nimport { vertexAI } from '@genkit-ai/vertexai'; // Example image model\nimport { parseDataUrl } from 'data-urls';\nimport { writeFile } from 'node:fs/promises';\n\nconst response = await ai.generate({\n model: vertexAI.model('imagen-3.0-fast-generate-001'),\n prompt: 'Image description',\n output: { format: 'media' }, // Request media output\n});\n\nconst imagePart = response.output;\nif (imagePart?.media?.url) {\n // URL is typically a data: URL\n const parsed = parseDataUrl(imagePart.media.url);\n if (parsed) {\n await writeFile('output.png', parsed.body);\n }\n}\n```\n\n**Supported Model Plugins (Examples):**\n\n- Vertex AI (`@genkit-ai/vertexai`): Gemini, Imagen, Claude on Vertex\n- Google AI (`@genkit-ai/googleai`): Gemini\n- Ollama (`@genkit-ai/ollama`): Llama 3, Gemma 2, etc. (self-hosted)\n- Community: Anthropic, OpenAI, Azure OpenAI, Cohere, Mistral, Groq\n\n**Key Concepts:**\n\n- **Flexibility:** Easily swap models (`model` parameter).\n- **Zod:** For defining and validating structured output schemas.\n- **Streaming:** For real-time output using `generateStream`.\n- **Multimodality:** Handle text, image, video, audio inputs (model-dependent).\n- **Media Generation:** Create images, etc. (model-dependent).\n\n\n\nAt the heart of generative AI are AI _models_. Currently, the two most prominent\nexamples of generative models are large language models (LLMs) and image\ngeneration models. These models take input, called a _prompt_ (most commonly\ntext, an image, or a combination of both), and from it produce as output text,\nan image, or even audio or video.\n\nThe output of these models can be surprisingly convincing: LLMs generate text\nthat appears as though it could have been written by a human being, and image\ngeneration models can produce images that are very close to real photographs or\nartwork created by humans.\n\nIn addition, LLMs have proven capable of tasks beyond simple text generation:\n\n- Writing computer programs\n- Planning subtasks that are required to complete a larger task\n- Organizing unorganized data\n- Understanding and extracting information data from a corpus of text\n- Following and performing automated activities based on a text description of\n the activity\n\nThere are many models available to you, from several different providers. Each\nmodel has its own strengths and weaknesses and one model might excel at one task\nbut perform less well at others. Apps making use of generative AI can often\nbenefit from using multiple different models depending on the task at hand.\n\nAs an app developer, you typically don't interact with generative AI\nmodels directly, but rather through services available as web APIs.\nAlthough these services often have similar functionality, they all provide them\nthrough different and incompatible APIs. If you want to make use of multiple\nmodel services, you have to use each of their proprietary SDKs, potentially\nincompatible with each other. And if you want to upgrade from one model to the\nnewest and most capable one, you might have to build that integration all over\nagain.\n\nGenkit addresses this challenge by providing a single interface that abstracts\naway the details of accessing potentially any generative AI model service, with\nseveral pre-built implementations already available. Building your AI-powered\napp around Genkit simplifies the process of making your first generative AI call\nand makes it equally easy to combine multiple models or swap one model for\nanother as new models emerge.\n\n### Before you begin\n\nIf you want to run the code examples on this page, first complete the steps in\nthe [Getting started](/docs/get-started) guide. All of the examples assume that you\nhave already installed Genkit as a dependency in your project.\n\n### Models supported by Genkit\n\nGenkit is designed to be flexible enough to use potentially any generative AI\nmodel service. Its core libraries define the common interface for working with\nmodels, and model plugins define the implementation details for working with a\nspecific model and its API.\n\nThe Genkit team maintains plugins for working with models provided by Vertex AI,\nGoogle Generative AI, and Ollama:\n\n- Gemini family of LLMs, through the\n [Google Cloud Vertex AI plugin](/docs/plugins/vertex-ai)\n- Gemini family of LLMs, through the [Google AI plugin](/docs/plugins/google-genai)\n- Imagen2 and Imagen3 image generation models, through Google Cloud Vertex AI\n- Anthropic's Claude 3 family of LLMs, through Google Cloud Vertex AI's model\n garden\n- Gemma 2, Llama 3, and many more open models, through the [Ollama\n plugin](/docs/plugins/ollama) (you must host the Ollama server yourself)\n\nIn addition, there are also several community-supported plugins that provide\ninterfaces to these models:\n\n- Claude 3 family of LLMs, through the [Anthropic plugin](https://thefireco.github.io/genkit-plugins/docs/plugins/genkitx-anthropic)\n- GPT family of LLMs through the [OpenAI plugin](https://thefireco.github.io/genkit-plugins/docs/plugins/genkitx-openai)\n- GPT family of LLMs through the [Azure OpenAI plugin](https://thefireco.github.io/genkit-plugins/docs/plugins/genkitx-azure-openai)\n- Command R family of LLMs through the [Cohere plugin](https://thefireco.github.io/genkit-plugins/docs/plugins/genkitx-cohere)\n- Mistral family of LLMs through the [Mistral plugin](https://thefireco.github.io/genkit-plugins/docs/plugins/genkitx-mistral)\n- Gemma 2, Llama 3, and many more open models hosted on Groq, through the\n [Groq plugin](https://thefireco.github.io/genkit-plugins/docs/plugins/genkitx-groq)\n\nYou can discover more by searching for [packages tagged with `genkit-model` on\nnpmjs.org](https://www.npmjs.com/search?q=keywords%3Agenkit-model).\n\n### Loading and configuring model plugins\n\nBefore you can use Genkit to start generating content, you need to load and\nconfigure a model plugin. If you're coming from the Getting Started guide,\nyou've already done this. Otherwise, see the [Getting Started](/docs/get-started)\nguide or the individual plugin's documentation and follow the steps there before\ncontinuing.\n\n### The generate() method\n\nIn Genkit, the primary interface through which you interact with generative AI\nmodels is the `generate()` method.\n\nThe simplest `generate()` call specifies the model you want to use and a text\nprompt:\n\n```ts\nimport { googleAI } from '@genkit-ai/googleai';\nimport { genkit } from 'genkit';\n\nconst ai = genkit({\n plugins: [googleAI()],\n // Optional. Specify a default model.\n model: googleAI.model('gemini-2.5-flash'),\n});\n\nasync function run() {\n const response = await ai.generate('Invent a menu item for a restaurant with a pirate theme.');\n console.log(response.text);\n}\n\nrun();\n```\n\nWhen you run this brief example, it will print out some debugging information\nfollowed by the output of the `generate()` call, which will usually be Markdown\ntext as in the following example:\n\n```md\n## The Blackheart's Bounty\n\n**A hearty stew of slow-cooked beef, spiced with rum and molasses, served in a\nhollowed-out cannonball with a side of crusty bread and a dollop of tangy\npineapple salsa.**\n\n**Description:** This dish is a tribute to the hearty meals enjoyed by pirates\non the high seas. The beef is tender and flavorful, infused with the warm spices\nof rum and molasses. The pineapple salsa adds a touch of sweetness and acidity,\nbalancing the richness of the stew. The cannonball serving vessel adds a fun and\nthematic touch, making this dish a perfect choice for any pirate-themed\nadventure.\n```\n\nRun the script again and you'll get a different output.\n\nThe preceding code sample sent the generation request to the default model,\nwhich you specified when you configured the Genkit instance.\n\nYou can also specify a model for a single `generate()` call:\n\n```ts\nimport { googleAI } from '@genkit-ai/googleai';\n\nconst response = await ai.generate({\n model: googleAI.model('gemini-2.5-flash'),\n prompt: 'Invent a menu item for a restaurant with a pirate theme.',\n});\n```\n\nThis example uses a model reference function provided by the model plugin. Model\nreferences carry static type information about the model and its options which\ncan be useful for code completion in the IDE and at compile time. Many plugins\nuse this pattern, but not all, so in cases where they don't, refer to the plugin\ndocumentation for their preferred way to create function references.\n\nSometimes you may see code samples where model references are imported as\nconstants:\n\n```ts\nimport { googleAI, gemini20Flash } from '@genkit-ai/googleai';\n\nconst ai = genkit({\n plugins: [googleAI()],\n model: gemini20Flash,\n});\n```\n\nSome plugins may still use this pattern. For plugins that switched to the new\nsyntax those constants are still there and continue to work, but new constants\nfor new future models may not to be added in the future.\n\nAnother option is to specify the model using a string identifier. This way will\nwork for all plugins regardless of how they chose to handle typed model\nreferences, however you won't have the help of static type checking:\n\n```ts\nconst response = await ai.generate({\n model: 'googleai/gemini-2.5-flash-001',\n prompt: 'Invent a menu item for a restaurant with a pirate theme.',\n});\n```\n\nA model string identifier looks like `providerid/modelid`, where the provider ID\n(in this case, `googleai`) identifies the plugin, and the model ID is a\nplugin-specific string identifier for a specific version of a model.\n\nSome model plugins, such as the Ollama plugin, provide access to potentially\ndozens of different models and therefore do not export individual model\nreferences. In these cases, you can only specify a model to `generate()` using\nits string identifier.\n\nThese examples also illustrate an important point: when you use\n`generate()` to make generative AI model calls, changing the model you want to\nuse is simply a matter of passing a different value to the model parameter. By\nusing `generate()` instead of the native model SDKs, you give yourself the\nflexibility to more easily use several different models in your app and change\nmodels in the future.\n\nSo far you have only seen examples of the simplest `generate()` calls. However,\n`generate()` also provides an interface for more advanced interactions with\ngenerative models, which you will see in the sections that follow.\n\n### System prompts\n\nSome models support providing a _system prompt_, which gives the model\ninstructions as to how you want it to respond to messages from the user. You can\nuse the system prompt to specify a persona you want the model to adopt, the tone\nof its responses, the format of its responses, and so on.\n\nIf the model you're using supports system prompts, you can provide one with the\n`system` parameter:\n\n```ts\nconst response = await ai.generate({\n prompt: 'What is your quest?',\n system: \"You are a knight from Monty Python's Flying Circus.\",\n});\n```\n\n### Multi-turn conversations with messages\n\nFor multi-turn conversations, you can use the `messages` parameter instead of `prompt` to provide a conversation history. This is particularly useful when you need to maintain context across multiple interactions with the model.\n\nThe `messages` parameter accepts an array of message objects, where each message has a `role` (one of `'system'`, `'user'`, `'model'`, or `'tool'`) and `content`:\n\n```ts\nconst response = await ai.generate({\n messages: [\n { role: 'user', content: 'Hello, can you help me plan a trip?' },\n { role: 'model', content: 'Of course! I\\'d be happy to help you plan a trip. Where are you thinking of going?' },\n { role: 'user', content: 'I want to visit Japan for two weeks in spring.' }\n ],\n});\n```\n\nYou can also combine `messages` with other parameters like `system` prompts:\n\n```ts\nconst response = await ai.generate({\n system: 'You are a helpful travel assistant.',\n messages: [\n { role: 'user', content: 'What should I pack for Japan in spring?' }\n ],\n});\n```\n\n**When to use `messages` vs. Chat API:**\n\n- Use the `messages` parameter for simple multi-turn conversations where you manually manage the conversation history\n- For persistent chat sessions with automatic history management, use the [Chat API](/docs/chat) instead\n\n### Model parameters\n\nThe `generate()` function takes a `config` parameter, through which you can\nspecify optional settings that control how the model generates content:\n\n```ts\nconst response = await ai.generate({\n prompt: 'Invent a menu item for a restaurant with a pirate theme.',\n config: {\n maxOutputTokens: 512,\n stopSequences: ['\\n'],\n temperature: 1.0,\n topP: 0.95,\n topK: 40,\n },\n});\n```\n\nThe exact parameters that are supported depend on the individual model and model\nAPI. However, the parameters in the previous example are common to almost every\nmodel. The following is an explanation of these parameters:\n\n#### Parameters that control output length\n\n**maxOutputTokens**\n\nLLMs operate on units called _tokens_. A token usually, but does not\nnecessarily, map to a specific sequence of characters. When you pass a prompt to\na model, one of the first steps it takes is to _tokenize_ your prompt string\ninto a sequence of tokens. Then, the LLM generates a sequence of tokens from the\ntokenized input. Finally, the sequence of tokens gets converted back into text,\nwhich is your output.\n\nThe maximum output tokens parameter simply sets a limit on how many tokens to\ngenerate using the LLM. Every model potentially uses a different tokenizer, but\na good rule of thumb is to consider a single English word to be made of 2 to 4\ntokens.\n\nAs stated earlier, some tokens might not map to character sequences. One such\nexample is that there is often a token that indicates the end of the sequence:\nwhen an LLM generates this token, it stops generating more. Therefore, it's\npossible and often the case that an LLM generates fewer tokens than the maximum\nbecause it generated the \"stop\" token.\n\n**stopSequences**\n\nYou can use this parameter to set the tokens or token sequences that, when\ngenerated, indicate the end of LLM output. The correct values to use here\ngenerally depend on how the model was trained, and are usually set by the model\nplugin. However, if you have prompted the model to generate another stop\nsequence, you might specify it here.\n\nNote that you are specifying character sequences, and not tokens per se. In most\ncases, you will specify a character sequence that the model's tokenizer maps to\na single token.\n\n#### Parameters that control \"creativity\"\n\nThe _temperature_, _top-p_, and _top-k_ parameters together control how\n\"creative\" you want the model to be. Below are very brief explanations of what\nthese parameters mean, but the more important point to take away is this: these\nparameters are used to adjust the character of an LLM's output. The optimal\nvalues for them depend on your goals and preferences, and are likely to be found\nonly through experimentation.\n\n**temperature**\n\nLLMs are fundamentally token-predicting machines. For a given sequence of tokens\n(such as the prompt) an LLM predicts, for each token in its vocabulary, the\nlikelihood that the token comes next in the sequence. The temperature is a\nscaling factor by which these predictions are divided before being normalized to\na probability between 0 and 1.\n\nLow temperature values—between 0.0 and 1.0—amplify the difference in\nlikelihoods between tokens, with the result that the model will be even less\nlikely to produce a token it already evaluated to be unlikely. This is often\nperceived as output that is less creative. Although 0.0 is technically not a\nvalid value, many models treat it as indicating that the model should behave\ndeterministically, and to only consider the single most likely token.\n\nHigh temperature values—those greater than 1.0—compress the\ndifferences in likelihoods between tokens, with the result that the model\nbecomes more likely to produce tokens it had previously evaluated to be\nunlikely. This is often perceived as output that is more creative. Some model\nAPIs impose a maximum temperature, often 2.0.\n\n**topP**\n\n_Top-p_ is a value between 0.0 and 1.0 that controls the number of possible\ntokens you want the model to consider, by specifying the cumulative probability\nof the tokens. For example, a value of 1.0 means to consider every possible\ntoken (but still take into account the probability of each token). A value of\n0.4 means to only consider the most likely tokens, whose probabilities add up to\n0.4, and to exclude the remaining tokens from consideration.\n\n**topK**\n\n_Top-k_ is an integer value that also controls the number of possible tokens you\nwant the model to consider, but this time by explicitly specifying the maximum\nnumber of tokens. Specifying a value of 1 means that the model should behave\ndeterministically.\n\n#### Experiment with model parameters\n\nYou can experiment with the effect of these parameters on the output generated\nby different model and prompt combinations by using the Developer UI. Start the\ndeveloper UI with the `genkit start` command and it will automatically load all\nof the models defined by the plugins configured in your project. You can quickly\ntry different prompts and configuration values without having to repeatedly make\nthese changes in code.\n\n### Structured output\n\n\n\nWhen using generative AI as a component in your application, you often want\noutput in a format other than plain text. Even if you're just generating content\nto display to the user, you can benefit from structured output simply for the\npurpose of presenting it more attractively to the user. But for more advanced\napplications of generative AI, such as programmatic use of the model's output,\nor feeding the output of one model into another, structured output is a must.\n\nIn Genkit, you can request structured output from a model by specifying a schema\nwhen you call `generate()`:\n\n```ts\nimport { z } from 'genkit';\n```\n\n```ts\nconst MenuItemSchema = z.object({\n name: z.string().describe('The name of the menu item.'),\n description: z.string().describe('A description of the menu item.'),\n calories: z.number().describe('The estimated number of calories.'),\n allergens: z.array(z.string()).describe('Any known allergens in the menu item.'),\n});\n\nconst response = await ai.generate({\n prompt: 'Suggest a menu item for a pirate-themed restaurant.',\n output: { schema: MenuItemSchema },\n});\n```\n\nModel output schemas are specified using the [Zod](https://zod.dev/)\nlibrary. In addition to a schema definition language, Zod also provides runtime\ntype checking, which bridges the gap between static TypeScript types and the\nunpredictable output of generative AI models. Zod lets you write code that can\nrely on the fact that a successful generate call will always return output that\nconforms to your TypeScript types.\n\nWhen you specify a schema in `generate()`, Genkit does several things behind the\nscenes:\n\n- Augments the prompt with additional guidance about the desired output format.\n This also has the side effect of specifying to the model what content exactly\n you want to generate (for example, not only suggest a menu item but also\n generate a description, a list of allergens, and so on).\n- Parses the model output into a JavaScript object.\n- Verifies that the output conforms with the schema.\n\nTo get structured output from a successful generate call, use the response\nobject's `output` property:\n\n```ts\nconst menuItem = response.output; // Typed as z.infer\nconsole.log(menuItem?.name);\n```\n\n#### Handling errors\n\nNote in the prior example that the `output` property can be `null`. This can\nhappen when the model fails to generate output that conforms to the schema.\nThe best strategy for dealing with such errors will depend on your exact use\ncase, but here are some general hints:\n\n- **Try a different model**. For structured output to succeed, the model must be\n capable of generating output in JSON. The most powerful LLMs, like Gemini and\n Claude, are versatile enough to do this; however, smaller models, such as some\n of the local models you would use with Ollama, might not be able to generate\n structured output reliably unless they have been specifically trained to do\n so.\n\n- **Make use of Zod's coercion abilities**: You can specify in your schemas that\n Zod should try to coerce non-conforming types into the type specified by the\n schema. If your schema includes primitive types other than strings, using Zod\n coercion can reduce the number of `generate()` failures you experience. The\n following version of `MenuItemSchema` uses type coercion to automatically\n correct situations where the model generates calorie information as a string\n instead of a number:\n\n ```ts\n const MenuItemSchema = z.object({\n name: z.string().describe('The name of the menu item.'),\n description: z.string().describe('A description of the menu item.'),\n calories: z.coerce.number().describe('The estimated number of calories.'),\n allergens: z.array(z.string()).describe('Any known allergens in the menu item.'),\n });\n ```\n\n- **Retry the generate() call**. If the model you've chosen only rarely fails to\n generate conformant output, you can treat the error as you would treat a\n network error, and simply retry the request using some kind of incremental\n back-off strategy.\n\n### Streaming\n\nWhen generating large amounts of text, you can improve the experience for your\nusers by presenting the output as it's generated—streaming the output. A\nfamiliar example of streaming in action can be seen in most LLM chat apps: users\ncan read the model's response to their message as it's being generated, which\nimproves the perceived responsiveness of the application and enhances the\nillusion of chatting with an intelligent counterpart.\n\nIn Genkit, you can stream output using the `generateStream()` method. Its\nsyntax is similar to the `generate()` method:\n\n```ts\nconst { stream, response } = ai.generateStream({\n prompt: 'Tell me a story about a boy and his dog.',\n});\n```\n\nThe response object has a `stream` property, which you can use to iterate over\nthe streaming output of the request as it's generated:\n\n```ts\nfor await (const chunk of stream) {\n console.log(chunk.text);\n}\n```\n\nYou can also get the complete output of the request, as you can with a\nnon-streaming request:\n\n```ts\nconst finalResponse = await response;\nconsole.log(finalResponse.text);\n```\n\nStreaming also works with structured output:\n\n```ts\nconst { stream, response } = ai.generateStream({\n prompt: 'Suggest three pirate-themed menu items.',\n output: { schema: z.array(MenuItemSchema) },\n});\n\nfor await (const chunk of stream) {\n console.log(chunk.output);\n}\n\nconst finalResponse = await response;\nconsole.log(finalResponse.output);\n```\n\nStreaming structured output works a little differently from streaming text: the\n`output` property of a response chunk is an object constructed from the\naccumulation of the chunks that have been produced so far, rather than an object\nrepresenting a single chunk (which might not be valid on its own). **Every chunk\nof structured output in a sense supersedes the chunk that came before it**.\n\nFor example, here's what the first five outputs from the prior example might\nlook like:\n\n```js\nnull;\n\n{\n starters: [{}];\n}\n\n{\n starters: [{ name: \"Captain's Treasure Chest\", description: 'A' }];\n}\n\n{\n starters: [\n {\n name: \"Captain's Treasure Chest\",\n description: 'A mix of spiced nuts, olives, and marinated cheese served in a treasure chest.',\n calories: 350,\n },\n ];\n}\n\n{\n starters: [\n {\n name: \"Captain's Treasure Chest\",\n description: 'A mix of spiced nuts, olives, and marinated cheese served in a treasure chest.',\n calories: 350,\n allergens: [Array],\n },\n { name: 'Shipwreck Salad', description: 'Fresh' },\n ];\n}\n```\n\n### Multimodal input\n\n\n\nThe examples you've seen so far have used text strings as model prompts. While\nthis remains the most common way to prompt generative AI models, many models can\nalso accept other media as prompts. Media prompts are most often used in\nconjunction with text prompts that instruct the model to perform some operation\non the media, such as to caption an image or transcribe an audio recording.\n\nThe ability to accept media input and the types of media you can use are\ncompletely dependent on the model and its API. For example, the Gemini 1.5\nseries of models can accept images, video, and audio as prompts.\n\nTo provide a media prompt to a model that supports it, instead of passing a\nsimple text prompt to `generate`, pass an array consisting of a media part and a\ntext part:\n\n```ts\nconst response = await ai.generate({\n prompt: [{ media: { url: 'https://.../image.jpg' } }, { text: 'What is in this image?' }],\n});\n```\n\nIn the above example, you specified an image using a publicly-accessible HTTPS\nURL. You can also pass media data directly by encoding it as a data URL. For\nexample:\n\n```ts\nimport { readFile } from 'node:fs/promises';\n```\n\n```ts\nconst data = await readFile('image.jpg');\nconst response = await ai.generate({\n prompt: [{ media: { url: `data:image/jpeg;base64,${data.toString('base64')}` } }, { text: 'What is in this image?' }],\n});\n```\n\nAll models that support media input support both data URLs and HTTPS URLs. Some\nmodel plugins add support for other media sources. For example, the Vertex AI\nplugin also lets you use Cloud Storage (`gs://`) URLs.\n\n### Generating media\n\nSo far, most of the examples on this page have dealt with generating text using\nLLMs. However, Genkit can also be used with image generation models. Using\n`generate()` with an image generation model is similar to using an LLM. For\nexample, to generate an image using the Imagen2 model through Vertex AI:\n\n1. Genkit uses `data:` URLs as the standard output format for generated media.\n This is a standard format with many libraries available to handle them. This\n example uses the `data-urls` package from `jsdom`:\n\n ```bash\n npm i --save data-urls\n\n npm i --save-dev @types/data-urls\n ```\n\n1. To generate an image and save it to a file, call `generate()`, specifying an\n image generation model and the media type of output format:\n\n ```ts\n import { vertexAI } from '@genkit-ai/vertexai';\n import { parseDataUrl } from 'data-urls'; // npm install data-urls\n import { writeFile } from 'node:fs/promises';\n\n const response = await ai.generate({\n model: vertexAI.model('imagen-3.0-fast-generate-001'),\n prompt: 'An illustration of a dog wearing a space suit, photorealistic',\n output: { format: 'media' },\n });\n\n const imagePart = response.output;\n if (imagePart?.media?.url) {\n const parsed = parseDataUrl(imagePart.media.url);\n if (parsed) {\n await writeFile('dog.png', parsed.body);\n }\n }\n ```\n\n### Next steps\n\n#### Learn more about Genkit\n\n- As an app developer, the primary way you influence the output of generative AI\n models is through prompting. Read [Prompt management](/docs/dotprompt) to learn how\n Genkit helps you develop effective prompts and manage them in your codebase.\n- Although `generate()` is the nucleus of every generative AI powered\n application, real-world applications usually require additional work before\n and after invoking a generative AI model. To reflect this, Genkit introduces\n the concept of _flows_, which are defined like functions but add additional\n features such as observability and simplified deployment. To learn more, see\n [Defining workflows](/docs/flows).\n\n#### Advanced LLM use\n\n- Many of your users will have interacted with large language models for the first time through chatbots. Although LLMs are capable of much more than simulating conversations, it remains a familiar and useful style of interaction. Even when your users will not be interacting directly with the model in this way, the conversational style of prompting is a powerful way to influence the output generated by an AI model. Read [Multi-turn chats](/docs/chat) to learn how to use Genkit as part of an LLM chat implementation.\n- One way to enhance the capabilities of LLMs is to prompt them with a list of\n ways they can request more information from you, or request you to perform\n some action. This is known as _tool calling_ or _function calling_. Models\n that are trained to support this capability can respond to a prompt with a\n specially-formatted response, which indicates to the calling application that\n it should perform some action and send the result back to the LLM along with\n the original prompt. Genkit has library functions that automate both the\n prompt generation and the call-response loop elements of a tool calling\n implementation. See [Tool calling](/docs/tool-calling) to learn more.\n- Retrieval-augmented generation (RAG) is a technique used to introduce\n domain-specific information into a model's output. This is accomplished by\n inserting relevant information into a prompt before passing it on to the\n language model. A complete RAG implementation requires you to bring several\n technologies together: text embedding generation models, vector databases, and\n large language models. See [Retrieval-augmented generation (RAG)](/docs/rag) to\n learn how Genkit simplifies the process of coordinating these various\n elements.\n\n#### Testing model output\n\nAs a software engineer, you're used to deterministic systems where the same\ninput always produces the same output. However, with AI models being\nprobabilistic, the output can vary based on subtle nuances in the input, the\nmodel's training data, and even randomness deliberately introduced by parameters\nlike temperature.\n\nGenkit's evaluators are structured ways to assess the quality of your LLM's\nresponses, using a variety of strategies. Read more on the\n[Evaluation](/docs/evaluation) page.\n", + "js/models.md": { + "text": "# Generating content with AI models\n\nAt the heart of generative AI are AI _models_. Currently, the two most prominent\nexamples of generative models are large language models (LLMs) and image\ngeneration models. These models take input, called a _prompt_ (most commonly\ntext, an image, or a combination of both), and from it produce as output text,\nan image, or even audio or video.\n\nThe output of these models can be surprisingly convincing: LLMs generate text\nthat appears as though it could have been written by a human being, and image\ngeneration models can produce images that are very close to real photographs or\nartwork created by humans.\n\nIn addition, LLMs have proven capable of tasks beyond simple text generation:\n\n- Writing computer programs\n- Planning subtasks that are required to complete a larger task\n- Organizing unorganized data\n- Understanding and extracting information data from a corpus of text\n- Following and performing automated activities based on a text description of\n the activity\n\nThere are many models available to you, from several different providers. Each\nmodel has its own strengths and weaknesses and one model might excel at one task\nbut perform less well at others. Apps making use of generative AI can often\nbenefit from using multiple different models depending on the task at hand.\n\nAs an app developer, you typically don't interact with generative AI\nmodels directly, but rather through services available as web APIs.\nAlthough these services often have similar functionality, they all provide them\nthrough different and incompatible APIs. If you want to make use of multiple\nmodel services, you have to use each of their proprietary SDKs, potentially\nincompatible with each other. And if you want to upgrade from one model to the\nnewest and most capable one, you might have to build that integration all over\nagain.\n\nGenkit addresses this challenge by providing a single interface that abstracts\naway the details of accessing potentially any generative AI model service, with\nseveral pre-built implementations already available. Building your AI-powered\napp around Genkit simplifies the process of making your first generative AI call\nand makes it equally easy to combine multiple models or swap one model for\nanother as new models emerge.\n\n### Before you begin\n\nIf you want to run the code examples on this page, first complete the steps in\nthe [Getting started](/docs/get-started) guide. All of the examples assume that you\nhave already installed Genkit as a dependency in your project.\n\n### Models supported by Genkit\n\nGenkit is designed to be flexible enough to use potentially any generative AI\nmodel service. Its core libraries define the common interface for working with\nmodels, and model plugins define the implementation details for working with a\nspecific model and its API.\n\nThe Genkit team maintains plugins for working with models provided by Vertex AI,\nGoogle Generative AI, and Ollama:\n\n- Gemini family of LLMs, through the\n [Google Cloud Vertex AI plugin](/docs/plugins/vertex-ai)\n- Gemini family of LLMs, through the [Google AI plugin](/docs/plugins/google-genai)\n- Imagen2 and Imagen3 image generation models, through Google Cloud Vertex AI\n- Anthropic's Claude 3 family of LLMs, through Google Cloud Vertex AI's model\n garden\n- Gemma 2, Llama 3, and many more open models, through the [Ollama\n plugin](/docs/plugins/ollama) (you must host the Ollama server yourself)\n- GPT, Dall-E and Whisper family of models, through the [OpenAI plugin](/docs/plugins/openai)\n- Grok family of models, through the [xAI plugin](/docs/plugins/xai)\n- DeepSeek Chat and DeepSeek Reasoner models, through the [DeepSeek plugin](/docs/plugins/deepseek)\n\nIn addition, there are also several community-supported plugins that provide\ninterfaces to these models:\n\n- Claude 3 family of LLMs, through the [Anthropic plugin](https://thefireco.github.io/genkit-plugins/docs/plugins/genkitx-anthropic)\n- GPT family of LLMs through the [Azure OpenAI plugin](https://thefireco.github.io/genkit-plugins/docs/plugins/genkitx-azure-openai)\n- Command R family of LLMs through the [Cohere plugin](https://thefireco.github.io/genkit-plugins/docs/plugins/genkitx-cohere)\n- Mistral family of LLMs through the [Mistral plugin](https://thefireco.github.io/genkit-plugins/docs/plugins/genkitx-mistral)\n- Gemma 2, Llama 3, and many more open models hosted on Groq, through the\n [Groq plugin](https://thefireco.github.io/genkit-plugins/docs/plugins/genkitx-groq)\n\nYou can discover more by searching for [packages tagged with `genkit-model` on\nnpmjs.org](https://www.npmjs.com/search?q=keywords%3Agenkit-model).\n\n### Loading and configuring model plugins\n\nBefore you can use Genkit to start generating content, you need to load and\nconfigure a model plugin. If you're coming from the Getting Started guide,\nyou've already done this. Otherwise, see the [Getting Started](/docs/get-started)\nguide or the individual plugin's documentation and follow the steps there before\ncontinuing.\n\n### The generate() method\n\nIn Genkit, the primary interface through which you interact with generative AI\nmodels is the `generate()` method.\n\nThe simplest `generate()` call specifies the model you want to use and a text\nprompt:\n\n```ts\nimport { googleAI } from '@genkit-ai/googleai';\nimport { genkit } from 'genkit';\n\nconst ai = genkit({\n plugins: [googleAI()],\n // Optional. Specify a default model.\n model: googleAI.model('gemini-2.5-flash'),\n});\n\nasync function run() {\n const response = await ai.generate('Invent a menu item for a restaurant with a pirate theme.');\n console.log(response.text);\n}\n\nrun();\n```\n\nWhen you run this brief example, it will print out some debugging information\nfollowed by the output of the `generate()` call, which will usually be Markdown\ntext as in the following example:\n\n```md\n## The Blackheart's Bounty\n\n**A hearty stew of slow-cooked beef, spiced with rum and molasses, served in a\nhollowed-out cannonball with a side of crusty bread and a dollop of tangy\npineapple salsa.**\n\n**Description:** This dish is a tribute to the hearty meals enjoyed by pirates\non the high seas. The beef is tender and flavorful, infused with the warm spices\nof rum and molasses. The pineapple salsa adds a touch of sweetness and acidity,\nbalancing the richness of the stew. The cannonball serving vessel adds a fun and\nthematic touch, making this dish a perfect choice for any pirate-themed\nadventure.\n```\n\nRun the script again and you'll get a different output.\n\nThe preceding code sample sent the generation request to the default model,\nwhich you specified when you configured the Genkit instance.\n\nYou can also specify a model for a single `generate()` call:\n\n```ts\nimport { googleAI } from '@genkit-ai/googleai';\n\nconst response = await ai.generate({\n model: googleAI.model('gemini-2.5-flash'),\n prompt: 'Invent a menu item for a restaurant with a pirate theme.',\n});\n```\n\nThis example uses a model reference function provided by the model plugin. Model\nreferences carry static type information about the model and its options which\ncan be useful for code completion in the IDE and at compile time. Many plugins\nuse this pattern, but not all, so in cases where they don't, refer to the plugin\ndocumentation for their preferred way to create function references.\n\nSometimes you may see code samples where model references are imported as\nconstants:\n\n```ts\nimport { googleAI, gemini20Flash } from '@genkit-ai/googleai';\n\nconst ai = genkit({\n plugins: [googleAI()],\n model: gemini20Flash,\n});\n```\n\nSome plugins may still use this pattern. For plugins that switched to the new\nsyntax those constants are still there and continue to work, but new constants\nfor new future models may not to be added in the future.\n\nAnother option is to specify the model using a string identifier. This way will\nwork for all plugins regardless of how they chose to handle typed model\nreferences, however you won't have the help of static type checking:\n\n```ts\nconst response = await ai.generate({\n model: 'googleai/gemini-2.5-flash-001',\n prompt: 'Invent a menu item for a restaurant with a pirate theme.',\n});\n```\n\nA model string identifier looks like `providerid/modelid`, where the provider ID\n(in this case, `googleai`) identifies the plugin, and the model ID is a\nplugin-specific string identifier for a specific version of a model.\n\nSome model plugins, such as the Ollama plugin, provide access to potentially\ndozens of different models and therefore do not export individual model\nreferences. In these cases, you can only specify a model to `generate()` using\nits string identifier.\n\nThese examples also illustrate an important point: when you use\n`generate()` to make generative AI model calls, changing the model you want to\nuse is simply a matter of passing a different value to the model parameter. By\nusing `generate()` instead of the native model SDKs, you give yourself the\nflexibility to more easily use several different models in your app and change\nmodels in the future.\n\nSo far you have only seen examples of the simplest `generate()` calls. However,\n`generate()` also provides an interface for more advanced interactions with\ngenerative models, which you will see in the sections that follow.\n\n### System prompts\n\nSome models support providing a _system prompt_, which gives the model\ninstructions as to how you want it to respond to messages from the user. You can\nuse the system prompt to specify a persona you want the model to adopt, the tone\nof its responses, the format of its responses, and so on.\n\nIf the model you're using supports system prompts, you can provide one with the\n`system` parameter:\n\n```ts\nconst response = await ai.generate({\n prompt: 'What is your quest?',\n system: \"You are a knight from Monty Python's Flying Circus.\",\n});\n```\n\n### Multi-turn conversations with messages\n\nFor multi-turn conversations, you can use the `messages` parameter instead of `prompt` to provide a conversation history. This is particularly useful when you need to maintain context across multiple interactions with the model.\n\nThe `messages` parameter accepts an array of message objects, where each message has a `role` (one of `'system'`, `'user'`, `'model'`, or `'tool'`) and `content`:\n\n```ts\nconst response = await ai.generate({\n messages: [\n { role: 'user', content: 'Hello, can you help me plan a trip?' },\n { role: 'model', content: 'Of course! I\\'d be happy to help you plan a trip. Where are you thinking of going?' },\n { role: 'user', content: 'I want to visit Japan for two weeks in spring.' }\n ],\n});\n```\n\nYou can also combine `messages` with other parameters like `system` prompts:\n\n```ts\nconst response = await ai.generate({\n system: 'You are a helpful travel assistant.',\n messages: [\n { role: 'user', content: 'What should I pack for Japan in spring?' }\n ],\n});\n```\n\n**When to use `messages` vs. Chat API:**\n\n- Use the `messages` parameter for simple multi-turn conversations where you manually manage the conversation history\n- For persistent chat sessions with automatic history management, use the [Chat API](/docs/chat) instead\n\n### Model parameters\n\nThe `generate()` function takes a `config` parameter, through which you can\nspecify optional settings that control how the model generates content:\n\n```ts\nconst response = await ai.generate({\n prompt: 'Invent a menu item for a restaurant with a pirate theme.',\n config: {\n maxOutputTokens: 512,\n stopSequences: ['\\n'],\n temperature: 1.0,\n topP: 0.95,\n topK: 40,\n },\n});\n```\n\nThe exact parameters that are supported depend on the individual model and model\nAPI. However, the parameters in the previous example are common to almost every\nmodel. The following is an explanation of these parameters:\n\n#### Parameters that control output length\n\n**maxOutputTokens**\n\nLLMs operate on units called _tokens_. A token usually, but does not\nnecessarily, map to a specific sequence of characters. When you pass a prompt to\na model, one of the first steps it takes is to _tokenize_ your prompt string\ninto a sequence of tokens. Then, the LLM generates a sequence of tokens from the\ntokenized input. Finally, the sequence of tokens gets converted back into text,\nwhich is your output.\n\nThe maximum output tokens parameter simply sets a limit on how many tokens to\ngenerate using the LLM. Every model potentially uses a different tokenizer, but\na good rule of thumb is to consider a single English word to be made of 2 to 4\ntokens.\n\nAs stated earlier, some tokens might not map to character sequences. One such\nexample is that there is often a token that indicates the end of the sequence:\nwhen an LLM generates this token, it stops generating more. Therefore, it's\npossible and often the case that an LLM generates fewer tokens than the maximum\nbecause it generated the \"stop\" token.\n\n**stopSequences**\n\nYou can use this parameter to set the tokens or token sequences that, when\ngenerated, indicate the end of LLM output. The correct values to use here\ngenerally depend on how the model was trained, and are usually set by the model\nplugin. However, if you have prompted the model to generate another stop\nsequence, you might specify it here.\n\nNote that you are specifying character sequences, and not tokens per se. In most\ncases, you will specify a character sequence that the model's tokenizer maps to\na single token.\n\n#### Parameters that control \"creativity\"\n\nThe _temperature_, _top-p_, and _top-k_ parameters together control how\n\"creative\" you want the model to be. Below are very brief explanations of what\nthese parameters mean, but the more important point to take away is this: these\nparameters are used to adjust the character of an LLM's output. The optimal\nvalues for them depend on your goals and preferences, and are likely to be found\nonly through experimentation.\n\n**temperature**\n\nLLMs are fundamentally token-predicting machines. For a given sequence of tokens\n(such as the prompt) an LLM predicts, for each token in its vocabulary, the\nlikelihood that the token comes next in the sequence. The temperature is a\nscaling factor by which these predictions are divided before being normalized to\na probability between 0 and 1.\n\nLow temperature values—between 0.0 and 1.0—amplify the difference in\nlikelihoods between tokens, with the result that the model will be even less\nlikely to produce a token it already evaluated to be unlikely. This is often\nperceived as output that is less creative. Although 0.0 is technically not a\nvalid value, many models treat it as indicating that the model should behave\ndeterministically, and to only consider the single most likely token.\n\nHigh temperature values—those greater than 1.0—compress the\ndifferences in likelihoods between tokens, with the result that the model\nbecomes more likely to produce tokens it had previously evaluated to be\nunlikely. This is often perceived as output that is more creative. Some model\nAPIs impose a maximum temperature, often 2.0.\n\n**topP**\n\n_Top-p_ is a value between 0.0 and 1.0 that controls the number of possible\ntokens you want the model to consider, by specifying the cumulative probability\nof the tokens. For example, a value of 1.0 means to consider every possible\ntoken (but still take into account the probability of each token). A value of\n0.4 means to only consider the most likely tokens, whose probabilities add up to\n0.4, and to exclude the remaining tokens from consideration.\n\n**topK**\n\n_Top-k_ is an integer value that also controls the number of possible tokens you\nwant the model to consider, but this time by explicitly specifying the maximum\nnumber of tokens. Specifying a value of 1 means that the model should behave\ndeterministically.\n\n#### Experiment with model parameters\n\nYou can experiment with the effect of these parameters on the output generated\nby different model and prompt combinations by using the Developer UI. Start the\ndeveloper UI with the `genkit start` command and it will automatically load all\nof the models defined by the plugins configured in your project. You can quickly\ntry different prompts and configuration values without having to repeatedly make\nthese changes in code.\n\n### Structured output\n\n\n\nWhen using generative AI as a component in your application, you often want\noutput in a format other than plain text. Even if you're just generating content\nto display to the user, you can benefit from structured output simply for the\npurpose of presenting it more attractively to the user. But for more advanced\napplications of generative AI, such as programmatic use of the model's output,\nor feeding the output of one model into another, structured output is a must.\n\nIn Genkit, you can request structured output from a model by specifying a schema\nwhen you call `generate()`:\n\n```ts\nimport { z } from 'genkit';\n```\n\n```ts\nconst MenuItemSchema = z.object({\n name: z.string().describe('The name of the menu item.'),\n description: z.string().describe('A description of the menu item.'),\n calories: z.number().describe('The estimated number of calories.'),\n allergens: z.array(z.string()).describe('Any known allergens in the menu item.'),\n});\n\nconst response = await ai.generate({\n prompt: 'Suggest a menu item for a pirate-themed restaurant.',\n output: { schema: MenuItemSchema },\n});\n```\n\nModel output schemas are specified using the [Zod](https://zod.dev/)\nlibrary. In addition to a schema definition language, Zod also provides runtime\ntype checking, which bridges the gap between static TypeScript types and the\nunpredictable output of generative AI models. Zod lets you write code that can\nrely on the fact that a successful generate call will always return output that\nconforms to your TypeScript types.\n\nWhen you specify a schema in `generate()`, Genkit does several things behind the\nscenes:\n\n- Augments the prompt with additional guidance about the desired output format.\n This also has the side effect of specifying to the model what content exactly\n you want to generate (for example, not only suggest a menu item but also\n generate a description, a list of allergens, and so on).\n- Parses the model output into a JavaScript object.\n- Verifies that the output conforms with the schema.\n\nTo get structured output from a successful generate call, use the response\nobject's `output` property:\n\n```ts\nconst menuItem = response.output; // Typed as z.infer\nconsole.log(menuItem?.name);\n```\n\n#### Handling errors\n\nNote in the prior example that the `output` property can be `null`. This can\nhappen when the model fails to generate output that conforms to the schema.\nThe best strategy for dealing with such errors will depend on your exact use\ncase, but here are some general hints:\n\n- **Try a different model**. For structured output to succeed, the model must be\n capable of generating output in JSON. The most powerful LLMs, like Gemini and\n Claude, are versatile enough to do this; however, smaller models, such as some\n of the local models you would use with Ollama, might not be able to generate\n structured output reliably unless they have been specifically trained to do\n so.\n\n- **Make use of Zod's coercion abilities**: You can specify in your schemas that\n Zod should try to coerce non-conforming types into the type specified by the\n schema. If your schema includes primitive types other than strings, using Zod\n coercion can reduce the number of `generate()` failures you experience. The\n following version of `MenuItemSchema` uses type coercion to automatically\n correct situations where the model generates calorie information as a string\n instead of a number:\n\n ```ts\n const MenuItemSchema = z.object({\n name: z.string().describe('The name of the menu item.'),\n description: z.string().describe('A description of the menu item.'),\n calories: z.coerce.number().describe('The estimated number of calories.'),\n allergens: z.array(z.string()).describe('Any known allergens in the menu item.'),\n });\n ```\n\n- **Retry the generate() call**. If the model you've chosen only rarely fails to\n generate conformant output, you can treat the error as you would treat a\n network error, and simply retry the request using some kind of incremental\n back-off strategy.\n\n### Streaming\n\nWhen generating large amounts of text, you can improve the experience for your\nusers by presenting the output as it's generated—streaming the output. A\nfamiliar example of streaming in action can be seen in most LLM chat apps: users\ncan read the model's response to their message as it's being generated, which\nimproves the perceived responsiveness of the application and enhances the\nillusion of chatting with an intelligent counterpart.\n\nIn Genkit, you can stream output using the `generateStream()` method. Its\nsyntax is similar to the `generate()` method:\n\n```ts\nconst { stream, response } = ai.generateStream({\n prompt: 'Tell me a story about a boy and his dog.',\n});\n```\n\nThe response object has a `stream` property, which you can use to iterate over\nthe streaming output of the request as it's generated:\n\n```ts\nfor await (const chunk of stream) {\n console.log(chunk.text);\n}\n```\n\nYou can also get the complete output of the request, as you can with a\nnon-streaming request:\n\n```ts\nconst finalResponse = await response;\nconsole.log(finalResponse.text);\n```\n\nStreaming also works with structured output:\n\n```ts\nconst { stream, response } = ai.generateStream({\n prompt: 'Suggest three pirate-themed menu items.',\n output: { schema: z.array(MenuItemSchema) },\n});\n\nfor await (const chunk of stream) {\n console.log(chunk.output);\n}\n\nconst finalResponse = await response;\nconsole.log(finalResponse.output);\n```\n\nStreaming structured output works a little differently from streaming text: the\n`output` property of a response chunk is an object constructed from the\naccumulation of the chunks that have been produced so far, rather than an object\nrepresenting a single chunk (which might not be valid on its own). **Every chunk\nof structured output in a sense supersedes the chunk that came before it**.\n\nFor example, here's what the first five outputs from the prior example might\nlook like:\n\n```js\nnull;\n\n{\n starters: [{}];\n}\n\n{\n starters: [{ name: \"Captain's Treasure Chest\", description: 'A' }];\n}\n\n{\n starters: [\n {\n name: \"Captain's Treasure Chest\",\n description: 'A mix of spiced nuts, olives, and marinated cheese served in a treasure chest.',\n calories: 350,\n },\n ];\n}\n\n{\n starters: [\n {\n name: \"Captain's Treasure Chest\",\n description: 'A mix of spiced nuts, olives, and marinated cheese served in a treasure chest.',\n calories: 350,\n allergens: [Array],\n },\n { name: 'Shipwreck Salad', description: 'Fresh' },\n ];\n}\n```\n\n### Multimodal input\n\n\n\nThe examples you've seen so far have used text strings as model prompts. While\nthis remains the most common way to prompt generative AI models, many models can\nalso accept other media as prompts. Media prompts are most often used in\nconjunction with text prompts that instruct the model to perform some operation\non the media, such as to caption an image or transcribe an audio recording.\n\nThe ability to accept media input and the types of media you can use are\ncompletely dependent on the model and its API. For example, the Gemini 1.5\nseries of models can accept images, video, and audio as prompts.\n\nTo provide a media prompt to a model that supports it, instead of passing a\nsimple text prompt to `generate`, pass an array consisting of a media part and a\ntext part:\n\n```ts\nconst response = await ai.generate({\n prompt: [{ media: { url: 'https://.../image.jpg' } }, { text: 'What is in this image?' }],\n});\n```\n\nIn the above example, you specified an image using a publicly-accessible HTTPS\nURL. You can also pass media data directly by encoding it as a data URL. For\nexample:\n\n```ts\nimport { readFile } from 'node:fs/promises';\n```\n\n```ts\nconst data = await readFile('image.jpg');\nconst response = await ai.generate({\n prompt: [{ media: { url: `data:image/jpeg;base64,${data.toString('base64')}` } }, { text: 'What is in this image?' }],\n});\n```\n\nAll models that support media input support both data URLs and HTTPS URLs. Some\nmodel plugins add support for other media sources. For example, the Vertex AI\nplugin also lets you use Cloud Storage (`gs://`) URLs.\n\n### Generating Media\n\nWhile most examples in this guide focus on generating text with LLMs, Genkit also supports generating other types of media, including **images** and **audio**. Thanks to its unified `generate()` interface, working with media models is just as straightforward as generating text.\n\n:::note\nGenkit returns generated media as a **data URL**, a widely supported format for handling binary media in both browsers and Node.js environments.\n:::\n\n#### Image Generation\n\nTo generate an image using a model like Imagen from Vertex AI, follow these steps:\n\n1. **Install a data URL parser.** Genkit outputs media as data URLs, so you'll need to decode them before saving to disk. This example uses [`data-urls`](https://www.npmjs.com/package/data-urls):\n\n ```bash\n npm install data-urls\n npm install --save-dev @types/data-urls\n ```\n\n2. **Generate the image and save it to a file:**\n\n ```ts\n import { vertexAI } from '@genkit-ai/vertexai';\n import { parseDataUrl } from 'data-urls';\n import { writeFile } from 'node:fs/promises';\n\n const response = await ai.generate({\n model: vertexAI.model('imagen-3.0-fast-generate-001'),\n prompt: 'An illustration of a dog wearing a space suit, photorealistic',\n output: { format: 'media' },\n });\n\n const imagePart = response.output;\n if (imagePart?.media?.url) {\n const parsed = parseDataUrl(imagePart.media.url);\n if (parsed) {\n await writeFile('dog.png', parsed.body);\n }\n }\n ```\n\nThis will generate an image and save it as a PNG file named `dog.png`.\n\n#### Audio Generation\n\nYou can also use Genkit to generate audio with a text-to-speech (TTS) models. This is especially useful for voice features, narration, or accessibility support.\n\nHere’s how to convert text into speech and save it as an audio file:\n\n```ts\nimport { googleAI } from '@genkit-ai/googleai';\nimport { writeFile } from 'node:fs/promises';\nimport { Buffer } from 'node:buffer';\n\nconst response = await ai.generate({\n model: googleAI.model('gemini-2.5-flash-preview-tts'),\n\n // Gemini-specific configuration for audio generation\n // Available configuration options will depend on model and provider\n config: {\n responseModalities: ['AUDIO'],\n speechConfig: {\n voiceConfig: {\n prebuiltVoiceConfig: { voiceName: 'Algenib' },\n },\n },\n },\n prompt: 'Say that Genkit is an amazing AI framework',\n});\n\n// Handle the audio data (returned as a data URL)\nif (response.media?.url) {\n // Extract base64 data from the data URL\n const audioBuffer = Buffer.from(\n response.media.url.substring(response.media.url.indexOf(',') + 1),\n 'base64'\n );\n\n // Save to a file\n await writeFile('output.wav', audioBuffer);\n}\n```\n\nThis code generates speech using the Gemini TTS model and saves the result to a file named `output.wav`.\n\n### Next steps\n\n#### Learn more about Genkit\n\n- As an app developer, the primary way you influence the output of generative AI\n models is through prompting. Read [Prompt management](/docs/dotprompt) to learn how\n Genkit helps you develop effective prompts and manage them in your codebase.\n- Although `generate()` is the nucleus of every generative AI powered\n application, real-world applications usually require additional work before\n and after invoking a generative AI model. To reflect this, Genkit introduces\n the concept of _flows_, which are defined like functions but add additional\n features such as observability and simplified deployment. To learn more, see\n [Defining workflows](/docs/flows).\n\n#### Advanced LLM use\n\n- Many of your users will have interacted with large language models for the first time through chatbots. Although LLMs are capable of much more than simulating conversations, it remains a familiar and useful style of interaction. Even when your users will not be interacting directly with the model in this way, the conversational style of prompting is a powerful way to influence the output generated by an AI model. Read [Multi-turn chats](/docs/chat) to learn how to use Genkit as part of an LLM chat implementation.\n- One way to enhance the capabilities of LLMs is to prompt them with a list of\n ways they can request more information from you, or request you to perform\n some action. This is known as _tool calling_ or _function calling_. Models\n that are trained to support this capability can respond to a prompt with a\n specially-formatted response, which indicates to the calling application that\n it should perform some action and send the result back to the LLM along with\n the original prompt. Genkit has library functions that automate both the\n prompt generation and the call-response loop elements of a tool calling\n implementation. See [Tool calling](/docs/tool-calling) to learn more.\n- Retrieval-augmented generation (RAG) is a technique used to introduce\n domain-specific information into a model's output. This is accomplished by\n inserting relevant information into a prompt before passing it on to the\n language model. A complete RAG implementation requires you to bring several\n technologies together: text embedding generation models, vector databases, and\n large language models. See [Retrieval-augmented generation (RAG)](/docs/rag) to\n learn how Genkit simplifies the process of coordinating these various\n elements.\n\n#### Testing model output\n\nAs a software engineer, you're used to deterministic systems where the same\ninput always produces the same output. However, with AI models being\nprobabilistic, the output can vary based on subtle nuances in the input, the\nmodel's training data, and even randomness deliberately introduced by parameters\nlike temperature.\n\nGenkit's evaluators are structured ways to assess the quality of your LLM's\nresponses, using a variety of strategies. Read more on the\n[Evaluation](/docs/evaluation) page.", "title": "Generating content with AI models", - "lang": "js" + "description": "Learn how to generate content with AI models using Genkit's unified interface, covering basic usage, configuration, structured output, streaming, and multimodal input/output.", + "lang": "js", + "headers": "### Before you begin\n### Models supported by Genkit\n### Loading and configuring model plugins\n### The generate() method\n## The Blackheart's Bounty\n### System prompts\n### Multi-turn conversations with messages\n### Model parameters\n#### Parameters that control output length\n#### Parameters that control \"creativity\"\n#### Experiment with model parameters\n### Structured output\n#### Handling errors\n### Streaming\n### Multimodal input\n### Generating Media\n#### Image Generation\n#### Audio Generation\n### Next steps\n#### Learn more about Genkit\n#### Advanced LLM use\n#### Testing model output\n" }, "js/multi-agent.md": { - "text": ":::caution[Beta]\nThis feature of Genkit is in **Beta,** which means it is not yet part of Genkit's stable API. APIs of beta features may change in minor version releases.\n:::\n\nA powerful application of large language models are LLM-powered agents. An agent\nis a system that can carry out complex tasks by planning how to break tasks into\nsmaller ones, and (with the help of [tool calling](/docs/tool-calling)) execute tasks\nthat interact with external resources such as databases or even physical\ndevices.\n\nHere are some excerpts from a very simple customer service agent built using a\nsingle prompt and several tools:\n\n```typescript\nconst menuLookupTool = ai.defineTool(\n {\n name: 'menuLookupTool',\n description: 'use this tool to look up the menu for a given date',\n inputSchema: z.object({\n date: z.string().describe('the date to look up the menu for'),\n }),\n outputSchema: z.string().describe('the menu for a given date'),\n },\n async (input) => {\n // Retrieve the menu from a database, website, etc.\n // ...\n },\n);\n\nconst reservationTool = ai.defineTool(\n {\n name: 'reservationTool',\n description: 'use this tool to try to book a reservation',\n inputSchema: z.object({\n partySize: z.coerce.number().describe('the number of guests'),\n date: z.string().describe('the date to book for'),\n }),\n outputSchema: z\n .string()\n .describe(\n \"true if the reservation was successfully booked and false if there's\" +\n ' no table available for the requested time',\n ),\n },\n async (input) => {\n // Access your database to try to make the reservation.\n // ...\n },\n);\n```\n\n```typescript\nconst chat = ai.chat({\n model: googleAI.model('gemini-2.5-flash'),\n system:\n \"You are an AI customer service agent for Pavel's Cafe. Use the tools \" +\n 'available to you to help the customer. If you cannot help the ' +\n 'customer with the available tools, politely explain so.',\n tools: [menuLookupTool, reservationTool],\n});\n```\n\nA simple architecture like the one shown above can be sufficient when your agent\nonly has a few capabilities. However, even for the limited example above, you\ncan see that there are some capabilities that customers would likely expect: for\nexample, listing the customer's current reservations, canceling a reservation,\nand so on. As you build more and more tools to implement these additional\ncapabilities, you start to run into some problems:\n\n- The more tools you add, the more you stretch the model's ability to\n consistently and correctly employ the right tool for the job.\n- Some tasks might best be served through a more focused back and forth\n between the user and the agent, rather than by a single tool call.\n- Some tasks might benefit from a specialized prompt. For example, if your\n agent is responding to an unhappy customer, you might want its tone to be\n more business-like, whereas the agent that greets the customer initially can\n have a more friendly and lighthearted tone.\n\nOne approach you can use to deal with these issues that arise when building\ncomplex agents is to create many specialized agents and use a general purpose\nagent to delegate tasks to them. Genkit supports this architecture by allowing\nyou to specify prompts as tools. Each prompt represents a single specialized\nagent, with its own set of tools available to it, and those agents are in turn\navailable as tools to your single orchestration agent, which is the primary\ninterface with the user.\n\nHere's what an expanded version of the previous example might look like as a\nmulti-agent system:\n\n```typescript\n// Define a prompt that represents a specialist agent\nconst reservationAgent = ai.definePrompt({\n name: 'reservationAgent',\n description: 'Reservation Agent can help manage guest reservations',\n tools: [reservationTool, reservationCancelationTool, reservationListTool],\n system: 'Help guests make and manage reservations',\n});\n\n// Or load agents from .prompt files\nconst menuInfoAgent = ai.prompt('menuInfoAgent');\nconst complaintAgent = ai.prompt('complaintAgent');\n\n// The triage agent is the agent that users interact with initially\nconst triageAgent = ai.definePrompt({\n name: 'triageAgent',\n description: 'Triage Agent',\n tools: [reservationAgent, menuInfoAgent, complaintAgent],\n system: `You are an AI customer service agent for Pavel's Cafe.\n Greet the user and ask them how you can help. If appropriate, transfer to an\n agent that can better handle the request. If you cannot help the customer with\n the available tools, politely explain so.`,\n});\n```\n\n```typescript\n// Start a chat session, initially with the triage agent\nconst chat = ai.chat(triageAgent);\n```\n", + "text": "# Building multi-agent systems\n\n:::caution[Beta]\nThis feature of Genkit is in **Beta,** which means it is not yet part of Genkit's stable API. APIs of beta features may change in minor version releases.\n:::\n\nA powerful application of large language models are LLM-powered agents. An agent\nis a system that can carry out complex tasks by planning how to break tasks into\nsmaller ones, and (with the help of [tool calling](/docs/tool-calling)) execute tasks\nthat interact with external resources such as databases or even physical\ndevices.\n\nHere are some excerpts from a very simple customer service agent built using a\nsingle prompt and several tools:\n\n```typescript\nconst menuLookupTool = ai.defineTool(\n {\n name: 'menuLookupTool',\n description: 'use this tool to look up the menu for a given date',\n inputSchema: z.object({\n date: z.string().describe('the date to look up the menu for'),\n }),\n outputSchema: z.string().describe('the menu for a given date'),\n },\n async (input) => {\n // Retrieve the menu from a database, website, etc.\n // ...\n },\n);\n\nconst reservationTool = ai.defineTool(\n {\n name: 'reservationTool',\n description: 'use this tool to try to book a reservation',\n inputSchema: z.object({\n partySize: z.coerce.number().describe('the number of guests'),\n date: z.string().describe('the date to book for'),\n }),\n outputSchema: z\n .string()\n .describe(\n \"true if the reservation was successfully booked and false if there's\" +\n ' no table available for the requested time',\n ),\n },\n async (input) => {\n // Access your database to try to make the reservation.\n // ...\n },\n);\n```\n\n```typescript\nconst chat = ai.chat({\n model: googleAI.model('gemini-2.5-flash'),\n system:\n \"You are an AI customer service agent for Pavel's Cafe. Use the tools \" +\n 'available to you to help the customer. If you cannot help the ' +\n 'customer with the available tools, politely explain so.',\n tools: [menuLookupTool, reservationTool],\n});\n```\n\nA simple architecture like the one shown above can be sufficient when your agent\nonly has a few capabilities. However, even for the limited example above, you\ncan see that there are some capabilities that customers would likely expect: for\nexample, listing the customer's current reservations, canceling a reservation,\nand so on. As you build more and more tools to implement these additional\ncapabilities, you start to run into some problems:\n\n- The more tools you add, the more you stretch the model's ability to\n consistently and correctly employ the right tool for the job.\n- Some tasks might best be served through a more focused back and forth\n between the user and the agent, rather than by a single tool call.\n- Some tasks might benefit from a specialized prompt. For example, if your\n agent is responding to an unhappy customer, you might want its tone to be\n more business-like, whereas the agent that greets the customer initially can\n have a more friendly and lighthearted tone.\n\nOne approach you can use to deal with these issues that arise when building\ncomplex agents is to create many specialized agents and use a general purpose\nagent to delegate tasks to them. Genkit supports this architecture by allowing\nyou to specify prompts as tools. Each prompt represents a single specialized\nagent, with its own set of tools available to it, and those agents are in turn\navailable as tools to your single orchestration agent, which is the primary\ninterface with the user.\n\nHere's what an expanded version of the previous example might look like as a\nmulti-agent system:\n\n```typescript\n// Define a prompt that represents a specialist agent\nconst reservationAgent = ai.definePrompt({\n name: 'reservationAgent',\n description: 'Reservation Agent can help manage guest reservations',\n tools: [reservationTool, reservationCancelationTool, reservationListTool],\n system: 'Help guests make and manage reservations',\n});\n\n// Or load agents from .prompt files\nconst menuInfoAgent = ai.prompt('menuInfoAgent');\nconst complaintAgent = ai.prompt('complaintAgent');\n\n// The triage agent is the agent that users interact with initially\nconst triageAgent = ai.definePrompt({\n name: 'triageAgent',\n description: 'Triage Agent',\n tools: [reservationAgent, menuInfoAgent, complaintAgent],\n system: `You are an AI customer service agent for Pavel's Cafe.\n Greet the user and ask them how you can help. If appropriate, transfer to an\n agent that can better handle the request. If you cannot help the customer with\n the available tools, politely explain so.`,\n});\n```\n\n```typescript\n// Start a chat session, initially with the triage agent\nconst chat = ai.chat(triageAgent);\n```\n", "title": "Building multi-agent systems", - "lang": "js" + "description": "Learn how to build multi-agent systems in Genkit by delegating tasks to specialized agents, addressing challenges of complex agentic workflows.", + "lang": "js", + "headers": "" }, - "js/nextjs.mdx": { - "text": "import { Tabs, TabItem } from '@astrojs/starlight/components';\n\nThis page shows how you can use Genkit flows in your Next.js applications using the official Genkit Next.js plugin. For complete API reference documentation, see the [Genkit Next.js Plugin API Reference](https://js.api.genkit.dev/modules/_genkit-ai_next.html).\n\n## Before you begin\n\nYou should be familiar with Genkit's concept of [flows](/docs/flows), and how to write them.\n\n## Create a Next.js project\n\nIf you don't already have a Next.js project that you want to add generative AI features to, you can create one for the purpose of following along with this page:\n\n```bash\nnpx create-next-app@latest --src-dir\n```\n\nThe `--src-dir` flag creates a `src/` directory to keep your project organized by separating source code from configuration files.\n\n## Install Genkit dependencies\n\nInstall the Genkit dependencies into your Next.js app:\n\n1. Install the core Genkit library and the Next.js plugin:\n\n ```bash\n npm install genkit @genkit-ai/next\n ```\n\n2. Install at least one model plugin.\n\n \n \n ```bash\n npm install @genkit-ai/googleai\n ```\n \n \n\n ```bash\n npm install @genkit-ai/vertexai\n ```\n \n \n\n3. Install the Genkit CLI globally. The tsx tool is also recommended as a development dependency, as it makes testing your code more convenient. Both of these dependencies are optional, however.\n\n ```bash\n npm install -g genkit-cli\n npm install --save-dev tsx\n ```\n\n## Define Genkit flows\n\nCreate a new directory in your Next.js project to contain your Genkit flows. Create `src/genkit/` and add your flow definitions there:\n\nFor example, create `src/genkit/menuSuggestionFlow.ts`:\n\n\n \n ```ts\n import { googleAI } from '@genkit-ai/googleai';\n import { genkit, z } from 'genkit';\n\n const ai = genkit({\n plugins: [googleAI()],\n });\n\n export const menuSuggestionFlow = ai.defineFlow(\n {\n name: 'menuSuggestionFlow',\n inputSchema: z.object({ theme: z.string() }),\n outputSchema: z.object({ menuItem: z.string() }),\n streamSchema: z.string(),\n },\n async ({ theme }, { sendChunk }) => {\n const { stream, response } = ai.generateStream({\n model: googleAI.model('gemini-2.5-flash'),\n prompt: `Invent a menu item for a ${theme} themed restaurant.`,\n });\n\n for await (const chunk of stream) {\n sendChunk(chunk.text);\n }\n\n const { text } = await response;\n return { menuItem: text };\n }\n );\n ```\n \n \n ```ts\n import { vertexAI } from '@genkit-ai/vertexai';\n import { genkit, z } from 'genkit';\n\n const ai = genkit({\n plugins: [vertexAI()],\n });\n\n export const menuSuggestionFlow = ai.defineFlow(\n {\n name: 'menuSuggestionFlow',\n inputSchema: z.object({ theme: z.string() }),\n outputSchema: z.object({ menuItem: z.string() }),\n streamSchema: z.string(),\n },\n async ({ theme }, { sendChunk }) => {\n const { stream, response } = ai.generateStream({\n model: vertexAI.model('gemini-2.5-flash'),\n prompt: `Invent a menu item for a ${theme} themed restaurant.`,\n });\n\n for await (const chunk of stream) {\n sendChunk(chunk.text);\n }\n\n const { text } = await response;\n return { menuItem: text };\n }\n );\n ```\n \n\n\n## Create API routes\n\nNow, create API routes that expose your flows using the Genkit Next.js plugin. For each flow, create a corresponding route file:\n\nCreate `src/app/api/menuSuggestion/route.ts`:\n\n```ts\nimport { menuSuggestionFlow } from '@/genkit/menuSuggestionFlow';\nimport { appRoute } from '@genkit-ai/next';\n\nexport const POST = appRoute(menuSuggestionFlow);\n```\n\n## Call your flows from the frontend\n\nIn your frontend code, you can now call your flows using the Genkit Next.js client:\n\n```tsx\n'use client';\n\nimport { useState } from 'react';\nimport { runFlow, streamFlow } from '@genkit-ai/next/client';\nimport { menuSuggestionFlow } from '@/genkit/menuSuggestionFlow';\n\nexport default function Home() {\n const [menuItem, setMenuItem] = useState('');\n const [isLoading, setIsLoading] = useState(false);\n const [streamedText, setStreamedText] = useState('');\n\n async function getMenuItem(formData: FormData) {\n const theme = formData.get('theme')?.toString() ?? '';\n setIsLoading(true);\n\n try {\n // Regular (non-streaming) approach\n const result = await runFlow({\n url: '/api/menuSuggestion',\n input: { theme },\n });\n\n setMenuItem(result.menuItem);\n } catch (error) {\n console.error('Error generating menu item:', error);\n } finally {\n setIsLoading(false);\n }\n }\n\n async function streamMenuItem(formData: FormData) {\n const theme = formData.get('theme')?.toString() ?? '';\n setIsLoading(true);\n setStreamedText('');\n\n try {\n // Streaming approach\n const result = streamFlow({\n url: '/api/menuSuggestion',\n input: { theme },\n });\n\n // Process the stream chunks as they arrive\n for await (const chunk of result.stream) {\n setStreamedText((prev) => prev + chunk);\n }\n\n // Get the final complete response\n const finalOutput = await result.output;\n setMenuItem(finalOutput.menuItem);\n } catch (error) {\n console.error('Error streaming menu item:', error);\n } finally {\n setIsLoading(false);\n }\n }\n\n return (\n
\n
\n \n \n
\n
\n \n {\n e.preventDefault();\n const formData = new FormData(e.currentTarget.form!);\n streamMenuItem(formData);\n }}\n >\n Stream Generation\n \n \n
\n\n {streamedText && (\n
\n

Streaming Output:

\n 
{streamedText}
\n
\n )}\n\n {menuItem && (\n
\n

Final Output:

\n 
{menuItem}
\n
\n )}\n
\n );\n}\n```\n\n## Authentication (Optional)\n\nIf you need to add authentication to your API routes, you can pass headers with your requests:\n\n```tsx\nconst result = await runFlow({\n url: '/api/menuSuggestion',\n headers: {\n Authorization: 'Bearer your-token-here',\n },\n input: { theme },\n});\n```\n\n## Test your app locally\n\nIf you want to run your app locally, you need to make credentials for the model API service you chose available.\n\n\n \n 1. [Generate an API key](https://aistudio.google.com/app/apikey) for the Gemini API using Google AI Studio.\n\n 2. Set the `GEMINI_API_KEY` environment variable to your key:\n\n ```bash\n export GEMINI_API_KEY=\n ```\n\n \n \n 1. In the Cloud console, [Enable the Vertex AI API](https://console.cloud.google.com/apis/library/aiplatform.googleapis.com?project=_) for your project.\n\n 2. Configure the [`gcloud`](https://cloud.google.com/sdk/gcloud) tool to set up application default credentials:\n\n ```bash\n gcloud config set project \n gcloud services enable aiplatform.googleapis.com\n ```\n\n \n\n\nThen, run your app locally as normal:\n\n```bash\nnpm run dev\n```\n\nFor Genkit development tools, you can still use:\n\n```bash\ngenkit start -- npx tsx --watch src/genkit/menuSuggestionFlow.ts\n```\n\n## Deploy your app\n\nWhen you deploy your app, you will need to make sure the credentials for any external services you use (such as your chosen model API service) are available to the deployed app. See the following pages for information specific to your chosen deployment platform:\n\n- [Cloud Functions for Firebase](/docs/firebase)\n- [Cloud Run](/docs/cloud-run)\n- [Other Node.js platforms](/docs/deploy-node)", + "js/nextjs.md": { + "text": "# Use Genkit in a Next.js app\n\nThis page shows how you can use Genkit flows in your Next.js applications using the official Genkit Next.js plugin. For complete API reference documentation, see the [Genkit Next.js Plugin API Reference](https://js.api.genkit.dev/modules/_genkit-ai_next.html).\n\n## Before you begin\n\nYou should be familiar with Genkit's concept of [flows](/docs/flows), and how to write them.\n\n## Create a Next.js project\n\nIf you don't already have a Next.js project that you want to add generative AI features to, you can create one for the purpose of following along with this page:\n\n```bash\nnpx create-next-app@latest --src-dir\n```\n\nThe `--src-dir` flag creates a `src/` directory to keep your project organized by separating source code from configuration files.\n\n## Install Genkit dependencies\n\nInstall the Genkit dependencies into your Next.js app:\n\n1. Install the core Genkit library and the Next.js plugin:\n\n ```bash\n npm install genkit @genkit-ai/next\n ```\n\n2. Install at least one model plugin.\n\n \n \n ```bash\n npm install @genkit-ai/googleai\n ```\n \n \n\n ```bash\n npm install @genkit-ai/vertexai\n ```\n \n \n\n3. Install the Genkit CLI globally. The tsx tool is also recommended as a development dependency, as it makes testing your code more convenient. Both of these dependencies are optional, however.\n\n ```bash\n npm install -g genkit-cli\n npm install --save-dev tsx\n ```\n\n## Define Genkit flows\n\nCreate a new directory in your Next.js project to contain your Genkit flows. Create `src/genkit/` and add your flow definitions there:\n\nFor example, create `src/genkit/menuSuggestionFlow.ts`:\n\n\n \n ```ts\n import { googleAI } from '@genkit-ai/googleai';\n import { genkit, z } from 'genkit';\n\n const ai = genkit({\n plugins: [googleAI()],\n });\n\n export const menuSuggestionFlow = ai.defineFlow(\n {\n name: 'menuSuggestionFlow',\n inputSchema: z.object({ theme: z.string() }),\n outputSchema: z.object({ menuItem: z.string() }),\n streamSchema: z.string(),\n },\n async ({ theme }, { sendChunk }) => {\n const { stream, response } = ai.generateStream({\n model: googleAI.model('gemini-2.5-flash'),\n prompt: `Invent a menu item for a ${theme} themed restaurant.`,\n });\n\n for await (const chunk of stream) {\n sendChunk(chunk.text);\n }\n\n const { text } = await response;\n return { menuItem: text };\n }\n );\n ```\n \n \n ```ts\n import { vertexAI } from '@genkit-ai/vertexai';\n import { genkit, z } from 'genkit';\n\n const ai = genkit({\n plugins: [vertexAI()],\n });\n\n export const menuSuggestionFlow = ai.defineFlow(\n {\n name: 'menuSuggestionFlow',\n inputSchema: z.object({ theme: z.string() }),\n outputSchema: z.object({ menuItem: z.string() }),\n streamSchema: z.string(),\n },\n async ({ theme }, { sendChunk }) => {\n const { stream, response } = ai.generateStream({\n model: vertexAI.model('gemini-2.5-flash'),\n prompt: `Invent a menu item for a ${theme} themed restaurant.`,\n });\n\n for await (const chunk of stream) {\n sendChunk(chunk.text);\n }\n\n const { text } = await response;\n return { menuItem: text };\n }\n );\n ```\n \n\n\n## Create API routes\n\nNow, create API routes that expose your flows using the Genkit Next.js plugin. For each flow, create a corresponding route file:\n\nCreate `src/app/api/menuSuggestion/route.ts`:\n\n```ts\nimport { menuSuggestionFlow } from '@/genkit/menuSuggestionFlow';\nimport { appRoute } from '@genkit-ai/next';\n\nexport const POST = appRoute(menuSuggestionFlow);\n```\n\n## Call your flows from the frontend\n\nIn your frontend code, you can now call your flows using the Genkit Next.js client:\n\n```tsx\n'use client';\n\nimport { useState } from 'react';\nimport { runFlow, streamFlow } from '@genkit-ai/next/client';\nimport { menuSuggestionFlow } from '@/genkit/menuSuggestionFlow';\n\nexport default function Home() {\n const [menuItem, setMenuItem] = useState('');\n const [isLoading, setIsLoading] = useState(false);\n const [streamedText, setStreamedText] = useState('');\n\n async function getMenuItem(formData: FormData) {\n const theme = formData.get('theme')?.toString() ?? '';\n setIsLoading(true);\n\n try {\n // Regular (non-streaming) approach\n const result = await runFlow({\n url: '/api/menuSuggestion',\n input: { theme },\n });\n\n setMenuItem(result.menuItem);\n } catch (error) {\n console.error('Error generating menu item:', error);\n } finally {\n setIsLoading(false);\n }\n }\n\n async function streamMenuItem(formData: FormData) {\n const theme = formData.get('theme')?.toString() ?? '';\n setIsLoading(true);\n setStreamedText('');\n\n try {\n // Streaming approach\n const result = streamFlow({\n url: '/api/menuSuggestion',\n input: { theme },\n });\n\n // Process the stream chunks as they arrive\n for await (const chunk of result.stream) {\n setStreamedText((prev) => prev + chunk);\n }\n\n // Get the final complete response\n const finalOutput = await result.output;\n setMenuItem(finalOutput.menuItem);\n } catch (error) {\n console.error('Error streaming menu item:', error);\n } finally {\n setIsLoading(false);\n }\n }\n\n return (\n
\n
\n \n \n
\n
\n \n {\n e.preventDefault();\n const formData = new FormData(e.currentTarget.form!);\n streamMenuItem(formData);\n }}\n >\n Stream Generation\n \n \n
\n\n {streamedText && (\n
\n

Streaming Output:

\n 
{streamedText}
\n
\n )}\n\n {menuItem && (\n
\n

Final Output:

\n 
{menuItem}
\n
\n )}\n
\n );\n}\n```\n\n## Authentication (Optional)\n\nIf you need to add authentication to your API routes, you can pass headers with your requests:\n\n```tsx\nconst result = await runFlow({\n url: '/api/menuSuggestion',\n headers: {\n Authorization: 'Bearer your-token-here',\n },\n input: { theme },\n});\n```\n\n## Test your app locally\n\nIf you want to run your app locally, you need to make credentials for the model API service you chose available.\n\n\n \n 1. [Generate an API key](https://aistudio.google.com/app/apikey) for the Gemini API using Google AI Studio.\n\n 2. Set the `GEMINI_API_KEY` environment variable to your key:\n\n ```bash\n export GEMINI_API_KEY=\n ```\n\n \n \n 1. In the Cloud console, [Enable the Vertex AI API](https://console.cloud.google.com/apis/library/aiplatform.googleapis.com?project=_) for your project.\n\n 2. Configure the [`gcloud`](https://cloud.google.com/sdk/gcloud) tool to set up application default credentials:\n\n ```bash\n gcloud config set project \n gcloud services enable aiplatform.googleapis.com\n ```\n\n \n\n\nThen, run your app locally as normal:\n\n```bash\nnpm run dev\n```\n\nFor Genkit development tools, you can still use:\n\n```bash\ngenkit start -- npx tsx --watch src/genkit/menuSuggestionFlow.ts\n```\n\n## Deploy your app\n\nWhen you deploy your app, you will need to make sure the credentials for any external services you use (such as your chosen model API service) are available to the deployed app. See the following pages for information specific to your chosen deployment platform:\n\n- [Cloud Functions for Firebase](/docs/firebase)\n- [Cloud Run](/docs/cloud-run)\n- [Other Node.js platforms](/docs/deploy-node)", "title": "Use Genkit in a Next.js app", - "lang": "js" + "description": "Learn how to integrate Genkit flows into your Next.js applications using the official Genkit Next.js plugin, covering project setup, flow definition, API routes, and client-side calls.", + "lang": "js", + "headers": "## Before you begin\n## Create a Next.js project\n## Install Genkit dependencies\n## Define Genkit flows\n## Create API routes\n## Call your flows from the frontend\n## Authentication (Optional)\n## Test your app locally\n## Deploy your app\n" }, "js/plugin-authoring-evaluator.md": { - "text": "You can extend Genkit to support custom evaluation, using either\nan LLM as a judge, or by programmatic (heuristic) evaluation.\n\n## Evaluator definition\n\nEvaluators are functions that assess an LLM's response. There are two main\napproaches to automated evaluation: heuristic evaluation and LLM-based\nevaluation. In the heuristic approach, you define a deterministic function.\nBy contrast, in an LLM-based assessment, the content is fed back to an LLM,\nand the LLM is asked to score the output according to criteria set in a\nprompt.\n\nThe `ai.defineEvaluator` method, which you use to define an\nevaluator action in Genkit, supports either approach. This\ndocument explores a couple of examples of how to use this\nmethod for heuristic and LLM-based evaluations.\n\n### LLM-based Evaluators\n\nAn LLM-based evaluator leverages an LLM to evaluate\nthe `input`, `context`, and `output` of your generative AI\nfeature.\n\nLLM-based evaluators in Genkit are made up of 3 components:\n\n- A prompt\n- A scoring function\n- An evaluator action\n\n#### Define the prompt\n\nFor this example, the evaluator leverages an LLM to determine whether a\nfood (the `output`) is delicious or not. First, provide context to the LLM,\nthen describe what you want it to do, and finally, give it a few examples\nto base its response on.\n\nGenkit’s `definePrompt` utility provides an easy way to define prompts with\ninput and output validation. The following code is an example of\nsetting up an evaluation prompt with `definePrompt`.\n\n```ts\nimport { z } from \"genkit\";\n\nconst DELICIOUSNESS_VALUES = ['yes', 'no', 'maybe'] as const;\n\nconst DeliciousnessDetectionResponseSchema = z.object({\n reason: z.string(),\n verdict: z.enum(DELICIOUSNESS_VALUES),\n});\n\nfunction getDeliciousnessPrompt(ai: Genkit) {\n return ai.definePrompt({\n name: 'deliciousnessPrompt',\n input: {\n schema: z.object({\n responseToTest: z.string(),\n }),\n },\n output: {\n schema: DeliciousnessDetectionResponseSchema,\n }\n prompt: `You are a food critic. Assess whether the provided output sounds delicious, giving only \"yes\" (delicious), \"no\" (not delicious), or \"maybe\" (undecided) as the verdict.\n\n Examples:\n Output: Chicken parm sandwich\n Response: { \"reason\": \"A classic and beloved dish.\", \"verdict\": \"yes\" }\n\n Output: Boston Logan Airport tarmac\n Response: { \"reason\": \"Not edible.\", \"verdict\": \"no\" }\n\n Output: A juicy piece of gossip\n Response: { \"reason\": \"Metaphorically 'tasty' but not food.\", \"verdict\": \"maybe\" }\n\n New Output: {{ responseToTest }}\n Response:\n `\n });\n}\n```\n\n#### Define the scoring function\n\nDefine a function that takes an example that includes `output` as\nrequired by the prompt, and scores the result. Genkit testcases include\n`input` as a required field, with `output` and `context` as optional fields.\nIt is the responsibility of the evaluator to validate that all fields\nrequired for evaluation are present.\n\n```ts\nimport { ModelArgument } from 'genkit';\nimport { BaseEvalDataPoint, Score } from 'genkit/evaluator';\n\n/**\n * Score an individual test case for delciousness.\n */\nexport async function deliciousnessScore(\n ai: Genkit,\n judgeLlm: ModelArgument,\n dataPoint: BaseEvalDataPoint,\n judgeConfig?: CustomModelOptions,\n): Promise {\n const d = dataPoint;\n // Validate the input has required fields\n if (!d.output) {\n throw new Error('Output is required for Deliciousness detection');\n }\n\n // Hydrate the prompt and generate an evaluation result\n const deliciousnessPrompt = getDeliciousnessPrompt(ai);\n const response = await deliciousnessPrompt(\n {\n responseToTest: d.output as string,\n },\n {\n model: judgeLlm,\n config: judgeConfig,\n },\n );\n\n // Parse the output\n const parsedResponse = response.output;\n if (!parsedResponse) {\n throw new Error(`Unable to parse evaluator response: ${response.text}`);\n }\n\n // Return a scored response\n return {\n score: parsedResponse.verdict,\n details: { reasoning: parsedResponse.reason },\n };\n}\n```\n\n#### Define the evaluator action\n\nThe final step is to write a function that defines the `EvaluatorAction`.\n\n```ts\nimport { EvaluatorAction } from 'genkit/evaluator';\n\n/**\n * Create the Deliciousness evaluator action.\n */\nexport function createDeliciousnessEvaluator(\n ai: Genkit,\n judge: ModelArgument,\n judgeConfig?: z.infer,\n): EvaluatorAction {\n return ai.defineEvaluator(\n {\n name: `myCustomEvals/deliciousnessEvaluator`,\n displayName: 'Deliciousness',\n definition: 'Determines if output is considered delicous.',\n isBilled: true,\n },\n async (datapoint: BaseEvalDataPoint) => {\n const score = await deliciousnessScore(ai, judge, datapoint, judgeConfig);\n return {\n testCaseId: datapoint.testCaseId,\n evaluation: score,\n };\n },\n );\n}\n```\n\nThe `defineEvaluator` method is similar to other Genkit constructors like\n`defineFlow` and `defineRetriever`. This method requires an `EvaluatorFn`\nto be provided as a callback. The `EvaluatorFn` method accepts a\n`BaseEvalDataPoint` object, which corresponds to a single entry in a\ndataset under evaluation, along with an optional custom-options\nparameter if specified. The function processes the datapoint and\nreturns an `EvalResponse` object.\n\nThe Zod Schemas for `BaseEvalDataPoint` and `EvalResponse` are\nas follows.\n\n##### `BaseEvalDataPoint`\n\n```ts\nexport const BaseEvalDataPoint = z.object({\n testCaseId: z.string(),\n input: z.unknown(),\n output: z.unknown().optional(),\n context: z.array(z.unknown()).optional(),\n reference: z.unknown().optional(),\n testCaseId: z.string().optional(),\n traceIds: z.array(z.string()).optional(),\n});\n\nexport const EvalResponse = z.object({\n sampleIndex: z.number().optional(),\n testCaseId: z.string(),\n traceId: z.string().optional(),\n spanId: z.string().optional(),\n evaluation: z.union([ScoreSchema, z.array(ScoreSchema)]),\n});\n```\n\n##### `ScoreSchema`\n\n```ts\nconst ScoreSchema = z.object({\n id: z.string().describe('Optional ID to differentiate multiple scores').optional(),\n score: z.union([z.number(), z.string(), z.boolean()]).optional(),\n error: z.string().optional(),\n details: z\n .object({\n reasoning: z.string().optional(),\n })\n .passthrough()\n .optional(),\n});\n```\n\nThe `defineEvaluator` object lets the user provide a name, a user-readable\ndisplay name, and a definition for the evaluator. The display name and\ndefiniton are displayed along with evaluation results in the Dev UI.\nIt also has an optional `isBilled` field that marks whether this evaluator\ncan result in billing (e.g., it uses a billed LLM or API). If an evaluator\nis billed, the UI prompts the user for a confirmation in the CLI before\nallowing them to run an evaluation. This step helps guard against\nunintended expenses.\n\n### Heuristic Evaluators\n\nA heuristic evaluator can be any function used to evaluate the `input`, `context`,\nor `output` of your generative AI feature.\n\nHeuristic evaluators in Genkit are made up of 2 components:\n\n- A scoring function\n- An evaluator action\n\n#### Define the scoring function\n\nAs with the LLM-based evaluator, define the scoring function. In this case,\nthe scoring function does not need a judge LLM.\n\n```ts\nimport { BaseEvalDataPoint, Score } from 'genkit/evaluator';\n\nconst US_PHONE_REGEX = /[\\+]?[(]?[0-9]{3}[)]?[-\\s\\.]?[0-9]{3}[-\\s\\.]?[0-9]{4}/i;\n\n/**\n * Scores whether a datapoint output contains a US Phone number.\n */\nexport async function usPhoneRegexScore(dataPoint: BaseEvalDataPoint): Promise {\n const d = dataPoint;\n if (!d.output || typeof d.output !== 'string') {\n throw new Error('String output is required for regex matching');\n }\n const matches = US_PHONE_REGEX.test(d.output as string);\n const reasoning = matches ? `Output matched US_PHONE_REGEX` : `Output did not match US_PHONE_REGEX`;\n return {\n score: matches,\n details: { reasoning },\n };\n}\n```\n\n#### Define the evaluator action\n\n```ts\nimport { Genkit } from 'genkit';\nimport { BaseEvalDataPoint, EvaluatorAction } from 'genkit/evaluator';\n\n/**\n * Configures a regex evaluator to match a US phone number.\n */\nexport function createUSPhoneRegexEvaluator(ai: Genkit): EvaluatorAction {\n return ai.defineEvaluator(\n {\n name: `myCustomEvals/usPhoneRegexEvaluator`,\n displayName: 'Regex Match for US PHONE NUMBER',\n definition: 'Uses Regex to check if output matches a US phone number',\n isBilled: false,\n },\n async (datapoint: BaseEvalDataPoint) => {\n const score = await usPhoneRegexScore(datapoint);\n return {\n testCaseId: datapoint.testCaseId,\n evaluation: score,\n };\n },\n );\n}\n```\n\n## Putting it together\n\n### Plugin definition\n\nPlugins are registered with the framework by installing them at the time of\ninitializing Genkit. To define a new plugin, use the `genkitPlugin` helper\nmethod to instantiate all Genkit actions within the plugin context.\n\nThis code sample shows two evaluators: the LLM-based deliciousness evaluator,\nand the regex-based US phone number evaluator. Instantiating these\nevaluators within the plugin context registers them with the plugin.\n\n```ts\nimport { GenkitPlugin, genkitPlugin } from 'genkit/plugin';\n\nexport function myCustomEvals(options: {\n judge: ModelArgument;\n judgeConfig?: ModelCustomOptions;\n}): GenkitPlugin {\n // Define the new plugin\n return genkitPlugin('myCustomEvals', async (ai: Genkit) => {\n const { judge, judgeConfig } = options;\n\n // The plugin instatiates our custom evaluators within the context\n // of the `ai` object, making them available\n // throughout our Genkit application.\n createDeliciousnessEvaluator(ai, judge, judgeConfig);\n createUSPhoneRegexEvaluator(ai);\n });\n}\nexport default myCustomEvals;\n```\n\n### Configure Genkit\n\nAdd the `myCustomEvals` plugin to your Genkit configuration.\n\nFor evaluation with Gemini, disable safety settings so that the evaluator can\naccept, detect, and score potentially harmful content.\n\n```ts\nimport { googleAI } from '@genkit-ai/googleai';\n\nconst ai = genkit({\n plugins: [\n vertexAI(),\n ...\n myCustomEvals({\n judge: googleAI.model(\"gemini-2.5-flash\"),\n }),\n ],\n ...\n});\n```\n\n## Using your custom evaluators\n\nOnce you instantiate your custom evaluators within the Genkit app context\n(either through a plugin or directly), they are ready to be used. The following\nexample illustrates how to try out the deliciousness evaluator with a few sample\ninputs and outputs.\n\n1. Create a json file `deliciousness_dataset.json` with the following content:\n\n```json\n[\n {\n \"testCaseId\": \"delicous_mango\",\n \"input\": \"What is a super delicious fruit\",\n \"output\": \"A perfectly ripe mango – sweet, juicy, and with a hint of tropical sunshine.\"\n },\n {\n \"testCaseId\": \"disgusting_soggy_cereal\",\n \"input\": \"What is something that is tasty when fresh but less tasty after some time?\",\n \"output\": \"Stale, flavorless cereal that's been sitting in the box too long.\"\n }\n]\n```\n\n2. Use the Genkit CLI to run the evaluator against these test cases.\n\n```bash\n# Start your genkit runtime\ngenkit start -- \n\ngenkit eval:run deliciousness_dataset.json --evaluators=myCustomEvals/deliciousnessEvaluator\n```\n\n3. Navigate to `localhost:4000/evaluate` to view your results in the Genkit UI.\n\nIt is important to note that confidence in custom evaluators increases as\nyou benchmark them with standard datasets or approaches. Iterate on the\nresults of such benchmarks to improve your evaluators' performance until it\nreaches the targeted level of quality.\n", + "text": "# Writing a Genkit Evaluator\n\nYou can extend Genkit to support custom evaluation, using either\nan LLM as a judge, or by programmatic (heuristic) evaluation.\n\n## Evaluator definition\n\nEvaluators are functions that assess an LLM's response. There are two main\napproaches to automated evaluation: heuristic evaluation and LLM-based\nevaluation. In the heuristic approach, you define a deterministic function.\nBy contrast, in an LLM-based assessment, the content is fed back to an LLM,\nand the LLM is asked to score the output according to criteria set in a\nprompt.\n\nThe `ai.defineEvaluator` method, which you use to define an\nevaluator action in Genkit, supports either approach. This\ndocument explores a couple of examples of how to use this\nmethod for heuristic and LLM-based evaluations.\n\n### LLM-based Evaluators\n\nAn LLM-based evaluator leverages an LLM to evaluate\nthe `input`, `context`, and `output` of your generative AI\nfeature.\n\nLLM-based evaluators in Genkit are made up of 3 components:\n\n- A prompt\n- A scoring function\n- An evaluator action\n\n#### Define the prompt\n\nFor this example, the evaluator leverages an LLM to determine whether a\nfood (the `output`) is delicious or not. First, provide context to the LLM,\nthen describe what you want it to do, and finally, give it a few examples\nto base its response on.\n\nGenkit’s `definePrompt` utility provides an easy way to define prompts with\ninput and output validation. The following code is an example of\nsetting up an evaluation prompt with `definePrompt`.\n\n```ts\nimport { z } from \"genkit\";\n\nconst DELICIOUSNESS_VALUES = ['yes', 'no', 'maybe'] as const;\n\nconst DeliciousnessDetectionResponseSchema = z.object({\n reason: z.string(),\n verdict: z.enum(DELICIOUSNESS_VALUES),\n});\n\nfunction getDeliciousnessPrompt(ai: Genkit) {\n return ai.definePrompt({\n name: 'deliciousnessPrompt',\n input: {\n schema: z.object({\n responseToTest: z.string(),\n }),\n },\n output: {\n schema: DeliciousnessDetectionResponseSchema,\n }\n prompt: `You are a food critic. Assess whether the provided output sounds delicious, giving only \"yes\" (delicious), \"no\" (not delicious), or \"maybe\" (undecided) as the verdict.\n\n Examples:\n Output: Chicken parm sandwich\n Response: { \"reason\": \"A classic and beloved dish.\", \"verdict\": \"yes\" }\n\n Output: Boston Logan Airport tarmac\n Response: { \"reason\": \"Not edible.\", \"verdict\": \"no\" }\n\n Output: A juicy piece of gossip\n Response: { \"reason\": \"Metaphorically 'tasty' but not food.\", \"verdict\": \"maybe\" }\n\n New Output: {{ responseToTest }}\n Response:\n `\n });\n}\n```\n\n#### Define the scoring function\n\nDefine a function that takes an example that includes `output` as\nrequired by the prompt, and scores the result. Genkit testcases include\n`input` as a required field, with `output` and `context` as optional fields.\nIt is the responsibility of the evaluator to validate that all fields\nrequired for evaluation are present.\n\n```ts\nimport { ModelArgument } from 'genkit';\nimport { BaseEvalDataPoint, Score } from 'genkit/evaluator';\n\n/**\n * Score an individual test case for delciousness.\n */\nexport async function deliciousnessScore(\n ai: Genkit,\n judgeLlm: ModelArgument,\n dataPoint: BaseEvalDataPoint,\n judgeConfig?: CustomModelOptions,\n): Promise {\n const d = dataPoint;\n // Validate the input has required fields\n if (!d.output) {\n throw new Error('Output is required for Deliciousness detection');\n }\n\n // Hydrate the prompt and generate an evaluation result\n const deliciousnessPrompt = getDeliciousnessPrompt(ai);\n const response = await deliciousnessPrompt(\n {\n responseToTest: d.output as string,\n },\n {\n model: judgeLlm,\n config: judgeConfig,\n },\n );\n\n // Parse the output\n const parsedResponse = response.output;\n if (!parsedResponse) {\n throw new Error(`Unable to parse evaluator response: ${response.text}`);\n }\n\n // Return a scored response\n return {\n score: parsedResponse.verdict,\n details: { reasoning: parsedResponse.reason },\n };\n}\n```\n\n#### Define the evaluator action\n\nThe final step is to write a function that defines the `EvaluatorAction`.\n\n```ts\nimport { EvaluatorAction } from 'genkit/evaluator';\n\n/**\n * Create the Deliciousness evaluator action.\n */\nexport function createDeliciousnessEvaluator(\n ai: Genkit,\n judge: ModelArgument,\n judgeConfig?: z.infer,\n): EvaluatorAction {\n return ai.defineEvaluator(\n {\n name: `myCustomEvals/deliciousnessEvaluator`,\n displayName: 'Deliciousness',\n definition: 'Determines if output is considered delicous.',\n isBilled: true,\n },\n async (datapoint: BaseEvalDataPoint) => {\n const score = await deliciousnessScore(ai, judge, datapoint, judgeConfig);\n return {\n testCaseId: datapoint.testCaseId,\n evaluation: score,\n };\n },\n );\n}\n```\n\nThe `defineEvaluator` method is similar to other Genkit constructors like\n`defineFlow` and `defineRetriever`. This method requires an `EvaluatorFn`\nto be provided as a callback. The `EvaluatorFn` method accepts a\n`BaseEvalDataPoint` object, which corresponds to a single entry in a\ndataset under evaluation, along with an optional custom-options\nparameter if specified. The function processes the datapoint and\nreturns an `EvalResponse` object.\n\nThe Zod Schemas for `BaseEvalDataPoint` and `EvalResponse` are\nas follows.\n\n##### `BaseEvalDataPoint`\n\n```ts\nexport const BaseEvalDataPoint = z.object({\n testCaseId: z.string(),\n input: z.unknown(),\n output: z.unknown().optional(),\n context: z.array(z.unknown()).optional(),\n reference: z.unknown().optional(),\n testCaseId: z.string().optional(),\n traceIds: z.array(z.string()).optional(),\n});\n\nexport const EvalResponse = z.object({\n sampleIndex: z.number().optional(),\n testCaseId: z.string(),\n traceId: z.string().optional(),\n spanId: z.string().optional(),\n evaluation: z.union([ScoreSchema, z.array(ScoreSchema)]),\n});\n```\n\n##### `ScoreSchema`\n\n```ts\nconst ScoreSchema = z.object({\n id: z.string().describe('Optional ID to differentiate multiple scores').optional(),\n score: z.union([z.number(), z.string(), z.boolean()]).optional(),\n error: z.string().optional(),\n details: z\n .object({\n reasoning: z.string().optional(),\n })\n .passthrough()\n .optional(),\n});\n```\n\nThe `defineEvaluator` object lets the user provide a name, a user-readable\ndisplay name, and a definition for the evaluator. The display name and\ndefiniton are displayed along with evaluation results in the Dev UI.\nIt also has an optional `isBilled` field that marks whether this evaluator\ncan result in billing (e.g., it uses a billed LLM or API). If an evaluator\nis billed, the UI prompts the user for a confirmation in the CLI before\nallowing them to run an evaluation. This step helps guard against\nunintended expenses.\n\n### Heuristic Evaluators\n\nA heuristic evaluator can be any function used to evaluate the `input`, `context`,\nor `output` of your generative AI feature.\n\nHeuristic evaluators in Genkit are made up of 2 components:\n\n- A scoring function\n- An evaluator action\n\n#### Define the scoring function\n\nAs with the LLM-based evaluator, define the scoring function. In this case,\nthe scoring function does not need a judge LLM.\n\n```ts\nimport { BaseEvalDataPoint, Score } from 'genkit/evaluator';\n\nconst US_PHONE_REGEX = /[\\+]?[(]?[0-9]{3}[)]?[-\\s\\.]?[0-9]{3}[-\\s\\.]?[0-9]{4}/i;\n\n/**\n * Scores whether a datapoint output contains a US Phone number.\n */\nexport async function usPhoneRegexScore(dataPoint: BaseEvalDataPoint): Promise {\n const d = dataPoint;\n if (!d.output || typeof d.output !== 'string') {\n throw new Error('String output is required for regex matching');\n }\n const matches = US_PHONE_REGEX.test(d.output as string);\n const reasoning = matches ? `Output matched US_PHONE_REGEX` : `Output did not match US_PHONE_REGEX`;\n return {\n score: matches,\n details: { reasoning },\n };\n}\n```\n\n#### Define the evaluator action\n\n```ts\nimport { Genkit } from 'genkit';\nimport { BaseEvalDataPoint, EvaluatorAction } from 'genkit/evaluator';\n\n/**\n * Configures a regex evaluator to match a US phone number.\n */\nexport function createUSPhoneRegexEvaluator(ai: Genkit): EvaluatorAction {\n return ai.defineEvaluator(\n {\n name: `myCustomEvals/usPhoneRegexEvaluator`,\n displayName: 'Regex Match for US PHONE NUMBER',\n definition: 'Uses Regex to check if output matches a US phone number',\n isBilled: false,\n },\n async (datapoint: BaseEvalDataPoint) => {\n const score = await usPhoneRegexScore(datapoint);\n return {\n testCaseId: datapoint.testCaseId,\n evaluation: score,\n };\n },\n );\n}\n```\n\n## Putting it together\n\n### Plugin definition\n\nPlugins are registered with the framework by installing them at the time of\ninitializing Genkit. To define a new plugin, use the `genkitPlugin` helper\nmethod to instantiate all Genkit actions within the plugin context.\n\nThis code sample shows two evaluators: the LLM-based deliciousness evaluator,\nand the regex-based US phone number evaluator. Instantiating these\nevaluators within the plugin context registers them with the plugin.\n\n```ts\nimport { GenkitPlugin, genkitPlugin } from 'genkit/plugin';\n\nexport function myCustomEvals(options: {\n judge: ModelArgument;\n judgeConfig?: ModelCustomOptions;\n}): GenkitPlugin {\n // Define the new plugin\n return genkitPlugin('myCustomEvals', async (ai: Genkit) => {\n const { judge, judgeConfig } = options;\n\n // The plugin instatiates our custom evaluators within the context\n // of the `ai` object, making them available\n // throughout our Genkit application.\n createDeliciousnessEvaluator(ai, judge, judgeConfig);\n createUSPhoneRegexEvaluator(ai);\n });\n}\nexport default myCustomEvals;\n```\n\n### Configure Genkit\n\nAdd the `myCustomEvals` plugin to your Genkit configuration.\n\nFor evaluation with Gemini, disable safety settings so that the evaluator can\naccept, detect, and score potentially harmful content.\n\n```ts\nimport { googleAI } from '@genkit-ai/googleai';\n\nconst ai = genkit({\n plugins: [\n vertexAI(),\n ...\n myCustomEvals({\n judge: googleAI.model(\"gemini-2.5-flash\"),\n }),\n ],\n ...\n});\n```\n\n## Using your custom evaluators\n\nOnce you instantiate your custom evaluators within the Genkit app context\n(either through a plugin or directly), they are ready to be used. The following\nexample illustrates how to try out the deliciousness evaluator with a few sample\ninputs and outputs.\n\n1. Create a json file `deliciousness_dataset.json` with the following content:\n\n```json\n[\n {\n \"testCaseId\": \"delicous_mango\",\n \"input\": \"What is a super delicious fruit\",\n \"output\": \"A perfectly ripe mango – sweet, juicy, and with a hint of tropical sunshine.\"\n },\n {\n \"testCaseId\": \"disgusting_soggy_cereal\",\n \"input\": \"What is something that is tasty when fresh but less tasty after some time?\",\n \"output\": \"Stale, flavorless cereal that's been sitting in the box too long.\"\n }\n]\n```\n\n2. Use the Genkit CLI to run the evaluator against these test cases.\n\n```bash\n# Start your genkit runtime\ngenkit start -- \n\ngenkit eval:run deliciousness_dataset.json --evaluators=myCustomEvals/deliciousnessEvaluator\n```\n\n3. Navigate to `localhost:4000/evaluate` to view your results in the Genkit UI.\n\nIt is important to note that confidence in custom evaluators increases as\nyou benchmark them with standard datasets or approaches. Iterate on the\nresults of such benchmarks to improve your evaluators' performance until it\nreaches the targeted level of quality.\n", "title": "Writing a Genkit Evaluator", - "lang": "js" + "description": "Learn how to write custom Genkit evaluators for heuristic and LLM-based assessments, including defining prompts, scoring functions, and evaluator actions.", + "lang": "js", + "headers": "## Evaluator definition\n### LLM-based Evaluators\n#### Define the prompt\n#### Define the scoring function\n#### Define the evaluator action\n##### `BaseEvalDataPoint`\n##### `ScoreSchema`\n### Heuristic Evaluators\n#### Define the scoring function\n#### Define the evaluator action\n## Putting it together\n### Plugin definition\n### Configure Genkit\n## Using your custom evaluators\n# Start your genkit runtime\n" }, "js/plugin-authoring.md": { - "text": "Genkit's capabilities are designed to be extended by plugins. Genkit plugins are configurable modules\nthat can provide models, retrievers, indexers, trace stores, and more. You've already seen plugins in\naction just by using Genkit:\n\n```ts\nimport { genkit } from 'genkit';\nimport { vertexAI } from '@genkit-ai/vertexai';\n\nconst ai = genkit({\n plugins: [vertexAI({ projectId: 'my-project' })],\n});\n```\n\nThe Vertex AI plugin takes configuration (such as the user's Google Cloud\nproject ID) and registers a variety of new models, embedders, and more with the\nGenkit registry. The registry powers Genkit's local UI for running and\ninspecting models, prompts, and more as well as serves as a lookup service for\nnamed actions at runtime.\n\n## Creating a Plugin\n\nTo create a plugin you'll generally want to create a new NPM package:\n\n```bash\nmkdir genkitx-my-plugin\n\ncd genkitx-my-plugin\n\nnpm init -y\n\nnpm install genkit\n\nnpm install --save-dev typescript\n\nnpx tsc --init\n```\n\nThen, define and export your plugin from your main entry point using the\n`genkitPlugin` helper:\n\n```ts\nimport { Genkit, z, modelActionMetadata } from 'genkit';\nimport { GenkitPlugin, genkitPlugin } from 'genkit/plugin';\nimport { ActionMetadata, ActionType } from 'genkit/registry';\n\ninterface MyPluginOptions {\n // add any plugin configuration here\n}\n\nexport function myPlugin(options?: MyPluginOptions): GenkitPlugin {\n return genkitPlugin(\n 'myPlugin',\n // Initializer function (required): Registers actions defined upfront.\n async (ai: Genkit) => {\n // Example: Define a model that's always available\n ai.defineModel({ name: 'myPlugin/always-available-model', ... });\n ai.defineEmbedder(/* ... */);\n // ... other upfront definitions\n },\n // Dynamic Action Resolver (optional): Defines actions on-demand.\n async (ai: Genkit, actionType: ActionType, actionName: string) => {\n // Called when an action (e.g., 'myPlugin/some-dynamic-model') is\n // requested but not found in the registry.\n if (actionType === 'model' && actionName === 'some-dynamic-model') {\n ai.defineModel({ name: `myPlugin/${actionName}`, ... });\n }\n // ... handle other dynamic actions\n },\n // List Actions function (optional): Lists all potential actions.\n async (): Promise => {\n // Returns metadata for all actions the plugin *could* provide,\n // even if not yet defined dynamically. Used by Dev UI, etc.\n // Example: Fetch available models from an API\n const availableModels = await fetchMyModelsFromApi();\n return availableModels.map(model => modelActionMetadata({\n type: 'model',\n name: `myPlugin/${model.id}`,\n // ... other metadata\n }));\n }\n );\n}\n```\n\nThe `genkitPlugin` function accepts up to three arguments:\n\n1. **Plugin Name (string, required):** A unique identifier for your plugin (e.g., `'myPlugin'`).\n2. **Initializer Function (`async (ai: Genkit) => void`, required):** This function runs when Genkit starts. Use it to register actions (models, embedders, etc.) that should always be available using `ai.defineModel()`, `ai.defineEmbedder()`, etc.\n3. **Dynamic Action Resolver (`async (ai: Genkit, actionType: ActionType, actionName: string) => void`, optional):** This function is called when Genkit tries to access an action (by type and name) that hasn't been registered yet. It lets you define actions dynamically, just-in-time. For example, if a user requests `model: 'myPlugin/some-model'`, and it wasn't defined in the initializer, this function runs, giving you a chance to define it using `ai.defineModel()`. This is useful when a plugin supports many possible actions (like numerous models) and you don't want to register them all at startup.\n4. **List Actions Function (`async () => Promise`, optional):** This function should return metadata for _all_ actions your plugin can potentially provide, including those that would be dynamically defined. This is primarily used by development tools like the Genkit Developer UI to populate lists of available models, embedders, etc., allowing users to discover and select them even if they haven't been explicitly defined yet. This function is generally _not_ called during normal flow execution.\n\n### Plugin options guidance\n\nIn general, your plugin should take a single `options` argument that includes\nany plugin-wide configuration necessary to function. For any plugin option that\nrequires a secret value, such as API keys, you should offer both an option and a\ndefault environment variable to configure it:\n\n```ts\nimport { GenkitError, Genkit, z } from 'genkit';\nimport { GenkitPlugin, genkitPlugin } from 'genkit/plugin';\n\ninterface MyPluginOptions {\n apiKey?: string;\n}\n\nexport function myPlugin(options?: MyPluginOptions) {\n return genkitPlugin('myPlugin', async (ai: Genkit) => {\n if (!apiKey)\n throw new GenkitError({\n source: 'my-plugin',\n status: 'INVALID_ARGUMENT',\n message:\n 'Must supply either `options.apiKey` or set `MY_PLUGIN_API_KEY` environment variable.',\n });\n\n ai.defineModel(...);\n ai.defineEmbedder(...)\n\n // ....\n });\n};\n```\n\n## Building your plugin\n\nA single plugin can activate many new things within Genkit. For example, the Vertex AI plugin activates several new models as well as an embedder.\n\n### Model plugins\n\nGenkit model plugins add one or more generative AI models to the Genkit registry. A model represents any generative\nmodel that is capable of receiving a prompt as input and generating text, media, or data as output.\nGenerally, a model plugin will make one or more `defineModel` calls in its initialization function.\n\nA custom model generally consists of three components:\n\n1. Metadata defining the model's capabilities.\n2. A configuration schema with any specific parameters supported by the model.\n3. A function that implements the model accepting `GenerateRequest` and\n returning `GenerateResponse`.\n\nTo build a model plugin, you'll need to use the `genkit/model` package:\n\nAt a high level, a model plugin might look something like this:\n\n```ts\nimport { genkitPlugin, GenkitPlugin } from 'genkit/plugin';\nimport { GenerationCommonConfigSchema } from 'genkit/model';\nimport { simulateSystemPrompt } from 'genkit/model/middleware';\nimport { Genkit, GenkitError, z } from 'genkit';\n\nexport interface MyPluginOptions {\n // ...\n}\n\nexport function myPlugin(options?: MyPluginOptions): GenkitPlugin {\n return genkitPlugin('my-plugin', async (ai: Genkit) => {\n ai.defineModel({\n // be sure to include your plugin as a provider prefix\n name: 'my-plugin/my-model',\n // label for your model as shown in Genkit Developer UI\n label: 'My Awesome Model',\n // optional list of supported versions of your model\n versions: ['my-model-001', 'my-model-001'],\n // model support attributes\n supports: {\n multiturn: true, // true if your model supports conversations\n media: true, // true if your model supports multimodal input\n tools: true, // true if your model supports tool/function calling\n systemRole: true, // true if your model supports the system role\n output: ['text', 'media', 'json'], // types of output your model supports\n },\n // Zod schema for your model's custom configuration\n configSchema: GenerationCommonConfigSchema.extend({\n safetySettings: z.object({...}),\n }),\n // list of middleware for your model to use\n use: [simulateSystemPrompt()]\n }, async request => {\n const myModelRequest = toMyModelRequest(request);\n const myModelResponse = await myModelApi(myModelRequest);\n return toGenerateResponse(myModelResponse);\n });\n });\n};\n```\n\n#### Transforming Requests and Responses\n\nThe primary work of a Genkit model plugin is transforming the\n`GenerateRequest` from Genkit's common format into a format that is recognized\nand supported by your model's API, and then transforming the response from your\nmodel into the `GenerateResponseData` format used by Genkit.\n\nSometimes, this may require massaging or manipulating data to work around model limitations. For example, if your model does not natively support a `system` message, you may need to transform a prompt's system message into a user/model message pair.\n\n#### Action References (Models, Embedders, etc.)\n\nWhile actions like models and embedders can always be referenced by their string name (e.g., `'myPlugin/my-model'`) after being defined (either upfront or dynamically), providing strongly-typed references offers better developer experience through improved type checking and IDE autocompletion.\n\nThe recommended pattern is to attach helper methods directly to your exported plugin function. These methods use reference builders like `modelRef` and `embedderRef` from Genkit core.\n\nFirst, define the type for your plugin function including the helper methods:\n\n```ts\nimport { GenkitPlugin } from 'genkit/plugin';\nimport { ModelReference, EmbedderReference, modelRef, embedderRef, z } from 'genkit';\n\n// Define your model's specific config schema if it has one\nconst MyModelConfigSchema = z.object({\n customParam: z.string().optional(),\n});\n\n// Define the type for your plugin function\nexport type MyPlugin = {\n // The main plugin function signature\n (options?: MyPluginOptions): GenkitPlugin;\n\n // Helper method for creating model references\n model(\n name: string, // e.g., 'some-model-name'\n config?: z.infer,\n ): ModelReference;\n\n // Helper method for creating embedder references\n embedder(\n name: string, // e.g., 'my-embedder'\n config?: Record, // Or a specific config schema\n ): EmbedderReference;\n\n // ... add helpers for other action types if needed\n};\n```\n\nThen, implement the plugin function and attach the helper methods before exporting:\n\n```ts\n// (Previous imports and MyPluginOptions interface definition)\nimport { modelRef, embedderRef } from 'genkit/model'; // Ensure modelRef/embedderRef are imported\n\nfunction myPluginFn(options?: MyPluginOptions): GenkitPlugin {\n return genkitPlugin(\n 'myPlugin',\n async (ai: Genkit) => {\n // Initializer...\n },\n async (ai, actionType, actionName) => {\n // Dynamic resolver...\n // Example: Define model if requested dynamically\n if (actionType === 'model') {\n ai.defineModel(\n {\n name: `myPlugin/${actionName}`,\n // ... other model definition properties\n configSchema: MyModelConfigSchema, // Use the defined schema\n },\n async (request) => {\n /* ... model implementation ... */\n },\n );\n }\n // Handle other dynamic actions...\n },\n async () => {\n // List actions...\n },\n );\n}\n\n// Create the final export conforming to the MyPlugin type\nexport const myPlugin = myPluginFn as MyPlugin;\n\n// Implement the helper methods\nmyPlugin.model = (\n name: string,\n config?: z.infer,\n): ModelReference => {\n return modelRef({\n name: `myPlugin/${name}`, // Automatically prefixes the name\n configSchema: MyModelConfigSchema,\n config,\n });\n};\n\nmyPlugin.embedder = (name: string, config?: Record): EmbedderReference => {\n return embedderRef({\n name: `myPlugin/${name}`,\n config,\n });\n};\n```\n\nNow, users can import your plugin and use the helper methods for type-safe action references:\n\n```ts\nimport { genkit } from 'genkit';\nimport { myPlugin } from 'genkitx-my-plugin'; // Assuming your package name\n\nconst ai = genkit({\n plugins: [\n myPlugin({\n /* options */\n }),\n ],\n});\n\nasync function run() {\n const { text } = await ai.generate({\n // Use the helper for a type-safe model reference\n model: myPlugin.model('some-model-name', { customParam: 'value' }),\n prompt: 'Tell me a story.',\n });\n console.log(text);\n\n const embeddings = await ai.embed({\n // Use the helper for a type-safe embedder reference\n embedder: myPlugin.embedder('my-embedder'),\n content: 'Embed this text.',\n });\n console.log(embeddings);\n}\n\nrun();\n```\n\nThis approach keeps the plugin definition clean while providing a convenient and type-safe way for users to reference the actions provided by your plugin. It works seamlessly with both statically and dynamically defined actions, as the references only contain metadata, not the implementation itself.\n\n## Publishing a plugin\n\nGenkit plugins can be published as normal NPM packages. To increase\ndiscoverability and maximize consistency, your package should be named\n`genkitx-{name}` to indicate it is a Genkit plugin and you should include as\nmany of the following `keywords` in your `package.json` as are relevant to your\nplugin:\n\n- `genkit-plugin`: always include this keyword in your package to indicate it is a Genkit plugin.\n- `genkit-model`: include this keyword if your package defines any models.\n- `genkit-retriever`: include this keyword if your package defines any retrievers.\n- `genkit-indexer`: include this keyword if your package defines any indexers.\n- `genkit-embedder`: include this keyword if your package defines any indexers.\n- `genkit-telemetry`: include this keyword if your package defines a telemetry provider.\n- `genkit-deploy`: include this keyword if your package includes helpers to deploy Genkit apps to cloud providers.\n- `genkit-flow`: include this keyword if your package enhances Genkit flows.\n\nA plugin that provided a retriever, embedder, and model might have a `package.json` that looks like:\n\n```js\n{\n \"name\": \"genkitx-my-plugin\",\n \"keywords\": [\"genkit-plugin\", \"genkit-retriever\", \"genkit-embedder\", \"genkit-model\"],\n // ... dependencies etc.\n}\n```\n", + "text": "# Writing Genkit plugins\n\nGenkit's capabilities are designed to be extended by plugins. Genkit plugins are configurable modules\nthat can provide models, retrievers, indexers, trace stores, and more. You've already seen plugins in\naction just by using Genkit:\n\n```ts\nimport { genkit } from 'genkit';\nimport { vertexAI } from '@genkit-ai/vertexai';\n\nconst ai = genkit({\n plugins: [vertexAI({ projectId: 'my-project' })],\n});\n```\n\nThe Vertex AI plugin takes configuration (such as the user's Google Cloud\nproject ID) and registers a variety of new models, embedders, and more with the\nGenkit registry. The registry powers Genkit's local UI for running and\ninspecting models, prompts, and more as well as serves as a lookup service for\nnamed actions at runtime.\n\n## Creating a Plugin\n\nTo create a plugin you'll generally want to create a new NPM package:\n\n```bash\nmkdir genkitx-my-plugin\n\ncd genkitx-my-plugin\n\nnpm init -y\n\nnpm install genkit\n\nnpm install --save-dev typescript\n\nnpx tsc --init\n```\n\nThen, define and export your plugin from your main entry point using the\n`genkitPlugin` helper:\n\n```ts\nimport { Genkit, z, modelActionMetadata } from 'genkit';\nimport { GenkitPlugin, genkitPlugin } from 'genkit/plugin';\nimport { ActionMetadata, ActionType } from 'genkit/registry';\n\ninterface MyPluginOptions {\n // add any plugin configuration here\n}\n\nexport function myPlugin(options?: MyPluginOptions): GenkitPlugin {\n return genkitPlugin(\n 'myPlugin',\n // Initializer function (required): Registers actions defined upfront.\n async (ai: Genkit) => {\n // Example: Define a model that's always available\n ai.defineModel({ name: 'myPlugin/always-available-model', ... });\n ai.defineEmbedder(/* ... */);\n // ... other upfront definitions\n },\n // Dynamic Action Resolver (optional): Defines actions on-demand.\n async (ai: Genkit, actionType: ActionType, actionName: string) => {\n // Called when an action (e.g., 'myPlugin/some-dynamic-model') is\n // requested but not found in the registry.\n if (actionType === 'model' && actionName === 'some-dynamic-model') {\n ai.defineModel({ name: `myPlugin/${actionName}`, ... });\n }\n // ... handle other dynamic actions\n },\n // List Actions function (optional): Lists all potential actions.\n async (): Promise => {\n // Returns metadata for all actions the plugin *could* provide,\n // even if not yet defined dynamically. Used by Dev UI, etc.\n // Example: Fetch available models from an API\n const availableModels = await fetchMyModelsFromApi();\n return availableModels.map(model => modelActionMetadata({\n type: 'model',\n name: `myPlugin/${model.id}`,\n // ... other metadata\n }));\n }\n );\n}\n```\n\nThe `genkitPlugin` function accepts up to three arguments:\n\n1. **Plugin Name (string, required):** A unique identifier for your plugin (e.g., `'myPlugin'`).\n2. **Initializer Function (`async (ai: Genkit) => void`, required):** This function runs when Genkit starts. Use it to register actions (models, embedders, etc.) that should always be available using `ai.defineModel()`, `ai.defineEmbedder()`, etc.\n3. **Dynamic Action Resolver (`async (ai: Genkit, actionType: ActionType, actionName: string) => void`, optional):** This function is called when Genkit tries to access an action (by type and name) that hasn't been registered yet. It lets you define actions dynamically, just-in-time. For example, if a user requests `model: 'myPlugin/some-model'`, and it wasn't defined in the initializer, this function runs, giving you a chance to define it using `ai.defineModel()`. This is useful when a plugin supports many possible actions (like numerous models) and you don't want to register them all at startup.\n4. **List Actions Function (`async () => Promise`, optional):** This function should return metadata for _all_ actions your plugin can potentially provide, including those that would be dynamically defined. This is primarily used by development tools like the Genkit Developer UI to populate lists of available models, embedders, etc., allowing users to discover and select them even if they haven't been explicitly defined yet. This function is generally _not_ called during normal flow execution.\n\n### Plugin options guidance\n\nIn general, your plugin should take a single `options` argument that includes\nany plugin-wide configuration necessary to function. For any plugin option that\nrequires a secret value, such as API keys, you should offer both an option and a\ndefault environment variable to configure it:\n\n```ts\nimport { GenkitError, Genkit, z } from 'genkit';\nimport { GenkitPlugin, genkitPlugin } from 'genkit/plugin';\n\ninterface MyPluginOptions {\n apiKey?: string;\n}\n\nexport function myPlugin(options?: MyPluginOptions) {\n return genkitPlugin('myPlugin', async (ai: Genkit) => {\n if (!apiKey)\n throw new GenkitError({\n source: 'my-plugin',\n status: 'INVALID_ARGUMENT',\n message:\n 'Must supply either `options.apiKey` or set `MY_PLUGIN_API_KEY` environment variable.',\n });\n\n ai.defineModel(...);\n ai.defineEmbedder(...)\n\n // ....\n });\n};\n```\n\n## Building your plugin\n\nA single plugin can activate many new things within Genkit. For example, the Vertex AI plugin activates several new models as well as an embedder.\n\n### Model plugins\n\nGenkit model plugins add one or more generative AI models to the Genkit registry. A model represents any generative\nmodel that is capable of receiving a prompt as input and generating text, media, or data as output.\nGenerally, a model plugin will make one or more `defineModel` calls in its initialization function.\n\nA custom model generally consists of three components:\n\n1. Metadata defining the model's capabilities.\n2. A configuration schema with any specific parameters supported by the model.\n3. A function that implements the model accepting `GenerateRequest` and\n returning `GenerateResponse`.\n\nTo build a model plugin, you'll need to use the `genkit/model` package:\n\nAt a high level, a model plugin might look something like this:\n\n```ts\nimport { genkitPlugin, GenkitPlugin } from 'genkit/plugin';\nimport { GenerationCommonConfigSchema } from 'genkit/model';\nimport { simulateSystemPrompt } from 'genkit/model/middleware';\nimport { Genkit, GenkitError, z } from 'genkit';\n\nexport interface MyPluginOptions {\n // ...\n}\n\nexport function myPlugin(options?: MyPluginOptions): GenkitPlugin {\n return genkitPlugin('my-plugin', async (ai: Genkit) => {\n ai.defineModel({\n // be sure to include your plugin as a provider prefix\n name: 'my-plugin/my-model',\n // label for your model as shown in Genkit Developer UI\n label: 'My Awesome Model',\n // optional list of supported versions of your model\n versions: ['my-model-001', 'my-model-001'],\n // model support attributes\n supports: {\n multiturn: true, // true if your model supports conversations\n media: true, // true if your model supports multimodal input\n tools: true, // true if your model supports tool/function calling\n systemRole: true, // true if your model supports the system role\n output: ['text', 'media', 'json'], // types of output your model supports\n },\n // Zod schema for your model's custom configuration\n configSchema: GenerationCommonConfigSchema.extend({\n safetySettings: z.object({...}),\n }),\n // list of middleware for your model to use\n use: [simulateSystemPrompt()]\n }, async request => {\n const myModelRequest = toMyModelRequest(request);\n const myModelResponse = await myModelApi(myModelRequest);\n return toGenerateResponse(myModelResponse);\n });\n });\n};\n```\n\n#### Transforming Requests and Responses\n\nThe primary work of a Genkit model plugin is transforming the\n`GenerateRequest` from Genkit's common format into a format that is recognized\nand supported by your model's API, and then transforming the response from your\nmodel into the `GenerateResponseData` format used by Genkit.\n\nSometimes, this may require massaging or manipulating data to work around model limitations. For example, if your model does not natively support a `system` message, you may need to transform a prompt's system message into a user/model message pair.\n\n#### Action References (Models, Embedders, etc.)\n\nWhile actions like models and embedders can always be referenced by their string name (e.g., `'myPlugin/my-model'`) after being defined (either upfront or dynamically), providing strongly-typed references offers better developer experience through improved type checking and IDE autocompletion.\n\nThe recommended pattern is to attach helper methods directly to your exported plugin function. These methods use reference builders like `modelRef` and `embedderRef` from Genkit core.\n\nFirst, define the type for your plugin function including the helper methods:\n\n```ts\nimport { GenkitPlugin } from 'genkit/plugin';\nimport { ModelReference, EmbedderReference, modelRef, embedderRef, z } from 'genkit';\n\n// Define your model's specific config schema if it has one\nconst MyModelConfigSchema = z.object({\n customParam: z.string().optional(),\n});\n\n// Define the type for your plugin function\nexport type MyPlugin = {\n // The main plugin function signature\n (options?: MyPluginOptions): GenkitPlugin;\n\n // Helper method for creating model references\n model(\n name: string, // e.g., 'some-model-name'\n config?: z.infer,\n ): ModelReference;\n\n // Helper method for creating embedder references\n embedder(\n name: string, // e.g., 'my-embedder'\n config?: Record, // Or a specific config schema\n ): EmbedderReference;\n\n // ... add helpers for other action types if needed\n};\n```\n\nThen, implement the plugin function and attach the helper methods before exporting:\n\n```ts\n// (Previous imports and MyPluginOptions interface definition)\nimport { modelRef, embedderRef } from 'genkit/model'; // Ensure modelRef/embedderRef are imported\n\nfunction myPluginFn(options?: MyPluginOptions): GenkitPlugin {\n return genkitPlugin(\n 'myPlugin',\n async (ai: Genkit) => {\n // Initializer...\n },\n async (ai, actionType, actionName) => {\n // Dynamic resolver...\n // Example: Define model if requested dynamically\n if (actionType === 'model') {\n ai.defineModel(\n {\n name: `myPlugin/${actionName}`,\n // ... other model definition properties\n configSchema: MyModelConfigSchema, // Use the defined schema\n },\n async (request) => {\n /* ... model implementation ... */\n },\n );\n }\n // Handle other dynamic actions...\n },\n async () => {\n // List actions...\n },\n );\n}\n\n// Create the final export conforming to the MyPlugin type\nexport const myPlugin = myPluginFn as MyPlugin;\n\n// Implement the helper methods\nmyPlugin.model = (\n name: string,\n config?: z.infer,\n): ModelReference => {\n return modelRef({\n name: `myPlugin/${name}`, // Automatically prefixes the name\n configSchema: MyModelConfigSchema,\n config,\n });\n};\n\nmyPlugin.embedder = (name: string, config?: Record): EmbedderReference => {\n return embedderRef({\n name: `myPlugin/${name}`,\n config,\n });\n};\n```\n\nNow, users can import your plugin and use the helper methods for type-safe action references:\n\n```ts\nimport { genkit } from 'genkit';\nimport { myPlugin } from 'genkitx-my-plugin'; // Assuming your package name\n\nconst ai = genkit({\n plugins: [\n myPlugin({\n /* options */\n }),\n ],\n});\n\nasync function run() {\n const { text } = await ai.generate({\n // Use the helper for a type-safe model reference\n model: myPlugin.model('some-model-name', { customParam: 'value' }),\n prompt: 'Tell me a story.',\n });\n console.log(text);\n\n const embeddings = await ai.embed({\n // Use the helper for a type-safe embedder reference\n embedder: myPlugin.embedder('my-embedder'),\n content: 'Embed this text.',\n });\n console.log(embeddings);\n}\n\nrun();\n```\n\nThis approach keeps the plugin definition clean while providing a convenient and type-safe way for users to reference the actions provided by your plugin. It works seamlessly with both statically and dynamically defined actions, as the references only contain metadata, not the implementation itself.\n\n## Publishing a plugin\n\nGenkit plugins can be published as normal NPM packages. To increase\ndiscoverability and maximize consistency, your package should be named\n`genkitx-{name}` to indicate it is a Genkit plugin and you should include as\nmany of the following `keywords` in your `package.json` as are relevant to your\nplugin:\n\n- `genkit-plugin`: always include this keyword in your package to indicate it is a Genkit plugin.\n- `genkit-model`: include this keyword if your package defines any models.\n- `genkit-retriever`: include this keyword if your package defines any retrievers.\n- `genkit-indexer`: include this keyword if your package defines any indexers.\n- `genkit-embedder`: include this keyword if your package defines any indexers.\n- `genkit-telemetry`: include this keyword if your package defines a telemetry provider.\n- `genkit-deploy`: include this keyword if your package includes helpers to deploy Genkit apps to cloud providers.\n- `genkit-flow`: include this keyword if your package enhances Genkit flows.\n\nA plugin that provided a retriever, embedder, and model might have a `package.json` that looks like:\n\n```js\n{\n \"name\": \"genkitx-my-plugin\",\n \"keywords\": [\"genkit-plugin\", \"genkit-retriever\", \"genkit-embedder\", \"genkit-model\"],\n // ... dependencies etc.\n}\n```\n", "title": "Writing Genkit plugins", - "lang": "js" + "description": "Learn how to extend Genkit's capabilities by writing custom plugins, covering plugin creation, options, building models, and publishing to NPM.", + "lang": "js", + "headers": "## Creating a Plugin\n### Plugin options guidance\n## Building your plugin\n### Model plugins\n#### Transforming Requests and Responses\n#### Action References (Models, Embedders, etc.)\n## Publishing a plugin\n" }, "js/rag.md": { - "text": "Genkit provides abstractions that help you build retrieval-augmented\ngeneration (RAG) flows, as well as plugins that provide integrations with\nrelated tools.\n\n## What is RAG?\n\nRetrieval-augmented generation is a technique used to incorporate external\nsources of information into an LLM’s responses. It's important to be able to do\nso because, while LLMs are typically trained on a broad body of material,\npractical use of LLMs often requires specific domain knowledge (for example, you\nmight want to use an LLM to answer customers' questions about your company’s\nproducts).\n\nOne solution is to fine-tune the model using more specific data. However, this\ncan be expensive both in terms of compute cost and in terms of the effort needed\nto prepare adequate training data.\n\nIn contrast, RAG works by incorporating external data sources into a prompt at\nthe time it's passed to the model. For example, you could imagine the prompt,\n\"What is Bart's relationship to Lisa?\" might be expanded (\"augmented\") by\nprepending some relevant information, resulting in the prompt, \"Homer and\nMarge's children are named Bart, Lisa, and Maggie. What is Bart's relationship\nto Lisa?\"\n\nThis approach has several advantages:\n\n- It can be more cost-effective because you don't have to retrain the model.\n- You can continuously update your data source and the LLM can immediately\n make use of the updated information.\n- You now have the potential to cite references in your LLM's responses.\n\nOn the other hand, using RAG naturally means longer prompts, and some LLM API\nservices charge for each input token you send. Ultimately, you must evaluate the\ncost tradeoffs for your applications.\n\nRAG is a very broad area and there are many different techniques used to achieve\nthe best quality RAG. The core Genkit framework offers three main abstractions\nto help you do RAG:\n\n- Indexers: add documents to an \"index\".\n- Embedders: transforms documents into a vector representation\n- Retrievers: retrieve documents from an \"index\", given a query.\n\nThese definitions are broad on purpose because Genkit is un-opinionated about\nwhat an \"index\" is or how exactly documents are retrieved from it. Genkit only\nprovides a `Document` format and everything else is defined by the retriever or\nindexer implementation provider.\n\n### Indexers\n\nThe index is responsible for keeping track of your documents in such a way that\nyou can quickly retrieve relevant documents given a specific query. This is most\noften accomplished using a vector database, which indexes your documents using\nmultidimensional vectors called embeddings. A text embedding (opaquely)\nrepresents the concepts expressed by a passage of text; these are generated\nusing special-purpose ML models. By indexing text using its embedding, a vector\ndatabase is able to cluster conceptually related text and retrieve documents\nrelated to a novel string of text (the query).\n\nBefore you can retrieve documents for the purpose of generation, you need to\ningest them into your document index. A typical ingestion flow does the\nfollowing:\n\n1. Split up large documents into smaller documents so that only relevant\n portions are used to augment your prompts – \"chunking\". This is necessary\n because many LLMs have a limited context window, making it impractical to\n include entire documents with a prompt.\n\n Genkit doesn't provide built-in chunking libraries; however, there are open\n source libraries available that are compatible with Genkit.\n\n2. Generate embeddings for each chunk. Depending on the database you're using,\n you might explicitly do this with an embedding generation model, or you might\n use the embedding generator provided by the database.\n3. Add the text chunk and its index to the database.\n\nYou might run your ingestion flow infrequently or only once if you are working\nwith a stable source of data. On the other hand, if you are working with data\nthat frequently changes, you might continuously run the ingestion flow (for\nexample, in a Cloud Firestore trigger, whenever a document is updated).\n\n### Embedders\n\nAn embedder is a function that takes content (text, images, audio, etc.) and\ncreates a numeric vector that encodes the semantic meaning of the original\ncontent. As mentioned above, embedders are leveraged as part of the process of\nindexing, however, they can also be used independently to create embeddings\nwithout an index.\n\n### Retrievers\n\nA retriever is a concept that encapsulates logic related to any kind of document\nretrieval. The most popular retrieval cases typically include retrieval from\nvector stores, however, in Genkit a retriever can be any function that returns\ndata.\n\nTo create a retriever, you can use one of the provided implementations or create\nyour own.\n\n## Supported indexers, retrievers, and embedders\n\nGenkit provides indexer and retriever support through its plugin system. The\nfollowing plugins are officially supported:\n\n- [Cloud Firestore vector store](/docs/plugins/firebase)\n- [Vertex AI Vector Search](/docs/plugins/vertex-ai)\n- [Chroma DB](/docs/plugins/chroma) vector database\n- [Pinecone](/docs/plugins/pinecone) cloud vector database\n\nIn addition, Genkit supports the following vector stores through predefined code\ntemplates, which you can customize for your database configuration and schema:\n\n- PostgreSQL with [`pgvector`](/docs/templates/pgvector)\n\nEmbedding model support is provided through the following plugins:\n\n| Plugin | Models |\n| ------------------------- | -------------------- |\n| [Google Generative AI][1] | Gecko text embedding |\n| [Google Vertex AI][2] | Gecko text embedding |\n\n[1]: /docs/plugins/google-genai\n[2]: /docs/plugins/vertex-ai\n\n## Defining a RAG Flow\n\nThe following examples show how you could ingest a collection of restaurant menu\nPDF documents into a vector database and retrieve them for use in a flow that\ndetermines what food items are available.\n\n### Install dependencies for processing PDFs\n\n```bash\nnpm install llm-chunk pdf-parse @genkit-ai/dev-local-vectorstore\n\nnpm install --save-dev @types/pdf-parse\n```\n\n### Add a local vector store to your configuration\n\n```ts\nimport { devLocalIndexerRef, devLocalVectorstore } from '@genkit-ai/dev-local-vectorstore';\nimport { vertexAI } from '@genkit-ai/vertexai';\nimport { z, genkit } from 'genkit';\n\nconst ai = genkit({\n plugins: [\n // vertexAI provides the textEmbedding004 embedder\n vertexAI(),\n\n // the local vector store requires an embedder to translate from text to vector\n devLocalVectorstore([\n {\n indexName: 'menuQA',\n embedder: vertexAI.embedder('text-embedding-005'),\n },\n ]),\n ],\n});\n```\n\n### Define an Indexer\n\nThe following example shows how to create an indexer to ingest a collection of\nPDF documents and store them in a local vector database.\n\nIt uses the local file-based vector similarity retriever that Genkit provides\nout-of-the-box for simple testing and prototyping (_do not use in production_)\n\n#### Create the indexer\n\n```ts\nexport const menuPdfIndexer = devLocalIndexerRef('menuQA');\n```\n\n#### Create chunking config\n\nThis example uses the `llm-chunk` library which provides a simple text splitter\nto break up documents into segments that can be vectorized.\n\nThe following definition configures the chunking function to guarantee a\ndocument segment of between 1000 and 2000 characters, broken at the end of a\nsentence, with an overlap between chunks of 100 characters.\n\n```ts\nconst chunkingConfig = {\n minLength: 1000,\n maxLength: 2000,\n splitter: 'sentence',\n overlap: 100,\n delimiters: '',\n} as any;\n```\n\nMore chunking options for this library can be found in the [llm-chunk\ndocumentation](https://www.npmjs.com/package/llm-chunk).\n\n#### Define your indexer flow\n\n```ts\nimport { Document } from 'genkit/retriever';\nimport { chunk } from 'llm-chunk';\nimport { readFile } from 'fs/promises';\nimport path from 'path';\nimport pdf from 'pdf-parse';\n\nasync function extractTextFromPdf(filePath: string) {\n const pdfFile = path.resolve(filePath);\n const dataBuffer = await readFile(pdfFile);\n const data = await pdf(dataBuffer);\n return data.text;\n}\n\nexport const indexMenu = ai.defineFlow(\n {\n name: 'indexMenu',\n inputSchema: z.object({ filePath: z.string().describe('PDF file path') }),\n outputSchema: z.object({\n success: z.boolean(),\n documentsIndexed: z.number(),\n error: z.string().optional(),\n }),\n },\n async ({ filePath }) => {\n try {\n filePath = path.resolve(filePath);\n\n // Read the pdf\n const pdfTxt = await ai.run('extract-text', () => extractTextFromPdf(filePath));\n\n // Divide the pdf text into segments\n const chunks = await ai.run('chunk-it', async () => chunk(pdfTxt, chunkingConfig));\n\n // Convert chunks of text into documents to store in the index.\n const documents = chunks.map((text) => {\n return Document.fromText(text, { filePath });\n });\n\n // Add documents to the index\n await ai.index({\n indexer: menuPdfIndexer,\n documents,\n });\n\n return {\n success: true,\n documentsIndexed: documents.length,\n };\n } catch (err) {\n // For unexpected errors that throw exceptions\n return {\n success: false,\n documentsIndexed: 0,\n error: err instanceof Error ? err.message : String(err)\n };\n }\n },\n);\n```\n\n#### Run the indexer flow\n\n```bash\ngenkit flow:run indexMenu '{\"filePath\": \"menu.pdf\"}'\n```\n\nAfter running the `indexMenu` flow, the vector database will be seeded with\ndocuments and ready to be used in Genkit flows with retrieval steps.\n\n### Define a flow with retrieval\n\nThe following example shows how you might use a retriever in a RAG flow. Like\nthe indexer example, this example uses Genkit's file-based vector retriever,\nwhich you should not use in production.\n\n```ts\nimport { devLocalRetrieverRef } from '@genkit-ai/dev-local-vectorstore';\nimport { vertexAI } from '@genkit-ai/vertexai';\n\n// Define the retriever reference\nexport const menuRetriever = devLocalRetrieverRef('menuQA');\n\nexport const menuQAFlow = ai.defineFlow(\n { \n name: 'menuQA', \n inputSchema: z.object({ query: z.string() }), \n outputSchema: z.object({ answer: z.string() }) \n },\n async ({ query }) => {\n // retrieve relevant documents\n const docs = await ai.retrieve({\n retriever: menuRetriever,\n query,\n options: { k: 3 },\n });\n\n // generate a response\n const { text } = await ai.generate({\n model: vertexAI.model('gemini-2.5-flash'),\n prompt: `\nYou are acting as a helpful AI assistant that can answer \nquestions about the food available on the menu at Genkit Grub Pub.\n\nUse only the context provided to answer the question.\nIf you don't know, do not make up an answer.\nDo not add or change items on the menu.\n\nQuestion: ${query}`,\n docs,\n });\n\n return { answer: text };\n },\n);\n```\n\n#### Run the retriever flow\n\n```bash\ngenkit flow:run menuQA '{\"query\": \"Recommend a dessert from the menu while avoiding dairy and nuts\"}'\n```\n\nThe output for this command should contain a response from the model, grounded\nin the indexed `menu.pdf` file.\n\n## Write your own indexers and retrievers\n\nIt's also possible to create your own retriever. This is useful if your\ndocuments are managed in a document store that is not supported in Genkit (eg:\nMySQL, Google Drive, etc.). The Genkit SDK provides flexible methods that let\nyou provide custom code for fetching documents. You can also define custom\nretrievers that build on top of existing retrievers in Genkit and apply advanced\nRAG techniques (such as reranking or prompt extensions) on top.\n\n### Simple Retrievers\n\nSimple retrievers let you easily convert existing code into retrievers:\n\n```ts\nimport { z } from 'genkit';\nimport { searchEmails } from './db';\n\nai.defineSimpleRetriever(\n {\n name: 'myDatabase',\n configSchema: z\n .object({\n limit: z.number().optional(),\n })\n .optional(),\n // we'll extract \"message\" from the returned email item\n content: 'message',\n // and several keys to use as metadata\n metadata: ['from', 'to', 'subject'],\n },\n async (query, config) => {\n const result = await searchEmails(query.text, { limit: config.limit });\n return result.data.emails;\n },\n);\n```\n\n### Custom Retrievers\n\n```ts\nimport { CommonRetrieverOptionsSchema } from 'genkit/retriever';\nimport { z } from 'genkit';\n\nexport const menuRetriever = devLocalRetrieverRef('menuQA');\n\nconst advancedMenuRetrieverOptionsSchema = CommonRetrieverOptionsSchema.extend({\n preRerankK: z.number().max(1000),\n});\n\nconst advancedMenuRetriever = ai.defineRetriever(\n {\n name: `custom/advancedMenuRetriever`,\n configSchema: advancedMenuRetrieverOptionsSchema,\n },\n async (input, options) => {\n const extendedPrompt = await extendPrompt(input);\n const docs = await ai.retrieve({\n retriever: menuRetriever,\n query: extendedPrompt,\n options: { k: options.preRerankK || 10 },\n });\n const rerankedDocs = await rerank(docs);\n return rerankedDocs.slice(0, options.k || 3);\n },\n);\n```\n\n(`extendPrompt` and `rerank` is something you would have to implement yourself,\nnot provided by the framework)\n\nAnd then you can just swap out your retriever:\n\n```ts\nconst docs = await ai.retrieve({\n retriever: advancedRetriever,\n query: input,\n options: { preRerankK: 7, k: 3 },\n});\n```\n\n### Rerankers and Two-Stage Retrieval\n\nA reranking model — also known as a cross-encoder — is a type of model that,\ngiven a query and document, will output a similarity score. We use this score to\nreorder the documents by relevance to our query. Reranker APIs take a list of\ndocuments (for example the output of a retriever) and reorders the documents\nbased on their relevance to the query. This step can be useful for fine-tuning\nthe results and ensuring that the most pertinent information is used in the\nprompt provided to a generative model.\n\n#### Reranker Example\n\nA reranker in Genkit is defined in a similar syntax to retrievers and indexers.\nHere is an example using a reranker in Genkit. This flow reranks a set of\ndocuments based on their relevance to the provided query using a predefined\nVertex AI reranker.\n\n```ts\nconst FAKE_DOCUMENT_CONTENT = [\n 'pythagorean theorem',\n 'e=mc^2',\n 'pi',\n 'dinosaurs',\n 'quantum mechanics',\n 'pizza',\n 'harry potter',\n];\n\nexport const rerankFlow = ai.defineFlow(\n {\n name: 'rerankFlow',\n inputSchema: z.object({ query: z.string() }),\n outputSchema: z.array(\n z.object({\n text: z.string(),\n score: z.number(),\n }),\n ),\n },\n async ({ query }) => {\n const documents = FAKE_DOCUMENT_CONTENT.map((text) => ({ content: text }));\n\n const rerankedDocuments = await ai.rerank({\n reranker: 'vertexai/semantic-ranker-512',\n query: { content: query },\n documents,\n });\n\n return rerankedDocuments.map((doc) => ({\n text: doc.content,\n score: doc.metadata.score,\n }));\n },\n);\n```\n\nThis reranker uses the Vertex AI genkit plugin with `semantic-ranker-512` to\nscore and rank documents. The higher the score, the more relevant the document\nis to the query.\n\n#### Custom Rerankers\n\nYou can also define custom rerankers to suit your specific use case. This is\nhelpful when you need to rerank documents using your own custom logic or a\ncustom model. Here’s a simple example of defining a custom reranker:\n\n```ts\nexport const customReranker = ai.defineReranker(\n {\n name: 'custom/reranker',\n configSchema: z.object({\n k: z.number().optional(),\n }),\n },\n async (query, documents, options) => {\n // Your custom reranking logic here\n const rerankedDocs = documents.map((doc) => {\n const score = Math.random(); // Assign random scores for demonstration\n return {\n ...doc,\n metadata: { ...doc.metadata, score },\n };\n });\n\n return rerankedDocs.sort((a, b) => b.metadata.score - a.metadata.score).slice(0, options.k || 3);\n },\n);\n```\n\nOnce defined, this custom reranker can be used just like any other reranker in\nyour RAG flows, giving you flexibility to implement advanced reranking\nstrategies.\n", + "text": "# Retrieval-augmented generation (RAG)\n\nGenkit provides abstractions that help you build retrieval-augmented\ngeneration (RAG) flows, as well as plugins that provide integrations with\nrelated tools.\n\n## What is RAG?\n\nRetrieval-augmented generation is a technique used to incorporate external\nsources of information into an LLM’s responses. It's important to be able to do\nso because, while LLMs are typically trained on a broad body of material,\npractical use of LLMs often requires specific domain knowledge (for example, you\nmight want to use an LLM to answer customers' questions about your company’s\nproducts).\n\nOne solution is to fine-tune the model using more specific data. However, this\ncan be expensive both in terms of compute cost and in terms of the effort needed\nto prepare adequate training data.\n\nIn contrast, RAG works by incorporating external data sources into a prompt at\nthe time it's passed to the model. For example, you could imagine the prompt,\n\"What is Bart's relationship to Lisa?\" might be expanded (\"augmented\") by\nprepending some relevant information, resulting in the prompt, \"Homer and\nMarge's children are named Bart, Lisa, and Maggie. What is Bart's relationship\nto Lisa?\"\n\nThis approach has several advantages:\n\n- It can be more cost-effective because you don't have to retrain the model.\n- You can continuously update your data source and the LLM can immediately\n make use of the updated information.\n- You now have the potential to cite references in your LLM's responses.\n\nOn the other hand, using RAG naturally means longer prompts, and some LLM API\nservices charge for each input token you send. Ultimately, you must evaluate the\ncost tradeoffs for your applications.\n\nRAG is a very broad area and there are many different techniques used to achieve\nthe best quality RAG. The core Genkit framework offers three main abstractions\nto help you do RAG:\n\n- Indexers: add documents to an \"index\".\n- Embedders: transforms documents into a vector representation\n- Retrievers: retrieve documents from an \"index\", given a query.\n\nThese definitions are broad on purpose because Genkit is un-opinionated about\nwhat an \"index\" is or how exactly documents are retrieved from it. Genkit only\nprovides a `Document` format and everything else is defined by the retriever or\nindexer implementation provider.\n\n### Indexers\n\nThe index is responsible for keeping track of your documents in such a way that\nyou can quickly retrieve relevant documents given a specific query. This is most\noften accomplished using a vector database, which indexes your documents using\nmultidimensional vectors called embeddings. A text embedding (opaquely)\nrepresents the concepts expressed by a passage of text; these are generated\nusing special-purpose ML models. By indexing text using its embedding, a vector\ndatabase is able to cluster conceptually related text and retrieve documents\nrelated to a novel string of text (the query).\n\nBefore you can retrieve documents for the purpose of generation, you need to\ningest them into your document index. A typical ingestion flow does the\nfollowing:\n\n1. Split up large documents into smaller documents so that only relevant\n portions are used to augment your prompts – \"chunking\". This is necessary\n because many LLMs have a limited context window, making it impractical to\n include entire documents with a prompt.\n\n Genkit doesn't provide built-in chunking libraries; however, there are open\n source libraries available that are compatible with Genkit.\n\n2. Generate embeddings for each chunk. Depending on the database you're using,\n you might explicitly do this with an embedding generation model, or you might\n use the embedding generator provided by the database.\n3. Add the text chunk and its index to the database.\n\nYou might run your ingestion flow infrequently or only once if you are working\nwith a stable source of data. On the other hand, if you are working with data\nthat frequently changes, you might continuously run the ingestion flow (for\nexample, in a Cloud Firestore trigger, whenever a document is updated).\n\n### Embedders\n\nAn embedder is a function that takes content (text, images, audio, etc.) and\ncreates a numeric vector that encodes the semantic meaning of the original\ncontent. As mentioned above, embedders are leveraged as part of the process of\nindexing, however, they can also be used independently to create embeddings\nwithout an index.\n\n### Retrievers\n\nA retriever is a concept that encapsulates logic related to any kind of document\nretrieval. The most popular retrieval cases typically include retrieval from\nvector stores, however, in Genkit a retriever can be any function that returns\ndata.\n\nTo create a retriever, you can use one of the provided implementations or create\nyour own.\n\n## Supported indexers, retrievers, and embedders\n\nGenkit provides indexer and retriever support through its plugin system. The\nfollowing plugins are officially supported:\n\n- [Astra DB](/docs/plugins/astra-db) - DataStax Astra DB vector database\n- [Chroma DB](/docs/plugins/chroma) vector database\n- [Cloud Firestore vector store](/docs/plugins/firebase)\n- [Cloud SQL for PostgreSQL](/docs/plugins/cloud-sql-pg) with pgvector extension\n- [LanceDB](/docs/plugins/lancedb) open-source vector database\n- [Neo4j](/docs/plugins/neo4j) graph database with vector search\n- [Pinecone](/docs/plugins/pinecone) cloud vector database\n- [Vertex AI Vector Search](/docs/plugins/vertex-ai)\n\nIn addition, Genkit supports the following vector stores through predefined code\ntemplates, which you can customize for your database configuration and schema:\n\n- PostgreSQL with [`pgvector`](/docs/templates/pgvector)\n\n## Defining a RAG Flow\n\nThe following examples show how you could ingest a collection of restaurant menu\nPDF documents into a vector database and retrieve them for use in a flow that\ndetermines what food items are available.\n\n### Install dependencies for processing PDFs\n\n```bash\nnpm install llm-chunk pdf-parse @genkit-ai/dev-local-vectorstore\n\nnpm install --save-dev @types/pdf-parse\n```\n\n### Add a local vector store to your configuration\n\n```ts\nimport { devLocalIndexerRef, devLocalVectorstore } from '@genkit-ai/dev-local-vectorstore';\nimport { googleAI } from '@genkit-ai/googleai';\nimport { z, genkit } from 'genkit';\n\nconst ai = genkit({\n plugins: [\n // googleAI provides the gemini-embedding-001 embedder\n googleAI(),\n\n // the local vector store requires an embedder to translate from text to vector\n devLocalVectorstore([\n {\n indexName: 'menuQA',\n embedder: googleAI.embedder('gemini-embedding-001'),\n },\n ]),\n ],\n});\n```\n\n### Define an Indexer\n\nThe following example shows how to create an indexer to ingest a collection of\nPDF documents and store them in a local vector database.\n\nIt uses the local file-based vector similarity retriever that Genkit provides\nout-of-the-box for simple testing and prototyping (_do not use in production_)\n\n#### Create the indexer\n\n```ts\nexport const menuPdfIndexer = devLocalIndexerRef('menuQA');\n```\n\n#### Create chunking config\n\nThis example uses the `llm-chunk` library which provides a simple text splitter\nto break up documents into segments that can be vectorized.\n\nThe following definition configures the chunking function to guarantee a\ndocument segment of between 1000 and 2000 characters, broken at the end of a\nsentence, with an overlap between chunks of 100 characters.\n\n```ts\nconst chunkingConfig = {\n minLength: 1000,\n maxLength: 2000,\n splitter: 'sentence',\n overlap: 100,\n delimiters: '',\n} as any;\n```\n\nMore chunking options for this library can be found in the [llm-chunk\ndocumentation](https://www.npmjs.com/package/llm-chunk).\n\n#### Define your indexer flow\n\n```ts\nimport { Document } from 'genkit/retriever';\nimport { chunk } from 'llm-chunk';\nimport { readFile } from 'fs/promises';\nimport path from 'path';\nimport pdf from 'pdf-parse';\n\nasync function extractTextFromPdf(filePath: string) {\n const pdfFile = path.resolve(filePath);\n const dataBuffer = await readFile(pdfFile);\n const data = await pdf(dataBuffer);\n return data.text;\n}\n\nexport const indexMenu = ai.defineFlow(\n {\n name: 'indexMenu',\n inputSchema: z.object({ filePath: z.string().describe('PDF file path') }),\n outputSchema: z.object({\n success: z.boolean(),\n documentsIndexed: z.number(),\n error: z.string().optional(),\n }),\n },\n async ({ filePath }) => {\n try {\n filePath = path.resolve(filePath);\n\n // Read the pdf\n const pdfTxt = await ai.run('extract-text', () => extractTextFromPdf(filePath));\n\n // Divide the pdf text into segments\n const chunks = await ai.run('chunk-it', async () => chunk(pdfTxt, chunkingConfig));\n\n // Convert chunks of text into documents to store in the index.\n const documents = chunks.map((text) => {\n return Document.fromText(text, { filePath });\n });\n\n // Add documents to the index\n await ai.index({\n indexer: menuPdfIndexer,\n documents,\n });\n\n return {\n success: true,\n documentsIndexed: documents.length,\n };\n } catch (err) {\n // For unexpected errors that throw exceptions\n return {\n success: false,\n documentsIndexed: 0,\n error: err instanceof Error ? err.message : String(err)\n };\n }\n },\n);\n```\n\n#### Run the indexer flow\n\n```bash\ngenkit flow:run indexMenu '{\"filePath\": \"menu.pdf\"}'\n```\n\nAfter running the `indexMenu` flow, the vector database will be seeded with\ndocuments and ready to be used in Genkit flows with retrieval steps.\n\n### Define a flow with retrieval\n\nThe following example shows how you might use a retriever in a RAG flow. Like\nthe indexer example, this example uses Genkit's file-based vector retriever,\nwhich you should not use in production.\n\n```ts\nimport { devLocalRetrieverRef } from '@genkit-ai/dev-local-vectorstore';\nimport { googleAI } from '@genkit-ai/googleai';\n\n// Define the retriever reference\nexport const menuRetriever = devLocalRetrieverRef('menuQA');\n\nexport const menuQAFlow = ai.defineFlow(\n { \n name: 'menuQA', \n inputSchema: z.object({ query: z.string() }), \n outputSchema: z.object({ answer: z.string() }) \n },\n async ({ query }) => {\n // retrieve relevant documents\n const docs = await ai.retrieve({\n retriever: menuRetriever,\n query,\n options: { k: 3 },\n });\n\n // generate a response\n const { text } = await ai.generate({\n model: googleAI.model('gemini-2.5-flash'),\n prompt: `\nYou are acting as a helpful AI assistant that can answer \nquestions about the food available on the menu at Genkit Grub Pub.\n\nUse only the context provided to answer the question.\nIf you don't know, do not make up an answer.\nDo not add or change items on the menu.\n\nQuestion: ${query}`,\n docs,\n });\n\n return { answer: text };\n },\n);\n```\n\n#### Run the retriever flow\n\n```bash\ngenkit flow:run menuQA '{\"query\": \"Recommend a dessert from the menu while avoiding dairy and nuts\"}'\n```\n\nThe output for this command should contain a response from the model, grounded\nin the indexed `menu.pdf` file.\n\n## Write your own indexers and retrievers\n\nIt's also possible to create your own retriever. This is useful if your\ndocuments are managed in a document store that is not supported in Genkit (eg:\nMySQL, Google Drive, etc.). The Genkit SDK provides flexible methods that let\nyou provide custom code for fetching documents. You can also define custom\nretrievers that build on top of existing retrievers in Genkit and apply advanced\nRAG techniques (such as reranking or prompt extensions) on top.\n\n### Simple Retrievers\n\nSimple retrievers let you easily convert existing code into retrievers:\n\n```ts\nimport { z } from 'genkit';\nimport { searchEmails } from './db';\n\nai.defineSimpleRetriever(\n {\n name: 'myDatabase',\n configSchema: z\n .object({\n limit: z.number().optional(),\n })\n .optional(),\n // we'll extract \"message\" from the returned email item\n content: 'message',\n // and several keys to use as metadata\n metadata: ['from', 'to', 'subject'],\n },\n async (query, config) => {\n const result = await searchEmails(query.text, { limit: config.limit });\n return result.data.emails;\n },\n);\n```\n\n### Custom Retrievers\n\n```ts\nimport { CommonRetrieverOptionsSchema } from 'genkit/retriever';\nimport { z } from 'genkit';\n\nexport const menuRetriever = devLocalRetrieverRef('menuQA');\n\nconst advancedMenuRetrieverOptionsSchema = CommonRetrieverOptionsSchema.extend({\n preRerankK: z.number().max(1000),\n});\n\nconst advancedMenuRetriever = ai.defineRetriever(\n {\n name: `custom/advancedMenuRetriever`,\n configSchema: advancedMenuRetrieverOptionsSchema,\n },\n async (input, options) => {\n const extendedPrompt = await extendPrompt(input);\n const docs = await ai.retrieve({\n retriever: menuRetriever,\n query: extendedPrompt,\n options: { k: options.preRerankK || 10 },\n });\n const rerankedDocs = await rerank(docs);\n return rerankedDocs.slice(0, options.k || 3);\n },\n);\n```\n\n(`extendPrompt` and `rerank` is something you would have to implement yourself,\nnot provided by the framework)\n\nAnd then you can just swap out your retriever:\n\n```ts\nconst docs = await ai.retrieve({\n retriever: advancedRetriever,\n query: input,\n options: { preRerankK: 7, k: 3 },\n});\n```\n\n### Rerankers and Two-Stage Retrieval\n\nA reranking model — also known as a cross-encoder — is a type of model that,\ngiven a query and document, will output a similarity score. We use this score to\nreorder the documents by relevance to our query. Reranker APIs take a list of\ndocuments (for example the output of a retriever) and reorders the documents\nbased on their relevance to the query. This step can be useful for fine-tuning\nthe results and ensuring that the most pertinent information is used in the\nprompt provided to a generative model.\n\n#### Reranker Example\n\nA reranker in Genkit is defined in a similar syntax to retrievers and indexers.\nHere is an example using a reranker in Genkit. This flow reranks a set of\ndocuments based on their relevance to the provided query using a predefined\nVertex AI reranker.\n\n```ts\nconst FAKE_DOCUMENT_CONTENT = [\n 'pythagorean theorem',\n 'e=mc^2',\n 'pi',\n 'dinosaurs',\n 'quantum mechanics',\n 'pizza',\n 'harry potter',\n];\n\nexport const rerankFlow = ai.defineFlow(\n {\n name: 'rerankFlow',\n inputSchema: z.object({ query: z.string() }),\n outputSchema: z.array(\n z.object({\n text: z.string(),\n score: z.number(),\n }),\n ),\n },\n async ({ query }) => {\n const documents = FAKE_DOCUMENT_CONTENT.map((text) => ({ content: text }));\n\n const rerankedDocuments = await ai.rerank({\n reranker: 'vertexai/semantic-ranker-512',\n query: { content: query },\n documents,\n });\n\n return rerankedDocuments.map((doc) => ({\n text: doc.content,\n score: doc.metadata.score,\n }));\n },\n);\n```\n\nThis reranker uses the Vertex AI genkit plugin with `semantic-ranker-512` to\nscore and rank documents. The higher the score, the more relevant the document\nis to the query.\n\n#### Custom Rerankers\n\nYou can also define custom rerankers to suit your specific use case. This is\nhelpful when you need to rerank documents using your own custom logic or a\ncustom model. Here’s a simple example of defining a custom reranker:\n\n```ts\nexport const customReranker = ai.defineReranker(\n {\n name: 'custom/reranker',\n configSchema: z.object({\n k: z.number().optional(),\n }),\n },\n async (query, documents, options) => {\n // Your custom reranking logic here\n const rerankedDocs = documents.map((doc) => {\n const score = Math.random(); // Assign random scores for demonstration\n return {\n ...doc,\n metadata: { ...doc.metadata, score },\n };\n });\n\n return rerankedDocs.sort((a, b) => b.metadata.score - a.metadata.score).slice(0, options.k || 3);\n },\n);\n```\n\nOnce defined, this custom reranker can be used just like any other reranker in\nyour RAG flows, giving you flexibility to implement advanced reranking\nstrategies.\n", "title": "Retrieval-augmented generation (RAG)", - "lang": "js" + "description": "Learn how Genkit simplifies retrieval-augmented generation (RAG) by providing abstractions and plugins for indexers, embedders, and retrievers to incorporate external data into LLM responses.", + "lang": "js", + "headers": "## What is RAG?\n### Indexers\n### Embedders\n### Retrievers\n## Supported indexers, retrievers, and embedders\n## Defining a RAG Flow\n### Install dependencies for processing PDFs\n### Add a local vector store to your configuration\n### Define an Indexer\n#### Create the indexer\n#### Create chunking config\n#### Define your indexer flow\n#### Run the indexer flow\n### Define a flow with retrieval\n#### Run the retriever flow\n## Write your own indexers and retrievers\n### Simple Retrievers\n### Custom Retrievers\n### Rerankers and Two-Stage Retrieval\n#### Reranker Example\n#### Custom Rerankers\n" }, - "js/tool-calling.mdx": { - "text": "import ExampleLink from '@/components/ExampleLink.astro';\nimport { Tabs, TabItem } from '@astrojs/starlight/components';\n\n_Tool calling_, also known as _function calling_, is a structured way to give\nLLMs the ability to make requests back to the application that called it. You\ndefine the tools you want to make available to the model, and the model will\nmake tool requests to your app as necessary to fulfill the prompts you give it.\n\nThe use cases of tool calling generally fall into a few themes:\n\n**Giving an LLM access to information it wasn't trained with**\n\n- Frequently changing information, such as a stock price or the current\n weather.\n- Information specific to your app domain, such as product information or user\n profiles.\n\nNote the overlap with [retrieval augmented generation](/docs/rag) (RAG), which is also\na way to let an LLM integrate factual information into its generations. RAG is a\nheavier solution that is most suited when you have a large amount of information\nor the information that's most relevant to a prompt is ambiguous. On the other\nhand, if retrieving the information the LLM needs is a simple function call or\ndatabase lookup, tool calling is more appropriate.\n\n**Introducing a degree of determinism into an LLM workflow**\n\n- Performing calculations that the LLM cannot reliably complete itself.\n- Forcing an LLM to generate verbatim text under certain circumstances, such\n as when responding to a question about an app's terms of service.\n\n**Performing an action when initiated by an LLM**\n\n- Turning on and off lights in an LLM-powered home assistant\n- Reserving table reservations in an LLM-powered restaurant agent\n\n## Before you begin\n\nIf you want to run the code examples on this page, first complete the steps in\nthe [Getting started](/docs/get-started) guide. All of the examples assume that you\nhave already set up a project with Genkit dependencies installed.\n\nThis page discusses one of the advanced features of Genkit model abstraction, so\nbefore you dive too deeply, you should be familiar with the content on the\n[Generating content with AI models](/docs/models) page. You should also be familiar\nwith Genkit's system for defining input and output schemas, which is discussed\non the [Flows](/docs/flows) page.\n\n## Overview of tool calling\n\n\n\nAt a high level, this is what a typical tool-calling interaction with an LLM\nlooks like:\n\n1. The calling application prompts the LLM with a request and also includes in\n the prompt a list of tools the LLM can use to generate a response.\n2. The LLM either generates a complete response or generates a tool call request\n in a specific format.\n3. If the caller receives a complete response, the request is fulfilled and the\n interaction ends; but if the caller receives a tool call, it performs\n whatever logic is appropriate and sends a new request to the LLM containing\n the original prompt or some variation of it as well as the result of the tool\n call.\n4. The LLM handles the new prompt as in Step 2.\n\nFor this to work, several requirements must be met:\n\n- The model must be trained to make tool requests when it's needed to complete\n a prompt. Most of the larger models provided through web APIs, such as\n Gemini and Claude, can do this, but smaller and more specialized models\n often cannot. Genkit will throw an error if you try to provide tools to a\n model that doesn't support it.\n- The calling application must provide tool definitions to the model in the\n format it expects.\n- The calling application must prompt the model to generate tool calling\n requests in the format the application expects.\n\n## Tool calling with Genkit\n\nGenkit provides a single interface for tool calling with models that support it.\nEach model plugin ensures that the last two of the above criteria are met, and\nthe Genkit instance's `generate()` function automatically carries out the tool\ncalling loop described earlier.\n\n### Model support\n\nTool calling support depends on the model, the model API, and the Genkit plugin.\nConsult the relevant documentation to determine if tool calling is likely to be\nsupported. In addition:\n\n- Genkit will throw an error if you try to provide tools to a model that\n doesn't support it.\n- If the plugin exports model references, the `info.supports.tools` property\n will indicate if it supports tool calling.\n\n### Defining tools\n\nUse the Genkit instance's `defineTool()` function to write tool definitions:\n\n```ts\nimport { genkit, z } from 'genkit';\nimport { googleAI } from '@genkitai/google-ai';\n\nconst ai = genkit({\n plugins: [googleAI()],\n model: googleAI.model('gemini-2.5-flash'),\n});\n\nconst getWeather = ai.defineTool(\n {\n name: 'getWeather',\n description: 'Gets the current weather in a given location',\n inputSchema: z.object({\n location: z.string().describe('The location to get the current weather for'),\n }),\n outputSchema: z.string(),\n },\n async (input) => {\n // Here, we would typically make an API call or database query. For this\n // example, we just return a fixed value.\n return `The current weather in ${input.location} is 63°F and sunny.`;\n },\n);\n```\n\nThe syntax here looks just like the `defineFlow()` syntax; however, `name`,\n`description`, and `inputSchema` parameters are required. When writing a tool\ndefinition, take special care with the wording and descriptiveness of these\nparameters. They are vital for the LLM to make effective use of the\navailable tools.\n\n### Using tools\n\nInclude defined tools in your prompts to generate content.\n\n\n \n ```ts\n const response = await ai.generate({\n prompt: \"What is the weather in Baltimore?\",\n tools: [getWeather],\n });\n ```\n \n \n ```ts\n const weatherPrompt = ai.definePrompt(\n {\n name: \"weatherPrompt\",\n tools: [getWeather],\n },\n \"What is the weather in {{location}}?\"\n );\n\n const response = await weatherPrompt({ location: \"Baltimore\" });\n ```\n\n \n \n ```dotprompt\n ---\n tools: [getWeather]\n input:\n schema:\n location: string\n ---\n\n What is the weather in {{location}}?\n ```\n\n Then you can execute the prompt in your code as follows:\n\n ```ts\n // assuming prompt file is named weatherPrompt.prompt\n const weatherPrompt = ai.prompt(\"weatherPrompt\");\n\n const response = await weatherPrompt({ location: \"Baltimore\" });\n ```\n\n \n \n ```ts\n const chat = ai.chat({\n system: \"Answer questions using the tools you have.\",\n tools: [getWeather],\n });\n\n const response = await chat.send(\"What is the weather in Baltimore?\");\n\n // Or, specify tools that are message-specific\n const response = await chat.send({\n prompt: \"What is the weather in Baltimore?\",\n tools: [getWeather],\n });\n ```\n\n \n\n\n### Streaming and Tool Calling\n\nWhen combining tool calling with streaming responses, you will receive `toolRequest` and `toolResponse` content parts in the chunks of the stream. For example, the following code:\n\n```ts\nconst { stream } = ai.generateStream({\n prompt: \"What is the weather in Baltimore?\",\n tools: [getWeather],\n});\n\nfor await (const chunk of stream) {\n console.log(chunk);\n}\n```\n\nMight produce a sequence of chunks similar to:\n\n```ts\n{index: 0, role: \"model\", content: [{text: \"Okay, I'll check the weather\"}]}\n{index: 0, role: \"model\", content: [{text: \"for Baltimore.\"}]}\n// toolRequests will be emitted as a single chunk by most models\n{index: 0, role: \"model\", content: [{toolRequest: {name: \"getWeather\", input: {location: \"Baltimore\"}}}]}\n// when streaming multiple messages, Genkit increments the index and indicates the new role\n{index: 1, role: \"tool\", content: [{toolResponse: {name: \"getWeather\", output: \"Temperature: 68 degrees\\nStatus: Cloudy.\"}}]}\n{index: 2, role: \"model\", content: [{text: \"The weather in Baltimore is 68 degrees and cloudy.\"}]}\n```\n\nYou can use these chunks to dynamically construct the full generated message sequence.\n\n### Limiting Tool Call Iterations with `maxTurns`\n\nWhen working with tools that might trigger multiple sequential calls, you can control resource usage and prevent runaway execution using the `maxTurns` parameter. This sets a hard limit on how many back-and-forth interactions the model can have with your tools in a single generation cycle.\n\n**Why use maxTurns?**\n- **Cost Control**: Prevents unexpected API usage charges from excessive tool calls\n- **Performance**: Ensures responses complete within reasonable timeframes\n- **Safety**: Guards against infinite loops in complex tool interactions\n- **Predictability**: Makes your application behavior more deterministic\n\nThe default value is 5 turns, which works well for most scenarios. Each \"turn\" represents one complete cycle where the model can make tool calls and receive responses.\n\n**Example: Web Research Agent**\n\nConsider a research agent that might need to search multiple times to find comprehensive information:\n\n```ts\nconst webSearch = ai.defineTool(\n {\n name: 'webSearch',\n description: 'Search the web for current information',\n inputSchema: z.object({\n query: z.string().describe('Search query'),\n }),\n outputSchema: z.string(),\n },\n async (input) => {\n // Simulate web search API call\n return `Search results for \"${input.query}\": [relevant information here]`;\n },\n);\n\nconst response = await ai.generate({\n prompt: 'Research the latest developments in quantum computing, including recent breakthroughs, key companies, and future applications.',\n tools: [webSearch],\n maxTurns: 8, // Allow up to 8 research iterations\n});\n```\n\n**Example: Financial Calculator**\n\nHere's a more complex scenario where an agent might need multiple calculation steps:\n\n```ts\nconst calculator = ai.defineTool(\n {\n name: 'calculator',\n description: 'Perform mathematical calculations',\n inputSchema: z.object({\n expression: z.string().describe('Mathematical expression to evaluate'),\n }),\n outputSchema: z.number(),\n },\n async (input) => {\n // Safe evaluation of mathematical expressions\n return eval(input.expression); // In production, use a safe math parser\n },\n);\n\nconst stockAnalyzer = ai.defineTool(\n {\n name: 'stockAnalyzer',\n description: 'Get current stock price and basic metrics',\n inputSchema: z.object({\n symbol: z.string().describe('Stock symbol (e.g., AAPL)'),\n }),\n outputSchema: z.object({\n price: z.number(),\n change: z.number(),\n volume: z.number(),\n }),\n },\n async (input) => {\n // Simulate stock API call\n return {\n price: 150.25,\n change: 2.50,\n volume: 45000000\n };\n },\n);\n```\n\n\n \n ```typescript\n const response = await ai.generate({\n prompt: 'Calculate the total value of my portfolio: 100 shares of AAPL, 50 shares of GOOGL, and 200 shares of MSFT. Also calculate what percentage each holding represents.',\n tools: [calculator, stockAnalyzer],\n maxTurns: 12, // Multiple stock lookups + calculations needed\n });\n ```\n \n \n ```typescript\n const portfolioAnalysisPrompt = ai.definePrompt(\n {\n name: \"portfolioAnalysis\",\n tools: [calculator, stockAnalyzer],\n maxTurns: 12,\n },\n \"Calculate the total value of my portfolio: {{holdings}}. Also calculate what percentage each holding represents.\"\n );\n\n const response = await portfolioAnalysisPrompt({ \n holdings: \"100 shares of AAPL, 50 shares of GOOGL, and 200 shares of MSFT\" \n });\n ```\n \n \n ```dotprompt\n ---\n tools: [calculator, stockAnalyzer]\n maxTurns: 12\n input:\n schema:\n holdings: string\n ---\n\n Calculate the total value of my portfolio: {{holdings}}. Also calculate what percentage each holding represents.\n ```\n\n Then execute the prompt:\n\n ```typescript\n const portfolioAnalysisPrompt = ai.prompt(\"portfolioAnalysis\");\n\n const response = await portfolioAnalysisPrompt({ \n holdings: \"100 shares of AAPL, 50 shares of GOOGL, and 200 shares of MSFT\" \n });\n ```\n \n \n ```typescript\n const chat = ai.chat({\n system: \"You are a financial analysis assistant. Use the available tools to provide accurate calculations and current market data.\",\n tools: [calculator, stockAnalyzer],\n maxTurns: 12,\n });\n\n const response = await chat.send(\"Calculate the total value of my portfolio: 100 shares of AAPL, 50 shares of GOOGL, and 200 shares of MSFT. Also calculate what percentage each holding represents.\");\n ```\n \n\n\n**What happens when maxTurns is reached?**\n\nWhen the limit is hit, Genkit stops the tool-calling loop and returns the model's current response, even if it was in the middle of using tools. The model will typically provide a partial answer or explain that it couldn't complete all the requested operations.\n\n### Dynamically defining tools at runtime\n\nAs most things in Genkit tools need to be predefined during your app's\ninitialization. This is necessary so that you would be able interact with your\ntools from the Genkit Dev UI. This is typically the recommended way. However\nthere are scenarios when the tool must be defined dynamically per user request.\n\nYou can dynamically define tools using `ai.dynamicTool` function. It is very\nsimilar to `ai.defineTool` method, however dynamic tools are not tracked by\nGenkit runtime, so cannot be interacted with from Genkit Dev UI and must be\npassed to the `ai.generate` call by reference (for regular tools you can also\nuse a string tool name).\n\n```ts\nimport { genkit, z } from 'genkit';\nimport { googleAI } from '@genkit-ai/googleai';\n\nconst ai = genkit({\n plugins: [googleAI()],\n model: googleAI.model('gemini-2.5-flash'),\n});\n\nai.defineFlow('weatherFlow', async () => {\n const getWeather = ai.dynamicTool(\n {\n name: 'getWeather',\n description: 'Gets the current weather in a given location',\n inputSchema: z.object({\n location: z.string().describe('The location to get the current weather for'),\n }),\n outputSchema: z.string(),\n },\n async (input) => {\n return `The current weather in ${input.location} is 63°F and sunny.`;\n },\n );\n\n const { text } = await ai.generate({\n prompt: 'What is the weather in Baltimore?',\n tools: [getWeather],\n });\n\n return text;\n});\n```\n\nWhen defining dynamic tools, to specify input and output schemas you can either\nuse Zod as shown in the previous example, or you can pass in manually\nconstructed JSON Schema.\n\n```ts\nconst getWeather = ai.dynamicTool(\n {\n name: 'getWeather',\n description: 'Gets the current weather in a given location',\n inputJsonSchema: myInputJsonSchema,\n outputJsonSchema: myOutputJsonSchema,\n },\n async (input) => {\n /* ... */\n },\n);\n```\n\nDynamic tools don't require the implementation function. If you don't pass in\nthe function the tool will behave like an [interrupt](/docs/interrupts) and you can\ndo manual tool call handling:\n\n```ts\nconst getWeather = ai.dynamicTool({\n name: 'getWeather',\n description: 'Gets the current weather in a given location',\n inputJsonSchema: myInputJsonSchema,\n outputJsonSchema: myOutputJsonSchema,\n});\n```\n\n### Pause the tool loop by using interrupts\n\nBy default, Genkit repeatedly calls the LLM until every tool call has been\nresolved. You can conditionally pause execution in situations where you want\nto, for example:\n\n- Ask the user a question or display UI.\n- Confirm a potentially risky action with the user.\n- Request out-of-band approval for an action.\n\n**Interrupts** are special tools that can halt the loop and return control\nto your code so that you can handle more advanced scenarios. Visit the\n[interrupts guide](/docs/interrupts) to learn how to use them.\n\n### Explicitly handling tool calls\n\nIf you want full control over this tool-calling loop, for example to\napply more complicated logic, set the `returnToolRequests` parameter to `true`.\nNow it's your responsibility to ensure all of the tool requests are fulfilled:\n\n```ts\nconst getWeather = ai.defineTool(\n {\n // ... tool definition ...\n },\n async ({ location }) => {\n // ... tool implementation ...\n },\n);\n\nconst generateOptions: GenerateOptions = {\n prompt: \"What's the weather like in Baltimore?\",\n tools: [getWeather],\n returnToolRequests: true,\n};\n\nlet llmResponse;\nwhile (true) {\n llmResponse = await ai.generate(generateOptions);\n const toolRequests = llmResponse.toolRequests;\n if (toolRequests.length < 1) {\n break;\n }\n const toolResponses: ToolResponsePart[] = await Promise.all(\n toolRequests.map(async (part) => {\n switch (part.toolRequest.name) {\n case 'specialTool':\n return {\n toolResponse: {\n name: part.toolRequest.name,\n ref: part.toolRequest.ref,\n output: await getWeather(part.toolRequest.input),\n },\n };\n default:\n throw Error('Tool not found');\n }\n }),\n );\n generateOptions.messages = llmResponse.messages;\n generateOptions.prompt = toolResponses;\n}\n", + "js/tool-calling.md": { + "text": "# Tool calling\n\n_Tool calling_, also known as _function calling_, is a structured way to give\nLLMs the ability to make requests back to the application that called it. You\ndefine the tools you want to make available to the model, and the model will\nmake tool requests to your app as necessary to fulfill the prompts you give it.\n\nThe use cases of tool calling generally fall into a few themes:\n\n**Giving an LLM access to information it wasn't trained with**\n\n- Frequently changing information, such as a stock price or the current\n weather.\n- Information specific to your app domain, such as product information or user\n profiles.\n\nNote the overlap with [retrieval augmented generation](/docs/rag) (RAG), which is also\na way to let an LLM integrate factual information into its generations. RAG is a\nheavier solution that is most suited when you have a large amount of information\nor the information that's most relevant to a prompt is ambiguous. On the other\nhand, if retrieving the information the LLM needs is a simple function call or\ndatabase lookup, tool calling is more appropriate.\n\n**Introducing a degree of determinism into an LLM workflow**\n\n- Performing calculations that the LLM cannot reliably complete itself.\n- Forcing an LLM to generate verbatim text under certain circumstances, such\n as when responding to a question about an app's terms of service.\n\n**Performing an action when initiated by an LLM**\n\n- Turning on and off lights in an LLM-powered home assistant\n- Reserving table reservations in an LLM-powered restaurant agent\n\n## Before you begin\n\nIf you want to run the code examples on this page, first complete the steps in\nthe [Getting started](/docs/get-started) guide. All of the examples assume that you\nhave already set up a project with Genkit dependencies installed.\n\nThis page discusses one of the advanced features of Genkit model abstraction, so\nbefore you dive too deeply, you should be familiar with the content on the\n[Generating content with AI models](/docs/models) page. You should also be familiar\nwith Genkit's system for defining input and output schemas, which is discussed\non the [Flows](/docs/flows) page.\n\n## Overview of tool calling\n\n\n\nAt a high level, this is what a typical tool-calling interaction with an LLM\nlooks like:\n\n1. The calling application prompts the LLM with a request and also includes in\n the prompt a list of tools the LLM can use to generate a response.\n2. The LLM either generates a complete response or generates a tool call request\n in a specific format.\n3. If the caller receives a complete response, the request is fulfilled and the\n interaction ends; but if the caller receives a tool call, it performs\n whatever logic is appropriate and sends a new request to the LLM containing\n the original prompt or some variation of it as well as the result of the tool\n call.\n4. The LLM handles the new prompt as in Step 2.\n\nFor this to work, several requirements must be met:\n\n- The model must be trained to make tool requests when it's needed to complete\n a prompt. Most of the larger models provided through web APIs, such as\n Gemini and Claude, can do this, but smaller and more specialized models\n often cannot. Genkit will throw an error if you try to provide tools to a\n model that doesn't support it.\n- The calling application must provide tool definitions to the model in the\n format it expects.\n- The calling application must prompt the model to generate tool calling\n requests in the format the application expects.\n\n## Tool calling with Genkit\n\nGenkit provides a single interface for tool calling with models that support it.\nEach model plugin ensures that the last two of the above criteria are met, and\nthe Genkit instance's `generate()` function automatically carries out the tool\ncalling loop described earlier.\n\n### Model support\n\nTool calling support depends on the model, the model API, and the Genkit plugin.\nConsult the relevant documentation to determine if tool calling is likely to be\nsupported. In addition:\n\n- Genkit will throw an error if you try to provide tools to a model that\n doesn't support it.\n- If the plugin exports model references, the `info.supports.tools` property\n will indicate if it supports tool calling.\n\n### Defining tools\n\nUse the Genkit instance's `defineTool()` function to write tool definitions:\n\n```ts\nimport { genkit, z } from 'genkit';\nimport { googleAI } from '@genkitai/google-ai';\n\nconst ai = genkit({\n plugins: [googleAI()],\n model: googleAI.model('gemini-2.5-flash'),\n});\n\nconst getWeather = ai.defineTool(\n {\n name: 'getWeather',\n description: 'Gets the current weather in a given location',\n inputSchema: z.object({\n location: z.string().describe('The location to get the current weather for'),\n }),\n outputSchema: z.string(),\n },\n async (input) => {\n // Here, we would typically make an API call or database query. For this\n // example, we just return a fixed value.\n return `The current weather in ${input.location} is 63°F and sunny.`;\n },\n);\n```\n\nThe syntax here looks just like the `defineFlow()` syntax; however, `name`,\n`description`, and `inputSchema` parameters are required. When writing a tool\ndefinition, take special care with the wording and descriptiveness of these\nparameters. They are vital for the LLM to make effective use of the\navailable tools.\n\n### Using tools\n\nInclude defined tools in your prompts to generate content.\n\n\n \n ```ts\n const response = await ai.generate({\n prompt: \"What is the weather in Baltimore?\",\n tools: [getWeather],\n });\n ```\n \n \n ```ts\n const weatherPrompt = ai.definePrompt(\n {\n name: \"weatherPrompt\",\n tools: [getWeather],\n },\n \"What is the weather in {{location}}?\"\n );\n\n const response = await weatherPrompt({ location: \"Baltimore\" });\n ```\n\n \n \n ```dotprompt\n ---\n tools: [getWeather]\n input:\n schema:\n location: string\n ---\n\n What is the weather in {{location}}?\n ```\n\n Then you can execute the prompt in your code as follows:\n\n ```ts\n // assuming prompt file is named weatherPrompt.prompt\n const weatherPrompt = ai.prompt(\"weatherPrompt\");\n\n const response = await weatherPrompt({ location: \"Baltimore\" });\n ```\n\n \n \n ```ts\n const chat = ai.chat({\n system: \"Answer questions using the tools you have.\",\n tools: [getWeather],\n });\n\n const response = await chat.send(\"What is the weather in Baltimore?\");\n\n // Or, specify tools that are message-specific\n const response = await chat.send({\n prompt: \"What is the weather in Baltimore?\",\n tools: [getWeather],\n });\n ```\n\n \n\n\n### Streaming and Tool Calling\n\nWhen combining tool calling with streaming responses, you will receive `toolRequest` and `toolResponse` content parts in the chunks of the stream. For example, the following code:\n\n```ts\nconst { stream } = ai.generateStream({\n prompt: \"What is the weather in Baltimore?\",\n tools: [getWeather],\n});\n\nfor await (const chunk of stream) {\n console.log(chunk);\n}\n```\n\nMight produce a sequence of chunks similar to:\n\n```ts\n{index: 0, role: \"model\", content: [{text: \"Okay, I'll check the weather\"}]}\n{index: 0, role: \"model\", content: [{text: \"for Baltimore.\"}]}\n// toolRequests will be emitted as a single chunk by most models\n{index: 0, role: \"model\", content: [{toolRequest: {name: \"getWeather\", input: {location: \"Baltimore\"}}}]}\n// when streaming multiple messages, Genkit increments the index and indicates the new role\n{index: 1, role: \"tool\", content: [{toolResponse: {name: \"getWeather\", output: \"Temperature: 68 degrees\\nStatus: Cloudy.\"}}]}\n{index: 2, role: \"model\", content: [{text: \"The weather in Baltimore is 68 degrees and cloudy.\"}]}\n```\n\nYou can use these chunks to dynamically construct the full generated message sequence.\n\n### Limiting Tool Call Iterations with `maxTurns`\n\nWhen working with tools that might trigger multiple sequential calls, you can control resource usage and prevent runaway execution using the `maxTurns` parameter. This sets a hard limit on how many back-and-forth interactions the model can have with your tools in a single generation cycle.\n\n**Why use maxTurns?**\n- **Cost Control**: Prevents unexpected API usage charges from excessive tool calls\n- **Performance**: Ensures responses complete within reasonable timeframes\n- **Safety**: Guards against infinite loops in complex tool interactions\n- **Predictability**: Makes your application behavior more deterministic\n\nThe default value is 5 turns, which works well for most scenarios. Each \"turn\" represents one complete cycle where the model can make tool calls and receive responses.\n\n**Example: Web Research Agent**\n\nConsider a research agent that might need to search multiple times to find comprehensive information:\n\n```ts\nconst webSearch = ai.defineTool(\n {\n name: 'webSearch',\n description: 'Search the web for current information',\n inputSchema: z.object({\n query: z.string().describe('Search query'),\n }),\n outputSchema: z.string(),\n },\n async (input) => {\n // Simulate web search API call\n return `Search results for \"${input.query}\": [relevant information here]`;\n },\n);\n\nconst response = await ai.generate({\n prompt: 'Research the latest developments in quantum computing, including recent breakthroughs, key companies, and future applications.',\n tools: [webSearch],\n maxTurns: 8, // Allow up to 8 research iterations\n});\n```\n\n**Example: Financial Calculator**\n\nHere's a more complex scenario where an agent might need multiple calculation steps:\n\n```ts\nconst calculator = ai.defineTool(\n {\n name: 'calculator',\n description: 'Perform mathematical calculations',\n inputSchema: z.object({\n expression: z.string().describe('Mathematical expression to evaluate'),\n }),\n outputSchema: z.number(),\n },\n async (input) => {\n // Safe evaluation of mathematical expressions\n return eval(input.expression); // In production, use a safe math parser\n },\n);\n\nconst stockAnalyzer = ai.defineTool(\n {\n name: 'stockAnalyzer',\n description: 'Get current stock price and basic metrics',\n inputSchema: z.object({\n symbol: z.string().describe('Stock symbol (e.g., AAPL)'),\n }),\n outputSchema: z.object({\n price: z.number(),\n change: z.number(),\n volume: z.number(),\n }),\n },\n async (input) => {\n // Simulate stock API call\n return {\n price: 150.25,\n change: 2.50,\n volume: 45000000\n };\n },\n);\n```\n\n\n \n ```typescript\n const response = await ai.generate({\n prompt: 'Calculate the total value of my portfolio: 100 shares of AAPL, 50 shares of GOOGL, and 200 shares of MSFT. Also calculate what percentage each holding represents.',\n tools: [calculator, stockAnalyzer],\n maxTurns: 12, // Multiple stock lookups + calculations needed\n });\n ```\n \n \n ```typescript\n const portfolioAnalysisPrompt = ai.definePrompt(\n {\n name: \"portfolioAnalysis\",\n tools: [calculator, stockAnalyzer],\n maxTurns: 12,\n },\n \"Calculate the total value of my portfolio: {{holdings}}. Also calculate what percentage each holding represents.\"\n );\n\n const response = await portfolioAnalysisPrompt({ \n holdings: \"100 shares of AAPL, 50 shares of GOOGL, and 200 shares of MSFT\" \n });\n ```\n \n \n ```dotprompt\n ---\n tools: [calculator, stockAnalyzer]\n maxTurns: 12\n input:\n schema:\n holdings: string\n ---\n\n Calculate the total value of my portfolio: {{holdings}}. Also calculate what percentage each holding represents.\n ```\n\n Then execute the prompt:\n\n ```typescript\n const portfolioAnalysisPrompt = ai.prompt(\"portfolioAnalysis\");\n\n const response = await portfolioAnalysisPrompt({ \n holdings: \"100 shares of AAPL, 50 shares of GOOGL, and 200 shares of MSFT\" \n });\n ```\n \n \n ```typescript\n const chat = ai.chat({\n system: \"You are a financial analysis assistant. Use the available tools to provide accurate calculations and current market data.\",\n tools: [calculator, stockAnalyzer],\n maxTurns: 12,\n });\n\n const response = await chat.send(\"Calculate the total value of my portfolio: 100 shares of AAPL, 50 shares of GOOGL, and 200 shares of MSFT. Also calculate what percentage each holding represents.\");\n ```\n \n\n\n**What happens when maxTurns is reached?**\n\nWhen the limit is hit, Genkit stops the tool-calling loop and returns the model's current response, even if it was in the middle of using tools. The model will typically provide a partial answer or explain that it couldn't complete all the requested operations.\n\n### Dynamically defining tools at runtime\n\nAs most things in Genkit tools need to be predefined during your app's\ninitialization. This is necessary so that you would be able interact with your\ntools from the Genkit Dev UI. This is typically the recommended way. However\nthere are scenarios when the tool must be defined dynamically per user request.\n\nYou can dynamically define tools using `ai.dynamicTool` function. It is very\nsimilar to `ai.defineTool` method, however dynamic tools are not tracked by\nGenkit runtime, so cannot be interacted with from Genkit Dev UI and must be\npassed to the `ai.generate` call by reference (for regular tools you can also\nuse a string tool name).\n\n```ts\nimport { genkit, z } from 'genkit';\nimport { googleAI } from '@genkit-ai/googleai';\n\nconst ai = genkit({\n plugins: [googleAI()],\n model: googleAI.model('gemini-2.5-flash'),\n});\n\nai.defineFlow('weatherFlow', async () => {\n const getWeather = ai.dynamicTool(\n {\n name: 'getWeather',\n description: 'Gets the current weather in a given location',\n inputSchema: z.object({\n location: z.string().describe('The location to get the current weather for'),\n }),\n outputSchema: z.string(),\n },\n async (input) => {\n return `The current weather in ${input.location} is 63°F and sunny.`;\n },\n );\n\n const { text } = await ai.generate({\n prompt: 'What is the weather in Baltimore?',\n tools: [getWeather],\n });\n\n return text;\n});\n```\n\nWhen defining dynamic tools, to specify input and output schemas you can either\nuse Zod as shown in the previous example, or you can pass in manually\nconstructed JSON Schema.\n\n```ts\nconst getWeather = ai.dynamicTool(\n {\n name: 'getWeather',\n description: 'Gets the current weather in a given location',\n inputJsonSchema: myInputJsonSchema,\n outputJsonSchema: myOutputJsonSchema,\n },\n async (input) => {\n /* ... */\n },\n);\n```\n\nDynamic tools don't require the implementation function. If you don't pass in\nthe function the tool will behave like an [interrupt](/docs/interrupts) and you can\ndo manual tool call handling:\n\n```ts\nconst getWeather = ai.dynamicTool({\n name: 'getWeather',\n description: 'Gets the current weather in a given location',\n inputJsonSchema: myInputJsonSchema,\n outputJsonSchema: myOutputJsonSchema,\n});\n```\n\n### Pause the tool loop by using interrupts\n\nBy default, Genkit repeatedly calls the LLM until every tool call has been\nresolved. You can conditionally pause execution in situations where you want\nto, for example:\n\n- Ask the user a question or display UI.\n- Confirm a potentially risky action with the user.\n- Request out-of-band approval for an action.\n\n**Interrupts** are special tools that can halt the loop and return control\nto your code so that you can handle more advanced scenarios. Visit the\n[interrupts guide](/docs/interrupts) to learn how to use them.\n\n### Explicitly handling tool calls\n\nIf you want full control over this tool-calling loop, for example to\napply more complicated logic, set the `returnToolRequests` parameter to `true`.\nNow it's your responsibility to ensure all of the tool requests are fulfilled:\n\n```ts\nconst getWeather = ai.defineTool(\n {\n // ... tool definition ...\n },\n async ({ location }) => {\n // ... tool implementation ...\n },\n);\n\nconst generateOptions: GenerateOptions = {\n prompt: \"What's the weather like in Baltimore?\",\n tools: [getWeather],\n returnToolRequests: true,\n};\n\nlet llmResponse;\nwhile (true) {\n llmResponse = await ai.generate(generateOptions);\n const toolRequests = llmResponse.toolRequests;\n if (toolRequests.length < 1) {\n break;\n }\n const toolResponses: ToolResponsePart[] = await Promise.all(\n toolRequests.map(async (part) => {\n switch (part.toolRequest.name) {\n case 'specialTool':\n return {\n toolResponse: {\n name: part.toolRequest.name,\n ref: part.toolRequest.ref,\n output: await getWeather(part.toolRequest.input),\n },\n };\n default:\n throw Error('Tool not found');\n }\n }),\n );\n generateOptions.messages = llmResponse.messages;\n generateOptions.prompt = toolResponses;\n}", "title": "Tool calling", - "lang": "js" + "description": "Learn how to enable LLMs to interact with external applications and data using Genkit's tool calling feature, covering tool definition, usage, and advanced scenarios.", + "lang": "js", + "headers": "## Before you begin\n## Overview of tool calling\n## Tool calling with Genkit\n### Model support\n### Defining tools\n### Using tools\n### Streaming and Tool Calling\n### Limiting Tool Call Iterations with `maxTurns`\n### Dynamically defining tools at runtime\n### Pause the tool loop by using interrupts\n### Explicitly handling tool calls\n" }, "js/tutorials/tutorial-chat-with-a-pdf.md": { - "text": "This tutorial demonstrates how to build a conversational application that\nallows users to extract information from PDF documents using natural language.\n\n1. [Set up your project](#1-set-up-your-project)\n2. [Import the required dependencies](#2-import-the-required-dependencies)\n3. [Configure Genkit and the default model](#3-configure-genkit-and-the-default-model)\n4. [Load and parse the PDF file](#4-load-and-parse-the-pdf)\n5. [Set up the prompt](#5-set-up-the-prompt)\n6. [Implement the UI](#6-implement-the-ui)\n7. [Implement the chat loop](#7-implement-the-chat-loop)\n8. [Run the app](#8-run-the-app)\n\n## Prerequisites\n\nBefore starting work, you should have these prerequisites set up:\n\n- [Node.js v20+](https://nodejs.org/en/download)\n- [npm](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm)\n\n## Implementation Steps\n\nAfter setting up your dependencies, you can build the project.\n\n### 1. Set up your project\n\n1. Create a directory structure and a file to hold\n your source code.\n\n ```bash\n mkdir -p chat-with-a-pdf/src && \\\n cd chat-with-a-pdf && \\\n touch src/index.ts\n ```\n\n2. Initialize a new TypeScript project.\n\n ```bash\n npm init -y\n ```\n\n3. Install the pdf-parse module.\n\n ```bash\n npm install pdf-parse && npm install --save-dev @types/pdf-parse\n ```\n\n4. Install the following Genkit dependencies to use Genkit in your project:\n\n ```bash\n npm install genkit @genkit-ai/googleai\n ```\n\n - `genkit` provides Genkit core capabilities.\n - `@genkit-ai/googleai` provides access to the Google AI Gemini models.\n\n5. Get and configure your model API key\n\n To use the Gemini API, which this tutorial uses, you must first\n configure an API key. If you don't already have one,\n [create a key](https://makersuite.google.com/app/apikey) in Google AI Studio.\n\n The Gemini API provides a generous free-of-charge tier and does not require a\n credit card to get started.\n\n After creating your API key, set the `GEMINI_API_KEY` environment\n variable to your key with the following command:\n\n ```bash\n export GEMINI_API_KEY=\n ```\n\n:::note\nAlthough this tutorial uses the Gemini API from AI Studio, Genkit\nsupports a wide variety of model providers, including:\n\n- [Gemini from Vertex AI.](/docs/plugins/vertex-ai#generative-ai-models)\n- Anthropic's Claude 3 models and Llama 3.1 through the\n [Vertex AI Model Garden](/docs/plugins/vertex-ai#anthropic-claude-3-on-vertex-ai-model-garden),\n as well as community plugins.\n- Open source models through\n [Ollama](/docs/plugins/ollama).\n- [Community-supported providers](/docs/models#models-supported-by-genkit) such as OpenAI and Cohere.\n :::\n\n### 2. Import the required dependencies\n\nIn the `index.ts` file that you created, add the\nfollowing lines to import the dependencies required for this project:\n\n```typescript\nimport { googleAI } from '@genkit-ai/googleai';\nimport { genkit } from 'genkit/beta'; // chat is a beta feature\nimport pdf from 'pdf-parse';\nimport fs from 'fs';\nimport { createInterface } from 'node:readline/promises';\n```\n\n- The first line imports the `googleAI`\n plugin from the `@genkit-ai/googleai` package, enabling access to\n Google's Gemini models.\n- The next two lines import the `pdf-parse` library for parsing PDF files\n and the `fs` module for file system operations.\n- The final line imports the `createInterface` function from the\n `node:readline/promises` module, which is used to create a command-line\n interface for user interaction.\n\n### 3. Configure Genkit and the default model\n\nAdd the following lines to configure Genkit and set Gemini 2.0 Flash as the\ndefault model.\n\n```typescript\nconst ai = genkit({\n plugins: [googleAI()],\n model: googleAI.model('gemini-2.5-flash'),\n});\n```\n\nYou can then add a skeleton for the code and error-handling.\n\n```typescript\n(async () => {\n try {\n // Step 1: get command line arguments\n // Step 2: load PDF file\n // Step 3: construct prompt\n // Step 4: start chat\n // Step 5: chat loop\n } catch (error) {\n console.error('Error parsing PDF or interacting with Genkit:', error);\n }\n})(); // <-- don't forget the trailing parentheses to call the function!\n```\n\n### 4. Load and parse the PDF\n\n1. Add code to read the PDF filename that was passed\n in from the command line.\n\n```typescript\n// Step 1: get command line arguments\nconst filename = process.argv[2];\nif (!filename) {\n console.error('Please provide a filename as a command line argument.');\n process.exit(1);\n}\n```\n\n2. Add code to load the contents of the PDF file.\n\n```typescript\n// Step 2: load PDF file\nlet dataBuffer = fs.readFileSync(filename);\nconst { text } = await pdf(dataBuffer);\n```\n\n### 5. Set up the prompt\n\nAdd code to set up the prompt:\n\n```typescript\n// Step 3: construct prompt\nconst prefix = process.argv[3] || \"Sample prompt: Answer the user's questions about the contents of this PDF file.\";\nconst prompt = `\n ${prefix}\n Context:\n ${text}\n `;\n```\n\n- The first `const` declaration defines a default prompt if the user doesn't\n pass in one of their own from the command line.\n- The second `const` declaration interpolates the prompt prefix and the full\n text of the PDF file into the prompt for the model.\n\n### 6. Implement the UI\n\nAdd the following code to start the chat and\nimplement the UI:\n\n```typescript\n// Step 4: start chat\nconst chat = ai.chat({ system: prompt });\nconst readline = createInterface(process.stdin, process.stdout);\nconsole.log(\"You're chatting with Gemini. Ctrl-C to quit.\\n\");\n```\n\nThe first `const` declaration starts the chat with the model by\ncalling the `chat` method, passing the prompt (which includes\nthe full text of the PDF file). The rest of the code instantiates\na text input, then displays a message to the user.\n\n### 7. Implement the chat loop\n\nUnder Step 5, add code to receive user input and\nsend that input to the model using `chat.send`. This part\nof the app loops until the user presses _CTRL + C_.\n\n```typescript\n// Step 5: chat loop\nwhile (true) {\n const userInput = await readline.question('> ');\n const { text } = await chat.send(userInput);\n console.log(text);\n}\n```\n\n### 8. Run the app\n\nTo run the app, open the terminal in the root\nfolder of your project, then run the following command:\n\n```typescript\nnpx tsx src/index.ts path/to/some.pdf\n```\n\nYou can then start chatting with the PDF file.\n", + "text": "# Chat with a PDF file\n\nThis tutorial demonstrates how to build a conversational application that\nallows users to extract information from PDF documents using natural language.\n\n1. [Set up your project](#1-set-up-your-project)\n2. [Import the required dependencies](#2-import-the-required-dependencies)\n3. [Configure Genkit and the default model](#3-configure-genkit-and-the-default-model)\n4. [Load and parse the PDF file](#4-load-and-parse-the-pdf)\n5. [Set up the prompt](#5-set-up-the-prompt)\n6. [Implement the UI](#6-implement-the-ui)\n7. [Implement the chat loop](#7-implement-the-chat-loop)\n8. [Run the app](#8-run-the-app)\n\n## Prerequisites\n\nBefore starting work, you should have these prerequisites set up:\n\n- [Node.js v20+](https://nodejs.org/en/download)\n- [npm](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm)\n\n## Implementation Steps\n\nAfter setting up your dependencies, you can build the project.\n\n### 1. Set up your project\n\n1. Create a directory structure and a file to hold\n your source code.\n\n ```bash\n mkdir -p chat-with-a-pdf/src && \\\n cd chat-with-a-pdf && \\\n touch src/index.ts\n ```\n\n2. Initialize a new TypeScript project.\n\n ```bash\n npm init -y\n ```\n\n3. Install the pdf-parse module.\n\n ```bash\n npm install pdf-parse && npm install --save-dev @types/pdf-parse\n ```\n\n4. Install the following Genkit dependencies to use Genkit in your project:\n\n ```bash\n npm install genkit @genkit-ai/googleai\n ```\n\n - `genkit` provides Genkit core capabilities.\n - `@genkit-ai/googleai` provides access to the Google AI Gemini models.\n\n5. Get and configure your model API key\n\n To use the Gemini API, which this tutorial uses, you must first\n configure an API key. If you don't already have one,\n [create a key](https://makersuite.google.com/app/apikey) in Google AI Studio.\n\n The Gemini API provides a generous free-of-charge tier and does not require a\n credit card to get started.\n\n After creating your API key, set the `GEMINI_API_KEY` environment\n variable to your key with the following command:\n\n ```bash\n export GEMINI_API_KEY=\n ```\n\n:::note\nAlthough this tutorial uses the Gemini API from AI Studio, Genkit\nsupports a wide variety of model providers, including:\n\n- [Gemini from Vertex AI.](/docs/plugins/vertex-ai#generative-ai-models)\n- Anthropic's Claude 3 models and Llama 3.1 through the\n [Vertex AI Model Garden](/docs/plugins/vertex-ai#anthropic-claude-3-on-vertex-ai-model-garden),\n as well as community plugins.\n- Open source models through\n [Ollama](/docs/plugins/ollama).\n- [Community-supported providers](/docs/models#models-supported-by-genkit) such as OpenAI and Cohere.\n :::\n\n### 2. Import the required dependencies\n\nIn the `index.ts` file that you created, add the\nfollowing lines to import the dependencies required for this project:\n\n```typescript\nimport { googleAI } from '@genkit-ai/googleai';\nimport { genkit } from 'genkit/beta'; // chat is a beta feature\nimport pdf from 'pdf-parse';\nimport fs from 'fs';\nimport { createInterface } from 'node:readline/promises';\n```\n\n- The first line imports the `googleAI`\n plugin from the `@genkit-ai/googleai` package, enabling access to\n Google's Gemini models.\n- The next two lines import the `pdf-parse` library for parsing PDF files\n and the `fs` module for file system operations.\n- The final line imports the `createInterface` function from the\n `node:readline/promises` module, which is used to create a command-line\n interface for user interaction.\n\n### 3. Configure Genkit and the default model\n\nAdd the following lines to configure Genkit and set Gemini 2.0 Flash as the\ndefault model.\n\n```typescript\nconst ai = genkit({\n plugins: [googleAI()],\n model: googleAI.model('gemini-2.5-flash'),\n});\n```\n\nYou can then add a skeleton for the code and error-handling.\n\n```typescript\n(async () => {\n try {\n // Step 1: get command line arguments\n // Step 2: load PDF file\n // Step 3: construct prompt\n // Step 4: start chat\n // Step 5: chat loop\n } catch (error) {\n console.error('Error parsing PDF or interacting with Genkit:', error);\n }\n})(); // <-- don't forget the trailing parentheses to call the function!\n```\n\n### 4. Load and parse the PDF\n\n1. Add code to read the PDF filename that was passed\n in from the command line.\n\n```typescript\n// Step 1: get command line arguments\nconst filename = process.argv[2];\nif (!filename) {\n console.error('Please provide a filename as a command line argument.');\n process.exit(1);\n}\n```\n\n2. Add code to load the contents of the PDF file.\n\n```typescript\n// Step 2: load PDF file\nlet dataBuffer = fs.readFileSync(filename);\nconst { text } = await pdf(dataBuffer);\n```\n\n### 5. Set up the prompt\n\nAdd code to set up the prompt:\n\n```typescript\n// Step 3: construct prompt\nconst prefix = process.argv[3] || \"Sample prompt: Answer the user's questions about the contents of this PDF file.\";\nconst prompt = `\n ${prefix}\n Context:\n ${text}\n `;\n```\n\n- The first `const` declaration defines a default prompt if the user doesn't\n pass in one of their own from the command line.\n- The second `const` declaration interpolates the prompt prefix and the full\n text of the PDF file into the prompt for the model.\n\n### 6. Implement the UI\n\nAdd the following code to start the chat and\nimplement the UI:\n\n```typescript\n// Step 4: start chat\nconst chat = ai.chat({ system: prompt });\nconst readline = createInterface(process.stdin, process.stdout);\nconsole.log(\"You're chatting with Gemini. Ctrl-C to quit.\\n\");\n```\n\nThe first `const` declaration starts the chat with the model by\ncalling the `chat` method, passing the prompt (which includes\nthe full text of the PDF file). The rest of the code instantiates\na text input, then displays a message to the user.\n\n### 7. Implement the chat loop\n\nUnder Step 5, add code to receive user input and\nsend that input to the model using `chat.send`. This part\nof the app loops until the user presses _CTRL + C_.\n\n```typescript\n// Step 5: chat loop\nwhile (true) {\n const userInput = await readline.question('> ');\n const { text } = await chat.send(userInput);\n console.log(text);\n}\n```\n\n### 8. Run the app\n\nTo run the app, open the terminal in the root\nfolder of your project, then run the following command:\n\n```typescript\nnpx tsx src/index.ts path/to/some.pdf\n```\n\nYou can then start chatting with the PDF file.\n", "title": "Chat with a PDF file", - "lang": "js" + "description": "Learn how to build a conversational application that allows users to extract information from PDF documents using natural language.", + "lang": "js", + "headers": "## Prerequisites\n## Implementation Steps\n### 1. Set up your project\n### 2. Import the required dependencies\n### 3. Configure Genkit and the default model\n### 4. Load and parse the PDF\n### 5. Set up the prompt\n### 6. Implement the UI\n### 7. Implement the chat loop\n### 8. Run the app\n" }, "js/tutorials/tutorial-summarize-youtube-videos.md": { - "text": "This tutorial demonstrates how to build a conversational application that\nallows users to summarize YouTube videos and chat about their contents using\nnatural language.\n\n1. [Set up your project](#1-set-up-your-project)\n2. [Import the required dependencies](#2-import-the-required-dependencies)\n3. [Configure Genkit and the default model](#3-configure-genkit-and-the-default-model)\n4. [Get the video URL from the command line](#4-parse-the-command-line-and-get-the-video-url)\n5. [Set up the prompt](#5-set-up-the-prompt)\n6. [Generate the response](#6-generate-the-response)\n7. [Run the app](#7-run-the-app)\n\n## Prerequisites\n\nBefore starting work, you should have these prerequisites set up:\n\n- [Node.js v20+](https://nodejs.org/en/download)\n- [npm](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm)\n\n## Implementation Steps\n\nAfter setting up your dependencies, you can build the project.\n\n### 1. Set up your project\n\n1. Create a directory structure and a file to hold\n your source code.\n\n ```bash\n mkdir -p summarize-a-video/src && \\\n cd summarize-a-video && \\\n touch src/index.ts\n ```\n\n2. Initialize a new TypeScript project.\n\n ```bash\n npm init -y\n ```\n\n3. Install the following Genkit dependencies to use Genkit in your project:\n\n ```bash\n npm install genkit @genkit-ai/googleai\n ```\n\n - `genkit` provides Genkit core capabilities.\n - `@genkit-ai/googleai` provides access to the Google AI Gemini models.\n\n4. Get and configure your model API key\n\n To use the Gemini API, which this tutorial uses, you must first\n configure an API key. If you don't already have one,\n [create a key](https://makersuite.google.com/app/apikey) in Google AI Studio.\n\n The Gemini API provides a generous free-of-charge tier and does not require a\n credit card to get started.\n\n After creating your API key, set the `GEMINI_API_KEY` environment\n variable to your key with the following command:\n\n ```bash\n export GEMINI_API_KEY=\n ```\n\n:::note\nAlthough this tutorial uses the Gemini API from AI Studio, Genkit\nsupports a wide variety of model providers, including:\n\n- [Gemini from Vertex AI.](/docs/plugins/vertex-ai#generative-ai-models)\n- Anthropic's Claude 3 models and Llama 3.1 through the\n [Vertex AI Model Garden](/docs/plugins/vertex-ai#anthropic-claude-3-on-vertex-ai-model-garden),\n as well as community plugins.\n- Open source models through\n [Ollama](/docs/plugins/ollama).\n- [Community-supported providers](/docs/models#models-supported-by-genkit) such as OpenAI and Cohere.\n:::\n\n### 2. Import the required dependencies\n\nIn the `index.ts` file that you created, add the\nfollowing lines to import the dependencies required for this project:\n\n```typescript\nimport { googleAI } from '@genkit-ai/googleai';\nimport { genkit } from 'genkit';\n```\n\n- The first line imports the `googleAI` plugin from the `@genkit-ai/googleai` package, enabling access to\n Google's Gemini models.\n\n### 3. Configure Genkit and the default model\n\nAdd the following lines to configure Genkit and set Gemini 2.0 Flash as the\ndefault model.\n\n```typescript\nconst ai = genkit({\n plugins: [googleAI()],\n model: googleAI.model('gemini-2.5-flash'),\n});\n```\n\nYou can then add a skeleton for the code and error-handling.\n\n```typescript\n(async () => {\n try {\n // Step 1: get command line arguments\n // Step 2: construct prompt\n // Step 3: process video\n } catch (error) {\n console.error('Error processing video:', error);\n }\n})(); // <-- don't forget the trailing parentheses to call the function!\n```\n\n### 4. Parse the command line and get the video URL\n\nAdd code to read the URL of the video that was passed in from the command line.\n\n```typescript\n// Step 1: get command line arguments\nconst videoURL = process.argv[2];\nif (!videoURL) {\n console.error('Please provide a video URL as a command line argument.');\n process.exit(1);\n}\n```\n\n### 5. Set up the prompt\n\nAdd code to set up the prompt:\n\n```typescript\n// Step 2: construct prompt\nconst prompt = process.argv[3] || 'Please summarize the following video:';\n```\n\n- This `const` declaration defines a default prompt if the user doesn't\n pass in one of their own from the command line.\n\n### 6. Generate the response\n\nAdd the following code to pass a multimodal prompt to the model:\n\n```typescript\n// Step 3: process video\nconst { text } = await ai.generate({\n prompt: [{ text: prompt }, { media: { url: videoURL, contentType: 'video/mp4' } }],\n});\nconsole.log(text);\n```\n\nThis code snippet calls the `ai.generate` method to send a multimodal prompt to\nthe model. The prompt consists of two parts:\n\n- `{ text: prompt }`: This is the text prompt that you defined earlier.\n- `{ media: { url: videoURL, contentType: \"video/mp4\" } }`: This is the URL of\n the video that you provided as a command-line argument. The `contentType`\n is set to `video/mp4` to indicate that the URL points to an MP4 video file.\n\nThe `ai.generate` method returns an object containing the generated text, which\nis then logged to the console.\n\n### 7. Run the app\n\nTo run the app, open the terminal in the root\nfolder of your project, then run the following command:\n\n```bash\nnpx tsx src/index.ts https://www.youtube.com/watch\\?v\\=YUgXJkNqH9Q\n```\n\nAfter a moment, a summary of the video you provided appears.\n\nYou can pass in other prompts as well. For example:\n\n```bash\nnpx tsx src/index.ts https://www.youtube.com/watch\\?v\\=YUgXJkNqH9Q \"Transcribe this video\"\n```\n\n:::note\nIf you get an error message saying \"no matches found\", you\nmight need to wrap the video URL in quotes.\n:::\n", + "text": "# Summarize YouTube videos\n\nThis tutorial demonstrates how to build a conversational application that\nallows users to summarize YouTube videos and chat about their contents using\nnatural language.\n\n1. [Set up your project](#1-set-up-your-project)\n2. [Import the required dependencies](#2-import-the-required-dependencies)\n3. [Configure Genkit and the default model](#3-configure-genkit-and-the-default-model)\n4. [Get the video URL from the command line](#4-parse-the-command-line-and-get-the-video-url)\n5. [Set up the prompt](#5-set-up-the-prompt)\n6. [Generate the response](#6-generate-the-response)\n7. [Run the app](#7-run-the-app)\n\n## Prerequisites\n\nBefore starting work, you should have these prerequisites set up:\n\n- [Node.js v20+](https://nodejs.org/en/download)\n- [npm](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm)\n\n## Implementation Steps\n\nAfter setting up your dependencies, you can build the project.\n\n### 1. Set up your project\n\n1. Create a directory structure and a file to hold\n your source code.\n\n ```bash\n mkdir -p summarize-a-video/src && \\\n cd summarize-a-video && \\\n touch src/index.ts\n ```\n\n2. Initialize a new TypeScript project.\n\n ```bash\n npm init -y\n ```\n\n3. Install the following Genkit dependencies to use Genkit in your project:\n\n ```bash\n npm install genkit @genkit-ai/googleai\n ```\n\n - `genkit` provides Genkit core capabilities.\n - `@genkit-ai/googleai` provides access to the Google AI Gemini models.\n\n4. Get and configure your model API key\n\n To use the Gemini API, which this tutorial uses, you must first\n configure an API key. If you don't already have one,\n [create a key](https://makersuite.google.com/app/apikey) in Google AI Studio.\n\n The Gemini API provides a generous free-of-charge tier and does not require a\n credit card to get started.\n\n After creating your API key, set the `GEMINI_API_KEY` environment\n variable to your key with the following command:\n\n ```bash\n export GEMINI_API_KEY=\n ```\n\n:::note\nAlthough this tutorial uses the Gemini API from AI Studio, Genkit\nsupports a wide variety of model providers, including:\n\n- [Gemini from Vertex AI.](/docs/plugins/vertex-ai#generative-ai-models)\n- Anthropic's Claude 3 models and Llama 3.1 through the\n [Vertex AI Model Garden](/docs/plugins/vertex-ai#anthropic-claude-3-on-vertex-ai-model-garden),\n as well as community plugins.\n- Open source models through\n [Ollama](/docs/plugins/ollama).\n- [Community-supported providers](/docs/models#models-supported-by-genkit) such as OpenAI and Cohere.\n:::\n\n### 2. Import the required dependencies\n\nIn the `index.ts` file that you created, add the\nfollowing lines to import the dependencies required for this project:\n\n```typescript\nimport { googleAI } from '@genkit-ai/googleai';\nimport { genkit } from 'genkit';\n```\n\n- The first line imports the `googleAI` plugin from the `@genkit-ai/googleai` package, enabling access to\n Google's Gemini models.\n\n### 3. Configure Genkit and the default model\n\nAdd the following lines to configure Genkit and set Gemini 2.0 Flash as the\ndefault model.\n\n```typescript\nconst ai = genkit({\n plugins: [googleAI()],\n model: googleAI.model('gemini-2.5-flash'),\n});\n```\n\nYou can then add a skeleton for the code and error-handling.\n\n```typescript\n(async () => {\n try {\n // Step 1: get command line arguments\n // Step 2: construct prompt\n // Step 3: process video\n } catch (error) {\n console.error('Error processing video:', error);\n }\n})(); // <-- don't forget the trailing parentheses to call the function!\n```\n\n### 4. Parse the command line and get the video URL\n\nAdd code to read the URL of the video that was passed in from the command line.\n\n```typescript\n// Step 1: get command line arguments\nconst videoURL = process.argv[2];\nif (!videoURL) {\n console.error('Please provide a video URL as a command line argument.');\n process.exit(1);\n}\n```\n\n### 5. Set up the prompt\n\nAdd code to set up the prompt:\n\n```typescript\n// Step 2: construct prompt\nconst prompt = process.argv[3] || 'Please summarize the following video:';\n```\n\n- This `const` declaration defines a default prompt if the user doesn't\n pass in one of their own from the command line.\n\n### 6. Generate the response\n\nAdd the following code to pass a multimodal prompt to the model:\n\n```typescript\n// Step 3: process video\nconst { text } = await ai.generate({\n prompt: [{ text: prompt }, { media: { url: videoURL, contentType: 'video/mp4' } }],\n});\nconsole.log(text);\n```\n\nThis code snippet calls the `ai.generate` method to send a multimodal prompt to\nthe model. The prompt consists of two parts:\n\n- `{ text: prompt }`: This is the text prompt that you defined earlier.\n- `{ media: { url: videoURL, contentType: \"video/mp4\" } }`: This is the URL of\n the video that you provided as a command-line argument. The `contentType`\n is set to `video/mp4` to indicate that the URL points to an MP4 video file.\n\nThe `ai.generate` method returns an object containing the generated text, which\nis then logged to the console.\n\n### 7. Run the app\n\nTo run the app, open the terminal in the root\nfolder of your project, then run the following command:\n\n```bash\nnpx tsx src/index.ts https://www.youtube.com/watch\\?v\\=YUgXJkNqH9Q\n```\n\nAfter a moment, a summary of the video you provided appears.\n\nYou can pass in other prompts as well. For example:\n\n```bash\nnpx tsx src/index.ts https://www.youtube.com/watch\\?v\\=YUgXJkNqH9Q \"Transcribe this video\"\n```\n\n:::note\nIf you get an error message saying \"no matches found\", you\nmight need to wrap the video URL in quotes.\n:::\n", "title": "Summarize YouTube videos", - "lang": "js" + "description": "Learn how to build a conversational application that allows users to summarize YouTube videos and chat about their contents using natural language.", + "lang": "js", + "headers": "## Prerequisites\n## Implementation Steps\n### 1. Set up your project\n### 2. Import the required dependencies\n### 3. Configure Genkit and the default model\n### 4. Parse the command line and get the video URL\n### 5. Set up the prompt\n### 6. Generate the response\n### 7. Run the app\n" }, "js/templates/pgvector.md": { - "text": "You can use PostgreSQL and `pgvector` as your retriever implementation. Use the\nfollowing example as a starting point and modify it to work with your database\nschema.\n\n```ts\nimport { genkit, z, Document } from 'genkit';\nimport { googleAI, textEmbedding004 } from '@genkit-ai/googleai';\nimport { toSql } from 'pgvector';\nimport postgres from 'postgres';\n\nconst ai = genkit({\n plugins: [googleAI()],\n});\n\nconst sql = postgres({ ssl: false, database: 'recaps' });\n\nconst QueryOptions = z.object({\n show: z.string(),\n k: z.number().optional(),\n});\n\nconst sqlRetriever = ai.defineRetriever(\n {\n name: 'pgvector-myTable',\n configSchema: QueryOptions,\n },\n async (input, options) => {\n const embedding = (\n await ai.embed({\n embedder: textEmbedding004,\n content: input,\n })\n )[0].embedding;\n const results = await sql`\n SELECT episode_id, season_number, chunk as content\n FROM embeddings\n WHERE show_id = ${options.show}\n ORDER BY embedding <#> ${toSql(embedding)} LIMIT ${options.k ?? 3}\n `;\n return {\n documents: results.map((row) => {\n const { content, ...metadata } = row;\n return Document.fromText(content, metadata);\n }),\n };\n },\n);\n```\n\nAnd here's how to use the retriever in a flow:\n\n```ts\n// Simple flow to use the sqlRetriever\nexport const askQuestionsOnGoT = ai.defineFlow(\n {\n name: 'askQuestionsOnGoT',\n inputSchema: z.string(),\n outputSchema: z.string(),\n },\n async (inputQuestion) => {\n const docs = await ai.retrieve({\n retriever: sqlRetriever,\n query: inputQuestion,\n options: {\n show: 'Game of Thrones',\n },\n });\n console.log(docs);\n\n // Continue with using retrieved docs\n // in RAG prompts.\n //...\n },\n);\n```\n", + "text": "# pgvector retriever template\n\nYou can use PostgreSQL and `pgvector` as your retriever implementation. Use the\nfollowing example as a starting point and modify it to work with your database\nschema.\n\n```ts\nimport { genkit, z, Document } from 'genkit';\nimport { googleAI } from '@genkit-ai/googleai';\nimport { toSql } from 'pgvector';\nimport postgres from 'postgres';\n\nconst ai = genkit({\n plugins: [googleAI()],\n});\n\nconst sql = postgres({ ssl: false, database: 'recaps' });\n\nconst QueryOptions = z.object({\n show: z.string(),\n k: z.number().optional(),\n});\n\nconst sqlRetriever = ai.defineRetriever(\n {\n name: 'pgvector-myTable',\n configSchema: QueryOptions,\n },\n async (input, options) => {\n const embedding = (\n await ai.embed({\n embedder: googleAI.embedder('gemini-embedding-001'),\n content: input,\n })\n )[0].embedding;\n const results = await sql`\n SELECT episode_id, season_number, chunk as content\n FROM embeddings\n WHERE show_id = ${options.show}\n ORDER BY embedding <#> ${toSql(embedding)} LIMIT ${options.k ?? 3}\n `;\n return {\n documents: results.map((row) => {\n const { content, ...metadata } = row;\n return Document.fromText(content, metadata);\n }),\n };\n },\n);\n```\n\nAnd here's how to use the retriever in a flow:\n\n```ts\n// Simple flow to use the sqlRetriever\nexport const askQuestionsOnGoT = ai.defineFlow(\n {\n name: 'askQuestionsOnGoT',\n inputSchema: z.object({ question: z.string() }),\n outputSchema: z.object({ answer: z.string() }),\n },\n async ({ question }) => {\n const docs = await ai.retrieve({\n retriever: sqlRetriever,\n query: question,\n options: {\n show: 'Game of Thrones',\n },\n });\n console.log(docs);\n\n // Continue with using retrieved docs\n // in RAG prompts.\n // For example:\n const { text } = await ai.generate({\n prompt: `Answer this question using the provided context: ${question}`,\n docs,\n });\n \n return { answer: text };\n },\n);\n```\n", "title": "pgvector retriever template", - "lang": "js" + "description": "This document provides a template for using PostgreSQL and pgvector as a retriever implementation in Genkit, with examples for configuration and usage in flows.", + "lang": "js", + "headers": "" }, "js/plugins/astra-db.md": { - "text": "This plugin provides a [Astra DB](https://docs.datastax.com/en/astra-db-serverless/index.html) retriever and indexer for Genkit.\n\n## Installation\n\n```bash\nnpm install genkitx-astra-db\n```\n\n## Prerequisites\n\nYou will need a DataStax account in which to run an Astra DB database. You can [sign up for a free DataStax account here](https://astra.datastax.com/signup).\n\nOnce you have an account, create a Serverless Vector database. After the database has been provisioned, create a collection. Ensure that you choose the same number of dimensions as the embedding provider you are going to use.\n\nYou will then need the database's API Endpoint, an Application Token and the name of the collection in order to configure the plugin.\n\n## Configuration\n\nTo use the Astra DB plugin, specify it when you call `configureGenkit()`:\n\n```typescript\nimport { genkit } from \"genkit\";\nimport { textEmbedding004 } from \"@genkit-ai/googleai\";\nimport { astraDB } from \"genkitx-astra-db\";\n\nconst ai = genkit({\n plugins: [\n astraDB([\n {\n clientParams: {\n applicationToken: \"your_application_token\",\n apiEndpoint: \"your_astra_db_endpoint\",\n keyspace: \"default_keyspace\",\n },\n collectionName: \"your_collection_name\",\n embedder: textEmbedding004,\n },\n ]),\n ],\n});\n```\n\n### Client Parameters\n\nYou will need an Application Token and API Endpoint from Astra DB. You can either provide them through the `clientParams` object or by setting the environment variables `ASTRA_DB_APPLICATION_TOKEN` and `ASTRA_DB_API_ENDPOINT`.\n\nIf you are using the default namespace, you do not need to pass it as config.\n\n### Configuration Options\n\nThe Astra DB plugin accepts the following configuration options:\n\n- `collectionName`: (required) The name of the collection in your Astra DB database\n- `embedder`: (required) The embedding model to use, like Google's `textEmbedding004`. Ensure that you have set up your collection with the correct number of dimensions for the embedder that you are using\n- `clientParams`: (optional) Astra DB connection configuration with the following properties:\n - `applicationToken`: Your Astra DB application token\n - `apiEndpoint`: Your Astra DB API endpoint\n - `keyspace`: (optional) Your Astra DB keyspace, defaults to \"default_keyspace\"\n\n### Astra DB Vectorize\n\nYou do not need to provide an `embedder` as you can use [Astra DB Vectorize](https://docs.datastax.com/en/astra-db-serverless/databases/embedding-generation.html) to generate your vectors. Ensure that you have [set up your collection with an embedding provider](https://docs.datastax.com/en/astra-db-serverless/databases/embedding-generation.html#external-embedding-provider-integrations). You can then skip the `embedder` option:\n\n```typescript\nimport { genkit } from \"genkit\";\nimport { astraDB } from \"genkitx-astra-db\";\n\nconst ai = genkit({\n plugins: [\n astraDB([\n {\n clientParams: {\n applicationToken: \"your_application_token\",\n apiEndpoint: \"your_astra_db_endpoint\",\n keyspace: \"default_keyspace\",\n },\n collectionName: \"your_collection_name\",\n },\n ]),\n ],\n});\n```\n\n## Usage\n\nImport the indexer and retriever references like so:\n\n```typescript\nimport { astraDBIndexerRef, astraDBRetrieverRef } from \"genkitx-astra-db\";\n```\n\nThen get a reference using the `collectionName` and an optional `displayName` and pass the relevant references to the Genkit functions `index()` or `retrieve()`.\n\n### Indexing\n\nUse the indexer reference with `ai.index()`:\n\n```typescript\nexport const astraDBIndexer = astraDBIndexerRef({\n collectionName: \"your_collection_name\",\n});\n\nawait ai.index({\n indexer: astraDBIndexer,\n documents,\n});\n```\n\n### Retrieval\n\nUse the retriever reference with `ai.retrieve()`:\n\n```typescript\nexport const astraDBRetriever = astraDBRetrieverRef({\n collectionName: \"your_collection_name\",\n});\n\nawait ai.retrieve({\n retriever: astraDBRetriever,\n query,\n});\n```\n\n#### Retrieval Options\n\nYou can pass options to `retrieve()` that will affect the retriever. The available options are:\n\n- `k`: The number of documents to return from the retriever. The default is 5.\n- `filter`: A `Filter` as defined by the [Astra DB library](https://docs.datastax.com/en/astra-api-docs/_attachments/typescript-client/types/Filter.html). See below for how to use a filter\n\n#### Advanced Retrieval\n\nIf you want to perform a vector search with additional filtering (hybrid search) you can pass a schema type to `astraDBRetrieverRef`. For example:\n\n```typescript\ntype Schema = {\n _id: string;\n text: string;\n score: number;\n};\n\nexport const astraDBRetriever = astraDBRetrieverRef({\n collectionName: \"your_collection_name\",\n});\n\nawait ai.retrieve({\n retriever: astraDBRetriever,\n query,\n options: {\n filter: {\n score: { $gt: 75 },\n },\n },\n});\n```\n\nYou can find the [operators that you can use in filters in the Astra DB documentation](https://docs.datastax.com/en/astra-db-serverless/api-reference/overview.html#operators).\n\nIf you don't provide a schema type, you can still filter but you won't get type-checking on the filtering options.\n\n## Further Information\n\nFor more on using indexers and retrievers with Genkit check out the documentation on [Retrieval-Augmented Generation with Genkit](/docs/rag).\n\n## Learn More\n\nFor more information, feedback, or to report issues, visit the [Astra DB plugin GitHub repository](https://github.com/datastax/genkitx-astra-db/tree/main).\n", + "text": "# Astra DB plugin\n\nThis plugin provides a [Astra DB](https://docs.datastax.com/en/astra-db-serverless/index.html) retriever and indexer for Genkit.\n\n## Installation\n\n```bash\nnpm install genkitx-astra-db\n```\n\n## Prerequisites\n\nYou will need a DataStax account in which to run an Astra DB database. You can [sign up for a free DataStax account here](https://astra.datastax.com/signup).\n\nOnce you have an account, create a Serverless Vector database. After the database has been provisioned, create a collection. Ensure that you choose the same number of dimensions as the embedding provider you are going to use.\n\nYou will then need the database's API Endpoint, an Application Token and the name of the collection in order to configure the plugin.\n\n## Configuration\n\nTo use the Astra DB plugin, specify it when you call `configureGenkit()`:\n\n```typescript\nimport { genkit } from \"genkit\";\nimport { googleAI } from \"@genkit-ai/googleai\";\nimport { astraDB } from \"genkitx-astra-db\";\n\nconst ai = genkit({\n plugins: [\n astraDB([\n {\n clientParams: {\n applicationToken: \"your_application_token\",\n apiEndpoint: \"your_astra_db_endpoint\",\n keyspace: \"default_keyspace\",\n },\n collectionName: \"your_collection_name\",\n embedder: googleAI.embedder('gemini-embedding-001'),\n },\n ]),\n ],\n});\n```\n\n### Client Parameters\n\nYou will need an Application Token and API Endpoint from Astra DB. You can either provide them through the `clientParams` object or by setting the environment variables `ASTRA_DB_APPLICATION_TOKEN` and `ASTRA_DB_API_ENDPOINT`.\n\nIf you are using the default namespace, you do not need to pass it as config.\n\n### Configuration Options\n\nThe Astra DB plugin accepts the following configuration options:\n\n- `collectionName`: (required) The name of the collection in your Astra DB database\n- `embedder`: (required) The embedding model to use, like Google's `googleAI.embedder('gemini-embedding-001')`. Ensure that you have set up your collection with the correct number of dimensions for the embedder that you are using\n- `clientParams`: (optional) Astra DB connection configuration with the following properties:\n - `applicationToken`: Your Astra DB application token\n - `apiEndpoint`: Your Astra DB API endpoint\n - `keyspace`: (optional) Your Astra DB keyspace, defaults to \"default_keyspace\"\n\n### Astra DB Vectorize\n\nYou do not need to provide an `embedder` as you can use [Astra DB Vectorize](https://docs.datastax.com/en/astra-db-serverless/databases/embedding-generation.html) to generate your vectors. Ensure that you have [set up your collection with an embedding provider](https://docs.datastax.com/en/astra-db-serverless/databases/embedding-generation.html#external-embedding-provider-integrations). You can then skip the `embedder` option:\n\n```typescript\nimport { genkit } from \"genkit\";\nimport { astraDB } from \"genkitx-astra-db\";\n\nconst ai = genkit({\n plugins: [\n astraDB([\n {\n clientParams: {\n applicationToken: \"your_application_token\",\n apiEndpoint: \"your_astra_db_endpoint\",\n keyspace: \"default_keyspace\",\n },\n collectionName: \"your_collection_name\",\n },\n ]),\n ],\n});\n```\n\n## Usage\n\nImport the indexer and retriever references like so:\n\n```typescript\nimport { astraDBIndexerRef, astraDBRetrieverRef } from \"genkitx-astra-db\";\n```\n\nThen get a reference using the `collectionName` and an optional `displayName` and pass the relevant references to the Genkit functions `index()` or `retrieve()`.\n\n### Indexing\n\nUse the indexer reference with `ai.index()`:\n\n```typescript\nexport const astraDBIndexer = astraDBIndexerRef({\n collectionName: \"your_collection_name\",\n});\n\nawait ai.index({\n indexer: astraDBIndexer,\n documents,\n});\n```\n\n### Retrieval\n\nUse the retriever reference with `ai.retrieve()`:\n\n```typescript\nexport const astraDBRetriever = astraDBRetrieverRef({\n collectionName: \"your_collection_name\",\n});\n\nawait ai.retrieve({\n retriever: astraDBRetriever,\n query,\n});\n```\n\n#### Retrieval Options\n\nYou can pass options to `retrieve()` that will affect the retriever. The available options are:\n\n- `k`: The number of documents to return from the retriever. The default is 5.\n- `filter`: A `Filter` as defined by the [Astra DB library](https://docs.datastax.com/en/astra-api-docs/_attachments/typescript-client/types/Filter.html). See below for how to use a filter\n\n#### Advanced Retrieval\n\nIf you want to perform a vector search with additional filtering (hybrid search) you can pass a schema type to `astraDBRetrieverRef`. For example:\n\n```typescript\ntype Schema = {\n _id: string;\n text: string;\n score: number;\n};\n\nexport const astraDBRetriever = astraDBRetrieverRef({\n collectionName: \"your_collection_name\",\n});\n\nawait ai.retrieve({\n retriever: astraDBRetriever,\n query,\n options: {\n filter: {\n score: { $gt: 75 },\n },\n },\n});\n```\n\nYou can find the [operators that you can use in filters in the Astra DB documentation](https://docs.datastax.com/en/astra-db-serverless/api-reference/overview.html#operators).\n\nIf you don't provide a schema type, you can still filter but you won't get type-checking on the filtering options.\n\n## Further Information\n\nFor more on using indexers and retrievers with Genkit check out the documentation on [Retrieval-Augmented Generation with Genkit](/docs/rag).\n\n## Learn More\n\nFor more information, feedback, or to report issues, visit the [Astra DB plugin GitHub repository](https://github.com/datastax/genkitx-astra-db/tree/main).\n", "title": "Astra DB plugin", - "lang": "js" + "description": "This document describes the Astra DB plugin for Genkit, providing a retriever and indexer for Astra DB, including installation, prerequisites, configuration, and usage.", + "lang": "js", + "headers": "## Installation\n## Prerequisites\n## Configuration\n### Client Parameters\n### Configuration Options\n### Astra DB Vectorize\n## Usage\n### Indexing\n### Retrieval\n#### Retrieval Options\n#### Advanced Retrieval\n## Further Information\n## Learn More\n" }, "js/plugins/auth0.md": { - "text": "The Auth0 AI plugin (`@auth0/ai-genkit`) is an SDK for building secure AI-powered applications using [Auth0](https://www.auth0.ai/), [Okta FGA](https://docs.fga.dev/) and Genkit.\n\n## Features\n\n- **Authorization for RAG**: Securely filter documents using Okta FGA as a [retriever](https://js.langchain.com/docs/concepts/retrievers/) for RAG applications. This smart retriever performs efficient batch access control checks, ensuring users only see documents they have permission to access.\n\n- **Tool Authorization with FGA**: Protect AI tool execution with fine-grained authorization policies through Okta FGA integration, controlling which users can invoke specific tools based on custom authorization rules.\n\n- **Client Initiated Backchannel Authentication (CIBA)**: Implement secure, out-of-band user authorization for sensitive AI operations using the [CIBA standard](https://openid.net/specs/openid-client-initiated-backchannel-authentication-core-1_0.html), enabling user confirmation without disrupting the main interaction flow.\n\n- **Federated API Access**: Seamlessly connect to third-party services by leveraging Auth0's Tokens For APIs feature, allowing AI tools to access users' connected services (like Google, Microsoft, etc.) with proper authorization.\n\n- **Device Authorization Flow**: Support headless and input-constrained environments with the [Device Authorization Flow](https://auth0.com/docs/get-started/authentication-and-authorization-flow/device-authorization-flow), enabling secure user authentication without direct input capabilities.\n\n## Installation\n\n:::caution\n`@auth0/ai-genkit` is currently **under heavy development**. We strictly follow [Semantic Versioning (SemVer)](https://semver.org/), meaning all **breaking changes will only occur in major versions**. However, please note that during this early phase, **major versions may be released frequently** as the API evolves. We recommend locking versions when using this in production.\n:::\n\n```bash\nnpm install @auth0/ai @auth0/ai-genkit\n```\n\n## Initialization\n\nInitialize the SDK with your Auth0 credentials:\n\n```javascript\nimport { Auth0AI, setAIContext } from \"@auth0/ai-genkit\";\nimport { genkit } from \"genkit/beta\";\nimport { googleAI } from \"@genkit-ai/googleai\";\n\n// Initialize Genkit\nconst ai = genkit({\n plugins: [googleAI()],\n model: googleAI.model('gemini-2.5-flash'),\n});\n\n// Initialize Auth0AI\nconst auth0AI = new Auth0AI({\n // Alternatively, you can use the `AUTH0_DOMAIN`, `AUTH0_CLIENT_ID`, and `AUTH0_CLIENT_SECRET`\n // environment variables.\n auth0: {\n domain: \"YOUR_AUTH0_DOMAIN\",\n clientId: \"YOUR_AUTH0_CLIENT_ID\",\n clientSecret: \"YOUR_AUTH0_CLIENT_SECRET\",\n },\n\n // store: new MemoryStore(), // Optional: Use a custom store\n\n genkit: ai\n});\n```\n\n## Calling APIs\n\nThe \"Tokens for API\" feature of Auth0 allows you to exchange refresh tokens for access tokens for third-party APIs. This is useful when you want to use a federated connection (like Google, Facebook, etc.) to authenticate users and then use the access token to call the API on behalf of the user.\n\nFirst initialize the Federated Connection Authorizer as follows:\n\n```javascript\nconst withGoogleAccess = auth0AI.withTokenForConnection({\n // An optional function to specify where to retrieve the token\n // This is the default:\n refreshToken: async (params) => {\n return context.refreshToken;\n },\n // The connection name:\n connection: \"google-oauth2\",\n // The scopes to request:\n scopes: [\"https://www.googleapis.com/auth/calendar.freebusy\"],\n});\n```\n\nThen use the `withGoogleAccess` to wrap the tool and use `getAccessTokenForConnection` from the SDK to get the access token:\n\n```javascript\nimport { getAccessTokenForConnection } from \"@auth0/ai-genkit\";\nimport { FederatedConnectionError } from \"@auth0/ai/interrupts\";\nimport { addHours } from \"date-fns\";\nimport { z } from \"zod\";\n\nexport const checkCalendarTool = ai.defineTool(\n ...withGoogleAccess({\n name: \"check_user_calendar\",\n description:\n \"Check user availability on a given date time on their calendar\",\n inputSchema: z.object({\n date: z.coerce.date(),\n }),\n outputSchema: z.object({\n available: z.boolean(),\n }),\n },\n async ({ date }) => {\n const accessToken = getAccessTokenForConnection();\n const body = JSON.stringify({\n timeMin: date,\n timeMax: addHours(date, 1),\n timeZone: \"UTC\",\n items: [{ id: \"primary\" }],\n });\n\n const response = await fetch(url, {\n method: \"POST\",\n headers: {\n Authorization: `Bearer ${accessToken}`,\n \"Content-Type\": \"application/json\",\n },\n body,\n });\n\n if (!response.ok) {\n if (response.status === 401) {\n throw new FederatedConnectionError(\n `Authorization required to access the Federated Connection`\n );\n }\n throw new Error(\n `Invalid response from Google Calendar API: ${\n response.status\n } - ${await response.text()}`\n );\n }\n const busyResp = await response.json();\n return { available: busyResp.calendars.primary.busy.length === 0 };\n }\n));\n```\n\n## CIBA: Client-Initiated Backchannel Authentication\n\nCIBA (Client-Initiated Backchannel Authentication) enables secure, user-in-the-loop authentication for sensitive operations. This flow allows you to request user authorization asynchronously and resume execution once authorization is granted.\n\n```javascript\nconst buyStockAuthorizer = auth0AI.withAsyncUserConfirmation({\n // A callback to retrieve the userID from tool context.\n userID: (_params, config) => {\n return config.configurable?.user_id;\n },\n // The message the user will see on the notification\n bindingMessage: async ({ qty , ticker }) => {\n return `Confirm the purchase of ${qty} ${ticker}`;\n },\n // The scopes and audience to request\n audience: process.env[\"AUDIENCE\"],\n scopes: [\"stock:trade\"]\n});\n```\n\nThen wrap the tool as follows:\n\n```javascript\nimport { z } from \"zod\";\nimport { getCIBACredentials } from \"@auth0/ai-genkit\";\n\nexport const buyTool = ai.defineTool(\n ...buyStockAuthorizer({\n name: \"buy_stock\",\n description: \"Execute a stock purchase given stock ticker and quantity\",\n inputSchema: z.object({\n tradeID: z\n .string()\n .uuid()\n .describe(\"The unique identifier for the trade provided by the user\"),\n userID: z\n .string()\n .describe(\"The user ID of the user who created the conditional trade\"),\n ticker: z.string().describe(\"The stock ticker to trade\"),\n qty: z\n .number()\n .int()\n .positive()\n .describe(\"The quantity of shares to trade\"),\n }),\n outputSchema: z.string(),\n },\n async ({ ticker, qty }) => {\n const { accessToken } = getCIBACredentials();\n fetch(\"http://yourapi.com/buy\", {\n method: \"POST\",\n headers: {\n \"Content-Type\": \"application/json\",\n Authorization: `Bearer ${accessToken}`,\n },\n body: JSON.stringify({ ticker, qty }),\n });\n return `Purchased ${qty} shares of ${ticker}`;\n })\n);\n```\n\n### CIBA with RAR (Rich Authorization Requests)\n\nAuth0 supports RAR (Rich Authorization Requests) for CIBA. This allows you to provide additional authorization parameters to be displayed during the user confirmation request.\n\nWhen defining the tool authorizer, you can specify the `authorizationDetails` parameter to include detailed information about the authorization being requested:\n\n```javascript\nconst buyStockAuthorizer = auth0AI.withAsyncUserConfirmation({\n // A callback to retrieve the userID from tool context.\n userID: (_params, config) => {\n return config.configurable?.user_id;\n },\n // The message the user will see on the notification\n bindingMessage: async ({ qty , ticker }) => {\n return `Confirm the purchase of ${qty} ${ticker}`;\n },\n authorizationDetails: async ({ qty, ticker }) => {\n return [{ type: \"trade_authorization\", qty, ticker, action: \"buy\" }];\n },\n // The scopes and audience to request\n audience: process.env[\"AUDIENCE\"],\n scopes: [\"stock:trade\"]\n});\n```\n\nTo use RAR with CIBA, you need to [set up authorization details](https://auth0.com/docs/get-started/apis/configure-rich-authorization-requests) in your Auth0 tenant. This includes defining the authorization request parameters and their types. Additionally, the [Guardian SDK](https://auth0.com/docs/secure/multi-factor-authentication/auth0-guardian) is required to handle these authorization details in your authorizer app.\n\nFor more information on setting up RAR with CIBA, refer to:\n- [Configure Rich Authorization Requests (RAR)](https://auth0.com/docs/get-started/apis/configure-rich-authorization-requests)\n- [User Authorization with CIBA](https://auth0.com/docs/get-started/authentication-and-authorization-flow/client-initiated-backchannel-authentication-flow/user-authorization-with-ciba)\n\n## Device Flow Authorizer\n\nThe Device Flow Authorizer enables secure, user-in-the-loop authentication for devices or tools that cannot directly authenticate users. It uses the OAuth 2.0 Device Authorization Grant to request user authorization and resume execution once authorization is granted.\n\n```javascript\nimport { auth0 } from \"./auth0\";\n\nexport const deviceFlowAuthorizer = auth0AI.withDeviceAuthorizationFlow({\n // The scopes and audience to request\n scopes: [\"read:data\", \"write:data\"],\n audience: \"https://api.example.com\",\n});\n```\n\nThen wrap the tool as follows:\n\n```javascript\nimport { z } from \"zod\";\nimport { getDeviceAuthorizerCredentials } from \"@auth0/ai-genkit\";\n\nexport const fetchData = ai.defineTool(\n ...deviceFlowAuthorizer({\n name: \"fetch_data\",\n description: \"Fetch data from a secure API\",\n inputSchema: z.object({\n resourceID: z.string().describe(\"The ID of the resource to fetch\"),\n }),\n outputSchema: z.any(),\n },\n async ({ resourceID }) => {\n const credentials = getDeviceAuthorizerCredentials();\n const response = await fetch(`https://api.example.com/resource/${resourceID}`, {\n headers: {\n Authorization: `Bearer ${credentials.accessToken}`,\n },\n });\n\n if (!response.ok) {\n throw new Error(`Failed to fetch resource: ${response.statusText}`);\n }\n\n return await response.json();\n })\n);\n```\n\n## FGA\n\n```javascript\nimport { Auth0AI } from \"@auth0/ai-llamaindex\";\n\nconst auth0AI = new Auth0AI.FGA({\n apiScheme,\n apiHost,\n storeId,\n credentials: {\n method: CredentialsMethod.ClientCredentials,\n config: {\n apiTokenIssuer,\n clientId,\n clientSecret,\n },\n },\n});\n// Alternatively you can use env variables: `FGA_API_SCHEME`, `FGA_API_HOST`, `FGA_STORE_ID`, `FGA_API_TOKEN_ISSUER`, `FGA_CLIENT_ID` and `FGA_CLIENT_SECRET`\n```\n\nThen initialize the tool wrapper:\n\n```javascript\nconst authorizedTool = fgaAI.withFGA(\n {\n buildQuery: async ({ userID, doc }) => ({\n user: userID,\n object: doc,\n relation: \"read\",\n }),\n },\n myAITool\n);\n\n// Or create a wrapper to apply to tools later\nconst authorizer = fgaAI.withFGA({\n buildQuery: async ({ userID, doc }) => ({\n user: userID,\n object: doc,\n relation: \"read\",\n }),\n});\n\nconst authorizedTool = authorizer(myAITool);\n```\n\n:::note\nThe parameters given to the `buildQuery` function are the same provided to the tool's `execute` function.\n:::\n\n## RAG with FGA\n\nAuth0 AI can leverage OpenFGA to authorize RAG applications. The `FGARetriever` can be used to filter documents based on access control checks defined in Okta FGA. This retriever performs batch checks on retrieved documents, returning only the ones that pass the specified access criteria.\n\nCreate a Retriever instance using the `FGARetriever.create` method:\n\n```javascript\n// From examples/langchain/retrievers-with-fga\nimport { FGARetriever } from \"@auth0/ai-genkit/RAG\";\nimport { MemoryStore, RetrievalChain } from \"./helpers/memory-store\";\nimport { readDocuments } from \"./helpers/read-documents\";\n\nasync function main() {\n // UserID\n const user = \"user1\";\n const documents = await readDocuments();\n // 1. Call helper function to load LangChain MemoryStore\n const vectorStore = await MemoryStore.fromDocuments(documents);\n // 2. Call helper function to create a LangChain retrieval chain.\n const retrievalChain = await RetrievalChain.create({\n // 3. Decorate the retriever with the FGARetriever to check permissions.\n retriever: FGARetriever.create({\n retriever: vectorStore.asRetriever(),\n buildQuery: (doc) => ({\n user: `user:${user}`,\n object: `doc:${doc.metadata.id}`,\n relation: \"viewer\",\n }),\n }),\n });\n\n // 4. Execute the query\n const { answer } = await retrievalChain.query({\n query: \"Show me forecast for ZEKO?\",\n });\n\n console.log(answer);\n}\n\nmain().catch(console.error);\n```\n\n## Handling Interrupts\n\nAuth0 AI uses interrupts thoroughly and it will never block a Graph. Whenever an authorizer requires some user interaction the graph will throw a `ToolInterruptError` with data that allows the client the resumption of the flow.\n\nHandle the interrupts as follows:\n\n```javascript\nimport { AuthorizationPendingInterrupt } from '@auth0/ai/interrupts\"\n\nconst tools = [ myProtectedTool ];\n\nconst response = await ai.generate({\n tools,\n prompt: \"Transfer $1000 to account ABC123\",\n});\n\nconst interrupt = response.interrupts[0];\n\nif (interrupt && AuthorizationPendingInterrupt.is(interrupt.metadata)) {\n //do something\n const tool = tools.find(t => t.name === interrupt.metadata.toolCall.toolName);\n tool.restart(\n interrupt,\n //resume data if needed\n );\n}\n```\n\n:::note\nSince Auth0 AI has persistence on the backend you typically don't need to reattach interrupt's information when resuming.\n:::\n\n## Learn More\n\nFor more information, feedback, or to report issues, visit the [Auth0 AI for Genkit GitHub repository](https://github.com/auth0-lab/auth0-ai-js/tree/main/packages/ai-genkit).\n", + "text": "# Auth0 AI plugin\n\nThe Auth0 AI plugin (`@auth0/ai-genkit`) is an SDK for building secure AI-powered applications using [Auth0](https://www.auth0.ai/), [Okta FGA](https://docs.fga.dev/) and Genkit.\n\n## Features\n\n- **Authorization for RAG**: Securely filter documents using Okta FGA as a [retriever](https://js.langchain.com/docs/concepts/retrievers/) for RAG applications. This smart retriever performs efficient batch access control checks, ensuring users only see documents they have permission to access.\n\n- **Tool Authorization with FGA**: Protect AI tool execution with fine-grained authorization policies through Okta FGA integration, controlling which users can invoke specific tools based on custom authorization rules.\n\n- **Client Initiated Backchannel Authentication (CIBA)**: Implement secure, out-of-band user authorization for sensitive AI operations using the [CIBA standard](https://openid.net/specs/openid-client-initiated-backchannel-authentication-core-1_0.html), enabling user confirmation without disrupting the main interaction flow.\n\n- **Federated API Access**: Seamlessly connect to third-party services by leveraging Auth0's Tokens For APIs feature, allowing AI tools to access users' connected services (like Google, Microsoft, etc.) with proper authorization.\n\n- **Device Authorization Flow**: Support headless and input-constrained environments with the [Device Authorization Flow](https://auth0.com/docs/get-started/authentication-and-authorization-flow/device-authorization-flow), enabling secure user authentication without direct input capabilities.\n\n## Installation\n\n:::caution\n`@auth0/ai-genkit` is currently **under heavy development**. We strictly follow [Semantic Versioning (SemVer)](https://semver.org/), meaning all **breaking changes will only occur in major versions**. However, please note that during this early phase, **major versions may be released frequently** as the API evolves. We recommend locking versions when using this in production.\n:::\n\n```bash\nnpm install @auth0/ai @auth0/ai-genkit\n```\n\n## Initialization\n\nInitialize the SDK with your Auth0 credentials:\n\n```javascript\nimport { Auth0AI, setAIContext } from \"@auth0/ai-genkit\";\nimport { genkit } from \"genkit/beta\";\nimport { googleAI } from \"@genkit-ai/googleai\";\n\n// Initialize Genkit\nconst ai = genkit({\n plugins: [googleAI()],\n model: googleAI.model('gemini-2.5-flash'),\n});\n\n// Initialize Auth0AI\nconst auth0AI = new Auth0AI({\n // Alternatively, you can use the `AUTH0_DOMAIN`, `AUTH0_CLIENT_ID`, and `AUTH0_CLIENT_SECRET`\n // environment variables.\n auth0: {\n domain: \"YOUR_AUTH0_DOMAIN\",\n clientId: \"YOUR_AUTH0_CLIENT_ID\",\n clientSecret: \"YOUR_AUTH0_CLIENT_SECRET\",\n },\n\n // store: new MemoryStore(), // Optional: Use a custom store\n\n genkit: ai\n});\n```\n\n## Calling APIs\n\nThe \"Tokens for API\" feature of Auth0 allows you to exchange refresh tokens for access tokens for third-party APIs. This is useful when you want to use a federated connection (like Google, Facebook, etc.) to authenticate users and then use the access token to call the API on behalf of the user.\n\nFirst initialize the Federated Connection Authorizer as follows:\n\n```javascript\nconst withGoogleAccess = auth0AI.withTokenForConnection({\n // An optional function to specify where to retrieve the token\n // This is the default:\n refreshToken: async (params) => {\n return context.refreshToken;\n },\n // The connection name:\n connection: \"google-oauth2\",\n // The scopes to request:\n scopes: [\"https://www.googleapis.com/auth/calendar.freebusy\"],\n});\n```\n\nThen use the `withGoogleAccess` to wrap the tool and use `getAccessTokenForConnection` from the SDK to get the access token:\n\n```javascript\nimport { getAccessTokenForConnection } from \"@auth0/ai-genkit\";\nimport { FederatedConnectionError } from \"@auth0/ai/interrupts\";\nimport { addHours } from \"date-fns\";\nimport { z } from \"zod\";\n\nexport const checkCalendarTool = ai.defineTool(\n ...withGoogleAccess({\n name: \"check_user_calendar\",\n description:\n \"Check user availability on a given date time on their calendar\",\n inputSchema: z.object({\n date: z.coerce.date(),\n }),\n outputSchema: z.object({\n available: z.boolean(),\n }),\n },\n async ({ date }) => {\n const accessToken = getAccessTokenForConnection();\n const body = JSON.stringify({\n timeMin: date,\n timeMax: addHours(date, 1),\n timeZone: \"UTC\",\n items: [{ id: \"primary\" }],\n });\n\n const response = await fetch(url, {\n method: \"POST\",\n headers: {\n Authorization: `Bearer ${accessToken}`,\n \"Content-Type\": \"application/json\",\n },\n body,\n });\n\n if (!response.ok) {\n if (response.status === 401) {\n throw new FederatedConnectionError(\n `Authorization required to access the Federated Connection`\n );\n }\n throw new Error(\n `Invalid response from Google Calendar API: ${\n response.status\n } - ${await response.text()}`\n );\n }\n const busyResp = await response.json();\n return { available: busyResp.calendars.primary.busy.length === 0 };\n }\n));\n```\n\n## CIBA: Client-Initiated Backchannel Authentication\n\nCIBA (Client-Initiated Backchannel Authentication) enables secure, user-in-the-loop authentication for sensitive operations. This flow allows you to request user authorization asynchronously and resume execution once authorization is granted.\n\n```javascript\nconst buyStockAuthorizer = auth0AI.withAsyncUserConfirmation({\n // A callback to retrieve the userID from tool context.\n userID: (_params, config) => {\n return config.configurable?.user_id;\n },\n // The message the user will see on the notification\n bindingMessage: async ({ qty , ticker }) => {\n return `Confirm the purchase of ${qty} ${ticker}`;\n },\n // The scopes and audience to request\n audience: process.env[\"AUDIENCE\"],\n scopes: [\"stock:trade\"]\n});\n```\n\nThen wrap the tool as follows:\n\n```javascript\nimport { z } from \"zod\";\nimport { getCIBACredentials } from \"@auth0/ai-genkit\";\n\nexport const buyTool = ai.defineTool(\n ...buyStockAuthorizer({\n name: \"buy_stock\",\n description: \"Execute a stock purchase given stock ticker and quantity\",\n inputSchema: z.object({\n tradeID: z\n .string()\n .uuid()\n .describe(\"The unique identifier for the trade provided by the user\"),\n userID: z\n .string()\n .describe(\"The user ID of the user who created the conditional trade\"),\n ticker: z.string().describe(\"The stock ticker to trade\"),\n qty: z\n .number()\n .int()\n .positive()\n .describe(\"The quantity of shares to trade\"),\n }),\n outputSchema: z.string(),\n },\n async ({ ticker, qty }) => {\n const { accessToken } = getCIBACredentials();\n fetch(\"http://yourapi.com/buy\", {\n method: \"POST\",\n headers: {\n \"Content-Type\": \"application/json\",\n Authorization: `Bearer ${accessToken}`,\n },\n body: JSON.stringify({ ticker, qty }),\n });\n return `Purchased ${qty} shares of ${ticker}`;\n })\n);\n```\n\n### CIBA with RAR (Rich Authorization Requests)\n\nAuth0 supports RAR (Rich Authorization Requests) for CIBA. This allows you to provide additional authorization parameters to be displayed during the user confirmation request.\n\nWhen defining the tool authorizer, you can specify the `authorizationDetails` parameter to include detailed information about the authorization being requested:\n\n```javascript\nconst buyStockAuthorizer = auth0AI.withAsyncUserConfirmation({\n // A callback to retrieve the userID from tool context.\n userID: (_params, config) => {\n return config.configurable?.user_id;\n },\n // The message the user will see on the notification\n bindingMessage: async ({ qty , ticker }) => {\n return `Confirm the purchase of ${qty} ${ticker}`;\n },\n authorizationDetails: async ({ qty, ticker }) => {\n return [{ type: \"trade_authorization\", qty, ticker, action: \"buy\" }];\n },\n // The scopes and audience to request\n audience: process.env[\"AUDIENCE\"],\n scopes: [\"stock:trade\"]\n});\n```\n\nTo use RAR with CIBA, you need to [set up authorization details](https://auth0.com/docs/get-started/apis/configure-rich-authorization-requests) in your Auth0 tenant. This includes defining the authorization request parameters and their types. Additionally, the [Guardian SDK](https://auth0.com/docs/secure/multi-factor-authentication/auth0-guardian) is required to handle these authorization details in your authorizer app.\n\nFor more information on setting up RAR with CIBA, refer to:\n- [Configure Rich Authorization Requests (RAR)](https://auth0.com/docs/get-started/apis/configure-rich-authorization-requests)\n- [User Authorization with CIBA](https://auth0.com/docs/get-started/authentication-and-authorization-flow/client-initiated-backchannel-authentication-flow/user-authorization-with-ciba)\n\n## Device Flow Authorizer\n\nThe Device Flow Authorizer enables secure, user-in-the-loop authentication for devices or tools that cannot directly authenticate users. It uses the OAuth 2.0 Device Authorization Grant to request user authorization and resume execution once authorization is granted.\n\n```javascript\nimport { auth0 } from \"./auth0\";\n\nexport const deviceFlowAuthorizer = auth0AI.withDeviceAuthorizationFlow({\n // The scopes and audience to request\n scopes: [\"read:data\", \"write:data\"],\n audience: \"https://api.example.com\",\n});\n```\n\nThen wrap the tool as follows:\n\n```javascript\nimport { z } from \"zod\";\nimport { getDeviceAuthorizerCredentials } from \"@auth0/ai-genkit\";\n\nexport const fetchData = ai.defineTool(\n ...deviceFlowAuthorizer({\n name: \"fetch_data\",\n description: \"Fetch data from a secure API\",\n inputSchema: z.object({\n resourceID: z.string().describe(\"The ID of the resource to fetch\"),\n }),\n outputSchema: z.any(),\n },\n async ({ resourceID }) => {\n const credentials = getDeviceAuthorizerCredentials();\n const response = await fetch(`https://api.example.com/resource/${resourceID}`, {\n headers: {\n Authorization: `Bearer ${credentials.accessToken}`,\n },\n });\n\n if (!response.ok) {\n throw new Error(`Failed to fetch resource: ${response.statusText}`);\n }\n\n return await response.json();\n })\n);\n```\n\n## FGA\n\n```javascript\nimport { Auth0AI } from \"@auth0/ai-llamaindex\";\n\nconst auth0AI = new Auth0AI.FGA({\n apiScheme,\n apiHost,\n storeId,\n credentials: {\n method: CredentialsMethod.ClientCredentials,\n config: {\n apiTokenIssuer,\n clientId,\n clientSecret,\n },\n },\n});\n// Alternatively you can use env variables: `FGA_API_SCHEME`, `FGA_API_HOST`, `FGA_STORE_ID`, `FGA_API_TOKEN_ISSUER`, `FGA_CLIENT_ID` and `FGA_CLIENT_SECRET`\n```\n\nThen initialize the tool wrapper:\n\n```javascript\nconst authorizedTool = fgaAI.withFGA(\n {\n buildQuery: async ({ userID, doc }) => ({\n user: userID,\n object: doc,\n relation: \"read\",\n }),\n },\n myAITool\n);\n\n// Or create a wrapper to apply to tools later\nconst authorizer = fgaAI.withFGA({\n buildQuery: async ({ userID, doc }) => ({\n user: userID,\n object: doc,\n relation: \"read\",\n }),\n});\n\nconst authorizedTool = authorizer(myAITool);\n```\n\n:::note\nThe parameters given to the `buildQuery` function are the same provided to the tool's `execute` function.\n:::\n\n## RAG with FGA\n\nAuth0 AI can leverage OpenFGA to authorize RAG applications. The `FGARetriever` can be used to filter documents based on access control checks defined in Okta FGA. This retriever performs batch checks on retrieved documents, returning only the ones that pass the specified access criteria.\n\nCreate a Retriever instance using the `FGARetriever.create` method:\n\n```javascript\n// From examples/langchain/retrievers-with-fga\nimport { FGARetriever } from \"@auth0/ai-genkit/RAG\";\nimport { MemoryStore, RetrievalChain } from \"./helpers/memory-store\";\nimport { readDocuments } from \"./helpers/read-documents\";\n\nasync function main() {\n // UserID\n const user = \"user1\";\n const documents = await readDocuments();\n // 1. Call helper function to load LangChain MemoryStore\n const vectorStore = await MemoryStore.fromDocuments(documents);\n // 2. Call helper function to create a LangChain retrieval chain.\n const retrievalChain = await RetrievalChain.create({\n // 3. Decorate the retriever with the FGARetriever to check permissions.\n retriever: FGARetriever.create({\n retriever: vectorStore.asRetriever(),\n buildQuery: (doc) => ({\n user: `user:${user}`,\n object: `doc:${doc.metadata.id}`,\n relation: \"viewer\",\n }),\n }),\n });\n\n // 4. Execute the query\n const { answer } = await retrievalChain.query({\n query: \"Show me forecast for ZEKO?\",\n });\n\n console.log(answer);\n}\n\nmain().catch(console.error);\n```\n\n## Handling Interrupts\n\nAuth0 AI uses interrupts thoroughly and it will never block a Graph. Whenever an authorizer requires some user interaction the graph will throw a `ToolInterruptError` with data that allows the client the resumption of the flow.\n\nHandle the interrupts as follows:\n\n```javascript\nimport { AuthorizationPendingInterrupt } from '@auth0/ai/interrupts\"\n\nconst tools = [ myProtectedTool ];\n\nconst response = await ai.generate({\n tools,\n prompt: \"Transfer $1000 to account ABC123\",\n});\n\nconst interrupt = response.interrupts[0];\n\nif (interrupt && AuthorizationPendingInterrupt.is(interrupt.metadata)) {\n //do something\n const tool = tools.find(t => t.name === interrupt.metadata.toolCall.toolName);\n tool.restart(\n interrupt,\n //resume data if needed\n );\n}\n```\n\n:::note\nSince Auth0 AI has persistence on the backend you typically don't need to reattach interrupt's information when resuming.\n:::\n\n## Learn More\n\nFor more information, feedback, or to report issues, visit the [Auth0 AI for Genkit GitHub repository](https://github.com/auth0-lab/auth0-ai-js/tree/main/packages/ai-genkit).\n", "title": "Auth0 AI plugin", - "lang": "js" + "description": "This document describes the Auth0 AI plugin for Genkit, which provides features for building secure AI-powered applications using Auth0, Okta FGA, and Genkit.", + "lang": "js", + "headers": "## Features\n## Installation\n## Initialization\n## Calling APIs\n## CIBA: Client-Initiated Backchannel Authentication\n### CIBA with RAR (Rich Authorization Requests)\n## Device Flow Authorizer\n## FGA\n## RAG with FGA\n## Handling Interrupts\n## Learn More\n" }, "js/plugins/chroma.md": { - "text": "The Chroma plugin provides indexer and retriever implementations that use the\n[Chroma](https://docs.trychroma.com/) vector database in client/server mode.\n\n## Installation\n\n```bash\nnpm install genkitx-chromadb\n```\n\n## Configuration\n\nTo use this plugin, specify it when you initialize Genkit:\n\n```ts\nimport { genkit } from 'genkit';\nimport { chroma } from 'genkitx-chromadb';\n\nconst ai = genkit({\n plugins: [\n chroma([\n {\n collectionName: 'bob_collection',\n embedder: textEmbedding004,\n },\n ]),\n ],\n});\n```\n\nYou must specify a Chroma collection and the embedding model you want to use. In\naddition, there are two optional parameters:\n\n- `clientParams`: If you're not running your Chroma server on the same machine as your Genkit flow, you need to specify auth options, or you're otherwise not running a default Chroma server configuration, you can specify a Chroma [`ChromaClientParams` object](https://docs.trychroma.com/js_reference/Client) to pass to the Chroma client:\n\n ```ts\n clientParams: {\n path: \"http://192.168.10.42:8000\",\n }\n ```\n\n- `embedderOptions`: Use this parameter to pass options to the embedder:\n\n ```ts\n embedderOptions: { taskType: 'RETRIEVAL_DOCUMENT' },\n ```\n\n## Usage\n\nImport retriever and indexer references like so:\n\n```ts\nimport { chromaRetrieverRef } from 'genkitx-chromadb';\nimport { chromaIndexerRef } from 'genkitx-chromadb';\n```\n\nThen, use the references with `ai.retrieve()` and `ai.index()`:\n\n```ts\n// To use the index you configured when you loaded the plugin:\nlet docs = await ai.retrieve({ retriever: chromaRetrieverRef, query });\n\n// To specify an index:\nexport const bobFactsRetriever = chromaRetrieverRef({\n collectionName: 'bob-facts',\n});\ndocs = await ai.retrieve({ retriever: bobFactsRetriever, query });\n```\n\n```ts\n// To use the index you configured when you loaded the plugin:\nawait ai.index({ indexer: chromaIndexerRef, documents });\n\n// To specify an index:\nexport const bobFactsIndexer = chromaIndexerRef({\n collectionName: 'bob-facts',\n});\nawait ai.index({ indexer: bobFactsIndexer, documents });\n```\n\nSee the [Retrieval-augmented generation](/docs/rag) page for a general\ndiscussion on indexers and retrievers.\n", + "text": "# Chroma plugin\n\nThe Chroma plugin provides indexer and retriever implementations that use the\n[Chroma](https://docs.trychroma.com/) vector database in client/server mode.\n\n## Installation\n\n```bash\nnpm install genkitx-chromadb\n```\n\n## Configuration\n\nTo use this plugin, specify it when you initialize Genkit:\n\n```ts\nimport { genkit } from 'genkit';\nimport { chroma } from 'genkitx-chromadb';\nimport { googleAI } from '@genkit-ai/googleai';\n\nconst ai = genkit({\n plugins: [\n chroma([\n {\n collectionName: 'bob_collection',\n embedder: googleAI.embedder('gemini-embedding-001'),\n },\n ]),\n ],\n});\n```\n\nYou must specify a Chroma collection and the embedding model you want to use. In\naddition, there are two optional parameters:\n\n- `clientParams`: If you're not running your Chroma server on the same machine as your Genkit flow, you need to specify auth options, or you're otherwise not running a default Chroma server configuration, you can specify a Chroma [`ChromaClientParams` object](https://docs.trychroma.com/js_reference/Client) to pass to the Chroma client:\n\n ```ts\n clientParams: {\n path: \"http://192.168.10.42:8000\",\n }\n ```\n\n- `embedderOptions`: Use this parameter to pass options to the embedder:\n\n ```ts\n embedderOptions: { taskType: 'RETRIEVAL_DOCUMENT' },\n ```\n\n## Usage\n\nImport retriever and indexer references like so:\n\n```ts\nimport { chromaRetrieverRef } from 'genkitx-chromadb';\nimport { chromaIndexerRef } from 'genkitx-chromadb';\n```\n\nThen, use the references with `ai.retrieve()` and `ai.index()`:\n\n```ts\n// To use the index you configured when you loaded the plugin:\nlet docs = await ai.retrieve({ retriever: chromaRetrieverRef, query });\n\n// To specify an index:\nexport const bobFactsRetriever = chromaRetrieverRef({\n collectionName: 'bob-facts',\n});\ndocs = await ai.retrieve({ retriever: bobFactsRetriever, query });\n```\n\n```ts\n// To use the index you configured when you loaded the plugin:\nawait ai.index({ indexer: chromaIndexerRef, documents });\n\n// To specify an index:\nexport const bobFactsIndexer = chromaIndexerRef({\n collectionName: 'bob-facts',\n});\nawait ai.index({ indexer: bobFactsIndexer, documents });\n```\n\nSee the [Retrieval-augmented generation](/docs/rag) page for a general\ndiscussion on indexers and retrievers.\n", "title": "Chroma plugin", - "lang": "js" + "description": "This document describes the Chroma plugin for Genkit, which provides indexer and retriever implementations for the Chroma vector database in client/server mode.", + "lang": "js", + "headers": "## Installation\n## Configuration\n## Usage\n" + }, + "js/plugins/cloud-sql-pg.md": { + "text": "# Cloud SQL for PostgreSQL plugin\n\nThe Cloud SQL for PostgreSQL plugin provides indexer and retriever implementations that use PostgreSQL with the pgvector extension for vector similarity search.\n\n## Installation\n\n```posix-terminal\nnpm i --save genkitx-cloud-sql-pg\n```\n\n## Configuration\n\nTo use this plugin, first create a `PostgresEngine` instance:\n\n```ts\nimport { PostgresEngine } from 'genkitx-cloud-sql-pg';\n\n// Create PostgresEngine instance\nconst engine = await PostgresEngine.fromInstance('my-project', 'us-central1', 'my-instance', 'my-database');\n\n// Create the vector store table\nawait engine.initVectorstoreTable('my-documents', 768);\n\n// Or create a custom vector store table\nawait engine.initVectorstoreTable('my-documents', 768, {\n schemaName: 'public',\n contentColumn: 'content',\n embeddingColumn: 'embedding',\n idColumn: 'custom_id', // Custom ID column name\n metadataColumns: [\n { name: 'source', dataType: 'TEXT' },\n { name: 'category', dataType: 'TEXT' }\n ],\n metadataJsonColumn: 'metadata',\n storeMetadata: true,\n overwriteExisting: true\n});\n```\n\nThen, specify the plugin when you initialize Genkit:\n\n```ts\nimport { genkit } from 'genkit';\nimport { postgres } from 'genkitx-cloud-sql-pg';\nimport { vertexAI } from '@genkit-ai/vertexai';\n\nconst ai = genkit({\n plugins: [\n postgres([\n {\n tableName: 'my-documents',\n engine: engine,\n embedder: vertexAI.embedder('gemini-embedding-001'),\n // Use additional fields to connect to a custom vector store table\n // schemaName: 'public',\n // contentColumn: 'custom_content',\n // embeddingColumn: 'custom_embedding',\n // idColumn: 'custom_id', // Match the ID column from table creation\n // metadataColumns: ['source', 'category'],\n // metadataJsonColumn: 'my_json_metadata',\n },\n ]),\n ],\n});\n\n// To use the table you configured when you loaded the plugin:\nawait ai.index({\n indexer: postgresIndexerRef,\n documents: [\n {\n content: [{ text: \"The product features include...\" }],\n metadata: {\n source: \"website\",\n category: \"product-docs\",\n custom_id: \"doc-123\" // This will be used as the document ID\n }\n }\n ]\n});\n\n// To retrieve from the configured table:\nconst query = \"What are the key features of the product?\";\nlet docs = await ai.retrieve({\n retriever: postgresRetrieverRef,\n query,\n options: {\n k: 5,\n filter: {\n category: 'product-docs',\n source: 'website'\n }\n }\n});\n```\n\n## Usage\n\nImport retriever and indexer references like so:\n\n```ts\nimport { postgresRetrieverRef, postgresIndexerRef } from 'genkitx-cloud-sql-pg';\n```\n\n### Index Documents\n\nYou can create reusable references for your indexers:\n\n```ts\nexport const myDocumentsIndexer = postgresIndexerRef({\n tableName: 'my-custom-documents',\n idColumn: 'custom_id',\n metadataColumns: ['source', 'category']\n});\n```\n\nThen use them to index documents:\n\n```ts\n// Index with custom ID from metadata\nconst docWithCustomId = new Document({\n content: [{ text: 'Document with custom ID' }],\n metadata: {\n source: 'test',\n category: 'docs',\n custom_id: 'custom-123'\n }\n});\n\nawait ai.index({\n indexer: myDocumentsIndexer,\n documents: [docWithCustomId]\n});\n\n// Index with custom batch size\nawait ai.index({\n indexer: myDocumentsIndexer,\n documents: [\n {\n content: [{ text: \"The product features include...\" }],\n metadata: {\n source: \"website\",\n category: \"product-docs\",\n custom_id: \"doc-456\"\n }\n }\n ],\n options: { batchSize: 10 }\n});\n```\n\n### Indexing Options\nThe indexer supports:\n\n* batchSize: Number of documents to process at once\n* Custom ID and metadata handling through table configuration\n\n\n### Retrieve Documents\n\nYou can create reusable references for your retrievers:\n\n```ts\nexport const myDocumentsRetriever = postgresRetrieverRef({\n tableName: 'my-documents',\n idColumn: 'custom_id',\n metadataColumns: ['source', 'category']\n});\n```\n\n\nThen use them to retrieve documents:\n\n```ts\n// Basic retrieval\nconst query = \"What are the key features of the product?\";\nlet docs = await ai.retrieve({\n retriever: myDocumentsRetriever,\n query,\n options: {\n k: 5, // Number of documents to return (default: 4, max: 1000)\n filter: \"source = 'website'\" // Optional SQL WHERE clause\n }\n});\n\n// Access retrieved documents and their metadata\nconsole.log(docs.documents[0].content); // Document content\nconsole.log(docs.documents[0].metadata.source); // Metadata fields\nconsole.log(docs.documents[0].metadata.category);\n```\n\n\n#### Retriever Options\n\nThe retriever supports the following options:\n\nk: Number of documents to return (default: 4, max: 1000)\nfilter: SQL WHERE clause to filter results (e.g., \"category = 'docs' AND source = 'website'\")\n\n#### Distance Strategies\n\nThe retriever supports different distance strategies for vector similarity search:\n\n```ts\nimport { DistanceStrategy } from 'genkitx-cloud-sql-pg';\n\n// Configure retriever with specific distance strategy\nconst myDocumentsRetriever = postgresRetrieverRef({\n tableName: 'my-documents',\n distanceStrategy: DistanceStrategy.COSINE_DISTANCE // or EUCLIDEAN_DISTANCE\n});\n```\n\n\nAvailable strategies:\n* COSINE_DISTANCE: Cosine similarity (default)\n* EUCLIDEAN_DISTANCE: Euclidean distance\n* DOT_PRODUCT: Dot product similarity\n\n#### Metadata Handling\n\nThe retriever preserves all metadata fields when returning documents. You can access both individual metadata columns and the JSON metadata column:\n\n```ts\n// Example 1: Search for product documentation\nconst productQuery = \"How do I configure the API rate limits?\";\nconst productDocs = await ai.retrieve({\n retriever: myDocumentsRetriever,\n query: productQuery,\n options: {\n k: 3,\n filter: \"category = 'api-docs' AND source = 'product-manual'\"\n }\n});\n\n// Example 2: Search for customer support articles\nconst supportQuery = \"What are the troubleshooting steps for connection issues?\";\nconst supportDocs = await ai.retrieve({\n retriever: myDocumentsRetriever,\n query: supportQuery,\n options: {\n k: 5,\n filter: \"category = 'troubleshooting' AND source = 'support-kb'\"\n }\n});\n\n// Access retrieved documents and their metadata\nconsole.log(productDocs.documents[0].content); // Document content\nconsole.log(productDocs.documents[0].metadata.source); // e.g., \"product-manual\"\nconsole.log(productDocs.documents[0].metadata.category); // e.g., \"api-docs\"\nconsole.log(productDocs.documents[0].metadata.lastUpdated); // e.g., \"2024-03-15\"\n```\n\nSee the [Retrieval-augmented generation](http://../rag.md) page for a general discussion on indexers and retrievers.\n", + "title": "Cloud SQL for PostgreSQL plugin", + "description": "This document describes the Cloud SQL for PostgreSQL plugin for Genkit, providing indexer and retriever implementations that use PostgreSQL with the pgvector extension for vector similarity search.", + "lang": "js", + "headers": "## Installation\n## Configuration\n## Usage\n### Index Documents\n### Indexing Options\n### Retrieve Documents\n#### Retriever Options\n#### Distance Strategies\n#### Metadata Handling\n" + }, + "js/plugins/compat-oai.md": { + "text": "# OpenAI-Compatible Plugin\n\nThe `@genkit-ai/compat-oai` package provides plugins for services that are compatible with the OpenAI API specification. This includes official OpenAI services as well as other model providers and local servers that expose an OpenAI-compatible endpoint.\n\nThis package contains four main exports:\n\n- `openAICompatible`: A general-purpose plugin for any OpenAI-compatible service.\n- [`openAI`](/docs/plugins/openai): A pre-configured plugin for OpenAI's own services (GPT models, DALL-E, etc.).\n- [`xai`](/docs/plugins/xai): A pre-configured plugin for xAI (Grok) models.\n- [`deepSeek`](/docs/plugins/deepseek): A pre-configured plugin for DeepSeek models.\n\n## Installation\n\n```bash\nnpm install @genkit-ai/compat-oai\n```\n\n## General-Purpose OpenAI-Compatible Plugin\n\nYou can use the `openAICompatible` plugin factory to connect to any service that exposes an OpenAI-compatible API. This is useful for custom or self-hosted models, such as those served via [Ollama](https://ollama.com/).\n\nTo use this plugin, import `openAICompatible` and specify it in your Genkit configuration. You must provide a unique `name` for each instance, and client options like `baseURL` and `apiKey`.\n\n### Configuration\n\nThe `openAICompatible` plugin takes an options object with the following parameters:\n\n- `name`: (Required) A unique name for the plugin instance (e.g., `'ollama'`, `'my-custom-llm'`).\n- `apiKey`: The API key for the service. For local services, this can often be a placeholder string like `'ollama'`.\n- `baseURL`: The base URL of the OpenAI-compatible API endpoint (e.g., `'http://localhost:11434/v1'` for Ollama).\n- Other options from the OpenAI Node.js SDK's `ClientOptions` can also be included, such as `timeout` or `defaultHeaders`.\n\nHere's an example of how to configure the plugin for a local Ollama instance:\n\n```ts\nimport { genkit } from 'genkit';\nimport { openAICompatible } from '@genkit-ai/compat-oai';\n\nexport const ai = genkit({\n plugins: [\n openAICompatible({\n name: 'localLlama',\n apiKey: 'ollama', // Required, but can be a placeholder for local servers\n baseURL: 'http://localhost:11434/v1', // Example for Ollama\n }),\n ],\n});\n```\n\n### Usage\n\nOnce configured, you need to define a `modelRef` to interact with your custom model. A `modelRef` is a reference that tells Genkit how to use a specific model, including its name and any supported features.\n\nThe model name in the `modelRef` should be prefixed with the `name` you gave the plugin instance, followed by a `/` and the model ID from the service.\n\n```ts\nimport { genkit, modelRef, z } from 'genkit';\nimport { openAICompatible } from '@genkit-ai/compat-oai';\n\n// In your Genkit config...\nconst ai = genkit({\n plugins: [\n openAICompatible({\n name: 'localLlama',\n apiKey: 'ollama',\n baseURL: 'http://localhost:11434/v1',\n }),\n ],\n});\n\n// Define a reference to your model\nexport const myLocalModel = modelRef({\n name: 'localLlama/llama3',\n // You can specify model-specific configuration here if needed.\n // For many custom models, Genkit's default capabilities are sufficient.\n});\n\n// Use the model in a flow\nexport const localLlamaFlow = ai.defineFlow(\n {\n name: 'localLlamaFlow',\n inputSchema: z.object({ subject: z.string() }),\n outputSchema: z.object({ joke: z.string() }),\n },\n async ({ subject }) => {\n const llmResponse = await ai.generate({\n model: myLocalModel,\n prompt: `Tell me a joke about ${subject}.`,\n });\n return { joke: llmResponse.text };\n }\n);\n```\n\nIn this example, `'localLlama/llama3'` tells Genkit to use the `llama3` model provided by the `localLlama` plugin instance.\n\n### Passing Model Configuration\n\nYou can pass configuration options to the model in the `generate` call. The available options depend on the specific model you are using. \nCommon options include `temperature`, `maxOutputTokens`, etc. These are passed through to the underlying service.\n\n```ts\nconst llmResponse = await ai.generate({\n model: myLocalModel,\n prompt: 'Tell me a joke about a llama.',\n config: {\n temperature: 0.9,\n },\n});\n```\n", + "title": "OpenAI-Compatible Plugin", + "description": "Learn how to configure and use Genkit OpenAI-comptiable plugin to access models through any OpenAI-compatible API.", + "lang": "js", + "headers": "## Installation\n## General-Purpose OpenAI-Compatible Plugin\n### Configuration\n### Usage\n### Passing Model Configuration\n" + }, + "js/plugins/deepseek.md": { + "text": "# DeepSeek Plugin\n\nThe `@genkit-ai/compat-oai` package includes a pre-configured plugin for [DeepSeek](https://www.deepseek.com/) models.\n\n:::note\nThe DeepSeek plugin is built on top of the `openAICompatible` plugin. It is pre-configured for DeepSeek's API endpoints, so you don't need to provide a `baseURL`.\n:::\n\n## Installation\n\n```bash\nnpm install @genkit-ai/compat-oai\n```\n\n## Configuration\n\nTo use this plugin, import `deepSeek` and specify it when you initialize Genkit.\n\n```ts\nimport { genkit } from 'genkit';\nimport { deepSeek } from '@genkit-ai/compat-oai/deepseek';\n\nexport const ai = genkit({\n plugins: [deepSeek()],\n});\n```\n\nYou must provide an API key from DeepSeek. You can get an API key from your [DeepSeek account settings](https://platform.deepseek.com/).\n\nConfigure the plugin to use your API key by doing one of the following:\n\n- Set the `DEEPSEEK_API_KEY` environment variable to your API key.\n- Specify the API key when you initialize the plugin:\n\n ```ts\n deepSeek({ apiKey: yourKey });\n ```\n\nAs always, avoid embedding API keys directly in your code.\n\n## Usage\n\nUse the `deepSeek.model()` helper to reference a DeepSeek model.\n\n```ts\nimport { genkit, z } from 'genkit';\nimport { deepSeek } from '@genkit-ai/compat-oai/deepseek';\n\nconst ai = genkit({\n plugins: [deepSeek({ apiKey: process.env.DEEPSEEK_API_KEY })],\n});\n\nexport const deepseekFlow = ai.defineFlow(\n {\n name: 'deepseekFlow',\n inputSchema: z.object({ subject: z.string() }),\n outputSchema: z.object({ information: z.string() }),\n },\n async ({ subject }) => {\n // Reference a model\n const deepseekChat = deepSeek.model('deepseek-chat');\n\n // Use it in a generate call\n const llmResponse = await ai.generate({\n model: deepseekChat,\n prompt: `Tell me something about ${subject}.`,\n });\n\n return { information: llmResponse.text };\n },\n);\n```\n\nYou can also pass model-specific configuration:\n\n```ts\nconst llmResponse = await ai.generate({\n model: deepSeek.model('deepseek-chat'),\n prompt: 'Tell me something about deep learning.',\n config: {\n temperature: 0.8,\n maxTokens: 1024,\n },\n});\n```\n\n## Advanced usage\n\n### Passthrough configuration\n\nYou can pass configuration options that are not defined in the plugin's custom config schema. This\npermits you to access new models and features without having to update your Genkit version.\n\n```ts\nimport { genkit } from 'genkit';\nimport { deepSeek } from '@genkit-ai/compat-oai/deepSeek';\n\nconst ai = genkit({\n plugins: [deepSeek()],\n});\n\nconst llmResponse = await ai.generate({\n prompt: `Tell me a cool story`,\n model: deepSeek.model('deepseek-new'), // hypothetical new model\n config: {\n new_feature_parameter: ... // hypothetical config needed for new model\n },\n});\n```\n\nGenkit passes this configuration as-is to the DeepSeek API giving you access to the new model features.\nNote that the field name and types are not validated by Genkit and should match the DeepSeek API\nspecification to work.\n", + "title": "DeepSeek Plugin", + "description": "Learn how to configure and use Genkit DeepSeek plugin to access DeepSeek models.", + "lang": "js", + "headers": "## Installation\n## Configuration\n## Usage\n## Advanced usage\n### Passthrough configuration\n" + }, + "js/plugins/express.md": { + "text": "# Express plugin\n\nThe Genkit Express plugin provides utilities for conveniently exposing Genkit flows and actions via an Express HTTP server as REST APIs. This allows you to integrate your Genkit applications with existing Express-based backends or deploy them to any platform that can serve an Express.js app.\n\n## Installation\n\nTo use this plugin, install it in your project:\n\n```bash\nnpm i @genkit-ai/express\n```\n\n## Usage\n\nYou can expose your Genkit flows and actions as REST API endpoints using the `expressHandler` function.\n\nFirst, define your Genkit flow:\n\n```typescript\nimport { genkit, z } from 'genkit';\nimport { googleAI } from '@genkit-ai/googleai';\nimport { expressHandler } from '@genkit-ai/express';\nimport express from 'express';\n\nconst ai = genkit({\n plugins: [googleAI()],\n model: googleAI.model('gemini-2.5-flash'),\n});\n\nconst simpleFlow = ai.defineFlow(\n {\n name: 'simpleFlow',\n inputSchema: z.object({ input: z.string() }),\n outputSchema: z.object({ output: z.string() }),\n },\n async ({ input }, { sendChunk }) => {\n const { text } = await ai.generate({\n prompt: input,\n onChunk: (c) => sendChunk(c.text),\n });\n return { output: text };\n },\n);\n\nconst app = express();\napp.use(express.json());\n\napp.post('/simpleFlow', expressHandler(simpleFlow));\n\napp.listen(8080, () => {\n console.log('Express server listening on port 8080');\n});\n```\n\n### Accessing flows from the client\n\nFlows and actions exposed using the `expressHandler` function can be accessed using the `genkit/beta/client` library:\n\n```typescript\nimport { runFlow, streamFlow } from 'genkit/beta/client';\n\n// Example: Running a flow\nconst result = await runFlow({\n url: `http://localhost:8080/simpleFlow`,\n input: { input: 'say hello' },\n});\nconsole.log(result); // { output: \"hello\" }\n\n// Example: Streaming a flow\nconst streamResult = streamFlow({\n url: `http://localhost:8080/simpleFlow`,\n input: { input: 'say hello' },\n});\nfor await (const chunk of streamResult.stream) {\n console.log(chunk);\n}\nconsole.log(await streamResult.output);\n```\n\n## Authentication\n\nYou can handle authentication for your Express endpoints using context providers with `expressHandler`. This allows you to implement custom authorization logic based on incoming request data.\n\n```typescript\nimport { UserFacingError } from 'genkit';\nimport { ContextProvider, RequestData } from 'genkit/context';\nimport { expressHandler } from '@genkit-ai/express';\nimport express from 'express';\n\n// Define a custom context provider for authentication\nconst authContext: ContextProvider = (req: RequestData) => {\n if (req.headers['authorization'] !== 'open sesame') {\n throw new UserFacingError('PERMISSION_DENIED', 'not authorized');\n }\n return {\n auth: {\n user: 'Ali Baba',\n },\n };\n};\n\n// Example middleware for authentication (optional, can be integrated directly into context provider)\nconst authMiddleware = (req: express.Request, res: express.Response, next: express.NextFunction) => {\n if (req.headers['authorization'] !== 'open sesame') {\n return res.status(403).send('Unauthorized');\n }\n next();\n};\n\nconst app = express();\napp.use(express.json());\n\n// Expose the flow with authentication middleware and context provider\napp.post(\n '/simpleFlow',\n authMiddleware, // Optional: Express middleware for early auth checks\n expressHandler(simpleFlow, { context: authContext })\n);\n\napp.listen(8080, () => {\n console.log('Express server with auth listening on port 8080');\n});\n```\n\nWhen using authentication policies, you can pass headers with the client library:\n\n```typescript\nimport { runFlow } from 'genkit/beta/client';\n\n// set auth headers (when using auth policies)\nconst result = await runFlow({\n url: `http://localhost:8080/simpleFlow`,\n headers: {\n Authorization: 'open sesame',\n },\n input: { input: 'say hello' },\n});\n\nconsole.log(result); // { output: \"hello\" }\n```\n\n### Using `withContextProvider`\n\nFor more advanced authentication scenarios, you can use `withContextProvider` to wrap your flows with a `ContextProvider`. This allows you to inject custom context, such as authentication details, into your flows.\n\nHere's an example of a custom context provider that checks for a specific header:\n\n```typescript\nimport { ContextProvider, RequestData, UserFacingError } from 'genkit/context';\nimport { startFlowServer, withContextProvider } from '@genkit-ai/express';\nimport { genkit, z } from 'genkit';\nimport { googleAI } from '@genkit-ai/googleai';\n\nconst ai = genkit({\n plugins: [googleAI()],\n model: googleAI.model('gemini-2.5-flash'),\n});\n\n// Define a custom context type\ninterface CustomAuthContext {\n auth?: {\n user: string;\n role: string;\n };\n}\n\n// Define a custom context provider\nconst customContextProvider: ContextProvider = async (req: RequestData) => {\n const customHeader = req.headers['x-custom-auth'];\n if (customHeader === 'my-secret-token') {\n return {\n auth: {\n user: 'authorized-user',\n role: 'admin',\n },\n };\n }\n throw new UserFacingError('UNAUTHENTICATED', 'Invalid or missing custom authentication header.');\n};\n\nexport const protectedFlow = ai.defineFlow(\n {\n name: 'protectedFlow',\n inputSchema: z.object({ input: z.string() }),\n outputSchema: z.object({ output: z.string() }),\n },\n async ({ input }, { context }) => {\n // Access context.auth populated by the CustomContextProvider\n if (!context.auth || context.auth.role !== 'admin') {\n throw new Error('Unauthorized access: Admin role required.');\n }\n return { output: `Hello, ${context.auth.user}! Your role is ${context.auth.role}. You said: ${input}` };\n }\n);\n\n// Secure the flow with the custom context provider\nstartFlowServer({\n flows: [withContextProvider(protectedFlow, customContextProvider)],\n});\n```\n\nTo call this secured flow from the client, include the custom header:\n\n```typescript\nimport { runFlow } from 'genkit/beta/client';\n\nconst result = await runFlow({\n url: `http://localhost:8080/protectedFlow`,\n headers: {\n 'X-Custom-Auth': 'my-secret-token', // Replace with your actual custom token\n },\n input: { input: 'sensitive data' },\n});\n\nconsole.log(result);\n```\n\n#### `apiKey` Context Provider\n\nThe `apiKey` context provider is a built-in `ContextProvider` that allows you to perform API key-based access checks. It can be used with `withContextProvider` to secure your flows.\n\nTo use it, you provide the expected API key. The `apiKey` provider will then check the `Authorization` header of incoming requests against the provided key. If it matches, it populates `context.auth` with `{ apiKey: 'api-key' }`.\n\n:::caution\n**Warning:** This type of API key authentication should only be used with trusted clients (e.g., server-to-server communication or internal applications). Since the API key is sent directly in the `Authorization` header, it can be easily intercepted if used in untrusted environments like client-side web applications. For public-facing applications, consider more robust authentication mechanisms like OAuth 2.0 or Firebase Authentication.\n:::\n\n```typescript\nimport { apiKey } from 'genkit/context';\nimport { startFlowServer, withContextProvider } from '@genkit-ai/express';\nimport { genkit, z } from 'genkit';\nimport { googleAI } from '@genkit-ai/googleai';\n\nconst ai = genkit({\n plugins: [googleAI()],\n model: googleAI.model('gemini-2.5-flash'),\n});\n\nexport const securedFlow = ai.defineFlow(\n {\n name: 'securedFlow',\n inputSchema: z.object({ sensitiveData: z.string() }),\n outputSchema: z.object({ output: z.string() }),\n },\n async ({ sensitiveData }, { context }) => {\n return { output: 'this is protected by API Key check' };\n }\n);\n\n// Secure the flow with an API key from environment variables\nstartFlowServer({\n flows: [withContextProvider(securedFlow, apiKey(process.env.MY_API_KEY))],\n});\n```\n\nTo call this secured flow from the client, include the API key in the `Authorization` header:\n\n```typescript\nimport { runFlow } from 'genkit/beta/client';\n\nconst result = await runFlow({\n url: `http://localhost:8080/securedFlow`,\n headers: {\n Authorization: `${process.env.MY_API_KEY}`, // Replace with your actual API key\n },\n input: { sensitiveData: 'sensitive data' },\n});\n\nconsole.log(result);\n```\n\n## Using `startFlowServer`\n\nYou can also use `startFlowServer` to quickly expose multiple flows and actions:\n\n```typescript\nimport { startFlowServer } from '@genkit-ai/express';\nimport { genkit, z } from 'genkit';\nimport { googleAI } from '@genkit-ai/googleai';\n\nconst ai = genkit({\n plugins: [googleAI()],\n model: googleAI.model('gemini-2.5-flash'),\n});\n\nexport const menuSuggestionFlow = ai.defineFlow(\n {\n name: 'menuSuggestionFlow',\n inputSchema: z.object({ theme: z.string() }),\n outputSchema: z.object({ menu: z.string() }),\n },\n async ({ theme }) => {\n // ... your flow logic here\n return { menu: `Suggested menu for ${theme}` };\n }\n);\n\nstartFlowServer({\n flows: [menuSuggestionFlow],\n});\n```\n\nYou can also configure the server with options like `port` and `cors`:\n\n```typescript\nstartFlowServer({\n flows: [menuSuggestionFlow],\n port: 4567,\n cors: {\n origin: '*',\n },\n});\n```\n\n`startFlowServer` options:\n\n```ts\nexport interface FlowServerOptions {\n /** List of flows to expose via the flow server. */\n flows: (Flow | FlowWithContextProvider)[];\n /** Port to run the server on. Defaults to env.PORT or 3400. */\n port?: number;\n /** CORS options for the server. */\n cors?: CorsOptions;\n /** HTTP method path prefix for the exposed flows. */\n pathPrefix?: string;\n /** JSON body parser options. */\n jsonParserOptions?: bodyParser.OptionsJson;\n}\n```\n", + "title": "Express plugin", + "description": "The Genkit Express plugin provides utilities for conveniently exposing Genkit flows and actions via an Express HTTP server as REST APIs.", + "lang": "js", + "headers": "## Installation\n## Usage\n### Accessing flows from the client\n## Authentication\n### Using `withContextProvider`\n#### `apiKey` Context Provider\n## Using `startFlowServer`\n" }, "js/plugins/firebase.md": { - "text": "The Firebase plugin provides integrations with Firebase services, so you can\nbuild intelligent and scalable AI applications. Key features include:\n\n- **Firestore Vector Store**: Use Firestore for indexing and retrieval\n with vector embeddings.\n- **Telemetry**: Export telemetry to\n [Google's Cloud operations suite](https://cloud.google.com/products/operations)\n that powers the Genkit Monitoring console.\n\n## Installation\n\nInstall the Firebase plugin with npm:\n\n```bash\nnpm install @genkit-ai/firebase\n```\n\n## Prerequisites\n\n### Firebase Project Setup\n\n1. All Firebase products require a Firebase project. You can create a new\n project or enable Firebase in an existing Google Cloud project using the\n [Firebase console](https://console.firebase.google.com/).\n1. If deploying flows with Cloud functions,\n [upgrade your Firebase project](https://console.firebase.google.com/project/_/overview?purchaseBillingPlan=metered)\n to the Blaze plan.\n1. If you want to run code locally that exports telemetry, you need the\n [Google Cloud CLI](https://cloud.google.com/sdk/docs/install) tool installed.\n\n### Firebase Admin SDK Initialization\n\nYou must initialize the Firebase Admin SDK in your application.\nThis is not handled automatically by the plugin.\n\n```js\nimport { initializeApp } from 'firebase-admin/app';\n\ninitializeApp({\n projectId: 'your-project-id',\n});\n```\n\nThe plugin requires you to specify your Firebase project ID. You can specify\nyour Firebase project ID in either of the following ways:\n\n- Set `projectId` in the `initializeApp()` configuration object as shown\n in the snippet above.\n\n- Set the `GCLOUD_PROJECT` environment variable. If you're running your\n flow from a Google Cloud environment (Cloud Functions, Cloud Run, and\n so on), `GCLOUD_PROJECT` is automatically set to the project ID of the\n environment.\n\n If you set `GCLOUD_PROJECT`, you can omit the configuration\n parameter in `initializeApp()`.\n\n### Credentials\n\nTo provide Firebase credentials, you also need to set up Google Cloud\nApplication Default Credentials. To specify your credentials:\n\n- If you're running your flow from a Google Cloud environment (Cloud Functions,\n Cloud Run, and so on), this is set automatically.\n\n- For other environments:\n\n 1. Generate service account credentials for your Firebase project and\n download the JSON key file. You can do so on the\n [Service account](https://console.firebase.google.com/project/_/settings/serviceaccounts/adminsdk)\n page of the Firebase console.\n 1. Set the environment variable `GOOGLE_APPLICATION_CREDENTIALS` to the file\n path of the JSON file that contains your service account key, or you can set\n the environment variable `GCLOUD_SERVICE_ACCOUNT_CREDS` to the content of the\n JSON file.\n\n## Features and usage\n\n### Telemetry\n\nThe Firebase plugin provides a telemetry implementation for sending metrics,\ntraces, and logs to Genkit Monitoring.\n\nTo get started, visit the [Getting started guide](/docs/observability/getting-started)\nfor installation and configuration instructions.\n\nSee the [Authentication and authorization guide](/docs/observability/authentication)\nto authenticate with Google Cloud.\n\nSee the [Advanced configuration guide](/docs/observability/advanced-configuration)\nfor configuration options.\n\nSee the [Telemetry collection](/docs/observability/telemetry-collection) for\ndetails on which Genkit metrics, traces, and logs collected.\n\n### Cloud Firestore vector search\n\nYou can use Cloud Firestore as a vector store for RAG indexing and retrieval.\n\nThis section contains information specific to the `firebase` plugin and Cloud\nFirestore's vector search feature. See the\n[Retrieval-augmented generation](/docs/rag) page for a more detailed\ndiscussion on implementing RAG using Genkit.\n\n#### Using `GCLOUD_SERVICE_ACCOUNT_CREDS` and Firestore\n\nIf you are using service account credentials by passing credentials directly\nvia `GCLOUD_SERVICE_ACCOUNT_CREDS` and are also using Firestore as a vector\nstore, you need to pass credentials directly to the Firestore instance\nduring initialization or the singleton may be initialized with application\ndefault credentials depending on plugin initialization order.\n\n```js\nimport { initializeApp } from 'firebase-admin/app';\nimport { getFirestore } from 'firebase-admin/firestore';\n\nconst app = initializeApp();\nlet firestore = getFirestore(app);\n\nif (process.env.GCLOUD_SERVICE_ACCOUNT_CREDS) {\n const serviceAccountCreds = JSON.parse(process.env.GCLOUD_SERVICE_ACCOUNT_CREDS);\n const authOptions = { credentials: serviceAccountCreds };\n firestore.settings(authOptions);\n}\n```\n\n#### Define a Firestore retriever\n\nUse `defineFirestoreRetriever()` to create a retriever for Firestore\nvector-based queries.\n\n```js\nimport { defineFirestoreRetriever } from '@genkit-ai/firebase';\nimport { initializeApp } from 'firebase-admin/app';\nimport { getFirestore } from 'firebase-admin/firestore';\n\nconst app = initializeApp();\nconst firestore = getFirestore(app);\n\nconst retriever = defineFirestoreRetriever(ai, {\n name: 'exampleRetriever',\n firestore,\n collection: 'documents',\n contentField: 'text', // Field containing document content\n vectorField: 'embedding', // Field containing vector embeddings\n embedder: yourEmbedderInstance, // Embedder to generate embeddings\n distanceMeasure: 'COSINE', // Default is 'COSINE'; other options: 'EUCLIDEAN', 'DOT_PRODUCT'\n});\n```\n\n#### Retrieve documents\n\nTo retrieve documents using the defined retriever, pass the retriever instance\nand query options to `ai.retrieve`.\n\n```js\nconst docs = await ai.retrieve({\n retriever,\n query: 'search query',\n options: {\n limit: 5, // Options: Return up to 5 documents\n where: { category: 'example' }, // Optional: Filter by field-value pairs\n collection: 'alternativeCollection', // Optional: Override default collection\n },\n});\n```\n\n#### Available Retrieval Options\n\nThe following options can be passed to the `options` field in `ai.retrieve`:\n\n- **`limit`**: _(number)_\n Specify the maximum number of documents to retrieve. Default is `10`.\n\n- **`where`**: _(Record\\)_\n Add additional filters based on Firestore fields. Example:\n\n ```js\n where: { category: 'news', status: 'published' }\n ```\n\n- **`collection`**: _(string)_\n Override the default collection specified in the retriever configuration. This is useful for querying subcollections or dynamically switching between collections.\n\n#### Populate Firestore with Embeddings\n\nTo populate your Firestore collection, use an embedding generator along with\nthe Admin SDK. For example, the menu ingestion script from the\n[Retrieval-augmented generation](/docs/rag) page could be adapted for\nFirestore in the following way:\n\n```js\nimport { genkit } from 'genkit';\nimport { vertexAI, textEmbedding004 } from \"@genkit-ai/vertexai\";\n\nimport { applicationDefault, initializeApp } from \"firebase-admin/app\";\nimport { FieldValue, getFirestore } from \"firebase-admin/firestore\";\n\nimport { chunk } from \"llm-chunk\";\nimport pdf from \"pdf-parse\";\n\nimport { readFile } from \"fs/promises\";\nimport path from \"path\";\n\n// Change these values to match your Firestore config/schema\nconst indexConfig = {\n collection: \"menuInfo\",\n contentField: \"text\",\n vectorField: \"embedding\",\n embedder: textEmbedding004,\n};\n\nconst ai = genkit({\n plugins: [vertexAI({ location: \"us-central1\" })],\n});\n\nconst app = initializeApp({ credential: applicationDefault() });\nconst firestore = getFirestore(app);\n\nexport async function indexMenu(filePath: string) {\n filePath = path.resolve(filePath);\n\n // Read the PDF.\n const pdfTxt = await extractTextFromPdf(filePath);\n\n // Divide the PDF text into segments.\n const chunks = await chunk(pdfTxt);\n\n // Add chunks to the index.\n await indexToFirestore(chunks);\n}\n\nasync function indexToFirestore(data: string[]) {\n for (const text of data) {\n const embedding = (await ai.embed({\n embedder: indexConfig.embedder,\n content: text,\n }))[0].embedding;\n await firestore.collection(indexConfig.collection).add({\n [indexConfig.vectorField]: FieldValue.vector(embedding),\n [indexConfig.contentField]: text,\n });\n }\n}\n\nasync function extractTextFromPdf(filePath: string) {\n const pdfFile = path.resolve(filePath);\n const dataBuffer = await readFile(pdfFile);\n const data = await pdf(dataBuffer);\n return data.text;\n}\n```\n\nFirestore depends on indexes to provide fast and efficient querying on\ncollections. (Note that \"index\" here refers to database indexes, and not\nGenkit's indexer and retriever abstractions.)\n\nThe prior example requires the `embedding` field to be indexed to work.\nTo create the index:\n\n- Run the `gcloud` command described in the\n [Create a single-field vector index](https://firebase.google.com/docs/firestore/vector-search?authuser=0#create_and_manage_vector_indexes)\n section of the Firestore docs.\n\n The command looks like the following:\n\n ```bash\n gcloud alpha firestore indexes composite create --project=your-project-id \\\n --collection-group=yourCollectionName --query-scope=COLLECTION \\\n --field-config=vector-config='{\"dimension\":\"768\",\"flat\": \"{}\"}',field-path=yourEmbeddingField\n ```\n\n However, the correct indexing configuration depends on the queries you\n make and the embedding model you're using.\n\n- Alternatively, call `ai.retrieve()` and Firestore will throw an error with\n the correct command to create the index.\n\n#### Learn more\n\n- See the [Retrieval-augmented generation](/docs/rag) page for a general discussion\n on indexers and retrievers in Genkit.\n- See [Search with vector embeddings](https://firebase.google.com/docs/firestore/vector-search)\n in the Cloud Firestore docs for more on the vector search feature.\n\n### Deploy flows as Cloud Functions\n\nTo deploy a flow with Cloud Functions, use the Firebase Functions library's\nbuilt-in support for genkit. The `onCallGenkit` method lets\nyou to create a [callable function](https://firebase.google.com/docs/functions/callable?gen=2nd)\nfrom a flow. It automatically supports\nstreaming and JSON requests. You can use the\n[Cloud Functions client SDKs](https://firebase.google.com/docs/functions/callable?gen=2nd#call_the_function)\nto call them.\n\n```js\nimport { onCallGenkit } from 'firebase-functions/https';\nimport { defineSecret } from 'firebase-functions/params';\n\nexport const exampleFlow = ai.defineFlow(\n {\n name: 'exampleFlow',\n },\n async (prompt) => {\n // Flow logic goes here.\n\n return response;\n },\n);\n\n// WARNING: This has no authentication or app check protections.\n// See genkit.dev/docs/auth for more information.\nexport const example = onCallGenkit({ secrets: [apiKey] }, exampleFlow);\n```\n\nDeploy your flow using the Firebase CLI:\n\n```bash\nfirebase deploy --only functions\n```\n", + "text": "# Firebase plugin\n\nThe Firebase plugin provides integrations with Firebase services, so you can\nbuild intelligent and scalable AI applications. Key features include:\n\n- **Firestore Vector Store**: Use Firestore for indexing and retrieval\n with vector embeddings.\n- **Telemetry**: Export telemetry to\n [Google's Cloud operations suite](https://cloud.google.com/products/operations)\n that powers the Genkit Monitoring console.\n\n## Installation\n\nInstall the Firebase plugin with npm:\n\n```bash\nnpm install @genkit-ai/firebase\n```\n\n## Prerequisites\n\n### Firebase Project Setup\n\n1. All Firebase products require a Firebase project. You can create a new\n project or enable Firebase in an existing Google Cloud project using the\n [Firebase console](https://console.firebase.google.com/).\n1. If deploying flows with Cloud functions,\n [upgrade your Firebase project](https://console.firebase.google.com/project/_/overview?purchaseBillingPlan=metered)\n to the Blaze plan.\n1. If you want to run code locally that exports telemetry, you need the\n [Google Cloud CLI](https://cloud.google.com/sdk/docs/install) tool installed.\n\n### Firebase Admin SDK Initialization\n\nYou must initialize the Firebase Admin SDK in your application.\nThis is not handled automatically by the plugin.\n\n```js\nimport { initializeApp } from 'firebase-admin/app';\n\ninitializeApp({\n projectId: 'your-project-id',\n});\n```\n\nThe plugin requires you to specify your Firebase project ID. You can specify\nyour Firebase project ID in either of the following ways:\n\n- Set `projectId` in the `initializeApp()` configuration object as shown\n in the snippet above.\n\n- Set the `GCLOUD_PROJECT` environment variable. If you're running your\n flow from a Google Cloud environment (Cloud Functions, Cloud Run, and\n so on), `GCLOUD_PROJECT` is automatically set to the project ID of the\n environment.\n\n If you set `GCLOUD_PROJECT`, you can omit the configuration\n parameter in `initializeApp()`.\n\n### Credentials\n\nTo provide Firebase credentials, you also need to set up Google Cloud\nApplication Default Credentials. To specify your credentials:\n\n- If you're running your flow from a Google Cloud environment (Cloud Functions,\n Cloud Run, and so on), this is set automatically.\n\n- For other environments:\n\n 1. Generate service account credentials for your Firebase project and\n download the JSON key file. You can do so on the\n [Service account](https://console.firebase.google.com/project/_/settings/serviceaccounts/adminsdk)\n page of the Firebase console.\n 1. Set the environment variable `GOOGLE_APPLICATION_CREDENTIALS` to the file\n path of the JSON file that contains your service account key, or you can set\n the environment variable `GCLOUD_SERVICE_ACCOUNT_CREDS` to the content of the\n JSON file.\n\n## Features and usage\n\n### Telemetry\n\nThe Firebase plugin provides a telemetry implementation for sending metrics,\ntraces, and logs to Genkit Monitoring.\n\nTo get started, visit the [Getting started guide](/docs/observability/getting-started)\nfor installation and configuration instructions.\n\nSee the [Authentication and authorization guide](/docs/observability/authentication)\nto authenticate with Google Cloud.\n\nSee the [Advanced configuration guide](/docs/observability/advanced-configuration)\nfor configuration options.\n\nSee the [Telemetry collection](/docs/observability/telemetry-collection) for\ndetails on which Genkit metrics, traces, and logs collected.\n\n### Cloud Firestore vector search\n\nYou can use Cloud Firestore as a vector store for RAG indexing and retrieval.\n\nThis section contains information specific to the `firebase` plugin and Cloud\nFirestore's vector search feature. See the\n[Retrieval-augmented generation](/docs/rag) page for a more detailed\ndiscussion on implementing RAG using Genkit.\n\n#### Using `GCLOUD_SERVICE_ACCOUNT_CREDS` and Firestore\n\nIf you are using service account credentials by passing credentials directly\nvia `GCLOUD_SERVICE_ACCOUNT_CREDS` and are also using Firestore as a vector\nstore, you need to pass credentials directly to the Firestore instance\nduring initialization or the singleton may be initialized with application\ndefault credentials depending on plugin initialization order.\n\n```js\nimport { initializeApp } from 'firebase-admin/app';\nimport { getFirestore } from 'firebase-admin/firestore';\n\nconst app = initializeApp();\nlet firestore = getFirestore(app);\n\nif (process.env.GCLOUD_SERVICE_ACCOUNT_CREDS) {\n const serviceAccountCreds = JSON.parse(process.env.GCLOUD_SERVICE_ACCOUNT_CREDS);\n const authOptions = { credentials: serviceAccountCreds };\n firestore.settings(authOptions);\n}\n```\n\n#### Define a Firestore retriever\n\nUse `defineFirestoreRetriever()` to create a retriever for Firestore\nvector-based queries.\n\n```js\nimport { defineFirestoreRetriever } from '@genkit-ai/firebase';\nimport { initializeApp } from 'firebase-admin/app';\nimport { getFirestore } from 'firebase-admin/firestore';\n\nconst app = initializeApp();\nconst firestore = getFirestore(app);\n\nconst retriever = defineFirestoreRetriever(ai, {\n name: 'exampleRetriever',\n firestore,\n collection: 'documents',\n contentField: 'text', // Field containing document content\n vectorField: 'embedding', // Field containing vector embeddings\n embedder: yourEmbedderInstance, // Embedder to generate embeddings\n distanceMeasure: 'COSINE', // Default is 'COSINE'; other options: 'EUCLIDEAN', 'DOT_PRODUCT'\n});\n```\n\n#### Retrieve documents\n\nTo retrieve documents using the defined retriever, pass the retriever instance\nand query options to `ai.retrieve`.\n\n```js\nconst docs = await ai.retrieve({\n retriever,\n query: 'search query',\n options: {\n limit: 5, // Options: Return up to 5 documents\n where: { category: 'example' }, // Optional: Filter by field-value pairs\n collection: 'alternativeCollection', // Optional: Override default collection\n },\n});\n```\n\n#### Available Retrieval Options\n\nThe following options can be passed to the `options` field in `ai.retrieve`:\n\n- **`limit`**: _(number)_\n Specify the maximum number of documents to retrieve. Default is `10`.\n\n- **`where`**: _(Record\\)_\n Add additional filters based on Firestore fields. Example:\n\n ```js\n where: { category: 'news', status: 'published' }\n ```\n\n- **`collection`**: _(string)_\n Override the default collection specified in the retriever configuration. This is useful for querying subcollections or dynamically switching between collections.\n\n#### Populate Firestore with Embeddings\n\nTo populate your Firestore collection, use an embedding generator along with\nthe Admin SDK. For example, the menu ingestion script from the\n[Retrieval-augmented generation](/docs/rag) page could be adapted for\nFirestore in the following way:\n\n```js\nimport { genkit } from 'genkit';\nimport { vertexAI } from \"@genkit-ai/vertexai\";\n\nimport { applicationDefault, initializeApp } from \"firebase-admin/app\";\nimport { FieldValue, getFirestore } from \"firebase-admin/firestore\";\n\nimport { chunk } from \"llm-chunk\";\nimport pdf from \"pdf-parse\";\n\nimport { readFile } from \"fs/promises\";\nimport path from \"path\";\n\n// Change these values to match your Firestore config/schema\nconst indexConfig = {\n collection: \"menuInfo\",\n contentField: \"text\",\n vectorField: \"embedding\",\n embedder: vertexAI.embedder('gemini-embedding-001'),\n};\n\nconst ai = genkit({\n plugins: [vertexAI({ location: \"us-central1\" })],\n});\n\nconst app = initializeApp({ credential: applicationDefault() });\nconst firestore = getFirestore(app);\n\nexport async function indexMenu(filePath: string) {\n filePath = path.resolve(filePath);\n\n // Read the PDF.\n const pdfTxt = await extractTextFromPdf(filePath);\n\n // Divide the PDF text into segments.\n const chunks = await chunk(pdfTxt);\n\n // Add chunks to the index.\n await indexToFirestore(chunks);\n}\n\nasync function indexToFirestore(data: string[]) {\n for (const text of data) {\n const embedding = (await ai.embed({\n embedder: indexConfig.embedder,\n content: text,\n }))[0].embedding;\n await firestore.collection(indexConfig.collection).add({\n [indexConfig.vectorField]: FieldValue.vector(embedding),\n [indexConfig.contentField]: text,\n });\n }\n}\n\nasync function extractTextFromPdf(filePath: string) {\n const pdfFile = path.resolve(filePath);\n const dataBuffer = await readFile(pdfFile);\n const data = await pdf(dataBuffer);\n return data.text;\n}\n```\n\nFirestore depends on indexes to provide fast and efficient querying on\ncollections. (Note that \"index\" here refers to database indexes, and not\nGenkit's indexer and retriever abstractions.)\n\nThe prior example requires the `embedding` field to be indexed to work.\nTo create the index:\n\n- Run the `gcloud` command described in the\n [Create a single-field vector index](https://firebase.google.com/docs/firestore/vector-search?authuser=0#create_and_manage_vector_indexes)\n section of the Firestore docs.\n\n The command looks like the following:\n\n ```bash\n gcloud alpha firestore indexes composite create --project=your-project-id \\\n --collection-group=yourCollectionName --query-scope=COLLECTION \\\n --field-config=vector-config='{\"dimension\":\"768\",\"flat\": \"{}\"}',field-path=yourEmbeddingField\n ```\n\n However, the correct indexing configuration depends on the queries you\n make and the embedding model you're using.\n\n- Alternatively, call `ai.retrieve()` and Firestore will throw an error with\n the correct command to create the index.\n\n#### Learn more\n\n- See the [Retrieval-augmented generation](/docs/rag) page for a general discussion\n on indexers and retrievers in Genkit.\n- See [Search with vector embeddings](https://firebase.google.com/docs/firestore/vector-search)\n in the Cloud Firestore docs for more on the vector search feature.\n\n### Deploy flows as Cloud Functions\n\nTo deploy a flow with Cloud Functions, use the Firebase Functions library's\nbuilt-in support for genkit. The `onCallGenkit` method lets\nyou to create a [callable function](https://firebase.google.com/docs/functions/callable?gen=2nd)\nfrom a flow. It automatically supports\nstreaming and JSON requests. You can use the\n[Cloud Functions client SDKs](https://firebase.google.com/docs/functions/callable?gen=2nd#call_the_function)\nto call them.\n\n```js\nimport { onCallGenkit } from 'firebase-functions/https';\nimport { defineSecret } from 'firebase-functions/params';\n\nexport const exampleFlow = ai.defineFlow(\n {\n name: 'exampleFlow',\n },\n async (prompt) => {\n // Flow logic goes here.\n\n return response;\n },\n);\n\n// WARNING: This has no authentication or app check protections.\n// See genkit.dev/docs/auth for more information.\nexport const example = onCallGenkit({ secrets: [apiKey] }, exampleFlow);\n```\n\nDeploy your flow using the Firebase CLI:\n\n```bash\nfirebase deploy --only functions\n```\n", "title": "Firebase plugin", - "lang": "js" + "description": "This document describes the Firebase plugin for Genkit, providing integrations with Firebase services like Firestore for vector search and telemetry export to Google Cloud Operations.", + "lang": "js", + "headers": "## Installation\n## Prerequisites\n### Firebase Project Setup\n### Firebase Admin SDK Initialization\n### Credentials\n## Features and usage\n### Telemetry\n### Cloud Firestore vector search\n#### Using `GCLOUD_SERVICE_ACCOUNT_CREDS` and Firestore\n#### Define a Firestore retriever\n#### Retrieve documents\n#### Available Retrieval Options\n#### Populate Firestore with Embeddings\n#### Learn more\n### Deploy flows as Cloud Functions\n" }, "js/plugins/google-genai.md": { - "text": "The Google Generative AI plugin provides interfaces to Google's Gemini models\nthrough the [Gemini API](https://ai.google.dev/docs/gemini_api_overview).\n\n## Installation\n\n```bash\nnpm install @genkit-ai/googleai\n```\n\n## Configuration\n\nTo use this plugin, specify it when you initialize Genkit:\n\n```ts\nimport { genkit } from 'genkit';\nimport { googleAI } from '@genkit-ai/googleai';\n\nconst ai = genkit({\n plugins: [googleAI()],\n});\n```\n\nThe plugin requires an API key for the Gemini API, which you can get from\n[Google AI Studio](https://aistudio.google.com/app/apikey).\n\nConfigure the plugin to use your API key by doing one of the following:\n\n- Set the `GEMINI_API_KEY` environment variable to your API key.\n- Specify the API key when you initialize the plugin:\n\n ```ts\n googleAI({ apiKey: yourKey });\n ```\n\n However, don't embed your API key directly in code! Use this feature only in\n conjunction with a service like Cloud Secret Manager or similar.\n\n## Usage\n\nThe recommended way to reference models is through the helper function provided by the plugin:\n\n```ts\nimport { googleAI } from '@genkit-ai/googleai';\n\n// Referencing models\nconst model = googleAI.model('gemini-2.5-flash');\nconst modelPro = googleAI.model('gemini-2.5-flash-lite');\n\n// Referencing embedders\nconst embedder = googleAI.embedder('text-embedding-004');\n```\n\nYou can use these references to specify which model `generate()` uses:\n\n```ts\nconst ai = genkit({\n plugins: [googleAI()],\n model: googleAI.model('gemini-2.5-flash'), // Set default model\n});\n\nconst llmResponse = await ai.generate('Tell me a joke.');\n```\n\nor use embedders (ex. `text-embedding-004`) with `embed` or retrievers:\n\n```ts\nconst ai = genkit({\n plugins: [googleAI()],\n});\n\nconst embeddings = await ai.embed({\n embedder: googleAI.embedder('text-embedding-004'),\n content: input,\n});\n```\n\n## Gemini Files API\n\nYou can use files uploaded to the Gemini Files API with Genkit:\n\n```ts\nimport { GoogleAIFileManager } from '@google/generative-ai/server';\nimport { genkit } from 'genkit';\nimport { googleAI } from '@genkit-ai/googleai';\n\nconst ai = genkit({\n plugins: [googleAI()],\n});\n\nconst fileManager = new GoogleAIFileManager(process.env.GEMINI_API_KEY);\nconst uploadResult = await fileManager.uploadFile('path/to/file.jpg', {\n mimeType: 'image/jpeg',\n displayName: 'Your Image',\n});\n\nconst response = await ai.generate({\n model: googleAI.model('gemini-2.5-flash'),\n prompt: [\n { text: 'Describe this image:' },\n {\n media: {\n contentType: uploadResult.file.mimeType,\n url: uploadResult.file.uri,\n },\n },\n ],\n});\n```\n\n## Fine-tuned models\n\nYou can use models fine-tuned with the Google Gemini API. Follow the\ninstructions from the\n[Gemini API](https://ai.google.dev/gemini-api/docs/model-tuning/tutorial?lang=python)\nor fine-tune a model using\n[AI Studio](https://aistudio.corp.google.com/app/tune).\n\nThe tuning process uses a base model—for example, Gemini 2.0 Flash—and your\nprovided examples to create a new tuned model. Remember the base model you\nused, and copy the new model's ID.\n\nWhen calling the tuned model in Genkit, use the base model as the `model`\nparameter, and pass the tuned model's ID as part of the `config` block. For\nexample, if you used Gemini 2.0 Flash as the base model, and got the model ID\n`tunedModels/my-example-model-apbm8oqbvuv2` you can call it with:\n\n```ts\nconst ai = genkit({\n plugins: [googleAI()],\n});\n\nconst llmResponse = await ai.generate({\n prompt: `Suggest an item for the menu of fish themed restruant`,\n model: googleAI.model('tunedModels/my-example-model-apbm8oqbvuv2'),\n});\n```\n\n## Context Caching\n\nThe Google Generative AI plugin supports **context caching**, which allows models to reuse previously cached content to optimize performance and reduce latency for repetitive tasks. This feature is especially useful for conversational flows or scenarios where the model references a large body of text consistently across multiple requests.\n\n### How to Use Context Caching\n\nTo enable context caching, ensure your model supports it. For example, `gemini-2.5-flash` and `gemini-1.5-pro` are models that support context caching.\n\nYou can define a caching mechanism in your application like this:\n\n```ts\nconst ai = genkit({\n plugins: [googleAI()],\n});\n\nconst llmResponse = await ai.generate({\n messages: [\n {\n role: 'user',\n content: [{ text: 'Here is the relevant text from War and Peace.' }],\n },\n {\n role: 'model',\n content: [\n {\n text: 'Based on War and Peace, here is some analysis of Pierre Bezukhov’s character.',\n },\n ],\n metadata: {\n cache: {\n ttlSeconds: 300, // Cache this message for 5 minutes\n },\n },\n },\n ],\n model: googleAI.model('gemini-2.5-flash-001'),\n prompt: 'Describe Pierre’s transformation throughout the novel.',\n});\n```\n\nIn this setup:\n\n- **`messages`**: Allows you to pass conversation history.\n- **`metadata.cache.ttlSeconds`**: Specifies the time-to-live (TTL) for caching a specific response.\n\n### Example: Leveraging Large Texts with Context\n\nFor applications referencing long documents, such as _War and Peace_ or _Lord of the Rings_, you can structure your queries to reuse cached contexts:\n\n```ts\nconst fs = require('fs/promises');\n\nconst textContent = await fs.readFile('path/to/war_and_peace.txt', 'utf-8');\n\nconst llmResponse = await ai.generate({\n messages: [\n {\n role: 'user',\n content: [{ text: textContent }], // Include the large text as context\n },\n {\n role: 'model',\n content: [\n {\n text: 'This analysis is based on the provided text from War and Peace.',\n },\n ],\n metadata: {\n cache: {\n ttlSeconds: 300, // Cache the response to avoid reloading the full text\n },\n },\n },\n ],\n model: googleAI.model('gemini-2.5-flash-001'),\n prompt: 'Analyze the relationship between Pierre and Natasha.',\n});\n```\n\n### Caching other modes of content\n\nThe Gemini models are multi-modal, and other modes of content are allowed to be cached as well.\n\nFor example, to cache a long piece of video content, you must first upload using the file manager from the Google AI SDK:\n\n```ts\nimport { GoogleAIFileManager } from '@google/generative-ai/server';\n```\n\n```ts\nconst fileManager = new GoogleAIFileManager(process.env.GEMINI_API_KEY);\n\n// Upload video to Google AI using the Gemini Files API\nconst uploadResult = await fileManager.uploadFile(videoFilePath, {\n mimeType: 'video/mp4', // Adjust according to the video format\n displayName: 'Uploaded Video for Analysis',\n});\n\nconst fileUri = uploadResult.file.uri;\n```\n\nNow you may configure the cache in your calls to `ai.generate`:\n\n```ts\nconst analyzeVideoResponse = await ai.generate({\n messages: [\n {\n role: 'user',\n content: [\n {\n media: {\n url: fileUri, // Use the uploaded file URL\n contentType: 'video/mp4',\n },\n },\n ],\n },\n {\n role: 'model',\n content: [\n {\n text: 'This video seems to contain several key moments. I will analyze it now and prepare to answer your questions.',\n },\n ],\n // Everything up to (including) this message will be cached.\n metadata: {\n cache: true,\n },\n },\n ],\n model: googleAI.model('gemini-2.5-flash-001'),\n prompt: query,\n});\n```\n\n### Supported Models for Context Caching\n\nOnly specific models, such as `gemini-2.5-flash` and `gemini-1.5-pro`, support context caching. If an unsupported model is used, an error will be raised, indicating that caching cannot be applied.\n\n### Further Reading\n\nSee more information regarding context caching on Google AI in their [documentation](https://ai.google.dev/gemini-api/docs/caching?lang=node).\n", + "text": "# Google Generative AI plugin\n\nThe Google Generative AI plugin provides interfaces to Google's Gemini models\nthrough the [Gemini API](https://ai.google.dev/docs/gemini_api_overview).\n\n## Installation\n\n```bash\nnpm install @genkit-ai/googleai\n```\n\n## Configuration\n\nTo use this plugin, specify it when you initialize Genkit:\n\n```ts\nimport { genkit } from 'genkit';\nimport { googleAI } from '@genkit-ai/googleai';\n\nconst ai = genkit({\n plugins: [googleAI()],\n});\n```\n\nThe plugin requires an API key for the Gemini API, which you can get from\n[Google AI Studio](https://aistudio.google.com/app/apikey).\n\nConfigure the plugin to use your API key by doing one of the following:\n\n- Set the `GEMINI_API_KEY` environment variable to your API key.\n- Specify the API key when you initialize the plugin:\n\n ```ts\n googleAI({ apiKey: yourKey });\n ```\n\n However, don't embed your API key directly in code! Use this feature only in\n conjunction with a service like Cloud Secret Manager or similar.\n\n## Usage\n\nThe recommended way to reference models is through the helper function provided by the plugin:\n\n```ts\nimport { googleAI } from '@genkit-ai/googleai';\n\n// Referencing models\nconst model = googleAI.model('gemini-2.5-flash');\nconst modelPro = googleAI.model('gemini-2.5-flash-lite');\n\n// Referencing embedders\nconst embedder = googleAI.embedder('gemini-embedding-001');\n```\n\nYou can use these references to specify which model `generate()` uses:\n\n```ts\nconst ai = genkit({\n plugins: [googleAI()],\n model: googleAI.model('gemini-2.5-flash'), // Set default model\n});\n\nconst llmResponse = await ai.generate('Tell me a joke.');\n```\n\nor use embedders (ex. `gemini-embedding-001`) with `embed` or retrievers:\n\n```ts\nconst ai = genkit({\n plugins: [googleAI()],\n});\n\nconst embeddings = await ai.embed({\n embedder: googleAI.embedder('gemini-embedding-001'),\n content: input,\n});\n```\n\n## Gemini Files API\n\nYou can use files uploaded to the Gemini Files API with Genkit:\n\n```ts\nimport { GoogleAIFileManager } from '@google/generative-ai/server';\nimport { genkit } from 'genkit';\nimport { googleAI } from '@genkit-ai/googleai';\n\nconst ai = genkit({\n plugins: [googleAI()],\n});\n\nconst fileManager = new GoogleAIFileManager(process.env.GEMINI_API_KEY);\nconst uploadResult = await fileManager.uploadFile('path/to/file.jpg', {\n mimeType: 'image/jpeg',\n displayName: 'Your Image',\n});\n\nconst response = await ai.generate({\n model: googleAI.model('gemini-2.5-flash'),\n prompt: [\n { text: 'Describe this image:' },\n {\n media: {\n contentType: uploadResult.file.mimeType,\n url: uploadResult.file.uri,\n },\n },\n ],\n});\n```\n\n## Fine-tuned models\n\nYou can use models fine-tuned with the Google Gemini API. Follow the\ninstructions from the\n[Gemini API](https://ai.google.dev/gemini-api/docs/model-tuning/tutorial?lang=python)\nor fine-tune a model using\n[AI Studio](https://aistudio.corp.google.com/app/tune).\n\nThe tuning process uses a base model—for example, Gemini 2.0 Flash—and your\nprovided examples to create a new tuned model. Remember the base model you\nused, and copy the new model's ID.\n\nWhen calling the tuned model in Genkit, use the base model as the `model`\nparameter, and pass the tuned model's ID as part of the `config` block. For\nexample, if you used Gemini 2.0 Flash as the base model, and got the model ID\n`tunedModels/my-example-model-apbm8oqbvuv2` you can call it with:\n\n```ts\nconst ai = genkit({\n plugins: [googleAI()],\n});\n\nconst llmResponse = await ai.generate({\n prompt: `Suggest an item for the menu of fish themed restruant`,\n model: googleAI.model('tunedModels/my-example-model-apbm8oqbvuv2'),\n});\n```\n\n## Text-to-Speech (TTS) Models\n\nThe Google Generative AI plugin provides access to text-to-speech capabilities through the Gemini TTS models. These models can convert text into natural-sounding speech for various applications such as voice assistants, accessibility features, or interactive content.\n\n### Basic Usage\n\nTo generate audio using the Gemini TTS model:\n\n```ts\nimport { googleAI } from '@genkit-ai/googleai';\nimport { writeFile } from 'node:fs/promises';\nimport wav from 'wav'; // npm install wav && npm install -D @types/wav\n\nconst ai = genkit({\n plugins: [googleAI()],\n});\n\nconst { media } = await ai.generate({\n model: googleAI.model('gemini-2.5-flash-preview-tts'),\n config: {\n responseModalities: ['AUDIO'],\n speechConfig: {\n voiceConfig: {\n prebuiltVoiceConfig: { voiceName: 'Algenib' },\n },\n },\n },\n prompt: 'Say that Genkit is an amazing Gen AI library',\n});\n\nif (!media) {\n throw new Error('no media returned');\n}\nconst audioBuffer = Buffer.from(\n media.url.substring(media.url.indexOf(',') + 1),\n 'base64'\n);\nawait writeFile('output.wav', await toWav(audioBuffer));\n\nasync function toWav(\n pcmData: Buffer,\n channels = 1,\n rate = 24000,\n sampleWidth = 2\n): Promise {\n return new Promise((resolve, reject) => {\n // This code depends on `wav` npm library.\n const writer = new wav.Writer({\n channels,\n sampleRate: rate,\n bitDepth: sampleWidth * 8,\n });\n\n let bufs = [] as any[];\n writer.on('error', reject);\n writer.on('data', function (d) {\n bufs.push(d);\n });\n writer.on('end', function () {\n resolve(Buffer.concat(bufs).toString('base64'));\n });\n\n writer.write(pcmData);\n writer.end();\n });\n}\n```\n\n### Multi-speaker Audio Generation\n\nYou can generate audio with multiple speakers, each with their own voice:\n\n```ts\nconst response = await ai.generate({\n model: googleAI.model('gemini-2.5-flash-preview-tts'),\n config: {\n responseModalities: ['AUDIO'],\n speechConfig: {\n multiSpeakerVoiceConfig: {\n speakerVoiceConfigs: [\n {\n speaker: 'Speaker1',\n voiceConfig: {\n prebuiltVoiceConfig: { voiceName: 'Algenib' },\n },\n },\n {\n speaker: 'Speaker2',\n voiceConfig: {\n prebuiltVoiceConfig: { voiceName: 'Achernar' },\n },\n },\n ],\n },\n },\n },\n prompt: `Here's the dialog:\n Speaker1: \"Genkit is an amazing Gen AI library!\"\n Speaker2: \"I thought it was a framework.\"`,\n});\n```\n\nWhen using multi-speaker configuration, the model automatically detects speaker labels in the text (like \"Speaker1:\" and \"Speaker2:\") and applies the corresponding voice to each speaker's lines.\n\n### Configuration Options\n\nThe Gemini TTS models support various configuration options:\n\n#### Voice Selection\n\nYou can choose from different pre-built voices with unique characteristics:\n\n```ts\nspeechConfig: {\n voiceConfig: {\n prebuiltVoiceConfig: {\n voiceName: 'Algenib' // Other options: 'Achernar', 'Ankaa', etc.\n },\n },\n}\n```\n\n#### Speech Emphasis\n\nYou can use markdown-style formatting in your prompt to add emphasis:\n\n- Bold text (`**like this**`) for stronger emphasis\n- Italic text (`*like this*`) for moderate emphasis\n\nExample:\n\n```ts\nprompt: 'Genkit is an **amazing** Gen AI *library*!'\n```\n\n#### Advanced Speech Parameters\n\nFor more control over the generated speech:\n\n```ts\nspeechConfig: {\n voiceConfig: {\n prebuiltVoiceConfig: {\n voiceName: 'Algenib',\n speakingRate: 1.0, // Range: 0.25 to 4.0, default is 1.0\n pitch: 0.0, // Range: -20.0 to 20.0, default is 0.0\n volumeGainDb: 0.0, // Range: -96.0 to 16.0, default is 0.0\n },\n },\n}\n```\n\n- `speakingRate`: Controls the speed of speech (higher values = faster speech)\n- `pitch`: Adjusts the pitch of the voice (higher values = higher pitch)\n- `volumeGainDb`: Controls the volume (higher values = louder)\n\nFor more detailed information about the Gemini TTS models and their configuration options, see the [Google AI Speech Generation documentation](https://ai.google.dev/gemini-api/docs/speech-generation).\n\n## Video Generation (Veo) Models\n\nThe Google Generative AI plugin provides access to video generation capabilities through the Veo models. These models can generate videos from text prompts or manipulate existing images to create dynamic video content.\n\n### Basic Usage: Text-to-Video Generation\n\nTo generate a video from a text prompt using the Veo model:\n\n```ts\nimport { googleAI } from '@genkit-ai/googleai';\nimport * as fs from 'fs';\nimport { Readable } from 'stream';\nimport { MediaPart } from 'genkit';\nimport { genkit } from 'genkit';\n\nconst ai = genkit({\n plugins: [googleAI()],\n});\n\nai.defineFlow('text-to-video-veo', async () => {\n let { operation } = await ai.generate({\n model: googleAI.model('veo-2.0-generate-001'),\n prompt: 'A majestic dragon soaring over a mystical forest at dawn.',\n config: {\n durationSeconds: 5,\n aspectRatio: '16:9',\n },\n });\n\n if (!operation) {\n throw new Error('Expected the model to return an operation');\n }\n\n // Wait until the operation completes.\n while (!operation.done) {\n operation = await ai.checkOperation(operation);\n // Sleep for 5 seconds before checking again.\n await new Promise((resolve) => setTimeout(resolve, 5000));\n }\n\n if (operation.error) {\n throw new Error('failed to generate video: ' + operation.error.message);\n }\n\n const video = operation.output?.message?.content.find((p) => !!p.media);\n if (!video) {\n throw new Error('Failed to find the generated video');\n }\n await downloadVideo(video, 'output.mp4');\n});\n\nasync function downloadVideo(video: MediaPart, path: string) {\n const fetch = (await import('node-fetch')).default;\n // Add API key before fetching the video.\n const videoDownloadResponse = await fetch(\n `${video.media!.url}&key=${process.env.GEMINI_API_KEY}`\n );\n if (\n !videoDownloadResponse ||\n videoDownloadResponse.status !== 200 ||\n !videoDownloadResponse.body\n ) {\n throw new Error('Failed to fetch video');\n }\n\n Readable.from(videoDownloadResponse.body).pipe(fs.createWriteStream(path));\n}\n```\n\nVeo 3 uses the exact same API, just make sure you only use supported config options (see below).\n\nTo use the Veo 3 model, reference `veo-3.0-generate-preview`:\n\n```ts\nlet { operation } = await ai.generate({\n model: googleAI.model('veo-3.0-generate-preview'),\n prompt: 'A cinematic shot of a an old car driving down a deserted road at sunset.',\n});\n```\n\n### Video Generation from Photo Reference\n\nTo use a photo as reference for the video using the Veo model (e.g. to make a static photo move), you can provide an image as part of the prompt.\n\n```ts\nconst startingImage = fs.readFileSync('photo.jpg', { encoding: 'base64' });\n\nlet { operation } = await ai.generate({\n model: googleAI.model('veo-2.0-generate-001'),\n prompt: [\n {\n text: 'make the subject in the photo move',\n },\n {\n media: {\n contentType: 'image/jpeg',\n url: `data:image/jpeg;base64,${startingImage}`,\n },\n },\n ],\n config: {\n durationSeconds: 5,\n aspectRatio: '9:16',\n personGeneration: 'allow_adult',\n },\n});\n```\n\n### Configuration Options\n\nThe Veo models support various configuration options.\n\n#### Veo Model Parameters\n\nFull list of options can be found at https://ai.google.dev/gemini-api/docs/video#veo-model-parameters\n\n- `negativePrompt`: Text string that describes anything you want to discourage the model from generating\n- `aspectRatio`: Changes the aspect ratio of the generated video.\n - `\"16:9\"`: Supported in Veo 3 and Veo 2.\n - `\"9:16\"`: Supported in Veo 2 only (defaults to \"16:9\").\n- `personGeneration`: Allow the model to generate videos of people. The following values are supported:\n - **Text-to-video generation**:\n - `\"allow_all\"`: Generate videos that include adults and children. Currently the only available `personGeneration` value for Veo 3.\n - `\"dont_allow\"`: Veo 2 only. Don't allow the inclusion of people or faces.\n - `\"allow_adult\"`: Veo 2 only. Generate videos that include adults, but not children.\n - **Image-to-video generation**: Veo 2 only\n - `\"dont_allow\"`: Don't allow the inclusion of people or faces.\n - `\"allow_adult\"`: Generate videos that include adults, but not children.\n- `numberOfVideos`: Output videos requested\n - `1`: Supported in Veo 3 and Veo 2\n - `2`: Supported in Veo 2 only.\n- `durationSeconds`: Veo 2 only. Length of each output video in seconds, between 5 and 8. Not configurable for Veo 3, default setting is 8 seconds.\n- `enhancePrompt`: Veo 2 only. Enable or disable the prompt rewriter. Enabled by default. Not configurable for Veo 3, default prompt enhancer is always on.\n\n## Context Caching\n\nThe Google Generative AI plugin supports **context caching**, which allows models to reuse previously cached content to optimize performance and reduce latency for repetitive tasks. This feature is especially useful for conversational flows or scenarios where the model references a large body of text consistently across multiple requests.\n\n### How to Use Context Caching\n\nTo enable context caching, ensure your model supports it. For example, `gemini-2.5-flash` and `gemini-1.5-pro` are models that support context caching.\n\nYou can define a caching mechanism in your application like this:\n\n```ts\nconst ai = genkit({\n plugins: [googleAI()],\n});\n\nconst llmResponse = await ai.generate({\n messages: [\n {\n role: 'user',\n content: [{ text: 'Here is the relevant text from War and Peace.' }],\n },\n {\n role: 'model',\n content: [\n {\n text: `Based on War and Peace, here is some analysis of Pierre Bezukhov's character.`,\n },\n ],\n metadata: {\n cache: {\n ttlSeconds: 300, // Cache this message for 5 minutes\n },\n },\n },\n ],\n model: googleAI.model('gemini-2.5-flash-001'),\n prompt: `Describe Pierre's transformation throughout the novel`,\n});\n```\n\nIn this setup:\n\n- **`messages`**: Allows you to pass conversation history.\n- **`metadata.cache.ttlSeconds`**: Specifies the time-to-live (TTL) for caching a specific response.\n\n### Example: Leveraging Large Texts with Context\n\nFor applications referencing long documents, such as _War and Peace_ or _Lord of the Rings_, you can structure your queries to reuse cached contexts:\n\n```ts\nconst fs = require('fs/promises');\n\nconst textContent = await fs.readFile('path/to/war_and_peace.txt', 'utf-8');\n\nconst llmResponse = await ai.generate({\n messages: [\n {\n role: 'user',\n content: [{ text: textContent }], // Include the large text as context\n },\n {\n role: 'model',\n content: [\n {\n text: 'This analysis is based on the provided text from War and Peace.',\n },\n ],\n metadata: {\n cache: {\n ttlSeconds: 300, // Cache the response to avoid reloading the full text\n },\n },\n },\n ],\n model: googleAI.model('gemini-2.5-flash-001'),\n prompt: 'Analyze the relationship between Pierre and Natasha.',\n});\n```\n\n### Caching other modes of content\n\nThe Gemini models are multi-modal, and other modes of content are allowed to be cached as well.\n\nFor example, to cache a long piece of video content, you must first upload using the file manager from the Google AI SDK:\n\n```ts\nimport { GoogleAIFileManager } from '@google/generative-ai/server';\n```\n\n```ts\nconst fileManager = new GoogleAIFileManager(process.env.GEMINI_API_KEY);\n\n// Upload video to Google AI using the Gemini Files API\nconst uploadResult = await fileManager.uploadFile(videoFilePath, {\n mimeType: 'video/mp4', // Adjust according to the video format\n displayName: 'Uploaded Video for Analysis',\n});\n\nconst fileUri = uploadResult.file.uri;\n```\n\nNow you may configure the cache in your calls to `ai.generate`:\n\n```ts\nconst analyzeVideoResponse = await ai.generate({\n messages: [\n {\n role: 'user',\n content: [\n {\n media: {\n url: fileUri, // Use the uploaded file URL\n contentType: 'video/mp4',\n },\n },\n ],\n },\n {\n role: 'model',\n content: [\n {\n text: 'This video seems to contain several key moments. I will analyze it now and prepare to answer your questions.',\n },\n ],\n // Everything up to (including) this message will be cached.\n metadata: {\n cache: true,\n },\n },\n ],\n model: googleAI.model('gemini-2.5-flash-001'),\n prompt: query,\n});\n```\n\n### Supported Models for Context Caching\n\nOnly specific models, such as `gemini-2.5-flash` and `gemini-1.5-pro`, support context caching. If an unsupported model is used, an error will be raised, indicating that caching cannot be applied.\n\n### Further Reading\n\nSee more information regarding context caching on Google AI in their [documentation](https://ai.google.dev/gemini-api/docs/caching?lang=node).\n", "title": "Google Generative AI plugin", - "lang": "js" + "description": "This document describes the Google Generative AI plugin for Genkit, providing interfaces to Google's Gemini models, including text-to-speech, video generation, and context caching.", + "lang": "js", + "headers": "## Installation\n## Configuration\n## Usage\n## Gemini Files API\n## Fine-tuned models\n## Text-to-Speech (TTS) Models\n### Basic Usage\n### Multi-speaker Audio Generation\n### Configuration Options\n#### Voice Selection\n#### Speech Emphasis\n#### Advanced Speech Parameters\n## Video Generation (Veo) Models\n### Basic Usage: Text-to-Video Generation\n### Video Generation from Photo Reference\n### Configuration Options\n#### Veo Model Parameters\n## Context Caching\n### How to Use Context Caching\n### Example: Leveraging Large Texts with Context\n### Caching other modes of content\n### Supported Models for Context Caching\n### Further Reading\n" }, "js/plugins/lancedb.md": { - "text": "The LanceDB plugin provides indexer and retriever implementations that use [LanceDB](https://lancedb.com/), an open-source vector database for AI applications.\n\n## Installation\n\n```bash\nnpm install genkitx-lancedb\n```\n\n## Configuration\n\nTo use this plugin, specify it when you initialize Genkit:\n\n```ts\nimport { genkit } from 'genkit';\nimport { lancedb } from 'genkitx-lancedb';\nimport { googleAI } from '@genkit-ai/googleai';\n\nconst ai = genkit({\n plugins: [\n // Google AI provides the text-embedding-004 embedder\n googleAI(),\n\n // LanceDB requires an embedder to translate from text to vector\n lancedb([\n {\n dbUri: '.db', // optional lancedb uri, default to .db\n tableName: 'table', // optional table name, default to table\n embedder: googleAI.embedder('text-embedding-004'),\n },\n ]),\n ],\n});\n```\n\nYou must specify an embedder to use with LanceDB. You can also optionally configure:\n\n- `dbUri`: The URI for the LanceDB database (defaults to `.db`)\n- `tableName`: The name of the table to use (defaults to `table`)\n\n## Usage\n\nImport retriever and indexer references like so:\n\n```ts\nimport { lancedbRetrieverRef, lancedbIndexerRef, WriteMode } from 'genkitx-lancedb';\n```\n\n### Retrieval\n\nUse the retriever reference with `ai.retrieve()`:\n\n```ts\n// To use the default configuration:\nlet docs = await ai.retrieve({ \n retriever: lancedbRetrieverRef, \n query \n});\n\n// To specify custom options:\nexport const menuRetriever = lancedbRetrieverRef({\n tableName: \"table\", // Use the same table name as the indexer\n displayName: \"Menu\", // Use a custom display name\n});\n\ndocs = await ai.retrieve({ \n retriever: menuRetriever, \n query,\n options: { \n k: 3, // Limit to 3 results\n },\n});\n```\n\n### Indexing\n\nUse the indexer reference with `ai.index()`:\n\n```ts\n// To use the default configuration:\nawait ai.index({ indexer: lancedbIndexerRef, documents });\n\n// To specify custom options:\nexport const menuPdfIndexer = lancedbIndexerRef({\n // Using all defaults for dbUri, tableName, and embedder\n});\n\nawait ai.index({ \n indexer: menuPdfIndexer, \n documents,\n options: {\n writeMode: WriteMode.Overwrite,\n }\n});\n```\n\n## Example: Creating a RAG Flow\n\nHere's a complete example of creating a RAG (Retrieval-Augmented Generation) flow with LanceDB:\n\n```ts\nimport { lancedbIndexerRef, lancedb, lancedbRetrieverRef, WriteMode } from 'genkitx-lancedb';\nimport { googleAI } from '@genkit-ai/googleai';\nimport { z, genkit } from 'genkit';\nimport { Document } from 'genkit/retriever';\nimport { chunk } from 'llm-chunk';\nimport { readFile } from 'fs/promises';\nimport path from 'path';\nimport pdf from 'pdf-parse/lib/pdf-parse';\n\nconst ai = genkit({\n plugins: [\n googleAI(),\n lancedb([\n {\n dbUri: '.db',\n tableName: 'table',\n embedder: googleAI.embedder('text-embedding-004'),\n },\n ]),\n ],\n});\n\n// Define indexer\nexport const menuPdfIndexer = lancedbIndexerRef({\n // Using all defaults\n});\n\nconst chunkingConfig = {\n minLength: 1000,\n maxLength: 2000,\n splitter: 'sentence',\n overlap: 100,\n delimiters: '',\n};\n\nasync function extractTextFromPdf(filePath: string) {\n const pdfFile = path.resolve(filePath);\n const dataBuffer = await readFile(pdfFile);\n const data = await pdf(dataBuffer);\n return data.text;\n}\n\n// Define indexing flow\nexport const indexMenu = ai.defineFlow(\n {\n name: 'indexMenu',\n inputSchema: z.object({ filePath: z.string().describe('PDF file path') }),\n outputSchema: z.object({\n success: z.boolean(),\n documentsIndexed: z.number(),\n error: z.string().optional(),\n }),\n },\n async ({ filePath }) => {\n try {\n filePath = path.resolve(filePath);\n\n // Read the pdf\n const pdfTxt = await ai.run('extract-text', () =>\n extractTextFromPdf(filePath)\n );\n\n // Divide the pdf text into segments\n const chunks = await ai.run('chunk-it', async () =>\n chunk(pdfTxt, chunkingConfig)\n );\n\n // Convert chunks of text into documents to store in the index\n const documents = chunks.map((text) => {\n return Document.fromText(text, { filePath });\n });\n\n // Add documents to the index\n await ai.index({\n indexer: menuPdfIndexer,\n documents,\n options: {\n writeMode: WriteMode.Overwrite,\n }\n });\n\n return {\n success: true,\n documentsIndexed: documents.length,\n };\n } catch (err) {\n // For unexpected errors that throw exceptions\n return {\n success: false,\n documentsIndexed: 0,\n error: err instanceof Error ? err.message : String(err)\n };\n }\n }\n);\n\n// Define retriever\nexport const menuRetriever = lancedbRetrieverRef({\n tableName: \"table\", // Use the same table name as the indexer\n displayName: \"Menu\", // Use a custom display name\n});\n\n// Define retrieval flow\nexport const menuQAFlow = ai.defineFlow(\n { \n name: \"Menu\", \n inputSchema: z.object({ query: z.string() }), \n outputSchema: z.object({ answer: z.string() }) \n },\n async ({ query }) => {\n // Retrieve relevant documents\n const docs = await ai.retrieve({\n retriever: menuRetriever,\n query,\n options: { \n k: 3,\n },\n });\n\n // Generate response using retrieved documents\n const { text } = await ai.generate({\n model: googleAI.model('gemini-2.5-flash'),\n prompt: `\n You are acting as a helpful AI assistant that can answer \n questions about the food available on the menu at Genkit Grub Pub.\n\n Use only the context provided to answer the question.\n If you don't know, do not make up an answer.\n Do not add or change items on the menu.\n\n Question: ${query}\n `,\n docs,\n });\n \n return { answer: text };\n }\n);\n```\n\nSee the [Retrieval-augmented generation](/docs/rag) page for a general discussion on indexers and retrievers.\n\n## Learn More\n\nFor more information, feedback, or to report issues, visit the [LanceDB plugin GitHub repository](https://github.com/lancedb/genkitx-lancedb).\n", + "text": "# LanceDB plugin\n\nThe LanceDB plugin provides indexer and retriever implementations that use [LanceDB](https://lancedb.com/), an open-source vector database for AI applications.\n\n## Installation\n\n```bash\nnpm install genkitx-lancedb\n```\n\n## Configuration\n\nTo use this plugin, specify it when you initialize Genkit:\n\n```ts\nimport { genkit } from 'genkit';\nimport { lancedb } from 'genkitx-lancedb';\nimport { googleAI } from '@genkit-ai/googleai';\n\nconst ai = genkit({\n plugins: [\n // Google AI provides the gemini-embedding-001 embedder\n googleAI(),\n\n // LanceDB requires an embedder to translate from text to vector\n lancedb([\n {\n dbUri: '.db', // optional lancedb uri, default to .db\n tableName: 'table', // optional table name, default to table\n embedder: googleAI.embedder('gemini-embedding-001'),\n },\n ]),\n ],\n});\n```\n\nYou must specify an embedder to use with LanceDB. You can also optionally configure:\n\n- `dbUri`: The URI for the LanceDB database (defaults to `.db`)\n- `tableName`: The name of the table to use (defaults to `table`)\n\n## Usage\n\nImport retriever and indexer references like so:\n\n```ts\nimport { lancedbRetrieverRef, lancedbIndexerRef, WriteMode } from 'genkitx-lancedb';\n```\n\n### Retrieval\n\nUse the retriever reference with `ai.retrieve()`:\n\n```ts\n// To use the default configuration:\nlet docs = await ai.retrieve({ \n retriever: lancedbRetrieverRef, \n query \n});\n\n// To specify custom options:\nexport const menuRetriever = lancedbRetrieverRef({\n tableName: \"table\", // Use the same table name as the indexer\n displayName: \"Menu\", // Use a custom display name\n});\n\ndocs = await ai.retrieve({ \n retriever: menuRetriever, \n query,\n options: { \n k: 3, // Limit to 3 results\n },\n});\n```\n\n### Indexing\n\nUse the indexer reference with `ai.index()`:\n\n```ts\n// To use the default configuration:\nawait ai.index({ indexer: lancedbIndexerRef, documents });\n\n// To specify custom options:\nexport const menuPdfIndexer = lancedbIndexerRef({\n // Using all defaults for dbUri, tableName, and embedder\n});\n\nawait ai.index({ \n indexer: menuPdfIndexer, \n documents,\n options: {\n writeMode: WriteMode.Overwrite,\n }\n});\n```\n\n## Example: Creating a RAG Flow\n\nHere's a complete example of creating a RAG (Retrieval-Augmented Generation) flow with LanceDB:\n\n```ts\nimport { lancedbIndexerRef, lancedb, lancedbRetrieverRef, WriteMode } from 'genkitx-lancedb';\nimport { googleAI } from '@genkit-ai/googleai';\nimport { z, genkit } from 'genkit';\nimport { Document } from 'genkit/retriever';\nimport { chunk } from 'llm-chunk';\nimport { readFile } from 'fs/promises';\nimport path from 'path';\nimport pdf from 'pdf-parse/lib/pdf-parse';\n\nconst ai = genkit({\n plugins: [\n googleAI(),\n lancedb([\n {\n dbUri: '.db',\n tableName: 'table',\n embedder: googleAI.embedder('gemini-embedding-001'),\n },\n ]),\n ],\n});\n\n// Define indexer\nexport const menuPdfIndexer = lancedbIndexerRef({\n // Using all defaults\n});\n\nconst chunkingConfig = {\n minLength: 1000,\n maxLength: 2000,\n splitter: 'sentence',\n overlap: 100,\n delimiters: '',\n};\n\nasync function extractTextFromPdf(filePath: string) {\n const pdfFile = path.resolve(filePath);\n const dataBuffer = await readFile(pdfFile);\n const data = await pdf(dataBuffer);\n return data.text;\n}\n\n// Define indexing flow\nexport const indexMenu = ai.defineFlow(\n {\n name: 'indexMenu',\n inputSchema: z.object({ filePath: z.string().describe('PDF file path') }),\n outputSchema: z.object({\n success: z.boolean(),\n documentsIndexed: z.number(),\n error: z.string().optional(),\n }),\n },\n async ({ filePath }) => {\n try {\n filePath = path.resolve(filePath);\n\n // Read the pdf\n const pdfTxt = await ai.run('extract-text', () =>\n extractTextFromPdf(filePath)\n );\n\n // Divide the pdf text into segments\n const chunks = await ai.run('chunk-it', async () =>\n chunk(pdfTxt, chunkingConfig)\n );\n\n // Convert chunks of text into documents to store in the index\n const documents = chunks.map((text) => {\n return Document.fromText(text, { filePath });\n });\n\n // Add documents to the index\n await ai.index({\n indexer: menuPdfIndexer,\n documents,\n options: {\n writeMode: WriteMode.Overwrite,\n }\n });\n\n return {\n success: true,\n documentsIndexed: documents.length,\n };\n } catch (err) {\n // For unexpected errors that throw exceptions\n return {\n success: false,\n documentsIndexed: 0,\n error: err instanceof Error ? err.message : String(err)\n };\n }\n }\n);\n\n// Define retriever\nexport const menuRetriever = lancedbRetrieverRef({\n tableName: \"table\", // Use the same table name as the indexer\n displayName: \"Menu\", // Use a custom display name\n});\n\n// Define retrieval flow\nexport const menuQAFlow = ai.defineFlow(\n { \n name: \"Menu\", \n inputSchema: z.object({ query: z.string() }), \n outputSchema: z.object({ answer: z.string() }) \n },\n async ({ query }) => {\n // Retrieve relevant documents\n const docs = await ai.retrieve({\n retriever: menuRetriever,\n query,\n options: { \n k: 3,\n },\n });\n\n // Generate response using retrieved documents\n const { text } = await ai.generate({\n model: googleAI.model('gemini-2.5-flash'),\n prompt: `\n You are acting as a helpful AI assistant that can answer \n questions about the food available on the menu at Genkit Grub Pub.\n\n Use only the context provided to answer the question.\n If you don't know, do not make up an answer.\n Do not add or change items on the menu.\n\n Question: ${query}\n `,\n docs,\n });\n \n return { answer: text };\n }\n);\n```\n\nSee the [Retrieval-augmented generation](/docs/rag) page for a general discussion on indexers and retrievers.\n\n## Learn More\n\nFor more information, feedback, or to report issues, visit the [LanceDB plugin GitHub repository](https://github.com/lancedb/genkitx-lancedb).\n", "title": "LanceDB plugin", - "lang": "js" + "description": "This document describes the LanceDB plugin for Genkit, which provides indexer and retriever implementations for LanceDB, an open-source vector database for AI applications.", + "lang": "js", + "headers": "## Installation\n## Configuration\n## Usage\n### Retrieval\n### Indexing\n## Example: Creating a RAG Flow\n## Learn More\n" + }, + "js/plugins/mcp.md": { + "text": "# Model Context Protocol (MCP) plugin\n\nThe Genkit MCP plugin provides integration between Genkit and the [Model Context Protocol](https://modelcontextprotocol.io) (MCP). MCP is an open standard allowing developers to build \"servers\" which provide tools, resources, and prompts to clients. Genkit MCP allows Genkit developers to:\n- Consume MCP tools, prompts, and resources as a client using `createMcpHost` or `createMcpClient`.\n- Provide Genkit tools and prompts as an MCP server using `createMcpServer`.\n\n## Installation\n\nTo get started, you'll need Genkit and the MCP plugin:\n\n```bash\nnpm i genkit @genkit-ai/mcp\n```\n\n## MCP Host\n\nTo connect to one or more MCP servers, you use the `createMcpHost` function. This function returns a `GenkitMcpHost` instance that manages connections to the configured MCP servers.\n\n```ts\nimport { googleAI } from '@genkit-ai/googleai';\nimport { createMcpHost } from '@genkit-ai/mcp';\nimport { genkit } from 'genkit';\n\nconst mcpHost = createMcpHost({\n name: 'myMcpClients', // A name for the host plugin itself\n mcpServers: {\n // Each key (e.g., 'fs', 'git') becomes a namespace for the server's tools.\n fs: {\n command: 'npx',\n args: ['-y', '@modelcontextprotocol/server-filesystem', process.cwd()],\n },\n memory: {\n command: 'npx',\n args: ['-y', '@modelcontextprotocol/server-memory'],\n },\n },\n});\n\nconst ai = genkit({\n plugins: [googleAI()],\n});\n\n(async () => {\n // Provide MCP tools to the model of your choice.\n const { text } = await ai.generate({\n model: googleAI.model('gemini-2.5-flash'),\n prompt: `Analyze all files in ${process.cwd()}.`,\n tools: await mcpHost.getActiveTools(ai),\n resources: await mcpHost.getActiveResources(ai),\n });\n\n console.log(text);\n\n await mcpHost.close();\n})();\n```\n\nThe `createMcpHost` function initializes a `GenkitMcpHost` instance, which handles the lifecycle and communication with the defined MCP servers.\n\n### `createMcpHost()` Options\n\n```ts\nexport interface McpHostOptions {\n /**\n * An optional client name for this MCP host. This name is advertised to MCP Servers\n * as the connecting client name. Defaults to 'genkit-mcp'.\n */\n name?: string;\n /**\n * An optional version for this MCP host. Primarily for\n * logging and identification within Genkit.\n * Defaults to '1.0.0'.\n */\n version?: string;\n /**\n * A record for configuring multiple MCP servers. Each server connection is\n * controlled by a `GenkitMcpClient` instance managed by `GenkitMcpHost`.\n * The key in the record is used as the identifier for the MCP server.\n */\n mcpServers?: Record;\n /**\n * If true, tool responses from the MCP server will be returned in their raw\n * MCP format. Otherwise (default), they are processed and potentially\n * simplified for better compatibility with Genkit's typical data structures.\n */\n rawToolResponses?: boolean;\n /**\n * When provided, each connected MCP server will be sent the roots specified here.\n * Overridden by any specific roots sent in the `mcpServers` config for a given server.\n */\n roots?: Root[];\n}\n\n/**\n * Configuration for an individual MCP server. The interface should be familiar\n * and compatible with existing tool configurations e.g. Cursor or Claude\n * Desktop.\n *\n * In addition to stdio servers, remote servers are supported via URL and\n * custom/arbitary transports are supported as well.\n */\nexport type McpServerConfig = (\n | McpStdioServerConfig\n | McpStreamableHttpConfig\n | McpTransportServerConfig\n) &\n McpServerControls;\n\n\nexport type McpStdioServerConfig = StdioServerParameters;\n\nexport type McpStreamableHttpConfig = {\n url: string;\n} & Omit;\n\nexport type McpTransportServerConfig = {\n transport: Transport;\n};\n\nexport interface McpServerControls {\n /** \n * when true, the server will be stopped and its registered components will\n * not appear in lists/plugins/etc \n */\n disabled?: boolean;\n /** MCP roots configuration. See: https://modelcontextprotocol.io/docs/concepts/roots */\n roots?: Root[];\n}\n\n// from '@modelcontextprotocol/sdk/client/stdio.js'\nexport type StdioServerParameters = {\n /**\n * The executable to run to start the server.\n */\n command: string;\n /**\n * Command line arguments to pass to the executable.\n */\n args?: string[];\n /**\n * The environment to use when spawning the process.\n *\n * If not specified, the result of getDefaultEnvironment() will be used.\n */\n env?: Record;\n /**\n * How to handle stderr of the child process. This matches the semantics of Node's `child_process.spawn`.\n *\n * The default is \"inherit\", meaning messages to stderr will be printed to the parent process's stderr.\n */\n stderr?: IOType | Stream | number;\n /**\n * The working directory to use when spawning the process.\n *\n * If not specified, the current working directory will be inherited.\n */\n cwd?: string;\n};\n\n// from '@modelcontextprotocol/sdk/client/streamableHttp.js'\nexport type StreamableHTTPClientTransportOptions = {\n /**\n * An OAuth client provider to use for authentication.\n *\n * When an `authProvider` is specified and the connection is started:\n * 1. The connection is attempted with any existing access token from the `authProvider`.\n * 2. If the access token has expired, the `authProvider` is used to refresh the token.\n * 3. If token refresh fails or no access token exists, and auth is required, `OAuthClientProvider.redirectToAuthorization` is called, and an `UnauthorizedError` will be thrown from `connect`/`start`.\n *\n * After the user has finished authorizing via their user agent, and is redirected back to the MCP client application, call `StreamableHTTPClientTransport.finishAuth` with the authorization code before retrying the connection.\n *\n * If an `authProvider` is not provided, and auth is required, an `UnauthorizedError` will be thrown.\n *\n * `UnauthorizedError` might also be thrown when sending any message over the transport, indicating that the session has expired, and needs to be re-authed and reconnected.\n */\n authProvider?: OAuthClientProvider;\n /**\n * Customizes HTTP requests to the server.\n */\n requestInit?: RequestInit;\n /**\n * Custom fetch implementation used for all network requests.\n */\n fetch?: FetchLike;\n /**\n * Options to configure the reconnection behavior.\n */\n reconnectionOptions?: StreamableHTTPReconnectionOptions;\n /**\n * Session ID for the connection. This is used to identify the session on the server.\n * When not provided and connecting to a server that supports session IDs, the server will generate a new session ID.\n */\n sessionId?: string;\n};\n```\n\n## MCP Client (Single Server)\n\nFor scenarios where you only need to connect to a single MCP server, or prefer to manage client instances individually, you can use `createMcpClient`.\n\n```ts\nimport { googleAI } from '@genkit-ai/googleai';\nimport { createMcpClient } from '@genkit-ai/mcp';\nimport { genkit } from 'genkit';\n\nconst myFsClient = createMcpClient({\n name: 'myFileSystemClient', // A unique name for this client instance\n mcpServer: {\n command: 'npx',\n args: ['-y', '@modelcontextprotocol/server-filesystem', process.cwd()],\n },\n // rawToolResponses: true, // Optional: get raw MCP responses\n});\n\n// In your Genkit configuration:\nconst ai = genkit({\n plugins: [googleAI()],\n});\n\n(async () => {\n await myFsClient.ready();\n\n // Retrieve tools from this specific client\n const fsTools = await myFsClient.getActiveTools(ai);\n\n const { text } = await ai.generate({\n model: googleAI.model('gemini-2.5-flash'), // Replace with your model\n prompt: 'List files in ' + process.cwd(),\n tools: fsTools,\n });\n console.log(text);\n\n await myFsClient.disable();\n})();\n```\n\n### `createMcpClient()` Options\n\nThe `createMcpClient` function takes an `McpClientOptions` object:\n- **`name`**: (required, string) A unique name for this client instance. This name will be used as the namespace for its tools and prompts.\n- **`version`**: (optional, string) Version for this client instance. Defaults to \"1.0.0\".\n- Additionally, it supports all options from `McpServerConfig` (e.g., `disabled`, `rawToolResponses`, and transport configurations), as detailed in the `createMcpHost` options section.\n\n### Using MCP Actions (Tools, Prompts)\n\nBoth `GenkitMcpHost` (via `getActiveTools()`) and `GenkitMcpClient` (via `getActiveTools()`) discover available tools from their connected and enabled MCP server(s). These tools are standard Genkit `ToolAction` instances and can be provided to Genkit models.\n\nMCP prompts can be fetched using `McpHost.getPrompt(serverName, promptName)` or `mcpClient.getPrompt(promptName)`. These return an `ExecutablePrompt`.\n\nAll MCP actions (tools, prompts, resources) are namespaced.\n- For `createMcpHost`, the namespace is the key you provide for that server in the `mcpServers` configuration (e.g., `localFs/read_file`).\n- For `createMcpClient`, the namespace is the `name` you provide in its options (e.g., `myFileSystemClient/list_resources`).\n\n### Tool Responses\n\nMCP tools return a `content` array as opposed to a structured response like most Genkit tools. The Genkit MCP plugin attempts to parse and coerce returned content:\n\n1. If the content is text and valid JSON, it is parsed and returned as a JSON object.\n2. If the content is text but not valid JSON, the raw text is returned.\n3. If the content contains a single non-text part (e.g., an image), that part is returned directly.\n4. If the content contains multiple or mixed parts (e.g., text and an image), the full content response array is returned.\n\n## MCP Server\n\nYou can also expose all of the tools and prompts from a Genkit instance as an MCP server using the `createMcpServer` function.\n\n```ts\nimport { googleAI } from '@genkit-ai/googleai';\nimport { createMcpServer } from '@genkit-ai/mcp';\nimport { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';\nimport { genkit, z } from 'genkit/beta';\n\nconst ai = genkit({\n plugins: [googleAI()],\n});\n\nai.defineTool(\n {\n name: 'add',\n description: 'add two numbers together',\n inputSchema: z.object({ a: z.number(), b: z.number() }),\n outputSchema: z.number(),\n },\n async ({ a, b }) => {\n return a + b;\n }\n);\n\nai.definePrompt(\n {\n name: 'happy',\n description: 'everybody together now',\n input: {\n schema: z.object({\n action: z.string().default('clap your hands').optional(),\n }),\n },\n },\n `If you're happy and you know it, {{action}}.`\n);\n\nai.defineResource(\n {\n name: 'my resouces',\n uri: 'my://resource',\n },\n async () => {\n return {\n content: [\n {\n text: 'my resource',\n },\n ],\n };\n }\n);\n\nai.defineResource(\n {\n name: 'file',\n template: 'file://{path}',\n },\n async ({ uri }) => {\n return {\n content: [\n {\n text: `file contents for ${uri}`,\n },\n ],\n };\n }\n);\n\n// Use createMcpServer\nconst server = createMcpServer(ai, {\n name: 'example_server',\n version: '0.0.1',\n});\n// Setup (async) then starts with stdio transport by default\nserver.setup().then(async () => {\n await server.start();\n const transport = new StdioServerTransport();\n await server!.server?.connect(transport);\n});\n```\n\nThe `createMcpServer` function returns a `GenkitMcpServer` instance. The `start()` method on this instance will start an MCP server (using the stdio transport by default) that exposes all registered Genkit tools and prompts. To start the server with a different MCP transport, you can pass the transport instance to the `start()` method (e.g., `server.start(customMcpTransport)`).\n\n### `createMcpServer()` Options\n- **`name`**: (required, string) The name you want to give your server for MCP inspection.\n- **`version`**: (optional, string) The version your server will advertise to clients. Defaults to \"1.0.0\".\n\n### Known Limitations\n\n- MCP prompts are only able to take string parameters, so inputs to schemas must be objects with only string property values.\n- MCP prompts only support `user` and `model` messages. `system` messages are not supported.\n- MCP prompts only support a single \"type\" within a message so you can't mix media and text in the same message.\n\n### Testing your MCP server\n\nYou can test your MCP server using the official inspector. For example, if your server code compiled into `dist/index.js`, you could run:\n\n npx @modelcontextprotocol/inspector dist/index.js\n\nOnce you start the inspector, you can list prompts and actions and test them out manually.\n", + "title": "Model Context Protocol (MCP) plugin", + "description": "The Genkit MCP plugin provides integration between Genkit and the Model Context Protocol (MCP).", + "lang": "js", + "headers": "## Installation\n## MCP Host\n### `createMcpHost()` Options\n## MCP Client (Single Server)\n### `createMcpClient()` Options\n### Using MCP Actions (Tools, Prompts)\n### Tool Responses\n## MCP Server\n### `createMcpServer()` Options\n### Known Limitations\n### Testing your MCP server\n" }, "js/plugins/neo4j.md": { - "text": "The Neo4j plugin provides indexer and retriever implementations that use the\n[Neo4j](https://neo4j.com/) graph database for vector search capabilities.\n\n## Installation\n\n```bash\nnpm install genkitx-neo4j\n```\n\n## Configuration\n\nTo use this plugin, specify it when you initialize Genkit:\n\n```ts\nimport { genkit } from 'genkit';\nimport { neo4j } from 'genkitx-neo4j';\nimport { textEmbedding004 } from 'genkitx-googleai';\n\nconst ai = genkit({\n plugins: [\n neo4j([\n {\n indexId: 'bob-facts',\n embedder: textEmbedding004,\n },\n ]),\n ],\n});\n```\n\nYou must specify a Neo4j index ID and the embedding model you want to use.\n\n### Connection Configuration\n\nThere are two ways to configure the Neo4j connection:\n\n1. Using environment variables:\n\n```\nNEO4J_URI=bolt://localhost:7687 # Neo4j's binary protocol\nNEO4J_USERNAME=neo4j\nNEO4J_PASSWORD=password\nNEO4J_DATABASE=neo4j # Optional: specify database name\n```\n\n2. Using the `clientParams` option:\n\n```ts\nneo4j([\n {\n indexId: 'bob-facts',\n embedder: textEmbedding004,\n clientParams: {\n url: 'bolt://localhost:7687', // Neo4j's binary protocol\n username: 'neo4j',\n password: 'password',\n database: 'neo4j', // Optional\n },\n },\n]),\n```\n\n> Note: The `bolt://` protocol is Neo4j's proprietary binary protocol designed for efficient client-server communication.\n\n### Configuration Options\n\nThe Neo4j plugin accepts the following configuration options:\n\n- `indexId`: (required) The name of the index to use in Neo4j\n- `embedder`: (required) The embedding model to use\n- `clientParams`: (optional) Neo4j connection configuration\n\n## Usage\n\nImport retriever and indexer references like so:\n\n```ts\nimport { neo4jRetrieverRef } from 'genkitx-neo4j';\nimport { neo4jIndexerRef } from 'genkitx-neo4j';\n```\n\n### Retrieval\n\nUse the retriever reference with `ai.retrieve()`:\n\n```ts\n// To use the index you configured when you loaded the plugin:\nlet docs = await ai.retrieve({ \n retriever: neo4jRetrieverRef, \n query,\n // Optional: limit number of results (max 1000)\n k: 5 \n});\n\n// To specify an index:\nexport const bobFactsRetriever = neo4jRetrieverRef({\n indexId: 'bob-facts',\n // Optional: custom display name\n displayName: 'Bob Facts Database'\n});\ndocs = await ai.retrieve({ \n retriever: bobFactsRetriever, \n query,\n k: 10 \n});\n```\n\n### Indexing\n\nUse the indexer reference with `ai.index()`:\n\n```ts\n// To use the index you configured when you loaded the plugin:\nawait ai.index({ indexer: neo4jIndexerRef, documents });\n\n// To specify an index:\nexport const bobFactsIndexer = neo4jIndexerRef({\n indexId: 'bob-facts',\n // Optional: custom display name\n displayName: 'Bob Facts Database'\n});\nawait ai.index({ indexer: bobFactsIndexer, documents });\n```\n\nSee the [Retrieval-augmented generation](/docs/rag) page for a general\ndiscussion on indexers and retrievers.\n\n## Learn More\n\nFor more information, feedback, or to report issues, visit the [Neo4j plugin GitHub repository](https://github.com/neo4j-partners/genkitx-neo4j/blob/main/README.md).\n", + "text": "# Neo4j plugin\n\nThe Neo4j plugin provides indexer and retriever implementations that use the\n[Neo4j](https://neo4j.com/) graph database for vector search capabilities.\n\n## Installation\n\n```bash\nnpm install genkitx-neo4j\n```\n\n## Configuration\n\nTo use this plugin, specify it when you initialize Genkit:\n\n```ts\nimport { genkit } from 'genkit';\nimport { neo4j } from 'genkitx-neo4j';\nimport { googleAI } from '@genkit-ai/googleai';\n\nconst ai = genkit({\n plugins: [\n neo4j([\n {\n indexId: 'bob-facts',\n embedder: googleAI.embedder('gemini-embedding-001'),\n },\n ]),\n ],\n});\n```\n\nYou must specify a Neo4j index ID and the embedding model you want to use.\n\n### Connection Configuration\n\nThere are two ways to configure the Neo4j connection:\n\n1. Using environment variables:\n\n```\nNEO4J_URI=bolt://localhost:7687 # Neo4j's binary protocol\nNEO4J_USERNAME=neo4j\nNEO4J_PASSWORD=password\nNEO4J_DATABASE=neo4j # Optional: specify database name\n```\n\n2. Using the `clientParams` option:\n\n```ts\nneo4j([\n {\n indexId: 'bob-facts',\n embedder: googleAI.embedder('gemini-embedding-001'),\n clientParams: {\n url: 'bolt://localhost:7687', // Neo4j's binary protocol\n username: 'neo4j',\n password: 'password',\n database: 'neo4j', // Optional\n },\n },\n]),\n```\n\n> Note: The `bolt://` protocol is Neo4j's proprietary binary protocol designed for efficient client-server communication.\n\n### Configuration Options\n\nThe Neo4j plugin accepts the following configuration options:\n\n- `indexId`: (required) The name of the index to use in Neo4j\n- `embedder`: (required) The embedding model to use\n- `clientParams`: (optional) Neo4j connection configuration\n\n## Usage\n\nImport retriever and indexer references like so:\n\n```ts\nimport { neo4jRetrieverRef } from 'genkitx-neo4j';\nimport { neo4jIndexerRef } from 'genkitx-neo4j';\n```\n\n### Retrieval\n\nUse the retriever reference with `ai.retrieve()`:\n\n```ts\n// To use the index you configured when you loaded the plugin:\nlet docs = await ai.retrieve({ \n retriever: neo4jRetrieverRef, \n query,\n // Optional: limit number of results (max 1000)\n k: 5 \n});\n\n// To specify an index:\nexport const bobFactsRetriever = neo4jRetrieverRef({\n indexId: 'bob-facts',\n // Optional: custom display name\n displayName: 'Bob Facts Database'\n});\ndocs = await ai.retrieve({ \n retriever: bobFactsRetriever, \n query,\n k: 10 \n});\n```\n\n### Indexing\n\nUse the indexer reference with `ai.index()`:\n\n```ts\n// To use the index you configured when you loaded the plugin:\nawait ai.index({ indexer: neo4jIndexerRef, documents });\n\n// To specify an index:\nexport const bobFactsIndexer = neo4jIndexerRef({\n indexId: 'bob-facts',\n // Optional: custom display name\n displayName: 'Bob Facts Database'\n});\nawait ai.index({ indexer: bobFactsIndexer, documents });\n```\n\nSee the [Retrieval-augmented generation](/docs/rag) page for a general\ndiscussion on indexers and retrievers.\n\n## Learn More\n\nFor more information, feedback, or to report issues, visit the [Neo4j plugin GitHub repository](https://github.com/neo4j-partners/genkitx-neo4j/blob/main/README.md).\n", "title": "Neo4j plugin", - "lang": "js" + "description": "This document describes the Neo4j plugin for Genkit, which provides indexer and retriever implementations that use the Neo4j graph database for vector search capabilities.", + "lang": "js", + "headers": "## Installation\n## Configuration\n### Connection Configuration\n### Configuration Options\n## Usage\n### Retrieval\n### Indexing\n## Learn More\n" }, "js/plugins/ollama.md": { - "text": "The Ollama plugin provides interfaces to any of the local LLMs supported by\n[Ollama](https://ollama.com/).\n\n## Installation\n\n```bash\nnpm install genkitx-ollama\n```\n\n## Configuration\n\nThis plugin requires that you first install and run the Ollama server. You can\nfollow the instructions on: [Download Ollama](https://ollama.com/download).\n\nYou can use the Ollama CLI to download the model you are interested in. For\nexample:\n\n```bash\nollama pull gemma\n```\n\nTo use this plugin, specify it when you initialize Genkit:\n\n```ts\nimport { genkit } from 'genkit';\nimport { ollama } from 'genkitx-ollama';\n\nconst ai = genkit({\n plugins: [\n ollama({\n models: [\n {\n name: 'gemma',\n type: 'generate', // type: 'chat' | 'generate' | undefined\n },\n ],\n serverAddress: 'http://127.0.0.1:11434', // default local address\n }),\n ],\n});\n```\n\n### Authentication\n\nIf you would like to access remote deployments of Ollama that require custom\nheaders (static, such as API keys, or dynamic, such as auth headers), you can\nspecify those in the Ollama config plugin:\n\nStatic headers:\n\n```ts\nollama({\n models: [{ name: 'gemma'}],\n requestHeaders: {\n 'api-key': 'API Key goes here'\n },\n serverAddress: 'https://my-deployment',\n}),\n```\n\nYou can also dynamically set headers per request. Here's an example of how to\nset an ID token using the Google Auth library:\n\n```ts\nimport { GoogleAuth } from 'google-auth-library';\nimport { ollama } from 'genkitx-ollama';\nimport { genkit } from 'genkit';\n\nconst ollamaCommon = { models: [{ name: 'gemma:2b' }] };\n\nconst ollamaDev = {\n ...ollamaCommon,\n serverAddress: 'http://127.0.0.1:11434',\n};\n\nconst ollamaProd = {\n ...ollamaCommon,\n serverAddress: 'https://my-deployment',\n requestHeaders: async (params) => {\n const headers = await fetchWithAuthHeader(params.serverAddress);\n return { Authorization: headers['Authorization'] };\n },\n};\n\nconst ai = genkit({\n plugins: [ollama(isDevEnv() ? ollamaDev : ollamaProd)],\n});\n\n// Function to lazily load GoogleAuth client\nlet auth: GoogleAuth;\nfunction getAuthClient() {\n if (!auth) {\n auth = new GoogleAuth();\n }\n return auth;\n}\n\n// Function to fetch headers, reusing tokens when possible\nasync function fetchWithAuthHeader(url: string) {\n const client = await getIdTokenClient(url);\n const headers = await client.getRequestHeaders(url); // Auto-manages token refresh\n return headers;\n}\n\nasync function getIdTokenClient(url: string) {\n const auth = getAuthClient();\n const client = await auth.getIdTokenClient(url);\n return client;\n}\n```\n\n## Usage\n\nThis plugin doesn't statically export model references. Specify one of the\nmodels you configured using a string identifier:\n\n```ts\nconst llmResponse = await ai.generate({\n model: 'ollama/gemma',\n prompt: 'Tell me a joke.',\n});\n```\n\n## Embedders\n\nThe Ollama plugin supports embeddings, which can be used for similarity searches\nand other NLP tasks.\n\n```ts\nconst ai = genkit({\n plugins: [\n ollama({\n serverAddress: 'http://localhost:11434',\n embedders: [{ name: 'nomic-embed-text', dimensions: 768 }],\n }),\n ],\n});\n\nasync function getEmbeddings() {\n const embeddings = (\n await ai.embed({\n embedder: 'ollama/nomic-embed-text',\n content: 'Some text to embed!',\n })\n )[0].embedding;\n\n return embeddings;\n}\n\ngetEmbeddings().then((e) => console.log(e));\n```\n", + "text": "# Ollama plugin\n\nThe Ollama plugin provides interfaces to any of the local LLMs supported by\n[Ollama](https://ollama.com/).\n\n## Installation\n\n```bash\nnpm install genkitx-ollama\n```\n\n## Configuration\n\nThis plugin requires that you first install and run the Ollama server. You can\nfollow the instructions on: [Download Ollama](https://ollama.com/download).\n\nYou can use the Ollama CLI to download the model you are interested in. For\nexample:\n\n```bash\nollama pull gemma\n```\n\nTo use this plugin, specify it when you initialize Genkit:\n\n```ts\nimport { genkit } from 'genkit';\nimport { ollama } from 'genkitx-ollama';\n\nconst ai = genkit({\n plugins: [\n ollama({\n models: [\n {\n name: 'gemma',\n type: 'generate', // type: 'chat' | 'generate' | undefined\n },\n ],\n serverAddress: 'http://127.0.0.1:11434', // default local address\n }),\n ],\n});\n```\n\n### Authentication\n\nIf you would like to access remote deployments of Ollama that require custom\nheaders (static, such as API keys, or dynamic, such as auth headers), you can\nspecify those in the Ollama config plugin:\n\nStatic headers:\n\n```ts\nollama({\n models: [{ name: 'gemma'}],\n requestHeaders: {\n 'api-key': 'API Key goes here'\n },\n serverAddress: 'https://my-deployment',\n}),\n```\n\nYou can also dynamically set headers per request. Here's an example of how to\nset an ID token using the Google Auth library:\n\n```ts\nimport { GoogleAuth } from 'google-auth-library';\nimport { ollama } from 'genkitx-ollama';\nimport { genkit } from 'genkit';\n\nconst ollamaCommon = { models: [{ name: 'gemma:2b' }] };\n\nconst ollamaDev = {\n ...ollamaCommon,\n serverAddress: 'http://127.0.0.1:11434',\n};\n\nconst ollamaProd = {\n ...ollamaCommon,\n serverAddress: 'https://my-deployment',\n requestHeaders: async (params) => {\n const headers = await fetchWithAuthHeader(params.serverAddress);\n return { Authorization: headers['Authorization'] };\n },\n};\n\nconst ai = genkit({\n plugins: [ollama(isDevEnv() ? ollamaDev : ollamaProd)],\n});\n\n// Function to lazily load GoogleAuth client\nlet auth: GoogleAuth;\nfunction getAuthClient() {\n if (!auth) {\n auth = new GoogleAuth();\n }\n return auth;\n}\n\n// Function to fetch headers, reusing tokens when possible\nasync function fetchWithAuthHeader(url: string) {\n const client = await getIdTokenClient(url);\n const headers = await client.getRequestHeaders(url); // Auto-manages token refresh\n return headers;\n}\n\nasync function getIdTokenClient(url: string) {\n const auth = getAuthClient();\n const client = await auth.getIdTokenClient(url);\n return client;\n}\n```\n\n## Usage\n\nThis plugin doesn't statically export model references. Specify one of the\nmodels you configured using a string identifier:\n\n```ts\nconst llmResponse = await ai.generate({\n model: 'ollama/gemma',\n prompt: 'Tell me a joke.',\n});\n```\n\n## Embedders\n\nThe Ollama plugin supports embeddings, which can be used for similarity searches\nand other NLP tasks.\n\n```ts\nconst ai = genkit({\n plugins: [\n ollama({\n serverAddress: 'http://localhost:11434',\n embedders: [{ name: 'nomic-embed-text', dimensions: 768 }],\n }),\n ],\n});\n\nasync function getEmbeddings() {\n const embeddings = (\n await ai.embed({\n embedder: 'ollama/nomic-embed-text',\n content: 'Some text to embed!',\n })\n )[0].embedding;\n\n return embeddings;\n}\n\ngetEmbeddings().then((e) => console.log(e));\n```\n", "title": "Ollama plugin", - "lang": "js" + "description": "This document describes the Ollama plugin for Genkit, which provides interfaces to local LLMs supported by Ollama, including installation, configuration, and usage for models and embedders.", + "lang": "js", + "headers": "## Installation\n## Configuration\n### Authentication\n## Usage\n## Embedders\n" + }, + "js/plugins/openai.md": { + "text": "# OpenAI Plugin\n\nThe `@genkit-ai/compat-oai` package includes a pre-configured plugin for official [OpenAI models](https://platform.openai.com/docs/models).\n\n:::note\nThe OpenAI plugin is built on top of the `openAICompatible` plugin. It is pre-configured for OpenAI's API endpoints.\n:::\n\n## Installation\n\n```bash\nnpm install @genkit-ai/compat-oai\n```\n\n## Configuration\n\nTo use this plugin, import `openAI` and specify it when you initialize Genkit:\n\n```ts\nimport { genkit } from 'genkit';\nimport { openAI } from '@genkit-ai/compat-oai/openai';\n\nexport const ai = genkit({\n plugins: [openAI()],\n});\n```\n\nThe plugin requires an API key for the OpenAI API. You can get one from the [OpenAI Platform](https://platform.openai.com/api-keys).\n\nConfigure the plugin to use your API key by doing one of the following:\n\n- Set the `OPENAI_API_KEY` environment variable to your API key.\n- Specify the API key when you initialize the plugin:\n\n ```ts\n openAI({ apiKey: yourKey });\n ```\n\n However, don't embed your API key directly in code! Use this feature only in\n conjunction with a service like Google Cloud Secret Manager or similar.\n\n## Usage\n\nThe plugin provides helpers to reference supported models and embedders.\n\n### Chat Models\n\nYou can reference chat models like `gpt-4o` and `gpt-4-turbo` using the `openAI.model()` helper.\n\n```ts\nimport { genkit, z } from 'genkit';\nimport { openAI } from '@genkit-ai/compat-oai/openai';\n\nconst ai = genkit({\n plugins: [openAI()],\n});\n\nexport const jokeFlow = ai.defineFlow(\n {\n name: 'jokeFlow',\n inputSchema: z.object({ subject: z.string() }),\n outputSchema: z.object({ joke: z.string() }),\n },\n async ({ subject }) => {\n const llmResponse = await ai.generate({\n prompt: `tell me a joke about ${subject}`,\n model: openAI.model('gpt-4o'),\n });\n return { joke: llmResponse.text };\n },\n);\n```\n\nYou can also pass model-specific configuration:\n\n```ts\nconst llmResponse = await ai.generate({\n prompt: `tell me a joke about ${subject}`,\n model: openAI.model('gpt-4o'),\n config: {\n temperature: 0.7,\n },\n});\n```\n\n### Image Generation Models\n\nThe plugin supports image generation models like DALL-E 3.\n\n```ts\nimport { genkit } from 'genkit';\nimport { openAI } from '@genkit-ai/compat-oai/openai';\n\nconst ai = genkit({\n plugins: [openAI()],\n});\n\n// Reference an image generation model\nconst dalle3 = openAI.model('dall-e-3');\n\n// Use it to generate an image\nconst imageResponse = await ai.generate({\n model: dalle3,\n prompt: 'A photorealistic image of a cat programming a computer.',\n config: {\n size: '1024x1024',\n style: 'vivid',\n },\n});\n\nconst imageUrl = imageResponse.media()?.url;\n```\n\n### Text Embedding Models\n\nYou can use text embedding models to create vector embeddings from text.\n\n```ts\nimport { genkit, z } from 'genkit';\nimport { openAI } from '@genkit-ai/compat-oai/openai';\n\nconst ai = genkit({\n plugins: [openAI()],\n});\n\nexport const embedFlow = ai.defineFlow(\n {\n name: 'embedFlow',\n inputSchema: z.object({ text: z.string() }),\n outputSchema: z.object({ embedding: z.string() }),\n },\n async ({ text }) => {\n const embedding = await ai.embed({\n embedder: openAI.embedder('text-embedding-ada-002'),\n content: text,\n });\n\n return { embedding: JSON.stringify(embedding) };\n },\n);\n```\n\n### Audio Transcription and Speech Models\n\nThe OpenAI plugin also supports audio models for transcription (speech-to-text) and speech generation (text-to-speech).\n\n#### Transcription (Speech-to-Text)\n\nUse models like `whisper-1` to transcribe audio files.\n\n```ts\nimport { genkit } from 'genkit';\nimport { openAI } from '@genkit-ai/compat-oai/openai';\nimport * as fs from 'fs';\n\nconst ai = genkit({\n plugins: [openAI()],\n});\n\nconst whisper = openAI.model('whisper-1');\nconst audioFile = fs.readFileSync('path/to/your/audio.mp3');\n\nconst transcription = await ai.generate({\n model: whisper,\n prompt: [\n {\n media: {\n contentType: 'audio/mp3',\n url: `data:audio/mp3;base64,${audioFile.toString('base64')}`,\n },\n },\n ],\n});\n\nconsole.log(transcription.text());\n```\n\n#### Speech Generation (Text-to-Speech)\n\nUse models like `tts-1` to generate speech from text.\n\n```ts\nimport { genkit } from 'genkit';\nimport { openAI } from '@genkit-ai/compat-oai/openai';\nimport * as fs from 'fs';\n\nconst ai = genkit({\n plugins: [openAI()],\n});\n\nconst tts = openAI.model('tts-1');\nconst speechResponse = await ai.generate({\n model: tts,\n prompt: 'Hello, world! This is a test of text-to-speech.',\n config: {\n voice: 'alloy',\n },\n});\n\nconst audioData = speechResponse.media();\nif (audioData) {\n fs.writeFileSync('output.mp3', Buffer.from(audioData.url.split(',')[1], 'base64'));\n}\n```\n\n## Advanced usage\n\n### Passthrough configuration\n\nYou can pass configuration options that are not defined in the plugin's custom configuration schema. This\npermits you to access new models and features without having to update your Genkit version.\n\n```ts\nimport { genkit } from 'genkit';\nimport { openAI } from '@genkit-ai/compat-oai/openai';\n\nconst ai = genkit({\n plugins: [openAI()],\n});\n\nconst llmResponse = await ai.generate({\n prompt: `Tell me a cool story`,\n model: openAI.model('gpt-4-new'), // hypothetical new model\n config: {\n seed: 123,\n new_feature_parameter: ... // hypothetical config needed for new model\n },\n});\n```\n\nGenkit passes this config as-is to the OpenAI API giving you access to the new model features.\nNote that the field name and types are not validated by Genkit and should match the OpenAI API\nspecification to work.\n\n### Web-search built-in tool\n\nSome OpenAI models support web search. You can enable it in the `config` block:\n\n```ts\nimport { genkit } from 'genkit';\nimport { openAI } from '@genkit-ai/compat-oai/openai';\n\nconst ai = genkit({\n plugins: [openAI()],\n});\n\nconst llmResponse = await ai.generate({\n prompt: `What was a positive news story from today?`,\n model: openAI.model('gpt-4o-search-preview'),\n config: {\n web_search_options: {},\n },\n});\n```\n", + "title": "OpenAI Plugin", + "description": "Learn how to configure and use Genkit OpenAI plugin to access various models and embedders from OpenAI.", + "lang": "js", + "headers": "## Installation\n## Configuration\n## Usage\n### Chat Models\n### Image Generation Models\n### Text Embedding Models\n### Audio Transcription and Speech Models\n#### Transcription (Speech-to-Text)\n#### Speech Generation (Text-to-Speech)\n## Advanced usage\n### Passthrough configuration\n### Web-search built-in tool\n" }, "js/plugins/pgvector.md": { - "text": "You can use PostgreSQL and `pgvector` as your retriever implementation. Use the\nfollowing examples as a starting point and modify it to work with your database\nschema.\n\n```ts\nimport { genkit, z, Document } from 'genkit';\nimport { googleAI, textEmbedding004 } from '@genkit-ai/googleai';\nimport { toSql } from 'pgvector';\nimport postgres from 'postgres';\n\nconst ai = genkit({\n plugins: [googleAI()],\n});\n\nconst sql = postgres({ ssl: false, database: 'recaps' });\n\nconst QueryOptions = z.object({\n show: z.string(),\n k: z.number().optional(),\n});\n\nconst sqlRetriever = ai.defineRetriever(\n {\n name: 'pgvector-myTable',\n configSchema: QueryOptions,\n },\n async (input, options) => {\n const embedding = (\n await ai.embed({\n embedder: textEmbedding004,\n content: input,\n })\n )[0].embedding;\n const results = await sql`\n SELECT episode_id, season_number, chunk as content\n FROM embeddings\n WHERE show_id = ${options.show}\n ORDER BY embedding <#> ${toSql(embedding)} LIMIT ${options.k ?? 3}\n `;\n return {\n documents: results.map((row) => {\n const { content, ...metadata } = row;\n return Document.fromText(content, metadata);\n }),\n };\n },\n);\n```\n\nAnd here's how to use the retriever in a flow:\n\n```ts\n// Simple flow to use the sqlRetriever\nexport const askQuestionsOnGoT = ai.defineFlow(\n {\n name: 'askQuestionsOnGoT',\n inputSchema: z.object({ question: z.string() }),\n outputSchema: z.object({ answer: z.string() }),\n },\n async ({ question }) => {\n const docs = await ai.retrieve({\n retriever: sqlRetriever,\n query: question,\n options: {\n show: 'Game of Thrones',\n },\n });\n console.log(docs);\n\n // Continue with using retrieved docs\n // in RAG prompts.\n //...\n \n // Return an answer (placeholder for actual implementation)\n return { answer: \"Answer would be generated here based on retrieved documents\" };\n },\n);\n```\n", + "text": "# pgvector retriever template\n\nYou can use PostgreSQL and `pgvector` as your retriever implementation. Use the\nfollowing examples as a starting point and modify it to work with your database\nschema.\n\n```ts\nimport { genkit, z, Document } from 'genkit';\nimport { googleAI } from '@genkit-ai/googleai';\nimport { toSql } from 'pgvector';\nimport postgres from 'postgres';\n\nconst ai = genkit({\n plugins: [googleAI()],\n});\n\nconst sql = postgres({ ssl: false, database: 'recaps' });\n\nconst QueryOptions = z.object({\n show: z.string(),\n k: z.number().optional(),\n});\n\nconst sqlRetriever = ai.defineRetriever(\n {\n name: 'pgvector-myTable',\n configSchema: QueryOptions,\n },\n async (input, options) => {\n const embedding = (\n await ai.embed({\n embedder: googleAI.embedder('gemini-embedding-001'),\n content: input,\n })\n )[0].embedding;\n const results = await sql`\n SELECT episode_id, season_number, chunk as content\n FROM embeddings\n WHERE show_id = ${options.show}\n ORDER BY embedding <#> ${toSql(embedding)} LIMIT ${options.k ?? 3}\n `;\n return {\n documents: results.map((row) => {\n const { content, ...metadata } = row;\n return Document.fromText(content, metadata);\n }),\n };\n },\n);\n```\n\nAnd here's how to use the retriever in a flow:\n\n```ts\n// Simple flow to use the sqlRetriever\nexport const askQuestionsOnGoT = ai.defineFlow(\n {\n name: 'askQuestionsOnGoT',\n inputSchema: z.object({ question: z.string() }),\n outputSchema: z.object({ answer: z.string() }),\n },\n async ({ question }) => {\n const docs = await ai.retrieve({\n retriever: sqlRetriever,\n query: question,\n options: {\n show: 'Game of Thrones',\n },\n });\n console.log(docs);\n\n // Continue with using retrieved docs\n // in RAG prompts.\n //...\n \n // Return an answer (placeholder for actual implementation)\n return { answer: \"Answer would be generated here based on retrieved documents\" };\n },\n);\n```\n", "title": "pgvector retriever template", - "lang": "js" + "description": "Learn how to use PostgreSQL and pgvector as a retriever implementation in Genkit Go.", + "lang": "js", + "headers": "" }, "js/plugins/pinecone.md": { - "text": "The Pinecone plugin provides indexer and retriever implementations that use the\n[Pinecone](https://www.pinecone.io/) cloud vector database.\n\n## Installation\n\n```bash\nnpm install genkitx-pinecone\n```\n\n## Configuration\n\nTo use this plugin, specify it when you initialize Genkit:\n\n```ts\nimport { genkit } from 'genkit';\nimport { pinecone } from 'genkitx-pinecone';\n\nconst ai = genkit({\n plugins: [\n pinecone([\n {\n indexId: 'bob-facts',\n embedder: textEmbedding004,\n },\n ]),\n ],\n});\n```\n\nYou must specify a Pinecone index ID and the embedding model you want to use.\n\nIn addition, you must configure Genkit with your Pinecone API key. There are two\nways to do this:\n\n- Set the `PINECONE_API_KEY` environment variable.\n- Specify it in the `clientParams` optional parameter:\n\n ```ts\n clientParams: {\n apiKey: ...,\n }\n ```\n\n The value of this parameter is a `PineconeConfiguration` object, which gets passed to the Pinecone client; you can use it to pass any parameter the client supports.\n\n## Usage\n\nImport retriever and indexer references like so:\n\n```ts\nimport { pineconeRetrieverRef } from 'genkitx-pinecone';\nimport { pineconeIndexerRef } from 'genkitx-pinecone';\n```\n\nThen, use these references with `ai.retrieve()` and `ai.index()`:\n\n```ts\n// To use the index you configured when you loaded the plugin:\nlet docs = await ai.retrieve({ retriever: pineconeRetrieverRef, query });\n\n// To specify an index:\nexport const bobFactsRetriever = pineconeRetrieverRef({\n indexId: 'bob-facts',\n});\ndocs = await ai.retrieve({ retriever: bobFactsRetriever, query });\n```\n\n```ts\n// To use the index you configured when you loaded the plugin:\nawait ai.index({ indexer: pineconeIndexerRef, documents });\n\n// To specify an index:\nexport const bobFactsIndexer = pineconeIndexerRef({\n indexId: 'bob-facts',\n});\nawait ai.index({ indexer: bobFactsIndexer, documents });\n```\n\nSee the [Retrieval-augmented generation](/docs/rag) page for a general\ndiscussion on indexers and retrievers.\n", + "text": "# Pinecone plugin\n\nThe Pinecone plugin provides indexer and retriever implementations that use the\n[Pinecone](https://www.pinecone.io/) cloud vector database.\n\n## Installation\n\n```bash\nnpm install genkitx-pinecone\n```\n\n## Configuration\n\nTo use this plugin, specify it when you initialize Genkit:\n\n```ts\nimport { genkit } from 'genkit';\nimport { pinecone } from 'genkitx-pinecone';\nimport { googleAI } from '@genkit-ai/googleai';\n\nconst ai = genkit({\n plugins: [\n pinecone([\n {\n indexId: 'bob-facts',\n embedder: googleAI.embedder('gemini-embedding-001'),\n },\n ]),\n ],\n});\n```\n\nYou must specify a Pinecone index ID and the embedding model you want to use.\n\nIn addition, you must configure Genkit with your Pinecone API key. There are two\nways to do this:\n\n- Set the `PINECONE_API_KEY` environment variable.\n- Specify it in the `clientParams` optional parameter:\n\n ```ts\n clientParams: {\n apiKey: ...,\n }\n ```\n\n The value of this parameter is a `PineconeConfiguration` object, which gets passed to the Pinecone client; you can use it to pass any parameter the client supports.\n\n## Usage\n\nImport retriever and indexer references like so:\n\n```ts\nimport { pineconeRetrieverRef } from 'genkitx-pinecone';\nimport { pineconeIndexerRef } from 'genkitx-pinecone';\n```\n\nThen, use these references with `ai.retrieve()` and `ai.index()`:\n\n```ts\n// To use the index you configured when you loaded the plugin:\nlet docs = await ai.retrieve({ retriever: pineconeRetrieverRef, query });\n\n// To specify an index:\nexport const bobFactsRetriever = pineconeRetrieverRef({\n indexId: 'bob-facts',\n});\ndocs = await ai.retrieve({ retriever: bobFactsRetriever, query });\n```\n\n```ts\n// To use the index you configured when you loaded the plugin:\nawait ai.index({ indexer: pineconeIndexerRef, documents });\n\n// To specify an index:\nexport const bobFactsIndexer = pineconeIndexerRef({\n indexId: 'bob-facts',\n});\nawait ai.index({ indexer: bobFactsIndexer, documents });\n```\n\nSee the [Retrieval-augmented generation](/docs/rag) page for a general\ndiscussion on indexers and retrievers.\n", "title": "Pinecone plugin", - "lang": "js" + "description": "This document describes the Pinecone plugin for Genkit, which provides indexer and retriever implementations for the Pinecone cloud vector database.", + "lang": "js", + "headers": "## Installation\n## Configuration\n## Usage\n" + }, + "js/plugins/toolbox.md": { + "text": "# MCP Toolbox for Databases\n\n[MCP Toolbox for Databases](https://github.com/googleapis/genai-toolbox) is an open source MCP server for databases. It was designed with enterprise-grade and production-quality in mind. It enables you to develop tools easier, faster, and more securely by handling the complexities such as connection pooling, authentication, and more.\n\nToolbox Tools can be seemlessly integrated with Genkit applications. For more\ninformation on [getting\nstarted](https://googleapis.github.io/genai-toolbox/getting-started/) or\n[configuring](https://googleapis.github.io/genai-toolbox/getting-started/configure/)\nToolbox, see the\n[documentation](https://googleapis.github.io/genai-toolbox/getting-started/introduction/).\n\n![architecture](./assets/mcp_db_toolbox.png)\n\n### Configure and deploy\n\nToolbox is an open source server that you deploy and manage yourself. For more\ninstructions on deploying and configuring, see the official Toolbox\ndocumentation:\n\n* [Installing the Server](https://googleapis.github.io/genai-toolbox/getting-started/introduction/#installing-the-server)\n* [Configuring Toolbox](https://googleapis.github.io/genai-toolbox/getting-started/configure/)\n\n### Install client SDK\n\nGenkit relies on the `@toolbox-sdk/core` node package to use Toolbox. Install the\npackage before getting started:\n\n```shell\nnpm install @toolbox-sdk/core\n```\n\n### Loading Toolbox Tools\n\nOnce you’re Toolbox server is configured and up and running, you can load tools\nfrom your server using ADK:\n\n```javascript\nimport {ToolboxClient} from '@toolbox-sdk/core';\nimport { genkit, z } from 'genkit';\n\nconst ai = genkit({\n plugins: [googleAI()],\n model: googleAI.model('gemini-1.5-pro'),\n});\n\n// Replace with your Toolbox Server URL\nconst URL = 'https://127.0.0.1:5000';\n\nlet client = ToolboxClient(URL);\ntoolboxTools = await client.loadToolset('toolsetName');\n\nconst getGenkitTool = (toolboxTool) => ai.defineTool({\n name: toolboxTool.getName(),\n description: toolboxTool.getDescription(),\n inputSchema: toolboxTool.getParams(),\n},\ntoolboxTool,\n);\n\nconst tools = toolboxTools.map(getGenkitTool);\n\nawait ai.generate({\n prompt: 'Ask some question.',\n tools: tools,\n});\n```\n\n### Advanced Toolbox Features\n\nToolbox has a variety of features to make developing Gen AI tools for databases.\nFor more information, read more about the following features:\n\n* [Authenticated Parameters](https://googleapis.github.io/genai-toolbox/resources/tools/#authenticated-parameters): bind tool inputs to values from OIDC tokens automatically, making it easy to run sensitive queries without potentially leaking data\n* [Authorized Invocations:](https://googleapis.github.io/genai-toolbox/resources/tools/#authorized-invocations) restrict access to use a tool based on the users Auth token\n* [OpenTelemetry](https://googleapis.github.io/genai-toolbox/how-to/export_telemetry/): get metrics and tracing from Toolbox with OpenTelemetry\n", + "title": "MCP Toolbox for Databases", + "description": "This document introduces the MCP Toolbox for Databases, an open-source MCP server designed for enterprise-grade database tool development, and explains its integration with Genkit applications.", + "lang": "js", + "headers": "### Configure and deploy\n### Install client SDK\n### Loading Toolbox Tools\n### Advanced Toolbox Features\n" }, "js/plugins/vertex-ai.md": { - "text": "The Vertex AI plugin provides interfaces to several AI services:\n\n- [Google generative AI models](https://cloud.google.com/vertex-ai/generative-ai/docs/):\n - Gemini text generation\n - Imagen2 and Imagen3 image generation\n - Text embedding generation\n - Multimodal embedding generation\n- A subset of evaluation metrics through the Vertex AI [Rapid Evaluation API](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/evaluation):\n - [BLEU](https://cloud.google.com/vertex-ai/docs/reference/rest/v1beta1/projects.locations/evaluateInstances#bleuinput)\n - [ROUGE](https://cloud.google.com/vertex-ai/docs/reference/rest/v1beta1/projects.locations/evaluateInstances#rougeinput)\n - [Fluency](https://cloud.google.com/vertex-ai/docs/reference/rest/v1beta1/projects.locations/evaluateInstances#fluencyinput)\n - [Safety](https://cloud.google.com/vertex-ai/docs/reference/rest/v1beta1/projects.locations/evaluateInstances#safetyinput)\n - [Groundeness](https://cloud.google.com/vertex-ai/docs/reference/rest/v1beta1/projects.locations/evaluateInstances#groundednessinput)\n - [Summarization Quality](https://cloud.google.com/vertex-ai/docs/reference/rest/v1beta1/projects.locations/evaluateInstances#summarizationqualityinput)\n - [Summarization Helpfulness](https://cloud.google.com/vertex-ai/docs/reference/rest/v1beta1/projects.locations/evaluateInstances#summarizationhelpfulnessinput)\n - [Summarization Verbosity](https://cloud.google.com/vertex-ai/docs/reference/rest/v1beta1/projects.locations/evaluateInstances#summarizationverbosityinput)\n- [Vector Search](https://cloud.google.com/vertex-ai/docs/vector-search/overview)\n\n## Installation\n\n```bash\nnpm install @genkit-ai/vertexai\n```\n\nIf you want to locally run flows that use this plugin, you also need the [Google Cloud CLI tool](https://cloud.google.com/sdk/docs/install) installed.\n\n## Configuration\n\nTo use this plugin, specify it when you initialize Genkit:\n\n```ts\nimport { genkit } from 'genkit';\nimport { vertexAI } from '@genkit-ai/vertexai';\n\nconst ai = genkit({\n plugins: [vertexAI({ location: 'us-central1' })],\n});\n```\n\nThe plugin requires you to specify your Google Cloud project ID, the [region](https://cloud.google.com/vertex-ai/generative-ai/docs/learn/locations) to which you want to make Vertex API requests, and your Google Cloud project credentials.\n\n- You can specify your Google Cloud project ID either by setting `projectId` in the `vertexAI()` configuration or by setting the `GCLOUD_PROJECT` environment variable. If you're running your flow from a Google Cloud environment (Cloud Functions, Cloud Run, and so on), `GCLOUD_PROJECT` is automatically set to the project ID of the environment.\n- You can specify the API location either by setting `location` in the `vertexAI()` configuration or by setting the `GCLOUD_LOCATION` environment variable.\n- To provide API credentials, you need to set up Google Cloud Application Default Credentials.\n\n 1. To specify your credentials:\n\n - If you're running your flow from a Google Cloud environment (Cloud Functions, Cloud Run, and so on), this is set automatically.\n - On your local dev environment, do this by running:\n\n ```bash\n gcloud auth application-default login --project YOUR_PROJECT_ID\n ```\n\n - For other environments, see the [Application Default Credentials](https://cloud.google.com/docs/authentication/provide-credentials-adc) docs.\n\n 1. In addition, make sure the account is granted the Vertex AI User IAM role (`roles/aiplatform.user`). See the Vertex AI [access control](https://cloud.google.com/vertex-ai/generative-ai/docs/access-control) docs.\n\n## Usage\n\n### Generative AI Models\n\nThe Vertex AI plugin allows you to use various Gemini, Imagen, and other Vertex AI models:\n\n```ts\nimport { vertexAI } from '@genkit-ai/vertexai';\n\nconst ai = genkit({\n plugins: [vertexAI({ location: 'us-central1' })],\n});\n\nconst llmResponse = await ai.generate({\n model: vertexAI.model('gemini-2.5-flash'),\n prompt: 'What should I do when I visit Melbourne?',\n});\n```\n\nThis plugin also supports grounding Gemini text responses using [Google Search](https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/ground-gemini#web-ground-gemini) or [your own data](https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/ground-gemini#private-ground-gemini).\n\nImportant: Vertex AI charges a fee for grounding requests in addition to the cost of making LLM requests. See the [Vertex AI pricing](https://cloud.google.com/vertex-ai/generative-ai/pricing) page and be sure you understand grounding request pricing before you use this feature.\n\nExample:\n\n```ts\nconst ai = genkit({\n plugins: [vertexAI({ location: 'us-central1' })],\n});\n\nawait ai.generate({\n model: vertexAI.model('gemini-2.5-flash'),\n prompt: '...',\n config: {\n googleSearchRetrieval: {\n disableAttribution: true,\n }\n vertexRetrieval: {\n datastore: {\n projectId: 'your-cloud-project',\n location: 'us-central1',\n collection: 'your-collection',\n },\n disableAttribution: true,\n }\n }\n})\n```\n\nYou can also use Vertex AI text embedding models for generating embeddings:\n\n```ts\nconst ai = genkit({\n plugins: [vertexAI({ location: 'us-central1' })],\n});\n\nconst embeddings = await ai.embed({\n embedder: vertexAI.embedder('text-embedding-004'),\n content: 'How many widgets do you have in stock?',\n});\n```\n\nFor Chroma DB or other vector databases, you can specify the embedder like this:\n\n```ts\nconst ai = genkit({\n plugins: [\n chroma([\n {\n embedder: vertexAI.embedder('text-embedding-004'),\n collectionName: 'my-collection',\n },\n ]),\n ],\n});\n```\n\nThis plugin can also handle multimodal embeddings:\n\n```ts\nimport { vertexAI } from '@genkit-ai/vertexai';\n\nconst ai = genkit({\n plugins: [vertexAI({ location: 'us-central1' })],\n});\n\nconst embeddings = await ai.embed({\n embedder: vertexAI.embedder('multimodal-embedding-001'),\n content: {\n content: [\n {\n media: {\n url: 'gs://cloud-samples-data/generative-ai/video/pixel8.mp4',\n contentType: 'video/mp4',\n },\n },\n ],\n },\n});\n```\n\nImagen3 model allows generating images from user prompt:\n\n```ts\nimport { vertexAI } from '@genkit-ai/vertexai';\n\nconst ai = genkit({\n plugins: [vertexAI({ location: 'us-central1' })],\n});\n\nconst response = await ai.generate({\n model: vertexAI.model('imagen-3.0-generate-002'),\n output: { format: 'media' },\n prompt: 'a banana riding a bicycle',\n});\n\nreturn response.media;\n```\n\nand even advanced editing of existing images:\n\n```ts\nconst ai = genkit({\n plugins: [vertexAI({ location: 'us-central1' })],\n});\n\nconst baseImg = fs.readFileSync('base.png', { encoding: 'base64' });\nconst maskImg = fs.readFileSync('mask.png', { encoding: 'base64' });\n\nconst response = await ai.generate({\n model: vertexAI.model('imagen-3.0-generate-002'),\n output: { format: 'media' },\n prompt: [\n { media: { url: `data:image/png;base64,${baseImg}` } },\n {\n media: { url: `data:image/png;base64,${maskImg}` },\n metadata: { type: 'mask' },\n },\n { text: 'replace the background with foo bar baz' },\n ],\n config: {\n editConfig: {\n editMode: 'outpainting',\n },\n },\n});\n\nreturn response.media;\n```\n\nRefer to [Imagen model documentation](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/imagen-api#edit_images_2) for more detailed options.\n\n#### Anthropic Claude 3 on Vertex AI Model Garden\n\nIf you have access to Claude 3 models ([haiku](https://console.cloud.google.com/vertex-ai/publishers/anthropic/model-garden/claude-3-haiku), [sonnet](https://console.cloud.google.com/vertex-ai/publishers/anthropic/model-garden/claude-3-sonnet) or [opus](https://console.cloud.google.com/vertex-ai/publishers/anthropic/model-garden/claude-3-opus)) in Vertex AI Model Garden you can use them with Genkit.\n\nHere's a sample configuration for enabling Vertex AI Model Garden models:\n\n```ts\nimport { genkit } from 'genkit';\nimport { vertexAIModelGarden } from '@genkit-ai/vertexai/modelgarden';\n\nconst ai = genkit({\n plugins: [\n vertexAIModelGarden({\n location: 'us-central1',\n models: ['claude-3-haiku', 'claude-3-sonnet', 'claude-3-opus'],\n }),\n ],\n});\n```\n\nThen use them as regular models:\n\n```ts\nconst llmResponse = await ai.generate({\n model: 'claude-3-sonnet',\n prompt: 'What should I do when I visit Melbourne?',\n});\n```\n\n#### Llama 3.1 405b on Vertex AI Model Garden\n\nFirst you'll need to enable [Llama 3.1 API Service](https://console.cloud.google.com/vertex-ai/publishers/meta/model-garden/llama3-405b-instruct-maas) in Vertex AI Model Garden.\n\nHere's sample configuration for Llama 3.1 405b in Vertex AI plugin:\n\n```ts\nimport { genkit } from 'genkit';\nimport { vertexAIModelGarden } from '@genkit-ai/vertexai/modelgarden';\n\nconst ai = genkit({\n plugins: [\n vertexAIModelGarden({\n location: 'us-central1',\n models: ['llama3-405b-instruct-maas'],\n }),\n ],\n});\n```\n\nThen use it as a regular model:\n\n```ts\nconst llmResponse = await ai.generate({\n model: 'llama3-405b-instruct-maas',\n prompt: 'Write a function that adds two numbers together',\n});\n```\n\n#### Mistral Models on Vertex AI Model Garden\n\nIf you have access to Mistral models ([Mistral Large](https://console.cloud.google.com/vertex-ai/publishers/mistralai/model-garden/mistral-large), [Mistral Nemo](https://console.cloud.google.com/vertex-ai/publishers/mistralai/model-garden/mistral-nemo), or [Codestral](https://console.cloud.google.com/vertex-ai/publishers/mistralai/model-garden/codestral)) in Vertex AI Model Garden, you can use them with Genkit.\n\nHere's a sample configuration for enabling Vertex AI Model Garden models:\n\n```ts\nimport { genkit } from 'genkit';\nimport { vertexAIModelGarden } from '@genkit-ai/vertexai/modelgarden';\n\nconst ai = genkit({\n plugins: [\n vertexAIModelGarden({\n location: 'us-central1',\n models: ['mistral-large', 'mistral-nemo', 'codestral'],\n }),\n ],\n});\n```\n\nThen use them as regular models:\n\n```ts\nconst llmResponse = await ai.generate({\n model: 'mistral-large',\n prompt: 'Write a function that adds two numbers together',\n config: {\n version: 'mistral-large-2411', // Optional: specify model version\n temperature: 0.7, // Optional: control randomness (0-1)\n maxOutputTokens: 1024, // Optional: limit response length\n topP: 0.9, // Optional: nucleus sampling parameter\n stopSequences: ['###'], // Optional: stop generation at sequences\n },\n});\n```\n\nThe models support:\n\n- `mistral-large`: Latest Mistral large model with function calling capabilities\n- `mistral-nemo`: Optimized for efficiency and speed\n- `codestral`: Specialized for code generation tasks\n\nEach model supports streaming responses and function calling:\n\n```ts\nconst response = await ai.generateStream({\n model: 'mistral-large',\n prompt: 'What should I cook tonight?',\n tools: ['recipe-finder'],\n config: {\n version: 'mistral-large-2411',\n temperature: 1,\n },\n});\n\nfor await (const chunk of response.stream) {\n console.log(chunk.text);\n}\n```\n\n### Evaluators\n\nTo use the evaluators from Vertex AI Rapid Evaluation, add an `evaluation` block to your `vertexAI` plugin configuration.\n\n```ts\nimport { genkit } from 'genkit';\nimport { vertexAIEvaluation, VertexAIEvaluationMetricType } from '@genkit-ai/vertexai/evaluation';\n\nconst ai = genkit({\n plugins: [\n vertexAIEvaluation({\n location: 'us-central1',\n metrics: [\n VertexAIEvaluationMetricType.SAFETY,\n {\n type: VertexAIEvaluationMetricType.ROUGE,\n metricSpec: {\n rougeType: 'rougeLsum',\n },\n },\n ],\n }),\n ],\n});\n```\n\nThe configuration above adds evaluators for the `Safety` and `ROUGE` metrics. The example shows two approaches- the `Safety` metric uses the default specification, whereas the `ROUGE` metric provides a customized specification that sets the rouge type to `rougeLsum`.\n\nBoth evaluators can be run using the `genkit eval:run` command with a compatible dataset: that is, a dataset with `output` and `reference` fields. The `Safety` evaluator can also be run using the `genkit eval:flow -e vertexai/safety` command since it only requires an `output`.\n\n### Indexers and retrievers\n\nThe Genkit Vertex AI plugin includes indexer and retriever implementations backed by the Vertex AI Vector Search service.\n\n(See the [Retrieval-augmented generation](http://../rag.md) page to learn how indexers and retrievers are used in a RAG implementation.)\n\nThe Vertex AI Vector Search service is a document index that works alongside the document store of your choice: the document store contains the content of documents, and the Vertex AI Vector Search index contains, for each document, its vector embedding and a reference to the document in the document store. After your documents are indexed by the Vertex AI Vector Search service, it can respond to search queries, producing lists of indexes into your document store.\n\nThe indexer and retriever implementations provided by the Vertex AI plugin use either Cloud Firestore or BigQuery as the document store. The plugin also includes interfaces you can implement to support other document stores.\n\nImportant: Pricing for Vector Search consists of both a charge for every gigabyte of data you ingest and an hourly charge for the VMs that host your deployed indexes. See [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing#vectorsearch). This is likely to be most cost-effective when you are serving high volumes of traffic. Be sure to understand the billing implications the service will have on your project before using it.\n\nTo use Vertex AI Vector Search:\n\n1. Choose an embedding model. This model is responsible for creating vector embeddings from text or media. Advanced users might use an embedding model optimized for their particular data sets, but for most users, Vertex AI's `text-embedding-004` model is a good choice for English text, the `text-multilingual-embedding-002` model is good for multilingual text, and the `multimodalEmbedding001` model is good for mixed text, images, and video.\n2. In the [Vector Search](https://console.cloud.google.com/vertex-ai/matching-engine/indexes) section of the Google Cloud console, create a new index. The most important settings are:\n\n - **Dimensions:** Specify the dimensionality of the vectors produced by your chosen embedding model. The `text-embedding-004` and `text-multilingual-embedding-002` models produce vectors of 768 dimensions. The `multimodalEmbedding001` model can produce vectors of 128, 256, 512, or 1408 dimensions for text and image, and will produce vectors of 1408 dimensions for video.\n - **Update method:** Select streaming updates.\n\n After you create the index, deploy it to a standard (public) endpoint.\n\n3. Get a document indexer and retriever for the document store you want to use:\n\n **Cloud Firestore**\n\n ```ts\n import { getFirestoreDocumentIndexer, getFirestoreDocumentRetriever } from '@genkit-ai/vertexai/vectorsearch';\n\n import { initializeApp } from 'firebase-admin/app';\n import { getFirestore } from 'firebase-admin/firestore';\n\n initializeApp({ projectId: PROJECT_ID });\n const db = getFirestore();\n\n const firestoreDocumentRetriever = getFirestoreDocumentRetriever(db, FIRESTORE_COLLECTION);\n const firestoreDocumentIndexer = getFirestoreDocumentIndexer(db, FIRESTORE_COLLECTION);\n ```\n\n **BigQuery**\n\n ```ts\n import { getBigQueryDocumentIndexer, getBigQueryDocumentRetriever } from '@genkit-ai/vertexai/vectorsearch';\n import { BigQuery } from '@google-cloud/bigquery';\n\n const bq = new BigQuery({ projectId: PROJECT_ID });\n\n const bigQueryDocumentRetriever = getBigQueryDocumentRetriever(bq, BIGQUERY_TABLE, BIGQUERY_DATASET);\n const bigQueryDocumentIndexer = getBigQueryDocumentIndexer(bq, BIGQUERY_TABLE, BIGQUERY_DATASET);\n ```\n\n **Other**\n\n To support other documents stores you can provide your own implementations of `DocumentRetriever` and `DocumentIndexer`:\n\n ```ts\n const myDocumentRetriever = async (neighbors) => {\n // Return the documents referenced by `neighbors`.\n // ...\n };\n const myDocumentIndexer = async (documents) => {\n // Add `documents` to storage.\n // ...\n };\n ```\n\n For an example, see [Sample Vertex AI Plugin Retriever and Indexer with Local File](https://github.com/firebase/genkit/tree/main/js/testapps/vertexai-vector-search-custom).\n\n4. Add a `vectorSearchOptions` block to your `vertexAI` plugin configuration:\n\n ```ts\n import { genkit } from 'genkit';\n import { vertexAIVectorSearch } from '@genkit-ai/vertexai/vectorsearch';\n\n const ai = genkit({\n plugins: [\n vertexAIVectorSearch({\n projectId: PROJECT_ID,\n location: LOCATION,\n vectorSearchOptions: [\n {\n indexId: VECTOR_SEARCH_INDEX_ID,\n indexEndpointId: VECTOR_SEARCH_INDEX_ENDPOINT_ID,\n deployedIndexId: VECTOR_SEARCH_DEPLOYED_INDEX_ID,\n publicDomainName: VECTOR_SEARCH_PUBLIC_DOMAIN_NAME,\n documentRetriever: firestoreDocumentRetriever,\n documentIndexer: firestoreDocumentIndexer,\n embedder: vertexAI.embedder('text-embedding-004'),\n },\n ],\n }),\n ],\n });\n ```\n\n Provide the embedder you chose in the first step and the document indexer and retriever you created in the previous step.\n\n To configure the plugin to use the Vector Search index you created earlier, you need to provide several values, which you can find in the Vector Search section of the Google Cloud console:\n\n - `indexId`: listed on the [Indexes](https://console.cloud.google.com/vertex-ai/matching-engine/indexes) tab\n - `indexEndpointId`: listed on the [Index Endpoints](https://console.cloud.google.com/vertex-ai/matching-engine/index-endpoints) tab\n - `deployedIndexId` and `publicDomainName`: listed on the \"Deployed index info\" page, which you can open by clicking the name of the deployed index on either of the tabs mentioned earlier\n\n5. Now that everything is configured, you can use the indexer and retriever in your Genkit application:\n\n ```ts\n import { vertexAiIndexerRef, vertexAiRetrieverRef } from '@genkit-ai/vertexai/vectorsearch';\n\n // ... inside your flow function:\n\n await ai.index({\n indexer: vertexAiIndexerRef({\n indexId: VECTOR_SEARCH_INDEX_ID,\n }),\n documents,\n });\n\n const res = await ai.retrieve({\n retriever: vertexAiRetrieverRef({\n indexId: VECTOR_SEARCH_INDEX_ID,\n }),\n query: queryDocument,\n });\n ```\n\nSee the code samples for:\n\n- [Vertex Vector Search + BigQuery](https://github.com/firebase/genkit/tree/main/js/testapps/vertexai-vector-search-bigquery)\n- [Vertex Vector Search + Firestore](https://github.com/firebase/genkit/tree/main/js/testapps/vertexai-vector-search-firestore)\n- [Vertex Vector Search + a custom DB](https://github.com/firebase/genkit/tree/main/js/testapps/vertexai-vector-search-custom)\n\n## Context Caching\n\nThe Vertex AI Genkit plugin supports **Context Caching**, which allows models to reuse previously cached content to optimize token usage when dealing with large pieces of content. This feature is especially useful for conversational flows or scenarios where the model references a large piece of content consistently across multiple requests.\n\n### How to Use Context Caching\n\nTo enable context caching, ensure your model supports it. For example, `gemini-2.5-flash` and `gemini-2.0-pro` are models that support context caching, and you will have to specify version number `001`.\n\nYou can define a caching mechanism in your application like this:\n\n```ts\nconst ai = genkit({\n plugins: [vertexAI({ location: 'us-central1' })],\n});\n\nconst llmResponse = await ai.generate({\n messages: [\n {\n role: 'user',\n content: [{ text: 'Here is the relevant text from War and Peace.' }],\n },\n {\n role: 'model',\n content: [\n {\n text: \"Based on War and Peace, here is some analysis of Pierre Bezukhov's character.\",\n },\n ],\n metadata: {\n cache: {\n ttlSeconds: 300, // Cache this message for 5 minutes\n },\n },\n },\n ],\n model: vertexAI.model('gemini-2.5-flash'),\n prompt: \"Describe Pierre's transformation throughout the novel.\",\n});\n```\n\nIn this setup:\n\n- **`messages`**: Allows you to pass conversation history.\n- **`metadata.cache.ttlSeconds`**: Specifies the time-to-live (TTL) for caching a specific response.\n\n### Example: Leveraging Large Texts with Context\n\nFor applications referencing long documents, such as _War and Peace_ or _Lord of the Rings_, you can structure your queries to reuse cached contexts:\n\n```ts\nconst textContent = await fs.readFile('path/to/war_and_peace.txt', 'utf-8');\n\nconst llmResponse = await ai.generate({\n messages: [\n {\n role: 'user',\n content: [{ text: textContent }], // Include the large text as context\n },\n {\n role: 'model',\n content: [\n {\n text: 'This analysis is based on the provided text from War and Peace.',\n },\n ],\n metadata: {\n cache: {\n ttlSeconds: 300, // Cache the response to avoid reloading the full text\n },\n },\n },\n ],\n model: vertexAI.model('gemini-2.5-flash'),\n prompt: 'Analyze the relationship between Pierre and Natasha.',\n});\n```\n\n### Benefits of Context Caching\n\n1. **Improved Performance**: Reduces the need for repeated processing of large inputs.\n2. **Cost Efficiency**: Decreases API usage for redundant data, optimizing token consumption.\n3. **Better Latency**: Speeds up response times for repeated or related queries.\n\n### Supported Models for Context Caching\n\nOnly specific models, such as `gemini-2.5-flash` and `gemini-2.0-pro`, support context caching, and currently only on version numbers `001`. If an unsupported model is used, an error will be raised, indicating that caching cannot be applied.\n\n### Further Reading\n\nSee more information regarding context caching on Vertex AI in their [documentation](https://cloud.google.com/vertex-ai/generative-ai/docs/context-cache/context-cache-overview).\n", + "text": "# Vertex AI plugin\n\nThe Vertex AI plugin provides interfaces to several AI services:\n\n- [Google generative AI models](https://cloud.google.com/vertex-ai/generative-ai/docs/):\n - Gemini text generation\n - Imagen2 and Imagen3 image generation\n - Text embedding generation\n - Multimodal embedding generation\n- A subset of evaluation metrics through the Vertex AI [Rapid Evaluation API](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/evaluation):\n - [BLEU](https://cloud.google.com/vertex-ai/docs/reference/rest/v1beta1/projects.locations/evaluateInstances#bleuinput)\n - [ROUGE](https://cloud.google.com/vertex-ai/docs/reference/rest/v1beta1/projects.locations/evaluateInstances#rougeinput)\n - [Fluency](https://cloud.google.com/vertex-ai/docs/reference/rest/v1beta1/projects.locations/evaluateInstances#fluencyinput)\n - [Safety](https://cloud.google.com/vertex-ai/docs/reference/rest/v1beta1/projects.locations/evaluateInstances#safetyinput)\n - [Groundeness](https://cloud.google.com/vertex-ai/docs/reference/rest/v1beta1/projects.locations/evaluateInstances#groundednessinput)\n - [Summarization Quality](https://cloud.google.com/vertex-ai/docs/reference/rest/v1beta1/projects.locations/evaluateInstances#summarizationqualityinput)\n - [Summarization Helpfulness](https://cloud.google.com/vertex-ai/docs/reference/rest/v1beta1/projects.locations/evaluateInstances#summarizationhelpfulnessinput)\n - [Summarization Verbosity](https://cloud.google.com/vertex-ai/docs/reference/rest/v1beta1/projects.locations/evaluateInstances#summarizationverbosityinput)\n- [Vector Search](https://cloud.google.com/vertex-ai/docs/vector-search/overview)\n\n## Installation\n\n```bash\nnpm install @genkit-ai/vertexai\n```\n\nIf you want to locally run flows that use this plugin, you also need the [Google Cloud CLI tool](https://cloud.google.com/sdk/docs/install) installed.\n\n## Configuration\n\nTo use this plugin, specify it when you initialize Genkit:\n\n```ts\nimport { genkit } from 'genkit';\nimport { vertexAI } from '@genkit-ai/vertexai';\n\nconst ai = genkit({\n plugins: [vertexAI({ location: 'us-central1' })],\n});\n```\n\nThe plugin requires you to specify your Google Cloud project ID, the [region](https://cloud.google.com/vertex-ai/generative-ai/docs/learn/locations) to which you want to make Vertex API requests, and your Google Cloud project credentials.\n\n- You can specify your Google Cloud project ID either by setting `projectId` in the `vertexAI()` configuration or by setting the `GCLOUD_PROJECT` environment variable. If you're running your flow from a Google Cloud environment (Cloud Functions, Cloud Run, and so on), `GCLOUD_PROJECT` is automatically set to the project ID of the environment.\n- You can specify the API location either by setting `location` in the `vertexAI()` configuration or by setting the `GCLOUD_LOCATION` environment variable.\n- To provide API credentials, you need to set up Google Cloud Application Default Credentials.\n\n 1. To specify your credentials:\n\n - If you're running your flow from a Google Cloud environment (Cloud Functions, Cloud Run, and so on), this is set automatically.\n - On your local dev environment, do this by running:\n\n ```bash\n gcloud auth application-default login --project YOUR_PROJECT_ID\n ```\n\n - For other environments, see the [Application Default Credentials](https://cloud.google.com/docs/authentication/provide-credentials-adc) docs.\n\n 1. In addition, make sure the account is granted the Vertex AI User IAM role (`roles/aiplatform.user`). See the Vertex AI [access control](https://cloud.google.com/vertex-ai/generative-ai/docs/access-control) docs.\n\n## Usage\n\n### Generative AI Models\n\nThe Vertex AI plugin allows you to use various Gemini, Imagen, and other Vertex AI models:\n\n```ts\nimport { vertexAI } from '@genkit-ai/vertexai';\n\nconst ai = genkit({\n plugins: [vertexAI({ location: 'us-central1' })],\n});\n\nconst llmResponse = await ai.generate({\n model: vertexAI.model('gemini-2.5-flash'),\n prompt: 'What should I do when I visit Melbourne?',\n});\n```\n\nThis plugin also supports grounding Gemini text responses using [Google Search](https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/ground-gemini#web-ground-gemini) or [your own data](https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/ground-gemini#private-ground-gemini).\n\nImportant: Vertex AI charges a fee for grounding requests in addition to the cost of making LLM requests. See the [Vertex AI pricing](https://cloud.google.com/vertex-ai/generative-ai/pricing) page and be sure you understand grounding request pricing before you use this feature.\n\nExample:\n\n```ts\nconst ai = genkit({\n plugins: [vertexAI({ location: 'us-central1' })],\n});\n\nawait ai.generate({\n model: vertexAI.model('gemini-2.5-flash'),\n prompt: '...',\n config: {\n googleSearchRetrieval: {\n disableAttribution: true,\n }\n vertexRetrieval: {\n datastore: {\n projectId: 'your-cloud-project',\n location: 'us-central1',\n collection: 'your-collection',\n },\n disableAttribution: true,\n }\n }\n})\n```\n\nYou can also use Vertex AI text embedding models for generating embeddings:\n\n```ts\nconst ai = genkit({\n plugins: [vertexAI({ location: 'us-central1' })],\n});\n\nconst embeddings = await ai.embed({\n embedder: vertexAI.embedder('gemini-embedding-001'),\n content: 'How many widgets do you have in stock?',\n});\n```\n\nFor Chroma DB or other vector databases, you can specify the embedder like this:\n\n```ts\nconst ai = genkit({\n plugins: [\n chroma([\n {\n embedder: vertexAI.embedder('gemini-embedding-001'),\n collectionName: 'my-collection',\n },\n ]),\n ],\n});\n```\n\nThis plugin can also handle multimodal embeddings:\n\n```ts\nimport { vertexAI } from '@genkit-ai/vertexai';\n\nconst ai = genkit({\n plugins: [vertexAI({ location: 'us-central1' })],\n});\n\nconst embeddings = await ai.embed({\n embedder: vertexAI.embedder('multimodal-embedding-001'),\n content: {\n content: [\n {\n media: {\n url: 'gs://cloud-samples-data/generative-ai/video/pixel8.mp4',\n contentType: 'video/mp4',\n },\n },\n ],\n },\n});\n```\n\nImagen3 model allows generating images from user prompt:\n\n```ts\nimport { vertexAI } from '@genkit-ai/vertexai';\n\nconst ai = genkit({\n plugins: [vertexAI({ location: 'us-central1' })],\n});\n\nconst response = await ai.generate({\n model: vertexAI.model('imagen-3.0-generate-002'),\n output: { format: 'media' },\n prompt: 'a banana riding a bicycle',\n});\n\nreturn response.media;\n```\n\nand even advanced editing of existing images:\n\n```ts\nconst ai = genkit({\n plugins: [vertexAI({ location: 'us-central1' })],\n});\n\nconst baseImg = fs.readFileSync('base.png', { encoding: 'base64' });\nconst maskImg = fs.readFileSync('mask.png', { encoding: 'base64' });\n\nconst response = await ai.generate({\n model: vertexAI.model('imagen-3.0-generate-002'),\n output: { format: 'media' },\n prompt: [\n { media: { url: `data:image/png;base64,${baseImg}` } },\n {\n media: { url: `data:image/png;base64,${maskImg}` },\n metadata: { type: 'mask' },\n },\n { text: 'replace the background with foo bar baz' },\n ],\n config: {\n editConfig: {\n editMode: 'outpainting',\n },\n },\n});\n\nreturn response.media;\n```\n\nRefer to [Imagen model documentation](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/imagen-api#edit_images_2) for more detailed options.\n\n#### Anthropic Claude 3 on Vertex AI Model Garden\n\nIf you have access to Claude 3 models ([haiku](https://console.cloud.google.com/vertex-ai/publishers/anthropic/model-garden/claude-3-haiku), [sonnet](https://console.cloud.google.com/vertex-ai/publishers/anthropic/model-garden/claude-3-sonnet) or [opus](https://console.cloud.google.com/vertex-ai/publishers/anthropic/model-garden/claude-3-opus)) in Vertex AI Model Garden you can use them with Genkit.\n\nHere's a sample configuration for enabling Vertex AI Model Garden models:\n\n```ts\nimport { genkit } from 'genkit';\nimport { vertexAIModelGarden } from '@genkit-ai/vertexai/modelgarden';\n\nconst ai = genkit({\n plugins: [\n vertexAIModelGarden({\n location: 'us-central1',\n models: ['claude-3-haiku', 'claude-3-sonnet', 'claude-3-opus'],\n }),\n ],\n});\n```\n\nThen use them as regular models:\n\n```ts\nconst llmResponse = await ai.generate({\n model: 'claude-3-sonnet',\n prompt: 'What should I do when I visit Melbourne?',\n});\n```\n\n#### Llama 3.1 405b on Vertex AI Model Garden\n\nFirst you'll need to enable [Llama 3.1 API Service](https://console.cloud.google.com/vertex-ai/publishers/meta/model-garden/llama3-405b-instruct-maas) in Vertex AI Model Garden.\n\nHere's sample configuration for Llama 3.1 405b in Vertex AI plugin:\n\n```ts\nimport { genkit } from 'genkit';\nimport { vertexAIModelGarden } from '@genkit-ai/vertexai/modelgarden';\n\nconst ai = genkit({\n plugins: [\n vertexAIModelGarden({\n location: 'us-central1',\n models: ['llama3-405b-instruct-maas'],\n }),\n ],\n});\n```\n\nThen use it as a regular model:\n\n```ts\nconst llmResponse = await ai.generate({\n model: 'llama3-405b-instruct-maas',\n prompt: 'Write a function that adds two numbers together',\n});\n```\n\n#### Mistral Models on Vertex AI Model Garden\n\nIf you have access to Mistral models ([Mistral Large](https://console.cloud.google.com/vertex-ai/publishers/mistralai/model-garden/mistral-large), [Mistral Nemo](https://console.cloud.google.com/vertex-ai/publishers/mistralai/model-garden/mistral-nemo), or [Codestral](https://console.cloud.google.com/vertex-ai/publishers/mistralai/model-garden/codestral)) in Vertex AI Model Garden, you can use them with Genkit.\n\nHere's a sample configuration for enabling Vertex AI Model Garden models:\n\n```ts\nimport { genkit } from 'genkit';\nimport { vertexAIModelGarden } from '@genkit-ai/vertexai/modelgarden';\n\nconst ai = genkit({\n plugins: [\n vertexAIModelGarden({\n location: 'us-central1',\n models: ['mistral-large', 'mistral-nemo', 'codestral'],\n }),\n ],\n});\n```\n\nThen use them as regular models:\n\n```ts\nconst llmResponse = await ai.generate({\n model: 'mistral-large',\n prompt: 'Write a function that adds two numbers together',\n config: {\n version: 'mistral-large-2411', // Optional: specify model version\n temperature: 0.7, // Optional: control randomness (0-1)\n maxOutputTokens: 1024, // Optional: limit response length\n topP: 0.9, // Optional: nucleus sampling parameter\n stopSequences: ['###'], // Optional: stop generation at sequences\n },\n});\n```\n\nThe models support:\n\n- `mistral-large`: Latest Mistral large model with function calling capabilities\n- `mistral-nemo`: Optimized for efficiency and speed\n- `codestral`: Specialized for code generation tasks\n\nEach model supports streaming responses and function calling:\n\n```ts\nconst response = await ai.generateStream({\n model: 'mistral-large',\n prompt: 'What should I cook tonight?',\n tools: ['recipe-finder'],\n config: {\n version: 'mistral-large-2411',\n temperature: 1,\n },\n});\n\nfor await (const chunk of response.stream) {\n console.log(chunk.text);\n}\n```\n\n### Evaluators\n\nTo use the evaluators from Vertex AI Rapid Evaluation, add an `evaluation` block to your `vertexAI` plugin configuration.\n\n```ts\nimport { genkit } from 'genkit';\nimport { vertexAIEvaluation, VertexAIEvaluationMetricType } from '@genkit-ai/vertexai/evaluation';\n\nconst ai = genkit({\n plugins: [\n vertexAIEvaluation({\n location: 'us-central1',\n metrics: [\n VertexAIEvaluationMetricType.SAFETY,\n {\n type: VertexAIEvaluationMetricType.ROUGE,\n metricSpec: {\n rougeType: 'rougeLsum',\n },\n },\n ],\n }),\n ],\n});\n```\n\nThe configuration above adds evaluators for the `Safety` and `ROUGE` metrics. The example shows two approaches- the `Safety` metric uses the default specification, whereas the `ROUGE` metric provides a customized specification that sets the rouge type to `rougeLsum`.\n\nBoth evaluators can be run using the `genkit eval:run` command with a compatible dataset: that is, a dataset with `output` and `reference` fields. The `Safety` evaluator can also be run using the `genkit eval:flow -e vertexai/safety` command since it only requires an `output`.\n\n### Indexers and retrievers\n\nThe Genkit Vertex AI plugin includes indexer and retriever implementations backed by the Vertex AI Vector Search service.\n\n(See the [Retrieval-augmented generation](http://../rag.md) page to learn how indexers and retrievers are used in a RAG implementation.)\n\nThe Vertex AI Vector Search service is a document index that works alongside the document store of your choice: the document store contains the content of documents, and the Vertex AI Vector Search index contains, for each document, its vector embedding and a reference to the document in the document store. After your documents are indexed by the Vertex AI Vector Search service, it can respond to search queries, producing lists of indexes into your document store.\n\nThe indexer and retriever implementations provided by the Vertex AI plugin use either Cloud Firestore or BigQuery as the document store. The plugin also includes interfaces you can implement to support other document stores.\n\nImportant: Pricing for Vector Search consists of both a charge for every gigabyte of data you ingest and an hourly charge for the VMs that host your deployed indexes. See [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing#vectorsearch). This is likely to be most cost-effective when you are serving high volumes of traffic. Be sure to understand the billing implications the service will have on your project before using it.\n\nTo use Vertex AI Vector Search:\n\n1. Choose an embedding model. This model is responsible for creating vector embeddings from text or media. Advanced users might use an embedding model optimized for their particular data sets, but for most users, Vertex AI's `gemini-embedding-001` model is a good choice for English text, the `text-multilingual-embedding-002` model is good for multilingual text, and the `multimodalEmbedding001` model is good for mixed text, images, and video.\n2. In the [Vector Search](https://console.cloud.google.com/vertex-ai/matching-engine/indexes) section of the Google Cloud console, create a new index. The most important settings are:\n\n - **Dimensions:** Specify the dimensionality of the vectors produced by your chosen embedding model. The `gemini-embedding-001` and `text-multilingual-embedding-002` models produce vectors of 768 dimensions. The `multimodalEmbedding001` model can produce vectors of 128, 256, 512, or 1408 dimensions for text and image, and will produce vectors of 1408 dimensions for video.\n - **Update method:** Select streaming updates.\n\n After you create the index, deploy it to a standard (public) endpoint.\n\n3. Get a document indexer and retriever for the document store you want to use:\n\n **Cloud Firestore**\n\n ```ts\n import { getFirestoreDocumentIndexer, getFirestoreDocumentRetriever } from '@genkit-ai/vertexai/vectorsearch';\n\n import { initializeApp } from 'firebase-admin/app';\n import { getFirestore } from 'firebase-admin/firestore';\n\n initializeApp({ projectId: PROJECT_ID });\n const db = getFirestore();\n\n const firestoreDocumentRetriever = getFirestoreDocumentRetriever(db, FIRESTORE_COLLECTION);\n const firestoreDocumentIndexer = getFirestoreDocumentIndexer(db, FIRESTORE_COLLECTION);\n ```\n\n **BigQuery**\n\n ```ts\n import { getBigQueryDocumentIndexer, getBigQueryDocumentRetriever } from '@genkit-ai/vertexai/vectorsearch';\n import { BigQuery } from '@google-cloud/bigquery';\n\n const bq = new BigQuery({ projectId: PROJECT_ID });\n\n const bigQueryDocumentRetriever = getBigQueryDocumentRetriever(bq, BIGQUERY_TABLE, BIGQUERY_DATASET);\n const bigQueryDocumentIndexer = getBigQueryDocumentIndexer(bq, BIGQUERY_TABLE, BIGQUERY_DATASET);\n ```\n\n **Other**\n\n To support other documents stores you can provide your own implementations of `DocumentRetriever` and `DocumentIndexer`:\n\n ```ts\n const myDocumentRetriever = async (neighbors) => {\n // Return the documents referenced by `neighbors`.\n // ...\n };\n const myDocumentIndexer = async (documents) => {\n // Add `documents` to storage.\n // ...\n };\n ```\n\n For an example, see [Sample Vertex AI Plugin Retriever and Indexer with Local File](https://github.com/firebase/genkit/tree/main/js/testapps/vertexai-vector-search-custom).\n\n4. Add a `vectorSearchOptions` block to your `vertexAI` plugin configuration:\n\n ```ts\n import { genkit } from 'genkit';\n import { vertexAIVectorSearch } from '@genkit-ai/vertexai/vectorsearch';\n\n const ai = genkit({\n plugins: [\n vertexAIVectorSearch({\n projectId: PROJECT_ID,\n location: LOCATION,\n vectorSearchOptions: [\n {\n indexId: VECTOR_SEARCH_INDEX_ID,\n indexEndpointId: VECTOR_SEARCH_INDEX_ENDPOINT_ID,\n deployedIndexId: VECTOR_SEARCH_DEPLOYED_INDEX_ID,\n publicDomainName: VECTOR_SEARCH_PUBLIC_DOMAIN_NAME,\n documentRetriever: firestoreDocumentRetriever,\n documentIndexer: firestoreDocumentIndexer,\n embedder: vertexAI.embedder('gemini-embedding-001'),\n },\n ],\n }),\n ],\n });\n ```\n\n Provide the embedder you chose in the first step and the document indexer and retriever you created in the previous step.\n\n To configure the plugin to use the Vector Search index you created earlier, you need to provide several values, which you can find in the Vector Search section of the Google Cloud console:\n\n - `indexId`: listed on the [Indexes](https://console.cloud.google.com/vertex-ai/matching-engine/indexes) tab\n - `indexEndpointId`: listed on the [Index Endpoints](https://console.cloud.google.com/vertex-ai/matching-engine/index-endpoints) tab\n - `deployedIndexId` and `publicDomainName`: listed on the \"Deployed index info\" page, which you can open by clicking the name of the deployed index on either of the tabs mentioned earlier\n\n5. Now that everything is configured, you can use the indexer and retriever in your Genkit application:\n\n ```ts\n import { vertexAiIndexerRef, vertexAiRetrieverRef } from '@genkit-ai/vertexai/vectorsearch';\n\n // ... inside your flow function:\n\n await ai.index({\n indexer: vertexAiIndexerRef({\n indexId: VECTOR_SEARCH_INDEX_ID,\n }),\n documents,\n });\n\n const res = await ai.retrieve({\n retriever: vertexAiRetrieverRef({\n indexId: VECTOR_SEARCH_INDEX_ID,\n }),\n query: queryDocument,\n });\n ```\n\nSee the code samples for:\n\n- [Vertex Vector Search + BigQuery](https://github.com/firebase/genkit/tree/main/js/testapps/vertexai-vector-search-bigquery)\n- [Vertex Vector Search + Firestore](https://github.com/firebase/genkit/tree/main/js/testapps/vertexai-vector-search-firestore)\n- [Vertex Vector Search + a custom DB](https://github.com/firebase/genkit/tree/main/js/testapps/vertexai-vector-search-custom)\n\n## Text-to-Speech (TTS) Models\n\nThe Vertex AI plugin provides access to text-to-speech capabilities through Gemini TTS models. These models can convert text into natural-sounding speech for various applications.\n\n### Basic Usage\n\nTo generate audio using a Vertex AI TTS model:\n\n```ts\nimport { vertexAI } from '@genkit-ai/vertexai';\nimport { writeFile } from 'node:fs/promises';\n\nconst ai = genkit({\n plugins: [vertexAI({ location: 'us-central1' })],\n});\n\nconst response = await ai.generate({\n model: vertexAI.model('gemini-2.5-flash-preview-tts'),\n config: {\n responseModalities: ['AUDIO'],\n speechConfig: {\n voiceConfig: {\n prebuiltVoiceConfig: { voiceName: 'Algenib' },\n },\n },\n },\n prompt: 'Say that Genkit is an amazing Gen AI library',\n});\n\n// Handle the audio data (returned as a data URL)\nif (response.media?.url) {\n // Extract base64 data from the data URL\n const audioBuffer = Buffer.from(\n response.media.url.substring(response.media.url.indexOf(',') + 1),\n 'base64'\n );\n \n // Save to a file\n await writeFile('output.wav', audioBuffer);\n}\n```\n\n### Multi-speaker Audio Generation\n\nYou can generate audio with multiple speakers, each with their own voice:\n\n```ts\nconst response = await ai.generate({\n model: vertexAI.model('gemini-2.5-flash-preview-tts'),\n config: {\n responseModalities: ['AUDIO'],\n speechConfig: {\n multiSpeakerVoiceConfig: {\n speakerVoiceConfigs: [\n {\n speaker: 'Speaker1',\n voiceConfig: {\n prebuiltVoiceConfig: { voiceName: 'Algenib' },\n },\n },\n {\n speaker: 'Speaker2',\n voiceConfig: {\n prebuiltVoiceConfig: { voiceName: 'Achernar' },\n },\n },\n ],\n },\n },\n },\n prompt: `Here's the dialog:\n Speaker1: \"Genkit is an amazing Gen AI library!\"\n Speaker2: \"I thought it was a framework.\"`,\n});\n```\n\n### Configuration Options\n\nThe Vertex AI TTS models support a wide range of configuration options:\n\n#### Voice Selection\n\nVertex AI offers multiple pre-built voices with different characteristics:\n\n```ts\nspeechConfig: {\n voiceConfig: {\n prebuiltVoiceConfig: { \n voiceName: 'Algenib' // Other options include: 'Achernar', 'Ankaa', etc.\n },\n },\n}\n```\n\n#### Speech Emphasis and Prosody Control\n\nYou can use markdown-style formatting in your prompt to add emphasis:\n\n- Bold text (`**like this**`) for stronger emphasis\n- Italic text (`*like this*`) for moderate emphasis\n\nExample:\n\n```ts\nprompt: 'Genkit is an **amazing** Gen AI *library*!'\n```\n\n#### Advanced Speech Parameters\n\nFor more precise control over the generated speech:\n\n```ts\nspeechConfig: {\n voiceConfig: {\n prebuiltVoiceConfig: { \n voiceName: 'Algenib',\n speakingRate: 1.0, // Range: 0.25 to 4.0, default is 1.0\n pitch: 0.0, // Range: -20.0 to 20.0, default is 0.0\n volumeGainDb: 0.0, // Range: -96.0 to 16.0, default is 0.0\n },\n },\n}\n```\n\n- `speakingRate`: Controls the speed of speech (higher values = faster speech)\n- `pitch`: Adjusts the pitch of the voice (higher values = higher pitch)\n- `volumeGainDb`: Controls the volume (higher values = louder)\n\n#### SSML Support\n\nFor advanced speech synthesis control, you can use Speech Synthesis Markup Language (SSML) in your prompts:\n\n```ts\nprompt: `\n Here is a pause.\n This text is spoken slowly and with a higher pitch.\n 12345\n`\n```\n\nNote: When using SSML, you must wrap your entire prompt in `` tags.\n\nFor more detailed information about the Vertex AI TTS models and their configuration options, see the [Vertex AI Speech Generation documentation](https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/generate-speech).\n\n## Context Caching\n\nThe Vertex AI Genkit plugin supports **Context Caching**, which allows models to reuse previously cached content to optimize token usage when dealing with large pieces of content. This feature is especially useful for conversational flows or scenarios where the model references a large piece of content consistently across multiple requests.\n\n### How to Use Context Caching\n\nTo enable context caching, ensure your model supports it. For example, `gemini-2.5-flash` and `gemini-2.0-pro` are models that support context caching, and you will have to specify version number `001`.\n\nYou can define a caching mechanism in your application like this:\n\n```ts\nconst ai = genkit({\n plugins: [vertexAI({ location: 'us-central1' })],\n});\n\nconst llmResponse = await ai.generate({\n messages: [\n {\n role: 'user',\n content: [{ text: 'Here is the relevant text from War and Peace.' }],\n },\n {\n role: 'model',\n content: [\n {\n text: \"Based on War and Peace, here is some analysis of Pierre Bezukhov's character.\",\n },\n ],\n metadata: {\n cache: {\n ttlSeconds: 300, // Cache this message for 5 minutes\n },\n },\n },\n ],\n model: vertexAI.model('gemini-2.5-flash'),\n prompt: \"Describe Pierre's transformation throughout the novel.\",\n});\n```\n\nIn this setup:\n\n- **`messages`**: Allows you to pass conversation history.\n- **`metadata.cache.ttlSeconds`**: Specifies the time-to-live (TTL) for caching a specific response.\n\n### Example: Leveraging Large Texts with Context\n\nFor applications referencing long documents, such as _War and Peace_ or _Lord of the Rings_, you can structure your queries to reuse cached contexts:\n\n```ts\nconst textContent = await fs.readFile('path/to/war_and_peace.txt', 'utf-8');\n\nconst llmResponse = await ai.generate({\n messages: [\n {\n role: 'user',\n content: [{ text: textContent }], // Include the large text as context\n },\n {\n role: 'model',\n content: [\n {\n text: 'This analysis is based on the provided text from War and Peace.',\n },\n ],\n metadata: {\n cache: {\n ttlSeconds: 300, // Cache the response to avoid reloading the full text\n },\n },\n },\n ],\n model: vertexAI.model('gemini-2.5-flash'),\n prompt: 'Analyze the relationship between Pierre and Natasha.',\n});\n```\n\n### Benefits of Context Caching\n\n1. **Improved Performance**: Reduces the need for repeated processing of large inputs.\n2. **Cost Efficiency**: Decreases API usage for redundant data, optimizing token consumption.\n3. **Better Latency**: Speeds up response times for repeated or related queries.\n\n### Supported Models for Context Caching\n\nOnly specific models, such as `gemini-2.5-flash` and `gemini-2.0-pro`, support context caching, and currently only on version numbers `001`. If an unsupported model is used, an error will be raised, indicating that caching cannot be applied.\n\n### Further Reading\n\nSee more information regarding context caching on Vertex AI in their [documentation](https://cloud.google.com/vertex-ai/generative-ai/docs/context-cache/context-cache-overview).\n", "title": "Vertex AI plugin", - "lang": "js" - }, - "js/plugins/third-party-plugins/index.mdx": { - "text": "import { Card, CardGrid } from '@astrojs/starlight/components';\nimport { Icon } from '@astrojs/starlight/components';\n\nThis page lists third-party plugins for Genkit that are built and maintained by Firebase or\nour partners.\n\n\n \n The Neo4j plugin integrates the{' '}\n \n Neo4j graph databases\n \n {' '}\n into the Genkit framework. You can leverage knowledge graphs and semantic search for building advanced AI\n applications, particularly for RAG scenarios where contextual information from a knowledge graph is crucial.\n
\n \n View plugin info\n \n \n \n \n The Astra DB plugin integrates{' '}\n \n Astra DB\n \n {' '}\n into the Genkit framework as an indexer and retriever. You can efficiently embed, index, and retrieve data within\n your Genkit applications.\n
\n \n View plugin info\n \n \n \n \n The Pinecone plugin provides indexer and retriever implementations that use the{' '}\n \n Pinecone\n \n {' '}\n cloud vector database.\n
\n View plugin info\n \n \n The Chroma plugin provides indexer and retriever implementations that use the{' '}\n \n Chroma vector database\n \n {' '}\n in client/server mode.\n
\n View plugin info\n \n \n The AuthO plugin enables you to build secure AI-powered applications using{' '}\n \n Auth0\n \n {' '}\n and{' '}\n \n Okta FGA\n \n \n
\n \n View plugin info\n \n \n \n \n The Ollama plugin provides interfaces to any of the local LLMs supported by{' '}\n \n Ollama\n \n \n
\n View plugin info\n \n \n The pgvector template is an example PostgreSQL and pgvector retriever implementation. You can use the provided\n examples as a starting point and modify them to work with your database schema.\n
\n View template\n \n\n", + "description": "This document describes the Vertex AI plugin for Genkit, providing interfaces to Google's generative AI models, evaluation metrics, Vector Search, and text-to-speech capabilities.", + "lang": "js", + "headers": "## Installation\n## Configuration\n## Usage\n### Generative AI Models\n#### Anthropic Claude 3 on Vertex AI Model Garden\n#### Llama 3.1 405b on Vertex AI Model Garden\n#### Mistral Models on Vertex AI Model Garden\n### Evaluators\n### Indexers and retrievers\n## Text-to-Speech (TTS) Models\n### Basic Usage\n### Multi-speaker Audio Generation\n### Configuration Options\n#### Voice Selection\n#### Speech Emphasis and Prosody Control\n#### Advanced Speech Parameters\n#### SSML Support\n## Context Caching\n### How to Use Context Caching\n### Example: Leveraging Large Texts with Context\n### Benefits of Context Caching\n### Supported Models for Context Caching\n### Further Reading\n" + }, + "js/plugins/xai.md": { + "text": "# xAI Plugin\n\nThe `@genkit-ai/compat-oai` package includes a pre-configured plugin for [xAI (Grok)](https://x.ai/) models.\nThe `xAI` plugin provides access to the `grok` family of models, including `grok-image` for image generation.\n\n:::note\nThe xAI plugin is built on top of the `openAICompatible` plugin. It is pre-configured for xAI's API endpoints.\n:::\n\n## Installation\n\n```bash\nnpm install @genkit-ai/compat-oai\n```\n\n## Configuration\n\nTo use this plugin, import `xAI` and specify it when you initialize Genkit:\n\n```ts\nimport { genkit } from 'genkit';\nimport { xAI } from '@genkit-ai/compat-oai/xai';\n\nexport const ai = genkit({\n plugins: [xAI()],\n});\n```\n\nYou must provide an API key from xAI. You can get an API key from your [xAI account settings](https://console.x.ai/).\n\nConfigure the plugin to use your API key by doing one of the following:\n\n- Set the `XAI_API_KEY` environment variable to your API key.\n- Specify the API key when you initialize the plugin:\n\n ```ts\n xAI({ apiKey: yourKey });\n ```\n\nAs always, avoid embedding API keys directly in your code.\n\n## Usage\n\nUse the `xAI.model()` helper to reference a Grok model.\n\n```ts\nimport { genkit, z } from 'genkit';\nimport { xAI } from '@genkit-ai/compat-oai/xai';\n\nconst ai = genkit({\n plugins: [xAI({ apiKey: process.env.XAI_API_KEY })],\n});\n\nexport const grokFlow = ai.defineFlow(\n {\n name: 'grokFlow',\n inputSchema: z.object({ subject: z.string() }),\n outputSchema: z.object({ fact: z.string() }),\n },\n async ({ subject }) => {\n const llmResponse = await ai.generate({\n model: xAI.model('grok-3-mini'),\n prompt: `tell me a fun fact about ${subject}`,\n });\n return { fact: llmResponse.text };\n },\n);\n```\n\n## Advanced usage\n\n### Passthrough configuration\n\nYou can pass configuration options that are not defined in the plugin's custom configuration schema. This\npermits you to access new models and features without having to update your Genkit version.\n\n```ts\nimport { genkit } from 'genkit';\nimport { xAI } from '@genkit-ai/compat-oai/xAI';\n\nconst ai = genkit({\n plugins: [xAI()],\n});\n\nconst llmResponse = await ai.generate({\n prompt: `Tell me a cool story`,\n model: xAI.model('grok-new'), // hypothetical new model\n config: {\n new_feature_parameter: ... // hypothetical config needed for new model\n },\n});\n```\n\nGenkit passes this configuration as-is to the xAI API giving you access to the new model features.\nNote that the field name and types are not validated by Genkit and should match the xAI API\nspecification to work.\n", + "title": "xAI Plugin", + "description": "Learn how to configure and use Genkit xAI plugin to access xAI (Grok) models.", + "lang": "js", + "headers": "## Installation\n## Configuration\n## Usage\n## Advanced usage\n### Passthrough configuration\n" + }, + "js/plugins/third-party-plugins/index.md": { + "text": "# Third-party plugins by Firebase and partners\n\nThis page lists third-party plugins for Genkit that are built and maintained by Firebase or\nour partners.\n\n\n \n The Neo4j plugin integrates the{' '}\n \n Neo4j graph databases\n \n {' '}\n into the Genkit framework. You can leverage knowledge graphs and semantic search for building advanced AI\n applications, particularly for RAG scenarios where contextual information from a knowledge graph is crucial.\n
\n \n View plugin info\n \n \n \n \n The Astra DB plugin integrates{' '}\n \n Astra DB\n \n {' '}\n into the Genkit framework as an indexer and retriever. You can efficiently embed, index, and retrieve data within\n your Genkit applications.\n
\n \n View plugin info\n \n \n \n \n The Pinecone plugin provides indexer and retriever implementations that use the{' '}\n \n Pinecone\n \n {' '}\n cloud vector database.\n
\n View plugin info\n \n \n The Chroma plugin provides indexer and retriever implementations that use the{' '}\n \n Chroma vector database\n \n {' '}\n in client/server mode.\n
\n View plugin info\n \n \n The AuthO plugin enables you to build secure AI-powered applications using{' '}\n \n Auth0\n \n {' '}\n and{' '}\n \n Okta FGA\n \n \n
\n \n View plugin info\n \n \n \n \n The Ollama plugin provides interfaces to any of the local LLMs supported by{' '}\n \n Ollama\n \n \n
\n View plugin info\n \n \n The pgvector template is an example PostgreSQL and pgvector retriever implementation. You can use the provided\n examples as a starting point and modify them to work with your database schema.\n
\n View template\n \n \n The Cloud SQL for PostgreSQL plugin provides indexer and retriever implementations that use the{' '}\n \n Cloud SQL for PostgreSQL\n \n {' '}\n cloud vector database.\n
\n View plugin info\n \n \n The MCP Toolbox plugin integrates the{' '}\n \n MCP Toolbox tools\n \n {' '}\n into the Genkit framework. It enables you to develop tools easier, faster, and more securely by handling the complexities such as connection pooling, authentication, and more.\n
\n View template\n \n", "title": "Third-party plugins by Firebase and partners", - "lang": "js" + "description": "This page lists third-party plugins for Genkit that are built and maintained by Firebase or our partners.", + "lang": "js", + "headers": "" }, "js/observability/advanced-configuration.md": { - "text": "This guide focuses on advanced configuration options for deployed features using\nthe Firebase telemetry plugin. Detailed descriptions of each configuration\noption can be found in our\n[JS API reference documentation](https://js.api.genkit.dev/interfaces/_genkit-ai_google-cloud.GcpTelemetryConfigOptions.html).\n\nThis documentation will describe how to fine-tune which telemetry is collected,\nhow often, and from what environments.\n\n## Default Configuration\n\nThe Firebase telemetry plugin provides default options, out of the box, to get\nyou up and running quickly. These are the provided defaults:\n\n```typescript\n{\n autoInstrumentation: true,\n autoInstrumentationConfig: {\n '@opentelemetry/instrumentation-dns': { enabled: false },\n }\n disableMetrics: false,\n disableTraces: false,\n disableLoggingInputAndOutput: false,\n forceDevExport: false,\n // 5 minutes\n metricExportIntervalMillis: 300_000,\n // 5 minutes\n metricExportTimeoutMillis: 300_000,\n // See https://js.api.genkit.dev/interfaces/_genkit-ai_google-cloud.GcpTelemetryConfigOptions.html#sampler\n sampler: AlwaysOnSampler()\n}\n```\n\n## Export local telemetry\n\nTo export telemetry when running locally set the `forceDevExport` option to\n`true`.\n\n```typescript\nimport { enableFirebaseTelemetry } from '@genkit-ai/firebase';\n\nenableFirebaseTelemetry({ forceDevExport: true });\n```\n\nDuring development and testing, you can decrease latency by adjusting the export\ninterval and timeout.\n\nNote: Shipping to production with a frequent export interval may\nincrease the cost for exported telemetry.\n\n```typescript\nimport { enableFirebaseTelemetry } from '@genkit-ai/firebase';\n\nenableFirebaseTelemetry({\n forceDevExport: true,\n metricExportIntervalMillis: 10_000, // 10 seconds\n metricExportTimeoutMillis: 10_000, // 10 seconds\n});\n```\n\n## Adjust auto instrumentation\n\nThe Firebase telemetry plugin will automatically collect traces and metrics for\npopular frameworks using OpenTelemetry [zero-code instrumentation](https://opentelemetry.io/docs/zero-code/js/).\n\nA full list of available instrumentations can be found in the\n[auto-instrumentations-node](https://github.com/open-telemetry/opentelemetry-js-contrib/blob/main/metapackages/auto-instrumentations-node/README.md#supported-instrumentations)\ndocumentation.\n\nTo selectively disable or enable instrumentations that are eligible for auto\ninstrumentation, update the `autoInstrumentationConfig` field:\n\n```typescript\nimport { enableFirebaseTelemetry } from '@genkit-ai/firebase';\n\nenableFirebaseTelemetry({\n autoInstrumentationConfig: {\n '@opentelemetry/instrumentation-fs': { enabled: false },\n '@opentelemetry/instrumentation-dns': { enabled: false },\n '@opentelemetry/instrumentation-net': { enabled: false },\n },\n});\n```\n\n## Disable telemetry\n\nGenkit Monitoring leverages a combination of logging, tracing, and\nmetrics to capture a holistic view of your Genkit interactions, however, you can\nalso disable each of these elements independently if needed.\n\n### Disable input and output logging\n\nBy default, the Firebase telemetry plugin will capture inputs and outputs for\neach Genkit feature or step.\n\nTo help you control how customer data is stored, you can disable the logging of\ninput and output by adding the following to your configuration:\n\n```typescript\nimport { enableFirebaseTelemetry } from '@genkit-ai/firebase';\n\nenableFirebaseTelemetry({\n disableLoggingInputAndOutput: true,\n});\n```\n\nWith this option set, input and output attributes will be redacted\nin the Genkit Monitoring trace viewer and will be missing\nfrom Google Cloud logging.\n\n### Disable metrics\n\nTo disable metrics collection, add the following to your configuration:\n\n```typescript\nimport { enableFirebaseTelemetry } from '@genkit-ai/firebase';\n\nenableFirebaseTelemetry({\n disableMetrics: true,\n});\n```\n\nWith this option set, you will no longer see stability metrics in the\nGenkit Monitoring dashboard and will be missing from Google Cloud\nMetrics.\n\n### Disable traces\n\nTo disable trace collection, add the following to your configuration:\n\n```typescript\nimport { enableFirebaseTelemetry } from '@genkit-ai/firebase';\n\nenableFirebaseTelemetry({\n disableTraces: true,\n});\n```\n\nWith this option set, you will no longer see traces in the Genkit\nMonitoring feature page, have access to the trace viewer, or see traces\npresent in Google Cloud Tracing.\n", + "text": "# Advanced Configuration\n\nThis guide focuses on advanced configuration options for deployed features using\nthe Firebase telemetry plugin. Detailed descriptions of each configuration\noption can be found in our\n[JS API reference documentation](https://js.api.genkit.dev/interfaces/_genkit-ai_google-cloud.GcpTelemetryConfigOptions.html).\n\nThis documentation will describe how to fine-tune which telemetry is collected,\nhow often, and from what environments.\n\n## Default Configuration\n\nThe Firebase telemetry plugin provides default options, out of the box, to get\nyou up and running quickly. These are the provided defaults:\n\n```typescript\n{\n autoInstrumentation: true,\n autoInstrumentationConfig: {\n '@opentelemetry/instrumentation-dns': { enabled: false },\n }\n disableMetrics: false,\n disableTraces: false,\n disableLoggingInputAndOutput: false,\n forceDevExport: false,\n // 5 minutes\n metricExportIntervalMillis: 300_000,\n // 5 minutes\n metricExportTimeoutMillis: 300_000,\n // See https://js.api.genkit.dev/interfaces/_genkit-ai_google-cloud.GcpTelemetryConfigOptions.html#sampler\n sampler: AlwaysOnSampler()\n}\n```\n\n## Export local telemetry\n\nTo export telemetry when running locally set the `forceDevExport` option to\n`true`.\n\n```typescript\nimport { enableFirebaseTelemetry } from '@genkit-ai/firebase';\n\nenableFirebaseTelemetry({ forceDevExport: true });\n```\n\nDuring development and testing, you can decrease latency by adjusting the export\ninterval and timeout.\n\nNote: Shipping to production with a frequent export interval may\nincrease the cost for exported telemetry.\n\n```typescript\nimport { enableFirebaseTelemetry } from '@genkit-ai/firebase';\n\nenableFirebaseTelemetry({\n forceDevExport: true,\n metricExportIntervalMillis: 10_000, // 10 seconds\n metricExportTimeoutMillis: 10_000, // 10 seconds\n});\n```\n\n## Adjust auto instrumentation\n\nThe Firebase telemetry plugin will automatically collect traces and metrics for\npopular frameworks using OpenTelemetry [zero-code instrumentation](https://opentelemetry.io/docs/zero-code/js/).\n\nA full list of available instrumentations can be found in the\n[auto-instrumentations-node](https://github.com/open-telemetry/opentelemetry-js-contrib/blob/main/metapackages/auto-instrumentations-node/README.md#supported-instrumentations)\ndocumentation.\n\nTo selectively disable or enable instrumentations that are eligible for auto\ninstrumentation, update the `autoInstrumentationConfig` field:\n\n```typescript\nimport { enableFirebaseTelemetry } from '@genkit-ai/firebase';\n\nenableFirebaseTelemetry({\n autoInstrumentationConfig: {\n '@opentelemetry/instrumentation-fs': { enabled: false },\n '@opentelemetry/instrumentation-dns': { enabled: false },\n '@opentelemetry/instrumentation-net': { enabled: false },\n },\n});\n```\n\n## Disable telemetry\n\nGenkit Monitoring leverages a combination of logging, tracing, and\nmetrics to capture a holistic view of your Genkit interactions, however, you can\nalso disable each of these elements independently if needed.\n\n### Disable input and output logging\n\nBy default, the Firebase telemetry plugin will capture inputs and outputs for\neach Genkit feature or step.\n\nTo help you control how customer data is stored, you can disable the logging of\ninput and output by adding the following to your configuration:\n\n```typescript\nimport { enableFirebaseTelemetry } from '@genkit-ai/firebase';\n\nenableFirebaseTelemetry({\n disableLoggingInputAndOutput: true,\n});\n```\n\nWith this option set, input and output attributes will be redacted\nin the Genkit Monitoring trace viewer and will be missing\nfrom Google Cloud logging.\n\n### Disable metrics\n\nTo disable metrics collection, add the following to your configuration:\n\n```typescript\nimport { enableFirebaseTelemetry } from '@genkit-ai/firebase';\n\nenableFirebaseTelemetry({\n disableMetrics: true,\n});\n```\n\nWith this option set, you will no longer see stability metrics in the\nGenkit Monitoring dashboard and will be missing from Google Cloud\nMetrics.\n\n### Disable traces\n\nTo disable trace collection, add the following to your configuration:\n\n```typescript\nimport { enableFirebaseTelemetry } from '@genkit-ai/firebase';\n\nenableFirebaseTelemetry({\n disableTraces: true,\n});\n```\n\nWith this option set, you will no longer see traces in the Genkit\nMonitoring feature page, have access to the trace viewer, or see traces\npresent in Google Cloud Tracing.\n", "title": "Advanced Configuration", - "lang": "js" + "description": "This guide covers advanced configuration options for Genkit's Firebase telemetry plugin, including fine-tuning telemetry collection, export settings, and disabling specific data types.", + "lang": "js", + "headers": "## Default Configuration\n## Export local telemetry\n## Adjust auto instrumentation\n## Disable telemetry\n### Disable input and output logging\n### Disable metrics\n### Disable traces\n" }, "js/observability/authentication.md": { - "text": "The Firebase telemetry plugin requires a Google Cloud or Firebase project ID\nand application credentials.\n\nIf you don't have a Google Cloud project and account, you can set one up in the\n[Firebase Console](https://console.firebase.google.com/) or in the\n[Google Cloud Console](https://cloud.google.com). All Firebase project IDs are\nGoogle Cloud project IDs.\n\n## Enable APIs\n\nPrior to adding the plugin, make sure the following APIs are enabled for\nyour project:\n\n- [Cloud Logging API](https://console.cloud.google.com/apis/library/logging.googleapis.com)\n- [Cloud Trace API](https://console.cloud.google.com/apis/library/cloudtrace.googleapis.com)\n- [Cloud Monitoring API](https://console.cloud.google.com/apis/library/monitoring.googleapis.com)\n\nThese APIs should be listed in the\n[API dashboard](https://console.cloud.google.com/apis/dashboard) for your\nproject.\nClick to learn more about how to [enable and disable APIs](https://support.google.com/googleapi/answer/6158841).\n\n## User Authentication\n\nTo export telemetry from your local development environment to Genkit\nMonitoring, you will need to authenticate yourself with Google Cloud.\n\nThe easiest way to authenticate as yourself is using the gcloud CLI, which will\nautomatically make your credentials available to the framework through\n[Application Default Credentials (ADC)](https://cloud.google.com/docs/authentication/application-default-credentials).\n\nIf you don't have the gcloud CLI installed, first follow the [installation instructions](https://cloud.google.com/sdk/docs/install#installation_instructions).\n\n1. Authenticate using the `gcloud` CLI:\n\n ```bash\n gcloud auth application-default login\n ```\n\n2. Set your project ID\n\n ```bash\n gcloud config set project PROJECT_ID\n ```\n\n## Deploy to Google Cloud\n\nIf deploying your code to a Google Cloud or Firebase environment (Cloud\nFunctions, Cloud Run, App Hosting, etc), the project ID and credentials will be\ndiscovered automatically with\n[Application Default Credentials](https://cloud.google.com/docs/authentication/provide-credentials-adc).\n\nYou will need to apply the following roles to the service account that is\nrunning your code (i.e. 'attached service account') using the\n[IAM Console](https://console.cloud.google.com/iam-admin/iam):\n\n- `roles/monitoring.metricWriter`\n- `roles/cloudtrace.agent`\n- `roles/logging.logWriter`\n\nNot sure which service account is the right one? See the\n[Find or create your service account](#find-or-create-your-service-account)\nsection.\n\n## Deploy outside of Google Cloud (with ADC)\n\nIf possible, use\n[Application Default Credentials](https://cloud.google.com/docs/authentication/provide-credentials-adc)\nto make credentials available to the plugin.\n\nTypically this involves generating a service account key and deploying\nthose credentials to your production environment.\n\n1. Follow the instructions to set up a\n [service account key](https://cloud.google.com/iam/docs/keys-create-delete#creating).\n\n2. Ensure the service account has the following roles:\n\n - `roles/monitoring.metricWriter`\n - `roles/cloudtrace.agent`\n - `roles/logging.logWriter`\n\n3. Deploy the credential file to production (**do not** check into source code)\n\n4. Set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable as the path to\n the credential file.\n\n ```bash\n GOOGLE_APPLICATION_CREDENTIALS = \"path/to/your/key/file\"\n ```\n\nNot sure which service account is the right one? See the\n[Find or create your service account](#find-or-create-your-service-account)\nsection.\n\n## Deploy outside of Google Cloud (without ADC)\n\nIn some serverless environments, you may not be able to deploy a credential\nfile.\n\n1. Follow the instructions to set up a\n [service account key](https://cloud.google.com/iam/docs/keys-create-delete#creating).\n\n2. Ensure the service account has the following roles:\n\n - `roles/monitoring.metricWriter`\n - `roles/cloudtrace.agent`\n - `roles/logging.logWriter`\n\n3. Download the credential file.\n\n4. Assign the contents of the credential file to the\n `GCLOUD_SERVICE_ACCOUNT_CREDS` environment variable as follows:\n\n```bash\nGCLOUD_SERVICE_ACCOUNT_CREDS='{\n \"type\": \"service_account\",\n \"project_id\": \"your-project-id\",\n \"private_key_id\": \"your-private-key-id\",\n \"private_key\": \"your-private-key\",\n \"client_email\": \"your-client-email\",\n \"client_id\": \"your-client-id\",\n \"auth_uri\": \"https://accounts.google.com/o/oauth2/auth\",\n \"token_uri\": \"https://accounts.google.com/o/oauth2/token\",\n \"auth_provider_x509_cert_url\": \"https://www.googleapis.com/oauth2/v1/certs\",\n \"client_x509_cert_url\": \"your-cert-url\"\n}'\n```\n\nNot sure which service account is the right one? See the\n[Find or create your service account](#find-or-create-your-service-account)\nsection.\n\n## Find or create your service account\n\nTo find the appropriate service account:\n\n1. Navigate to the [service accounts page](https://console.cloud.google.com/iam-admin/serviceaccounts)\n in the Google Cloud Console\n2. Select your project\n3. Find the appropriate service account. Common default service accounts are as follows:\n\n- Firebase functions & Cloud Run\n\n `PROJECT ID-compute@developer.gserviceaccount.com`\n\n- App Engine\n\n `PROJECT ID@appspot.gserviceaccount.com`\n\n- App Hosting\n\n `firebase-app-hosting-compute@PROJECT ID.iam.gserviceaccount.com`\n\nIf you are deploying outside of the Google ecosystem or don't want to use a\ndefault service account, you can\n[create a service account](https://cloud.google.com/iam/docs/service-accounts-create#creating)\nin the Google Cloud console.\n", + "text": "# Authentication and authorization\n\nThe Firebase telemetry plugin requires a Google Cloud or Firebase project ID\nand application credentials.\n\nIf you don't have a Google Cloud project and account, you can set one up in the\n[Firebase Console](https://console.firebase.google.com/) or in the\n[Google Cloud Console](https://cloud.google.com). All Firebase project IDs are\nGoogle Cloud project IDs.\n\n## Enable APIs\n\nPrior to adding the plugin, make sure the following APIs are enabled for\nyour project:\n\n- [Cloud Logging API](https://console.cloud.google.com/apis/library/logging.googleapis.com)\n- [Cloud Trace API](https://console.cloud.google.com/apis/library/cloudtrace.googleapis.com)\n- [Cloud Monitoring API](https://console.cloud.google.com/apis/library/monitoring.googleapis.com)\n\nThese APIs should be listed in the\n[API dashboard](https://console.cloud.google.com/apis/dashboard) for your\nproject.\nClick to learn more about how to [enable and disable APIs](https://support.google.com/googleapi/answer/6158841).\n\n## User Authentication\n\nTo export telemetry from your local development environment to Genkit\nMonitoring, you will need to authenticate yourself with Google Cloud.\n\nThe easiest way to authenticate as yourself is using the gcloud CLI, which will\nautomatically make your credentials available to the framework through\n[Application Default Credentials (ADC)](https://cloud.google.com/docs/authentication/application-default-credentials).\n\nIf you don't have the gcloud CLI installed, first follow the [installation instructions](https://cloud.google.com/sdk/docs/install#installation_instructions).\n\n1. Authenticate using the `gcloud` CLI:\n\n ```bash\n gcloud auth application-default login\n ```\n\n2. Set your project ID\n\n ```bash\n gcloud config set project PROJECT_ID\n ```\n\n## Deploy to Google Cloud\n\nIf deploying your code to a Google Cloud or Firebase environment (Cloud\nFunctions, Cloud Run, App Hosting, etc), the project ID and credentials will be\ndiscovered automatically with\n[Application Default Credentials](https://cloud.google.com/docs/authentication/provide-credentials-adc).\n\nYou will need to apply the following roles to the service account that is\nrunning your code (i.e. 'attached service account') using the\n[IAM Console](https://console.cloud.google.com/iam-admin/iam):\n\n- `roles/monitoring.metricWriter`\n- `roles/cloudtrace.agent`\n- `roles/logging.logWriter`\n\nNot sure which service account is the right one? See the\n[Find or create your service account](#find-or-create-your-service-account)\nsection.\n\n## Deploy outside of Google Cloud (with ADC)\n\nIf possible, use\n[Application Default Credentials](https://cloud.google.com/docs/authentication/provide-credentials-adc)\nto make credentials available to the plugin.\n\nTypically this involves generating a service account key and deploying\nthose credentials to your production environment.\n\n1. Follow the instructions to set up a\n [service account key](https://cloud.google.com/iam/docs/keys-create-delete#creating).\n\n2. Ensure the service account has the following roles:\n\n - `roles/monitoring.metricWriter`\n - `roles/cloudtrace.agent`\n - `roles/logging.logWriter`\n\n3. Deploy the credential file to production (**do not** check into source code)\n\n4. Set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable as the path to\n the credential file.\n\n ```bash\n GOOGLE_APPLICATION_CREDENTIALS = \"path/to/your/key/file\"\n ```\n\nNot sure which service account is the right one? See the\n[Find or create your service account](#find-or-create-your-service-account)\nsection.\n\n## Deploy outside of Google Cloud (without ADC)\n\nIn some serverless environments, you may not be able to deploy a credential\nfile.\n\n1. Follow the instructions to set up a\n [service account key](https://cloud.google.com/iam/docs/keys-create-delete#creating).\n\n2. Ensure the service account has the following roles:\n\n - `roles/monitoring.metricWriter`\n - `roles/cloudtrace.agent`\n - `roles/logging.logWriter`\n\n3. Download the credential file.\n\n4. Assign the contents of the credential file to the\n `GCLOUD_SERVICE_ACCOUNT_CREDS` environment variable as follows:\n\n```bash\nGCLOUD_SERVICE_ACCOUNT_CREDS='{\n \"type\": \"service_account\",\n \"project_id\": \"your-project-id\",\n \"private_key_id\": \"your-private-key-id\",\n \"private_key\": \"your-private-key\",\n \"client_email\": \"your-client-email\",\n \"client_id\": \"your-client-id\",\n \"auth_uri\": \"https://accounts.google.com/o/oauth2/auth\",\n \"token_uri\": \"https://accounts.google.com/o/oauth2/token\",\n \"auth_provider_x509_cert_url\": \"https://www.googleapis.com/oauth2/v1/certs\",\n \"client_x509_cert_url\": \"your-cert-url\"\n}'\n```\n\nNot sure which service account is the right one? See the\n[Find or create your service account](#find-or-create-your-service-account)\nsection.\n\n## Find or create your service account\n\nTo find the appropriate service account:\n\n1. Navigate to the [service accounts page](https://console.cloud.google.com/iam-admin/serviceaccounts)\n in the Google Cloud Console\n2. Select your project\n3. Find the appropriate service account. Common default service accounts are as follows:\n\n- Firebase functions & Cloud Run\n\n `PROJECT ID-compute@developer.gserviceaccount.com`\n\n- App Engine\n\n `PROJECT ID@appspot.gserviceaccount.com`\n\n- App Hosting\n\n `firebase-app-hosting-compute@PROJECT ID.iam.gserviceaccount.com`\n\nIf you are deploying outside of the Google ecosystem or don't want to use a\ndefault service account, you can\n[create a service account](https://cloud.google.com/iam/docs/service-accounts-create#creating)\nin the Google Cloud console.\n", "title": "Authentication and authorization", - "lang": "js" + "description": "Learn how to authenticate and authorize the Firebase telemetry plugin for Genkit, covering API enablement, user authentication, and deployment to Google Cloud or other environments.", + "lang": "js", + "headers": "## Enable APIs\n## User Authentication\n## Deploy to Google Cloud\n## Deploy outside of Google Cloud (with ADC)\n## Deploy outside of Google Cloud (without ADC)\n## Find or create your service account\n" }, "js/observability/getting-started.md": { - "text": "This quickstart guide describes how to set up Genkit Monitoring for\nyour deployed Genkit features, so that you can collect and view real-time\ntelemetry data. With Genkit Monitoring, you get visibility into how\nyour Genkit features are performing in production.\n\nKey capabilities of Genkit Monitoring include:\n\n- Viewing quantitative metrics like Genkit feature latency, errors, and\n token usage.\n- Inspecting traces to see your Genkit's feature steps, inputs, and outputs,\n to help with debugging and quality improvement.\n- Exporting production traces to run evals within Genkit.\n\nSetting up Genkit Monitoring requires completing tasks in both your codebase\nand on the Google Cloud Console.\n\n## Before you begin\n\n1. If you haven't already, create a Firebase project.\n\n In the [Firebase console](https://console.firebase.google.com), click\n **Add a project**, then follow the on-screen instructions. You can\n create a new project or add Firebase services to an already-existing Google Cloud project.\n\n2. Ensure your project is on the\n [Blaze pricing plan](https://firebase.google.com/pricing).\n\n Genkit Monitoring relies on telemetry data written to Google Cloud\n Logging, Metrics, and Trace, which are paid services. View the\n [Google Cloud Observability pricing](https://cloud.google.com/stackdriver/pricing) page for pricing details and to learn about free-of-charge tier limits.\n\n3. Write a Genkit feature by following the [Get Started Guide](/docs/get-started), and prepare your code for deployment by using one of the following guides:\n\n a. [Deploy flows using Cloud Functions for Firebase](/docs/firebase)\n b. [Deploy flows using Cloud Run](/docs/cloud-run)\n c. [Deploy flows to any Node.js platform](/docs/deploy-node)\n\n## Step 1. Add the Firebase plugin\n\nInstall the `@genkit-ai/firebase` plugin in your project:\n\n```bash\nnpm install @genkit-ai/firebase\n```\n\nImport `enableFirebaseTelemetry` into your Genkit configuration file (the\nfile where `genkit(...)` is initalized), and call it:\n\n```typescript\nimport { enableFirebaseTelemetry } from '@genkit-ai/firebase';\n\nenableFirebaseTelemetry();\n```\n\n## Step 2. Enable the required APIs\n\nMake sure that the following APIs are enabled for your Google Cloud project:\n\n- [Cloud Logging API](https://console.cloud.google.com/apis/library/logging.googleapis.com)\n- [Cloud Trace API](https://console.cloud.google.com/apis/library/cloudtrace.googleapis.com)\n- [Cloud Monitoring API](https://console.cloud.google.com/apis/library/monitoring.googleapis.com)\n\nThese APIs should be listed in the\n[API dashboard](https://console.cloud.google.com/apis/dashboard) for your\nproject.\n\n## Step 3. Set up permissions\n\nThe Firebase plugin needs to use a _service account_ to authenticate with\nGoogle Cloud Logging, Metrics, and Trace services.\n\nGrant the following roles to whichever service account is configured to run your code within the [Google Cloud IAM Console](https://console.cloud.google.com/iam-admin/iam). For Cloud Functions for Firebase and Cloud Run, that's typically the default compute service account.\n\n- **Monitoring Metric Writer** (`roles/monitoring.metricWriter`)\n- **Cloud Trace Agent** (`roles/cloudtrace.agent`)\n- **Logs Writer** (`roles/logging.logWriter`)\n\n## Step 4. (Optional) Test your configuration locally\n\nBefore deploying, you can run your Genkit code locally to confirm that\ntelemetry data is being collected, and is viewable in the Genkit Monitoring\ndashboard.\n\n1. In your Genkit code, set `forceDevExport` to `true` to send telemetry from\n your local environment.\n\n2. Use your service account to authenticate and test your configuration.\n\n :::tip\n In order to impersonate the service account, you will need to have\n the `roles/iam.serviceAccountTokenCreator`\n [IAM role](https://console.cloud.google.com/iam-admin/iam) applied to your\n user account.\n :::\n\n With the\n [Google Cloud CLI tool](https://cloud.google.com/sdk/docs/install?authuser=0),\n authenticate using the service account:\n\n ```bash\n gcloud auth application-default login --impersonate-service-account SERVICE_ACCT_EMAIL\n ```\n\n3. Run and invoke your Genkit feature, and then view metrics on the\n [Genkit Monitoring dashboard](https://console.firebase.google.com/project/_/genai_monitoring).\n Allow for up to 5 minutes to collect the first metric. You can reduce this\n delay by setting `metricExportIntervalMillis` in the telemetry configuration.\n\n4. If metrics are not appearing in the Genkit Monitoring dashboard, view the\n [Troubleshooting](/docs/observability/troubleshooting) guide for steps\n to debug.\n\n## Step 5. Re-build and deploy code\n\nRe-build, deploy, and invoke your Genkit feature to start collecting data.\nAfter Genkit Monitoring receives your metrics, you can view them by\nvisiting the\n[Genkit Monitoring dashboard](https://console.firebase.google.com/project/_/genai_monitoring)\n\n:::note\nIt may take up to 5 minutes to collect the first metric (based on the default `metricExportIntervalMillis` setting in the telemetry configuration).\n:::\n", + "text": "# Get started with Genkit Monitoring\n\nThis quickstart guide describes how to set up Genkit Monitoring for\nyour deployed Genkit features, so that you can collect and view real-time\ntelemetry data. With Genkit Monitoring, you get visibility into how\nyour Genkit features are performing in production.\n\nKey capabilities of Genkit Monitoring include:\n\n- Viewing quantitative metrics like Genkit feature latency, errors, and\n token usage.\n- Inspecting traces to see your Genkit's feature steps, inputs, and outputs,\n to help with debugging and quality improvement.\n- Exporting production traces to run evals within Genkit.\n\nSetting up Genkit Monitoring requires completing tasks in both your codebase\nand on the Google Cloud Console.\n\n## Before you begin\n\n1. If you haven't already, create a Firebase project.\n\n In the [Firebase console](https://console.firebase.google.com), click\n **Add a project**, then follow the on-screen instructions. You can\n create a new project or add Firebase services to an already-existing Google Cloud project.\n\n2. Ensure your project is on the\n [Blaze pricing plan](https://firebase.google.com/pricing).\n\n Genkit Monitoring relies on telemetry data written to Google Cloud\n Logging, Metrics, and Trace, which are paid services. View the\n [Google Cloud Observability pricing](https://cloud.google.com/stackdriver/pricing) page for pricing details and to learn about free-of-charge tier limits.\n\n3. Write a Genkit feature by following the [Get Started Guide](/docs/get-started), and prepare your code for deployment by using one of the following guides:\n\n a. [Deploy flows using Cloud Functions for Firebase](/docs/firebase)\n b. [Deploy flows using Cloud Run](/docs/cloud-run)\n c. [Deploy flows to any Node.js platform](/docs/deploy-node)\n\n## Step 1. Add the Firebase plugin\n\nInstall the `@genkit-ai/firebase` plugin in your project:\n\n```bash\nnpm install @genkit-ai/firebase\n```\n\n### Environment-based configuration\n\nIf you intend to use the default configuration for Firebase Genkit\nMonitoring, you can enable telemetry by setting the\n`ENABLE_FIREBASE_MONITORING` environment variable in your deployment\nenvironment.\n\n```bash\nexport ENABLE_FIREBASE_MONITORING=true\n```\n\n:::note\nThis will use default configuration values. To\noverride configuration options, use \"Programmatic configuration\".\n:::\n\n### Programmatic configuration\n\nYou can also enable Firebase Genkit Monitoring in code. This is useful\nif you want to tweak any configuration settings like the metric export\ninterval or to set up your local environment to export telemetry data.\n\nImport `enableFirebaseTelemetry` into your Genkit configuration file (the\nfile where `genkit(...)` is initalized), and call it:\n\n```typescript\nimport { enableFirebaseTelemetry } from '@genkit-ai/firebase';\n\nenableFirebaseTelemetry();\n```\n\n## Step 2. Enable the required APIs\n\nMake sure that the following APIs are enabled for your Google Cloud project:\n\n- [Cloud Logging API](https://console.cloud.google.com/apis/library/logging.googleapis.com)\n- [Cloud Trace API](https://console.cloud.google.com/apis/library/cloudtrace.googleapis.com)\n- [Cloud Monitoring API](https://console.cloud.google.com/apis/library/monitoring.googleapis.com)\n\nThese APIs should be listed in the\n[API dashboard](https://console.cloud.google.com/apis/dashboard) for your\nproject.\n\n## Step 3. Set up permissions\n\nThe Firebase plugin needs to use a _service account_ to authenticate with\nGoogle Cloud Logging, Metrics, and Trace services.\n\nGrant the following roles to whichever service account is configured to run your code within the [Google Cloud IAM Console](https://console.cloud.google.com/iam-admin/iam). For Cloud Functions for Firebase and Cloud Run, that's typically the default compute service account.\n\n- **Monitoring Metric Writer** (`roles/monitoring.metricWriter`)\n- **Cloud Trace Agent** (`roles/cloudtrace.agent`)\n- **Logs Writer** (`roles/logging.logWriter`)\n\n## Step 4. (Optional) Test your configuration locally\n\nBefore deploying, you can run your Genkit code locally to confirm that\ntelemetry data is being collected, and is viewable in the Genkit Monitoring\ndashboard.\n\n1. In your Genkit code, set `forceDevExport` to `true` to send telemetry from\n your local environment.\n\n2. Use your service account to authenticate and test your configuration.\n\n :::tip\n In order to impersonate the service account, you will need to have\n the `roles/iam.serviceAccountTokenCreator`\n [IAM role](https://console.cloud.google.com/iam-admin/iam) applied to your\n user account.\n :::\n\n With the\n [Google Cloud CLI tool](https://cloud.google.com/sdk/docs/install?authuser=0),\n authenticate using the service account:\n\n ```bash\n gcloud auth application-default login --impersonate-service-account SERVICE_ACCT_EMAIL\n ```\n\n3. Run and invoke your Genkit feature, and then view metrics on the\n [Genkit Monitoring dashboard](https://console.firebase.google.com/project/_/genai_monitoring).\n Allow for up to 5 minutes to collect the first metric. You can reduce this\n delay by setting `metricExportIntervalMillis` in the telemetry configuration.\n\n4. If metrics are not appearing in the Genkit Monitoring dashboard, view the\n [Troubleshooting](/docs/observability/troubleshooting) guide for steps\n to debug.\n\n## Step 5. Re-build and deploy code\n\nRe-build, deploy, and invoke your Genkit feature to start collecting data.\nAfter Genkit Monitoring receives your metrics, you can view them by\nvisiting the\n[Genkit Monitoring dashboard](https://console.firebase.google.com/project/_/genai_monitoring)\n\n:::note\nIt may take up to 5 minutes to collect the first metric (based on the default `metricExportIntervalMillis` setting in the telemetry configuration).\n:::\n", "title": "Get started with Genkit Monitoring", - "lang": "js" + "description": "This quickstart guide explains how to set up Genkit Monitoring for your deployed Genkit features to collect and view real-time telemetry data, including metrics, traces, and production trace exports for evaluations.", + "lang": "js", + "headers": "## Before you begin\n## Step 1. Add the Firebase plugin\n### Environment-based configuration\n### Programmatic configuration\n## Step 2. Enable the required APIs\n## Step 3. Set up permissions\n## Step 4. (Optional) Test your configuration locally\n## Step 5. Re-build and deploy code\n" }, "js/observability/telemetry-collection.md": { - "text": "The Firebase telemetry plugin exports a combination of metrics, traces, and\nlogs to Google Cloud Observability. This document details which metrics, trace\nattributes, and logs will be collected and what you can expect in terms of\nlatency, quotas, and cost.\n\n## Telemetry delay\n\nThere may be a slight delay before telemetry from a given invocation is\navailable in Firebase. This is dependent on your export interval (5 minutes\nby default).\n\n## Quotas and limits\n\nThere are several quotas that are important to keep in mind:\n\n- [Cloud Trace Quotas](http://cloud.google.com/trace/docs/quotas)\n- [Cloud Logging Quotas](http://cloud.google.com/logging/quotas)\n- [Cloud Monitoring Quotas](http://cloud.google.com/monitoring/quotas)\n\n## Cost\n\nCloud Logging, Cloud Trace, and Cloud Monitoring have generous free-of-charge\ntiers. Specific pricing can be found at the following links:\n\n- [Cloud Logging Pricing](http://cloud.google.com/stackdriver/pricing#google-cloud-observability-pricing)\n- [Cloud Trace Pricing](https://cloud.google.com/trace#pricing)\n- [Cloud Monitoring Pricing](https://cloud.google.com/stackdriver/pricing#monitoring-pricing-summary)\n\n## Metrics\n\nThe Firebase telemetry plugin collects a number of different metrics to support\nthe various Genkit action types detailed in the following sections.\n\n### Feature metrics\n\nFeatures are the top-level entry-point to your Genkit code. In most cases, this\nwill be a flow. Otherwise, this will be the top-most span in a trace.\n\n| Name | Type | Description |\n| ----------------------- | --------- | ----------------------- |\n| genkit/feature/requests | Counter | Number of requests |\n| genkit/feature/latency | Histogram | Execution latency in ms |\n\nEach feature metric contains the following dimensions:\n\n| Name | Description |\n| ------------- | -------------------------------------------------------------------------------- |\n| name | The name of the feature. In most cases, this is the top-level Genkit flow |\n| status | 'success' or 'failure' depending on whether or not the feature request succeeded |\n| error | Only set when `status=failure`. Contains the error type that caused the failure |\n| source | The Genkit source language. Eg. 'ts' |\n| sourceVersion | The Genkit framework version |\n\n### Action metrics\n\nActions represent a generic step of execution within Genkit. Each of these steps\nwill have the following metrics tracked:\n\n| Name | Type | Description |\n| ---------------------- | --------- | --------------------------------------------- |\n| genkit/action/requests | Counter | Number of times this action has been executed |\n| genkit/action/latency | Histogram | Execution latency in ms |\n\nEach action metric contains the following dimensions:\n\n| Name | Description |\n| ------------- | ---------------------------------------------------------------------------------------------------- |\n| name | The name of the action |\n| featureName | The name of the parent feature being executed |\n| path | The path of execution from the feature root to this action. eg. '/myFeature/parentAction/thisAction' |\n| status | 'success' or 'failure' depending on whether or not the action succeeded |\n| error | Only set when `status=failure`. Contains the error type that caused the failure |\n| source | The Genkit source language. Eg. 'ts' |\n| sourceVersion | The Genkit framework version |\n\n### Generate metrics\n\nThese are special action metrics relating to actions that interact with a model.\nIn addition to requests and latency, input and output are also tracked, with\nmodel specific dimensions that make debugging and configuration tuning easier.\n\n| Name | Type | Description |\n| ------------------------------------ | --------- | ------------------------------------------ |\n| genkit/ai/generate/requests | Counter | Number of times this model has been called |\n| genkit/ai/generate/latency | Histogram | Execution latency in ms |\n| genkit/ai/generate/input/tokens | Counter | Input tokens |\n| genkit/ai/generate/output/tokens | Counter | Output tokens |\n| genkit/ai/generate/input/characters | Counter | Input characters |\n| genkit/ai/generate/output/characters | Counter | Output characters |\n| genkit/ai/generate/input/images | Counter | Input images |\n| genkit/ai/generate/output/images | Counter | Output images |\n| genkit/ai/generate/input/audio | Counter | Input audio files |\n| genkit/ai/generate/output/audio | Counter | Output audio files |\n\nEach generate metric contains the following dimensions:\n\n| Name | Description |\n| ------------- | ---------------------------------------------------------------------------------------------------- |\n| modelName | The name of the model |\n| featureName | The name of the parent feature being executed |\n| path | The path of execution from the feature root to this action. eg. '/myFeature/parentAction/thisAction' |\n| latencyMs | The response time taken by the model |\n| status | 'success' or 'failure' depending on whether or not the feature request succeeded |\n| error | Only set when `status=failure`. Contains the error type that caused the failure |\n| source | The Genkit source language. Eg. 'ts' |\n| sourceVersion | The Genkit framework version |\n\n## Traces\n\nAll Genkit actions are automatically instrumented to provide detailed traces for\nyour AI features. Locally, traces are visible in the Developer UI. For deployed\napps enable Genkit Monitoring to get the same level of visibility.\n\nThe following sections describe what trace attributes you can expect based on\nthe Genkit action type for a particular span in the trace.\n\n### Root Spans\n\nRoot spans have special attributes to help disambiguate the state attributes for\nthe whole trace versus an individual span.\n\n| Attribute name | Description |\n| ---------------- | ----------------------------------------------------------------------------------------------------------------------- |\n| genkit/feature | The name of the parent feature being executed |\n| genkit/isRoot | Marked true if this span is the root span |\n| genkit/rootState | The state of the overall execution as `success` or `error`. This does not indicate that this step failed in particular. |\n\n### Flow\n\n| Attribute name | Description |\n| ----------------------- | ---------------------------------------------------------------------------------------------------------- |\n| genkit/input | The input to the flow. This will always be `` because of trace attribute size limits. |\n| genkit/metadata/subtype | The type of Genkit action. For flows it will be `flow`. |\n| genkit/name | The name of this Genkit action. In this case the name of the flow |\n| genkit/output | The output generated in the flow. This will always be `` because of trace attribute size limits. |\n| genkit/path | The fully qualified execution path that lead to this step in the trace, including type information. |\n| genkit/state | The state of this span's execution as `success` or `error`. |\n| genkit/type | The type of Genkit primitive that corresponds to this span. For flows, this will be `action`. |\n\n### Util\n\n| Attribute name | Description |\n| -------------- | ---------------------------------------------------------------------------------------------------------- |\n| genkit/input | The input to the util. This will always be `` because of trace attribute size limits. |\n| genkit/name | The name of this Genkit action. In this case the name of the flow |\n| genkit/output | The output generated in the util. This will always be `` because of trace attribute size limits. |\n| genkit/path | The fully qualified execution path that lead to this step in the trace, including type information. |\n| genkit/state | The state of this span's execution as `success` or `error`. |\n| genkit/type | The type of Genkit primitive that corresponds to this span. For flows, this will be `util`. |\n\n### Model\n\n| Attribute name | Description |\n| ----------------------- | ----------------------------------------------------------------------------------------------------------- |\n| genkit/input | The input to the model. This will always be `` because of trace attribute size limits. |\n| genkit/metadata/subtype | The type of Genkit action. For models it will be `model`. |\n| genkit/model | The name of the model. |\n| genkit/name | The name of this Genkit action. In this case the name of the model. |\n| genkit/output | The output generated by the model. This will always be `` because of trace attribute size limits. |\n| genkit/path | The fully qualified execution path that lead to this step in the trace, including type information. |\n| genkit/state | The state of this span's execution as `success` or `error`. |\n| genkit/type | The type of Genkit primitive that corresponds to this span. For flows, this will be `action`. |\n\n### Tool\n\n| Attribute name | Description |\n| ----------------------- | ----------------------------------------------------------------------------------------------------------- |\n| genkit/input | The input to the model. This will always be `` because of trace attribute size limits. |\n| genkit/metadata/subtype | The type of Genkit action. For tools it will be `tool`. |\n| genkit/name | The name of this Genkit action. In this case the name of the model. |\n| genkit/output | The output generated by the model. This will always be `` because of trace attribute size limits. |\n| genkit/path | The fully qualified execution path that lead to this step in the trace, including type information. |\n| genkit/state | The state of this span's execution as `success` or `error`. |\n| genkit/type | The type of Genkit primitive that corresponds to this span. For flows, this will be `action`. |\n\n## Logs\n\nFor deployed apps with Genkit Monitoring, logs are used to capture\ninput, output, and configuration metadata that provides rich detail about\neach step in your AI feature.\n\nAll logs will include the following shared metadata fields:\n\n| Field name | Description |\n| ----------------- | --------------------------------------------------------------------------------------------------------------------------------- |\n| insertId | Unique id for the log entry |\n| jsonPayload | Container for variable information that is unique to each log type |\n| labels | `{module: genkit}` |\n| logName | `projects/weather-gen-test-next/logs/genkit_log` |\n| receivedTimestamp | Time the log was received by Cloud |\n| resource | Information about the source of the log including deployment information region, and projectId |\n| severity | The log level written. See Cloud's [LogSeverity](https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry#logseverity) |\n| spanId | Identifier for the span that created this log |\n| timestamp | Time that the client logged a message |\n| trace | Identifier for the trace of the format `projects//traces/` |\n| traceSampled | Boolean representing whether the trace was sampled. Logs are not sampled. |\n\nEach log type will have a different json payload described in each section.\n\n### Input\n\nJSON payload:\n\n| Field name | Description |\n| ---------- | -------------------------------------------------------------------------------------------- |\n| message | `[genkit] Input[, ]` including `(message X of N)` for multi-part messages |\n| metadata | Additional context including the input message sent to the action |\n\nMetadata:\n\n| Field name | Description |\n| ---------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- |\n| content | The input message content sent to this Genkit action |\n| featureName | The name of the Genkit flow, action, tool, util, or helper. |\n| messageIndex \\* | Index indicating the order of messages for inputs that contain multiple messages. For single messages, this will always be 0. |\n| model \\* | Model name. |\n| path | The execution path that generated this log of the format `step1 > step2 > step3` |\n| partIndex \\* | Index indicating the order of parts within a message for multi-part messages. This is typical when combining text and images in a single input. |\n| qualifiedPath | The execution path that generated this log, including type information of the format: `/{flow1,t:flow}/{generate,t:util}/{modelProvider/model,t:action,s:model` |\n| totalMessages \\* | The total number of messages for this input. For single messages, this will always be 1. |\n| totalParts \\* | Total number of parts for this message. For single-part messages, this will always be 1. |\n\n(\\*) Starred items are only present on Input logs for model interactions.\n\n### Output\n\nJSON payload:\n\n| Field name | Description |\n| ---------- | --------------------------------------------------------------------------------------------- |\n| message | `[genkit] Output[, ]` including `(message X of N)` for multi-part messages |\n| metadata | Additional context including the input message sent to the action |\n\nMetadata:\n\n| Field name | Description |\n| ------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- |\n| candidateIndex \\* (deprecated) | Index indicating the order of candidates for outputs that contain multiple candidates. For logs with single candidates, this will always be 0. |\n| content | The output message generated by the Genkit action |\n| featureName | The name of the Genkit flow, action, tool, util, or helper. |\n| messageIndex \\* | Index indicating the order of messages for inputs that contain multiple messages. For single messages, this will always be 0. |\n| model \\* | Model name. |\n| path | The execution path that generated this log of the format `step1 > step2 > step3 |\n| partIndex \\* | Index indicating the order of parts within a message for multi-part messages. This is typical when combining text and images in a single output. |\n| qualifiedPath | The execution path that generated this log, including type information of the format: `/{flow1,t:flow}/{generate,t:util}/{modelProvider/model,t:action,s:model` |\n| totalCandidates \\* (deprecated) | Total number of candidates generated as output. For single-candidate messages, this will always be 1. |\n| totalParts \\* | Total number of parts for this message. For single-part messages, this will always be 1. |\n\n(\\*) Starred items are only present on Output logs for model interactions.\n\n### Config\n\nJSON payload:\n\n| Field name | Description |\n| ---------- | ----------------------------------------------------------------- |\n| message | `[genkit] Config[, ]` |\n| metadata | Additional context including the input message sent to the action |\n\nMetadata:\n\n| Field name | Description |\n| ------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- |\n| featureName | The name of the Genkit flow, action, tool, util, or helper. |\n| model | Model name. |\n| path | The execution path that generated this log of the format `step1 > step2 > step3 |\n| qualifiedPath | The execution path that generated this log, including type information of the format: `/{flow1,t:flow}/{generate,t:util}/{modelProvider/model,t:action,s:model` |\n| source | The Genkit library language used. This will always be set to 'ts' as it is the only supported language. |\n| sourceVersion | The Genkit library version. |\n| temperature | Model temperature used. |\n\n### Paths\n\nJSON payload:\n\n| Field name | Description |\n| ---------- | ----------------------------------------------------------------- |\n| message | `[genkit] Paths[, ]` |\n| metadata | Additional context including the input message sent to the action |\n\nMetadata:\n\n| Field name | Description |\n| ---------- | ---------------------------------------------------------------- |\n| flowName | The name of the Genkit flow, action, tool, util, or helper. |\n| paths | An array containing all execution paths for the collected spans. |\n", + "text": "# Telemetry Collection\n\nThe Firebase telemetry plugin exports a combination of metrics, traces, and\nlogs to Google Cloud Observability. This document details which metrics, trace\nattributes, and logs will be collected and what you can expect in terms of\nlatency, quotas, and cost.\n\n## Telemetry delay\n\nThere may be a slight delay before telemetry from a given invocation is\navailable in Firebase. This is dependent on your export interval (5 minutes\nby default).\n\n## Quotas and limits\n\nThere are several quotas that are important to keep in mind:\n\n- [Cloud Trace Quotas](http://cloud.google.com/trace/docs/quotas)\n- [Cloud Logging Quotas](http://cloud.google.com/logging/quotas)\n- [Cloud Monitoring Quotas](http://cloud.google.com/monitoring/quotas)\n\n## Cost\n\nCloud Logging, Cloud Trace, and Cloud Monitoring have generous free-of-charge\ntiers. Specific pricing can be found at the following links:\n\n- [Cloud Logging Pricing](http://cloud.google.com/stackdriver/pricing#google-cloud-observability-pricing)\n- [Cloud Trace Pricing](https://cloud.google.com/trace#pricing)\n- [Cloud Monitoring Pricing](https://cloud.google.com/stackdriver/pricing#monitoring-pricing-summary)\n\n## Metrics\n\nThe Firebase telemetry plugin collects a number of different metrics to support\nthe various Genkit action types detailed in the following sections.\n\n### Feature metrics\n\nFeatures are the top-level entry-point to your Genkit code. In most cases, this\nwill be a flow. Otherwise, this will be the top-most span in a trace.\n\n| Name | Type | Description |\n| ----------------------- | --------- | ----------------------- |\n| genkit/feature/requests | Counter | Number of requests |\n| genkit/feature/latency | Histogram | Execution latency in ms |\n\nEach feature metric contains the following dimensions:\n\n| Name | Description |\n| ------------- | -------------------------------------------------------------------------------- |\n| name | The name of the feature. In most cases, this is the top-level Genkit flow |\n| status | 'success' or 'failure' depending on whether or not the feature request succeeded |\n| error | Only set when `status=failure`. Contains the error type that caused the failure |\n| source | The Genkit source language. Eg. 'ts' |\n| sourceVersion | The Genkit framework version |\n\n### Action metrics\n\nActions represent a generic step of execution within Genkit. Each of these steps\nwill have the following metrics tracked:\n\n| Name | Type | Description |\n| ---------------------- | --------- | --------------------------------------------- |\n| genkit/action/requests | Counter | Number of times this action has been executed |\n| genkit/action/latency | Histogram | Execution latency in ms |\n\nEach action metric contains the following dimensions:\n\n| Name | Description |\n| ------------- | ---------------------------------------------------------------------------------------------------- |\n| name | The name of the action |\n| featureName | The name of the parent feature being executed |\n| path | The path of execution from the feature root to this action. eg. '/myFeature/parentAction/thisAction' |\n| status | 'success' or 'failure' depending on whether or not the action succeeded |\n| error | Only set when `status=failure`. Contains the error type that caused the failure |\n| source | The Genkit source language. Eg. 'ts' |\n| sourceVersion | The Genkit framework version |\n\n### Generate metrics\n\nThese are special action metrics relating to actions that interact with a model.\nIn addition to requests and latency, input and output are also tracked, with\nmodel specific dimensions that make debugging and configuration tuning easier.\n\n| Name | Type | Description |\n| ------------------------------------ | --------- | ------------------------------------------ |\n| genkit/ai/generate/requests | Counter | Number of times this model has been called |\n| genkit/ai/generate/latency | Histogram | Execution latency in ms |\n| genkit/ai/generate/input/tokens | Counter | Input tokens |\n| genkit/ai/generate/output/tokens | Counter | Output tokens |\n| genkit/ai/generate/input/characters | Counter | Input characters |\n| genkit/ai/generate/output/characters | Counter | Output characters |\n| genkit/ai/generate/input/images | Counter | Input images |\n| genkit/ai/generate/output/images | Counter | Output images |\n| genkit/ai/generate/input/audio | Counter | Input audio files |\n| genkit/ai/generate/output/audio | Counter | Output audio files |\n\nEach generate metric contains the following dimensions:\n\n| Name | Description |\n| ------------- | ---------------------------------------------------------------------------------------------------- |\n| modelName | The name of the model |\n| featureName | The name of the parent feature being executed |\n| path | The path of execution from the feature root to this action. eg. '/myFeature/parentAction/thisAction' |\n| latencyMs | The response time taken by the model |\n| status | 'success' or 'failure' depending on whether or not the feature request succeeded |\n| error | Only set when `status=failure`. Contains the error type that caused the failure |\n| source | The Genkit source language. Eg. 'ts' |\n| sourceVersion | The Genkit framework version |\n\n## Traces\n\nAll Genkit actions are automatically instrumented to provide detailed traces for\nyour AI features. Locally, traces are visible in the Developer UI. For deployed\napps enable Genkit Monitoring to get the same level of visibility.\n\nThe following sections describe what trace attributes you can expect based on\nthe Genkit action type for a particular span in the trace.\n\n### Root Spans\n\nRoot spans have special attributes to help disambiguate the state attributes for\nthe whole trace versus an individual span.\n\n| Attribute name | Description |\n| ---------------- | ----------------------------------------------------------------------------------------------------------------------- |\n| genkit/feature | The name of the parent feature being executed |\n| genkit/isRoot | Marked true if this span is the root span |\n| genkit/rootState | The state of the overall execution as `success` or `error`. This does not indicate that this step failed in particular. |\n\n### Flow\n\n| Attribute name | Description |\n| ----------------------- | ---------------------------------------------------------------------------------------------------------- |\n| genkit/input | The input to the flow. This will always be `` because of trace attribute size limits. |\n| genkit/metadata/subtype | The type of Genkit action. For flows it will be `flow`. |\n| genkit/name | The name of this Genkit action. In this case the name of the flow |\n| genkit/output | The output generated in the flow. This will always be `` because of trace attribute size limits. |\n| genkit/path | The fully qualified execution path that lead to this step in the trace, including type information. |\n| genkit/state | The state of this span's execution as `success` or `error`. |\n| genkit/type | The type of Genkit primitive that corresponds to this span. For flows, this will be `action`. |\n\n### Util\n\n| Attribute name | Description |\n| -------------- | ---------------------------------------------------------------------------------------------------------- |\n| genkit/input | The input to the util. This will always be `` because of trace attribute size limits. |\n| genkit/name | The name of this Genkit action. In this case the name of the flow |\n| genkit/output | The output generated in the util. This will always be `` because of trace attribute size limits. |\n| genkit/path | The fully qualified execution path that lead to this step in the trace, including type information. |\n| genkit/state | The state of this span's execution as `success` or `error`. |\n| genkit/type | The type of Genkit primitive that corresponds to this span. For flows, this will be `util`. |\n\n### Model\n\n| Attribute name | Description |\n| ----------------------- | ----------------------------------------------------------------------------------------------------------- |\n| genkit/input | The input to the model. This will always be `` because of trace attribute size limits. |\n| genkit/metadata/subtype | The type of Genkit action. For models it will be `model`. |\n| genkit/model | The name of the model. |\n| genkit/name | The name of this Genkit action. In this case the name of the model. |\n| genkit/output | The output generated by the model. This will always be `` because of trace attribute size limits. |\n| genkit/path | The fully qualified execution path that lead to this step in the trace, including type information. |\n| genkit/state | The state of this span's execution as `success` or `error`. |\n| genkit/type | The type of Genkit primitive that corresponds to this span. For flows, this will be `action`. |\n\n### Tool\n\n| Attribute name | Description |\n| ----------------------- | ----------------------------------------------------------------------------------------------------------- |\n| genkit/input | The input to the model. This will always be `` because of trace attribute size limits. |\n| genkit/metadata/subtype | The type of Genkit action. For tools it will be `tool`. |\n| genkit/name | The name of this Genkit action. In this case the name of the model. |\n| genkit/output | The output generated by the model. This will always be `` because of trace attribute size limits. |\n| genkit/path | The fully qualified execution path that lead to this step in the trace, including type information. |\n| genkit/state | The state of this span's execution as `success` or `error`. |\n| genkit/type | The type of Genkit primitive that corresponds to this span. For flows, this will be `action`. |\n\n## Logs\n\nFor deployed apps with Genkit Monitoring, logs are used to capture\ninput, output, and configuration metadata that provides rich detail about\neach step in your AI feature.\n\nAll logs will include the following shared metadata fields:\n\n| Field name | Description |\n| ----------------- | --------------------------------------------------------------------------------------------------------------------------------- |\n| insertId | Unique id for the log entry |\n| jsonPayload | Container for variable information that is unique to each log type |\n| labels | `{module: genkit}` |\n| logName | `projects/weather-gen-test-next/logs/genkit_log` |\n| receivedTimestamp | Time the log was received by Cloud |\n| resource | Information about the source of the log including deployment information region, and projectId |\n| severity | The log level written. See Cloud's [LogSeverity](https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry#logseverity) |\n| spanId | Identifier for the span that created this log |\n| timestamp | Time that the client logged a message |\n| trace | Identifier for the trace of the format `projects//traces/` |\n| traceSampled | Boolean representing whether the trace was sampled. Logs are not sampled. |\n\nEach log type will have a different json payload described in each section.\n\n### Input\n\nJSON payload:\n\n| Field name | Description |\n| ---------- | -------------------------------------------------------------------------------------------- |\n| message | `[genkit] Input[, ]` including `(message X of N)` for multi-part messages |\n| metadata | Additional context including the input message sent to the action |\n\nMetadata:\n\n| Field name | Description |\n| ---------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- |\n| content | The input message content sent to this Genkit action |\n| featureName | The name of the Genkit flow, action, tool, util, or helper. |\n| messageIndex \\* | Index indicating the order of messages for inputs that contain multiple messages. For single messages, this will always be 0. |\n| model \\* | Model name. |\n| path | The execution path that generated this log of the format `step1 > step2 > step3` |\n| partIndex \\* | Index indicating the order of parts within a message for multi-part messages. This is typical when combining text and images in a single input. |\n| qualifiedPath | The execution path that generated this log, including type information of the format: `/{flow1,t:flow}/{generate,t:util}/{modelProvider/model,t:action,s:model` |\n| totalMessages \\* | The total number of messages for this input. For single messages, this will always be 1. |\n| totalParts \\* | Total number of parts for this message. For single-part messages, this will always be 1. |\n\n(\\*) Starred items are only present on Input logs for model interactions.\n\n### Output\n\nJSON payload:\n\n| Field name | Description |\n| ---------- | --------------------------------------------------------------------------------------------- |\n| message | `[genkit] Output[, ]` including `(message X of N)` for multi-part messages |\n| metadata | Additional context including the input message sent to the action |\n\nMetadata:\n\n| Field name | Description |\n| ------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- |\n| candidateIndex \\* (deprecated) | Index indicating the order of candidates for outputs that contain multiple candidates. For logs with single candidates, this will always be 0. |\n| content | The output message generated by the Genkit action |\n| featureName | The name of the Genkit flow, action, tool, util, or helper. |\n| messageIndex \\* | Index indicating the order of messages for inputs that contain multiple messages. For single messages, this will always be 0. |\n| model \\* | Model name. |\n| path | The execution path that generated this log of the format `step1 > step2 > step3 |\n| partIndex \\* | Index indicating the order of parts within a message for multi-part messages. This is typical when combining text and images in a single output. |\n| qualifiedPath | The execution path that generated this log, including type information of the format: `/{flow1,t:flow}/{generate,t:util}/{modelProvider/model,t:action,s:model` |\n| totalCandidates \\* (deprecated) | Total number of candidates generated as output. For single-candidate messages, this will always be 1. |\n| totalParts \\* | Total number of parts for this message. For single-part messages, this will always be 1. |\n\n(\\*) Starred items are only present on Output logs for model interactions.\n\n### Config\n\nJSON payload:\n\n| Field name | Description |\n| ---------- | ----------------------------------------------------------------- |\n| message | `[genkit] Config[, ]` |\n| metadata | Additional context including the input message sent to the action |\n\nMetadata:\n\n| Field name | Description |\n| ------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- |\n| featureName | The name of the Genkit flow, action, tool, util, or helper. |\n| model | Model name. |\n| path | The execution path that generated this log of the format `step1 > step2 > step3 |\n| qualifiedPath | The execution path that generated this log, including type information of the format: `/{flow1,t:flow}/{generate,t:util}/{modelProvider/model,t:action,s:model` |\n| source | The Genkit library language used. This will always be set to 'ts' as it is the only supported language. |\n| sourceVersion | The Genkit library version. |\n| temperature | Model temperature used. |\n\n### Paths\n\nJSON payload:\n\n| Field name | Description |\n| ---------- | ----------------------------------------------------------------- |\n| message | `[genkit] Paths[, ]` |\n| metadata | Additional context including the input message sent to the action |\n\nMetadata:\n\n| Field name | Description |\n| ---------- | ---------------------------------------------------------------- |\n| flowName | The name of the Genkit flow, action, tool, util, or helper. |\n| paths | An array containing all execution paths for the collected spans. |\n", "title": "Telemetry Collection", - "lang": "js" + "description": "This document details the metrics, trace attributes, and logs collected by the Firebase telemetry plugin for Genkit, along with information on latency, quotas, and cost.", + "lang": "js", + "headers": "## Telemetry delay\n## Quotas and limits\n## Cost\n## Metrics\n### Feature metrics\n### Action metrics\n### Generate metrics\n## Traces\n### Root Spans\n### Flow\n### Util\n### Model\n### Tool\n## Logs\n### Input\n### Output\n### Config\n### Paths\n" }, "js/observability/troubleshooting.md": { - "text": "The following sections detail solutions to common issues that developers run\ninto when using Genkit Monitoring.\n\n## I can't see traces or metrics in Genkit Monitoring\n\n1. Ensure that the following APIs are enabled for your underlying Google Cloud project:\n - [Cloud Logging API](https://console.cloud.google.com/apis/library/logging.googleapis.com)\n - [Cloud Trace API](https://console.cloud.google.com/apis/library/cloudtrace.googleapis.com)\n - [Cloud Monitoring API](https://console.cloud.google.com/apis/library/monitoring.googleapis.com)\n2. Ensure that the following roles are applied to the service account that is running your code (or service account that has been configured as part of the plugin options) in [Cloud IAM](https://console.cloud.google.com/iam-admin/iam).\n - **Monitoring Metric Writer** (`roles/monitoring.metricWriter`)\n - **Cloud Trace Agent** (`roles/cloudtrace.agent`)\n - **Logs Writer** (`roles/logging.logWriter`)\n3. Inspect the application logs for errors writing to Cloud Logging, Cloud Trace, and Cloud Monitoring. On Google Cloud infrastructure such as Firebase Functions and Cloud Run, even when telemetry is misconfigured, logs to `stdout/stderr` are automatically ingested by the Cloud Logging Agent, allowing you to diagnose issues in the in the [Cloud Logging Console](https://console.cloud.google.com/logs).\n\n4. Debug locally:\n\n Enable dev export:\n\n ```typescript\n enableFirebaseTelemetry({\n forceDevExport: true,\n });\n ```\n\n To test with your personal user credentials, use the\n [gcloud CLI](https://cloud.google.com/sdk/docs/install) to authenticate with\n Google Cloud. Doing so can help diagnose enabled or disabled APIs, but does\n not test the gcloud auth application-default login.\n\n Alternatively, impersonating the service account lets you test\n production-like access. You must have the\n `roles/iam. serviceAccountTokenCreator` IAM role applied to your user account\n in order to impersonate service accounts:\n\n ```bash\n gcloud auth application-default login --impersonate-service-account \n ```\n\n See the\n [ADC](https://cloud.google.com/docs/authentication/set-up-adc-local-dev-environment)\n documentation for more information.\n\n### Telemetry upload reliability in Firebase Functions / Cloud Run\n\nWhen Genkit is hosted in Google Cloud Run (including Cloud Functions for\nFirebase), telemetry-data upload may be less reliable as the container switches\nto the \"idle\"\n[lifecycle state](https://cloud.google.com/blog/topics/developers-practitioners/lifecycle-container-cloud-run).\nIf higher reliability is important to you, consider changing\n[CPU allocation](https://cloud.google.com/run/docs/configuring/cpu-allocation)\nto **always allocated** in the Google Cloud Console.\n\n:::note\nThe **always allocated** setting impacts pricing.\n:::\n", + "text": "# Genkit Monitoring - Troubleshooting\n\nThe following sections detail solutions to common issues that developers run\ninto when using Genkit Monitoring.\n\n## I can't see traces or metrics in Genkit Monitoring\n\n1. Ensure that the following APIs are enabled for your underlying Google Cloud project:\n - [Cloud Logging API](https://console.cloud.google.com/apis/library/logging.googleapis.com)\n - [Cloud Trace API](https://console.cloud.google.com/apis/library/cloudtrace.googleapis.com)\n - [Cloud Monitoring API](https://console.cloud.google.com/apis/library/monitoring.googleapis.com)\n2. Ensure that the following roles are applied to the service account that is running your code (or service account that has been configured as part of the plugin options) in [Cloud IAM](https://console.cloud.google.com/iam-admin/iam).\n - **Monitoring Metric Writer** (`roles/monitoring.metricWriter`)\n - **Cloud Trace Agent** (`roles/cloudtrace.agent`)\n - **Logs Writer** (`roles/logging.logWriter`)\n3. Inspect the application logs for errors writing to Cloud Logging, Cloud Trace, and Cloud Monitoring. On Google Cloud infrastructure such as Firebase Functions and Cloud Run, even when telemetry is misconfigured, logs to `stdout/stderr` are automatically ingested by the Cloud Logging Agent, allowing you to diagnose issues in the in the [Cloud Logging Console](https://console.cloud.google.com/logs).\n\n4. Debug locally:\n\n Enable dev export:\n\n ```typescript\n enableFirebaseTelemetry({\n forceDevExport: true,\n });\n ```\n\n To test with your personal user credentials, use the\n [gcloud CLI](https://cloud.google.com/sdk/docs/install) to authenticate with\n Google Cloud. Doing so can help diagnose enabled or disabled APIs, but does\n not test the gcloud auth application-default login.\n\n Alternatively, impersonating the service account lets you test\n production-like access. You must have the\n `roles/iam. serviceAccountTokenCreator` IAM role applied to your user account\n in order to impersonate service accounts:\n\n ```bash\n gcloud auth application-default login --impersonate-service-account \n ```\n\n See the\n [ADC](https://cloud.google.com/docs/authentication/set-up-adc-local-dev-environment)\n documentation for more information.\n\n## Request count does not match traces count\n\nAt low volumes (<1 query per second), you may notice that your metric counts,\nlike requests or failed paths, do not match the number of traces shown in the\ntraces table. Below are three common reasons for this happening.\n\n### Metric and trace export intervals can be different\n\nIn some cases, the dashboard shows traces that have exported but metrics that\nhave not, or vice versa.\n\nYou can reduce the likelihood of this happening by adjusting the metric export\ninterval to be more frequent. By default, metrics are exported every 5 minutes.\nThe minimum allowable export interval is 5 seconds.\n\n:::note\nExporting metrics more frequently can result in increased costs.\n:::\n\n```typescript\nenableFirebaseTelemetry({\n // Override the export interval to 3 minutes\n metricExportIntervalMillis: 180_000,\n // Override the export timeout to 3 minutes\n metricExportTimeoutMillis: 180_000,\n});\n```\n\n### Intermittent network issues\n\nOccasionally you may have transient network issues that result in a failure to\nupload telemetry data. These failures are logged to Google Cloud Logging. To\nsee the specific failure reason, look for a log that starts with --\n\n> Unable to send telemetry to Google Cloud: Error: Send TimeSeries failed:\n\n### Telemetry upload reliability in Firebase Functions or Cloud Run\n\nWhen your Genkit codei is hosted in Google Cloud Run or Cloud Functions for\nFirebase, telemetry-data upload may be less reliable as the container switches\nto the \"idle\"\n[lifecycle state](https://cloud.google.com/blog/topics/developers-practitioners/lifecycle-container-cloud-run).\nIf higher reliability is important to you, consider changing\n[CPU allocation](https://cloud.google.com/run/docs/configuring/cpu-allocation)\nto **Instance-based billing** (previously called **CPU always allocated**) in the Google Cloud Console.\n\n:::note\nThe **Instance-based billing** setting impacts pricing. Check [Cloud Run pricing](https://cloud.google.com/run/pricing) before enabling this setting.\n:::\n\nTo switch to instance-based billing, run\n\n```bash\ngcloud run services update YOUR-SERVICE --no-cpu-throttling\n```\n", "title": "Genkit Monitoring - Troubleshooting", - "lang": "js" + "description": "This guide provides solutions to common issues encountered when using Genkit Monitoring, including problems with traces, metrics, and telemetry export.", + "lang": "js", + "headers": "## I can't see traces or metrics in Genkit Monitoring\n## Request count does not match traces count\n### Metric and trace export intervals can be different\n### Intermittent network issues\n### Telemetry upload reliability in Firebase Functions or Cloud Run\n" }, "js/errors/no-new-actions-at-runtime.md": { - "text": "Defining new actions at runtime is not allowed.\n\n✅ DO:\n\n```ts\nconst prompt = defineDotprompt({...})\n\nconst flow = defineFlow({...}, async (input) => {\n await prompt.generate(...);\n})\n```\n\n❌ DON'T:\n\n```ts\nconst flow = defineFlow({...}, async (input) => {\n const prompt = defineDotprompt({...})\n prompt.generate(...);\n})\n```\n", + "text": "# No new actions at runtime error\n\nDefining new actions at runtime is not allowed.\n\n✅ DO:\n\n```ts\nconst prompt = ai.definePrompt({...})\n\nconst myFlow = ai.defineFlow({...}, async (input) => {\n await prompt(...);\n})\n```\n\n❌ DON'T:\n\n```ts\nconst myFlow = ai.defineFlow({...}, async (input) => {\n const prompt = ai.definePrompt({...})\n await prompt(...);\n})\n```\n", "title": "No new actions at runtime error", - "lang": "js" + "description": "Learn why defining new actions at runtime is not allowed in Genkit and how to correctly define them.", + "lang": "js", + "headers": "" }, "js/errors/types.md": { - "text": "Genkit knows about two specialized types: `GenkitError` and `UserFacingError`.\n`GenkitError` is intended for use by Genkit itself or Genkit plugins.\n`UserFacingError` is intended for [`ContextProviders`](/docs/deploy-node) and\nyour code. The separation between these two error types helps you better understand\nwhere your error is coming from.\n\nGenkit plugins for web hosting (e.g. [`@genkit-ai/express`](https://js.api.genkit.dev/modules/_genkit-ai_express.html) or [`@genkit-ai/next`](https://js.api.genkit.dev/modules/_genkit-ai_next.html))\nSHOULD capture all other Error types and instead report them as an internal error in the response.\nThis adds a layer of security to your application by ensuring that internal details of your application\ndo not leak to attackers.\n", + "text": "# Error Types\n\nGenkit knows about two specialized types: `GenkitError` and `UserFacingError`.\n`GenkitError` is intended for use by Genkit itself or Genkit plugins.\n`UserFacingError` is intended for [`ContextProviders`](/docs/deploy-node) and\nyour code. The separation between these two error types helps you better understand\nwhere your error is coming from.\n\nGenkit plugins for web hosting (e.g. [`@genkit-ai/express`](https://js.api.genkit.dev/modules/_genkit-ai_express.html) or [`@genkit-ai/next`](https://js.api.genkit.dev/modules/_genkit-ai_next.html))\nSHOULD capture all other Error types and instead report them as an internal error in the response.\nThis adds a layer of security to your application by ensuring that internal details of your application\ndo not leak to attackers.\n", "title": "Error Types", - "lang": "js" + "description": "Learn about Genkit's specialized error types, GenkitError and UserFacingError, and how they are used to differentiate between internal and user-facing issues.", + "lang": "js", + "headers": "" }, "go/cloud-run.md": { - "text": "You can deploy Genkit flows as web services using Cloud Run. This page,\nas an example, walks you through the process of deploying the default sample\nflow.\n\n1. Install the [Google Cloud CLI](https://cloud.google.com/sdk/docs/install) if\n you haven't already.\n\n2. Create a new Google Cloud project using the\n [Cloud console](https://console.cloud.google.com) or choose an existing one.\n The project must be linked to a billing account.\n\n After you create or choose a project, configure the Google Cloud CLI to use\n it:\n\n ```bash\n gcloud auth login\n\n gcloud init\n ```\n\n3. Create a directory for the Genkit sample project:\n\n ```bash\n mkdir -p ~/tmp/genkit-cloud-project\n\n cd ~/tmp/genkit-cloud-project\n ```\n\n If you're going to use an IDE, open it to this directory.\n\n4. Initialize a Go module in your project directory:\n\n ```bash\n go mod init example/cloudrun\n\n go mod get github.com/firebase/genkit/go\n ```\n\n5. Create a sample app using Genkit:\n\n ```go\n package main\n\n import (\n \"context\"\n \"fmt\"\n \"log\"\n \"net/http\"\n \"os\"\n\n \"github.com/firebase/genkit/go/ai\"\n \"github.com/firebase/genkit/go/genkit\"\n \"github.com/firebase/genkit/go/plugins/googlegenai\"\n \"github.com/firebase/genkit/go/plugins/server\"\n )\n\n func main() {\n ctx := context.Background()\n\n // Initialize Genkit with the Google AI plugin and Gemini 2.0 Flash.\n // Alternatively, use &googlegenai.VertexAI{} and \"vertexai/gemini-2.5-flash\"\n // to use Vertex AI as the provider instead.\n g, err := genkit.Init(ctx,\n genkit.WithPlugins(&googlegenai.GoogleAI{}),\n genkit.WithDefaultModel(\"googleai/gemini-2.5-flash\"),\n )\n if err != nil {\n log.Fatalf(\"failed to initialize Genkit: %w\", err)\n }\n\n flow := genkit.DefineFlow(g, \"jokesFlow\", func(ctx context.Context, topic string) (string, error) {\n resp, err := genkit.Generate(ctx, g,\n ai.WithPrompt(`Tell a short joke about %s. Be creative!`, topic),\n )\n if err != nil {\n return \"\", fmt.Errorf(\"failed to generate joke: %w\", err)\n }\n\n return resp.Text(), nil\n })\n\n mux := http.NewServeMux()\n mux.HandleFunc(\"POST /jokesFlow\", genkit.Handler(flow))\n log.Fatal(server.Start(ctx, \"127.0.0.1:\"+os.Getenv(\"PORT\"), mux))\n }\n ```\n\n6. Make API credentials available to your deployed function. Choose which\n credentials you need based on your choice in the sample above:\n\n
\n Gemini (Google AI)\n\n 1. Make sure Google AI is\n [available in your region](https://ai.google.dev/available_regions).\n\n 2. [Generate an API key](https://aistudio.google.com/app/apikey) for the\n Gemini API using Google AI Studio.\n\n 3. Make the API key available in the Cloud Run environment:\n\n 1. In the Cloud console, enable the\n [Secret Manager API](https://console.cloud.google.com/apis/library/secretmanager.googleapis.com?project=_).\n 2. On the\n [Secret Manager](https://console.cloud.google.com/security/secret-manager?project=_)\n page, create a new secret containing your API key.\n 3. After you create the secret, on the same page, grant your default\n compute service account access to the secret with the\n **Secret Manager Secret Accessor** role. (You can look up the name\n of the default compute service account on the IAM page.)\n\n In a later step, when you deploy your service, you will need to\n reference the name of this secret.\n\n
\n\n
\n Gemini (Vertex AI)\n\n 1. In the Cloud console,\n [Enable the Vertex AI API](https://console.cloud.google.com/apis/library/aiplatform.googleapis.com?project=_)\n for your project.\n\n 2. On the [IAM](https://console.cloud.google.com/iam-admin/iam?project=_)\n page, ensure that the **Default compute service account** is granted\n the **Vertex AI User** role.\n\n
\n\n The only secret you need to set up for this tutorial is for the model\n provider, but in general, you must do something similar for each service\n your flow uses.\n\n7. **Optional**: Try your flow in the developer UI:\n\n 1. Set up your local environment for the model provider you chose:\n\n
\n Gemini (Google AI)\n\n ```bash\n export GEMINI_API_KEY=\n ```\n\n
\n\n
\n Gemini (Vertex AI)\n\n ```bash\n export GOOGLE_CLOUD_PROJECT=\n\n export GOOGLE_CLOUD_LOCATION=us-central1\n\n gcloud auth application-default login\n ```\n\n
\n\n 2. Start the UI:\n\n ```bash\n genkit start -- go run .\n ```\n\n 3. In the developer UI (`http://localhost:4000/`), run the flow:\n\n 1. Click **jokesFlow**.\n\n 2. On the **Input JSON** tab, provide a subject for the model:\n\n ```json\n \"bananas\"\n ```\n\n 3. Click **Run**.\n\n8. If everything's working as expected so far, you can build and deploy the\n flow:\n\n
\n Gemini (Google AI)\n\n ```bash\n gcloud run deploy --port 3400 \\\n --update-secrets=GEMINI_API_KEY=:latest\n ```\n\n
\n\n
\n Gemini (Vertex AI)\n\n ```bash\n gcloud run deploy --port 3400 \\\n --set-env-vars GOOGLE_CLOUD_PROJECT= \\\n --set-env-vars GOOGLE_CLOUD_LOCATION=us-central1\n ```\n\n (`GOOGLE_CLOUD_LOCATION` configures the Vertex API region you want to\n use.)\n\n
\n\n Choose `N` when asked if you want to allow unauthenticated invocations.\n Answering `N` will configure your service to require IAM credentials. See\n [Authentication](https://cloud.google.com/run/docs/authenticating/overview)\n in the Cloud Run docs for information on providing these credentials.\n\nAfter deployment finishes, the tool will print the service URL. You can test\nit with `curl`:\n\n```bash\ncurl -X POST https:///menuSuggestionFlow \\\n -H \"Authorization: Bearer $(gcloud auth print-identity-token)\" \\\n -H \"Content-Type: application/json\" -d '\"bananas\"'\n```\n", + "text": "# Genkit with Cloud Run\n\nYou can deploy Genkit flows as web services using Cloud Run. This page,\nas an example, walks you through the process of deploying the default sample\nflow.\n\n1. Install the [Google Cloud CLI](https://cloud.google.com/sdk/docs/install) if\n you haven't already.\n\n2. Create a new Google Cloud project using the\n [Cloud console](https://console.cloud.google.com) or choose an existing one.\n The project must be linked to a billing account.\n\n After you create or choose a project, configure the Google Cloud CLI to use\n it:\n\n ```bash\n gcloud auth login\n\n gcloud init\n ```\n\n3. Create a directory for the Genkit sample project:\n\n ```bash\n mkdir -p ~/tmp/genkit-cloud-project\n\n cd ~/tmp/genkit-cloud-project\n ```\n\n If you're going to use an IDE, open it to this directory.\n\n4. Initialize a Go module in your project directory:\n\n ```bash\n go mod init example/cloudrun\n\n go mod get github.com/firebase/genkit/go\n ```\n\n5. Create a sample app using Genkit:\n\n ```go\n package main\n\n import (\n \"context\"\n \"fmt\"\n \"log\"\n \"net/http\"\n \"os\"\n\n \"github.com/firebase/genkit/go/ai\"\n \"github.com/firebase/genkit/go/genkit\"\n \"github.com/firebase/genkit/go/plugins/googlegenai\"\n \"github.com/firebase/genkit/go/plugins/server\"\n )\n\n func main() {\n ctx := context.Background()\n\n // Initialize Genkit with the Google AI plugin and Gemini 2.0 Flash.\n // Alternatively, use &googlegenai.VertexAI{} and \"vertexai/gemini-2.5-flash\"\n // to use Vertex AI as the provider instead.\n g, err := genkit.Init(ctx,\n genkit.WithPlugins(&googlegenai.GoogleAI{}),\n genkit.WithDefaultModel(\"googleai/gemini-2.5-flash\"),\n )\n if err != nil {\n log.Fatalf(\"failed to initialize Genkit: %w\", err)\n }\n\n flow := genkit.DefineFlow(g, \"jokesFlow\", func(ctx context.Context, topic string) (string, error) {\n resp, err := genkit.Generate(ctx, g,\n ai.WithPrompt(`Tell a short joke about %s. Be creative!`, topic),\n )\n if err != nil {\n return \"\", fmt.Errorf(\"failed to generate joke: %w\", err)\n }\n\n return resp.Text(), nil\n })\n\n mux := http.NewServeMux()\n mux.HandleFunc(\"POST /jokesFlow\", genkit.Handler(flow))\n log.Fatal(server.Start(ctx, \"0.0.0.0:\"+os.Getenv(\"PORT\"), mux))\n }\n ```\n\n6. Make API credentials available to your deployed function. Choose which\n credentials you need based on your choice in the sample above:\n\n
\n Gemini (Google AI)\n\n 1. Make sure Google AI is\n [available in your region](https://ai.google.dev/available_regions).\n\n 2. [Generate an API key](https://aistudio.google.com/app/apikey) for the\n Gemini API using Google AI Studio.\n\n 3. Make the API key available in the Cloud Run environment:\n\n 1. In the Cloud console, enable the\n [Secret Manager API](https://console.cloud.google.com/apis/library/secretmanager.googleapis.com?project=_).\n 2. On the\n [Secret Manager](https://console.cloud.google.com/security/secret-manager?project=_)\n page, create a new secret containing your API key.\n 3. After you create the secret, on the same page, grant your default\n compute service account access to the secret with the\n **Secret Manager Secret Accessor** role. (You can look up the name\n of the default compute service account on the IAM page.)\n\n In a later step, when you deploy your service, you will need to\n reference the name of this secret.\n\n
\n\n
\n Gemini (Vertex AI)\n\n 1. In the Cloud console,\n [Enable the Vertex AI API](https://console.cloud.google.com/apis/library/aiplatform.googleapis.com?project=_)\n for your project.\n\n 2. On the [IAM](https://console.cloud.google.com/iam-admin/iam?project=_)\n page, ensure that the **Default compute service account** is granted\n the **Vertex AI User** role.\n\n
\n\n The only secret you need to set up for this tutorial is for the model\n provider, but in general, you must do something similar for each service\n your flow uses.\n\n7. **Optional**: Try your flow in the developer UI:\n\n 1. Set up your local environment for the model provider you chose:\n\n
\n Gemini (Google AI)\n\n ```bash\n export GEMINI_API_KEY=\n ```\n\n
\n\n
\n Gemini (Vertex AI)\n\n ```bash\n export GOOGLE_CLOUD_PROJECT=\n\n export GOOGLE_CLOUD_LOCATION=us-central1\n\n gcloud auth application-default login\n ```\n\n
\n\n 2. Start the UI:\n\n ```bash\n genkit start -- go run .\n ```\n\n 3. In the developer UI (`http://localhost:4000/`), run the flow:\n\n 1. Click **jokesFlow**.\n\n 2. On the **Input JSON** tab, provide a subject for the model:\n\n ```json\n \"bananas\"\n ```\n\n 3. Click **Run**.\n\n8. If everything's working as expected so far, you can build and deploy the\n flow:\n\n
\n Gemini (Google AI)\n\n ```bash\n gcloud run deploy --port 3400 \\\n --update-secrets=GEMINI_API_KEY=:latest\n ```\n\n
\n\n
\n Gemini (Vertex AI)\n\n ```bash\n gcloud run deploy --port 3400 \\\n --set-env-vars GOOGLE_CLOUD_PROJECT= \\\n --set-env-vars GOOGLE_CLOUD_LOCATION=us-central1\n ```\n\n (`GOOGLE_CLOUD_LOCATION` configures the Vertex API region you want to\n use.)\n\n
\n\n Choose `N` when asked if you want to allow unauthenticated invocations.\n Answering `N` will configure your service to require IAM credentials. See\n [Authentication](https://cloud.google.com/run/docs/authenticating/overview)\n in the Cloud Run docs for information on providing these credentials.\n\nAfter deployment finishes, the tool will print the service URL. You can test\nit with `curl`:\n\n```bash\ncurl -X POST https:///jokesFlow \\\n -H \"Authorization: Bearer $(gcloud auth print-identity-token)\" \\\n -H \"Content-Type: application/json\" -d '{\"data\": \"bananas\"}'\n```\n", "title": "Genkit with Cloud Run", - "lang": "go" + "description": "Learn how to deploy Genkit Go flows as web services using Cloud Run.", + "lang": "go", + "headers": "" }, "go/deploy.md": { - "text": "You can deploy Genkit flows as web services using any service that can host a Go\nbinary. This page, as an example, walks you through the general process of\ndeploying the default sample flow, and points out where you must take\nprovider-specific actions.\n\n1. Create a directory for the Genkit sample project:\n\n ```bash\n mkdir -p ~/tmp/genkit-cloud-project\n\n cd ~/tmp/genkit-cloud-project\n ```\n\n If you're going to use an IDE, open it to this directory.\n\n2. Initialize a Go module in your project directory:\n\n ```bash\n go mod init example/cloudrun\n\n go get github.com/firebase/genkit/go\n ```\n\n3. Create a sample app using Genkit:\n\n ```go\n package main\n\n import (\n \"context\"\n \"fmt\"\n \"log\"\n \"net/http\"\n \"os\"\n\n \"github.com/firebase/genkit/go/ai\"\n \"github.com/firebase/genkit/go/genkit\"\n \"github.com/firebase/genkit/go/plugins/googlegenai\"\n \"github.com/firebase/genkit/go/plugins/server\"\n )\n\n func main() {\n ctx := context.Background()\n\n // Initialize Genkit with the Google AI plugin and Gemini 2.0 Flash.\n // Alternatively, use &googlegenai.VertexAI{} and \"vertexai/gemini-2.5-flash\"\n // to use Vertex AI as the provider instead.\n g, err := genkit.Init(ctx,\n genkit.WithPlugins(&googlegenai.GoogleAI{}),\n genkit.WithDefaultModel(\"googleai/gemini-2.5-flash\"),\n )\n if err != nil {\n log.Fatalf(\"failed to initialize Genkit: %w\", err)\n }\n\n flow := genkit.DefineFlow(g, \"jokesFlow\", func(ctx context.Context, topic string) (string, error) {\n resp, err := genkit.Generate(ctx, g,\n ai.WithPrompt(`Tell a short joke about %s. Be creative!`, topic),\n )\n if err != nil {\n return \"\", fmt.Errorf(\"failed to generate joke: %w\", err)\n }\n\n return resp.Text(), nil\n })\n\n mux := http.NewServeMux()\n mux.HandleFunc(\"POST /jokesFlow\", genkit.Handler(flow))\n log.Fatal(server.Start(ctx, \"127.0.0.1:\"+os.Getenv(\"PORT\"), mux))\n }\n ```\n\n4. Implement some form of authentication and authorization to gate access to\n the flows you plan to deploy.\n\n Because most generative AI services are metered, you most likely do not want\n to allow open access to any endpoints that call them. Some hosting services\n provide an authentication layer as a frontend to apps deployed on them,\n which you can use for this purpose.\n\n5. Make API credentials available to your deployed function. Do one of the\n following, depending on the model provider you chose:\n\n
\n Gemini (Google AI)\n\n 1. Make sure Google AI is [available in your\n region](https://ai.google.dev/available_regions).\n\n 2. [Generate an API key](https://aistudio.google.com/app/apikey) for the\n Gemini API using Google AI Studio.\n\n 3. Make the API key available in the deployed environment.\n\n Most app hosts provide some system for securely handling secrets such as\n API keys. Often, these secrets are available to your app in the form of\n environment variables. If you can assign your API key to the\n `GEMINI_API_KEY` variable, Genkit will use it automatically. Otherwise,\n you need to modify the `googlegenai.GoogleAI` plugin struct to explicitly\n set the key. (But don't embed the key directly in code! Use the secret\n management facilities provided by your hosting provider.)\n\n
\n\n
\n Gemini (Vertex AI)\n\n 1. In the Cloud console, [Enable the Vertex AI API](https://console.cloud.google.com/apis/library/aiplatform.googleapis.com?project=_)\n for your project.\n\n 2. On the [IAM](https://console.cloud.google.com/iam-admin/iam?project=_)\n page, create a service account for accessing the Vertex AI API if you\n don't alreacy have one.\n\n Grant the account the **Vertex AI User** role.\n\n 3. [Set up Application Default Credentials](https://cloud.google.com/docs/authentication/provide-credentials-adc#on-prem)\n in your hosting environment.\n\n 4. Configure the plugin with your Google Cloud project ID and the Vertex\n AI API location you want to use. You can do so either by setting the\n `GOOGLE_CLOUD_PROJECT` and `GOOGLE_CLOUD_LOCATION` environment\n variables in your hosting environment, or in your\n `googlegenai.VertexAI{}` constructor.\n\n The only secret you need to set up for this tutorial is for the model\n provider, but in general, you must do something similar for each\n service your flow uses.\n\n
\n\n6. **Optional**: Try your flow in the developer UI:\n\n 1. Set up your local environment for the model provider you chose:\n\n
\n Gemini (Google AI)\n\n ```bash\n export GEMINI_API_KEY=\n ```\n\n
\n\n
\n Gemini (Vertex AI)\n\n ```bash\n export GOOGLE_CLOUD_PROJECT=\n\n export GOOGLE_CLOUD_LOCATION=us-central1\n\n gcloud auth application-default login\n ```\n\n
\n\n 2. Start the UI:\n\n ```bash\n genkit start -- go run .\n ```\n\n 3. In the developer UI (`http://localhost:4000/`), run the flow:\n\n 4. Click **jokesFlow**.\n\n 5. On the **Input JSON** tab, provide a subject for the model:\n\n ```json\n \"bananas\"\n ```\n\n 6. Click **Run**.\n\n7. If everything's working as expected so far, you can build and deploy the\n flow using your provider's tools.\n", + "text": "# Deploy flows to any app hosting platform\n\nYou can deploy Genkit flows as web services using any service that can host a Go\nbinary. This page, as an example, walks you through the general process of\ndeploying the default sample flow, and points out where you must take\nprovider-specific actions.\n\n1. Create a directory for the Genkit sample project:\n\n ```bash\n mkdir -p ~/tmp/genkit-cloud-project\n\n cd ~/tmp/genkit-cloud-project\n ```\n\n If you're going to use an IDE, open it to this directory.\n\n2. Initialize a Go module in your project directory:\n\n ```bash\n go mod init example/cloudrun\n\n go get github.com/firebase/genkit/go\n ```\n\n3. Create a sample app using Genkit:\n\n ```go\n package main\n\n import (\n \"context\"\n \"fmt\"\n \"log\"\n \"net/http\"\n \"os\"\n\n \"github.com/firebase/genkit/go/ai\"\n \"github.com/firebase/genkit/go/genkit\"\n \"github.com/firebase/genkit/go/plugins/googlegenai\"\n \"github.com/firebase/genkit/go/plugins/server\"\n )\n\n func main() {\n ctx := context.Background()\n\n // Initialize Genkit with the Google AI plugin and Gemini 2.0 Flash.\n // Alternatively, use &googlegenai.VertexAI{} and \"vertexai/gemini-2.5-flash\"\n // to use Vertex AI as the provider instead.\n g, err := genkit.Init(ctx,\n genkit.WithPlugins(&googlegenai.GoogleAI{}),\n genkit.WithDefaultModel(\"googleai/gemini-2.5-flash\"),\n )\n if err != nil {\n log.Fatalf(\"failed to initialize Genkit: %w\", err)\n }\n\n flow := genkit.DefineFlow(g, \"jokesFlow\", func(ctx context.Context, topic string) (string, error) {\n resp, err := genkit.Generate(ctx, g,\n ai.WithPrompt(`Tell a short joke about %s. Be creative!`, topic),\n )\n if err != nil {\n return \"\", fmt.Errorf(\"failed to generate joke: %w\", err)\n }\n\n return resp.Text(), nil\n })\n\n mux := http.NewServeMux()\n mux.HandleFunc(\"POST /jokesFlow\", genkit.Handler(flow))\n log.Fatal(server.Start(ctx, \"127.0.0.1:\"+os.Getenv(\"PORT\"), mux))\n }\n ```\n\n4. Implement some form of authentication and authorization to gate access to\n the flows you plan to deploy.\n\n Because most generative AI services are metered, you most likely do not want\n to allow open access to any endpoints that call them. Some hosting services\n provide an authentication layer as a frontend to apps deployed on them,\n which you can use for this purpose.\n\n5. Make API credentials available to your deployed function. Do one of the\n following, depending on the model provider you chose:\n\n
\n Gemini (Google AI)\n\n 1. Make sure Google AI is [available in your\n region](https://ai.google.dev/available_regions).\n\n 2. [Generate an API key](https://aistudio.google.com/app/apikey) for the\n Gemini API using Google AI Studio.\n\n 3. Make the API key available in the deployed environment.\n\n Most app hosts provide some system for securely handling secrets such as\n API keys. Often, these secrets are available to your app in the form of\n environment variables. If you can assign your API key to the\n `GEMINI_API_KEY` variable, Genkit will use it automatically. Otherwise,\n you need to modify the `googlegenai.GoogleAI` plugin struct to explicitly\n set the key. (But don't embed the key directly in code! Use the secret\n management facilities provided by your hosting provider.)\n\n
\n\n
\n Gemini (Vertex AI)\n\n 1. In the Cloud console, [Enable the Vertex AI API](https://console.cloud.google.com/apis/library/aiplatform.googleapis.com?project=_)\n for your project.\n\n 2. On the [IAM](https://console.cloud.google.com/iam-admin/iam?project=_)\n page, create a service account for accessing the Vertex AI API if you\n don't alreacy have one.\n\n Grant the account the **Vertex AI User** role.\n\n 3. [Set up Application Default Credentials](https://cloud.google.com/docs/authentication/provide-credentials-adc#on-prem)\n in your hosting environment.\n\n 4. Configure the plugin with your Google Cloud project ID and the Vertex\n AI API location you want to use. You can do so either by setting the\n `GOOGLE_CLOUD_PROJECT` and `GOOGLE_CLOUD_LOCATION` environment\n variables in your hosting environment, or in your\n `googlegenai.VertexAI{}` constructor.\n\n The only secret you need to set up for this tutorial is for the model\n provider, but in general, you must do something similar for each\n service your flow uses.\n\n
\n\n6. **Optional**: Try your flow in the developer UI:\n\n 1. Set up your local environment for the model provider you chose:\n\n
\n Gemini (Google AI)\n\n ```bash\n export GEMINI_API_KEY=\n ```\n\n
\n\n
\n Gemini (Vertex AI)\n\n ```bash\n export GOOGLE_CLOUD_PROJECT=\n\n export GOOGLE_CLOUD_LOCATION=us-central1\n\n gcloud auth application-default login\n ```\n\n
\n\n 2. Start the UI:\n\n ```bash\n genkit start -- go run .\n ```\n\n 3. In the developer UI (`http://localhost:4000/`), run the flow:\n\n 4. Click **jokesFlow**.\n\n 5. On the **Input JSON** tab, provide a subject for the model:\n\n ```json\n \"bananas\"\n ```\n\n 6. Click **Run**.\n\n7. If everything's working as expected so far, you can build and deploy the\n flow using your provider's tools.\n", "title": "Deploy flows to any app hosting platform", - "lang": "go" + "description": "Learn how to deploy Genkit Go flows as web services using any service that can host a Go binary.", + "lang": "go", + "headers": "" }, "go/dotprompt.md": { - "text": "Prompt engineering is the primary way that you, as an app developer, influence\nthe output of generative AI models. For example, when using LLMs, you can craft\nprompts that influence the tone, format, length, and other characteristics of\nthe models' responses.\n\nThe way you write these prompts will depend on the model you're using; a prompt\nwritten for one model might not perform well when used with another model.\nSimilarly, the model parameters you set (temperature, top-k, and so on) will\nalso affect output differently depending on the model.\n\nGetting all three of these factors—the model, the model parameters, and\nthe prompt—working together to produce the output you want is rarely a\ntrivial process and often involves substantial iteration and experimentation.\nGenkit provides a library and file format called Dotprompt, that aims to make\nthis iteration faster and more convenient.\n\n[Dotprompt](https://github.com/google/dotprompt) is designed around the premise\nthat **prompts are code**. You define your prompts along with the models and\nmodel parameters they're intended for separately from your application code.\nThen, you (or, perhaps someone not even involved with writing application code)\ncan rapidly iterate on the prompts and model parameters using the Genkit\nDeveloper UI. Once your prompts are working the way you want, you can import\nthem into your application and run them using Genkit.\n\nYour prompt definitions each go in a file with a `.prompt` extension. Here's an\nexample of what these files look like:\n\n```dotprompt\n---\nmodel: googleai/gemini-1.5-flash\nconfig:\n temperature: 0.9\ninput:\n schema:\n location: string\n style?: string\n name?: string\n default:\n location: a restaurant\n---\n\nYou are the world's most welcoming AI assistant and are currently working at {{location}}.\n\nGreet a guest{{#if name}} named {{name}}{{/if}}{{#if style}} in the style of {{style}}{{/if}}.\n```\n\nThe portion in the triple-dashes is YAML front matter, similar to the front\nmatter format used by GitHub Markdown and Jekyll; the rest of the file is the\nprompt, which can optionally use\n[Handlebars](https://handlebarsjs.com/guide/) templates. The\nfollowing sections will go into more detail about each of the parts that make a\n`.prompt` file and how to use them.\n\n## Before you begin\n\nBefore reading this page, you should be familiar with the content covered on the\n[Generating content with AI models](/go/docs/models) page.\n\nIf you want to run the code examples on this page, first complete the steps in\nthe [Get started](/go/docs/get-started-go) guide. All of the examples assume that you\nhave already installed Genkit as a dependency in your project.\n\n## Creating prompt files\n\nAlthough Dotprompt provides several [different ways](#defining-prompts-in-code) to create\nand load prompts, it's optimized for projects that organize their prompts as\n`.prompt` files within a single directory (or subdirectories thereof). This\nsection shows you how to create and load prompts using this recommended setup.\n\n### Creating a prompt directory\n\nThe Dotprompt library expects to find your prompts in a directory at your\nproject root and automatically loads any prompts it finds there. By default,\nthis directory is named `prompts`. For example, using the default directory\nname, your project structure might look something like this:\n\n```text\nyour-project/\n├── prompts/\n│ └── hello.prompt\n├── main.go\n├── go.mod\n└── go.sum\n```\n\nIf you want to use a different directory, you can specify it when you configure\nGenkit:\n\n```go\ng, err := genkit.Init(ctx.Background(), ai.WithPromptDir(\"./llm_prompts\"))\n```\n\n### Creating a prompt file\n\nThere are two ways to create a `.prompt` file: using a text editor, or with the\ndeveloper UI.\n\n#### Using a text editor\n\nIf you want to create a prompt file using a text editor, create a text file with\nthe `.prompt` extension in your prompts directory: for example,\n`prompts/hello.prompt`.\n\nHere is a minimal example of a prompt file:\n\n```dotprompt\n---\nmodel: vertexai/gemini-1.5-flash\n---\nYou are the world's most welcoming AI assistant. Greet the user and offer your\nassistance.\n```\n\nThe portion in the dashes is YAML front matter, similar to the front matter\nformat used by GitHub Markdown and Jekyll; the rest of the file is the prompt,\nwhich can optionally use Handlebars templates. The front matter section is\noptional, but most prompt files will at least contain metadata specifying a\nmodel. The remainder of this page shows you how to go beyond this, and make use\nof Dotprompt's features in your prompt files.\n\n#### Using the developer UI\n\nYou can also create a prompt file using the model runner in the developer UI.\nStart with application code that imports the Genkit library and configures it to\nuse the model plugin you're interested in. For example:\n\n```go\npackage main\n\nimport (\n\t\"context\"\n\t\"log\"\n\n\t\"github.com/firebase/genkit/go/ai\"\n\t\"github.com/firebase/genkit/go/genkit\"\n\t\"github.com/firebase/genkit/go/plugins/googlegenai\"\n)\n\nfunc main() {\n\tg, err := genkit.Init(context.Background(), genkit.WithPlugins(&googlegenai.GoogleAI{}))\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\t// Blocks end of program execution to use the developer UI.\n\tselect {}\n}\n\n```\n\nLoad the developer UI in the same project:\n\n```bash\ngenkit start -- go run .\n```\n\nIn the **Model** section, choose the model you want to use from the list of\nmodels provided by the plugin.\n\n![Genkit developer UI model runner](./resources/developer_ui_model_runner.png)\n\nThen, experiment with the prompt and configuration until you get results you're\nhappy with. When you're ready, press the Export button and save the file to your\nprompts directory.\n\n## Running prompts\n\nAfter you've created prompt files, you can run them from your application code,\nor using the tooling provided by Genkit. Regardless of how you want to run your\nprompts, first start with application code that imports the Genkit library and\nthe model plugins you're interested in. For example:\n\n```go\npackage main\n\nimport (\n\t\"context\"\n\t\"log\"\n\n\t\"github.com/firebase/genkit/go/ai\"\n\t\"github.com/firebase/genkit/go/genkit\"\n\t\"github.com/firebase/genkit/go/plugins/googlegenai\"\n)\n\nfunc main() {\n\tg, err := genkit.Init(context.Background(), genkit.WithPlugins(&googlegenai.GoogleAI{}))\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\t// Blocks end of program execution to use the developer UI.\n\tselect {}\n}\n\n```\n\nIf you're storing your prompts in a directory other than the default, be sure to\nspecify it when you configure Genkit.\n\n### Run prompts from code\n\nTo use a prompt, first load it using the `genkit.LookupPrompt()` function:\n\n```go\nhelloPrompt := genkit.LookupPrompt(g, \"hello\")\n```\n\nAn executable prompt has similar options to that of `genkit.Generate()` and many\nof them are overridable at execution time, including things like input (see the\nsection about [specifying input schemas](#input-and-output-schemas)), configuration, and more:\n\n```go\nresp, err := helloPrompt.Execute(context.Background(),\n\tai.WithModelName(\"googleai/gemini-2.5-flash\"),\n\tai.WithInput(map[string]any{\"name\": \"John\"}),\n\tai.WithConfig(&googlegenai.GeminiConfig{Temperature: 0.5})\n)\n```\n\nAny parameters you pass to the prompt call will override the same parameters\nspecified in the prompt file.\n\nSee [Generate content with AI models](/go/docs/models) for descriptions of the available\noptions.\n\n### Using the developer UI\n\nAs you're refining your app's prompts, you can run them in the Genkit developer\nUI to quickly iterate on prompts and model configurations, independently from\nyour application code.\n\nLoad the developer UI from your project directory:\n\n```bash\ngenkit start -- go run .\n```\n\n![Genkit developer UI prompt runner](./resources/prompts-in-developer-ui.png)\n\nOnce you've loaded prompts into the developer UI, you can run them with\ndifferent input values, and experiment with how changes to the prompt wording or\nthe configuration parameters affect the model output. When you're happy with the\nresult, you can click the **Export prompt** button to save the modified prompt\nback into your project directory.\n\n## Model configuration\n\nIn the front matter block of your prompt files, you can optionally specify model\nconfiguration values for your prompt:\n\n```yaml\n---\nmodel: googleai/gemini-2.5-flash\nconfig:\n temperature: 1.4\n topK: 50\n topP: 0.4\n maxOutputTokens: 400\n stopSequences:\n - ''\n - ''\n---\n```\n\nThese values map directly to the `WithConfig()` option accepted by the\nexecutable prompt:\n\n```go\nresp, err := helloPrompt.Execute(context.Background(),\n ai.WithConfig(&googlegenai.GeminiConfig{\n Temperature: 1.4,\n TopK: 50,\n TopP: 0.4,\n MaxOutputTokens: 400,\n StopSequences: []string{\"\", \"\"},\n }))\n```\n\nSee [Generate content with AI models](/go/docs/models) for descriptions of the available\noptions.\n\n## Input and output schemas\n\nYou can specify input and output schemas for your prompt by defining them in the\nfront matter section. These schemas are used in much the same way as those\npassed to a `genkit.Generate()` request or a flow definition:\n\n```dotprompt\n---\nmodel: googleai/gemini-2.5-flash\ninput:\n schema:\n theme?: string\n default:\n theme: \"pirate\"\noutput:\n schema:\n dishname: string\n description: string\n calories: integer\n allergens(array): string\n---\nInvent a menu item for a {{theme}} themed\nrestaurant.\n```\n\nThis code produces the following structured output:\n\n```go\npackage main\n\nimport (\n\t\"context\"\n\t\"log\"\n\n\t\"github.com/firebase/genkit/go/ai\"\n\t\"github.com/firebase/genkit/go/genkit\"\n\t\"github.com/firebase/genkit/go/plugins/googlegenai\"\n)\n\nfunc main() {\n\tctx := context.Background()\n\tg, err := genkit.Init(ctx, genkit.WithPlugins(&googlegenai.GoogleAI{}))\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\tmenuPrompt := genkit.LookupPrompt(g, \"menu\")\n\tif menuPrompt == nil {\n\t\tlog.Fatal(\"no prompt named 'menu' found\")\n\t}\n\n\tresp, err := menuPrompt.Execute(ctx,\n\t\tai.WithInput(map[string]any{\"theme\": \"medieval\"}),\n\t)\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\tvar output map[string]any\n\tif err := resp.Output(&output); err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\tlog.Println(output[\"dishname\"])\n\tlog.Println(output[\"description\"])\n\n\t// Blocks end of program execution to use the developer UI.\n\tselect {}\n}\n```\n\nYou have several options for defining schemas in a `.prompt` file: Dotprompt's\nown schema definition format, Picoschema; standard JSON Schema; or, as\nreferences to schemas defined in your application code. The following sections\ndescribe each of these options in more detail.\n\n### Picoschema\n\nThe schemas in the example above are defined in a format called Picoschema.\nPicoschema is a compact, YAML-optimized schema definition format that simplifies\ndefining the most important attributes of a schema for LLM usage. Here's a\nlonger example of a schema, which specifies the information an app might store\nabout an article:\n\n```yaml\nschema:\n title: string # string, number, and boolean types are defined like this\n subtitle?: string # optional fields are marked with a `?`\n draft?: boolean, true when in draft state\n status?(enum, approval status): [PENDING, APPROVED]\n date: string, the date of publication e.g. '2024-04-09' # descriptions follow a comma\n tags(array, relevant tags for article): string # arrays are denoted via parentheses\n authors(array):\n name: string\n email?: string\n metadata?(object): # objects are also denoted via parentheses\n updatedAt?: string, ISO timestamp of last update\n approvedBy?: integer, id of approver\n extra?: any, arbitrary extra data\n (*): string, wildcard field\n```\n\nThe above schema is equivalent to the following Go type:\n\n```go\ntype Article struct {\n\tTitle string `json:\"title\"`\n\tSubtitle string `json:\"subtitle,omitempty\" jsonschema:\"required=false\"`\n\tDraft bool `json:\"draft,omitempty\"` // True when in draft state\n\tStatus string `json:\"status,omitempty\" jsonschema:\"enum=PENDING,enum=APPROVED\"` // Approval status\n\tDate string `json:\"date\"` // The date of publication e.g. '2025-04-07'\n\tTags []string `json:\"tags\"` // Relevant tags for article\n\tAuthors []struct {\n\t\tName string `json:\"name\"`\n\t\tEmail string `json:\"email,omitempty\"`\n\t} `json:\"authors\"`\n\tMetadata struct {\n\t\tUpdatedAt string `json:\"updatedAt,omitempty\"` // ISO timestamp of last update\n\t\tApprovedBy int `json:\"approvedBy,omitempty\"` // ID of approver\n\t} `json:\"metadata,omitempty\"`\n\tExtra any `json:\"extra\"` // Arbitrary extra data\n}\n```\n\nPicoschema supports scalar types `string`, `integer`, `number`, `boolean`, and\n`any`. Objects, arrays, and enums are denoted by a parenthetical after the field\nname.\n\nObjects defined by Picoschema have all properties required unless denoted\noptional by `?`, and don't allow additional properties. When a property is\nmarked as optional, it is also made nullable to provide more leniency for LLMs\nto return null instead of omitting a field.\n\nIn an object definition, the special key `(*)` can be used to declare a\n\"wildcard\" field definition. This will match any additional properties not\nsupplied by an explicit key.\n\n### JSON Schema\n\nPicoschema does not support many of the capabilities of full JSON schema. If you\nrequire more robust schemas, you may supply a JSON Schema instead:\n\n```yaml\noutput:\n schema:\n type: object\n properties:\n field1:\n type: number\n minimum: 20\n```\n\n{# TODO: Talk about defining schemas in code to reference in .prompt file once implemented. #}\n\n## Prompt templates\n\nThe portion of a `.prompt` file that follows the front matter (if present) is\nthe prompt itself, which will be passed to the model. While this prompt could be\na simple text string, very often you will want to incorporate user input into\nthe prompt. To do so, you can specify your prompt using the\n[Handlebars](https://handlebarsjs.com/guide/) templating language.\nPrompt templates can include placeholders that refer to the values defined by\nyour prompt's input schema.\n\nYou already saw this in action in the section on input and output schemas:\n\n```dotprompt\n---\nmodel: googleai/gemini-2.5-flash\ninput:\n schema:\n theme?: string\n default:\n theme: \"pirate\"\noutput:\n schema:\n dishname: string\n description: string\n calories: integer\n allergens(array): string\n---\nInvent a menu item for a {{theme}} themed restaurant.\n```\n\nIn this example, the Handlebars expression, `{{theme}}`, resolves to the value of the input's `theme` property when you\nrun the prompt. To pass input to the prompt, call the prompt as in the following\nexample:\n\n```go\nmenuPrompt = genkit.LookupPrompt(g, \"menu\")\n\nresp, err := menuPrompt.Execute(context.Background(),\n ai.WithInput(map[string]any{\"theme\": \"medieval\"}),\n)\n```\n\nNote that because the input schema declared the `theme` property to be optional\nand provided a default, you could have omitted the property, and the prompt\nwould have resolved using the default value.\n\nHandlebars templates also support some limited logical constructs. For example,\nas an alternative to providing a default, you could define the prompt using\nHandlebars's `#if` helper:\n\n```dotprompt\n---\nmodel: googleai/gemini-2.5-flash\ninput:\n schema:\n theme?: string\n---\nInvent a menu item for a {{#if theme}}{{theme}}{{else}}themed{{/if}} restaurant.\n```\n\nIn this example, the prompt renders as \"Invent a menu item for a restaurant\"\nwhen the `theme` property is unspecified.\n\nSee the [Handlebars\ndocumentation](https://handlebarsjs.com/guide/builtin-helpers.html)\nfor information on all of the built-in logical helpers.\n\nIn addition to properties defined by your input schema, your templates can also\nrefer to values automatically defined by Genkit. The next few sections describe\nthese automatically-defined values and how you can use them.\n\n### Multi-message prompts\n\nBy default, Dotprompt constructs a single message with a \"user\" role.\nHowever, some prompts, such as a system prompt, are best expressed as\ncombinations of multiple messages.\n\nThe `{{role}}` helper provides a straightforward way to construct multi-message prompts:\n\n```dotprompt\n---\nmodel: vertexai/gemini-2.5-flash\ninput:\n schema:\n userQuestion: string\n---\n{{role \"system\"}}\nYou are a helpful AI assistant that really loves to talk about food. Try to work\nfood items into all of your conversations.\n\n{{role \"user\"}}\n{{userQuestion}}\n```\n\n### Multi-modal prompts\n\nFor models that support multimodal input, such as images alongside text, you can\nuse the `{{media}}` helper:\n\n```dotprompt\n---\nmodel: vertexai/gemini-2.5-flash\ninput:\n schema:\n photoUrl: string\n---\nDescribe this image in a detailed paragraph:\n\n{{media url=photoUrl}}\n```\n\nThe URL can be `https:` or base64-encoded `data:` URIs for \"inline\" image usage.\nIn code, this would be:\n\n```go\nmultimodalPrompt = genkit.LookupPrompt(g, \"multimodal\")\n\nresp, err := multimodalPrompt.Execute(context.Background(),\n ai.WithInput(map[string]any{\"photoUrl\": \"https://example.com/photo.jpg\"}),\n)\n```\n\nSee also [Multimodal input](/go/docs/models#multimodal-input), on the\n[Generating content with AI models](/go/docs/models) page, for an\nexample of constructing a `data:` URL.\n\n### Partials\n\nPartials are reusable templates that can be included inside any prompt. Partials\ncan be especially helpful for related prompts that share common behavior.\n\nWhen loading a prompt directory, any file prefixed with an underscore (`_`) is\nconsidered a partial. So a file `_personality.prompt` might contain:\n\n```dotprompt\nYou should speak like a {{#if style}}{{style}}{{else}}helpful assistant.{{/if}}.\n```\n\nThis can then be included in other prompts:\n\n```dotprompt\n---\nmodel: googleai/gemini-2.5-flash\ninput:\n schema:\n name: string\n style?: string\n---\n{{ role \"system\" }}\n{{>personality style=style}}\n\n{{ role \"user\" }}\nGive the user a friendly greeting.\n\nUser's Name: {{name}}\n```\n\nPartials are inserted using the `{{>NAME_OF_PARTIAL args...}}` syntax. If no arguments are provided to the partial, it executes\nwith the same context as the parent prompt.\n\nPartials accept named arguments or a single positional argument\nrepresenting the context. This can be helpful for tasks such as rendering\nmembers of a list.\n\n**\\_destination.prompt**\n\n```dotprompt\n- {{name}} ({{country}})\n```\n\n**chooseDestination.prompt**\n\n```dotprompt\n---\nmodel: googleai/gemini-2.5-flash\ninput:\n schema:\n destinations(array):\n name: string\n country: string\n---\nHelp the user decide between these vacation destinations:\n\n{{#each destinations}}\n{{>destination this}}\n{{/each}}\n```\n\n#### Defining partials in code\n\nYou can also define partials in code using `genkit.DefinePartial()`:\n\n```go\ngenkit.DefinePartial(g, \"personality\", \"Talk like a {% verbatim %}{{#if style}}{{style}}{{else}}{% endverbatim %}helpful assistant{% verbatim %}{{/if}}{% endverbatim %}.\")\n```\n\nCode-defined partials are available in all prompts.\n\n### Defining Custom Helpers\n\nYou can define custom helpers to process and manage data inside of a prompt.\nHelpers are registered globally using `genkit.DefineHelper()`:\n\n```go\ngenkit.DefineHelper(g, \"shout\", func(input string) string {\n return strings.ToUpper(input)\n})\n```\n\nOnce a helper is defined you can use it in any prompt:\n\n```dotprompt\n---\nmodel: googleai/gemini-2.5-flash\ninput:\n schema:\n name: string\n---\n\nHELLO, {{shout name}}!!!\n```\n\n## Prompt variants\n\nBecause prompt files are just text, you can (and should!) commit them to your\nversion control system, simplifying the process of comparing changes over time.\nOften, tweaked versions of prompts can only be fully tested in a\nproduction environment side-by-side with existing versions. Dotprompt supports\nthis through its variants feature.\n\nTo create a variant, create a `[name].[variant].prompt` file. For example, if\nyou were using Gemini 2.0 Flash in your prompt but wanted to see if Gemini 2.5\nPro would perform better, you might create two files:\n\n- `myPrompt.prompt`: the \"baseline\" prompt\n- `myPrompt.gemini25pro.prompt`: a variant named `gemini25pro`\n\nTo use a prompt variant, specify the variant option when loading:\n\n```go\nmyPrompt := genkit.LookupPrompt(g, \"myPrompt.gemini25Pro\")\n```\n\nThe name of the variant is included in the metadata of generation traces, so you\ncan compare and contrast actual performance between variants in the Genkit trace\ninspector.\n\n## Defining prompts in code\n\nAll of the examples discussed so far have assumed that your prompts are defined\nin individual `.prompt` files in a single directory (or subdirectories thereof),\naccessible to your app at runtime. Dotprompt is designed around this setup, and\nits authors consider it to be the best developer experience overall.\n\nHowever, if you have use cases that are not well supported by this setup, you\ncan also define prompts in code using the `genkit.DefinePrompt()` function:\n\n```go\ntype GeoQuery struct {\n CountryCount int `json:\"countryCount\"`\n}\n\ntype CountryList struct {\n Countries []string `json:\"countries\"`\n}\n\ngeographyPrompt, err := genkit.DefinePrompt(\n g, \"GeographyPrompt\",\n ai.WithSystem(\"You are a geography teacher. Respond only when the user asks about geography.\"),\n ai.WithPrompt(\"Give me the {% verbatim %}{{countryCount}}{% endverbatim %} biggest countries in the world by inhabitants.\"),\n ai.WithConfig(&googlegenai.GeminiConfig{Temperature: 0.5}),\n ai.WithInputType(GeoQuery{CountryCount: 10}) // Defaults to 10.\n ai.WithOutputType(CountryList{}),\n)\nif err != nil {\n log.Fatal(err)\n}\n\nresp, err := geographyPrompt.Execute(context.Background(), ai.WithInput(GeoQuery{CountryCount: 15}))\nif err != nil {\n log.Fatal(err)\n}\n\nvar list CountryList\nif err := resp.Output(&list); err != nil {\n log.Fatal(err)\n}\n\nlog.Println(\"Countries: %s\", list.Countries)\n```\n\nPrompts may also be rendered into a `GenerateActionOptions` which may then be\nprocessed and passed into `genkit.GenerateWithRequest()`:\n\n```go\nactionOpts, err := geographyPrompt.Render(ctx, ai.WithInput(GeoQuery{CountryCount: 15}))\nif err != nil {\n log.Fatal(err)\n}\n\n// Do something with the value...\nactionOpts.Config = &googlegenai.GeminiConfig{Temperature: 0.8}\n\nresp, err := genkit.GenerateWithRequest(ctx, g, actionOpts, nil, nil) // No middleware or streaming\n```\n\nNote that all prompt options carry over to `GenerateActionOptions` with the\nexception of `WithMiddleware()`, which must be passed separately if using\n`Prompt.Render()` instead of `Prompt.Execute()`.\n", + "text": "# Managing prompts with Dotprompt\n\nPrompt engineering is the primary way that you, as an app developer, influence\nthe output of generative AI models. For example, when using LLMs, you can craft\nprompts that influence the tone, format, length, and other characteristics of\nthe models' responses.\n\nThe way you write these prompts will depend on the model you're using; a prompt\nwritten for one model might not perform well when used with another model.\nSimilarly, the model parameters you set (temperature, top-k, and so on) will\nalso affect output differently depending on the model.\n\nGetting all three of these factors—the model, the model parameters, and\nthe prompt—working together to produce the output you want is rarely a\ntrivial process and often involves substantial iteration and experimentation.\nGenkit provides a library and file format called Dotprompt, that aims to make\nthis iteration faster and more convenient.\n\n[Dotprompt](https://github.com/google/dotprompt) is designed around the premise\nthat **prompts are code**. You define your prompts along with the models and\nmodel parameters they're intended for separately from your application code.\nThen, you (or, perhaps someone not even involved with writing application code)\ncan rapidly iterate on the prompts and model parameters using the Genkit\nDeveloper UI. Once your prompts are working the way you want, you can import\nthem into your application and run them using Genkit.\n\nYour prompt definitions each go in a file with a `.prompt` extension. Here's an\nexample of what these files look like:\n\n```dotprompt\n---\nmodel: googleai/gemini-1.5-flash\nconfig:\n temperature: 0.9\ninput:\n schema:\n location: string\n style?: string\n name?: string\n default:\n location: a restaurant\n---\n\nYou are the world's most welcoming AI assistant and are currently working at {{location}}.\n\nGreet a guest{{#if name}} named {{name}}{{/if}}{{#if style}} in the style of {{style}}{{/if}}.\n```\n\nThe portion in the triple-dashes is YAML front matter, similar to the front\nmatter format used by GitHub Markdown and Jekyll; the rest of the file is the\nprompt, which can optionally use\n[Handlebars](https://handlebarsjs.com/guide/) templates. The\nfollowing sections will go into more detail about each of the parts that make a\n`.prompt` file and how to use them.\n\n## Before you begin\n\nBefore reading this page, you should be familiar with the content covered on the\n[Generating content with AI models](/go/docs/models) page.\n\nIf you want to run the code examples on this page, first complete the steps in\nthe [Get started](/go/docs/get-started-go) guide. All of the examples assume that you\nhave already installed Genkit as a dependency in your project.\n\n## Creating prompt files\n\nAlthough Dotprompt provides several [different ways](#defining-prompts-in-code) to create\nand load prompts, it's optimized for projects that organize their prompts as\n`.prompt` files within a single directory (or subdirectories thereof). This\nsection shows you how to create and load prompts using this recommended setup.\n\n### Creating a prompt directory\n\nThe Dotprompt library expects to find your prompts in a directory at your\nproject root and automatically loads any prompts it finds there. By default,\nthis directory is named `prompts`. For example, using the default directory\nname, your project structure might look something like this:\n\n```text\nyour-project/\n├── prompts/\n│ └── hello.prompt\n├── main.go\n├── go.mod\n└── go.sum\n```\n\nIf you want to use a different directory, you can specify it when you configure\nGenkit:\n\n```go\ng, err := genkit.Init(ctx.Background(), ai.WithPromptDir(\"./llm_prompts\"))\n```\n\n### Creating a prompt file\n\nThere are two ways to create a `.prompt` file: using a text editor, or with the\ndeveloper UI.\n\n#### Using a text editor\n\nIf you want to create a prompt file using a text editor, create a text file with\nthe `.prompt` extension in your prompts directory: for example,\n`prompts/hello.prompt`.\n\nHere is a minimal example of a prompt file:\n\n```dotprompt\n---\nmodel: vertexai/gemini-1.5-flash\n---\nYou are the world's most welcoming AI assistant. Greet the user and offer your\nassistance.\n```\n\nThe portion in the dashes is YAML front matter, similar to the front matter\nformat used by GitHub Markdown and Jekyll; the rest of the file is the prompt,\nwhich can optionally use Handlebars templates. The front matter section is\noptional, but most prompt files will at least contain metadata specifying a\nmodel. The remainder of this page shows you how to go beyond this, and make use\nof Dotprompt's features in your prompt files.\n\n#### Using the developer UI\n\nYou can also create a prompt file using the model runner in the developer UI.\nStart with application code that imports the Genkit library and configures it to\nuse the model plugin you're interested in. For example:\n\n```go\npackage main\n\nimport (\n\t\"context\"\n\t\"log\"\n\n\t\"github.com/firebase/genkit/go/ai\"\n\t\"github.com/firebase/genkit/go/genkit\"\n\t\"github.com/firebase/genkit/go/plugins/googlegenai\"\n)\n\nfunc main() {\n\tg, err := genkit.Init(context.Background(), genkit.WithPlugins(&googlegenai.GoogleAI{}))\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\t// Blocks end of program execution to use the developer UI.\n\tselect {}\n}\n\n```\n\nLoad the developer UI in the same project:\n\n```bash\ngenkit start -- go run .\n```\n\nIn the **Model** section, choose the model you want to use from the list of\nmodels provided by the plugin.\n\n![Genkit developer UI model runner](./resources/developer_ui_model_runner.png)\n\nThen, experiment with the prompt and configuration until you get results you're\nhappy with. When you're ready, press the Export button and save the file to your\nprompts directory.\n\n## Running prompts\n\nAfter you've created prompt files, you can run them from your application code,\nor using the tooling provided by Genkit. Regardless of how you want to run your\nprompts, first start with application code that imports the Genkit library and\nthe model plugins you're interested in. For example:\n\n```go\npackage main\n\nimport (\n\t\"context\"\n\t\"log\"\n\n\t\"github.com/firebase/genkit/go/ai\"\n\t\"github.com/firebase/genkit/go/genkit\"\n\t\"github.com/firebase/genkit/go/plugins/googlegenai\"\n)\n\nfunc main() {\n\tg, err := genkit.Init(context.Background(), genkit.WithPlugins(&googlegenai.GoogleAI{}))\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\t// Blocks end of program execution to use the developer UI.\n\tselect {}\n}\n\n```\n\nIf you're storing your prompts in a directory other than the default, be sure to\nspecify it when you configure Genkit.\n\n### Run prompts from code\n\nTo use a prompt, first load it using the `genkit.LookupPrompt()` function:\n\n```go\nhelloPrompt := genkit.LookupPrompt(g, \"hello\")\n```\n\nAn executable prompt has similar options to that of `genkit.Generate()` and many\nof them are overridable at execution time, including things like input (see the\nsection about [specifying input schemas](#input-and-output-schemas)), configuration, and more:\n\n```go\nresp, err := helloPrompt.Execute(context.Background(),\n\tai.WithModelName(\"googleai/gemini-2.5-flash\"),\n\tai.WithInput(map[string]any{\"name\": \"John\"}),\n\tai.WithConfig(&googlegenai.GeminiConfig{Temperature: 0.5})\n)\n```\n\nAny parameters you pass to the prompt call will override the same parameters\nspecified in the prompt file.\n\nSee [Generate content with AI models](/go/docs/models) for descriptions of the available\noptions.\n\n### Using the developer UI\n\nAs you're refining your app's prompts, you can run them in the Genkit developer\nUI to quickly iterate on prompts and model configurations, independently from\nyour application code.\n\nLoad the developer UI from your project directory:\n\n```bash\ngenkit start -- go run .\n```\n\n![Genkit developer UI prompt runner](./resources/prompts-in-developer-ui.png)\n\nOnce you've loaded prompts into the developer UI, you can run them with\ndifferent input values, and experiment with how changes to the prompt wording or\nthe configuration parameters affect the model output. When you're happy with the\nresult, you can click the **Export prompt** button to save the modified prompt\nback into your project directory.\n\n## Model configuration\n\nIn the front matter block of your prompt files, you can optionally specify model\nconfiguration values for your prompt:\n\n```yaml\n---\nmodel: googleai/gemini-2.5-flash\nconfig:\n temperature: 1.4\n topK: 50\n topP: 0.4\n maxOutputTokens: 400\n stopSequences:\n - ''\n - ''\n---\n```\n\nThese values map directly to the `WithConfig()` option accepted by the\nexecutable prompt:\n\n```go\nresp, err := helloPrompt.Execute(context.Background(),\n ai.WithConfig(&googlegenai.GeminiConfig{\n Temperature: 1.4,\n TopK: 50,\n TopP: 0.4,\n MaxOutputTokens: 400,\n StopSequences: []string{\"\", \"\"},\n }))\n```\n\nSee [Generate content with AI models](/go/docs/models) for descriptions of the available\noptions.\n\n## Input and output schemas\n\nYou can specify input and output schemas for your prompt by defining them in the\nfront matter section. These schemas are used in much the same way as those\npassed to a `genkit.Generate()` request or a flow definition:\n\n```dotprompt\n---\nmodel: googleai/gemini-2.5-flash\ninput:\n schema:\n theme?: string\n default:\n theme: \"pirate\"\noutput:\n schema:\n dishname: string\n description: string\n calories: integer\n allergens(array): string\n---\nInvent a menu item for a {{theme}} themed\nrestaurant.\n```\n\nThis code produces the following structured output:\n\n```go\npackage main\n\nimport (\n\t\"context\"\n\t\"log\"\n\n\t\"github.com/firebase/genkit/go/ai\"\n\t\"github.com/firebase/genkit/go/genkit\"\n\t\"github.com/firebase/genkit/go/plugins/googlegenai\"\n)\n\nfunc main() {\n\tctx := context.Background()\n\tg, err := genkit.Init(ctx, genkit.WithPlugins(&googlegenai.GoogleAI{}))\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\tmenuPrompt := genkit.LookupPrompt(g, \"menu\")\n\tif menuPrompt == nil {\n\t\tlog.Fatal(\"no prompt named 'menu' found\")\n\t}\n\n\tresp, err := menuPrompt.Execute(ctx,\n\t\tai.WithInput(map[string]any{\"theme\": \"medieval\"}),\n\t)\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\tvar output map[string]any\n\tif err := resp.Output(&output); err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\tlog.Println(output[\"dishname\"])\n\tlog.Println(output[\"description\"])\n\n\t// Blocks end of program execution to use the developer UI.\n\tselect {}\n}\n```\n\nYou have several options for defining schemas in a `.prompt` file: Dotprompt's\nown schema definition format, Picoschema; standard JSON Schema; or, as\nreferences to schemas defined in your application code. The following sections\ndescribe each of these options in more detail.\n\n### Picoschema\n\nThe schemas in the example above are defined in a format called Picoschema.\nPicoschema is a compact, YAML-optimized schema definition format that simplifies\ndefining the most important attributes of a schema for LLM usage. Here's a\nlonger example of a schema, which specifies the information an app might store\nabout an article:\n\n```yaml\nschema:\n title: string # string, number, and boolean types are defined like this\n subtitle?: string # optional fields are marked with a `?`\n draft?: boolean, true when in draft state\n status?(enum, approval status): [PENDING, APPROVED]\n date: string, the date of publication e.g. '2024-04-09' # descriptions follow a comma\n tags(array, relevant tags for article): string # arrays are denoted via parentheses\n authors(array):\n name: string\n email?: string\n metadata?(object): # objects are also denoted via parentheses\n updatedAt?: string, ISO timestamp of last update\n approvedBy?: integer, id of approver\n extra?: any, arbitrary extra data\n (*): string, wildcard field\n```\n\nThe above schema is equivalent to the following Go type:\n\n```go\ntype Article struct {\n\tTitle string `json:\"title\"`\n\tSubtitle string `json:\"subtitle,omitempty\" jsonschema:\"required=false\"`\n\tDraft bool `json:\"draft,omitempty\"` // True when in draft state\n\tStatus string `json:\"status,omitempty\" jsonschema:\"enum=PENDING,enum=APPROVED\"` // Approval status\n\tDate string `json:\"date\"` // The date of publication e.g. '2025-04-07'\n\tTags []string `json:\"tags\"` // Relevant tags for article\n\tAuthors []struct {\n\t\tName string `json:\"name\"`\n\t\tEmail string `json:\"email,omitempty\"`\n\t} `json:\"authors\"`\n\tMetadata struct {\n\t\tUpdatedAt string `json:\"updatedAt,omitempty\"` // ISO timestamp of last update\n\t\tApprovedBy int `json:\"approvedBy,omitempty\"` // ID of approver\n\t} `json:\"metadata,omitempty\"`\n\tExtra any `json:\"extra\"` // Arbitrary extra data\n}\n```\n\nPicoschema supports scalar types `string`, `integer`, `number`, `boolean`, and\n`any`. Objects, arrays, and enums are denoted by a parenthetical after the field\nname.\n\nObjects defined by Picoschema have all properties required unless denoted\noptional by `?`, and don't allow additional properties. When a property is\nmarked as optional, it is also made nullable to provide more leniency for LLMs\nto return null instead of omitting a field.\n\nIn an object definition, the special key `(*)` can be used to declare a\n\"wildcard\" field definition. This will match any additional properties not\nsupplied by an explicit key.\n\n### JSON Schema\n\nPicoschema does not support many of the capabilities of full JSON schema. If you\nrequire more robust schemas, you may supply a JSON Schema instead:\n\n```yaml\noutput:\n schema:\n type: object\n properties:\n field1:\n type: number\n minimum: 20\n```\n\n{# TODO: Talk about defining schemas in code to reference in .prompt file once implemented. #}\n\n## Prompt templates\n\nThe portion of a `.prompt` file that follows the front matter (if present) is\nthe prompt itself, which will be passed to the model. While this prompt could be\na simple text string, very often you will want to incorporate user input into\nthe prompt. To do so, you can specify your prompt using the\n[Handlebars](https://handlebarsjs.com/guide/) templating language.\nPrompt templates can include placeholders that refer to the values defined by\nyour prompt's input schema.\n\nYou already saw this in action in the section on input and output schemas:\n\n```dotprompt\n---\nmodel: googleai/gemini-2.5-flash\ninput:\n schema:\n theme?: string\n default:\n theme: \"pirate\"\noutput:\n schema:\n dishname: string\n description: string\n calories: integer\n allergens(array): string\n---\nInvent a menu item for a {{theme}} themed restaurant.\n```\n\nIn this example, the Handlebars expression, `{{theme}}`, resolves to the value of the input's `theme` property when you\nrun the prompt. To pass input to the prompt, call the prompt as in the following\nexample:\n\n```go\nmenuPrompt = genkit.LookupPrompt(g, \"menu\")\n\nresp, err := menuPrompt.Execute(context.Background(),\n ai.WithInput(map[string]any{\"theme\": \"medieval\"}),\n)\n```\n\nNote that because the input schema declared the `theme` property to be optional\nand provided a default, you could have omitted the property, and the prompt\nwould have resolved using the default value.\n\nHandlebars templates also support some limited logical constructs. For example,\nas an alternative to providing a default, you could define the prompt using\nHandlebars's `#if` helper:\n\n```dotprompt\n---\nmodel: googleai/gemini-2.5-flash\ninput:\n schema:\n theme?: string\n---\nInvent a menu item for a {{#if theme}}{{theme}}{{else}}themed{{/if}} restaurant.\n```\n\nIn this example, the prompt renders as \"Invent a menu item for a restaurant\"\nwhen the `theme` property is unspecified.\n\nSee the [Handlebars\ndocumentation](https://handlebarsjs.com/guide/builtin-helpers.html)\nfor information on all of the built-in logical helpers.\n\nIn addition to properties defined by your input schema, your templates can also\nrefer to values automatically defined by Genkit. The next few sections describe\nthese automatically-defined values and how you can use them.\n\n### Multi-message prompts\n\nBy default, Dotprompt constructs a single message with a \"user\" role.\nHowever, some prompts, such as a system prompt, are best expressed as\ncombinations of multiple messages.\n\nThe `{{role}}` helper provides a straightforward way to construct multi-message prompts:\n\n```dotprompt\n---\nmodel: vertexai/gemini-2.5-flash\ninput:\n schema:\n userQuestion: string\n---\n{{role \"system\"}}\nYou are a helpful AI assistant that really loves to talk about food. Try to work\nfood items into all of your conversations.\n\n{{role \"user\"}}\n{{userQuestion}}\n```\n\n### Multi-modal prompts\n\nFor models that support multimodal input, such as images alongside text, you can\nuse the `{{media}}` helper:\n\n```dotprompt\n---\nmodel: vertexai/gemini-2.5-flash\ninput:\n schema:\n photoUrl: string\n---\nDescribe this image in a detailed paragraph:\n\n{{media url=photoUrl}}\n```\n\nThe URL can be `https:` or base64-encoded `data:` URIs for \"inline\" image usage.\nIn code, this would be:\n\n```go\nmultimodalPrompt = genkit.LookupPrompt(g, \"multimodal\")\n\nresp, err := multimodalPrompt.Execute(context.Background(),\n ai.WithInput(map[string]any{\"photoUrl\": \"https://example.com/photo.jpg\"}),\n)\n```\n\nSee also [Multimodal input](/go/docs/models#multimodal-input), on the\n[Generating content with AI models](/go/docs/models) page, for an\nexample of constructing a `data:` URL.\n\n### Partials\n\nPartials are reusable templates that can be included inside any prompt. Partials\ncan be especially helpful for related prompts that share common behavior.\n\nWhen loading a prompt directory, any file prefixed with an underscore (`_`) is\nconsidered a partial. So a file `_personality.prompt` might contain:\n\n```dotprompt\nYou should speak like a {{#if style}}{{style}}{{else}}helpful assistant.{{/if}}.\n```\n\nThis can then be included in other prompts:\n\n```dotprompt\n---\nmodel: googleai/gemini-2.5-flash\ninput:\n schema:\n name: string\n style?: string\n---\n{{ role \"system\" }}\n{{>personality style=style}}\n\n{{ role \"user\" }}\nGive the user a friendly greeting.\n\nUser's Name: {{name}}\n```\n\nPartials are inserted using the `{{>NAME_OF_PARTIAL args...}}` syntax. If no arguments are provided to the partial, it executes\nwith the same context as the parent prompt.\n\nPartials accept named arguments or a single positional argument\nrepresenting the context. This can be helpful for tasks such as rendering\nmembers of a list.\n\n**\\_destination.prompt**\n\n```dotprompt\n- {{name}} ({{country}})\n```\n\n**chooseDestination.prompt**\n\n```dotprompt\n---\nmodel: googleai/gemini-2.5-flash\ninput:\n schema:\n destinations(array):\n name: string\n country: string\n---\nHelp the user decide between these vacation destinations:\n\n{{#each destinations}}\n{{>destination this}}\n{{/each}}\n```\n\n#### Defining partials in code\n\nYou can also define partials in code using `genkit.DefinePartial()`:\n\n```go\ngenkit.DefinePartial(g, \"personality\", \"Talk like a {% verbatim %}{{#if style}}{{style}}{{else}}{% endverbatim %}helpful assistant{% verbatim %}{{/if}}{% endverbatim %}.\")\n```\n\nCode-defined partials are available in all prompts.\n\n### Defining Custom Helpers\n\nYou can define custom helpers to process and manage data inside of a prompt.\nHelpers are registered globally using `genkit.DefineHelper()`:\n\n```go\ngenkit.DefineHelper(g, \"shout\", func(input string) string {\n return strings.ToUpper(input)\n})\n```\n\nOnce a helper is defined you can use it in any prompt:\n\n```dotprompt\n---\nmodel: googleai/gemini-2.5-flash\ninput:\n schema:\n name: string\n---\n\nHELLO, {{shout name}}!!!\n```\n\n## Prompt variants\n\nBecause prompt files are just text, you can (and should!) commit them to your\nversion control system, simplifying the process of comparing changes over time.\nOften, tweaked versions of prompts can only be fully tested in a\nproduction environment side-by-side with existing versions. Dotprompt supports\nthis through its variants feature.\n\nTo create a variant, create a `[name].[variant].prompt` file. For example, if\nyou were using Gemini 2.0 Flash in your prompt but wanted to see if Gemini 2.5\nPro would perform better, you might create two files:\n\n- `myPrompt.prompt`: the \"baseline\" prompt\n- `myPrompt.gemini25pro.prompt`: a variant named `gemini25pro`\n\nTo use a prompt variant, specify the variant option when loading:\n\n```go\nmyPrompt := genkit.LookupPrompt(g, \"myPrompt.gemini25Pro\")\n```\n\nThe name of the variant is included in the metadata of generation traces, so you\ncan compare and contrast actual performance between variants in the Genkit trace\ninspector.\n\n## Defining prompts in code\n\nAll of the examples discussed so far have assumed that your prompts are defined\nin individual `.prompt` files in a single directory (or subdirectories thereof),\naccessible to your app at runtime. Dotprompt is designed around this setup, and\nits authors consider it to be the best developer experience overall.\n\nHowever, if you have use cases that are not well supported by this setup, you\ncan also define prompts in code using the `genkit.DefinePrompt()` function:\n\n```go\ntype GeoQuery struct {\n CountryCount int `json:\"countryCount\"`\n}\n\ntype CountryList struct {\n Countries []string `json:\"countries\"`\n}\n\ngeographyPrompt, err := genkit.DefinePrompt(\n g, \"GeographyPrompt\",\n ai.WithSystem(\"You are a geography teacher. Respond only when the user asks about geography.\"),\n ai.WithPrompt(\"Give me the {% verbatim %}{{countryCount}}{% endverbatim %} biggest countries in the world by inhabitants.\"),\n ai.WithConfig(&googlegenai.GeminiConfig{Temperature: 0.5}),\n ai.WithInputType(GeoQuery{CountryCount: 10}) // Defaults to 10.\n ai.WithOutputType(CountryList{}),\n)\nif err != nil {\n log.Fatal(err)\n}\n\nresp, err := geographyPrompt.Execute(context.Background(), ai.WithInput(GeoQuery{CountryCount: 15}))\nif err != nil {\n log.Fatal(err)\n}\n\nvar list CountryList\nif err := resp.Output(&list); err != nil {\n log.Fatal(err)\n}\n\nlog.Println(\"Countries: %s\", list.Countries)\n```\n\nPrompts may also be rendered into a `GenerateActionOptions` which may then be\nprocessed and passed into `genkit.GenerateWithRequest()`:\n\n```go\nactionOpts, err := geographyPrompt.Render(ctx, ai.WithInput(GeoQuery{CountryCount: 15}))\nif err != nil {\n log.Fatal(err)\n}\n\n// Do something with the value...\nactionOpts.Config = &googlegenai.GeminiConfig{Temperature: 0.8}\n\nresp, err := genkit.GenerateWithRequest(ctx, g, actionOpts, nil, nil) // No middleware or streaming\n```\n\nNote that all prompt options carry over to `GenerateActionOptions` with the\nexception of `WithMiddleware()`, which must be passed separately if using\n`Prompt.Render()` instead of `Prompt.Execute()`.\n", "title": "Managing prompts with Dotprompt", - "lang": "go" + "description": "Learn how to use Dotprompt to manage prompts, models, and parameters for generative AI models in Genkit Go.", + "lang": "go", + "headers": "## Before you begin\n## Creating prompt files\n### Creating a prompt directory\n### Creating a prompt file\n#### Using a text editor\n#### Using the developer UI\n## Running prompts\n### Run prompts from code\n### Using the developer UI\n## Model configuration\n## Input and output schemas\n### Picoschema\n### JSON Schema\n## Prompt templates\n### Multi-message prompts\n### Multi-modal prompts\n### Partials\n#### Defining partials in code\n### Defining Custom Helpers\n## Prompt variants\n## Defining prompts in code\n" }, "go/evaluation.md": { - "text": "Evaluation is a form of testing that helps you validate your LLM's responses and\nensure they meet your quality bar.\n\nGenkit supports third-party evaluation tools through plugins, paired\nwith powerful observability features that provide insight into the runtime state\nof your LLM-powered applications. Genkit tooling helps you automatically extract\ndata including inputs, outputs, and information from intermediate steps to\nevaluate the end-to-end quality of LLM responses as well as understand the\nperformance of your system's building blocks.\n\n### Types of evaluation\n\nGenkit supports two types of evaluation:\n\n- **Inference-based evaluation**: This type of evaluation runs against a\n collection of pre-determined inputs, assessing the corresponding outputs for\n quality.\n\n This is the most common evaluation type, suitable for most use cases. This\n approach tests a system's actual output for each evaluation run.\n\n You can perform the quality assessment manually, by visually inspecting the\n results. Alternatively, you can automate the assessment by using an\n evaluation metric.\n\n- **Raw evaluation**: This type of evaluation directly assesses the quality of\n inputs without any inference. This approach typically is used with automated\n evaluation using metrics. All required fields for evaluation (e.g., `input`,\n `context`, `output` and `reference`) must be present in the input dataset. This\n is useful when you have data coming from an external source (e.g., collected\n from your production traces) and you want to have an objective measurement of\n the quality of the collected data.\n\n For more information, see the [Advanced use](#advanced-use) section of this\n page.\n\nThis section explains how to perform inference-based evaluation using Genkit.\n\n## Quick start\n\nPerform these steps to get started quickly with Genkit.\n\n### Setup\n\n1. Use an existing Genkit app or create a new one by following our\n [Get started](/go/docs/get-started-go) guide.\n\n2. Add the following code to define a simple RAG application to evaluate. For\n this guide, we use a dummy retriever that always returns the same documents.\n\n ```go\n package main\n\n import (\n \"context\"\n \"fmt\"\n \"log\"\n\n \"github.com/firebase/genkit/go/ai\"\n \"github.com/firebase/genkit/go/genkit\"\n \"github.com/firebase/genkit/go/plugins/googlegenai\"\n )\n\n func main() {\n ctx := context.Background()\n\n // Initialize Genkit\n g, err := genkit.Init(ctx,\n genkit.WithPlugins(&googlegenai.GoogleAI{}),\n genkit.WithDefaultModel(\"googleai/gemini-2.5-flash\"),\n )\n if err != nil {\n log.Fatalf(\"Genkit initialization error: %v\", err)\n }\n\n // Dummy retriever that always returns the same facts\n dummyRetrieverFunc := func(ctx context.Context, req *ai.RetrieverRequest) (*ai.RetrieverResponse, error) {\n facts := []string{\n \"Dog is man's best friend\",\n \"Dogs have evolved and were domesticated from wolves\",\n }\n // Just return facts as documents.\n var docs []*ai.Document\n for _, fact := range facts {\n docs = append(docs, ai.DocumentFromText(fact, nil))\n }\n return &ai.RetrieverResponse{Documents: docs}, nil\n }\n factsRetriever := genkit.DefineRetriever(g, \"local\", \"dogFacts\", dummyRetrieverFunc)\n\n m := googlegenai.GoogleAIModel(g, \"gemini-2.5-flash\")\n if m == nil {\n log.Fatal(\"failed to find model\")\n }\n\n // A simple question-answering flow\n genkit.DefineFlow(g, \"qaFlow\", func(ctx context.Context, query string) (string, error) {\n factDocs, err := ai.Retrieve(ctx, factsRetriever, ai.WithTextDocs(query))\n if err != nil {\n return \"\", fmt.Errorf(\"retrieval failed: %w\", err)\n }\n llmResponse, err := genkit.Generate(ctx, g,\n ai.WithModelName(\"googleai/gemini-2.5-flash\"),\n ai.WithPrompt(\"Answer this question with the given context: %s\", query),\n ai.WithDocs(factDocs.Documents...)\n )\n if err != nil {\n return \"\", fmt.Errorf(\"generation failed: %w\", err)\n }\n return llmResponse.Text(), nil\n })\n }\n ```\n\n3. You can optionally add evaluation metrics to your application to use while\n evaluating. This guide uses the `EvaluatorRegex` metric from the\n `evaluators` package.\n\n ```go\n import (\n \"github.com/firebase/genkit/go/plugins/evaluators\"\n )\n\n func main() {\n // ...\n\n metrics := []evaluators.MetricConfig{\n {\n MetricType: evaluators.EvaluatorRegex,\n },\n }\n\n // Initialize Genkit\n g, err := genkit.Init(ctx,\n genkit.WithPlugins(\n &googlegenai.GoogleAI{},\n &evaluators.GenkitEval{Metrics: metrics}, // Add this plugin\n ),\n genkit.WithDefaultModel(\"googleai/gemini-2.5-flash\"),\n )\n }\n ```\n\n **Note:** Ensure that the `evaluators` package is\n installed in your go project:\n\n ```bash\n go get github.com/firebase/genkit/go/plugins/evaluators\n ```\n\n4. Start your Genkit application.\n\n ```bash\n genkit start -- go run main.go\n ```\n\n### Create a dataset\n\nCreate a dataset to define the examples we want to use for evaluating our flow.\n\n1. Go to the Dev UI at `http://localhost:4000` and click the **Datasets**\n button to open the Datasets page.\n\n2. Click the **Create Dataset** button to open the create dataset dialog.\n\n a. Provide a `datasetId` for your new dataset. This guide uses\n `myFactsQaDataset`.\n\n b. Select `Flow` dataset type.\n\n c. Leave the validation target field empty and click **Save**\n\n3. Your new dataset page appears, showing an empty dataset. Add examples to it\n by following these steps:\n\n a. Click the **Add example** button to open the example editor panel.\n\n b. Only the `Input` field is required. Enter `\"Who is man's best friend?\"`\n in the `Input` field, and click **Save** to add the example has to your\n dataset.\n\n If you have configured the `EvaluatorRegex` metric and would like\n to try it out, you need to specify a Reference string that contains the\n pattern to match the output against. For the preceding input, set the\n `Reference output` text to `\"(?i)dog\"`, which is a case-insensitive regular-\n expression pattern to match the word \"dog\" in the flow output.\n\n c. Repeat steps (a) and (b) a couple of more times to add more examples.\n This guide adds the following example inputs to the dataset:\n\n ```text\n \"Can I give milk to my cats?\"\n \"From which animals did dogs evolve?\"\n ```\n\n If you are using the regular-expression evaluator, use the corresponding\n reference strings:\n\n ```text\n \"(?i)don't know\"\n \"(?i)wolf|wolves\"\n ```\n\n Note that this is a contrived example and the regular-expression\n evaluator may not be the right choice to evaluate the responses\n from `qaFlow`. However, this guide can be applied to any\n Genkit Go evaluator of your choice.\n\n By the end of this step, your dataset should have 3 examples in it, with the\n values mentioned above.\n\n### Run evaluation and view results\n\nTo start evaluating the flow, click the **Run new evaluation** button on your\ndataset page. You can also start a new evaluation from the _Evaluations_ tab.\n\n1. Select the `Flow` radio button to evaluate a flow.\n\n2. Select `qaFlow` as the target flow to evaluate.\n\n3. Select `myFactsQaDataset` as the target dataset to use for evaluation.\n\n4. If you have installed an evaluator metric using Genkit plugins,\n you can see these metrics in this page. Select the metrics that you want to\n use with this evaluation run. This is entirely optional: Omitting this step\n will still return the results in the evaluation run, but without any\n associated metrics.\n\n If you have not provided any reference values and are using the\n `EvaluatorRegex` metric, your evaluation will fail since this metric needs\n reference to be set.\n\n5. Click **Run evaluation** to start evaluation. Depending on the flow\n you're testing, this may take a while. Once the evaluation is complete, a\n success message appears with a link to view the results. Click the link\n to go to the _Evaluation details_ page.\n\nYou can see the details of your evaluation on this page, including original\ninput, extracted context and metrics (if any).\n\n## Core concepts\n\n### Terminology\n\nKnowing the following terms can help ensure that you correctly understand\nthe information provided on this page:\n\n- **Evaluation**: An evaluation is a process that assesses system performance.\n In Genkit, such a system is usually a Genkit primitive, such as a flow or a\n model. An evaluation can be automated or manual (human evaluation).\n\n- **Bulk inference** Inference is the act of running an input on a flow or\n model to get the corresponding output. Bulk inference involves performing\n inference on multiple inputs simultaneously.\n\n- **Metric** An evaluation metric is a criterion on which an inference is\n scored. Examples include accuracy, faithfulness, maliciousness, whether the\n output is in English, etc.\n\n- **Dataset** A dataset is a collection of examples to use for inference-based\n evaluation. A dataset typically consists of `Input` and optional `Reference`\n fields. The `Reference` field does not affect the inference step of evaluation\n but it is passed verbatim to any evaluation metrics. In Genkit, you can create\n a dataset through the Dev UI. There are two types of datasets in Genkit:\n _Flow_ datasets and _Model_ datasets.\n\n## Supported evaluators\n\nGenkit supports several evaluators, some built-in, and others\nprovided externally.\n\n### Genkit evaluators\n\nGenkit includes a small number of built-in evaluators, ported from\nthe [JS evaluators plugin](https://js.api.genkit.dev/enums/_genkit-ai_evaluator.GenkitMetric.html),\nto help you get started:\n\n- EvaluatorDeepEqual -- Checks if the generated output is deep-equal to the\n reference output provided.\n- EvaluatorRegex -- Checks if the generated output matches the regular\n expression provided in the reference field.\n- EvaluatorJsonata -- Checks if the generated output matches the\n [JSONATA](https://jsonata.org/) expression provided in the\n reference field.\n\n## Advanced use\n\nAlong with its basic functionality, Genkit also provides advanced support for\ncertain evaluation use cases.\n\n### Evaluation using the CLI\n\nGenkit CLI provides a rich API for performing evaluation. This is especially\nuseful in environments where the Dev UI is not available (e.g. in a CI/CD\nworkflow).\n\nGenkit CLI provides 3 main evaluation commands: `eval:flow`, `eval:extractData`,\nand `eval:run`.\n\n#### Evaluation `eval:flow` command\n\nThe `eval:flow` command runs inference-based evaluation on an input dataset.\nThis dataset may be provided either as a JSON file or by referencing an existing\ndataset in your Genkit runtime.\n\n```bash\n# Referencing an existing dataset\ngenkit eval:flow qaFlow --input myFactsQaDataset\n\n# or, using a dataset from a file\ngenkit eval:flow qaFlow --input testInputs.json\n```\n\n**Note:** Make sure that you start your genkit app before running these CLI\ncommands.\n\n```bash\ngenkit start -- go run main.go\n```\n\nHere, `testInputs.json` should be an array of objects containing an `input`\nfield and an optional `reference` field, like below:\n\n```json\n[\n {\n \"input\": \"What is the French word for Cheese?\"\n },\n {\n \"input\": \"What green vegetable looks like cauliflower?\",\n \"reference\": \"Broccoli\"\n }\n]\n```\n\nIf your flow requires auth, you may specify it using the `--context` argument:\n\n```bash\ngenkit eval:flow qaFlow --input testInputs.json --context '{\"auth\": {\"email_verified\": true}}'\n```\n\nBy default, the `eval:flow` and `eval:run` commands use all available metrics\nfor evaluation. To run on a subset of the configured evaluators, use the\n`--evaluators` flag and provide a comma-separated list of evaluators by name:\n\n```bash\ngenkit eval:flow qaFlow --input testInputs.json --evaluators=genkitEval/regex,genkitEval/jsonata\n```\n\nYou can view the results of your evaluation run in the Dev UI at\n`localhost:4000/evaluate`.\n\n#### `eval:extractData` and `eval:run` commands\n\nTo support _raw evaluation_, Genkit provides tools to extract data from traces\nand run evaluation metrics on extracted data. This is useful, for example, if\nyou are using a different framework for evaluation or if you are collecting\ninferences from a different environment to test locally for output quality.\n\nYou can batch run your Genkit flow and extract an _evaluation dataset_ from the\nresultant traces. A raw evaluation dataset is a collection of inputs for\nevaluation metrics, _without_ running any prior inference.\n\nRun your flow over your test inputs:\n\n```bash\ngenkit flow:batchRun qaFlow testInputs.json\n```\n\nExtract the evaluation data:\n\n```bash\ngenkit eval:extractData qaFlow --maxRows 2 --output factsEvalDataset.json\n```\n\nThe exported data has a format different from the dataset format presented\nearlier. This is because this data is intended to be used with evaluation\nmetrics directly, without any inference step. Here is the syntax of the\nextracted data.\n\n```json\nArray<{\n \"testCaseId\": string,\n \"input\": any,\n \"output\": any,\n \"context\": any[],\n \"traceIds\": string[],\n}>;\n```\n\nThe data extractor automatically locates retrievers and adds the produced docs\nto the context array. You can run evaluation metrics on this extracted dataset\nusing the `eval:run` command.\n\n```bash\ngenkit eval:run factsEvalDataset.json\n```\n\nBy default, `eval:run` runs against all configured evaluators, and as with\n`eval:flow`, results for `eval:run` appear in the evaluation page of Developer\nUI, located at `localhost:4000/evaluate`.\n", + "text": "# Evaluation\n\nEvaluation is a form of testing that helps you validate your LLM's responses and\nensure they meet your quality bar.\n\nGenkit supports third-party evaluation tools through plugins, paired\nwith powerful observability features that provide insight into the runtime state\nof your LLM-powered applications. Genkit tooling helps you automatically extract\ndata including inputs, outputs, and information from intermediate steps to\nevaluate the end-to-end quality of LLM responses as well as understand the\nperformance of your system's building blocks.\n\n### Types of evaluation\n\nGenkit supports two types of evaluation:\n\n- **Inference-based evaluation**: This type of evaluation runs against a\n collection of pre-determined inputs, assessing the corresponding outputs for\n quality.\n\n This is the most common evaluation type, suitable for most use cases. This\n approach tests a system's actual output for each evaluation run.\n\n You can perform the quality assessment manually, by visually inspecting the\n results. Alternatively, you can automate the assessment by using an\n evaluation metric.\n\n- **Raw evaluation**: This type of evaluation directly assesses the quality of\n inputs without any inference. This approach typically is used with automated\n evaluation using metrics. All required fields for evaluation (e.g., `input`,\n `context`, `output` and `reference`) must be present in the input dataset. This\n is useful when you have data coming from an external source (e.g., collected\n from your production traces) and you want to have an objective measurement of\n the quality of the collected data.\n\n For more information, see the [Advanced use](#advanced-use) section of this\n page.\n\nThis section explains how to perform inference-based evaluation using Genkit.\n\n## Quick start\n\nPerform these steps to get started quickly with Genkit.\n\n### Setup\n\n1. Use an existing Genkit app or create a new one by following our\n [Get started](/go/docs/get-started-go) guide.\n\n2. Add the following code to define a simple RAG application to evaluate. For\n this guide, we use a dummy retriever that always returns the same documents.\n\n ```go\n package main\n\n import (\n \"context\"\n \"fmt\"\n \"log\"\n\n \"github.com/firebase/genkit/go/ai\"\n \"github.com/firebase/genkit/go/genkit\"\n \"github.com/firebase/genkit/go/plugins/googlegenai\"\n )\n\n func main() {\n ctx := context.Background()\n\n // Initialize Genkit\n g, err := genkit.Init(ctx,\n genkit.WithPlugins(&googlegenai.GoogleAI{}),\n genkit.WithDefaultModel(\"googleai/gemini-2.5-flash\"),\n )\n if err != nil {\n log.Fatalf(\"Genkit initialization error: %v\", err)\n }\n\n // Dummy retriever that always returns the same facts\n dummyRetrieverFunc := func(ctx context.Context, req *ai.RetrieverRequest) (*ai.RetrieverResponse, error) {\n facts := []string{\n \"Dog is man's best friend\",\n \"Dogs have evolved and were domesticated from wolves\",\n }\n // Just return facts as documents.\n var docs []*ai.Document\n for _, fact := range facts {\n docs = append(docs, ai.DocumentFromText(fact, nil))\n }\n return &ai.RetrieverResponse{Documents: docs}, nil\n }\n factsRetriever := genkit.DefineRetriever(g, \"local\", \"dogFacts\", dummyRetrieverFunc)\n\n m := googlegenai.GoogleAIModel(g, \"gemini-2.5-flash\")\n if m == nil {\n log.Fatal(\"failed to find model\")\n }\n\n // A simple question-answering flow\n genkit.DefineFlow(g, \"qaFlow\", func(ctx context.Context, query string) (string, error) {\n factDocs, err := ai.Retrieve(ctx, factsRetriever, ai.WithTextDocs(query))\n if err != nil {\n return \"\", fmt.Errorf(\"retrieval failed: %w\", err)\n }\n llmResponse, err := genkit.Generate(ctx, g,\n ai.WithModelName(\"googleai/gemini-2.5-flash\"),\n ai.WithPrompt(\"Answer this question with the given context: %s\", query),\n ai.WithDocs(factDocs.Documents...)\n )\n if err != nil {\n return \"\", fmt.Errorf(\"generation failed: %w\", err)\n }\n return llmResponse.Text(), nil\n })\n }\n ```\n\n3. You can optionally add evaluation metrics to your application to use while\n evaluating. This guide uses the `EvaluatorRegex` metric from the\n `evaluators` package.\n\n ```go\n import (\n \"github.com/firebase/genkit/go/plugins/evaluators\"\n )\n\n func main() {\n // ...\n\n metrics := []evaluators.MetricConfig{\n {\n MetricType: evaluators.EvaluatorRegex,\n },\n }\n\n // Initialize Genkit\n g, err := genkit.Init(ctx,\n genkit.WithPlugins(\n &googlegenai.GoogleAI{},\n &evaluators.GenkitEval{Metrics: metrics}, // Add this plugin\n ),\n genkit.WithDefaultModel(\"googleai/gemini-2.5-flash\"),\n )\n }\n ```\n\n **Note:** Ensure that the `evaluators` package is\n installed in your go project:\n\n ```bash\n go get github.com/firebase/genkit/go/plugins/evaluators\n ```\n\n4. Start your Genkit application.\n\n ```bash\n genkit start -- go run main.go\n ```\n\n### Create a dataset\n\nCreate a dataset to define the examples we want to use for evaluating our flow.\n\n1. Go to the Dev UI at `http://localhost:4000` and click the **Datasets**\n button to open the Datasets page.\n\n2. Click the **Create Dataset** button to open the create dataset dialog.\n\n a. Provide a `datasetId` for your new dataset. This guide uses\n `myFactsQaDataset`.\n\n b. Select `Flow` dataset type.\n\n c. Leave the validation target field empty and click **Save**\n\n3. Your new dataset page appears, showing an empty dataset. Add examples to it\n by following these steps:\n\n a. Click the **Add example** button to open the example editor panel.\n\n b. Only the `Input` field is required. Enter `\"Who is man's best friend?\"`\n in the `Input` field, and click **Save** to add the example has to your\n dataset.\n\n If you have configured the `EvaluatorRegex` metric and would like\n to try it out, you need to specify a Reference string that contains the\n pattern to match the output against. For the preceding input, set the\n `Reference output` text to `\"(?i)dog\"`, which is a case-insensitive regular-\n expression pattern to match the word \"dog\" in the flow output.\n\n c. Repeat steps (a) and (b) a couple of more times to add more examples.\n This guide adds the following example inputs to the dataset:\n\n ```text\n \"Can I give milk to my cats?\"\n \"From which animals did dogs evolve?\"\n ```\n\n If you are using the regular-expression evaluator, use the corresponding\n reference strings:\n\n ```text\n \"(?i)don't know\"\n \"(?i)wolf|wolves\"\n ```\n\n Note that this is a contrived example and the regular-expression\n evaluator may not be the right choice to evaluate the responses\n from `qaFlow`. However, this guide can be applied to any\n Genkit Go evaluator of your choice.\n\n By the end of this step, your dataset should have 3 examples in it, with the\n values mentioned above.\n\n### Run evaluation and view results\n\nTo start evaluating the flow, click the **Run new evaluation** button on your\ndataset page. You can also start a new evaluation from the _Evaluations_ tab.\n\n1. Select the `Flow` radio button to evaluate a flow.\n\n2. Select `qaFlow` as the target flow to evaluate.\n\n3. Select `myFactsQaDataset` as the target dataset to use for evaluation.\n\n4. If you have installed an evaluator metric using Genkit plugins,\n you can see these metrics in this page. Select the metrics that you want to\n use with this evaluation run. This is entirely optional: Omitting this step\n will still return the results in the evaluation run, but without any\n associated metrics.\n\n If you have not provided any reference values and are using the\n `EvaluatorRegex` metric, your evaluation will fail since this metric needs\n reference to be set.\n\n5. Click **Run evaluation** to start evaluation. Depending on the flow\n you're testing, this may take a while. Once the evaluation is complete, a\n success message appears with a link to view the results. Click the link\n to go to the _Evaluation details_ page.\n\nYou can see the details of your evaluation on this page, including original\ninput, extracted context and metrics (if any).\n\n## Core concepts\n\n### Terminology\n\nKnowing the following terms can help ensure that you correctly understand\nthe information provided on this page:\n\n- **Evaluation**: An evaluation is a process that assesses system performance.\n In Genkit, such a system is usually a Genkit primitive, such as a flow or a\n model. An evaluation can be automated or manual (human evaluation).\n\n- **Bulk inference** Inference is the act of running an input on a flow or\n model to get the corresponding output. Bulk inference involves performing\n inference on multiple inputs simultaneously.\n\n- **Metric** An evaluation metric is a criterion on which an inference is\n scored. Examples include accuracy, faithfulness, maliciousness, whether the\n output is in English, etc.\n\n- **Dataset** A dataset is a collection of examples to use for inference-based\n evaluation. A dataset typically consists of `Input` and optional `Reference`\n fields. The `Reference` field does not affect the inference step of evaluation\n but it is passed verbatim to any evaluation metrics. In Genkit, you can create\n a dataset through the Dev UI. There are two types of datasets in Genkit:\n _Flow_ datasets and _Model_ datasets.\n\n## Supported evaluators\n\nGenkit supports several evaluators, some built-in, and others\nprovided externally.\n\n### Genkit evaluators\n\nGenkit includes a small number of built-in evaluators, ported from\nthe [JS evaluators plugin](https://js.api.genkit.dev/enums/_genkit-ai_evaluator.GenkitMetric.html),\nto help you get started:\n\n- EvaluatorDeepEqual -- Checks if the generated output is deep-equal to the\n reference output provided.\n- EvaluatorRegex -- Checks if the generated output matches the regular\n expression provided in the reference field.\n- EvaluatorJsonata -- Checks if the generated output matches the\n [JSONATA](https://jsonata.org/) expression provided in the\n reference field.\n\n## Advanced use\n\nAlong with its basic functionality, Genkit also provides advanced support for\ncertain evaluation use cases.\n\n### Evaluation using the CLI\n\nGenkit CLI provides a rich API for performing evaluation. This is especially\nuseful in environments where the Dev UI is not available (e.g. in a CI/CD\nworkflow).\n\nGenkit CLI provides 3 main evaluation commands: `eval:flow`, `eval:extractData`,\nand `eval:run`.\n\n#### Evaluation `eval:flow` command\n\nThe `eval:flow` command runs inference-based evaluation on an input dataset.\nThis dataset may be provided either as a JSON file or by referencing an existing\ndataset in your Genkit runtime.\n\n```bash\n# Referencing an existing dataset\ngenkit eval:flow qaFlow --input myFactsQaDataset\n\n# or, using a dataset from a file\ngenkit eval:flow qaFlow --input testInputs.json\n```\n\n**Note:** Make sure that you start your genkit app before running these CLI\ncommands.\n\n```bash\ngenkit start -- go run main.go\n```\n\nHere, `testInputs.json` should be an array of objects containing an `input`\nfield and an optional `reference` field, like below:\n\n```json\n[\n {\n \"input\": \"What is the French word for Cheese?\"\n },\n {\n \"input\": \"What green vegetable looks like cauliflower?\",\n \"reference\": \"Broccoli\"\n }\n]\n```\n\nIf your flow requires auth, you may specify it using the `--context` argument:\n\n```bash\ngenkit eval:flow qaFlow --input testInputs.json --context '{\"auth\": {\"email_verified\": true}}'\n```\n\nBy default, the `eval:flow` and `eval:run` commands use all available metrics\nfor evaluation. To run on a subset of the configured evaluators, use the\n`--evaluators` flag and provide a comma-separated list of evaluators by name:\n\n```bash\ngenkit eval:flow qaFlow --input testInputs.json --evaluators=genkitEval/regex,genkitEval/jsonata\n```\n\nYou can view the results of your evaluation run in the Dev UI at\n`localhost:4000/evaluate`.\n\n#### `eval:extractData` and `eval:run` commands\n\nTo support _raw evaluation_, Genkit provides tools to extract data from traces\nand run evaluation metrics on extracted data. This is useful, for example, if\nyou are using a different framework for evaluation or if you are collecting\ninferences from a different environment to test locally for output quality.\n\nYou can batch run your Genkit flow and extract an _evaluation dataset_ from the\nresultant traces. A raw evaluation dataset is a collection of inputs for\nevaluation metrics, _without_ running any prior inference.\n\nRun your flow over your test inputs:\n\n```bash\ngenkit flow:batchRun qaFlow testInputs.json\n```\n\nExtract the evaluation data:\n\n```bash\ngenkit eval:extractData qaFlow --maxRows 2 --output factsEvalDataset.json\n```\n\nThe exported data has a format different from the dataset format presented\nearlier. This is because this data is intended to be used with evaluation\nmetrics directly, without any inference step. Here is the syntax of the\nextracted data.\n\n```json\nArray<{\n \"testCaseId\": string,\n \"input\": any,\n \"output\": any,\n \"context\": any[],\n \"traceIds\": string[],\n}>;\n```\n\nThe data extractor automatically locates retrievers and adds the produced docs\nto the context array. You can run evaluation metrics on this extracted dataset\nusing the `eval:run` command.\n\n```bash\ngenkit eval:run factsEvalDataset.json\n```\n\nBy default, `eval:run` runs against all configured evaluators, and as with\n`eval:flow`, results for `eval:run` appear in the evaluation page of Developer\nUI, located at `localhost:4000/evaluate`.\n", "title": "Evaluation", - "lang": "go" + "description": "Learn how to evaluate Genkit Go flows and models using built-in and third-party tools.", + "lang": "go", + "headers": "### Types of evaluation\n## Quick start\n### Setup\n### Create a dataset\n### Run evaluation and view results\n## Core concepts\n### Terminology\n## Supported evaluators\n### Genkit evaluators\n## Advanced use\n### Evaluation using the CLI\n#### Evaluation `eval:flow` command\n# Referencing an existing dataset\n# or, using a dataset from a file\n#### `eval:extractData` and `eval:run` commands\n" }, "go/flows.md": { - "text": "The core of your app's AI features is generative model requests, but it's rare\nthat you can simply take user input, pass it to the model, and display the model\noutput back to the user. Usually, there are pre- and post-processing steps that\nmust accompany the model call. For example:\n\n- Retrieving contextual information to send with the model call.\n- Retrieving the history of the user's current session, for example in a chat\n app.\n- Using one model to reformat the user input in a way that's suitable to pass\n to another model.\n- Evaluating the \"safety\" of a model's output before presenting it to the\n user.\n- Combining the output of several models.\n\nEvery step of this workflow must work together for any AI-related task to\nsucceed.\n\nIn Genkit, you represent this tightly-linked logic using a construction called a\nflow. Flows are written just like functions, using ordinary Go code, but\nthey add additional capabilities intended to ease the development of AI\nfeatures:\n\n- **Type safety**: Input and output schemas, which provides both static and\n runtime type checking.\n- **Integration with developer UI**: Debug flows independently of your\n application code using the developer UI. In the developer UI, you can run\n flows and view traces for each step of the flow.\n- **Simplified deployment**: Deploy flows directly as web API endpoints, using\n any platform that can host a web app.\n\nGenkit's flows are lightweight and unobtrusive, and don't force your app to\nconform to any specific abstraction. All of the flow's logic is written in\nstandard Go, and code inside a flow doesn't need to be flow-aware.\n\n## Defining and calling flows\n\nIn its simplest form, a flow just wraps a function. The following example wraps\na function that calls `genkit.Generate()`:\n\n```go\nmenuSuggestionFlow := genkit.DefineFlow(g, \"menuSuggestionFlow\",\n func(ctx context.Context, theme string) (string, error) {\n resp, err := genkit.Generate(ctx, g,\n ai.WithPrompt(\"Invent a menu item for a %s themed restaurant.\", theme),\n )\n if err != nil {\n return \"\", err\n }\n\n return resp.Text(), nil\n })\n```\n\nJust by wrapping your `genkit.Generate()` calls like this, you add some\nfunctionality: Doing so lets you run the flow from the Genkit CLI and from the\ndeveloper UI, and is a requirement for several of Genkit's features,\nincluding deployment and observability (later sections discuss these topics).\n\n### Input and output schemas\n\nOne of the most important advantages Genkit flows have over directly calling a\nmodel API is type safety of both inputs and outputs. When defining flows, you\ncan define schemas, in much the same way as you define the output schema of a\n`genkit.Generate()` call; however, unlike with `genkit.Generate()`, you can also\nspecify an input schema.\n\nHere's a refinement of the last example, which defines a flow that takes a\nstring as input and outputs an object:\n\n```go\ntype MenuItem struct {\n Name string `json:\"name\"`\n Description string `json:\"description\"`\n}\n\nmenuSuggestionFlow := genkit.DefineFlow(g, \"menuSuggestionFlow\",\n func(ctx context.Context, theme string) (MenuItem, error) {\n return genkit.GenerateData[MenuItem](ctx, g,\n ai.WithPrompt(\"Invent a menu item for a %s themed restaurant.\", theme),\n )\n })\n```\n\nNote that the schema of a flow does not necessarily have to line up with the\nschema of the `genkit.Generate()` calls within the flow (in fact, a flow might\nnot even contain `genkit.Generate()` calls). Here's a variation of the example\nthat calls `genkit.GenerateData()`, but uses the structured\noutput to format a simple string, which the flow returns. Note how we pass\n`MenuItem` as a type parameter; this is the equivalent of passing the\n`WithOutputType()` option and getting a value of that type in response.\n\n```go\ntype MenuItem struct {\n Name string `json:\"name\"`\n Description string `json:\"description\"`\n}\n\nmenuSuggestionMarkdownFlow := genkit.DefineFlow(g, \"menuSuggestionMarkdownFlow\",\n func(ctx context.Context, theme string) (string, error) {\n item, _, err := genkit.GenerateData[MenuItem](ctx, g,\n ai.WithPrompt(\"Invent a menu item for a %s themed restaurant.\", theme),\n )\n if err != nil {\n return \"\", err\n }\n\n return fmt.Sprintf(\"**%s**: %s\", item.Name, item.Description), nil\n })\n```\n\n### Calling flows\n\nOnce you've defined a flow, you can call it from your Go code:\n\n```go\nitem, err := menuSuggestionFlow.Run(context.Background(), \"bistro\")\n```\n\nThe argument to the flow must conform to the input schema.\n\nIf you defined an output schema, the flow response will conform to it. For\nexample, if you set the output schema to `MenuItem`, the flow output will\ncontain its properties:\n\n```go\nitem, err := menuSuggestionFlow.Run(context.Background(), \"bistro\")\nif err != nil {\n log.Fatal(err)\n}\n\nlog.Println(item.Name)\nlog.Println(item.Description)\n```\n\n## Streaming flows\n\nFlows support streaming using an interface similar to `genkit.Generate()`'s\nstreaming interface. Streaming is useful when your flow generates a large\namount of output, because you can present the output to the user as it's being\ngenerated, which improves the perceived responsiveness of your app. As a\nfamiliar example, chat-based LLM interfaces often stream their responses to the\nuser as they are generated.\n\nHere's an example of a flow that supports streaming:\n\n```go\ntype Menu struct {\n Theme string `json:\"theme\"`\n Items []MenuItem `json:\"items\"`\n}\n\ntype MenuItem struct {\n Name string `json:\"name\"`\n Description string `json:\"description\"`\n}\n\nmenuSuggestionFlow := genkit.DefineStreamingFlow(g, \"menuSuggestionFlow\",\n func(ctx context.Context, theme string, callback core.StreamCallback[string]) (Menu, error) {\n item, _, err := genkit.GenerateData[MenuItem](ctx, g,\n ai.WithPrompt(\"Invent a menu item for a %s themed restaurant.\", theme),\n ai.WithStreaming(func(ctx context.Context, chunk *ai.ModelResponseChunk) error {\n // Here, you could process the chunk in some way before sending it to\n // the output stream using StreamCallback. In this example, we output\n // the text of the chunk, unmodified.\n return callback(ctx, chunk.Text())\n }),\n )\n if err != nil {\n return nil, err\n }\n\n return Menu{\n Theme: theme,\n Items: []MenuItem{item},\n }, nil\n })\n```\n\nThe `string` type in `StreamCallback[string]` specifies the type of\nvalues your flow streams. This does not necessarily need to be the same\ntype as the return type, which is the type of the flow's complete output\n(`Menu` in this example).\n\nIn this example, the values streamed by the flow are directly coupled to\nthe values streamed by the `genkit.Generate()` call inside the flow.\nAlthough this is often the case, it doesn't have to be: you can output values\nto the stream using the callback as often as is useful for your flow.\n\n### Calling streaming flows\n\nStreaming flows can be run like non-streaming flows with\n`menuSuggestionFlow.Run(ctx, \"bistro\")` or they can be streamed:\n\n```go\nstreamCh, err := menuSuggestionFlow.Stream(context.Background(), \"bistro\")\nif err != nil {\n log.Fatal(err)\n}\n\nfor result := range streamCh {\n if result.Err != nil {\n log.Fatal(\"Stream error: %v\", result.Err)\n }\n if result.Done {\n log.Printf(\"Menu with %s theme:\\n\", result.Output.Theme)\n for item := range result.Output.Items {\n log.Println(\" - %s: %s\", item.Name, item.Description)\n }\n } else {\n log.Println(\"Stream chunk:\", result.Stream)\n }\n}\n```\n\n## Running flows from the command line\n\nYou can run flows from the command line using the Genkit CLI tool:\n\n```bash\ngenkit flow:run menuSuggestionFlow '\"French\"'\n```\n\nFor streaming flows, you can print the streaming output to the console by adding\nthe `-s` flag:\n\n```bash\ngenkit flow:run menuSuggestionFlow '\"French\"' -s\n```\n\nRunning a flow from the command line is useful for testing a flow, or for\nrunning flows that perform tasks needed on an ad hoc basis—for example, to\nrun a flow that ingests a document into your vector database.\n\n## Debugging flows\n\nOne of the advantages of encapsulating AI logic within a flow is that you can\ntest and debug the flow independently from your app using the Genkit developer\nUI.\n\nThe developer UI relies on the Go app continuing to run, even if the logic has\ncompleted. If you are just getting started and Genkit is not part of a broader\napp, add `select {}` as the last line of `main()` to prevent the app from\nshutting down so that you can inspect it in the UI.\n\nTo start the developer UI, run the following command from your project\ndirectory:\n\n```bash\ngenkit start -- go run .\n```\n\nFrom the **Run** tab of developer UI, you can run any of the flows defined in\nyour project:\n\n![Screenshot of the Flow runner](./resources/devui-flows.png)\n\nAfter you've run a flow, you can inspect a trace of the flow invocation by\neither clicking **View trace** or looking at the **Inspect** tab.\n\n## Deploying flows\n\nYou can deploy your flows directly as web API endpoints, ready for you to call\nfrom your app clients. Deployment is discussed in detail on several other pages,\nbut this section gives brief overviews of your deployment options.\n\n### `net/http` Server\n\nTo deploy a flow using any Go hosting platform, such as Cloud Run, define\nyour flow using `genkit.DefineFlow()` and start a `net/http` server with the\nprovided flow handler using `genkit.Handler()`:\n\n```go\npackage main\n\nimport (\n\t\"context\"\n\t\"log\"\n\t\"net/http\"\n\n\t\"github.com/firebase/genkit/go/ai\"\n\t\"github.com/firebase/genkit/go/genkit\"\n\t\"github.com/firebase/genkit/go/plugins/googlegenai\"\n\t\"github.com/firebase/genkit/go/plugins/server\"\n)\n\ntype MenuItem struct {\n\tName string `json:\"name\"`\n\tDescription string `json:\"description\"`\n}\n\nfunc main() {\n\tctx := context.Background()\n\n\tg, err := genkit.Init(ctx, genkit.WithPlugins(&googlegenai.GoogleAI{}))\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\tmenuSuggestionFlow := genkit.DefineFlow(g, \"menuSuggestionFlow\",\n\t\tfunc(ctx context.Context, theme string) (MenuItem, error) {\n\t\t\titem, _, err := genkit.GenerateData[MenuItem](ctx, g,\n\t\t\t\tai.WithPrompt(\"Invent a menu item for a %s themed restaurant.\", theme),\n\t\t\t)\n\t\t\treturn item, err\n\t\t})\n\n\tmux := http.NewServeMux()\n\tmux.HandleFunc(\"POST /menuSuggestionFlow\", genkit.Handler(menuSuggestionFlow))\n\tlog.Fatal(server.Start(ctx, \"127.0.0.1:3400\", mux))\n}\n```\n\n`server.Start()` is an optional helper function that starts the server and\nmanages its lifecycle, including capturing interrupt signals to ease local\ndevelopment, but you may use your own method.\n\nTo serve all the flows defined in your codebase, you can use\n`genkit.ListFlows()`:\n\n```go\nmux := http.NewServeMux()\nfor _, flow := range genkit.ListFlows(g) {\n mux.HandleFunc(\"POST /\"+flow.Name(), genkit.Handler(flow))\n}\nlog.Fatal(server.Start(ctx, \"127.0.0.1:3400\", mux))\n```\n\nYou can call a flow endpoint with a POST request as follows:\n\n```bash\ncurl -X POST \"http://localhost:3400/menuSuggestionFlow\" \\\n -H \"Content-Type: application/json\" -d '{\"data\": \"banana\"}'\n```\n\n### Other server frameworks\n\nYou can also use other server frameworks to deploy your flows. For\nexample, you can use [Gin](https://gin-gonic.com/) with just a few lines:\n\n```go\nrouter := gin.Default()\nfor _, flow := range genkit.ListFlows(g) {\n router.POST(\"/\"+flow.Name(), func(c *gin.Context) {\n genkit.Handler(flow)(c.Writer, c.Request)\n })\n}\nlog.Fatal(router.Run(\":3400\"))\n```\n\nFor information on deploying to specific platforms, see\n[Genkit with Cloud Run](/go/docs/cloud-run).\n", + "text": "# Defining AI workflows\n\nThe core of your app's AI features is generative model requests, but it's rare\nthat you can simply take user input, pass it to the model, and display the model\noutput back to the user. Usually, there are pre- and post-processing steps that\nmust accompany the model call. For example:\n\n- Retrieving contextual information to send with the model call.\n- Retrieving the history of the user's current session, for example in a chat\n app.\n- Using one model to reformat the user input in a way that's suitable to pass\n to another model.\n- Evaluating the \"safety\" of a model's output before presenting it to the\n user.\n- Combining the output of several models.\n\nEvery step of this workflow must work together for any AI-related task to\nsucceed.\n\nIn Genkit, you represent this tightly-linked logic using a construction called a\nflow. Flows are written just like functions, using ordinary Go code, but\nthey add additional capabilities intended to ease the development of AI\nfeatures:\n\n- **Type safety**: Input and output schemas, which provides both static and\n runtime type checking.\n- **Integration with developer UI**: Debug flows independently of your\n application code using the developer UI. In the developer UI, you can run\n flows and view traces for each step of the flow.\n- **Simplified deployment**: Deploy flows directly as web API endpoints, using\n any platform that can host a web app.\n\nGenkit's flows are lightweight and unobtrusive, and don't force your app to\nconform to any specific abstraction. All of the flow's logic is written in\nstandard Go, and code inside a flow doesn't need to be flow-aware.\n\n## Defining and calling flows\n\nIn its simplest form, a flow just wraps a function. The following example wraps\na function that calls `genkit.Generate()`:\n\n```go\nmenuSuggestionFlow := genkit.DefineFlow(g, \"menuSuggestionFlow\",\n func(ctx context.Context, theme string) (string, error) {\n resp, err := genkit.Generate(ctx, g,\n ai.WithPrompt(\"Invent a menu item for a %s themed restaurant.\", theme),\n )\n if err != nil {\n return \"\", err\n }\n\n return resp.Text(), nil\n })\n```\n\nJust by wrapping your `genkit.Generate()` calls like this, you add some\nfunctionality: Doing so lets you run the flow from the Genkit CLI and from the\ndeveloper UI, and is a requirement for several of Genkit's features,\nincluding deployment and observability (later sections discuss these topics).\n\n### Input and output schemas\n\nOne of the most important advantages Genkit flows have over directly calling a\nmodel API is type safety of both inputs and outputs. When defining flows, you\ncan define schemas, in much the same way as you define the output schema of a\n`genkit.Generate()` call; however, unlike with `genkit.Generate()`, you can also\nspecify an input schema.\n\nHere's a refinement of the last example, which defines a flow that takes a\nstring as input and outputs an object:\n\n```go\ntype MenuItem struct {\n Name string `json:\"name\"`\n Description string `json:\"description\"`\n}\n\nmenuSuggestionFlow := genkit.DefineFlow(g, \"menuSuggestionFlow\",\n func(ctx context.Context, theme string) (MenuItem, error) {\n return genkit.GenerateData[MenuItem](ctx, g,\n ai.WithPrompt(\"Invent a menu item for a %s themed restaurant.\", theme),\n )\n })\n```\n\nNote that the schema of a flow does not necessarily have to line up with the\nschema of the `genkit.Generate()` calls within the flow (in fact, a flow might\nnot even contain `genkit.Generate()` calls). Here's a variation of the example\nthat calls `genkit.GenerateData()`, but uses the structured\noutput to format a simple string, which the flow returns. Note how we pass\n`MenuItem` as a type parameter; this is the equivalent of passing the\n`WithOutputType()` option and getting a value of that type in response.\n\n```go\ntype MenuItem struct {\n Name string `json:\"name\"`\n Description string `json:\"description\"`\n}\n\nmenuSuggestionMarkdownFlow := genkit.DefineFlow(g, \"menuSuggestionMarkdownFlow\",\n func(ctx context.Context, theme string) (string, error) {\n item, _, err := genkit.GenerateData[MenuItem](ctx, g,\n ai.WithPrompt(\"Invent a menu item for a %s themed restaurant.\", theme),\n )\n if err != nil {\n return \"\", err\n }\n\n return fmt.Sprintf(\"**%s**: %s\", item.Name, item.Description), nil\n })\n```\n\n### Calling flows\n\nOnce you've defined a flow, you can call it from your Go code:\n\n```go\nitem, err := menuSuggestionFlow.Run(context.Background(), \"bistro\")\n```\n\nThe argument to the flow must conform to the input schema.\n\nIf you defined an output schema, the flow response will conform to it. For\nexample, if you set the output schema to `MenuItem`, the flow output will\ncontain its properties:\n\n```go\nitem, err := menuSuggestionFlow.Run(context.Background(), \"bistro\")\nif err != nil {\n log.Fatal(err)\n}\n\nlog.Println(item.Name)\nlog.Println(item.Description)\n```\n\n## Streaming flows\n\nFlows support streaming using an interface similar to `genkit.Generate()`'s\nstreaming interface. Streaming is useful when your flow generates a large\namount of output, because you can present the output to the user as it's being\ngenerated, which improves the perceived responsiveness of your app. As a\nfamiliar example, chat-based LLM interfaces often stream their responses to the\nuser as they are generated.\n\nHere's an example of a flow that supports streaming:\n\n```go\ntype Menu struct {\n Theme string `json:\"theme\"`\n Items []MenuItem `json:\"items\"`\n}\n\ntype MenuItem struct {\n Name string `json:\"name\"`\n Description string `json:\"description\"`\n}\n\nmenuSuggestionFlow := genkit.DefineStreamingFlow(g, \"menuSuggestionFlow\",\n func(ctx context.Context, theme string, callback core.StreamCallback[string]) (Menu, error) {\n item, _, err := genkit.GenerateData[MenuItem](ctx, g,\n ai.WithPrompt(\"Invent a menu item for a %s themed restaurant.\", theme),\n ai.WithStreaming(func(ctx context.Context, chunk *ai.ModelResponseChunk) error {\n // Here, you could process the chunk in some way before sending it to\n // the output stream using StreamCallback. In this example, we output\n // the text of the chunk, unmodified.\n return callback(ctx, chunk.Text())\n }),\n )\n if err != nil {\n return nil, err\n }\n\n return Menu{\n Theme: theme,\n Items: []MenuItem{item},\n }, nil\n })\n```\n\nThe `string` type in `StreamCallback[string]` specifies the type of\nvalues your flow streams. This does not necessarily need to be the same\ntype as the return type, which is the type of the flow's complete output\n(`Menu` in this example).\n\nIn this example, the values streamed by the flow are directly coupled to\nthe values streamed by the `genkit.Generate()` call inside the flow.\nAlthough this is often the case, it doesn't have to be: you can output values\nto the stream using the callback as often as is useful for your flow.\n\n### Calling streaming flows\n\nStreaming flows can be run like non-streaming flows with\n`menuSuggestionFlow.Run(ctx, \"bistro\")` or they can be streamed:\n\n```go\nstreamCh, err := menuSuggestionFlow.Stream(context.Background(), \"bistro\")\nif err != nil {\n log.Fatal(err)\n}\n\nfor result := range streamCh {\n if result.Err != nil {\n log.Fatal(\"Stream error: %v\", result.Err)\n }\n if result.Done {\n log.Printf(\"Menu with %s theme:\\n\", result.Output.Theme)\n for item := range result.Output.Items {\n log.Println(\" - %s: %s\", item.Name, item.Description)\n }\n } else {\n log.Println(\"Stream chunk:\", result.Stream)\n }\n}\n```\n\n## Running flows from the command line\n\nYou can run flows from the command line using the Genkit CLI tool:\n\n```bash\ngenkit flow:run menuSuggestionFlow '\"French\"'\n```\n\nFor streaming flows, you can print the streaming output to the console by adding\nthe `-s` flag:\n\n```bash\ngenkit flow:run menuSuggestionFlow '\"French\"' -s\n```\n\nRunning a flow from the command line is useful for testing a flow, or for\nrunning flows that perform tasks needed on an ad hoc basis—for example, to\nrun a flow that ingests a document into your vector database.\n\n## Debugging flows\n\nOne of the advantages of encapsulating AI logic within a flow is that you can\ntest and debug the flow independently from your app using the Genkit developer\nUI.\n\nThe developer UI relies on the Go app continuing to run, even if the logic has\ncompleted. If you are just getting started and Genkit is not part of a broader\napp, add `select {}` as the last line of `main()` to prevent the app from\nshutting down so that you can inspect it in the UI.\n\nTo start the developer UI, run the following command from your project\ndirectory:\n\n```bash\ngenkit start -- go run .\n```\n\nFrom the **Run** tab of developer UI, you can run any of the flows defined in\nyour project:\n\n![Screenshot of the Flow runner](./resources/devui-flows.png)\n\nAfter you've run a flow, you can inspect a trace of the flow invocation by\neither clicking **View trace** or looking at the **Inspect** tab.\n\n## Deploying flows\n\nYou can deploy your flows directly as web API endpoints, ready for you to call\nfrom your app clients. Deployment is discussed in detail on several other pages,\nbut this section gives brief overviews of your deployment options.\n\n### `net/http` Server\n\nTo deploy a flow using any Go hosting platform, such as Cloud Run, define\nyour flow using `genkit.DefineFlow()` and start a `net/http` server with the\nprovided flow handler using `genkit.Handler()`:\n\n```go\npackage main\n\nimport (\n\t\"context\"\n\t\"log\"\n\t\"net/http\"\n\n\t\"github.com/firebase/genkit/go/ai\"\n\t\"github.com/firebase/genkit/go/genkit\"\n\t\"github.com/firebase/genkit/go/plugins/googlegenai\"\n\t\"github.com/firebase/genkit/go/plugins/server\"\n)\n\ntype MenuItem struct {\n\tName string `json:\"name\"`\n\tDescription string `json:\"description\"`\n}\n\nfunc main() {\n\tctx := context.Background()\n\n\tg, err := genkit.Init(ctx, genkit.WithPlugins(&googlegenai.GoogleAI{}))\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\tmenuSuggestionFlow := genkit.DefineFlow(g, \"menuSuggestionFlow\",\n\t\tfunc(ctx context.Context, theme string) (MenuItem, error) {\n\t\t\titem, _, err := genkit.GenerateData[MenuItem](ctx, g,\n\t\t\t\tai.WithPrompt(\"Invent a menu item for a %s themed restaurant.\", theme),\n\t\t\t)\n\t\t\treturn item, err\n\t\t})\n\n\tmux := http.NewServeMux()\n\tmux.HandleFunc(\"POST /menuSuggestionFlow\", genkit.Handler(menuSuggestionFlow))\n\tlog.Fatal(server.Start(ctx, \"127.0.0.1:3400\", mux))\n}\n```\n\n`server.Start()` is an optional helper function that starts the server and\nmanages its lifecycle, including capturing interrupt signals to ease local\ndevelopment, but you may use your own method.\n\nTo serve all the flows defined in your codebase, you can use\n`genkit.ListFlows()`:\n\n```go\nmux := http.NewServeMux()\nfor _, flow := range genkit.ListFlows(g) {\n mux.HandleFunc(\"POST /\"+flow.Name(), genkit.Handler(flow))\n}\nlog.Fatal(server.Start(ctx, \"127.0.0.1:3400\", mux))\n```\n\nYou can call a flow endpoint with a POST request as follows:\n\n```bash\ncurl -X POST \"http://localhost:3400/menuSuggestionFlow\" \\\n -H \"Content-Type: application/json\" -d '{\"data\": \"banana\"}'\n```\n\n### Other server frameworks\n\nYou can also use other server frameworks to deploy your flows. For\nexample, you can use [Gin](https://gin-gonic.com/) with just a few lines:\n\n```go\nrouter := gin.Default()\nfor _, flow := range genkit.ListFlows(g) {\n router.POST(\"/\"+flow.Name(), func(c *gin.Context) {\n genkit.Handler(flow)(c.Writer, c.Request)\n })\n}\nlog.Fatal(router.Run(\":3400\"))\n```\n\nFor information on deploying to specific platforms, see\n[Genkit with Cloud Run](/go/docs/cloud-run).\n", "title": "Defining AI workflows", - "lang": "go" + "description": "Learn how to define and use Genkit flows in Go to structure your AI logic.", + "lang": "go", + "headers": "## Defining and calling flows\n### Input and output schemas\n### Calling flows\n## Streaming flows\n### Calling streaming flows\n## Running flows from the command line\n## Debugging flows\n## Deploying flows\n### `net/http` Server\n### Other server frameworks\n" }, "go/get-started-go.md": { - "text": "This guide shows you how to get started with Genkit in a Go app.\n\nIf you discover issues with the libraries or this documentation please report\nthem in our [GitHub repository](https://github.com/firebase/genkit/).\n\n## Make your first request\n\n1. Install Go 1.24 or later. See [Download and install](https://go.dev/doc/install)\n in the official Go docs.\n\n2. Initialize a new Go project directory with the Genkit package:\n\n ```bash\n mkdir genkit-intro && cd genkit-intro\n\n go mod init example/genkit-intro\n\n go get github.com/firebase/genkit/go\n ```\n\n3. Create a `main.go` file with the following sample code:\n\n ```go\n package main\n\n import (\n \"context\"\n \"log\"\n\n \"github.com/firebase/genkit/go/ai\"\n \"github.com/firebase/genkit/go/genkit\"\n \"github.com/firebase/genkit/go/plugins/googlegenai\"\n )\n\n func main() {\n ctx := context.Background()\n\n // Initialize Genkit with the Google AI plugin and Gemini 2.0 Flash.\n g, err := genkit.Init(ctx,\n genkit.WithPlugins(&googlegenai.GoogleAI{}),\n genkit.WithDefaultModel(\"googleai/gemini-2.5-flash\"),\n )\n if err != nil {\n log.Fatalf(\"could not initialize Genkit: %v\", err)\n }\n\n resp, err := genkit.Generate(ctx, g, ai.WithPrompt(\"What is the meaning of life?\"))\n if err != nil {\n log.Fatalf(\"could not generate model response: %v\", err)\n }\n\n log.Println(resp.Text())\n }\n ```\n\n4. Configure your Gemini API key by setting the `GEMINI_API_KEY` environment\n variable:\n\n ```bash\n export GEMINI_API_KEY=\n ```\n\n If you don't already have one, [create a key in Google AI Studio](https://aistudio.google.com/apikey).\n Google AI provides a generous free-of-charge tier and does not require a\n credit card to get started.\n\n5. Run the app to see the model response:\n\n ```bash\n go run .\n # Example output (may vary):\n # There is no single universally agreed-upon meaning of life; it's a deeply\n # personal question. Many find meaning through connection, growth,\n # contribution, happiness, or discovering their own purpose.\n ```\n\n## Next steps\n\nNow that you're set up to make model requests with Genkit, learn how to use more\nGenkit capabilities to build your AI-powered apps and workflows. To get started\nwith additional Genkit capabilities, see the following guides:\n\n- [Developer tools](/docs/devtools): Learn how to set up and use\n Genkit's CLI and developer UI to help you locally test and debug your app.\n- [Generating content](/go/docs/models): Learn how to use Genkit's\n unified generation API to generate text and structured data from any\n supported model.\n- [Creating flows](/go/docs/flows): Learn how to use special Genkit\n functions, called flows, that provide end-to-end observability for workflows\n and rich debugging from Genkit tooling.\n- [Managing prompts](/go/docs/dotprompt): Learn how Genkit helps you\n manage your prompts and configuration together as code.\n", + "text": "# Get started with Genkit using Go\n\nThis guide shows you how to get started with Genkit in a Go app.\n\nIf you discover issues with the libraries or this documentation please report\nthem in our [GitHub repository](https://github.com/firebase/genkit/).\n\n## Make your first request\n\n1. Install Go 1.24 or later. See [Download and install](https://go.dev/doc/install)\n in the official Go docs.\n\n2. Initialize a new Go project directory with the Genkit package:\n\n ```bash\n mkdir genkit-intro && cd genkit-intro\n\n go mod init example/genkit-intro\n\n go get github.com/firebase/genkit/go\n ```\n\n3. Create a `main.go` file with the following sample code:\n\n ```go\n package main\n\n import (\n \"context\"\n \"log\"\n\n \"github.com/firebase/genkit/go/ai\"\n \"github.com/firebase/genkit/go/genkit\"\n \"github.com/firebase/genkit/go/plugins/googlegenai\"\n )\n\n func main() {\n ctx := context.Background()\n\n // Initialize Genkit with the Google AI plugin and Gemini 2.0 Flash.\n g, err := genkit.Init(ctx,\n genkit.WithPlugins(&googlegenai.GoogleAI{}),\n genkit.WithDefaultModel(\"googleai/gemini-2.5-flash\"),\n )\n if err != nil {\n log.Fatalf(\"could not initialize Genkit: %v\", err)\n }\n\n resp, err := genkit.Generate(ctx, g, ai.WithPrompt(\"What is the meaning of life?\"))\n if err != nil {\n log.Fatalf(\"could not generate model response: %v\", err)\n }\n\n log.Println(resp.Text())\n }\n ```\n\n4. Configure your Gemini API key by setting the `GEMINI_API_KEY` environment\n variable:\n\n ```bash\n export GEMINI_API_KEY=\n ```\n\n If you don't already have one, [create a key in Google AI Studio](https://aistudio.google.com/apikey).\n Google AI provides a generous free-of-charge tier and does not require a\n credit card to get started.\n\n5. Run the app to see the model response:\n\n ```bash\n go run .\n # Example output (may vary):\n # There is no single universally agreed-upon meaning of life; it's a deeply\n # personal question. Many find meaning through connection, growth,\n # contribution, happiness, or discovering their own purpose.\n ```\n\n## Next steps\n\nNow that you're set up to make model requests with Genkit, learn how to use more\nGenkit capabilities to build your AI-powered apps and workflows. To get started\nwith additional Genkit capabilities, see the following guides:\n\n- [Developer tools](/docs/devtools): Learn how to set up and use\n Genkit's CLI and developer UI to help you locally test and debug your app.\n- [Generating content](/go/docs/models): Learn how to use Genkit's\n unified generation API to generate text and structured data from any\n supported model.\n- [Creating flows](/go/docs/flows): Learn how to use special Genkit\n functions, called flows, that provide end-to-end observability for workflows\n and rich debugging from Genkit tooling.\n- [Managing prompts](/go/docs/dotprompt): Learn how Genkit helps you\n manage your prompts and configuration together as code.\n", "title": "Get started with Genkit using Go", - "lang": "go" + "description": "Learn how to set up Genkit and make your first generative AI request in a Go application.", + "lang": "go", + "headers": "## Make your first request\n## Next steps\n" }, "go/models.md": { - "text": "At the heart of generative AI are AI _models_. The two most prominent\nexamples of generative models are large language models (LLMs) and image\ngeneration models. These models take input, called a _prompt_ (most commonly\ntext, an image, or a combination of both), and from it produce as output text,\nan image, or even audio or video.\n\nThe output of these models can be surprisingly convincing: LLMs generate text\nthat appears as though it could have been written by a human being, and image\ngeneration models can produce images that are very close to real photographs or\nartwork created by humans.\n\nIn addition, LLMs have proven capable of tasks beyond simple text generation:\n\n- Writing computer programs.\n- Planning subtasks that are required to complete a larger task.\n- Organizing unorganized data.\n- Understanding and extracting information data from a corpus of text.\n- Following and performing automated activities based on a text description of\n the activity.\n\nThere are many models available to you, from several different providers. Each\nmodel has its own strengths and weaknesses and one model might excel at one task\nbut perform less well at others. Apps making use of generative AI can often\nbenefit from using multiple different models depending on the task at hand.\n\nAs an app developer, you typically don't interact with generative AI models\ndirectly, but rather through services available as web APIs. Although these\nservices often have similar functionality, they all provide them through\ndifferent and incompatible APIs. If you want to make use of multiple model\nservices, you have to use each of their proprietary SDKs, potentially\nincompatible with each other. And if you want to upgrade from one model to the\nnewest and most capable one, you might have to build that integration all over\nagain.\n\nGenkit addresses this challenge by providing a single interface that abstracts\naway the details of accessing potentially any generative AI model service, with\nseveral prebuilt implementations already available. Building your AI-powered\napp around Genkit simplifies the process of making your first generative AI call\nand makes it equally straightforward to combine multiple models or swap one\nmodel for another as new models emerge.\n\n### Before you begin\n\nIf you want to run the code examples on this page, first complete the steps in\nthe [Get started](/go/docs/get-started-go) guide. All of the examples assume that you\nhave already installed Genkit as a dependency in your project.\n\n### Models supported by Genkit\n\nGenkit is designed to be flexible enough to use potentially any generative AI\nmodel service. Its core libraries define the common interface for working with\nmodels, and model plugins define the implementation details for working with a\nspecific model and its API.\n\nThe Genkit team maintains plugins for working with models provided by Vertex AI,\nGoogle Generative AI, and Ollama:\n\n- Gemini family of LLMs, through the\n [Google GenAI plugin](/go/docs/plugins/google-genai).\n- Gemma 3, Llama 4, and many more open models, through the\n [Ollama plugin](/go/docs/plugins/ollama)\n (you must host the Ollama server yourself).\n\n### Loading and configuring model plugins\n\nBefore you can use Genkit to start generating content, you need to load and\nconfigure a model plugin. If you're coming from the Get Started guide,\nyou've already done this. Otherwise, see the [Get Started](/go/docs/get-started-go)\nguide or the individual plugin's documentation and follow the steps there before\ncontinuing.\n\n### The `genkit.Generate()` function\n\nIn Genkit, the primary interface through which you interact with generative AI\nmodels is the `genkit.Generate()` function.\n\nThe simplest `genkit.Generate()` call specifies the model you want to use and a\ntext prompt:\n\n```go\npackage main\n\nimport (\n \"context\"\n \"log\"\n\n \"github.com/firebase/genkit/go/ai\"\n \"github.com/firebase/genkit/go/genkit\"\n \"github.com/firebase/genkit/go/plugins/googlegenai\"\n)\n\nfunc main() {\n\tctx := context.Background()\n\n\tg, err := genkit.Init(ctx,\n\t\tgenkit.WithPlugins(&googlegenai.GoogleAI{}),\n\t\tgenkit.WithDefaultModel(\"googleai/gemini-2.5-flash\"),\n\t)\n\tif err != nil {\n\t\tlog.Fatalf(\"could not initialize Genkit: %v\", err)\n\t}\n\n\tresp, err := genkit.Generate(ctx, g,\n\t\tai.WithPrompt(\"Invent a menu item for a pirate themed restaurant.\"),\n\t)\n\tif err != nil {\n\t\tlog.Fatalf(\"could not generate model response: %v\", err)\n\t}\n\n\tlog.Println(resp.Text())\n}\n```\n\nWhen you run this brief example, it will print out some debugging information\nfollowed by the output of the `genkit.Generate()` call, which will usually be\nMarkdown text as in the following example:\n\n```md\n## The Blackheart's Bounty\n\n**A hearty stew of slow-cooked beef, spiced with rum and molasses, served in a\nhollowed-out cannonball with a side of crusty bread and a dollop of tangy\npineapple salsa.**\n\n**Description:** This dish is a tribute to the hearty meals enjoyed by pirates\non the high seas. The beef is tender and flavorful, infused with the warm spices\nof rum and molasses. The pineapple salsa adds a touch of sweetness and acidity,\nbalancing the richness of the stew. The cannonball serving vessel adds a fun and\nthematic touch, making this dish a perfect choice for any pirate-themed\nadventure.\n```\n\nRun the script again and you'll get a different output.\n\nThe preceding code sample sent the generation request to the default model,\nwhich you specified when you configured the Genkit instance.\n\nYou can also specify a model for a single `genkit.Generate()` call:\n\n```go\nresp, err := genkit.Generate(ctx, g,\n\tai.WithModelName(\"googleai/gemini-2.5-pro\"),\n\tai.WithPrompt(\"Invent a menu item for a pirate themed restaurant.\"),\n)\n```\n\nA model string identifier looks like `providerid/modelid`, where the provider ID\n(in this case, `googleai`) identifies the plugin, and the model ID is a\nplugin-specific string identifier for a specific version of a model.\n\nThese examples also illustrate an important point: when you use\n`genkit.Generate()` to make generative AI model calls, changing the model you\nwant to use is a matter of passing a different value to the model\nparameter. By using `genkit.Generate()` instead of the native model SDKs, you\ngive yourself the flexibility to more easily use several different models in\nyour app and change models in the future.\n\nSo far you have only seen examples of the simplest `genkit.Generate()` calls.\nHowever, `genkit.Generate()` also provides an interface for more advanced\ninteractions with generative models, which you will see in the sections that\nfollow.\n\n### System prompts\n\nSome models support providing a _system prompt_, which gives the model\ninstructions as to how you want it to respond to messages from the user. You can\nuse the system prompt to specify characteristics such as a persona you want the\nmodel to adopt, the tone of its responses, and the format of its responses.\n\nIf the model you're using supports system prompts, you can provide one with the\n`ai.WithSystem()` option:\n\n```go\nresp, err := genkit.Generate(ctx, g,\n\tai.WithSystem(\"You are a food industry marketing consultant.\"),\n\tai.WithPrompt(\"Invent a menu item for a pirate themed restaurant.\"),\n)\n```\n\nFor models that don't support system prompts, `ai.WithSystem()` simulates it by\nmodifying the request to appear _like_ a system prompt.\n\n### Model parameters\n\nThe `genkit.Generate()` function takes a `ai.WithConfig()` option, through which\nyou can specify optional settings that control how the model generates content:\n\n```go\nresp, err := genkit.Generate(ctx, g,\n\tai.WithModelName(\"googleai/gemini-2.5-flash\"),\n\tai.WithPrompt(\"Invent a menu item for a pirate themed restaurant.\"),\n\tai.WithConfig(&googlegenai.GeminiConfig{\n\t\tMaxOutputTokens: 500,\n\t\tStopSequences: [\"\", \"\"],\n\t\tTemperature: 0.5,\n\t\tTopP: 0.4,\n\t\tTopK: 50,\n\t}),\n)\n```\n\nThe exact parameters that are supported depend on the individual model and model\nAPI. However, the parameters in the previous example are common to almost every\nmodel. The following is an explanation of these parameters:\n\n#### Parameters that control output length\n\n**MaxOutputTokens**\n\nLLMs operate on units called _tokens_. A token usually, but does not\nnecessarily, map to a specific sequence of characters. When you pass a prompt to\na model, one of the first steps it takes is to _tokenize_ your prompt string\ninto a sequence of tokens. Then, the LLM generates a sequence of tokens from the\ntokenized input. Finally, the sequence of tokens gets converted back into text,\nwhich is your output.\n\nThe maximum output tokens parameter sets a limit on how many tokens to\ngenerate using the LLM. Every model potentially uses a different tokenizer, but\na good rule of thumb is to consider a single English word to be made of 2 to 4\ntokens.\n\nAs stated earlier, some tokens might not map to character sequences. One such\nexample is that there is often a token that indicates the end of the sequence:\nwhen an LLM generates this token, it stops generating more. Therefore, it's\npossible and often the case that an LLM generates fewer tokens than the maximum\nbecause it generated the \"stop\" token.\n\n**StopSequences**\n\nYou can use this parameter to set the tokens or token sequences that, when\ngenerated, indicate the end of LLM output. The correct values to use here\ngenerally depend on how the model was trained, and are usually set by the model\nplugin. However, if you have prompted the model to generate another stop\nsequence, you might specify it here.\n\nNote that you are specifying character sequences, and not tokens per se. In most\ncases, you will specify a character sequence that the model's tokenizer maps to\na single token.\n\n#### Parameters that control \"creativity\"\n\nThe _temperature_, _top-p_, and _top-k_ parameters together control how\n\"creative\" you want the model to be. This section provides very brief\nexplanations of what these parameters mean, but the more important point is\nthis: these parameters are used to adjust the character of an LLM's output. The\noptimal values for them depend on your goals and preferences, and are likely to\nbe found only through experimentation.\n\n**Temperature**\n\nLLMs are fundamentally token-predicting machines. For a given sequence of tokens\n(such as the prompt) an LLM predicts, for each token in its vocabulary, the\nlikelihood that the token comes next in the sequence. The temperature is a\nscaling factor by which these predictions are divided before being normalized to\na probability between 0 and 1.\n\nLow temperature values—between 0.0 and 1.0—amplify the difference in\nlikelihoods between tokens, with the result that the model will be even less\nlikely to produce a token it already evaluated to be unlikely. This is often\nperceived as output that is less creative. Although 0.0 is technically not a\nvalid value, many models treat it as indicating that the model should behave\ndeterministically, and to only consider the single most likely token.\n\nHigh temperature values—those greater than 1.0—compress the\ndifferences in likelihoods between tokens, with the result that the model\nbecomes more likely to produce tokens it had previously evaluated to be\nunlikely. This is often perceived as output that is more creative. Some model\nAPIs impose a maximum temperature, often 2.0.\n\n**TopP**\n\n_Top-p_ is a value between 0.0 and 1.0 that controls the number of possible\ntokens you want the model to consider, by specifying the cumulative probability\nof the tokens. For example, a value of 1.0 means to consider every possible\ntoken (but still take into account the probability of each token). A value of\n0.4 means to only consider the most likely tokens, whose probabilities add up to\n0.4, and to exclude the remaining tokens from consideration.\n\n**TopK**\n\n_Top-k_ is an integer value that also controls the number of possible tokens you\nwant the model to consider, but this time by explicitly specifying the maximum\nnumber of tokens. Specifying a value of 1 means that the model should behave\ndeterministically.\n\n#### Experiment with model parameters\n\nYou can experiment with the effect of these parameters on the output generated\nby different model and prompt combinations by using the Developer UI. Start the\ndeveloper UI with the `genkit start` command and it will automatically load all\nof the models defined by the plugins configured in your project. You can quickly\ntry different prompts and configuration values without having to repeatedly make\nthese changes in code.\n\n#### Pair model with its config\n\nGiven that each provider or even a specific model may have its own configuration\nschema or warrant certain settings, it may be error prone to set separate\noptions using `ai.WithModelName()` and `ai.WithConfig()` since the latter is not\nstrongly typed to the former.\n\nTo pair a model with its config, you can create a model reference that you can\npass into the generate call instead:\n\n```go\nmodel := googlegenai.GoogleAIModelRef(\"gemini-2.5-flash\", &googlegenai.GeminiConfig{\n\tMaxOutputTokens: 500,\n\tStopSequences: [\"\", \"\"],\n\tTemperature: 0.5,\n\tTopP: 0.4,\n\tTopK: 50,\n})\n\nresp, err := genkit.Generate(ctx, g,\n\tai.WithModel(model),\n\tai.WithPrompt(\"Invent a menu item for a pirate themed restaurant.\"),\n)\nif err != nil {\n\tlog.Fatal(err)\n}\n```\n\nThe constructor for the model reference will enforce that the correct config\ntype is provided which may reduce mismatches.\n\n### Structured output\n\nWhen using generative AI as a component in your application, you often want\noutput in a format other than plain text. Even if you're just generating content\nto display to the user, you can benefit from structured output simply for the\npurpose of presenting it more attractively to the user. But for more advanced\napplications of generative AI, such as programmatic use of the model's output,\nor feeding the output of one model into another, structured output is a must.\n\nIn Genkit, you can request structured output from a model by specifying an\noutput type when you call `genkit.Generate()`:\n\n```go\ntype MenuItem struct {\n\tName string `json:\"name\"`\n\tDescription string `json:\"description\"`\n\tCalories int `json:\"calories\"`\n\tAllergens []string `json:\"allergens\"`\n}\n\nresp, err := genkit.Generate(ctx, g,\n\tai.WithPrompt(\"Invent a menu item for a pirate themed restaurant.\"),\n\tai.WithOutputType(MenuItem{}),\n)\nif err != nil {\n\tlog.Fatal(err) // One possible error is that the response does not conform to the type.\n}\n```\n\nModel output types are specified as JSON schema using the\n[`invopop/jsonschema`](https://github.com/invopop/jsonschema) package. This\nprovides runtime type checking, which bridges the gap between static Go types\nand the unpredictable output of generative AI models. This system lets you write\ncode that can rely on the fact that a successful generate call will always\nreturn output that conforms to your Go types.\n\nWhen you specify an output type in `genkit.Generate()`, Genkit does several\nthings behind the scenes:\n\n- Augments the prompt with additional guidance about the selected output\n format. This also has the side effect of specifying to the model what\n content exactly you want to generate (for example, not only suggest a menu\n item but also generate a description, a list of allergens, and so on).\n- Verifies that the output conforms to the schema.\n- Marshals the model output into a Go type.\n\nTo get structured output from a successful generate call, call `Output()` on the\nmodel response with an empty value of the type:\n\n```go\nvar item MenuItem\nif err := resp.Output(&item); err != nil {\n\tlog.Fatalf(err)\n}\n\nlog.Printf(\"%s (%d calories, %d allergens): %s\\n\",\n\titem.Name, item.Calories, len(item.Allergens), item.Description)\n```\n\nAlternatively, you can use `genkit.GenerateData()` for a more succinct call:\n\n```go\nitem, resp, err := genkit.GenerateData[MenuItem](ctx, g,\n\tai.WithPrompt(\"Invent a menu item for a pirate themed restaurant.\"),\n)\nif err != nil {\n\tlog.Fatal(err)\n}\n\nlog.Printf(\"%s (%d calories, %d allergens): %s\\n\",\n\titem.Name, item.Calories, len(item.Allergens), item.Description)\n```\n\nThis function requires the output type parameter but automatically sets the\n`ai.WithOutputType()` option and calls `ModelResponse.Output()` before returning\nthe value.\n\n#### Handling errors\n\nNote in the prior example that the `genkit.Generate()` call can result in an\nerror. One possible error can happen when the model fails to generate output\nthat conforms to the schema. The best strategy for dealing with such errors will\ndepend on your exact use case, but here are some general hints:\n\n- **Try a different model**. For structured output to succeed, the model must\n be capable of generating output in JSON. The most powerful LLMs like Gemini\n are versatile enough to do this; however, smaller models, such as some of\n the local models you would use with Ollama, might not be able to generate\n structured output reliably unless they have been specifically trained to do\n so.\n\n- **Simplify the schema**. LLMs may have trouble generating complex or deeply\n nested types. Try using clear names, fewer fields, or a flattened structure\n if you are not able to reliably generate structured data.\n\n- **Retry the `genkit.Generate()` call**. If the model you've chosen only\n rarely fails to generate conformant output, you can treat the error as you\n would treat a network error, and retry the request using some kind of\n incremental back-off strategy.\n\n### Streaming\n\nWhen generating large amounts of text, you can improve the experience for your\nusers by presenting the output as it's generated—streaming the output. A\nfamiliar example of streaming in action can be seen in most LLM chat apps: users\ncan read the model's response to their message as it's being generated, which\nimproves the perceived responsiveness of the application and enhances the\nillusion of chatting with an intelligent counterpart.\n\nIn Genkit, you can stream output using the `ai.WithStreaming()` option:\n\n```go\nresp, err := genkit.Generate(ctx, g,\n\tai.WithPrompt(\"Suggest a complete menu for a pirate themed restaurant.\"),\n\tai.WithStreaming(func(ctx context.Context, chunk *ai.ModelResponseChunk) error {\n\t\t// Do something with the chunk...\n\t\tlog.Println(chunk.Text())\n\t\treturn nil\n\t}),\n)\nif err != nil {\n\tlog.Fatal(err)\n}\n\nlog.Println(resp.Text())\n```\n\n\n\n### Multimodal input\n\nThe examples you've seen so far have used text strings as model prompts. While\nthis remains the most common way to prompt generative AI models, many models can\nalso accept other media as prompts. Media prompts are most often used in\nconjunction with text prompts that instruct the model to perform some operation\non the media, such as to caption an image or transcribe an audio recording.\n\nThe ability to accept media input and the types of media you can use are\ncompletely dependent on the model and its API. For example, the Gemini 2.0\nseries of models can accept images, video, and audio as prompts.\n\nTo provide a media prompt to a model that supports it, instead of passing a\nsimple text prompt to `genkit.Generate()`, pass an array consisting of a media\npart and a text part. This example specifies an image using a\npublicly-accessible HTTPS URL.\n\n```go\nresp, err := genkit.Generate(ctx, g,\n\tai.WithModelName(\"googleai/gemini-2.5-flash\"),\n\tai.WithMessages(\n\t\tNewUserMessage(\n\t\t\tNewMediaPart(\"image/jpeg\", \"https://example.com/photo.jpg\"),\n\t\t\tNewTextPart(\"Compose a poem about this image.\"),\n\t\t),\n\t),\n)\n```\n\nYou can also pass media data directly by encoding it as a data URL. For\nexample:\n\n```go\nimage, err := ioutil.ReadFile(\"photo.jpg\")\nif err != nil {\n\tlog.Fatal(err)\n}\n\nresp, err := genkit.Generate(ctx, g,\n\tai.WithModelName(\"googleai/gemini-2.5-flash\"),\n\tai.WithMessages(\n\t\tNewUserMessage(\n\t\t\tNewMediaPart(\"image/jpeg\", \"data:image/jpeg;base64,\" + base64.StdEncoding.EncodeToString(image)),\n\t\t\tNewTextPart(\"Compose a poem about this image.\"),\n\t\t),\n\t),\n)\n```\n\nAll models that support media input support both data URLs and HTTPS URLs. Some\nmodel plugins add support for other media sources. For example, the Vertex AI\nplugin also lets you use Cloud Storage (`gs://`) URLs.\n\n\n\n### Next steps\n\n#### Learn more about Genkit\n\n- As an app developer, the primary way you influence the output of generative\n AI models is through prompting. Read\n [Managing prompts with Dotprompt](/go/docs/dotprompt) to learn\n how Genkit helps you develop effective prompts and manage them in your\n codebase.\n- Although `genkit.Generate()` is the nucleus of every generative AI powered\n application, real-world applications usually require additional work before\n and after invoking a generative AI model. To reflect this, Genkit introduces\n the concept of _flows_, which are defined like functions but add additional\n features such as observability and simplified deployment. To learn more, see\n [Defining AI workflows](/go/docs/flows).\n\n#### Advanced LLM use\n\nThere are techniques your app can use to reap even more benefit from LLMs.\n\n- One way to enhance the capabilities of LLMs is to prompt them with a list of\n ways they can request more information from you, or request you to perform\n some action. This is known as _tool calling_ or _function calling_. Models\n that are trained to support this capability can respond to a prompt with a\n specially-formatted response, which indicates to the calling application\n that it should perform some action and send the result back to the LLM along\n with the original prompt. Genkit has library functions that automate both\n the prompt generation and the call-response loop elements of a tool calling\n implementation. See [Tool calling](/go/docs/tool-calling) to learn more.\n- Retrieval-augmented generation (RAG) is a technique used to introduce\n domain-specific information into a model's output. This is accomplished by\n inserting relevant information into a prompt before passing it on to the\n language model. A complete RAG implementation requires you to bring several\n technologies together: text embedding generation models, vector databases, and\n large language models. See [Retrieval-augmented generation (RAG)](/go/docs/rag) to\n learn how Genkit simplifies the process of coordinating these various\n elements.\n", + "text": "# Generating content with AI models\n\nAt the heart of generative AI are AI _models_. The two most prominent\nexamples of generative models are large language models (LLMs) and image\ngeneration models. These models take input, called a _prompt_ (most commonly\ntext, an image, or a combination of both), and from it produce as output text,\nan image, or even audio or video.\n\nThe output of these models can be surprisingly convincing: LLMs generate text\nthat appears as though it could have been written by a human being, and image\ngeneration models can produce images that are very close to real photographs or\nartwork created by humans.\n\nIn addition, LLMs have proven capable of tasks beyond simple text generation:\n\n- Writing computer programs.\n- Planning subtasks that are required to complete a larger task.\n- Organizing unorganized data.\n- Understanding and extracting information data from a corpus of text.\n- Following and performing automated activities based on a text description of\n the activity.\n\nThere are many models available to you, from several different providers. Each\nmodel has its own strengths and weaknesses and one model might excel at one task\nbut perform less well at others. Apps making use of generative AI can often\nbenefit from using multiple different models depending on the task at hand.\n\nAs an app developer, you typically don't interact with generative AI models\ndirectly, but rather through services available as web APIs. Although these\nservices often have similar functionality, they all provide them through\ndifferent and incompatible APIs. If you want to make use of multiple model\nservices, you have to use each of their proprietary SDKs, potentially\nincompatible with each other. And if you want to upgrade from one model to the\nnewest and most capable one, you might have to build that integration all over\nagain.\n\nGenkit addresses this challenge by providing a single interface that abstracts\naway the details of accessing potentially any generative AI model service, with\nseveral prebuilt implementations already available. Building your AI-powered\napp around Genkit simplifies the process of making your first generative AI call\nand makes it equally straightforward to combine multiple models or swap one\nmodel for another as new models emerge.\n\n### Before you begin\n\nIf you want to run the code examples on this page, first complete the steps in\nthe [Get started](/go/docs/get-started-go) guide. All of the examples assume that you\nhave already installed Genkit as a dependency in your project.\n\n### Models supported by Genkit\n\nGenkit is designed to be flexible enough to use potentially any generative AI\nmodel service. Its core libraries define the common interface for working with\nmodels, and model plugins define the implementation details for working with a\nspecific model and its API.\n\nThe Genkit team maintains plugins for working with models provided by Vertex AI,\nGoogle Generative AI, and Ollama:\n\n- Gemini family of LLMs, through the\n [Google GenAI plugin](/go/docs/plugins/google-genai).\n- Gemma 3, Llama 4, and many more open models, through the\n [Ollama plugin](/go/docs/plugins/ollama)\n (you must host the Ollama server yourself).\n\n### Loading and configuring model plugins\n\nBefore you can use Genkit to start generating content, you need to load and\nconfigure a model plugin. If you're coming from the Get Started guide,\nyou've already done this. Otherwise, see the [Get Started](/go/docs/get-started-go)\nguide or the individual plugin's documentation and follow the steps there before\ncontinuing.\n\n### The `genkit.Generate()` function\n\nIn Genkit, the primary interface through which you interact with generative AI\nmodels is the `genkit.Generate()` function.\n\nThe simplest `genkit.Generate()` call specifies the model you want to use and a\ntext prompt:\n\n```go\npackage main\n\nimport (\n \"context\"\n \"log\"\n\n \"github.com/firebase/genkit/go/ai\"\n \"github.com/firebase/genkit/go/genkit\"\n \"github.com/firebase/genkit/go/plugins/googlegenai\"\n)\n\nfunc main() {\n\tctx := context.Background()\n\n\tg, err := genkit.Init(ctx,\n\t\tgenkit.WithPlugins(&googlegenai.GoogleAI{}),\n\t\tgenkit.WithDefaultModel(\"googleai/gemini-2.5-flash\"),\n\t)\n\tif err != nil {\n\t\tlog.Fatalf(\"could not initialize Genkit: %v\", err)\n\t}\n\n\tresp, err := genkit.Generate(ctx, g,\n\t\tai.WithPrompt(\"Invent a menu item for a pirate themed restaurant.\"),\n\t)\n\tif err != nil {\n\t\tlog.Fatalf(\"could not generate model response: %v\", err)\n\t}\n\n\tlog.Println(resp.Text())\n}\n```\n\nWhen you run this brief example, it will print out some debugging information\nfollowed by the output of the `genkit.Generate()` call, which will usually be\nMarkdown text as in the following example:\n\n```md\n## The Blackheart's Bounty\n\n**A hearty stew of slow-cooked beef, spiced with rum and molasses, served in a\nhollowed-out cannonball with a side of crusty bread and a dollop of tangy\npineapple salsa.**\n\n**Description:** This dish is a tribute to the hearty meals enjoyed by pirates\non the high seas. The beef is tender and flavorful, infused with the warm spices\nof rum and molasses. The pineapple salsa adds a touch of sweetness and acidity,\nbalancing the richness of the stew. The cannonball serving vessel adds a fun and\nthematic touch, making this dish a perfect choice for any pirate-themed\nadventure.\n```\n\nRun the script again and you'll get a different output.\n\nThe preceding code sample sent the generation request to the default model,\nwhich you specified when you configured the Genkit instance.\n\nYou can also specify a model for a single `genkit.Generate()` call:\n\n```go\nresp, err := genkit.Generate(ctx, g,\n\tai.WithModelName(\"googleai/gemini-2.5-pro\"),\n\tai.WithPrompt(\"Invent a menu item for a pirate themed restaurant.\"),\n)\n```\n\nA model string identifier looks like `providerid/modelid`, where the provider ID\n(in this case, `googleai`) identifies the plugin, and the model ID is a\nplugin-specific string identifier for a specific version of a model.\n\nThese examples also illustrate an important point: when you use\n`genkit.Generate()` to make generative AI model calls, changing the model you\nwant to use is a matter of passing a different value to the model\nparameter. By using `genkit.Generate()` instead of the native model SDKs, you\ngive yourself the flexibility to more easily use several different models in\nyour app and change models in the future.\n\nSo far you have only seen examples of the simplest `genkit.Generate()` calls.\nHowever, `genkit.Generate()` also provides an interface for more advanced\ninteractions with generative models, which you will see in the sections that\nfollow.\n\n### System prompts\n\nSome models support providing a _system prompt_, which gives the model\ninstructions as to how you want it to respond to messages from the user. You can\nuse the system prompt to specify characteristics such as a persona you want the\nmodel to adopt, the tone of its responses, and the format of its responses.\n\nIf the model you're using supports system prompts, you can provide one with the\n`ai.WithSystem()` option:\n\n```go\nresp, err := genkit.Generate(ctx, g,\n\tai.WithSystem(\"You are a food industry marketing consultant.\"),\n\tai.WithPrompt(\"Invent a menu item for a pirate themed restaurant.\"),\n)\n```\n\nFor models that don't support system prompts, `ai.WithSystem()` simulates it by\nmodifying the request to appear _like_ a system prompt.\n\n### Model parameters\n\nThe `genkit.Generate()` function takes a `ai.WithConfig()` option, through which\nyou can specify optional settings that control how the model generates content:\n\n```go\nresp, err := genkit.Generate(ctx, g,\n\tai.WithModelName(\"googleai/gemini-2.5-flash\"),\n\tai.WithPrompt(\"Invent a menu item for a pirate themed restaurant.\"),\n\tai.WithConfig(&googlegenai.GeminiConfig{\n\t\tMaxOutputTokens: 500,\n\t\tStopSequences: [\"\", \"\"],\n\t\tTemperature: 0.5,\n\t\tTopP: 0.4,\n\t\tTopK: 50,\n\t}),\n)\n```\n\nThe exact parameters that are supported depend on the individual model and model\nAPI. However, the parameters in the previous example are common to almost every\nmodel. The following is an explanation of these parameters:\n\n#### Parameters that control output length\n\n**MaxOutputTokens**\n\nLLMs operate on units called _tokens_. A token usually, but does not\nnecessarily, map to a specific sequence of characters. When you pass a prompt to\na model, one of the first steps it takes is to _tokenize_ your prompt string\ninto a sequence of tokens. Then, the LLM generates a sequence of tokens from the\ntokenized input. Finally, the sequence of tokens gets converted back into text,\nwhich is your output.\n\nThe maximum output tokens parameter sets a limit on how many tokens to\ngenerate using the LLM. Every model potentially uses a different tokenizer, but\na good rule of thumb is to consider a single English word to be made of 2 to 4\ntokens.\n\nAs stated earlier, some tokens might not map to character sequences. One such\nexample is that there is often a token that indicates the end of the sequence:\nwhen an LLM generates this token, it stops generating more. Therefore, it's\npossible and often the case that an LLM generates fewer tokens than the maximum\nbecause it generated the \"stop\" token.\n\n**StopSequences**\n\nYou can use this parameter to set the tokens or token sequences that, when\ngenerated, indicate the end of LLM output. The correct values to use here\ngenerally depend on how the model was trained, and are usually set by the model\nplugin. However, if you have prompted the model to generate another stop\nsequence, you might specify it here.\n\nNote that you are specifying character sequences, and not tokens per se. In most\ncases, you will specify a character sequence that the model's tokenizer maps to\na single token.\n\n#### Parameters that control \"creativity\"\n\nThe _temperature_, _top-p_, and _top-k_ parameters together control how\n\"creative\" you want the model to be. This section provides very brief\nexplanations of what these parameters mean, but the more important point is\nthis: these parameters are used to adjust the character of an LLM's output. The\noptimal values for them depend on your goals and preferences, and are likely to\nbe found only through experimentation.\n\n**Temperature**\n\nLLMs are fundamentally token-predicting machines. For a given sequence of tokens\n(such as the prompt) an LLM predicts, for each token in its vocabulary, the\nlikelihood that the token comes next in the sequence. The temperature is a\nscaling factor by which these predictions are divided before being normalized to\na probability between 0 and 1.\n\nLow temperature values—between 0.0 and 1.0—amplify the difference in\nlikelihoods between tokens, with the result that the model will be even less\nlikely to produce a token it already evaluated to be unlikely. This is often\nperceived as output that is less creative. Although 0.0 is technically not a\nvalid value, many models treat it as indicating that the model should behave\ndeterministically, and to only consider the single most likely token.\n\nHigh temperature values—those greater than 1.0—compress the\ndifferences in likelihoods between tokens, with the result that the model\nbecomes more likely to produce tokens it had previously evaluated to be\nunlikely. This is often perceived as output that is more creative. Some model\nAPIs impose a maximum temperature, often 2.0.\n\n**TopP**\n\n_Top-p_ is a value between 0.0 and 1.0 that controls the number of possible\ntokens you want the model to consider, by specifying the cumulative probability\nof the tokens. For example, a value of 1.0 means to consider every possible\ntoken (but still take into account the probability of each token). A value of\n0.4 means to only consider the most likely tokens, whose probabilities add up to\n0.4, and to exclude the remaining tokens from consideration.\n\n**TopK**\n\n_Top-k_ is an integer value that also controls the number of possible tokens you\nwant the model to consider, but this time by explicitly specifying the maximum\nnumber of tokens. Specifying a value of 1 means that the model should behave\ndeterministically.\n\n#### Experiment with model parameters\n\nYou can experiment with the effect of these parameters on the output generated\nby different model and prompt combinations by using the Developer UI. Start the\ndeveloper UI with the `genkit start` command and it will automatically load all\nof the models defined by the plugins configured in your project. You can quickly\ntry different prompts and configuration values without having to repeatedly make\nthese changes in code.\n\n#### Pair model with its config\n\nGiven that each provider or even a specific model may have its own configuration\nschema or warrant certain settings, it may be error prone to set separate\noptions using `ai.WithModelName()` and `ai.WithConfig()` since the latter is not\nstrongly typed to the former.\n\nTo pair a model with its config, you can create a model reference that you can\npass into the generate call instead:\n\n```go\nmodel := googlegenai.GoogleAIModelRef(\"gemini-2.5-flash\", &googlegenai.GeminiConfig{\n\tMaxOutputTokens: 500,\n\tStopSequences: [\"\", \"\"],\n\tTemperature: 0.5,\n\tTopP: 0.4,\n\tTopK: 50,\n})\n\nresp, err := genkit.Generate(ctx, g,\n\tai.WithModel(model),\n\tai.WithPrompt(\"Invent a menu item for a pirate themed restaurant.\"),\n)\nif err != nil {\n\tlog.Fatal(err)\n}\n```\n\nThe constructor for the model reference will enforce that the correct config\ntype is provided which may reduce mismatches.\n\n### Structured output\n\nWhen using generative AI as a component in your application, you often want\noutput in a format other than plain text. Even if you're just generating content\nto display to the user, you can benefit from structured output simply for the\npurpose of presenting it more attractively to the user. But for more advanced\napplications of generative AI, such as programmatic use of the model's output,\nor feeding the output of one model into another, structured output is a must.\n\nIn Genkit, you can request structured output from a model by specifying an\noutput type when you call `genkit.Generate()`:\n\n```go\ntype MenuItem struct {\n\tName string `json:\"name\"`\n\tDescription string `json:\"description\"`\n\tCalories int `json:\"calories\"`\n\tAllergens []string `json:\"allergens\"`\n}\n\nresp, err := genkit.Generate(ctx, g,\n\tai.WithPrompt(\"Invent a menu item for a pirate themed restaurant.\"),\n\tai.WithOutputType(MenuItem{}),\n)\nif err != nil {\n\tlog.Fatal(err) // One possible error is that the response does not conform to the type.\n}\n```\n\nModel output types are specified as JSON schema using the\n[`invopop/jsonschema`](https://github.com/invopop/jsonschema) package. This\nprovides runtime type checking, which bridges the gap between static Go types\nand the unpredictable output of generative AI models. This system lets you write\ncode that can rely on the fact that a successful generate call will always\nreturn output that conforms to your Go types.\n\nWhen you specify an output type in `genkit.Generate()`, Genkit does several\nthings behind the scenes:\n\n- Augments the prompt with additional guidance about the selected output\n format. This also has the side effect of specifying to the model what\n content exactly you want to generate (for example, not only suggest a menu\n item but also generate a description, a list of allergens, and so on).\n- Verifies that the output conforms to the schema.\n- Marshals the model output into a Go type.\n\nTo get structured output from a successful generate call, call `Output()` on the\nmodel response with an empty value of the type:\n\n```go\nvar item MenuItem\nif err := resp.Output(&item); err != nil {\n\tlog.Fatalf(err)\n}\n\nlog.Printf(\"%s (%d calories, %d allergens): %s\\n\",\n\titem.Name, item.Calories, len(item.Allergens), item.Description)\n```\n\nAlternatively, you can use `genkit.GenerateData()` for a more succinct call:\n\n```go\nitem, resp, err := genkit.GenerateData[MenuItem](ctx, g,\n\tai.WithPrompt(\"Invent a menu item for a pirate themed restaurant.\"),\n)\nif err != nil {\n\tlog.Fatal(err)\n}\n\nlog.Printf(\"%s (%d calories, %d allergens): %s\\n\",\n\titem.Name, item.Calories, len(item.Allergens), item.Description)\n```\n\nThis function requires the output type parameter but automatically sets the\n`ai.WithOutputType()` option and calls `ModelResponse.Output()` before returning\nthe value.\n\n#### Handling errors\n\nNote in the prior example that the `genkit.Generate()` call can result in an\nerror. One possible error can happen when the model fails to generate output\nthat conforms to the schema. The best strategy for dealing with such errors will\ndepend on your exact use case, but here are some general hints:\n\n- **Try a different model**. For structured output to succeed, the model must\n be capable of generating output in JSON. The most powerful LLMs like Gemini\n are versatile enough to do this; however, smaller models, such as some of\n the local models you would use with Ollama, might not be able to generate\n structured output reliably unless they have been specifically trained to do\n so.\n\n- **Simplify the schema**. LLMs may have trouble generating complex or deeply\n nested types. Try using clear names, fewer fields, or a flattened structure\n if you are not able to reliably generate structured data.\n\n- **Retry the `genkit.Generate()` call**. If the model you've chosen only\n rarely fails to generate conformant output, you can treat the error as you\n would treat a network error, and retry the request using some kind of\n incremental back-off strategy.\n\n### Streaming\n\nWhen generating large amounts of text, you can improve the experience for your\nusers by presenting the output as it's generated—streaming the output. A\nfamiliar example of streaming in action can be seen in most LLM chat apps: users\ncan read the model's response to their message as it's being generated, which\nimproves the perceived responsiveness of the application and enhances the\nillusion of chatting with an intelligent counterpart.\n\nIn Genkit, you can stream output using the `ai.WithStreaming()` option:\n\n```go\nresp, err := genkit.Generate(ctx, g,\n\tai.WithPrompt(\"Suggest a complete menu for a pirate themed restaurant.\"),\n\tai.WithStreaming(func(ctx context.Context, chunk *ai.ModelResponseChunk) error {\n\t\t// Do something with the chunk...\n\t\tlog.Println(chunk.Text())\n\t\treturn nil\n\t}),\n)\nif err != nil {\n\tlog.Fatal(err)\n}\n\nlog.Println(resp.Text())\n```\n\n\n\n### Multimodal input\n\nThe examples you've seen so far have used text strings as model prompts. While\nthis remains the most common way to prompt generative AI models, many models can\nalso accept other media as prompts. Media prompts are most often used in\nconjunction with text prompts that instruct the model to perform some operation\non the media, such as to caption an image or transcribe an audio recording.\n\nThe ability to accept media input and the types of media you can use are\ncompletely dependent on the model and its API. For example, the Gemini 2.0\nseries of models can accept images, video, and audio as prompts.\n\nTo provide a media prompt to a model that supports it, instead of passing a\nsimple text prompt to `genkit.Generate()`, pass an array consisting of a media\npart and a text part. This example specifies an image using a\npublicly-accessible HTTPS URL.\n\n```go\nresp, err := genkit.Generate(ctx, g,\n\tai.WithModelName(\"googleai/gemini-2.5-flash\"),\n\tai.WithMessages(\n\t\tNewUserMessage(\n\t\t\tNewMediaPart(\"image/jpeg\", \"https://example.com/photo.jpg\"),\n\t\t\tNewTextPart(\"Compose a poem about this image.\"),\n\t\t),\n\t),\n)\n```\n\nYou can also pass media data directly by encoding it as a data URL. For\nexample:\n\n```go\nimage, err := ioutil.ReadFile(\"photo.jpg\")\nif err != nil {\n\tlog.Fatal(err)\n}\n\nresp, err := genkit.Generate(ctx, g,\n\tai.WithModelName(\"googleai/gemini-2.5-flash\"),\n\tai.WithMessages(\n\t\tNewUserMessage(\n\t\t\tNewMediaPart(\"image/jpeg\", \"data:image/jpeg;base64,\" + base64.StdEncoding.EncodeToString(image)),\n\t\t\tNewTextPart(\"Compose a poem about this image.\"),\n\t\t),\n\t),\n)\n```\n\nAll models that support media input support both data URLs and HTTPS URLs. Some\nmodel plugins add support for other media sources. For example, the Vertex AI\nplugin also lets you use Cloud Storage (`gs://`) URLs.\n\n\n\n### Next steps\n\n#### Learn more about Genkit\n\n- As an app developer, the primary way you influence the output of generative\n AI models is through prompting. Read\n [Managing prompts with Dotprompt](/go/docs/dotprompt) to learn\n how Genkit helps you develop effective prompts and manage them in your\n codebase.\n- Although `genkit.Generate()` is the nucleus of every generative AI powered\n application, real-world applications usually require additional work before\n and after invoking a generative AI model. To reflect this, Genkit introduces\n the concept of _flows_, which are defined like functions but add additional\n features such as observability and simplified deployment. To learn more, see\n [Defining AI workflows](/go/docs/flows).\n\n#### Advanced LLM use\n\nThere are techniques your app can use to reap even more benefit from LLMs.\n\n- One way to enhance the capabilities of LLMs is to prompt them with a list of\n ways they can request more information from you, or request you to perform\n some action. This is known as _tool calling_ or _function calling_. Models\n that are trained to support this capability can respond to a prompt with a\n specially-formatted response, which indicates to the calling application\n that it should perform some action and send the result back to the LLM along\n with the original prompt. Genkit has library functions that automate both\n the prompt generation and the call-response loop elements of a tool calling\n implementation. See [Tool calling](/go/docs/tool-calling) to learn more.\n- Retrieval-augmented generation (RAG) is a technique used to introduce\n domain-specific information into a model's output. This is accomplished by\n inserting relevant information into a prompt before passing it on to the\n language model. A complete RAG implementation requires you to bring several\n technologies together: text embedding generation models, vector databases, and\n large language models. See [Retrieval-augmented generation (RAG)](/go/docs/rag) to\n learn how Genkit simplifies the process of coordinating these various\n elements.\n", "title": "Generating content with AI models", - "lang": "go" + "description": "Learn how to use Genkit's unified API in Go to generate content with various AI models like LLMs and image generators.", + "lang": "go", + "headers": "### Before you begin\n### Models supported by Genkit\n### Loading and configuring model plugins\n### The `genkit.Generate()` function\n## The Blackheart's Bounty\n### System prompts\n### Model parameters\n#### Parameters that control output length\n#### Parameters that control \"creativity\"\n#### Experiment with model parameters\n#### Pair model with its config\n### Structured output\n#### Handling errors\n### Streaming\n### Multimodal input\n### Next steps\n#### Learn more about Genkit\n#### Advanced LLM use\n" }, "go/monitoring.md": { - "text": "Genkit provides two complementary monitoring features: OpenTelemetry\nexport and trace inspection using the developer UI.\n\n## OpenTelemetry export\n\nGenkit is fully instrumented with [OpenTelemetry](https://opentelemetry.io/) and\nprovides hooks to export telemetry data.\n\nThe [Google Cloud plugin](/go/docs/plugins/google-cloud) exports telemetry to\nCloud's operations suite.\n\n## Trace store\n\nThe trace store feature is complementary to the telemetry instrumentation. It\nlets you inspect your traces for your flow runs in the Genkit Developer UI.\n\nThis feature is enabled whenever you run a Genkit flow in a dev environment\n(such as when using `genkit start` or `genkit flow:run`).\n", + "text": "# Monitoring\n\nGenkit provides two complementary monitoring features: OpenTelemetry\nexport and trace inspection using the developer UI.\n\n## OpenTelemetry export\n\nGenkit is fully instrumented with [OpenTelemetry](https://opentelemetry.io/) and\nprovides hooks to export telemetry data.\n\nThe [Google Cloud plugin](/go/docs/plugins/google-cloud) exports telemetry to\nCloud's operations suite.\n\n## Trace store\n\nThe trace store feature is complementary to the telemetry instrumentation. It\nlets you inspect your traces for your flow runs in the Genkit Developer UI.\n\nThis feature is enabled whenever you run a Genkit flow in a dev environment\n(such as when using `genkit start` or `genkit flow:run`).\n", "title": "Monitoring", - "lang": "go" + "description": "Learn about Genkit's monitoring features, including OpenTelemetry export and trace inspection in the Developer UI for Go applications.", + "lang": "go", + "headers": "## OpenTelemetry export\n## Trace store\n" }, "go/plugin-authoring-models.md": { - "text": "Genkit model plugins add one or more generative AI models to the Genkit\nregistry. A model represents any generative model that is capable of receiving a\nprompt as input and generating text, media, or data as output.\n\n## Before you begin\n\nRead [Writing Genkit plugins](/go/docs/plugin-authoring) for information about writing\nany kind of Genkit plug-in, including model plugins. In particular, note that\nevery plugin must export a type that conforms to the `genkit.Plugin` interface,\nwhich includes a `Name()` and a `Init()` function.\n\n## Model definitions\n\nGenerally, a model plugin will make one or more `genkit.DefineModel()` calls in\nits `Init` function—once for each model the plugin is providing an\ninterface to.\n\nA model definition consists of three components:\n\n1. Metadata declaring the model's capabilities.\n2. A configuration type with any specific parameters supported by the model.\n3. A generation function that accepts an `ai.ModelRequest` and returns an\n `ai.ModelResponse`, presumably using an AI model to generate the latter.\n\nAt a high level, here's what it looks like in code:\n\n```go\npackage myplugin\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\n\t\"github.com/firebase/genkit/go/ai\"\n\t\"github.com/firebase/genkit/go/genkit\"\n)\n\nconst providerID = \"myProvider\" // Unique ID for your plugin provider\n\n// MyModelConfig defines the configuration options for your model.\n// Embed ai.GenerationCommonConfig for common options.\ntype MyModelConfig struct {\n\tai.GenerationCommonConfig\n\tAnotherCustomOption string `json:\"anotherCustomOption,omitempty\"`\n\tCustomOption int `json:\"customOption,omitempty\"`\n}\n\n// DefineModel registers your custom model with Genkit.\nfunc DefineMyModel(g *genkit.Genkit) {\n\tgenkit.DefineModel(g, providerID, \"my-model\",\n\t\t&ai.ModelInfo{\n\t\t\tLabel: \"My Model\", // User-friendly label\n\t\t\tSupports: &ai.ModelSupports{\n\t\t\t\tMultiturn: true, // Does the model support multi-turn chats?\n\t\t\t\tSystemRole: true, // Does the model support system messages?\n\t\t\t\tMedia: false, // Can the model accept media input?\n\t\t\t\tTools: false, // Does the model support function calling (tools)?\n\t\t\t},\n\t\t\tVersions: []string{\"my-model-001\"}, // List supported versions/aliases\n\t\t},\n\t\t// The generation function\n\t\tfunc(ctx context.Context, mr *ai.ModelRequest, cb ai.ModelStreamCallback) (*ai.ModelResponse, error) {\n\t\t\t// Verify that the request includes a configuration that conforms to your schema.\n\t\t\tvar cfg MyModelConfig\n\t\t\tif mr.Config != nil {\n\t\t\t\t// Attempt to cast the config; handle potential type mismatch\n\t\t\t\tif typedCfg, ok := mr.Config.(*MyModelConfig); ok {\n\t\t\t\t\tcfg = *typedCfg\n\t\t\t\t} else {\n\t\t\t\t\t// Handle incorrect config type if necessary, or rely on default values\n\t\t\t\t\t// For simplicity, this example proceeds with default cfg if cast fails\n\t\t\t\t}\n\t\t\t}\n\t\t\t// Now 'cfg' holds the configuration, either from the request or default.\n\n\t\t\t// Use your custom logic to convert Genkit's ai.ModelRequest into a form\n\t\t\t// usable by the model's native API.\n\t\t\tapiRequest, err := apiRequestFromGenkitRequest(mr, cfg) // Pass config too\n\t\t\tif err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"failed to create API request: %w\", err)\n\t\t\t}\n\n\t\t\t// Send the request to the model API, using your own code or the model\n\t\t\t// API's client library.\n\t\t\tapiResponse, err := callModelAPI(ctx, apiRequest) // Pass context if needed\n\t\t\tif err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"model API call failed: %w\", err)\n\t\t\t}\n\n\t\t\t// Use your custom logic to convert the model's response to Genkit's ai.ModelResponse.\n\t\t\tresponse, err := genResponseFromAPIResponse(apiResponse)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"failed to convert API response: %w\", err)\n\t\t\t}\n\n\t\t\treturn response, nil\n\t\t},\n\t)\n}\n\n// Placeholder for the function that converts Genkit request to your API's format\nfunc apiRequestFromGenkitRequest(mr *ai.ModelRequest, cfg MyModelConfig) (interface{}, error) {\n\t// Implementation depends on your specific model API\n\tfmt.Printf(\"Converting Genkit request with config: %+v\\n\", cfg)\n\t// ... conversion logic ...\n\treturn \"your-api-request-format\", nil // Replace with actual request object\n}\n\n// Placeholder for the function that calls your model's API\nfunc callModelAPI(ctx context.Context, apiRequest interface{}) (interface{}, error) {\n\t// Implementation depends on your specific model API client library\n\t// ... API call logic ...\n\treturn \"your-api-response-format\", nil // Replace with actual response object\n}\n\n// Placeholder for the function that converts your API's response to Genkit's format\nfunc genResponseFromAPIResponse(apiResponse interface{}) (*ai.ModelResponse, error) {\n\t// Implementation depends on your specific model API response format\n\t// ... conversion logic ...\n\treturn &ai.ModelResponse{\n\t\tCandidates: []*ai.Candidate{\n\t\t\t{\n\t\t\t\tMessage: &ai.Message{\n\t\t\t\t\tContent: []*ai.Part{ai.NewTextPart(\"Generated response text\")},\n\t\t\t\t\tRole: ai.RoleModel,\n\t\t\t\t},\n\t\t\t\tFinishReason: ai.FinishReasonStop,\n\t\t\t},\n\t\t},\n\t}, nil // Replace with actual response conversion\n}\n\n// Example Plugin implementation\ntype MyPlugin struct{}\n\nfunc (p *MyPlugin) Name() string {\n\treturn providerID\n}\n\nfunc (p *MyPlugin) Init(ctx context.Context, g *genkit.Genkit) error {\n\tDefineMyModel(g)\n\t// Define other models or resources here\n\treturn nil\n}\n\n// Ensure MyPlugin implements genkit.Plugin\nvar _ genkit.Plugin = &MyPlugin{}\n```\n\n### Declaring model capabilities\n\nEvery model definition must contain, as part of its metadata, an `ai.ModelInfo`\nvalue that declares which features the model supports. Genkit uses this\ninformation to determine certain behaviors, such as verifying whether certain\ninputs are valid for the model. For example, if the model doesn't support\nmulti-turn interactions, then it's an error to pass it a message history.\n\nNote that these declarations refer to the capabilities of the model as provided\nby your plugin, and do not necessarily map one-to-one to the capabilities of the\nunderlying model and model API. For example, even if the model API doesn't\nprovide a specific way to define system messages, your plugin might still\ndeclare support for the system role, and implement it as special logic that\ninserts system messages into the user prompt.\n\n### Defining your model's config schema\n\nTo specify the generation options a model supports, define and export a\nconfiguration type. Genkit has an `ai.GenerationCommonConfig` type that contains\noptions frequently supported by generative AI model services, which you can\nembed or use outright.\n\nYour generation function should verify that the request contains the correct\noptions type.\n\n### Transforming requests and responses\n\nThe generation function carries out the primary work of a Genkit model plugin:\ntransforming the `ai.ModelRequest` from Genkit's common format into a format\nthat is supported by your model's API, and then transforming the response from\nyour model into the `ai.ModelResponse` format used by Genkit.\n\nSometimes, this may require massaging or manipulating data to work around model\nlimitations. For example, if your model does not natively support a `system`\nmessage, you may need to transform a prompt's system message into a user-model\nmessage pair.\n\n## Exports\n\nIn addition to the resources that all plugins must export, a model plugin should\nalso export the following:\n\n- A generation config type, as discussed [earlier](#defining-your-models-config-schema).\n\n- A `Model()` function, which returns references to your plugin's defined\n models. Often, this can be:\n\n ```go\n func Model(g *genkit.Genkit, name string) *ai.Model {\n return genkit.LookupModel(g, providerID, name)\n }\n ```\n\n- A `ModelRef` function, which creates a model reference paired with its\n config that can validate the type and be passed around together:\n\n ```go\n func ModelRef(name string, config *MyModelConfig) *ai.ModelRef {\n return ai.NewModelRef(name, config)\n }\n ```\n\n- **Optional**: A `DefineModel()` function, which lets users define models\n that your plugin can provide, but that you do not automatically define.\n There are two main reasons why you might want to provide such a function:\n\n - Your plugin provides access to too many models to practically register\n each one. For example, the Ollama plugin can provide access to dozens of\n different models, with more added frequently. For this reason, it\n doesn't automatically define any models, and instead requires the user\n to call `DefineModel()` for each model they want to use.\n\n - To give your users the ability to use newly-released models that you\n have not yet added to your plugin.\n\n A plugin's `DefineModel()` function is typically a frontend to\n `genkit.DefineModel()` that defines a generation function, but lets the user\n specify the model name and model capabilities.\n", + "text": "# Writing a Genkit model plugin\n\nGenkit model plugins add one or more generative AI models to the Genkit\nregistry. A model represents any generative model that is capable of receiving a\nprompt as input and generating text, media, or data as output.\n\n## Before you begin\n\nRead [Writing Genkit plugins](/go/docs/plugin-authoring) for information about writing\nany kind of Genkit plug-in, including model plugins. In particular, note that\nevery plugin must export a type that conforms to the `genkit.Plugin` interface,\nwhich includes a `Name()` and a `Init()` function.\n\n## Model definitions\n\nGenerally, a model plugin will make one or more `genkit.DefineModel()` calls in\nits `Init` function—once for each model the plugin is providing an\ninterface to.\n\nA model definition consists of three components:\n\n1. Metadata declaring the model's capabilities.\n2. A configuration type with any specific parameters supported by the model.\n3. A generation function that accepts an `ai.ModelRequest` and returns an\n `ai.ModelResponse`, presumably using an AI model to generate the latter.\n\nAt a high level, here's what it looks like in code:\n\n```go\npackage myplugin\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\n\t\"github.com/firebase/genkit/go/ai\"\n\t\"github.com/firebase/genkit/go/genkit\"\n)\n\nconst providerID = \"myProvider\" // Unique ID for your plugin provider\n\n// MyModelConfig defines the configuration options for your model.\n// Embed ai.GenerationCommonConfig for common options.\ntype MyModelConfig struct {\n\tai.GenerationCommonConfig\n\tAnotherCustomOption string `json:\"anotherCustomOption,omitempty\"`\n\tCustomOption int `json:\"customOption,omitempty\"`\n}\n\n// DefineModel registers your custom model with Genkit.\nfunc DefineMyModel(g *genkit.Genkit) {\n\tgenkit.DefineModel(g, providerID, \"my-model\",\n\t\t&ai.ModelInfo{\n\t\t\tLabel: \"My Model\", // User-friendly label\n\t\t\tSupports: &ai.ModelSupports{\n\t\t\t\tMultiturn: true, // Does the model support multi-turn chats?\n\t\t\t\tSystemRole: true, // Does the model support system messages?\n\t\t\t\tMedia: false, // Can the model accept media input?\n\t\t\t\tTools: false, // Does the model support function calling (tools)?\n\t\t\t},\n\t\t\tVersions: []string{\"my-model-001\"}, // List supported versions/aliases\n\t\t},\n\t\t// The generation function\n\t\tfunc(ctx context.Context, mr *ai.ModelRequest, cb ai.ModelStreamCallback) (*ai.ModelResponse, error) {\n\t\t\t// Verify that the request includes a configuration that conforms to your schema.\n\t\t\tvar cfg MyModelConfig\n\t\t\tif mr.Config != nil {\n\t\t\t\t// Attempt to cast the config; handle potential type mismatch\n\t\t\t\tif typedCfg, ok := mr.Config.(*MyModelConfig); ok {\n\t\t\t\t\tcfg = *typedCfg\n\t\t\t\t} else {\n\t\t\t\t\t// Handle incorrect config type if necessary, or rely on default values\n\t\t\t\t\t// For simplicity, this example proceeds with default cfg if cast fails\n\t\t\t\t}\n\t\t\t}\n\t\t\t// Now 'cfg' holds the configuration, either from the request or default.\n\n\t\t\t// Use your custom logic to convert Genkit's ai.ModelRequest into a form\n\t\t\t// usable by the model's native API.\n\t\t\tapiRequest, err := apiRequestFromGenkitRequest(mr, cfg) // Pass config too\n\t\t\tif err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"failed to create API request: %w\", err)\n\t\t\t}\n\n\t\t\t// Send the request to the model API, using your own code or the model\n\t\t\t// API's client library.\n\t\t\tapiResponse, err := callModelAPI(ctx, apiRequest) // Pass context if needed\n\t\t\tif err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"model API call failed: %w\", err)\n\t\t\t}\n\n\t\t\t// Use your custom logic to convert the model's response to Genkit's ai.ModelResponse.\n\t\t\tresponse, err := genResponseFromAPIResponse(apiResponse)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"failed to convert API response: %w\", err)\n\t\t\t}\n\n\t\t\treturn response, nil\n\t\t},\n\t)\n}\n\n// Placeholder for the function that converts Genkit request to your API's format\nfunc apiRequestFromGenkitRequest(mr *ai.ModelRequest, cfg MyModelConfig) (interface{}, error) {\n\t// Implementation depends on your specific model API\n\tfmt.Printf(\"Converting Genkit request with config: %+v\\n\", cfg)\n\t// ... conversion logic ...\n\treturn \"your-api-request-format\", nil // Replace with actual request object\n}\n\n// Placeholder for the function that calls your model's API\nfunc callModelAPI(ctx context.Context, apiRequest interface{}) (interface{}, error) {\n\t// Implementation depends on your specific model API client library\n\t// ... API call logic ...\n\treturn \"your-api-response-format\", nil // Replace with actual response object\n}\n\n// Placeholder for the function that converts your API's response to Genkit's format\nfunc genResponseFromAPIResponse(apiResponse interface{}) (*ai.ModelResponse, error) {\n\t// Implementation depends on your specific model API response format\n\t// ... conversion logic ...\n\treturn &ai.ModelResponse{\n\t\tCandidates: []*ai.Candidate{\n\t\t\t{\n\t\t\t\tMessage: &ai.Message{\n\t\t\t\t\tContent: []*ai.Part{ai.NewTextPart(\"Generated response text\")},\n\t\t\t\t\tRole: ai.RoleModel,\n\t\t\t\t},\n\t\t\t\tFinishReason: ai.FinishReasonStop,\n\t\t\t},\n\t\t},\n\t}, nil // Replace with actual response conversion\n}\n\n// Example Plugin implementation\ntype MyPlugin struct{}\n\nfunc (p *MyPlugin) Name() string {\n\treturn providerID\n}\n\nfunc (p *MyPlugin) Init(ctx context.Context, g *genkit.Genkit) error {\n\tDefineMyModel(g)\n\t// Define other models or resources here\n\treturn nil\n}\n\n// Ensure MyPlugin implements genkit.Plugin\nvar _ genkit.Plugin = &MyPlugin{}\n```\n\n### Declaring model capabilities\n\nEvery model definition must contain, as part of its metadata, an `ai.ModelInfo`\nvalue that declares which features the model supports. Genkit uses this\ninformation to determine certain behaviors, such as verifying whether certain\ninputs are valid for the model. For example, if the model doesn't support\nmulti-turn interactions, then it's an error to pass it a message history.\n\nNote that these declarations refer to the capabilities of the model as provided\nby your plugin, and do not necessarily map one-to-one to the capabilities of the\nunderlying model and model API. For example, even if the model API doesn't\nprovide a specific way to define system messages, your plugin might still\ndeclare support for the system role, and implement it as special logic that\ninserts system messages into the user prompt.\n\n### Defining your model's config schema\n\nTo specify the generation options a model supports, define and export a\nconfiguration type. Genkit has an `ai.GenerationCommonConfig` type that contains\noptions frequently supported by generative AI model services, which you can\nembed or use outright.\n\nYour generation function should verify that the request contains the correct\noptions type.\n\n### Transforming requests and responses\n\nThe generation function carries out the primary work of a Genkit model plugin:\ntransforming the `ai.ModelRequest` from Genkit's common format into a format\nthat is supported by your model's API, and then transforming the response from\nyour model into the `ai.ModelResponse` format used by Genkit.\n\nSometimes, this may require massaging or manipulating data to work around model\nlimitations. For example, if your model does not natively support a `system`\nmessage, you may need to transform a prompt's system message into a user-model\nmessage pair.\n\n## Exports\n\nIn addition to the resources that all plugins must export, a model plugin should\nalso export the following:\n\n- A generation config type, as discussed [earlier](#defining-your-models-config-schema).\n\n- A `Model()` function, which returns references to your plugin's defined\n models. Often, this can be:\n\n ```go\n func Model(g *genkit.Genkit, name string) *ai.Model {\n return genkit.LookupModel(g, providerID, name)\n }\n ```\n\n- A `ModelRef` function, which creates a model reference paired with its\n config that can validate the type and be passed around together:\n\n ```go\n func ModelRef(name string, config *MyModelConfig) *ai.ModelRef {\n return ai.NewModelRef(name, config)\n }\n ```\n\n- **Optional**: A `DefineModel()` function, which lets users define models\n that your plugin can provide, but that you do not automatically define.\n There are two main reasons why you might want to provide such a function:\n\n - Your plugin provides access to too many models to practically register\n each one. For example, the Ollama plugin can provide access to dozens of\n different models, with more added frequently. For this reason, it\n doesn't automatically define any models, and instead requires the user\n to call `DefineModel()` for each model they want to use.\n\n - To give your users the ability to use newly-released models that you\n have not yet added to your plugin.\n\n A plugin's `DefineModel()` function is typically a frontend to\n `genkit.DefineModel()` that defines a generation function, but lets the user\n specify the model name and model capabilities.\n", "title": "Writing a Genkit model plugin", - "lang": "go" + "description": "Learn how to create a Genkit model plugin in Go to integrate new generative AI models.", + "lang": "go", + "headers": "## Before you begin\n## Model definitions\n### Declaring model capabilities\n### Defining your model's config schema\n### Transforming requests and responses\n## Exports\n" }, "go/plugin-authoring-telemetry.md": { - "text": "The Genkit libraries are instrumented with [OpenTelemetry](http://opentelemetry.io)\nto support collecting traces, metrics, and logs. Genkit users can export this\ntelemetry data to monitoring and visualization tools by installing a plugin that\nconfigures the [OpenTelemetry Go SDK](https://opentelemetry.io/docs/languages/go/getting-started/)\nto export to a particular OpenTelemetry-capable system.\n\nGenkit includes a plugin that configures OpenTelemetry to export data to\n[Google Cloud Monitoring and Cloud Logging](/go/docs/plugins/google-cloud). To support\nother monitoring systems, you can extend Genkit by writing a telemetry plugin,\nas described on this page.\n\n## Before you begin\n\nRead [Writing Genkit plugins](/go/docs/plugin-authoring) for information about writing\nany kind of Genkit plugin, including telemetry plugins. In particular, note that\nevery plugin must export an `Init` function, which users are expected to call\nbefore using the plugin.\n\n## Exporters and Loggers\n\nAs stated earlier, the primary job of a telemetry plugin is to configure\nOpenTelemetry (which Genkit has already been instrumented with) to export data\nto a particular service. To do so, you need the following:\n\n- An implementation of OpenTelemetry's [`SpanExporter`](https://pkg.go.dev/go.opentelemetry.io/otel/sdk/trace#SpanExporter)\n interface that exports data to the service of your choice.\n- An implementation of OpenTelemetry's [`metric.Exporter`](https://pkg.go.dev/go.opentelemetry.io/otel/sdk/metric#Exporter)\n interface that exports data to the service of your choice.\n- Either a [`slog.Logger`](https://pkg.go.dev/log/slog#Logger)\n or an implementation of the [`slog.Handler`](https://pkg.go.dev/log/slog#Handler)\n interface, that exports logs to the service of your choice.\n\nDepending on the service you're interested in exporting to, this might be a\nrelatively minor effort or a large one.\n\nBecause OpenTelemetry is an industry standard, many monitoring services already\nhave libraries that implement these interfaces. For example, the `googlecloud`\nplugin for Genkit makes use of the\n[`opentelemetry-operations-go`](https://github.com/GoogleCloudPlatform/opentelemetry-operations-go)\nlibrary, maintained by the Google Cloud team.\nSimilarly, many monitoring services provide libraries that implement the\nstandard `slog` interfaces.\n\nOn the other hand, if no such libraries are available for your service,\nimplementing the necessary interfaces can be a substantial project.\n\nCheck the [OpenTelemetry registry](https://opentelemetry.io/ecosystem/registry/?component=exporter&language=go)\nor the monitoring service's docs to see if integrations are already available.\n\nIf you need to build these integrations yourself, take a look at the source of\nthe [official OpenTelemetry exporters](https://github.com/open-telemetry/opentelemetry-go/tree/main/exporters)\nand the page [A Guide to Writing `slog` Handlers](https://github.com/golang/example/blob/master/slog-handler-guide/README).\n\n## Building the plugin\n\n### Dependencies\n\nEvery telemetry plugin needs to import the Genkit core library and several\nOpenTelemetry libraries:\n\n```go\n\t// Import the Genkit core library.\n\n\t\"github.com/firebase/genkit/go/genkit\"\n\n\t// Import the OpenTelemetry libraries.\n\t\"go.opentelemetry.io/otel\"\n\t\"go.opentelemetry.io/otel/sdk/metric\"\n\t\"go.opentelemetry.io/otel/sdk/trace\"\n```\n\nIf you are building a plugin around an existing OpenTelemetry or `slog`\nintegration, you will also need to import them.\n\n### `Config`\n\nA telemetry plugin should, at a minimum, support the following configuration\noptions:\n\n```go\ntype Config struct {\n\t// Export even in the dev environment.\n\tForceExport bool\n\n\t// The interval for exporting metric data.\n\t// The default is 60 seconds.\n\tMetricInterval time.Duration\n\n\t// The minimum level at which logs will be written.\n\t// Defaults to [slog.LevelInfo].\n\tLogLevel slog.Leveler\n}\n```\n\nThe examples that follow assume you are making these options available and will\nprovide some guidance on how to handle them.\n\nMost plugins will also include configuration settings for the service it's\nexporting to (API key, project name, and so on).\n\n### `Init()`\n\nThe `Init()` function of a telemetry plugin should do all of the following:\n\n- Return early if Genkit is running in a development environment (such as when\n running with with `genkit start`) and the `Config.ForceExport` option isn't\n set:\n\n ```go\n shouldExport := cfg.ForceExport || os.Getenv(\"GENKIT_ENV\") != \"dev\"\n if !shouldExport {\n \treturn nil\n }\n ```\n\n- Initialize your trace span exporter and register it with Genkit:\n\n ```go\n spanProcessor := trace.NewBatchSpanProcessor(YourCustomSpanExporter{})\n genkit.RegisterSpanProcessor(g, spanProcessor)\n ```\n\n- Initialize your metric exporter and register it with the OpenTelemetry\n library:\n\n ```go\n r := metric.NewPeriodicReader(\n \tYourCustomMetricExporter{},\n \tmetric.WithInterval(cfg.MetricInterval),\n )\n mp := metric.NewMeterProvider(metric.WithReader(r))\n otel.SetMeterProvider(mp)\n ```\n\n Use the user-configured collection interval (`Config.MetricInterval`) when\n initializing the `PeriodicReader`.\n\n- Register your `slog` handler as the default logger:\n\n ```go\n logger := slog.New(YourCustomHandler{\n \tOptions: &slog.HandlerOptions{Level: cfg.LogLevel},\n })\n slog.SetDefault(logger)\n ```\n\n You should configure your handler to honor the user-specified minimum log\n level (`Config.LogLevel`).\n\n### PII redaction\n\nBecause most generative AI flows begin with user input of some kind, it's a\nlikely possibility that some flow traces contain personally-identifiable\ninformation (PII). To protect your users' information, you should redact PII\nfrom traces before you export them.\n\nIf you are building your own span exporter, you can build this functionality\ninto it.\n\nIf you're building your plugin around an existing OpenTelemetry integration, you\ncan wrap the provided span exporter with a custom exporter that carries out this\ntask. For example, the `googlecloud` plugin removes the `genkit:input` and\n`genkit:output` attributes from every span before exporting them using a wrapper\nsimilar to the following:\n\n```go\ntype redactingSpanExporter struct {\n\ttrace.SpanExporter\n}\n\nfunc (e *redactingSpanExporter) ExportSpans(ctx context.Context, spanData []trace.ReadOnlySpan) error {\n\tvar redacted []trace.ReadOnlySpan\n\tfor _, s := range spanData {\n\t\tredacted = append(redacted, redactedSpan{s})\n\t}\n\treturn e.SpanExporter.ExportSpans(ctx, redacted)\n}\n\nfunc (e *redactingSpanExporter) Shutdown(ctx context.Context) error {\n\treturn e.SpanExporter.Shutdown(ctx)\n}\n\ntype redactedSpan struct {\n\ttrace.ReadOnlySpan\n}\n\nfunc (s redactedSpan) Attributes() []attribute.KeyValue {\n\t// Omit input and output, which may contain PII.\n\tvar ts []attribute.KeyValue\n\tfor _, a := range s.ReadOnlySpan.Attributes() {\n\t\tif a.Key == \"genkit:input\" || a.Key == \"genkit:output\" {\n\t\t\tcontinue\n\t\t}\n\t\tts = append(ts, a)\n\t}\n\treturn ts\n}\n```\n\n## Troubleshooting\n\nIf you're having trouble getting data to show up where you expect, OpenTelemetry\nprovides a useful [diagnostic tool](https://opentelemetry.io/docs/languages/js/getting-started/nodejs/#troubleshooting)\nthat helps locate the source of the problem.\n", + "text": "# Writing a Genkit telemetry plugin\n\nThe Genkit libraries are instrumented with [OpenTelemetry](http://opentelemetry.io)\nto support collecting traces, metrics, and logs. Genkit users can export this\ntelemetry data to monitoring and visualization tools by installing a plugin that\nconfigures the [OpenTelemetry Go SDK](https://opentelemetry.io/docs/languages/go/getting-started/)\nto export to a particular OpenTelemetry-capable system.\n\nGenkit includes a plugin that configures OpenTelemetry to export data to\n[Google Cloud Monitoring and Cloud Logging](/go/docs/plugins/google-cloud). To support\nother monitoring systems, you can extend Genkit by writing a telemetry plugin,\nas described on this page.\n\n## Before you begin\n\nRead [Writing Genkit plugins](/go/docs/plugin-authoring) for information about writing\nany kind of Genkit plugin, including telemetry plugins. In particular, note that\nevery plugin must export an `Init` function, which users are expected to call\nbefore using the plugin.\n\n## Exporters and Loggers\n\nAs stated earlier, the primary job of a telemetry plugin is to configure\nOpenTelemetry (which Genkit has already been instrumented with) to export data\nto a particular service. To do so, you need the following:\n\n- An implementation of OpenTelemetry's [`SpanExporter`](https://pkg.go.dev/go.opentelemetry.io/otel/sdk/trace#SpanExporter)\n interface that exports data to the service of your choice.\n- An implementation of OpenTelemetry's [`metric.Exporter`](https://pkg.go.dev/go.opentelemetry.io/otel/sdk/metric#Exporter)\n interface that exports data to the service of your choice.\n- Either a [`slog.Logger`](https://pkg.go.dev/log/slog#Logger)\n or an implementation of the [`slog.Handler`](https://pkg.go.dev/log/slog#Handler)\n interface, that exports logs to the service of your choice.\n\nDepending on the service you're interested in exporting to, this might be a\nrelatively minor effort or a large one.\n\nBecause OpenTelemetry is an industry standard, many monitoring services already\nhave libraries that implement these interfaces. For example, the `googlecloud`\nplugin for Genkit makes use of the\n[`opentelemetry-operations-go`](https://github.com/GoogleCloudPlatform/opentelemetry-operations-go)\nlibrary, maintained by the Google Cloud team.\nSimilarly, many monitoring services provide libraries that implement the\nstandard `slog` interfaces.\n\nOn the other hand, if no such libraries are available for your service,\nimplementing the necessary interfaces can be a substantial project.\n\nCheck the [OpenTelemetry registry](https://opentelemetry.io/ecosystem/registry/?component=exporter&language=go)\nor the monitoring service's docs to see if integrations are already available.\n\nIf you need to build these integrations yourself, take a look at the source of\nthe [official OpenTelemetry exporters](https://github.com/open-telemetry/opentelemetry-go/tree/main/exporters)\nand the page [A Guide to Writing `slog` Handlers](https://github.com/golang/example/blob/master/slog-handler-guide/README).\n\n## Building the plugin\n\n### Dependencies\n\nEvery telemetry plugin needs to import the Genkit core library and several\nOpenTelemetry libraries:\n\n```go\n\t// Import the Genkit core library.\n\n\t\"github.com/firebase/genkit/go/genkit\"\n\n\t// Import the OpenTelemetry libraries.\n\t\"go.opentelemetry.io/otel\"\n\t\"go.opentelemetry.io/otel/sdk/metric\"\n\t\"go.opentelemetry.io/otel/sdk/trace\"\n```\n\nIf you are building a plugin around an existing OpenTelemetry or `slog`\nintegration, you will also need to import them.\n\n### `Config`\n\nA telemetry plugin should, at a minimum, support the following configuration\noptions:\n\n```go\ntype Config struct {\n\t// Export even in the dev environment.\n\tForceExport bool\n\n\t// The interval for exporting metric data.\n\t// The default is 60 seconds.\n\tMetricInterval time.Duration\n\n\t// The minimum level at which logs will be written.\n\t// Defaults to [slog.LevelInfo].\n\tLogLevel slog.Leveler\n}\n```\n\nThe examples that follow assume you are making these options available and will\nprovide some guidance on how to handle them.\n\nMost plugins will also include configuration settings for the service it's\nexporting to (API key, project name, and so on).\n\n### `Init()`\n\nThe `Init()` function of a telemetry plugin should do all of the following:\n\n- Return early if Genkit is running in a development environment (such as when\n running with with `genkit start`) and the `Config.ForceExport` option isn't\n set:\n\n ```go\n shouldExport := cfg.ForceExport || os.Getenv(\"GENKIT_ENV\") != \"dev\"\n if !shouldExport {\n \treturn nil\n }\n ```\n\n- Initialize your trace span exporter and register it with Genkit:\n\n ```go\n spanProcessor := trace.NewBatchSpanProcessor(YourCustomSpanExporter{})\n genkit.RegisterSpanProcessor(g, spanProcessor)\n ```\n\n- Initialize your metric exporter and register it with the OpenTelemetry\n library:\n\n ```go\n r := metric.NewPeriodicReader(\n \tYourCustomMetricExporter{},\n \tmetric.WithInterval(cfg.MetricInterval),\n )\n mp := metric.NewMeterProvider(metric.WithReader(r))\n otel.SetMeterProvider(mp)\n ```\n\n Use the user-configured collection interval (`Config.MetricInterval`) when\n initializing the `PeriodicReader`.\n\n- Register your `slog` handler as the default logger:\n\n ```go\n logger := slog.New(YourCustomHandler{\n \tOptions: &slog.HandlerOptions{Level: cfg.LogLevel},\n })\n slog.SetDefault(logger)\n ```\n\n You should configure your handler to honor the user-specified minimum log\n level (`Config.LogLevel`).\n\n### PII redaction\n\nBecause most generative AI flows begin with user input of some kind, it's a\nlikely possibility that some flow traces contain personally-identifiable\ninformation (PII). To protect your users' information, you should redact PII\nfrom traces before you export them.\n\nIf you are building your own span exporter, you can build this functionality\ninto it.\n\nIf you're building your plugin around an existing OpenTelemetry integration, you\ncan wrap the provided span exporter with a custom exporter that carries out this\ntask. For example, the `googlecloud` plugin removes the `genkit:input` and\n`genkit:output` attributes from every span before exporting them using a wrapper\nsimilar to the following:\n\n```go\ntype redactingSpanExporter struct {\n\ttrace.SpanExporter\n}\n\nfunc (e *redactingSpanExporter) ExportSpans(ctx context.Context, spanData []trace.ReadOnlySpan) error {\n\tvar redacted []trace.ReadOnlySpan\n\tfor _, s := range spanData {\n\t\tredacted = append(redacted, redactedSpan{s})\n\t}\n\treturn e.SpanExporter.ExportSpans(ctx, redacted)\n}\n\nfunc (e *redactingSpanExporter) Shutdown(ctx context.Context) error {\n\treturn e.SpanExporter.Shutdown(ctx)\n}\n\ntype redactedSpan struct {\n\ttrace.ReadOnlySpan\n}\n\nfunc (s redactedSpan) Attributes() []attribute.KeyValue {\n\t// Omit input and output, which may contain PII.\n\tvar ts []attribute.KeyValue\n\tfor _, a := range s.ReadOnlySpan.Attributes() {\n\t\tif a.Key == \"genkit:input\" || a.Key == \"genkit:output\" {\n\t\t\tcontinue\n\t\t}\n\t\tts = append(ts, a)\n\t}\n\treturn ts\n}\n```\n\n## Troubleshooting\n\nIf you're having trouble getting data to show up where you expect, OpenTelemetry\nprovides a useful [diagnostic tool](https://opentelemetry.io/docs/languages/js/getting-started/nodejs/#troubleshooting)\nthat helps locate the source of the problem.\n", "title": "Writing a Genkit telemetry plugin", - "lang": "go" + "description": "Learn how to create a Genkit telemetry plugin in Go to export traces, metrics, and logs using OpenTelemetry.", + "lang": "go", + "headers": "## Before you begin\n## Exporters and Loggers\n## Building the plugin\n### Dependencies\n### `Config`\n### `Init()`\n### PII redaction\n## Troubleshooting\n" }, "go/plugin-authoring.md": { - "text": "Genkit's capabilities are designed to be extended by plugins. Genkit plugins are\nconfigurable modules that can provide models, retrievers, indexers, trace\nstores, and more. You've already seen plugins in action just by using Genkit:\n\n```go\nimport (\n \"github.com/firebase/genkit/go/ai\"\n \"github.com/firebase/genkit/go/genkit\"\n \"github.com/firebase/genkit/go/plugins/googlegenai\"\n \"github.com/firebase/genkit/go/plugins/server\"\n)\n```\n\n```go\ng, err := genkit.Init(ctx,\n ai.WithPlugins(\n &googlegenai.GoogleAI{APIKey: ...},\n &googlegenai.VertexAI{ProjectID: \"my-project\", Location: \"us-central1\"},\n ),\n)\n```\n\nThe Vertex AI plugin takes configuration (such as the user's Google Cloud\nproject ID) and registers a variety of new models, embedders, and more with the\nGenkit registry. The registry serves as a lookup service for named actions at\nruntime, and powers Genkit's local UI for running and inspecting models,\nprompts, and more.\n\n## Creating a plugin\n\nIn Go, a Genkit plugin is a package that adheres to a small set of\nconventions. A single module can contain several plugins.\n\n### Provider ID\n\nEvery plugin must have a unique identifier string that distinguishes it from\nother plugins. Genkit uses this identifier as a namespace for every resource\nyour plugin defines, to prevent naming conflicts with other plugins.\n\nFor example, if your plugin has an ID `yourplugin` and provides a model called\n`text-generator`, the full model identifier will be `yourplugin/text-generator`.\n\nThis provider ID needs to be exported and you should define it once for your\nplugin and use it consistently when required by a Genkit function.\n\n```go\npackage yourplugin\n\nconst providerID = \"yourplugin\"\n```\n\n### Standard exports\n\nEvery plugin should define and export the following symbols to conform to the\n`genkit.Plugin` interface:\n\n- A struct type that encapsulates all of the configuration options accepted by\n the plugin.\n\n For any plugin options that are secret values, such as API keys, you should\n offer both a config option and a default environment variable to configure\n it. This lets your plugin take advantage of the secret-management features\n offered by many hosting providers (such as Cloud Secret Manager, which you\n can use with Cloud Run). For example:\n\n ```go\n type MyPlugin struct {\n APIKey string\n // Other options you may allow to configure...\n }\n ```\n\n- A `Name()` method on the struct that returns the provider ID.\n\n- An `Init()` method on the struct with a declaration like the following:\n\n ```go\n func (m *MyPlugin) Init(ctx context.Context, g *genkit.Genkit) error\n ```\n\n In this function, perform any setup steps required by your plugin. For\n example:\n\n - Confirm that any required configuration values are specified and assign\n default values to any unspecified optional settings.\n - Verify that the given configuration options are valid together.\n - Create any shared resources required by the rest of your plugin. For\n example, create clients for any services your plugin accesses.\n\n To the extent possible, the resources provided by your plugin shouldn't\n assume that any other plugins have been installed before this one.\n\n This method will be called automatically during `genkit.Init()` when the\n user passes the plugin into the `WithPlugins()` option.\n\n## Building plugin features\n\nA single plugin can activate many new things within Genkit. For example, the\nVertex AI plugin activates several new models as well as an embedder.\n\n### Model plugins\n\nGenkit model plugins add one or more generative AI models to the Genkit\nregistry. A model represents any generative model that is capable of receiving a\nprompt as input and generating text, media, or data as output.\n\nSee [Writing a Genkit model plugin](/go/docs/plugin-authoring-models).\n\n### Telemetry plugins\n\nGenkit telemetry plugins configure Genkit's OpenTelemetry instrumentation to\nexport traces, metrics, and logs to a particular monitoring or visualization\ntool.\n\nSee [Writing a Genkit telemetry plugin](/go/docs/plugin-authoring-telemetry).\n\n## Publishing a plugin\n\nGenkit plugins can be published as normal Go packages. To increase\ndiscoverability, your package should have `genkit` somewhere in its name so it\ncan be found with a simple search on\n[`pkg.go.dev`](https://pkg.go.dev/search?q=genkit). Any of the following are\ngood choices:\n\n- `github.com/yourorg/genkit-plugins/servicename`\n- `github.com/yourorg/your-repo/genkit/servicename`\n", + "text": "# Writing Genkit plugins\n\nGenkit's capabilities are designed to be extended by plugins. Genkit plugins are\nconfigurable modules that can provide models, retrievers, trace\nstores, and more. You've already seen plugins in action just by using Genkit:\n\n```go\nimport (\n \"github.com/firebase/genkit/go/ai\"\n \"github.com/firebase/genkit/go/genkit\"\n \"github.com/firebase/genkit/go/plugins/googlegenai\"\n \"github.com/firebase/genkit/go/plugins/server\"\n)\n```\n\n```go\ng, err := genkit.Init(ctx,\n ai.WithPlugins(\n &googlegenai.GoogleAI{APIKey: ...},\n &googlegenai.VertexAI{ProjectID: \"my-project\", Location: \"us-central1\"},\n ),\n)\n```\n\nThe Vertex AI plugin takes configuration (such as the user's Google Cloud\nproject ID) and registers a variety of new models, embedders, and more with the\nGenkit registry. The registry serves as a lookup service for named actions at\nruntime, and powers Genkit's local UI for running and inspecting models,\nprompts, and more.\n\n## Creating a plugin\n\nIn Go, a Genkit plugin is a package that adheres to a small set of\nconventions. A single module can contain several plugins.\n\n### Provider ID\n\nEvery plugin must have a unique identifier string that distinguishes it from\nother plugins. Genkit uses this identifier as a namespace for every resource\nyour plugin defines, to prevent naming conflicts with other plugins.\n\nFor example, if your plugin has an ID `yourplugin` and provides a model called\n`text-generator`, the full model identifier will be `yourplugin/text-generator`.\n\nThis provider ID needs to be exported and you should define it once for your\nplugin and use it consistently when required by a Genkit function.\n\n```go\npackage yourplugin\n\nconst providerID = \"yourplugin\"\n```\n\n### Standard exports\n\nEvery plugin should define and export the following symbols to conform to the\n`genkit.Plugin` interface:\n\n- A struct type that encapsulates all of the configuration options accepted by\n the plugin.\n\n For any plugin options that are secret values, such as API keys, you should\n offer both a config option and a default environment variable to configure\n it. This lets your plugin take advantage of the secret-management features\n offered by many hosting providers (such as Cloud Secret Manager, which you\n can use with Cloud Run). For example:\n\n ```go\n type MyPlugin struct {\n APIKey string\n // Other options you may allow to configure...\n }\n ```\n\n- A `Name()` method on the struct that returns the provider ID.\n\n- An `Init()` method on the struct with a declaration like the following:\n\n ```go\n func (m *MyPlugin) Init(ctx context.Context, g *genkit.Genkit) error\n ```\n\n In this function, perform any setup steps required by your plugin. For\n example:\n\n - Confirm that any required configuration values are specified and assign\n default values to any unspecified optional settings.\n - Verify that the given configuration options are valid together.\n - Create any shared resources required by the rest of your plugin. For\n example, create clients for any services your plugin accesses.\n\n To the extent possible, the resources provided by your plugin shouldn't\n assume that any other plugins have been installed before this one.\n\n This method will be called automatically during `genkit.Init()` when the\n user passes the plugin into the `WithPlugins()` option.\n\n## Building plugin features\n\nA single plugin can activate many new things within Genkit. For example, the\nVertex AI plugin activates several new models as well as an embedder.\n\n### Model plugins\n\nGenkit model plugins add one or more generative AI models to the Genkit\nregistry. A model represents any generative model that is capable of receiving a\nprompt as input and generating text, media, or data as output.\n\nSee [Writing a Genkit model plugin](/go/docs/plugin-authoring-models).\n\n### Telemetry plugins\n\nGenkit telemetry plugins configure Genkit's OpenTelemetry instrumentation to\nexport traces, metrics, and logs to a particular monitoring or visualization\ntool.\n\nSee [Writing a Genkit telemetry plugin](/go/docs/plugin-authoring-telemetry).\n\n## Publishing a plugin\n\nGenkit plugins can be published as normal Go packages. To increase\ndiscoverability, your package should have `genkit` somewhere in its name so it\ncan be found with a simple search on\n[`pkg.go.dev`](https://pkg.go.dev/search?q=genkit). Any of the following are\ngood choices:\n\n- `github.com/yourorg/genkit-plugins/servicename`\n- `github.com/yourorg/your-repo/genkit/servicename`\n", "title": "Writing Genkit plugins", - "lang": "go" + "description": "Learn the fundamentals of creating Genkit plugins in Go to extend its capabilities with new models, retrievers, and more.", + "lang": "go", + "headers": "## Creating a plugin\n### Provider ID\n### Standard exports\n## Building plugin features\n### Model plugins\n### Telemetry plugins\n## Publishing a plugin\n" }, "go/rag.md": { - "text": "Genkit provides abstractions that help you build retrieval-augmented generation\n(RAG) flows, as well as plugins that provide integrations with related tools.\n\n## What is RAG?\n\nRetrieval-augmented generation is a technique used to incorporate external\nsources of information into an LLM’s responses. It's important to be able to do\nso because, while LLMs are typically trained on a broad body of\nmaterial, practical use of LLMs often requires specific domain knowledge (for\nexample, you might want to use an LLM to answer customers' questions about your\ncompany’s products).\n\nOne solution is to fine-tune the model using more specific data. However, this\ncan be expensive both in terms of compute cost and in terms of the effort needed\nto prepare adequate training data.\n\nIn contrast, RAG works by incorporating external data sources into a prompt at\nthe time it's passed to the model. For example, you could imagine the prompt,\n\"What is Bart's relationship to Lisa?\" might be expanded (\"augmented\") by\nprepending some relevant information, resulting in the prompt, \"Homer and\nMarge's children are named Bart, Lisa, and Maggie. What is Bart's relationship\nto Lisa?\"\n\nThis approach has several advantages:\n\n- It can be more cost effective because you don't have to retrain the model.\n- You can continuously update your data source and the LLM can immediately make\n use of the updated information.\n- You now have the potential to cite references in your LLM's responses.\n\nOn the other hand, using RAG naturally means longer prompts, and some LLM API\nservices charge for each input token you send. Ultimately, you must evaluate the\ncost tradeoffs for your applications.\n\nRAG is a very broad area and there are many different techniques used to achieve\nthe best quality RAG. The core Genkit framework offers two main abstractions to\nhelp you do RAG:\n\n- Indexers: add documents to an \"index\".\n- Embedders: transforms documents into a vector representation\n- Retrievers: retrieve documents from an \"index\", given a query.\n\nThese definitions are broad on purpose because Genkit is un-opinionated about\nwhat an \"index\" is or how exactly documents are retrieved from it. Genkit only\nprovides a `Document` format and everything else is defined by the retriever or\nindexer implementation provider.\n\n### Indexers\n\nThe index is responsible for keeping track of your documents in such a way that\nyou can quickly retrieve relevant documents given a specific query. This is most\noften accomplished using a vector database, which indexes your documents using\nmultidimensional vectors called embeddings. A text embedding (opaquely)\nrepresents the concepts expressed by a passage of text; these are generated\nusing special-purpose ML models. By indexing text using its embedding, a vector\ndatabase is able to cluster conceptually related text and retrieve documents\nrelated to a novel string of text (the query).\n\nBefore you can retrieve documents for the purpose of generation, you need to\ningest them into your document index. A typical ingestion flow does the\nfollowing:\n\n1. Split up large documents into smaller documents so that only relevant\n portions are used to augment your prompts – \"chunking\". This is necessary\n because many LLMs have a limited context window, making it impractical to\n include entire documents with a prompt.\n\n Genkit doesn't provide built-in chunking libraries; however, there are open\n source libraries available that are compatible with Genkit.\n\n2. Generate embeddings for each chunk. Depending on the database you're using,\n you might explicitly do this with an embedding generation model, or you\n might use the embedding generator provided by the database.\n\n3. Add the text chunk and its index to the database.\n\nYou might run your ingestion flow infrequently or only once if you are working\nwith a stable source of data. On the other hand, if you are working with data\nthat frequently changes, you might continuously run the ingestion flow (for\nexample, in a Cloud Firestore trigger, whenever a document is updated).\n\n### Embedders\n\nAn embedder is a function that takes content (text, images, audio, etc.) and\ncreates a numeric vector that encodes the semantic meaning of the original\ncontent. As mentioned above, embedders are leveraged as part of the process of\nindexing. However, they can also be used independently to create embeddings\nwithout an index.\n\n### Retrievers\n\nA retriever is a concept that encapsulates logic related to any kind of document\nretrieval. The most popular retrieval cases typically include retrieval from\nvector stores. However, in Genkit a retriever can be any function that returns\ndata.\n\nTo create a retriever, you can use one of the provided implementations or\ncreate your own.\n\n## Supported indexers, retrievers, and embedders\n\nGenkit provides indexer and retriever support through its plugin system. The\nfollowing plugins are officially supported:\n\n- [Pinecone](/go/docs/plugins/pinecone) cloud vector database\n\nIn addition, Genkit supports the following vector stores through predefined\ncode templates, which you can customize for your database configuration and\nschema:\n\n- PostgreSQL with [`pgvector`](/go/docs/plugins/pgvector)\n\nEmbedding model support is provided through the following plugins:\n\n| Plugin | Models |\n| ----------------------------------------------------- | -------------- |\n| [Google Generative AI](/go/docs/plugins/google-genai) | Text embedding |\n\n## Defining a RAG Flow\n\nThe following examples show how you could ingest a collection of restaurant menu\nPDF documents into a vector database and retrieve them for use in a flow that\ndetermines what food items are available.\n\n### Install dependencies\n\nIn this example, we will use the `textsplitter` library from `langchaingo` and\nthe `ledongthuc/pdf` PDF parsing Library:\n\n```bash\ngo get github.com/tmc/langchaingo/textsplitter\ngo get github.com/ledongthuc/pdf\n```\n\n### Define an Indexer\n\nThe following example shows how to create an indexer to ingest a collection of\nPDF documents and store them in a local vector database.\n\nIt uses the local file-based vector similarity retriever that Genkit provides\nout-of-the box for simple testing and prototyping. _Do not use this\nin production._\n\n#### Create the indexer\n\n```go\n// Import Genkit's file-based vector retriever, (Don't use in production.)\nimport \"github.com/firebase/genkit/go/plugins/localvec\"\n\n// Vertex AI provides the text-embedding-004 embedder model.\nimport \"github.com/firebase/genkit/go/plugins/vertexai\"\n```\n\n```go\nctx := context.Background()\n\ng, err := genkit.Init(ctx, genkit.WithPlugins(&googlegenai.VertexAI{}))\nif err != nil {\n\tlog.Fatal(err)\n}\n\nif err = localvec.Init(); err != nil {\n\tlog.Fatal(err)\n}\n\nmenuPDFIndexer, _, err := localvec.DefineIndexerAndRetriever(g, \"menuQA\",\n\t localvec.Config{Embedder: googlegenai.VertexAIEmbedder(g, \"text-embedding-004\")})\nif err != nil {\n\tlog.Fatal(err)\n}\n```\n\n#### Create chunking config\n\nThis example uses the `textsplitter` library which provides a simple text\nsplitter to break up documents into segments that can be vectorized.\n\nThe following definition configures the chunking function to return document\nsegments of 200 characters, with an overlap between chunks of 20 characters.\n\n```go\nsplitter := textsplitter.NewRecursiveCharacter(\n textsplitter.WithChunkSize(200),\n textsplitter.WithChunkOverlap(20),\n)\n```\n\nMore chunking options for this library can be found in the\n[`langchaingo` documentation](https://pkg.go.dev/github.com/tmc/langchaingo/textsplitter#Option).\n\n#### Define your indexer flow\n\n```go\ngenkit.DefineFlow(\n g, \"indexMenu\",\n func(ctx context.Context, path string) (any, error) {\n // Extract plain text from the PDF. Wrap the logic in Run so it\n // appears as a step in your traces.\n pdfText, err := genkit.Run(ctx, \"extract\", func() (string, error) {\n return readPDF(path)\n })\n if err != nil {\n return nil, err\n }\n\n // Split the text into chunks. Wrap the logic in Run so it appears as a\n // step in your traces.\n docs, err := genkit.Run(ctx, \"chunk\", func() ([]*ai.Document, error) {\n chunks, err := splitter.SplitText(pdfText)\n if err != nil {\n return nil, err\n }\n\n var docs []*ai.Document\n for _, chunk := range chunks {\n docs = append(docs, ai.DocumentFromText(chunk, nil))\n }\n return docs, nil\n })\n if err != nil {\n return nil, err\n }\n\n // Add chunks to the index.\n err = ai.Index(ctx, menuPDFIndexer, ai.WithDocs(docs...))\n return nil, err\n },\n)\n```\n\n```go\n// Helper function to extract plain text from a PDF. Excerpted from\n// https://github.com/ledongthuc/pdf\nfunc readPDF(path string) (string, error) {\n f, r, err := pdf.Open(path)\n if f != nil {\n defer f.Close()\n }\n if err != nil {\n return \"\", err\n }\n\n reader, err := r.GetPlainText()\n if err != nil {\n return \"\", err\n }\n\n bytes, err := io.ReadAll(reader)\n if err != nil {\n return \"\", err\n }\n\n return string(bytes), nil\n}\n```\n\n#### Run the indexer flow\n\n```bash\ngenkit flow:run indexMenu \"'menu.pdf'\"\n```\n\nAfter running the `indexMenu` flow, the vector database will be seeded with\ndocuments and ready to be used in Genkit flows with retrieval steps.\n\n### Define a flow with retrieval\n\nThe following example shows how you might use a retriever in a RAG flow. Like\nthe indexer example, this example uses Genkit's file-based vector retriever,\nwhich you should not use in production.\n\n```go\nctx := context.Background()\n\ng, err := genkit.Init(ctx, genkit.WithPlugins(&googlegenai.VertexAI{}))\nif err != nil {\n log.Fatal(err)\n}\n\nif err = localvec.Init(); err != nil {\n log.Fatal(err)\n}\n\nmodel := googlegenai.VertexAIModel(g, \"gemini-1.5-flash\")\n\n_, menuPdfRetriever, err := localvec.DefineIndexerAndRetriever(\n g, \"menuQA\", localvec.Config{Embedder: googlegenai.VertexAIEmbedder(g, \"text-embedding-004\")},\n)\nif err != nil {\n log.Fatal(err)\n}\n\ngenkit.DefineFlow(\n g, \"menuQA\",\n func(ctx context.Context, question string) (string, error) {\n // Retrieve text relevant to the user's question.\n resp, err := ai.Retrieve(ctx, menuPdfRetriever, ai.WithTextDocs(question))\n\n\n if err != nil {\n return \"\", err\n }\n\n // Call Generate, including the menu information in your prompt.\n return genkit.GenerateText(ctx, g,\n ai.WithModelName(\"googleai/gemini-2.5-flash\"),\n ai.WithDocs(resp.Documents),\n ai.WithSystem(`\nYou are acting as a helpful AI assistant that can answer questions about the\nfood available on the menu at Genkit Grub Pub.\nUse only the context provided to answer the question. If you don't know, do not\nmake up an answer. Do not add or change items on the menu.`)\n ai.WithPrompt(question),\n })\n```\n\n## Write your own indexers and retrievers\n\nIt's also possible to create your own retriever. This is useful if your\ndocuments are managed in a document store that is not supported in Genkit (eg:\nMySQL, Google Drive, etc.). The Genkit SDK provides flexible methods that let\nyou provide custom code for fetching documents.\n\nYou can also define custom retrievers that build on top of existing retrievers\nin Genkit and apply advanced RAG techniques (such as reranking or prompt\nextension) on top.\n\nFor example, suppose you have a custom re-ranking function you want to use. The\nfollowing example defines a custom retriever that applies your function to the\nmenu retriever defined earlier:\n\n```go\ntype CustomMenuRetrieverOptions struct {\n K int\n PreRerankK int\n}\n\nadvancedMenuRetriever := genkit.DefineRetriever(\n g, \"custom\", \"advancedMenuRetriever\",\n func(ctx context.Context, req *ai.RetrieverRequest) (*ai.RetrieverResponse, error) {\n // Handle options passed using our custom type.\n opts, _ := req.Options.(CustomMenuRetrieverOptions)\n // Set fields to default values when either the field was undefined\n // or when req.Options is not a CustomMenuRetrieverOptions.\n if opts.K == 0 {\n opts.K = 3\n }\n if opts.PreRerankK == 0 {\n opts.PreRerankK = 10\n }\n\n // Call the retriever as in the simple case.\n resp, err := ai.Retrieve(ctx, menuPDFRetriever,\n ai.WithDocs(req.Query),\n ai.WithConfig(ocalvec.RetrieverOptions{K: opts.PreRerankK}),\n )\n if err != nil {\n return nil, err\n }\n\n // Re-rank the returned documents using your custom function.\n rerankedDocs := rerank(response.Documents)\n response.Documents = rerankedDocs[:opts.K]\n\n return response, nil\n },\n)\n```\n", + "text": "# Retrieval-augmented generation (RAG)\n\nGenkit provides abstractions that help you build retrieval-augmented generation\n(RAG) flows, as well as plugins that provide integrations with related tools.\n\n## What is RAG?\n\nRetrieval-augmented generation is a technique used to incorporate external\nsources of information into an LLM’s responses. It's important to be able to do\nso because, while LLMs are typically trained on a broad body of\nmaterial, practical use of LLMs often requires specific domain knowledge (for\nexample, you might want to use an LLM to answer customers' questions about your\ncompany’s products).\n\nOne solution is to fine-tune the model using more specific data. However, this\ncan be expensive both in terms of compute cost and in terms of the effort needed\nto prepare adequate training data.\n\nIn contrast, RAG works by incorporating external data sources into a prompt at\nthe time it's passed to the model. For example, you could imagine the prompt,\n\"What is Bart's relationship to Lisa?\" might be expanded (\"augmented\") by\nprepending some relevant information, resulting in the prompt, \"Homer and\nMarge's children are named Bart, Lisa, and Maggie. What is Bart's relationship\nto Lisa?\"\n\nThis approach has several advantages:\n\n- It can be more cost effective because you don't have to retrain the model.\n- You can continuously update your data source and the LLM can immediately make\n use of the updated information.\n- You now have the potential to cite references in your LLM's responses.\n\nOn the other hand, using RAG naturally means longer prompts, and some LLM API\nservices charge for each input token you send. Ultimately, you must evaluate the\ncost tradeoffs for your applications.\n\nRAG is a very broad area and there are many different techniques used to achieve\nthe best quality RAG. The core Genkit framework offers two main abstractions to\nhelp you do RAG:\n\n- Embedders: transforms documents into a vector representation\n- Retrievers: retrieve documents from an \"index\", given a query.\n\nThese definitions are broad on purpose because Genkit is un-opinionated about\nwhat an \"index\" is or how exactly documents are retrieved from it. Genkit only\nprovides a `Document` format and everything else is defined by the retriever or\nindexer implementation provider.\n\n### Embedders\n\nAn embedder is a function that takes content (text, images, audio, etc.) and\ncreates a numeric vector that encodes the semantic meaning of the original\ncontent. As mentioned above, embedders are leveraged as part of the process of\nindexing. However, they can also be used independently to create embeddings\nwithout an index.\n\n### Retrievers\n\nA retriever is a concept that encapsulates logic related to any kind of document\nretrieval. The most popular retrieval cases typically include retrieval from\nvector stores. However, in Genkit a retriever can be any function that returns\ndata.\n\nTo create a retriever, you can use one of the provided implementations or\ncreate your own.\n\n## Supported retrievers, and embedders\n\nGenkit provides retriever support through its plugin system. The\nfollowing plugins are officially supported:\n\n- [Pinecone](/go/docs/plugins/pinecone) cloud vector database\n\nIn addition, Genkit supports the following vector stores through predefined\ncode templates, which you can customize for your database configuration and\nschema:\n\n- PostgreSQL with [`pgvector`](/go/docs/plugins/pgvector)\n\nEmbedding model support is provided through the following plugins:\n\n| Plugin | Models |\n| ----------------------------------------------------- | -------------- |\n| [Google Generative AI](/go/docs/plugins/google-genai) | Text embedding |\n\n## Defining a RAG Flow\n\nThe following examples show how you could ingest a collection of restaurant menu\nPDF documents into a vector database and retrieve them for use in a flow that\ndetermines what food items are available.\n*Note*: Although retriever functions are defined using Genkit, users are expected to add their own functionality to index the documents.\n\n### Install dependencies\n\nIn this example, we will use the `textsplitter` library from `langchaingo` and\nthe `ledongthuc/pdf` PDF parsing Library:\n\n```bash\ngo get github.com/tmc/langchaingo/textsplitter\ngo get github.com/ledongthuc/pdf\n```\n\n#### Create chunking config\n\nThis example uses the `textsplitter` library which provides a simple text\nsplitter to break up documents into segments that can be vectorized.\n\nThe following definition configures the chunking function to return document\nsegments of 200 characters, with an overlap between chunks of 20 characters.\n\n```go\nsplitter := textsplitter.NewRecursiveCharacter(\n textsplitter.WithChunkSize(200),\n textsplitter.WithChunkOverlap(20),\n)\n```\n\nMore chunking options for this library can be found in the\n[`langchaingo` documentation](https://pkg.go.dev/github.com/tmc/langchaingo/textsplitter#Option).\n\n#### Define your indexer flow\n\n```go\ngenkit.DefineFlow(\n g, \"indexMenu\",\n func(ctx context.Context, path string) (any, error) {\n // Extract plain text from the PDF. Wrap the logic in Run so it\n // appears as a step in your traces.\n pdfText, err := genkit.Run(ctx, \"extract\", func() (string, error) {\n return readPDF(path)\n })\n if err != nil {\n return nil, err\n }\n\n // Split the text into chunks. Wrap the logic in Run so it appears as a\n // step in your traces.\n docs, err := genkit.Run(ctx, \"chunk\", func() ([]*ai.Document, error) {\n chunks, err := splitter.SplitText(pdfText)\n if err != nil {\n return nil, err\n }\n\n var docs []*ai.Document\n for _, chunk := range chunks {\n docs = append(docs, ai.DocumentFromText(chunk, nil))\n }\n return docs, nil\n })\n if err != nil {\n return nil, err\n }\n\n // Add chunks to the index using custom index function\n },\n)\n```\n\n```go\n// Helper function to extract plain text from a PDF. Excerpted from\n// https://github.com/ledongthuc/pdf\nfunc readPDF(path string) (string, error) {\n f, r, err := pdf.Open(path)\n if f != nil {\n defer f.Close()\n }\n if err != nil {\n return \"\", err\n }\n\n reader, err := r.GetPlainText()\n if err != nil {\n return \"\", err\n }\n\n bytes, err := io.ReadAll(reader)\n if err != nil {\n return \"\", err\n }\n\n return string(bytes), nil\n}\n```\n\n#### Run the indexer flow\n\n```bash\ngenkit flow:run indexMenu \"'menu.pdf'\"\n```\n\nAfter running the `indexMenu` flow, the vector database will be seeded with\ndocuments and ready to be used in Genkit flows with retrieval steps.\n\n### Define a flow with retrieval\n\nThe following example shows how you might use a retriever in a RAG flow. This \nexample uses Genkit's file-based vector retriever, which you should not use in production.\n\n```go\nctx := context.Background()\n\ng, err := genkit.Init(ctx, genkit.WithPlugins(&googlegenai.VertexAI{}))\nif err != nil {\n log.Fatal(err)\n}\n\nif err = localvec.Init(); err != nil {\n log.Fatal(err)\n}\n\nmodel := googlegenai.VertexAIModel(g, \"gemini-1.5-flash\")\n\n_, menuPdfRetriever, err := localvec.DefineRetriever(\n g, \"menuQA\", localvec.Config{Embedder: googlegenai.VertexAIEmbedder(g, \"text-embedding-004\")},\n)\nif err != nil {\n log.Fatal(err)\n}\n\ngenkit.DefineFlow(\n g, \"menuQA\",\n func(ctx context.Context, question string) (string, error) {\n // Retrieve text relevant to the user's question.\n resp, err := ai.Retrieve(ctx, menuPdfRetriever, ai.WithTextDocs(question))\n\n\n if err != nil {\n return \"\", err\n }\n\n // Call Generate, including the menu information in your prompt.\n return genkit.GenerateText(ctx, g,\n ai.WithModelName(\"googleai/gemini-2.5-flash\"),\n ai.WithDocs(resp.Documents),\n ai.WithSystem(`\nYou are acting as a helpful AI assistant that can answer questions about the\nfood available on the menu at Genkit Grub Pub.\nUse only the context provided to answer the question. If you don't know, do not\nmake up an answer. Do not add or change items on the menu.`)\n ai.WithPrompt(question))\n })\n```\n\n## Write your own retrievers\n\nIt's also possible to create your own retriever. This is useful if your\ndocuments are managed in a document store that is not supported in Genkit (eg:\nMySQL, Google Drive, etc.). The Genkit SDK provides flexible methods that let\nyou provide custom code for fetching documents.\n\nYou can also define custom retrievers that build on top of existing retrievers\nin Genkit and apply advanced RAG techniques (such as reranking or prompt\nextension) on top.\n\nFor example, suppose you have a custom re-ranking function you want to use. The\nfollowing example defines a custom retriever that applies your function to the\nmenu retriever defined earlier:\n\n```go\ntype CustomMenuRetrieverOptions struct {\n K int\n PreRerankK int\n}\n\nadvancedMenuRetriever := genkit.DefineRetriever(\n g, \"custom\", \"advancedMenuRetriever\",\n func(ctx context.Context, req *ai.RetrieverRequest) (*ai.RetrieverResponse, error) {\n // Handle options passed using our custom type.\n opts, _ := req.Options.(CustomMenuRetrieverOptions)\n // Set fields to default values when either the field was undefined\n // or when req.Options is not a CustomMenuRetrieverOptions.\n if opts.K == 0 {\n opts.K = 3\n }\n if opts.PreRerankK == 0 {\n opts.PreRerankK = 10\n }\n\n // Call the retriever as in the simple case.\n resp, err := ai.Retrieve(ctx, menuPDFRetriever,\n ai.WithDocs(req.Query),\n ai.WithConfig(ocalvec.RetrieverOptions{K: opts.PreRerankK}),\n )\n if err != nil {\n return nil, err\n }\n\n // Re-rank the returned documents using your custom function.\n rerankedDocs := rerank(response.Documents)\n response.Documents = rerankedDocs[:opts.K]\n\n return response, nil\n },\n)\n```\n", "title": "Retrieval-augmented generation (RAG)", - "lang": "go" + "description": "Learn how to build Retrieval-Augmented Generation (RAG) flows in Genkit Go using indexers, embedders, and retrievers.", + "lang": "go", + "headers": "## What is RAG?\n### Embedders\n### Retrievers\n## Supported retrievers, and embedders\n## Defining a RAG Flow\n### Install dependencies\n#### Create chunking config\n#### Define your indexer flow\n#### Run the indexer flow\n### Define a flow with retrieval\n## Write your own retrievers\n" }, "go/tool-calling.md": { - "text": "_Tool calling_, also known as _function calling_, is a structured way to give\nLLMs the ability to make requests back to the application that called it. You\ndefine the tools you want to make available to the model, and the model will\nmake tool requests to your app as necessary to fulfill the prompts you give it.\n\nThe use cases of tool calling generally fall into a few themes:\n\n**Giving an LLM access to information it wasn't trained with**\n\n- Frequently changing information, such as a stock price or the current\n weather.\n- Information specific to your app domain, such as product information or user\n profiles.\n\nNote the overlap with [retrieval augmented generation](/go/docs/rag) (RAG), which is also\na way to let an LLM integrate factual information into its generations. RAG is a\nheavier solution that is most suited when you have a large amount of information\nor the information that's most relevant to a prompt is ambiguous. On the other\nhand, if a function call or database lookup is all that's necessary for\nretrieving the information the LLM needs, tool calling is more appropriate.\n\n**Introducing a degree of determinism into an LLM workflow**\n\n- Performing calculations that the LLM cannot reliably complete itself.\n- Forcing an LLM to generate verbatim text under certain circumstances, such\n as when responding to a question about an app's terms of service.\n\n**Performing an action when initiated by an LLM**\n\n- Turning on and off lights in an LLM-powered home assistant\n- Reserving table reservations in an LLM-powered restaurant agent\n\n## Before you begin\n\nIf you want to run the code examples on this page, first complete the steps in\nthe [Get started](/go/docs/get-started-go) guide. All of the examples assume that you\nhave already set up a project with Genkit dependencies installed.\n\nThis page discusses one of the advanced features of Genkit model abstraction, so\nbefore you dive too deeply, you should be familiar with the content on the\n[Generating content with AI models](/go/docs/models) page. You should also be familiar\nwith Genkit's system for defining input and output schemas, which is discussed\non the [Flows](/go/docs/flows) page.\n\n## Overview of tool calling\n\nAt a high level, this is what a typical tool-calling interaction with an LLM\nlooks like:\n\n1. The calling application prompts the LLM with a request and also includes in\n the prompt a list of tools the LLM can use to generate a response.\n2. The LLM either generates a complete response or generates a tool call\n request in a specific format.\n3. If the caller receives a complete response, the request is fulfilled and the\n interaction ends; but if the caller receives a tool call, it performs\n whatever logic is appropriate and sends a new request to the LLM containing\n the original prompt or some variation of it as well as the result of the\n tool call.\n4. The LLM handles the new prompt as in Step 2.\n\nFor this to work, several requirements must be met:\n\n- The model must be trained to make tool requests when it's needed to complete\n a prompt. Most of the larger models provided through web APIs such as Gemini\n can do this, but smaller and more specialized models often cannot. Genkit\n will throw an error if you try to provide tools to a model that doesn't\n support it.\n- The calling application must provide tool definitions to the model in the\n format it expects.\n- The calling application must prompt the model to generate tool calling\n requests in the format the application expects.\n\n## Tool calling with Genkit\n\nGenkit provides a single interface for tool calling with models that support it.\nEach model plugin ensures that the last two criteria mentioned in the previous\nsection are met, and the `genkit.Generate()` function automatically carries out\nthe tool-calling loop described earlier.\n\n### Model support\n\nTool calling support depends on the model, the model API, and the Genkit plugin.\nConsult the relevant documentation to determine if tool calling is likely to be\nsupported. In addition:\n\n- Genkit will throw an error if you try to provide tools to a model that\n doesn't support it.\n- If the plugin exports model references, the `ModelInfo.Supports.Tools`\n property will indicate if it supports tool calling.\n\n### Defining tools\n\nUse the `genkit.DefineTool()` function to write tool definitions:\n\n```go\npackage main\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"log\"\n\n\t\"github.com/firebase/genkit/go/ai\"\n\t\"github.com/firebase/genkit/go/genkit\"\n\t\"github.com/firebase/genkit/go/plugins/googlegenai\"\n)\n\n// Define the input structure for the tool\ntype WeatherInput struct {\n\tLocation string `json:\"location\" jsonschema_description:\"Location to get weather for\"`\n}\n\nfunc main() {\n\tctx := context.Background()\n\n\tg, err := genkit.Init(ctx,\n\t\tgenkit.WithPlugins(&googlegenai.GoogleAI{}),\n\t\tgenkit.WithDefaultModel(\"googleai/gemini-1.5-flash\"), // Updated model name\n\t)\n\tif err != nil {\n\t\tlog.Fatalf(\"Genkit initialization failed: %v\", err)\n\t}\n\n\tgenkit.DefineTool(\n\t\tg, \"getWeather\", \"Gets the current weather in a given location\",\n\t\tfunc(ctx context.Context, input WeatherInput) (string, error) {\n\t\t\t// Here, we would typically make an API call or database query. For this\n\t\t\t// example, we just return a fixed value.\n\t\t\tlog.Printf(\"Tool 'getWeather' called for location: %s\", input.Location)\n\t\t\treturn fmt.Sprintf(\"The current weather in %s is 63°F and sunny.\", input.Location), nil\n\t\t})\n}\n```\n\nThe syntax here looks just like the `genkit.DefineFlow()` syntax; however, you\nmust write a description. Take special care with the wording and descriptiveness\nof the description as it is vital for the LLM to decide to use it appropriately.\n\n### Using tools\n\nInclude defined tools in your prompts to generate content.\n\n**Using `genkit.Generate()`:**\n\n```go\nresp, err := genkit.Generate(ctx, g,\n\tai.WithPrompt(\"What is the weather in San Francisco?\"),\n\tai.WithTools(getWeatherTool),\n)\n```\n\n**Using `genkit.DefinePrompt()`:**\n\n```go\nweatherPrompt, err := genkit.DefinePrompt(g, \"weatherPrompt\",\n\tai.WithPrompt(\"What is the weather in {% verbatim %}{{location}}{% endverbatim %}?\"),\n\tai.WithTools(getWeatherTool),\n)\nif err != nil {\n\tlog.Fatal(err)\n}\n\nresp, err := weatherPrompt.Execute(ctx,\n\twith.Input(map[string]any{\"location\": \"San Francisco\"}),\n)\n```\n\n**Using a `.prompt` file:**\n\nCreate a file named `prompts/weatherPrompt.prompt` (assuming default prompt directory):\n\n```dotprompt\n---\nsystem: \"Answer questions using the tools you have.\"\ntools: [getWeather]\ninput:\n schema:\n location: string\n---\n\nWhat is the weather in {{location}}?\n```\n\nThen execute it in your Go code:\n\n```go\n// Assuming prompt file named weatherPrompt.prompt exists in ./prompts dir.\nweatherPrompt := genkit.LookupPrompt(\"weatherPrompt\")\nif weatherPrompt == nil {\n\tlog.Fatal(\"no prompt named 'weatherPrompt' found\")\n}\n\nresp, err := weatherPrompt.Execute(ctx,\n\tai.WithInput(map[string]any{\"location\": \"San Francisco\"}),\n)\n```\n\nGenkit will automatically handle the tool call if the LLM needs to use the\n`getWeather` tool to answer the prompt.\n\n### Explicitly handling tool calls\n\nIf you want full control over this tool-calling loop, for example to apply more\ncomplicated logic, set the `WithReturnToolRequests()` option to `true`. Now it's\nyour responsibility to ensure all of the tool requests are fulfilled:\n\n```go\ngetWeatherTool := genkit.DefineTool(\n g, \"getWeather\", \"Gets the current weather in a given location\",\n func(ctx *ai.ToolContext, location struct {\n Location string `jsonschema_description:\"Location to get weather for\"`\n }) (string, error) {\n // Tool implementation...\n return \"sunny\", nil\n },\n)\n\nresp, err := genkit.Generate(ctx, g,\n ai.WithPrompt(\"What is the weather in San Francisco?\"),\n ai.WithTools(getWeatherTool),\n ai.WithReturnToolRequests(true),\n)\nif err != nil {\n log.Fatal(err)\n}\n\nparts := []*ai.Part{}\nfor _, req := range resp.ToolRequests() {\n tool := genkit.LookupTool(g, req.Name)\n if tool == nil {\n log.Fatalf(\"tool %q not found\", req.Name)\n }\n\n output, err := tool.RunRaw(ctx, req.Input)\n if err != nil {\n log.Fatalf(\"tool %q execution failed: %v\", tool.Name(), err)\n }\n\n parts = append(parts,\n ai.NewToolResponsePart(&ai.ToolResponse{\n Name: req.Name,\n Ref: req.Ref,\n Output: output,\n }))\n}\n\nresp, err = genkit.Generate(ctx, g,\n ai.WithMessages(append(resp.History(), ai.NewMessage(ai.RoleTool, nil, parts...))...),\n)\nif err != nil {\n log.Fatal(err)\n}\n```\n", + "text": "# Tool calling\n\n_Tool calling_, also known as _function calling_, is a structured way to give\nLLMs the ability to make requests back to the application that called it. You\ndefine the tools you want to make available to the model, and the model will\nmake tool requests to your app as necessary to fulfill the prompts you give it.\n\nThe use cases of tool calling generally fall into a few themes:\n\n**Giving an LLM access to information it wasn't trained with**\n\n- Frequently changing information, such as a stock price or the current\n weather.\n- Information specific to your app domain, such as product information or user\n profiles.\n\nNote the overlap with [retrieval augmented generation](/go/docs/rag) (RAG), which is also\na way to let an LLM integrate factual information into its generations. RAG is a\nheavier solution that is most suited when you have a large amount of information\nor the information that's most relevant to a prompt is ambiguous. On the other\nhand, if a function call or database lookup is all that's necessary for\nretrieving the information the LLM needs, tool calling is more appropriate.\n\n**Introducing a degree of determinism into an LLM workflow**\n\n- Performing calculations that the LLM cannot reliably complete itself.\n- Forcing an LLM to generate verbatim text under certain circumstances, such\n as when responding to a question about an app's terms of service.\n\n**Performing an action when initiated by an LLM**\n\n- Turning on and off lights in an LLM-powered home assistant\n- Reserving table reservations in an LLM-powered restaurant agent\n\n## Before you begin\n\nIf you want to run the code examples on this page, first complete the steps in\nthe [Get started](/go/docs/get-started-go) guide. All of the examples assume that you\nhave already set up a project with Genkit dependencies installed.\n\nThis page discusses one of the advanced features of Genkit model abstraction, so\nbefore you dive too deeply, you should be familiar with the content on the\n[Generating content with AI models](/go/docs/models) page. You should also be familiar\nwith Genkit's system for defining input and output schemas, which is discussed\non the [Flows](/go/docs/flows) page.\n\n## Overview of tool calling\n\nAt a high level, this is what a typical tool-calling interaction with an LLM\nlooks like:\n\n1. The calling application prompts the LLM with a request and also includes in\n the prompt a list of tools the LLM can use to generate a response.\n2. The LLM either generates a complete response or generates a tool call\n request in a specific format.\n3. If the caller receives a complete response, the request is fulfilled and the\n interaction ends; but if the caller receives a tool call, it performs\n whatever logic is appropriate and sends a new request to the LLM containing\n the original prompt or some variation of it as well as the result of the\n tool call.\n4. The LLM handles the new prompt as in Step 2.\n\nFor this to work, several requirements must be met:\n\n- The model must be trained to make tool requests when it's needed to complete\n a prompt. Most of the larger models provided through web APIs such as Gemini\n can do this, but smaller and more specialized models often cannot. Genkit\n will throw an error if you try to provide tools to a model that doesn't\n support it.\n- The calling application must provide tool definitions to the model in the\n format it expects.\n- The calling application must prompt the model to generate tool calling\n requests in the format the application expects.\n\n## Tool calling with Genkit\n\nGenkit provides a single interface for tool calling with models that support it.\nEach model plugin ensures that the last two criteria mentioned in the previous\nsection are met, and the `genkit.Generate()` function automatically carries out\nthe tool-calling loop described earlier.\n\n### Model support\n\nTool calling support depends on the model, the model API, and the Genkit plugin.\nConsult the relevant documentation to determine if tool calling is likely to be\nsupported. In addition:\n\n- Genkit will throw an error if you try to provide tools to a model that\n doesn't support it.\n- If the plugin exports model references, the `ModelInfo.Supports.Tools`\n property will indicate if it supports tool calling.\n\n### Defining tools\n\nUse the `genkit.DefineTool()` function to write tool definitions:\n\n```go\npackage main\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"log\"\n\n\t\"github.com/firebase/genkit/go/ai\"\n\t\"github.com/firebase/genkit/go/genkit\"\n\t\"github.com/firebase/genkit/go/plugins/googlegenai\"\n)\n\n// Define the input structure for the tool\ntype WeatherInput struct {\n\tLocation string `json:\"location\" jsonschema_description:\"Location to get weather for\"`\n}\n\nfunc main() {\n\tctx := context.Background()\n\n\tg, err := genkit.Init(ctx,\n\t\tgenkit.WithPlugins(&googlegenai.GoogleAI{}),\n\t\tgenkit.WithDefaultModel(\"googleai/gemini-1.5-flash\"), // Updated model name\n\t)\n\tif err != nil {\n\t\tlog.Fatalf(\"Genkit initialization failed: %v\", err)\n\t}\n\n\tgenkit.DefineTool(\n\t\tg, \"getWeather\", \"Gets the current weather in a given location\",\n\t\tfunc(ctx context.Context, input WeatherInput) (string, error) {\n\t\t\t// Here, we would typically make an API call or database query. For this\n\t\t\t// example, we just return a fixed value.\n\t\t\tlog.Printf(\"Tool 'getWeather' called for location: %s\", input.Location)\n\t\t\treturn fmt.Sprintf(\"The current weather in %s is 63°F and sunny.\", input.Location), nil\n\t\t})\n}\n```\n\nThe syntax here looks just like the `genkit.DefineFlow()` syntax; however, you\nmust write a description. Take special care with the wording and descriptiveness\nof the description as it is vital for the LLM to decide to use it appropriately.\n\n### Using tools\n\nInclude defined tools in your prompts to generate content.\n\n**Using `genkit.Generate()`:**\n\n```go\nresp, err := genkit.Generate(ctx, g,\n\tai.WithPrompt(\"What is the weather in San Francisco?\"),\n\tai.WithTools(getWeatherTool),\n)\n```\n\n**Using `genkit.DefinePrompt()`:**\n\n```go\nweatherPrompt, err := genkit.DefinePrompt(g, \"weatherPrompt\",\n\tai.WithPrompt(\"What is the weather in {% verbatim %}{{location}}{% endverbatim %}?\"),\n\tai.WithTools(getWeatherTool),\n)\nif err != nil {\n\tlog.Fatal(err)\n}\n\nresp, err := weatherPrompt.Execute(ctx,\n\twith.Input(map[string]any{\"location\": \"San Francisco\"}),\n)\n```\n\n**Using a `.prompt` file:**\n\nCreate a file named `prompts/weatherPrompt.prompt` (assuming default prompt directory):\n\n```dotprompt\n---\nsystem: \"Answer questions using the tools you have.\"\ntools: [getWeather]\ninput:\n schema:\n location: string\n---\n\nWhat is the weather in {{location}}?\n```\n\nThen execute it in your Go code:\n\n```go\n// Assuming prompt file named weatherPrompt.prompt exists in ./prompts dir.\nweatherPrompt := genkit.LookupPrompt(\"weatherPrompt\")\nif weatherPrompt == nil {\n\tlog.Fatal(\"no prompt named 'weatherPrompt' found\")\n}\n\nresp, err := weatherPrompt.Execute(ctx,\n\tai.WithInput(map[string]any{\"location\": \"San Francisco\"}),\n)\n```\n\nGenkit will automatically handle the tool call if the LLM needs to use the\n`getWeather` tool to answer the prompt.\n\n### Explicitly handling tool calls\n\nIf you want full control over this tool-calling loop, for example to apply more\ncomplicated logic, set the `WithReturnToolRequests()` option to `true`. Now it's\nyour responsibility to ensure all of the tool requests are fulfilled:\n\n```go\ngetWeatherTool := genkit.DefineTool(\n g, \"getWeather\", \"Gets the current weather in a given location\",\n func(ctx *ai.ToolContext, location struct {\n Location string `jsonschema_description:\"Location to get weather for\"`\n }) (string, error) {\n // Tool implementation...\n return \"sunny\", nil\n },\n)\n\nresp, err := genkit.Generate(ctx, g,\n ai.WithPrompt(\"What is the weather in San Francisco?\"),\n ai.WithTools(getWeatherTool),\n ai.WithReturnToolRequests(true),\n)\nif err != nil {\n log.Fatal(err)\n}\n\nparts := []*ai.Part{}\nfor _, req := range resp.ToolRequests() {\n tool := genkit.LookupTool(g, req.Name)\n if tool == nil {\n log.Fatalf(\"tool %q not found\", req.Name)\n }\n\n output, err := tool.RunRaw(ctx, req.Input)\n if err != nil {\n log.Fatalf(\"tool %q execution failed: %v\", tool.Name(), err)\n }\n\n parts = append(parts,\n ai.NewToolResponsePart(&ai.ToolResponse{\n Name: req.Name,\n Ref: req.Ref,\n Output: output,\n }))\n}\n\nresp, err = genkit.Generate(ctx, g,\n ai.WithMessages(append(resp.History(), ai.NewMessage(ai.RoleTool, nil, parts...))...),\n)\nif err != nil {\n log.Fatal(err)\n}\n```\n", "title": "Tool calling", - "lang": "go" + "description": "Learn how to use tool calling (function calling) with Genkit Go to give LLMs access to external information and actions.", + "lang": "go", + "headers": "## Before you begin\n## Overview of tool calling\n## Tool calling with Genkit\n### Model support\n### Defining tools\n### Using tools\n### Explicitly handling tool calls\n" + }, + "go/plugins/alloydb.md": { + "text": "# AlloyDB plugin\n\nThe AlloyDB plugin provides provides the retriever implementation to search a [AlloyDB](https://cloud.google.com/alloydb/docs) database using the [pgvector](https://github.com/pgvector/pgvector) extension.\n\n## Configuration\n\n## Configuration\n\nTo use this plugin, follow these steps:\n\n1. Import the plugin\n\n\t```go\n\timport \"github.com/firebase/genkit/go/plugins/alloydb\"\n\t```\n\n2. Create a `PostgresEngine` instance:\n\n\t- Using basic authentication\n\t\t```go\n\t\tpEngine, err := alloydb.NewPostgresEngine(ctx,\n\t\t\t\tWithUser('user'),\n\t\t\t\tWithPassword('password'),\n\t\t\t\tWithAlloyDBInstance('my-project', 'us-central1', 'my-cluster', 'my-instance'),\n\t\t\t\tWithDatabase('my-database')\n\t\t```\n\t- Using email authentication\n\t\t```go\n\t\tpEngine, err := alloydb.NewPostgresEngine(ctx,\n\t\t\tWithAlloyDBInstance('my-project', 'us-central1', 'my-cluster', 'my-instance'),\n\t\t\tWithDatabase('my-database'),\n\t\t\tWithIAMAccountEmail('mail@company.com'))\n\t\t```\n\t- Using custom pool\n\t\t```go\n\n\n\n\t\tpool, err := pgxpool.New(ctx, \"add_your_connection_string\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tpEngine, err := alloydb.NewPostgresEngine(ctx,\n\t\t\tWithDatabase(\"db_test\"),\n\t\t\tWithPool(pool))\n\n\t\t```\n\n3. Create the Postgres plugin\n\t- Using plugin method Init\n\n\n\t\t```go\n\t\t\tpostgres := &alloydb.Postgres{\n\t\t\t\tengine: pEngine,\n\t\t\t}\n\n\t\t\tif err := (postgres).Init(ctx, g); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t```\n\n\t- Using the genkit method init\n\n\t\t```go\n\t\t\tpostgres := &alloydb.Postgres{\n\t\t\t\tengine: pEngine,\n\t\t\t}\n\n\t\t\tg, err := genkit.Init(ctx, genkit.WithPlugins(postgres))\n\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\n\t\t```\n\n## Usage\n\nTo add documents to a AlloyDB index, first create an index definition that specifies the features of the table:\n\n```go\ncfg := &alloydb.Config{\n\tTableName: 'documents',\n\tSchemaName: 'public',\n\tContentColumn: \"content\",\n\tEmbeddingColumn: \"embedding\",\n\tMetadataColumns: []string{\"source\", \"category\"},\n\tIDColumn: \"custom_id\",\n\tMetadataJSONColumn: \"custom_metadata\",\n\tEmbedder: embedder,\n\tEmbedderOptions: nil,\n}\n\ndoc, retriever, err := postgresql.DefineRetriever(ctx, g, postgres, cfg)\nif err != nil {\n return err\n}\n\ndocs := []*ai.Document{{\n Content: []*ai.Part{{\n Kind: ai.PartText,\n ContentType: \"text/plain\",\n Text: \"The product features include...\",\n }},\n Metadata: map[string]any{\"source\": \"website\", \"category\": \"product-docs\", \"custom_id\": \"doc-123\"},\n }}\n\nif err := doc.Index(ctx, docs); err != nil {\n return err\n}\n```\n\nSimilarly, to retrieve documents from an index,use the retriever\nmethod:\n\n```go\ndoc, retriever, err := alloydb.DefineRetriever(ctx, g, postgres, cfg)\nif err != nil {\n return err\n}\n\nd2 := ai.DocumentFromText( \"The product features include...\" , nil)\n\nresp, err := retriever.Retrieve(ctx, &ai.RetrieverRequest{\n Query: d2,\n k:5,\n filter: \"source='website' AND category='product-docs'\"\n})\n\nif err != nil {\n return err\n}\n```\n\nIt's also possible to use the Retrieve method from Retriever\n\n```go\n\nd2 := ai.DocumentFromText( \"The product features include...\" , nil)\n\nretrieverOptions := &alloydb.RetrieverOptions{\n\tk:5,\n filter: \"source='website' AND category='product-docs'\"\n}\n\nresp, err := ai.Retrieve(ctx, retriever,ai.WithDocs(d2), &ai.WithConfig(retrieverOptions))\nif err != nil {\n return err\n}\n```\n\n\nSee the [Retrieval-augmented generation](/go/docs/rag) page for a general\ndiscussion on using retrievers for RAG.\n", + "title": "AlloyDB plugin", + "description": "Learn how to configure and use the AlloyDB plugin as a retriever implementation in Genkit Go.", + "lang": "go", + "headers": "## Configuration\n## Configuration\n## Usage\n" + }, + "go/plugins/cloud-sql-pg.md": { + "text": "# Cloud SQL for PostgreSQL plugin\n\nThe Postgresql plugin provides the retriever implementation to search a [Cloud SQL for Postgresql](https://cloud.google.com/sql/docs/postgres) database using the [pgvector](https://github.com/pgvector/pgvector) extension.\n\n## Configuration\n\nTo use this plugin, follow these steps:\n\n1. Import the plugin\n\n\t```go\n\timport \"github.com/firebase/genkit/go/plugins/postgresql\"\n\t```\n\n2. Create a `PostgresEngine` instance:\n\n\t- Using basic authentication\n\t\t```go\n\t\tpEngine, err := NewPostgresEngine(ctx,\n\t\t\t\tWithUser('user'),\n\t\t\t\tWithPassword('password'),\n\t\t\t\tWithCloudSQLInstance('my-project', 'us-central1', 'my-instance'),\n\t\t\t\tWithDatabase('my-database')\n\t\t```\n\t- Using email authentication\n\t\t```go\n\t\tpEngine, err := NewPostgresEngine(ctx,\n\t\t\tWithCloudSQLInstance('my-project', 'us-central1', 'my-instance'),\n\t\t\tWithDatabase('my-database'),\n\t\t\tWithIAMAccountEmail('mail@company.com'))\n\t\t```\n\t- Using custom pool\n\t\t```go\n\t\tpool, err := pgxpool.New(ctx, \"add_your_connection_string\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tpEngine, err := NewPostgresEngine(ctx,\n\t\t\tWithDatabase(\"db_test\"),\n\t\t\tWithPool(pool))\n\n\t\t```\n\n3. Create the Postgres plugin\n\t- Using plugin method Init\n\n\n\t\t```go\n\t\t\tpostgres := &postgresql.Postgres{\n\t\t\t\tengine: pEngine,\n\t\t\t}\n\n\t\t\tif err := (postgres).Init(ctx, g); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t```\n\n\t- Using the genkit method init\n\n\t\t```go\n\t\t\tpostgres := &postgresql.Postgres{\n\t\t\t\tengine: pEngine,\n\t\t\t}\n\n\t\t\tg, err := genkit.Init(ctx, genkit.WithPlugins(postgres))\n\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\n\t\t```\n\n## Usage\n\nTo add documents to a Postgresql index, first create a retrieve definition that specifies the features of the table:\n\n```go\ncfg := &postgresql.Config{\n\tTableName: 'documents',\n\tSchemaName: 'public',\n\tContentColumn: \"content\",\n\tEmbeddingColumn: \"embedding\",\n\tMetadataColumns: []string{\"source\", \"category\"},\n\tIDColumn: \"custom_id\",\n\tMetadataJSONColumn: \"custom_metadata\",\n\tEmbedder: embedder,\n\tEmbedderOptions: nil,\n}\n\ndoc, retriever, err := postgresql.DefineRetriever(ctx, g, postgres, cfg)\nif err != nil {\n return err\n}\n\ndocs := []*ai.Document{{\n Content: []*ai.Part{{\n Kind: ai.PartText,\n ContentType: \"text/plain\",\n Text: \"The product features include...\",\n }},\n Metadata: map[string]any{\"source\": \"website\", \"category\": \"product-docs\", \"custom_id\": \"doc-123\"},\n }}\n\nif err := doc.Index(ctx, docs); err != nil {\n return err\n}\n\n```\n\nSimilarly, to retrieve documents from an index, use the retrieve method:\n\n```go\nd2 := ai.DocumentFromText( \"The product features include...\" , nil)\n\nresp, err := retriever.Retrieve(ctx, &ai.RetrieverRequest{\n Query: d2,\n k:5,\n filter: \"source='website' AND category='product-docs'\"\n})\n\nif err != nil {\n return err\n}\n```\n\nIt's also possible to use the Retrieve method from Retriever\n\n```go\n_, retriever, err := postgresql.DefineRetriever(ctx, g, postgres, cfg)\nif err != nil {\n return err\n}\n\nd2 := ai.DocumentFromText( \"The product features include...\" , nil)\n\nretrieverOptions := &postgresql.RetrieverOptions{\n\tk:5,\n filter: \"source='website' AND category='product-docs'\"\n}\n\nresp, err := ai.Retrieve(ctx, retriever,ai.WithDocs(d2), &ai.WithConfig(retrieverOptions))\nif err != nil {\n return err\n}\n```\n\n\nSee the [Retrieval-augmented generation](/go/docs/rag) page for a general\ndiscussion on using retrievers for RAG.\n", + "title": "Cloud SQL for PostgreSQL plugin", + "description": "Learn how to configure and use the PostgreSQL plugin as a retriever implementation in Genkit Go.", + "lang": "go", + "headers": "## Configuration\n## Usage\n" }, "go/plugins/firebase.md": { - "text": "The Firebase plugin provides integration with Firebase services for Genkit applications. It enables you to use Firebase Firestore as a vector database for retrieval-augmented generation (RAG) applications by defining retrievers and indexers.\n\n## Prerequisites\n\nThis plugin requires:\n\n- A Firebase project - Create one at the [Firebase Console](https://console.firebase.google.com/)\n- Firestore database enabled in your Firebase project\n- Firebase credentials configured for your application\n\n### Firebase Setup\n\n1. **Create a Firebase project** at [Firebase Console](https://console.firebase.google.com/)\n2. **Enable Firestore** in your project:\n - Go to Firestore Database in the Firebase console\n - Click \"Create database\"\n - Choose your security rules and location\n3. **Set up authentication** using one of these methods:\n - For local development: `firebase login` and `firebase use `\n - For production: Service account key or Application Default Credentials\n\n## Configuration\n\n### Basic Configuration\n\nTo use this plugin, import the `firebase` package and initialize it with your project:\n\n```go\nimport \"github.com/firebase/genkit/go/plugins/firebase\"\n```\n\n```go\n// Option 1: Using project ID (recommended)\nfirebasePlugin := &firebase.Firebase{\n ProjectId: \"your-firebase-project-id\",\n}\n\ng, err := genkit.Init(context.Background(), genkit.WithPlugins(firebasePlugin))\nif err != nil {\n log.Fatal(err)\n}\n```\n\n### Environment Variable Configuration\n\nYou can also configure the project ID using environment variables:\n\n```bash\nexport FIREBASE_PROJECT_ID=your-firebase-project-id\n```\n\n```go\n// Plugin will automatically use FIREBASE_PROJECT_ID environment variable\nfirebasePlugin := &firebase.Firebase{}\ng, err := genkit.Init(context.Background(), genkit.WithPlugins(firebasePlugin))\n```\n\n### Advanced Configuration\n\nFor advanced use cases, you can provide a pre-configured Firebase app:\n\n```go\nimport firebasev4 \"firebase.google.com/go/v4\"\n\n// Create Firebase app with custom configuration\napp, err := firebasev4.NewApp(ctx, &firebasev4.Config{\n ProjectID: \"your-project-id\",\n // Additional Firebase configuration options\n})\nif err != nil {\n log.Fatal(err)\n}\n\nfirebasePlugin := &firebase.Firebase{\n App: app,\n}\n```\n\n## Usage\n\n### Defining Firestore Retrievers\n\nThe primary use case for the Firebase plugin is creating retrievers for RAG applications:\n\n```go\n// Define a Firestore retriever\nretrieverOptions := firebase.RetrieverOptions{\n Name: \"my-documents\",\n Collection: \"documents\",\n VectorField: \"embedding\",\n EmbedderName: \"text-embedding-3-small\",\n TopK: 10,\n}\n\nretriever, err := firebase.DefineRetriever(ctx, g, retrieverOptions)\nif err != nil {\n log.Fatal(err)\n}\n```\n\n### Using Retrievers in RAG Workflows\n\nOnce defined, you can use the retriever in your RAG workflows:\n\n```go\n// Retrieve relevant documents\nresults, err := ai.Retrieve(ctx, retriever, ai.WithDocs(\"What is machine learning?\"))\nif err != nil {\n log.Fatal(err)\n}\n\n// Use retrieved documents in generation\nvar contextDocs []string\nfor _, doc := range results.Documents {\n contextDocs = append(contextDocs, doc.Content[0].Text)\n}\n\ncontext := strings.Join(contextDocs, \"\\n\\n\")\nresp, err := genkit.Generate(ctx, g,\n ai.WithPrompt(fmt.Sprintf(\"Context: %s\\n\\nQuestion: %s\", context, \"What is machine learning?\")),\n)\n```\n\n### Complete RAG Example\n\nHere's a complete example showing how to set up a RAG system with Firebase:\n\n```go\npackage main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"strings\"\n\n \"github.com/firebase/genkit/go/ai\"\n \"github.com/firebase/genkit/go/genkit\"\n \"github.com/firebase/genkit/go/plugins/firebase\"\n \"github.com/firebase/genkit/go/plugins/compat_oai/openai\"\n)\n\nfunc main() {\n ctx := context.Background()\n\n // Initialize plugins\n firebasePlugin := &firebase.Firebase{\n ProjectId: \"my-firebase-project\",\n }\n \n openaiPlugin := &openai.OpenAI{\n APIKey: \"your-openai-api-key\",\n }\n\n g, err := genkit.Init(ctx, genkit.WithPlugins(firebasePlugin, openaiPlugin))\n if err != nil {\n log.Fatal(err)\n }\n\n // Define retriever for knowledge base\n retriever, err := firebase.DefineRetriever(ctx, g, firebase.RetrieverOptions{\n Name: \"knowledge-base\",\n Collection: \"documents\",\n VectorField: \"embedding\",\n EmbedderName: \"text-embedding-3-small\",\n TopK: 5,\n })\n if err != nil {\n log.Fatal(err)\n }\n\n // RAG query function\n query := \"How does machine learning work?\"\n \n // Step 1: Retrieve relevant documents\n retrievalResults, err := ai.Retrieve(ctx, retriever, ai.WithDocs(query))\n if err != nil {\n log.Fatal(err)\n }\n\n // Step 2: Prepare context from retrieved documents\n var contextParts []string\n for _, doc := range retrievalResults.Documents {\n contextParts = append(contextParts, doc.Content[0].Text)\n }\n context := strings.Join(contextParts, \"\\n\\n\")\n\n // Step 3: Generate answer with context\n model := openaiPlugin.Model(g, \"gpt-4o\")\n response, err := genkit.Generate(ctx, g,\n ai.WithModel(model),\n ai.WithPrompt(fmt.Sprintf(`\nBased on the following context, answer the question:\n\nContext:\n%s\n\nQuestion: %s\n\nAnswer:`, context, query)),\n )\n if err != nil {\n log.Fatal(err)\n }\n\n fmt.Printf(\"Answer: %s\\n\", response.Text())\n}\n```\n\n## Firestore Data Structure\n\n### Document Storage Format\n\nYour Firestore documents should follow this structure for optimal retrieval:\n\n```json\n{\n \"content\": \"Your document text content here...\",\n \"embedding\": [0.1, -0.2, 0.3, ...],\n \"metadata\": {\n \"title\": \"Document Title\",\n \"author\": \"Author Name\",\n \"category\": \"Technology\",\n \"timestamp\": \"2024-01-15T10:30:00Z\"\n }\n}\n```\n\n### Indexing Documents\n\nTo add documents to your Firestore collection with embeddings:\n\n```go\n// Example of adding documents with embeddings\nembedder := openaiPlugin.Embedder(g, \"text-embedding-3-small\")\n\ndocuments := []struct {\n Content string\n Metadata map[string]interface{}\n}{\n {\n Content: \"Machine learning is a subset of artificial intelligence...\",\n Metadata: map[string]interface{}{\n \"title\": \"Introduction to ML\",\n \"category\": \"Technology\",\n },\n },\n // More documents...\n}\n\nfor _, doc := range documents {\n // Generate embedding\n embeddingResp, err := ai.Embed(ctx, embedder, ai.WithDocs(doc.Content))\n if err != nil {\n log.Fatal(err)\n }\n\n // Store in Firestore\n firestoreClient, _ := firebasePlugin.App.Firestore(ctx)\n _, err = firestoreClient.Collection(\"documents\").Doc().Set(ctx, map[string]interface{}{\n \"content\": doc.Content,\n \"embedding\": embeddingResp.Embeddings[0].Embedding,\n \"metadata\": doc.Metadata,\n })\n if err != nil {\n log.Fatal(err)\n }\n}\n```\n\n## Configuration Options\n\n### Firebase struct\n\n```go\ntype Firebase struct {\n // ProjectId is your Firebase project ID\n // If empty, uses FIREBASE_PROJECT_ID environment variable\n ProjectId string\n \n // App is a pre-configured Firebase app instance\n // Use either ProjectId or App, not both\n App *firebasev4.App\n}\n```\n\n### RetrieverOptions\n\n```go\ntype RetrieverOptions struct {\n // Name is a unique identifier for the retriever\n Name string\n \n // Collection is the Firestore collection name containing documents\n Collection string\n \n // VectorField is the field name containing the embedding vectors\n VectorField string\n \n // EmbedderName is the name of the embedder to use for query vectorization\n EmbedderName string\n \n // TopK is the number of top similar documents to retrieve\n TopK int\n \n // Additional filtering and configuration options\n}\n```\n\n## Authentication\n\n### Local Development\n\nFor local development, use the Firebase CLI:\n\n```bash\n# Install Firebase CLI\nnpm install -g firebase-tools\n\n# Login and set project\nfirebase login\nfirebase use your-project-id\n```\n\n### Production Deployment\n\nFor production, use one of these authentication methods:\n\n#### Service Account Key\n\n```go\nimport \"google.golang.org/api/option\"\n\napp, err := firebasev4.NewApp(ctx, &firebasev4.Config{\n ProjectID: \"your-project-id\",\n}, option.WithCredentialsFile(\"path/to/serviceAccountKey.json\"))\n```\n\n#### Application Default Credentials\n\nSet the environment variable:\n\n```bash\nexport GOOGLE_APPLICATION_CREDENTIALS=\"path/to/serviceAccountKey.json\"\n```\n\nOr use the metadata server on Google Cloud Platform.\n\n## Error Handling\n\nHandle Firebase-specific errors appropriately:\n\n```go\nretriever, err := firebase.DefineRetriever(ctx, g, options)\nif err != nil {\n if strings.Contains(err.Error(), \"plugin not found\") {\n log.Fatal(\"Firebase plugin not initialized. Make sure to include it in genkit.Init()\")\n }\n log.Fatalf(\"Failed to create retriever: %v\", err)\n}\n\n// Handle retrieval errors\nresults, err := ai.Retrieve(ctx, retriever, ai.WithDocs(query))\nif err != nil {\n log.Printf(\"Retrieval failed: %v\", err)\n // Implement fallback logic\n}\n```\n\n## Best Practices\n\n### Performance Optimization\n\n- **Batch Operations**: Use Firestore batch writes when adding multiple documents\n- **Index Configuration**: Set up appropriate Firestore indexes for your queries\n- **Caching**: Implement caching for frequently accessed documents\n- **Pagination**: Use pagination for large result sets\n\n### Security\n\n- **Firestore Rules**: Configure proper security rules for your collections\n- **API Keys**: Never expose Firebase configuration in client-side code\n- **Authentication**: Implement proper user authentication for sensitive data\n\n### Cost Management\n\n- **Document Size**: Keep documents reasonably sized to minimize read costs\n- **Query Optimization**: Design efficient queries to reduce operation costs\n- **Storage Management**: Regularly clean up unused documents and embeddings\n\n## Integration Examples\n\n### With Multiple Embedders\n\n```go\n// Use different embedders for different types of content\ntechnicalRetriever, err := firebase.DefineRetriever(ctx, g, firebase.RetrieverOptions{\n Name: \"technical-docs\",\n Collection: \"technical_documents\",\n VectorField: \"embedding\",\n EmbedderName: \"text-embedding-3-large\", // More accurate for technical content\n TopK: 5,\n})\n\ngeneralRetriever, err := firebase.DefineRetriever(ctx, g, firebase.RetrieverOptions{\n Name: \"general-knowledge\",\n Collection: \"general_documents\", \n VectorField: \"embedding\",\n EmbedderName: \"text-embedding-3-small\", // Faster for general content\n TopK: 10,\n})\n```\n\n### With Flows\n\n```go\nragFlow := genkit.DefineFlow(g, \"rag-qa\", func(ctx context.Context, query string) (string, error) {\n // Retrieve context\n results, err := ai.Retrieve(ctx, retriever, ai.WithDocs(query))\n if err != nil {\n return \"\", err\n }\n \n // Generate response\n response, err := genkit.Generate(ctx, g,\n ai.WithPrompt(buildPromptWithContext(query, results)),\n )\n if err != nil {\n return \"\", err\n }\n \n return response.Text(), nil\n})\n```\n", + "text": "# Firebase plugin\n\nThe Firebase plugin provides integration with Firebase services for Genkit applications. It enables you to use Firebase Firestore as a vector database for retrieval-augmented generation (RAG) applications by defining retrievers.\n\n## Prerequisites\n\nThis plugin requires:\n\n- A Firebase project - Create one at the [Firebase Console](https://console.firebase.google.com/)\n- Firestore database enabled in your Firebase project\n- Firebase credentials configured for your application\n\n### Firebase Setup\n\n1. **Create a Firebase project** at [Firebase Console](https://console.firebase.google.com/)\n2. **Enable Firestore** in your project:\n - Go to Firestore Database in the Firebase console\n - Click \"Create database\"\n - Choose your security rules and location\n3. **Set up authentication** using one of these methods:\n - For local development: `firebase login` and `firebase use `\n - For production: Service account key or Application Default Credentials\n\n## Configuration\n\n### Basic Configuration\n\nTo use this plugin, import the `firebase` package and initialize it with your project:\n\n```go\nimport \"github.com/firebase/genkit/go/plugins/firebase\"\n```\n\n```go\n// Option 1: Using project ID (recommended)\nfirebasePlugin := &firebase.Firebase{\n ProjectId: \"your-firebase-project-id\",\n}\n\ng, err := genkit.Init(context.Background(), genkit.WithPlugins(firebasePlugin))\nif err != nil {\n log.Fatal(err)\n}\n```\n\n### Environment Variable Configuration\n\nYou can also configure the project ID using environment variables:\n\n```bash\nexport FIREBASE_PROJECT_ID=your-firebase-project-id\n```\n\n```go\n// Plugin will automatically use FIREBASE_PROJECT_ID environment variable\nfirebasePlugin := &firebase.Firebase{}\ng, err := genkit.Init(context.Background(), genkit.WithPlugins(firebasePlugin))\n```\n\n### Advanced Configuration\n\nFor advanced use cases, you can provide a pre-configured Firebase app:\n\n```go\nimport firebasev4 \"firebase.google.com/go/v4\"\n\n// Create Firebase app with custom configuration\napp, err := firebasev4.NewApp(ctx, &firebasev4.Config{\n ProjectID: \"your-project-id\",\n // Additional Firebase configuration options\n})\nif err != nil {\n log.Fatal(err)\n}\n\nfirebasePlugin := &firebase.Firebase{\n App: app,\n}\n```\n\n## Usage\n\n### Defining Firestore Retrievers\n\nThe primary use case for the Firebase plugin is creating retrievers for RAG applications:\n\n```go\n// Define a Firestore retriever\nretrieverOptions := firebase.RetrieverOptions{\n Name: \"my-documents\",\n Collection: \"documents\",\n VectorField: \"embedding\",\n EmbedderName: \"text-embedding-3-small\",\n TopK: 10,\n}\n\nretriever, err := firebase.DefineRetriever(ctx, g, retrieverOptions)\nif err != nil {\n log.Fatal(err)\n}\n```\n\n### Using Retrievers in RAG Workflows\n\nOnce defined, you can use the retriever in your RAG workflows:\n\n```go\n// Retrieve relevant documents\nresults, err := ai.Retrieve(ctx, retriever, ai.WithDocs(\"What is machine learning?\"))\nif err != nil {\n log.Fatal(err)\n}\n\n// Use retrieved documents in generation\nvar contextDocs []string\nfor _, doc := range results.Documents {\n contextDocs = append(contextDocs, doc.Content[0].Text)\n}\n\ncontext := strings.Join(contextDocs, \"\\n\\n\")\nresp, err := genkit.Generate(ctx, g,\n ai.WithPrompt(fmt.Sprintf(\"Context: %s\\n\\nQuestion: %s\", context, \"What is machine learning?\")),\n)\n```\n\n### Complete RAG Example\n\nHere's a complete example showing how to set up a RAG system with Firebase:\n\n```go\npackage main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"strings\"\n\n \"github.com/firebase/genkit/go/ai\"\n \"github.com/firebase/genkit/go/genkit\"\n \"github.com/firebase/genkit/go/plugins/firebase\"\n \"github.com/firebase/genkit/go/plugins/compat_oai/openai\"\n)\n\nfunc main() {\n ctx := context.Background()\n\n // Initialize plugins\n firebasePlugin := &firebase.Firebase{\n ProjectId: \"my-firebase-project\",\n }\n \n openaiPlugin := &openai.OpenAI{\n APIKey: \"your-openai-api-key\",\n }\n\n g, err := genkit.Init(ctx, genkit.WithPlugins(firebasePlugin, openaiPlugin))\n if err != nil {\n log.Fatal(err)\n }\n\n // Define retriever for knowledge base\n retriever, err := firebase.DefineRetriever(ctx, g, firebase.RetrieverOptions{\n Name: \"knowledge-base\",\n Collection: \"documents\",\n VectorField: \"embedding\",\n EmbedderName: \"text-embedding-3-small\",\n TopK: 5,\n })\n if err != nil {\n log.Fatal(err)\n }\n\n // RAG query function\n query := \"How does machine learning work?\"\n \n // Step 1: Retrieve relevant documents\n retrievalResults, err := ai.Retrieve(ctx, retriever, ai.WithDocs(query))\n if err != nil {\n log.Fatal(err)\n }\n\n // Step 2: Prepare context from retrieved documents\n var contextParts []string\n for _, doc := range retrievalResults.Documents {\n contextParts = append(contextParts, doc.Content[0].Text)\n }\n context := strings.Join(contextParts, \"\\n\\n\")\n\n // Step 3: Generate answer with context\n model := openaiPlugin.Model(g, \"gpt-4o\")\n response, err := genkit.Generate(ctx, g,\n ai.WithModel(model),\n ai.WithPrompt(fmt.Sprintf(`\nBased on the following context, answer the question:\n\nContext:\n%s\n\nQuestion: %s\n\nAnswer:`, context, query)),\n )\n if err != nil {\n log.Fatal(err)\n }\n\n fmt.Printf(\"Answer: %s\\n\", response.Text())\n}\n```\n\n## Firestore Data Structure\n\n### Document Storage Format\n\nYour Firestore documents should follow this structure for optimal retrieval:\n\n```json\n{\n \"content\": \"Your document text content here...\",\n \"embedding\": [0.1, -0.2, 0.3, ...],\n \"metadata\": {\n \"title\": \"Document Title\",\n \"author\": \"Author Name\",\n \"category\": \"Technology\",\n \"timestamp\": \"2024-01-15T10:30:00Z\"\n }\n}\n```\n\n### Indexing Documents\n\nTo add documents to your Firestore collection with embeddings:\n\n```go\n// Example of adding documents with embeddings\nembedder := openaiPlugin.Embedder(g, \"text-embedding-3-small\")\n\ndocuments := []struct {\n Content string\n Metadata map[string]interface{}\n}{\n {\n Content: \"Machine learning is a subset of artificial intelligence...\",\n Metadata: map[string]interface{}{\n \"title\": \"Introduction to ML\",\n \"category\": \"Technology\",\n },\n },\n // More documents...\n}\n\nfor _, doc := range documents {\n // Generate embedding\n embeddingResp, err := ai.Embed(ctx, embedder, ai.WithDocs(doc.Content))\n if err != nil {\n log.Fatal(err)\n }\n\n // Store in Firestore\n firestoreClient, _ := firebasePlugin.App.Firestore(ctx)\n _, err = firestoreClient.Collection(\"documents\").Doc().Set(ctx, map[string]interface{}{\n \"content\": doc.Content,\n \"embedding\": embeddingResp.Embeddings[0].Embedding,\n \"metadata\": doc.Metadata,\n })\n if err != nil {\n log.Fatal(err)\n }\n}\n```\n\n## Configuration Options\n\n### Firebase struct\n\n```go\ntype Firebase struct {\n // ProjectId is your Firebase project ID\n // If empty, uses FIREBASE_PROJECT_ID environment variable\n ProjectId string\n \n // App is a pre-configured Firebase app instance\n // Use either ProjectId or App, not both\n App *firebasev4.App\n}\n```\n\n### RetrieverOptions\n\n```go\ntype RetrieverOptions struct {\n // Name is a unique identifier for the retriever\n Name string\n \n // Collection is the Firestore collection name containing documents\n Collection string\n \n // VectorField is the field name containing the embedding vectors\n VectorField string\n \n // EmbedderName is the name of the embedder to use for query vectorization\n EmbedderName string\n \n // TopK is the number of top similar documents to retrieve\n TopK int\n \n // Additional filtering and configuration options\n}\n```\n\n## Authentication\n\n### Local Development\n\nFor local development, use the Firebase CLI:\n\n```bash\n# Install Firebase CLI\nnpm install -g firebase-tools\n\n# Login and set project\nfirebase login\nfirebase use your-project-id\n```\n\n### Production Deployment\n\nFor production, use one of these authentication methods:\n\n#### Service Account Key\n\n```go\nimport \"google.golang.org/api/option\"\n\napp, err := firebasev4.NewApp(ctx, &firebasev4.Config{\n ProjectID: \"your-project-id\",\n}, option.WithCredentialsFile(\"path/to/serviceAccountKey.json\"))\n```\n\n#### Application Default Credentials\n\nSet the environment variable:\n\n```bash\nexport GOOGLE_APPLICATION_CREDENTIALS=\"path/to/serviceAccountKey.json\"\n```\n\nOr use the metadata server on Google Cloud Platform.\n\n## Error Handling\n\nHandle Firebase-specific errors appropriately:\n\n```go\nretriever, err := firebase.DefineRetriever(ctx, g, options)\nif err != nil {\n if strings.Contains(err.Error(), \"plugin not found\") {\n log.Fatal(\"Firebase plugin not initialized. Make sure to include it in genkit.Init()\")\n }\n log.Fatalf(\"Failed to create retriever: %v\", err)\n}\n\n// Handle retrieval errors\nresults, err := ai.Retrieve(ctx, retriever, ai.WithDocs(query))\nif err != nil {\n log.Printf(\"Retrieval failed: %v\", err)\n // Implement fallback logic\n}\n```\n\n## Best Practices\n\n### Performance Optimization\n\n- **Batch Operations**: Use Firestore batch writes when adding multiple documents\n- **Index Configuration**: Set up appropriate Firestore indexes for your queries\n- **Caching**: Implement caching for frequently accessed documents\n- **Pagination**: Use pagination for large result sets\n\n### Security\n\n- **Firestore Rules**: Configure proper security rules for your collections\n- **API Keys**: Never expose Firebase configuration in client-side code\n- **Authentication**: Implement proper user authentication for sensitive data\n\n### Cost Management\n\n- **Document Size**: Keep documents reasonably sized to minimize read costs\n- **Query Optimization**: Design efficient queries to reduce operation costs\n- **Storage Management**: Regularly clean up unused documents and embeddings\n\n## Integration Examples\n\n### With Multiple Embedders\n\n```go\n// Use different embedders for different types of content\ntechnicalRetriever, err := firebase.DefineRetriever(ctx, g, firebase.RetrieverOptions{\n Name: \"technical-docs\",\n Collection: \"technical_documents\",\n VectorField: \"embedding\",\n EmbedderName: \"text-embedding-3-large\", // More accurate for technical content\n TopK: 5,\n})\n\ngeneralRetriever, err := firebase.DefineRetriever(ctx, g, firebase.RetrieverOptions{\n Name: \"general-knowledge\",\n Collection: \"general_documents\", \n VectorField: \"embedding\",\n EmbedderName: \"text-embedding-3-small\", // Faster for general content\n TopK: 10,\n})\n```\n\n### With Flows\n\n```go\nragFlow := genkit.DefineFlow(g, \"rag-qa\", func(ctx context.Context, query string) (string, error) {\n // Retrieve context\n results, err := ai.Retrieve(ctx, retriever, ai.WithDocs(query))\n if err != nil {\n return \"\", err\n }\n \n // Generate response\n response, err := genkit.Generate(ctx, g,\n ai.WithPrompt(buildPromptWithContext(query, results)),\n )\n if err != nil {\n return \"\", err\n }\n \n return response.Text(), nil\n})\n```\n", "title": "Firebase plugin", - "lang": "go" + "description": "Learn how to configure and use the Genkit Firebase plugin for Go to integrate with Firebase services including Firestore for RAG applications.", + "lang": "go", + "headers": "## Prerequisites\n### Firebase Setup\n## Configuration\n### Basic Configuration\n### Environment Variable Configuration\n### Advanced Configuration\n## Usage\n### Defining Firestore Retrievers\n### Using Retrievers in RAG Workflows\n### Complete RAG Example\n## Firestore Data Structure\n### Document Storage Format\n### Indexing Documents\n## Configuration Options\n### Firebase struct\n### RetrieverOptions\n## Authentication\n### Local Development\n# Install Firebase CLI\n# Login and set project\n### Production Deployment\n#### Service Account Key\n#### Application Default Credentials\n## Error Handling\n## Best Practices\n### Performance Optimization\n### Security\n### Cost Management\n## Integration Examples\n### With Multiple Embedders\n### With Flows\n" }, "go/plugins/google-cloud.md": { - "text": "The Google Cloud plugin exports Genkit's telemetry and logging data to\n[Google Cloud's operation suite](https://cloud.google.com/products/operations).\n\nNote: Logging is facilitated by the `slog` package\nin favor of the [OpenTelemetry](https://opentelemetry.io/) logging APIs. Export\nof logs is done via a dedicated Google Cloud exporter.\n\n## Prerequisites\n\nIf you want to locally run flows that use this plugin, you need the\n[Google Cloud CLI tool](https://cloud.google.com/sdk/docs/install) installed.\n\n## Set up a Google Cloud account\n\nThis plugin requires a Google Cloud account ([sign up](https://cloud.google.com/gcp) if you don't already have one) and a Google Cloud project.\n\nPrior to adding the plugin, make sure that the following APIs are enabled for your project:\n\n- [Cloud Logging API](https://console.cloud.google.com/apis/library/logging.googleapis.com)\n- [Cloud Trace API](https://console.cloud.google.com/apis/library/cloudtrace.googleapis.com)\n- [Cloud Monitoring API](https://console.cloud.google.com/apis/library/monitoring.googleapis.com)\n\nThese APIs should be listed in the [API dashboard](https://console.cloud.google.com/apis/dashboard) for your project.\n\nClick [here](https://support.google.com/googleapi/answer/6158841) to learn more about enabling and disabling APIs.\n\n## Configuration\n\nTo enable exporting to Google Cloud Tracing, Logging, and Monitoring, import the\n`googlecloud` package and run `Init()`. After calling `Init()`, your telemetry\ngets automatically exported.\n\n```go\nimport \"github.com/firebase/genkit/go/plugins/googlecloud\"\n```\n\n```go\nif err := (&googlecloud.GoogleCloud{ProjectID: \"your-google-cloud-project\"}).Init(ctx, g); err != nil {\n\treturn err\n}\n```\n\nYou must specify the Google Cloud project to which you want to export telemetry\ndata. There are also some optional parameters:\n\n- `ProjectID`: (Required) Your Google Cloud project ID.\n- `ForceExport`: Export telemetry data even when running in a dev environment\n (such as when using `genkit start` or `genkit flow:run`). This is a quick way\n to test your integration and send your first events for monitoring in Google\n Cloud.\n\n If you use this option, you also need to make your Cloud credentials available\n locally:\n\n ```bash\n gcloud auth application-default login\n ```\n\n- `MetricInterval`: The interval (`time.Duration`) at which to export telemetry\n information. By default, this is 60 seconds.\n- `LogLevel`: The minimum severity level (`slog.Level`) of log entries to export. By default,\n `slog.LevelInfo`.\n\nThe plugin requires your Google Cloud project credentials. If you're running\nyour flows from a Google Cloud environment (Cloud Run, etc), the credentials are\nset automatically. Running in other environments requires setting up\n[Application Default Credentials](https://cloud.google.com/docs/authentication/provide-credentials-adc).\n\n## Production monitoring via Google Cloud's operations suite\n\nOnce a flow is deployed, navigate to [Google Cloud's operations suite](https://console.cloud.google.com/) and select your project.\n\n![Google Cloud Operations Suite dashboard](../resources/cloud-ops-suite.png)\n\n### Logs and traces\n\nFrom the side menu, find 'Logging' and click 'Logs explorer'.\n\n![Logs Explorer menu item in Cloud Logging](../resources/cloud-ops-logs-explorer-menu.png)\n\nYou will see all logs that are associated with your deployed flow, including `console.log()`. Any log which has the prefix `[genkit]` is a Genkit-internal log that contains information that may be interesting for debugging purposes. For example, Genkit logs in the format `Config[...]` contain metadata such as the temperature and topK values for specific LLM inferences. Logs in the format `Output[...]` contain LLM responses while `Input[...]` logs contain the prompts. Cloud Logging has robust ACLs that allow fine grained control over sensitive logs.\n\n> Note: Prompts and LLM responses are redacted from trace attributes in Cloud Trace.\n\nFor specific log lines, it is possible to navigate to their respective traces by clicking on the extended menu ![Log line menu icon](../resources/cloud-ops-log-menu-icon.png) icon and selecting \"View in trace details\".\n\n![View in trace details option in log menu](../resources/cloud-ops-view-trace-details.png)\n\nThis will bring up a trace preview pane providing a quick glance of the details of the trace. To get to the full details, click the \"View in Trace\" link at the top right of the pane.\n\n![View in Trace link in trace preview pane](../resources/cloud-ops-view-in-trace.png)\n\nThe most prominent navigation element in Cloud Trace is the trace scatter plot. It contains all collected traces in a given time span.\n\n![Cloud Trace scatter plot](../resources/cloud-ops-trace-graph.png)\n\nClicking on each data point will show its details below the scatter plot.\n\n![Cloud Trace details view](../resources/cloud-ops-trace-view.png)\n\nThe detailed view contains the flow shape, including all steps, and important timing information. Cloud Trace has the ability to interleave all logs associated with a given trace within this view. Select the \"Show expanded\" option in the \"Logs & events\" drop down.\n\n![Show expanded option in Logs & events dropdown](../resources/cloud-ops-show-expanded.png)\n\nThe resultant view allows detailed examination of logs in the context of the trace, including prompts and LLM responses.\n\n![Trace details view with expanded logs](../resources/cloud-ops-output-logs.png)\n\n### Metrics\n\nViewing all metrics that Genkit exports can be done by selecting \"Logging\" from the side menu and clicking on \"Metrics management\".\n\n![Metrics Management menu item in Cloud Logging](../resources/cloud-ops-metrics-mgmt.png)\n\nThe metrics management console contains a tabular view of all collected metrics, including those that pertain to Cloud Run and its surrounding environment. Clicking on the 'Workload' option will reveal a list that includes Genkit-collected metrics. Any metric with the `genkit` prefix constitutes an internal Genkit metric.\n\n![Metrics table showing Genkit metrics](../resources/cloud-ops-metrics-table.png)\n\nGenkit collects several categories of metrics, including flow-level, action-level, and generate-level metrics. Each metric has several useful dimensions facilitating robust filtering and grouping.\n\nCommon dimensions include:\n\n- `flow_name` - the top-level name of the flow.\n- `flow_path` - the span and its parent span chain up to the root span.\n- `error_code` - in case of an error, the corresponding error code.\n- `error_message` - in case of an error, the corresponding error message.\n- `model` - the name of the model.\n- `temperature` - the inference temperature [value](https://ai.google.dev/docs/concepts#model-parameters).\n- `topK` - the inference topK [value](https://ai.google.dev/docs/concepts#model-parameters).\n- `topP` - the inference topP [value](https://ai.google.dev/docs/concepts#model-parameters).\n\n#### Flow-level metrics\n\n| Name | Dimensions |\n| -------------------- | ------------------------------------ |\n| genkit/flow/requests | flow_name, error_code, error_message |\n| genkit/flow/latency | flow_name |\n\n#### Action-level metrics\n\n| Name | Dimensions |\n| ---------------------- | ------------------------------------ |\n| genkit/action/requests | flow_name, error_code, error_message |\n| genkit/action/latency | flow_name |\n\n#### Generate-level metrics\n\n| Name | Dimensions |\n| ------------------------------------ | -------------------------------------------------------------------- |\n| genkit/ai/generate | flow_path, model, temperature, topK, topP, error_code, error_message |\n| genkit/ai/generate/input_tokens | flow_path, model, temperature, topK, topP |\n| genkit/ai/generate/output_tokens | flow_path, model, temperature, topK, topP |\n| genkit/ai/generate/input_characters | flow_path, model, temperature, topK, topP |\n| genkit/ai/generate/output_characters | flow_path, model, temperature, topK, topP |\n| genkit/ai/generate/input_images | flow_path, model, temperature, topK, topP |\n| genkit/ai/generate/output_images | flow_path, model, temperature, topK, topP |\n| genkit/ai/generate/latency | flow_path, model, temperature, topK, topP, error_code, error_message |\n\nVisualizing metrics can be done through the Metrics Explorer. Using the side menu, select 'Logging' and click 'Metrics explorer'\n\n![Metrics Explorer menu item in Cloud Logging](../resources/cloud-ops-metrics-explorer.png)\n\nSelect a metrics by clicking on the \"Select a metric\" dropdown, selecting 'Generic Node', 'Genkit', and a metric.\n\n![Selecting a Genkit metric in Metrics Explorer](../resources/cloud-ops-metrics-generic-node.png)\n\nThe visualization of the metric will depend on its type (counter, histogram, etc). The Metrics Explorer provides robust aggregation and querying facilities to help graph metrics by their various dimensions.\n\n![Metrics Explorer showing a Genkit metric graph](../resources/cloud-ops-metrics-metric.png)\n\n## Telemetry Delay\n\nThere may be a slight delay before telemetry for a particular execution of a flow is displayed in Cloud's operations suite. In most cases, this delay is under 1 minute.\n\n## Quotas and limits\n\nThere are several quotas that are important to keep in mind:\n\n- [Cloud Trace Quotas](http://cloud.google.com/trace/docs/quotas)\n - 128 bytes per attribute key\n - 256 bytes per attribute value\n- [Cloud Logging Quotas](http://cloud.google.com/logging/quotas)\n - 256 KB per log entry\n- [Cloud Monitoring Quotas](http://cloud.google.com/monitoring/quotas)\n\n## Cost\n\nCloud Logging, Cloud Trace, and Cloud Monitoring have generous free tiers. Specific pricing can be found at the following links:\n\n- [Cloud Logging Pricing](http://cloud.google.com/stackdriver/pricing#google-cloud-observability-pricing)\n- [Cloud Trace Pricing](https://cloud.google.com/trace#pricing)\n- [Cloud Monitoring Pricing](https://cloud.google.com/stackdriver/pricing#monitoring-pricing-summary)\n", + "text": "# Google Cloud telemetry and logging plugin\n\nThe Google Cloud plugin exports Genkit's telemetry and logging data to\n[Google Cloud's operation suite](https://cloud.google.com/products/operations).\n\nNote: Logging is facilitated by the `slog` package\nin favor of the [OpenTelemetry](https://opentelemetry.io/) logging APIs. Export\nof logs is done via a dedicated Google Cloud exporter.\n\n## Prerequisites\n\nIf you want to locally run flows that use this plugin, you need the\n[Google Cloud CLI tool](https://cloud.google.com/sdk/docs/install) installed.\n\n## Set up a Google Cloud account\n\nThis plugin requires a Google Cloud account ([sign up](https://cloud.google.com/gcp) if you don't already have one) and a Google Cloud project.\n\nPrior to adding the plugin, make sure that the following APIs are enabled for your project:\n\n- [Cloud Logging API](https://console.cloud.google.com/apis/library/logging.googleapis.com)\n- [Cloud Trace API](https://console.cloud.google.com/apis/library/cloudtrace.googleapis.com)\n- [Cloud Monitoring API](https://console.cloud.google.com/apis/library/monitoring.googleapis.com)\n\nThese APIs should be listed in the [API dashboard](https://console.cloud.google.com/apis/dashboard) for your project.\n\nClick [here](https://support.google.com/googleapi/answer/6158841) to learn more about enabling and disabling APIs.\n\n## Configuration\n\nTo enable exporting to Google Cloud Tracing, Logging, and Monitoring, import the\n`googlecloud` package and run `Init()`. After calling `Init()`, your telemetry\ngets automatically exported.\n\n```go\nimport \"github.com/firebase/genkit/go/plugins/googlecloud\"\n```\n\n```go\nif err := (&googlecloud.GoogleCloud{ProjectID: \"your-google-cloud-project\"}).Init(ctx, g); err != nil {\n\treturn err\n}\n```\n\nYou must specify the Google Cloud project to which you want to export telemetry\ndata. There are also some optional parameters:\n\n- `ProjectID`: (Required) Your Google Cloud project ID.\n- `ForceExport`: Export telemetry data even when running in a dev environment\n (such as when using `genkit start` or `genkit flow:run`). This is a quick way\n to test your integration and send your first events for monitoring in Google\n Cloud.\n\n If you use this option, you also need to make your Cloud credentials available\n locally:\n\n ```bash\n gcloud auth application-default login\n ```\n\n- `MetricInterval`: The interval (`time.Duration`) at which to export telemetry\n information. By default, this is 60 seconds.\n- `LogLevel`: The minimum severity level (`slog.Level`) of log entries to export. By default,\n `slog.LevelInfo`.\n\nThe plugin requires your Google Cloud project credentials. If you're running\nyour flows from a Google Cloud environment (Cloud Run, etc), the credentials are\nset automatically. Running in other environments requires setting up\n[Application Default Credentials](https://cloud.google.com/docs/authentication/provide-credentials-adc).\n\n## Production monitoring via Google Cloud's operations suite\n\nOnce a flow is deployed, navigate to [Google Cloud's operations suite](https://console.cloud.google.com/) and select your project.\n\n![Google Cloud Operations Suite dashboard](../resources/cloud-ops-suite.png)\n\n### Logs and traces\n\nFrom the side menu, find 'Logging' and click 'Logs explorer'.\n\n![Logs Explorer menu item in Cloud Logging](../resources/cloud-ops-logs-explorer-menu.png)\n\nYou will see all logs that are associated with your deployed flow, including `console.log()`. Any log which has the prefix `[genkit]` is a Genkit-internal log that contains information that may be interesting for debugging purposes. For example, Genkit logs in the format `Config[...]` contain metadata such as the temperature and topK values for specific LLM inferences. Logs in the format `Output[...]` contain LLM responses while `Input[...]` logs contain the prompts. Cloud Logging has robust ACLs that allow fine grained control over sensitive logs.\n\n> Note: Prompts and LLM responses are redacted from trace attributes in Cloud Trace.\n\nFor specific log lines, it is possible to navigate to their respective traces by clicking on the extended menu ![Log line menu icon](../resources/cloud-ops-log-menu-icon.png) icon and selecting \"View in trace details\".\n\n![View in trace details option in log menu](../resources/cloud-ops-view-trace-details.png)\n\nThis will bring up a trace preview pane providing a quick glance of the details of the trace. To get to the full details, click the \"View in Trace\" link at the top right of the pane.\n\n![View in Trace link in trace preview pane](../resources/cloud-ops-view-in-trace.png)\n\nThe most prominent navigation element in Cloud Trace is the trace scatter plot. It contains all collected traces in a given time span.\n\n![Cloud Trace scatter plot](../resources/cloud-ops-trace-graph.png)\n\nClicking on each data point will show its details below the scatter plot.\n\n![Cloud Trace details view](../resources/cloud-ops-trace-view.png)\n\nThe detailed view contains the flow shape, including all steps, and important timing information. Cloud Trace has the ability to interleave all logs associated with a given trace within this view. Select the \"Show expanded\" option in the \"Logs & events\" drop down.\n\n![Show expanded option in Logs & events dropdown](../resources/cloud-ops-show-expanded.png)\n\nThe resultant view allows detailed examination of logs in the context of the trace, including prompts and LLM responses.\n\n![Trace details view with expanded logs](../resources/cloud-ops-output-logs.png)\n\n### Metrics\n\nViewing all metrics that Genkit exports can be done by selecting \"Logging\" from the side menu and clicking on \"Metrics management\".\n\n![Metrics Management menu item in Cloud Logging](../resources/cloud-ops-metrics-mgmt.png)\n\nThe metrics management console contains a tabular view of all collected metrics, including those that pertain to Cloud Run and its surrounding environment. Clicking on the 'Workload' option will reveal a list that includes Genkit-collected metrics. Any metric with the `genkit` prefix constitutes an internal Genkit metric.\n\n![Metrics table showing Genkit metrics](../resources/cloud-ops-metrics-table.png)\n\nGenkit collects several categories of metrics, including flow-level, action-level, and generate-level metrics. Each metric has several useful dimensions facilitating robust filtering and grouping.\n\nCommon dimensions include:\n\n- `flow_name` - the top-level name of the flow.\n- `flow_path` - the span and its parent span chain up to the root span.\n- `error_code` - in case of an error, the corresponding error code.\n- `error_message` - in case of an error, the corresponding error message.\n- `model` - the name of the model.\n- `temperature` - the inference temperature [value](https://ai.google.dev/docs/concepts#model-parameters).\n- `topK` - the inference topK [value](https://ai.google.dev/docs/concepts#model-parameters).\n- `topP` - the inference topP [value](https://ai.google.dev/docs/concepts#model-parameters).\n\n#### Flow-level metrics\n\n| Name | Dimensions |\n| -------------------- | ------------------------------------ |\n| genkit/flow/requests | flow_name, error_code, error_message |\n| genkit/flow/latency | flow_name |\n\n#### Action-level metrics\n\n| Name | Dimensions |\n| ---------------------- | ------------------------------------ |\n| genkit/action/requests | flow_name, error_code, error_message |\n| genkit/action/latency | flow_name |\n\n#### Generate-level metrics\n\n| Name | Dimensions |\n| ------------------------------------ | -------------------------------------------------------------------- |\n| genkit/ai/generate | flow_path, model, temperature, topK, topP, error_code, error_message |\n| genkit/ai/generate/input_tokens | flow_path, model, temperature, topK, topP |\n| genkit/ai/generate/output_tokens | flow_path, model, temperature, topK, topP |\n| genkit/ai/generate/input_characters | flow_path, model, temperature, topK, topP |\n| genkit/ai/generate/output_characters | flow_path, model, temperature, topK, topP |\n| genkit/ai/generate/input_images | flow_path, model, temperature, topK, topP |\n| genkit/ai/generate/output_images | flow_path, model, temperature, topK, topP |\n| genkit/ai/generate/latency | flow_path, model, temperature, topK, topP, error_code, error_message |\n\nVisualizing metrics can be done through the Metrics Explorer. Using the side menu, select 'Logging' and click 'Metrics explorer'\n\n![Metrics Explorer menu item in Cloud Logging](../resources/cloud-ops-metrics-explorer.png)\n\nSelect a metrics by clicking on the \"Select a metric\" dropdown, selecting 'Generic Node', 'Genkit', and a metric.\n\n![Selecting a Genkit metric in Metrics Explorer](../resources/cloud-ops-metrics-generic-node.png)\n\nThe visualization of the metric will depend on its type (counter, histogram, etc). The Metrics Explorer provides robust aggregation and querying facilities to help graph metrics by their various dimensions.\n\n![Metrics Explorer showing a Genkit metric graph](../resources/cloud-ops-metrics-metric.png)\n\n## Telemetry Delay\n\nThere may be a slight delay before telemetry for a particular execution of a flow is displayed in Cloud's operations suite. In most cases, this delay is under 1 minute.\n\n## Quotas and limits\n\nThere are several quotas that are important to keep in mind:\n\n- [Cloud Trace Quotas](http://cloud.google.com/trace/docs/quotas)\n - 128 bytes per attribute key\n - 256 bytes per attribute value\n- [Cloud Logging Quotas](http://cloud.google.com/logging/quotas)\n - 256 KB per log entry\n- [Cloud Monitoring Quotas](http://cloud.google.com/monitoring/quotas)\n\n## Cost\n\nCloud Logging, Cloud Trace, and Cloud Monitoring have generous free tiers. Specific pricing can be found at the following links:\n\n- [Cloud Logging Pricing](http://cloud.google.com/stackdriver/pricing#google-cloud-observability-pricing)\n- [Cloud Trace Pricing](https://cloud.google.com/trace#pricing)\n- [Cloud Monitoring Pricing](https://cloud.google.com/stackdriver/pricing#monitoring-pricing-summary)\n", "title": "Google Cloud telemetry and logging plugin", - "lang": "go" + "description": "Learn how to configure the Genkit Google Cloud plugin to export telemetry and logs to Cloud's operations suite for Go applications.", + "lang": "go", + "headers": "## Prerequisites\n## Set up a Google Cloud account\n## Configuration\n## Production monitoring via Google Cloud's operations suite\n### Logs and traces\n### Metrics\n#### Flow-level metrics\n#### Action-level metrics\n#### Generate-level metrics\n## Telemetry Delay\n## Quotas and limits\n## Cost\n" }, "go/plugins/google-genai.md": { - "text": "The Google Generative AI plugin provides interfaces to Google's Gemini models through either the Gemini API or the Vertex AI Gemini API.\n\n## Configuration\n\nThe configuration depends on which provider you choose:\n\n### Google AI\n\nTo use this plugin, import the `googlegenai` package and pass \n`googlegenai.GoogleAI` to `WithPlugins()` in the Genkit initializer:\n\n```go\nimport \"github.com/firebase/genkit/go/plugins/googlegenai\"\n```\n\n```go\ng, err := genkit.Init(context.Background(), ai.WithPlugins(&googlegenai.GoogleAI{}))\n```\n\nThe plugin requires an API key for the Gemini API, which you can get from\n[Google AI Studio](https://aistudio.google.com/app/apikey).\n\nConfigure the plugin to use your API key by doing one of the following:\n\n- Set the `GEMINI_API_KEY` environment variable to your API key.\n\n- Specify the API key when you initialize the plugin:\n\n ```go\n ai.WithPlugins(&googlegenai.GoogleAI{APIKey: \"YOUR_API_KEY\"})\n ```\n\n However, don't embed your API key directly in code! Use this feature only\n in conjunction with a service like Cloud Secret Manager or similar.\n\n### Vertex AI\n\nTo use this plugin, import the `googlegenai` package and pass\n`googlegenai.VertexAI` to `WithPlugins()` in the Genkit initializer:\n\n```go\nimport \"github.com/firebase/genkit/go/plugins/googlegenai\"\n```\n\n```go\ng, err := genkit.Init(context.Background(), genkit.WithPlugins(&googlegenai.VertexAI{}))\n```\n\nThe plugin requires you to specify your Google Cloud project ID, the\n[region](https://cloud.google.com/vertex-ai/generative-ai/docs/learn/locations)\nto which you want to make Vertex API requests, and your Google Cloud project\ncredentials.\n\n- By default, `googlegenai.VertexAI` gets your Google Cloud project ID from the\n `GOOGLE_CLOUD_PROJECT` environment variable.\n\n You can also pass this value directly:\n\n ```go\n genkit.WithPlugins(&googlegenai.VertexAI{ProjectID: \"my-project-id\"})\n ```\n\n- By default, `googlegenai.VertexAI` gets the Vertex AI API location from the\n `GOOGLE_CLOUD_LOCATION` environment variable.\n\n You can also pass this value directly:\n\n ```go\n genkit.WithPlugins(&googlegenai.VertexAI{Location: \"us-central1\"})\n ```\n\n- To provide API credentials, you need to set up Google Cloud Application\n Default Credentials.\n\n 1. To specify your credentials:\n\n - If you're running your flow from a Google Cloud environment (Cloud\n Functions, Cloud Run, and so on), this is set automatically.\n\n - On your local dev environment, do this by running:\n\n ```shell\n gcloud auth application-default login\n ```\n\n - For other environments, see the [Application Default Credentials](https://cloud.google.com/docs/authentication/provide-credentials-adc)\n docs.\n\n 2. In addition, make sure the account is granted the Vertex AI User IAM role\n (`roles/aiplatform.user`). See the Vertex AI [access control](https://cloud.google.com/vertex-ai/generative-ai/docs/access-control)\n docs.\n\n## Usage\n\n### Generative models\n\nTo get a reference to a supported model, specify its identifier to\neither `googlegenai.GoogleAIModel` or `googlgenai.VertexAIModel`:\n\n```go\nmodel := googlegenai.GoogleAIModel(g, \"gemini-2.5-flash\")\n```\n\nAlternatively, you may create a `ModelRef` which pairs the model name with its\nconfig:\n\n```go\nmodelRef := googlegenai.GoogleAIModelRef(\"gemini-2.5-flash\", &googlegenai.GeminiConfig{\n Temperature: 0.5,\n MaxOutputTokens: 500,\n // Other configuration...\n})\n```\n\nThe following models are supported: `gemini-1.5-pro`, `gemini-1.5-flash`,\n`gemini-2.0-pro`, `gemini-2.5-flash`, and other experimental models.\n\nModel references have a `Generate()` method that calls the Google API:\n\n```go\nresp, err := genkit.Generate(ctx, g, ai.WithModel(modelRef), ai.WithPrompt(\"Tell me a joke.\"))\nif err != nil {\n return err\n}\n\nlog.Println(resp.Text())\n```\n\nSee [Generating content with AI models](/go/docs/models) for more information.\n\n### Embedding models\n\nTo get a reference to a supported embedding model, specify its identifier to\neither `googlegenai.GoogleAIEmbedder` or `googlgenai.VertexAIEmbedder`:\n\n```go\nembeddingModel := googlegenai.GoogleAIEmbedder(g, \"text-embedding-004\")\n```\n\nThe following models are supported:\n\n- **Google AI**\n\n `text-embedding-004` and `embedding-001`\n\n- **Vertex AI**\n\n `textembedding-gecko@003`, `textembedding-gecko@002`, \n `textembedding-gecko@001`, `text-embedding-004`,\n `textembedding-gecko-multilingual@001`, `text-multilingual-embedding-002`, \n and `multimodalembedding`\n\nEmbedder references have an `Embed()` method that calls the Google AI API:\n\n```go\nresp, err := ai.Embed(ctx, embeddingModel, ai.WithDocs(userInput))\nif err != nil {\n return err\n}\n```\n\nYou can also pass an Embedder to an indexer's `Index()` method and a retriever's\n`Retrieve()` method:\n\n```go\nif err := ai.Index(ctx, myIndexer, ai.WithDocs(docsToIndex...)); err != nil {\n return err\n}\n```\n\n```go\nresp, err := ai.Retrieve(ctx, myRetriever, ai.WithDocs(userInput))\nif err != nil {\n return err\n}\n```\n\nSee [Retrieval-augmented generation (RAG)](/go/docs/rag) for more information.", + "text": "# Google Generative AI plugin\n\nThe Google Generative AI plugin provides interfaces to Google's Gemini models through either the Gemini API or the Vertex AI Gemini API.\n\n## Configuration\n\nThe configuration depends on which provider you choose:\n\n### Google AI\n\nTo use this plugin, import the `googlegenai` package and pass \n`googlegenai.GoogleAI` to `WithPlugins()` in the Genkit initializer:\n\n```go\nimport \"github.com/firebase/genkit/go/plugins/googlegenai\"\n```\n\n```go\ng, err := genkit.Init(context.Background(), ai.WithPlugins(&googlegenai.GoogleAI{}))\n```\n\nThe plugin requires an API key for the Gemini API, which you can get from\n[Google AI Studio](https://aistudio.google.com/app/apikey).\n\nConfigure the plugin to use your API key by doing one of the following:\n\n- Set the `GEMINI_API_KEY` environment variable to your API key.\n\n- Specify the API key when you initialize the plugin:\n\n ```go\n ai.WithPlugins(&googlegenai.GoogleAI{APIKey: \"YOUR_API_KEY\"})\n ```\n\n However, don't embed your API key directly in code! Use this feature only\n in conjunction with a service like Cloud Secret Manager or similar.\n\n### Vertex AI\n\nTo use this plugin, import the `googlegenai` package and pass\n`googlegenai.VertexAI` to `WithPlugins()` in the Genkit initializer:\n\n```go\nimport \"github.com/firebase/genkit/go/plugins/googlegenai\"\n```\n\n```go\ng, err := genkit.Init(context.Background(), genkit.WithPlugins(&googlegenai.VertexAI{}))\n```\n\nThe plugin requires you to specify your Google Cloud project ID, the\n[region](https://cloud.google.com/vertex-ai/generative-ai/docs/learn/locations)\nto which you want to make Vertex API requests, and your Google Cloud project\ncredentials.\n\n- By default, `googlegenai.VertexAI` gets your Google Cloud project ID from the\n `GOOGLE_CLOUD_PROJECT` environment variable.\n\n You can also pass this value directly:\n\n ```go\n genkit.WithPlugins(&googlegenai.VertexAI{ProjectID: \"my-project-id\"})\n ```\n\n- By default, `googlegenai.VertexAI` gets the Vertex AI API location from the\n `GOOGLE_CLOUD_LOCATION` environment variable.\n\n You can also pass this value directly:\n\n ```go\n genkit.WithPlugins(&googlegenai.VertexAI{Location: \"us-central1\"})\n ```\n\n- To provide API credentials, you need to set up Google Cloud Application\n Default Credentials.\n\n 1. To specify your credentials:\n\n - If you're running your flow from a Google Cloud environment (Cloud\n Functions, Cloud Run, and so on), this is set automatically.\n\n - On your local dev environment, do this by running:\n\n ```shell\n gcloud auth application-default login\n ```\n\n - For other environments, see the [Application Default Credentials](https://cloud.google.com/docs/authentication/provide-credentials-adc)\n docs.\n\n 2. In addition, make sure the account is granted the Vertex AI User IAM role\n (`roles/aiplatform.user`). See the Vertex AI [access control](https://cloud.google.com/vertex-ai/generative-ai/docs/access-control)\n docs.\n\n## Usage\n\n### Generative models\n\nTo get a reference to a supported model, specify its identifier to\neither `googlegenai.GoogleAIModel` or `googlgenai.VertexAIModel`:\n\n```go\nmodel := googlegenai.GoogleAIModel(g, \"gemini-2.5-flash\")\n```\n\nAlternatively, you may create a `ModelRef` which pairs the model name with its\nconfig:\n\n```go\nmodelRef := googlegenai.GoogleAIModelRef(\"gemini-2.5-flash\", &googlegenai.GeminiConfig{\n Temperature: 0.5,\n MaxOutputTokens: 500,\n // Other configuration...\n})\n```\n\nThe following models are supported: `gemini-1.5-pro`, `gemini-1.5-flash`,\n`gemini-2.0-pro`, `gemini-2.5-flash`, and other experimental models.\n\nModel references have a `Generate()` method that calls the Google API:\n\n```go\nresp, err := genkit.Generate(ctx, g, ai.WithModel(modelRef), ai.WithPrompt(\"Tell me a joke.\"))\nif err != nil {\n return err\n}\n\nlog.Println(resp.Text())\n```\n\nSee [Generating content with AI models](/go/docs/models) for more information.\n\n### Embedding models\n\nTo get a reference to a supported embedding model, specify its identifier to\neither `googlegenai.GoogleAIEmbedder` or `googlgenai.VertexAIEmbedder`:\n\n```go\nembeddingModel := googlegenai.GoogleAIEmbedder(g, \"text-embedding-004\")\n```\n\nThe following models are supported:\n\n- **Google AI**\n\n `text-embedding-004` and `embedding-001`\n\n- **Vertex AI**\n\n `textembedding-gecko@003`, `textembedding-gecko@002`, \n `textembedding-gecko@001`, `text-embedding-004`,\n `textembedding-gecko-multilingual@001`, `text-multilingual-embedding-002`, \n and `multimodalembedding`\n\nEmbedder references have an `Embed()` method that calls the Google AI API:\n\n```go\nresp, err := ai.Embed(ctx, embeddingModel, ai.WithDocs(userInput))\nif err != nil {\n return err\n}\n```\n\nYou can retrieve docs by passing in an input to a Retriever's `Retrieve()` method:\n\n```go\nresp, err := ai.Retrieve(ctx, myRetriever, ai.WithDocs(userInput))\nif err != nil {\n return err\n}\n```\n\nSee [Retrieval-augmented generation (RAG)](/go/docs/rag) for more information.", "title": "Google Generative AI plugin", - "lang": "go" + "description": "Learn how to configure and use the Genkit Google Generative AI plugin for Go to access Gemini models via the Gemini API or Vertex AI API.", + "lang": "go", + "headers": "## Configuration\n### Google AI\n### Vertex AI\n## Usage\n### Generative models\n### Embedding models\n" }, "go/plugins/mcp.md": { - "text": "The MCP (Model Context Protocol) plugin enables integration with MCP servers and allows you to expose Genkit tools as MCP servers. You can connect to external MCP servers to use their tools and prompts, manage multiple server connections, or turn your Genkit application into an MCP server.\n\n## Prerequisites\n\nThis plugin requires MCP servers to be available. For testing and development, you can use:\n\n- `mcp-server-time` - Spmple server Exposing time operations\n- `@modelcontextprotocol/server-everything` - A comprehensive MCP server for testing\n- Custom MCP servers written in Python, TypeScript, or other languages\n\n## Configuration\n\n### Single Server Connection\n\nTo connect to a single MCP server, import the `mcp` package and create a `GenkitMCPClient`:\n\n```go\nimport \"github.com/firebase/genkit/go/plugins/mcp\"\n```\n\n```go\nctx := context.Background()\ng, err := genkit.Init(ctx)\nif err != nil {\n log.Fatal(err)\n}\n\nclient, err := mcp.NewGenkitMCPClient(mcp.MCPClientOptions{\n Name: \"mcp-server-time\",\n Stdio: &mcp.StdioConfig{\n Command: \"uvx\",\n Args: []string{\"mcp-server-time\"},\n },\n})\nif err != nil {\n log.Fatal(err)\n}\n```\n\n### Multiple Server Management\n\nTo manage connections to multiple MCP servers, use `GenkitMCPManager`:\n\n```go\nimport \"github.com/firebase/genkit/go/plugins/mcp\"\n```\n\n```go\nmanager, err := mcp.NewMCPManager(mcp.MCPManagerOptions{\n Name: \"my-app\",\n MCPServers: []mcp.MCPServerConfig{\n {\n Name: \"everything-server\",\n Config: mcp.MCPClientOptions{\n Name: \"everything-server\",\n Stdio: &mcp.StdioConfig{\n Command: \"npx\",\n Args: []string{\"-y\", \"@modelcontextprotocol/server-everything\"},\n },\n },\n },\n {\n Name: \"mcp-server-time\",\n Config: mcp.MCPClientOptions{\n Name: \"mcp-server-time\",\n Stdio: &mcp.StdioConfig{\n Command: \"uvx\",\n Args: []string{\"mcp-server-time\"},\n },\n },\n },\n },\n})\nif err != nil {\n log.Fatal(err)\n}\n```\n\n### Exposing as MCP Server\n\nTo expose your Genkit tools as an MCP server, create an `MCPServer`:\n\n```go\nimport \"github.com/firebase/genkit/go/plugins/mcp\"\n```\n\n```go\n// Define your tools first\naddTool := genkit.DefineTool(g, \"add\", \"Add two numbers\",\n func(ctx *ai.ToolContext, input struct{A, B int}) (int, error) {\n return input.A + input.B, nil\n })\n\n// Create MCP server\nserver := mcp.NewMCPServer(g, mcp.MCPServerOptions{\n Name: \"genkit-calculator\",\n Version: \"1.0.0\",\n})\n```\n\n## Usage\n\n### Using Tools from MCP Servers\n\nOnce connected to an MCP server, you can retrieve and use its tools:\n\n```go\n// Get a specific tool\nechoTool, err := client.GetTool(ctx, g, \"echo\")\nif err != nil {\n log.Fatal(err)\n}\n\n// Use the tool in your workflow\nresp, err := genkit.Generate(ctx, g, \n ai.WithModel(myModel),\n ai.WithPrompt(\"Use the echo tool to repeat this message\"),\n ai.WithTools(echoTool),\n)\nif err != nil {\n log.Fatal(err)\n}\n```\n\n### Using Prompts from MCP Servers\n\nRetrieve and use prompts from connected MCP servers:\n\n```go\n// Get a specific prompt\nsimplePrompt, err := client.GetPrompt(ctx, g, \"simple_prompt\")\nif err != nil {\n log.Fatal(err)\n}\n\n// Use the prompt\nresp, err := genkit.Generate(ctx, g,\n ai.WithModel(myModel),\n ai.WithPrompt(simplePrompt),\n)\n```\n\n### Managing Multiple Servers\n\nWith `GenkitMCPManager`, you can dynamically manage server connections:\n\n```go\n// Connect to a new server at runtime\nerr = manager.Connect(\"weather\", mcp.MCPClientOptions{\n Name: \"weather-server\",\n Stdio: &mcp.StdioConfig{\n Command: \"python\",\n Args: []string{\"weather_server.py\"},\n },\n})\nif err != nil {\n log.Fatal(err)\n}\n\n// Disconnect a server completely\nerr = manager.Disconnect(\"weather\")\nif err != nil {\n log.Fatal(err)\n}\n\n// Get all tools from all active servers\ntools, err := manager.GetActiveTools(ctx, g)\nif err != nil {\n log.Fatal(err)\n}\n\n// Get a specific prompt from a specific server\nprompt, err := manager.GetPrompt(ctx, g, \"mcp-server-time\", \"current_time\", nil)\nif err != nil {\n log.Fatal(err)\n}\n```\n\nFor individual client management (disable/enable without disconnecting), you would access the clients directly. The manager focuses on connection lifecycle management.\n\n### Running as MCP Server\n\nTo run your Genkit application as an MCP server:\n\n```go\n// Option 1: Auto-expose all defined tools\nserver := mcp.NewMCPServer(g, mcp.MCPServerOptions{\n Name: \"genkit-calculator\",\n Version: \"1.0.0\",\n})\n\n// Option 2: Expose only specific tools\nserver = mcp.NewMCPServer(g, mcp.MCPServerOptions{\n Name: \"genkit-calculator\", \n Version: \"1.0.0\",\n Tools: []ai.Tool{addTool, multiplyTool},\n})\n\n// Start the MCP server\nlog.Println(\"Starting MCP server...\")\nif err := server.ServeStdio(ctx); err != nil {\n log.Fatal(err)\n}\n```\n\n## Transport Options\n\n### Stdio Transport\n\nYou can use either Stdio or SSE \n\n```go\nStdio: &mcp.StdioConfig{\n Command: \"uvx\",\n Args: []string{\"mcp-server-time\"},\n Env: []string{\"DEBUG=1\"},\n}\n```\n\n```go\nSSE: &mcp.SSEConfig{\n BaseURL: \"http://localhost:3000/sse\",\n}\n```\n\n## Testing\n\n### Testing Your MCP Server\n\nTo test your Genkit application as an MCP server:\n\n```bash\n# Run your server\ngo run main.go\n\n# Test with MCP Inspector in another terminal\nnpx @modelcontextprotocol/inspector go run main.go\n```\n\n## Configuration Options\n\n### MCPClientOptions\n\n```go\ntype MCPClientOptions struct {\n Name string // Server identifier\n Version string // Version number (defaults to \"1.0.0\")\n Disabled bool // Disabled flag to temporarily disable this client\n Stdio *StdioConfig // Stdio transport config\n SSE *SSEConfig // SSE transport config\n}\n```\n\n### StdioConfig\n\n```go\ntype StdioConfig struct {\n Command string // Command to run\n Args []string // Command arguments\n Env []string // Environment variables\n}\n```\n\n### MCPServerConfig\n\n```go\ntype MCPServerConfig struct {\n Name string // Name for this server\n Config MCPClientOptions // Client configuration options\n}\n```\n\n### MCPManagerOptions\n\n```go\ntype MCPManagerOptions struct {\n Name string // Manager instance name\n Version string // Manager version (defaults to \"1.0.0\")\n MCPServers []MCPServerConfig // Array of server configurations\n}\n```\n\n### MCPServerOptions\n\n```go\ntype MCPServerOptions struct {\n Name string // Server name\n Version string // Server version\n Tools []ai.Tool // Specific tools to expose (optional)\n}\n```", + "text": "# MCP (Model Context Protocol) plugin\n\nThe MCP (Model Context Protocol) plugin enables integration with MCP servers and allows you to expose Genkit tools as MCP servers. You can connect to external MCP servers to use their tools and prompts, manage multiple server connections, or turn your Genkit application into an MCP server.\n\n## Prerequisites\n\nThis plugin requires MCP servers to be available. For testing and development, you can use:\n\n- `mcp-server-time` - Spmple server Exposing time operations\n- `@modelcontextprotocol/server-everything` - A comprehensive MCP server for testing\n- Custom MCP servers written in Python, TypeScript, or other languages\n\n## Configuration\n\n### Single Server Connection\n\nTo connect to a single MCP server, import the `mcp` package and create a `GenkitMCPClient`:\n\n```go\nimport \"github.com/firebase/genkit/go/plugins/mcp\"\n```\n\n```go\nctx := context.Background()\ng, err := genkit.Init(ctx)\nif err != nil {\n log.Fatal(err)\n}\n\nclient, err := mcp.NewGenkitMCPClient(mcp.MCPClientOptions{\n Name: \"mcp-server-time\",\n Stdio: &mcp.StdioConfig{\n Command: \"uvx\",\n Args: []string{\"mcp-server-time\"},\n },\n})\nif err != nil {\n log.Fatal(err)\n}\n```\n\n### Multiple Server Management\n\nTo manage connections to multiple MCP servers, use `GenkitMCPManager`:\n\n```go\nimport \"github.com/firebase/genkit/go/plugins/mcp\"\n```\n\n```go\nmanager, err := mcp.NewMCPManager(mcp.MCPManagerOptions{\n Name: \"my-app\",\n MCPServers: []mcp.MCPServerConfig{\n {\n Name: \"everything-server\",\n Config: mcp.MCPClientOptions{\n Name: \"everything-server\",\n Stdio: &mcp.StdioConfig{\n Command: \"npx\",\n Args: []string{\"-y\", \"@modelcontextprotocol/server-everything\"},\n },\n },\n },\n {\n Name: \"mcp-server-time\",\n Config: mcp.MCPClientOptions{\n Name: \"mcp-server-time\",\n Stdio: &mcp.StdioConfig{\n Command: \"uvx\",\n Args: []string{\"mcp-server-time\"},\n },\n },\n },\n },\n})\nif err != nil {\n log.Fatal(err)\n}\n```\n\n### Exposing as MCP Server\n\nTo expose your Genkit tools as an MCP server, create an `MCPServer`:\n\n```go\nimport \"github.com/firebase/genkit/go/plugins/mcp\"\n```\n\n```go\n// Define your tools first\naddTool := genkit.DefineTool(g, \"add\", \"Add two numbers\",\n func(ctx *ai.ToolContext, input struct{A, B int}) (int, error) {\n return input.A + input.B, nil\n })\n\n// Create MCP server\nserver := mcp.NewMCPServer(g, mcp.MCPServerOptions{\n Name: \"genkit-calculator\",\n Version: \"1.0.0\",\n})\n```\n\n## Usage\n\n### Using Tools from MCP Servers\n\nOnce connected to an MCP server, you can retrieve and use its tools:\n\n```go\n// Get a specific tool\nechoTool, err := client.GetTool(ctx, g, \"echo\")\nif err != nil {\n log.Fatal(err)\n}\n\n// Use the tool in your workflow\nresp, err := genkit.Generate(ctx, g, \n ai.WithModel(myModel),\n ai.WithPrompt(\"Use the echo tool to repeat this message\"),\n ai.WithTools(echoTool),\n)\nif err != nil {\n log.Fatal(err)\n}\n```\n\n### Using Prompts from MCP Servers\n\nRetrieve and use prompts from connected MCP servers:\n\n```go\n// Get a specific prompt\nsimplePrompt, err := client.GetPrompt(ctx, g, \"simple_prompt\")\nif err != nil {\n log.Fatal(err)\n}\n\n// Use the prompt\nresp, err := genkit.Generate(ctx, g,\n ai.WithModel(myModel),\n ai.WithPrompt(simplePrompt),\n)\n```\n\n### Managing Multiple Servers\n\nWith `GenkitMCPManager`, you can dynamically manage server connections:\n\n```go\n// Connect to a new server at runtime\nerr = manager.Connect(\"weather\", mcp.MCPClientOptions{\n Name: \"weather-server\",\n Stdio: &mcp.StdioConfig{\n Command: \"python\",\n Args: []string{\"weather_server.py\"},\n },\n})\nif err != nil {\n log.Fatal(err)\n}\n\n// Disconnect a server completely\nerr = manager.Disconnect(\"weather\")\nif err != nil {\n log.Fatal(err)\n}\n\n// Get all tools from all active servers\ntools, err := manager.GetActiveTools(ctx, g)\nif err != nil {\n log.Fatal(err)\n}\n\n// Get a specific prompt from a specific server\nprompt, err := manager.GetPrompt(ctx, g, \"mcp-server-time\", \"current_time\", nil)\nif err != nil {\n log.Fatal(err)\n}\n```\n\nFor individual client management (disable/enable without disconnecting), you would access the clients directly. The manager focuses on connection lifecycle management.\n\n### Running as MCP Server\n\nTo run your Genkit application as an MCP server:\n\n```go\n// Option 1: Auto-expose all defined tools\nserver := mcp.NewMCPServer(g, mcp.MCPServerOptions{\n Name: \"genkit-calculator\",\n Version: \"1.0.0\",\n})\n\n// Option 2: Expose only specific tools\nserver = mcp.NewMCPServer(g, mcp.MCPServerOptions{\n Name: \"genkit-calculator\", \n Version: \"1.0.0\",\n Tools: []ai.Tool{addTool, multiplyTool},\n})\n\n// Start the MCP server\nlog.Println(\"Starting MCP server...\")\nif err := server.ServeStdio(ctx); err != nil {\n log.Fatal(err)\n}\n```\n\n## Transport Options\n\n### Stdio Transport\n\nYou can use either Stdio or SSE \n\n```go\nStdio: &mcp.StdioConfig{\n Command: \"uvx\",\n Args: []string{\"mcp-server-time\"},\n Env: []string{\"DEBUG=1\"},\n}\n```\n\n```go\nSSE: &mcp.SSEConfig{\n BaseURL: \"http://localhost:3000/sse\",\n}\n```\n\n## Testing\n\n### Testing Your MCP Server\n\nTo test your Genkit application as an MCP server:\n\n```bash\n# Run your server\ngo run main.go\n\n# Test with MCP Inspector in another terminal\nnpx @modelcontextprotocol/inspector go run main.go\n```\n\n## Configuration Options\n\n### MCPClientOptions\n\n```go\ntype MCPClientOptions struct {\n Name string // Server identifier\n Version string // Version number (defaults to \"1.0.0\")\n Disabled bool // Disabled flag to temporarily disable this client\n Stdio *StdioConfig // Stdio transport config\n SSE *SSEConfig // SSE transport config\n}\n```\n\n### StdioConfig\n\n```go\ntype StdioConfig struct {\n Command string // Command to run\n Args []string // Command arguments\n Env []string // Environment variables\n}\n```\n\n### MCPServerConfig\n\n```go\ntype MCPServerConfig struct {\n Name string // Name for this server\n Config MCPClientOptions // Client configuration options\n}\n```\n\n### MCPManagerOptions\n\n```go\ntype MCPManagerOptions struct {\n Name string // Manager instance name\n Version string // Manager version (defaults to \"1.0.0\")\n MCPServers []MCPServerConfig // Array of server configurations\n}\n```\n\n### MCPServerOptions\n\n```go\ntype MCPServerOptions struct {\n Name string // Server name\n Version string // Server version\n Tools []ai.Tool // Specific tools to expose (optional)\n}\n```", "title": "MCP (Model Context Protocol) plugin", - "lang": "go" + "description": "Learn how to integrate MCP servers with Genkit for Go and expose Genkit tools as MCP servers.", + "lang": "go", + "headers": "## Prerequisites\n## Configuration\n### Single Server Connection\n### Multiple Server Management\n### Exposing as MCP Server\n## Usage\n### Using Tools from MCP Servers\n### Using Prompts from MCP Servers\n### Managing Multiple Servers\n### Running as MCP Server\n## Transport Options\n### Stdio Transport\n## Testing\n### Testing Your MCP Server\n# Run your server\n# Test with MCP Inspector in another terminal\n## Configuration Options\n### MCPClientOptions\n### StdioConfig\n### MCPServerConfig\n### MCPManagerOptions\n### MCPServerOptions\n" }, "go/plugins/ollama.md": { - "text": "The Ollama plugin provides interfaces to any of the local LLMs supported by\n[Ollama](https://ollama.com/).\n\n## Prerequisites\n\nThis plugin requires that you first install and run the Ollama server. You can\nfollow the instructions on the [Download Ollama](https://ollama.com/download)\npage.\n\nUse the Ollama CLI to download the models you are interested in. For example:\n\n```bash\nollama pull gemma3\n```\n\nFor development, you can run Ollama on your development machine. Deployed apps\nusually run Ollama on a GPU-accelerated machine that is different from the one\nhosting the app backend running Genkit.\n\n## Configuration\n\nTo use this plugin, pass `ollama.Ollama` to `WithPlugins()` in the Genkit\ninitializer, specifying the address of your Ollama server:\n\n```go\nimport \"github.com/firebase/genkit/go/plugins/ollama\"\n```\n\n```go\ng, err := genkit.Init(context.Background(), genkit.WithPlugins(&ollama.Ollama{ServerAddress: \"http://127.0.0.1:11434\"}))\n```\n\n## Usage\n\nTo generate content, you first need to create a model definition based on the\nmodel you installed and want to use. For example, if you installed Gemma 2:\n\n```go\nmodel := ollama.DefineModel(\n ollama.ModelDefinition{\n Name: \"gemma3\",\n Type: \"chat\", // \"chat\" or \"generate\"\n },\n &ai.ModelInfo{\n Multiturn: true,\n SystemRole: true,\n Tools: false,\n Media: false,\n },\n)\n```\n\nThen, you can use the model reference to send requests to your Ollama server:\n\n```go\nresp, err := genkit.Generate(ctx, g, ai.WithModel(model), ai.WithPrompt(\"Tell me a joke.\"))\nif err != nil {\n return err\n}\n\nlog.Println(resp.Text())\n```\n\nSee [Generating content](/go/docs/models) for more information.\n", + "text": "# Ollama plugin\n\nThe Ollama plugin provides interfaces to any of the local LLMs supported by\n[Ollama](https://ollama.com/).\n\n## Prerequisites\n\nThis plugin requires that you first install and run the Ollama server. You can\nfollow the instructions on the [Download Ollama](https://ollama.com/download)\npage.\n\nUse the Ollama CLI to download the models you are interested in. For example:\n\n```bash\nollama pull gemma3\n```\n\nFor development, you can run Ollama on your development machine. Deployed apps\nusually run Ollama on a GPU-accelerated machine that is different from the one\nhosting the app backend running Genkit.\n\n## Configuration\n\nTo use this plugin, pass `ollama.Ollama` to `WithPlugins()` in the Genkit\ninitializer, specifying the address of your Ollama server:\n\n```go\nimport \"github.com/firebase/genkit/go/plugins/ollama\"\n```\n\n```go\ng, err := genkit.Init(context.Background(), genkit.WithPlugins(&ollama.Ollama{ServerAddress: \"http://127.0.0.1:11434\"}))\n```\n\n## Usage\n\nTo generate content, you first need to create a model definition based on the\nmodel you installed and want to use. For example, if you installed Gemma 2:\n\n```go\nmodel := ollama.DefineModel(\n ollama.ModelDefinition{\n Name: \"gemma3\",\n Type: \"chat\", // \"chat\" or \"generate\"\n },\n &ai.ModelInfo{\n Multiturn: true,\n SystemRole: true,\n Tools: false,\n Media: false,\n },\n)\n```\n\nThen, you can use the model reference to send requests to your Ollama server:\n\n```go\nresp, err := genkit.Generate(ctx, g, ai.WithModel(model), ai.WithPrompt(\"Tell me a joke.\"))\nif err != nil {\n return err\n}\n\nlog.Println(resp.Text())\n```\n\nSee [Generating content](/go/docs/models) for more information.\n", "title": "Ollama plugin", - "lang": "go" + "description": "Learn how to configure and use the Genkit Ollama plugin for Go to interact with local LLMs like Gemma and Llama.", + "lang": "go", + "headers": "## Prerequisites\n## Configuration\n## Usage\n" }, "go/plugins/openai.md": { - "text": "The OpenAI-Compatible API plugin (`compat_oai`) provides a unified interface for accessing multiple AI providers that implement OpenAI's API specification. This includes OpenAI, Anthropic, and other compatible services.\n\n## Overview\n\nThe `compat_oai` package serves as a foundation for building plugins that work with OpenAI-compatible APIs. It includes:\n\n- **Base Implementation**: Common functionality for OpenAI-compatible APIs\n- **OpenAI Plugin**: Direct access to OpenAI's models and embeddings\n- **Anthropic Plugin**: Access to Claude models through OpenAI-compatible endpoints\n- **Extensible Framework**: Build custom plugins for other compatible providers\n\n## Prerequisites\n\nDepending on which provider you use, you'll need:\n\n- **OpenAI**: API key from [OpenAI API Keys page](https://platform.openai.com/api-keys)\n- **Anthropic**: API key from [Anthropic Console](https://console.anthropic.com/)\n- **Other providers**: API keys from the respective services\n\n## OpenAI Provider\n\n### Configuration\n\n```go\nimport \"github.com/firebase/genkit/go/plugins/compat_oai/openai\"\n```\n\n```go\ng, err := genkit.Init(context.Background(), genkit.WithPlugins(&openai.OpenAI{\n APIKey: \"YOUR_OPENAI_API_KEY\", // or set OPENAI_API_KEY env var\n}))\n```\n\n### Supported Models\n\n#### Latest Models\n- **gpt-4.1** - Latest GPT-4.1 with multimodal support\n- **gpt-4.1-mini** - Faster, cost-effective GPT-4.1 variant\n- **gpt-4.1-nano** - Ultra-efficient GPT-4.1 variant\n- **gpt-4.5-preview** - Preview of GPT-4.5 with advanced capabilities\n\n#### Production Models\n- **gpt-4o** - Advanced GPT-4 with vision and tool support\n- **gpt-4o-mini** - Fast and cost-effective GPT-4o variant\n- **gpt-4-turbo** - High-performance GPT-4 with large context window\n\n#### Reasoning Models\n- **o3-mini** - Latest compact reasoning model\n- **o1** - Advanced reasoning model for complex problems\n- **o1-mini** - Compact reasoning model\n- **o1-preview** - Preview reasoning model\n\n#### Legacy Models\n- **gpt-4** - Original GPT-4 model\n- **gpt-3.5-turbo** - Fast and efficient language model\n\n### Embedding Models\n- **text-embedding-3-large** - Most capable embedding model\n- **text-embedding-3-small** - Fast and efficient embedding model \n- **text-embedding-ada-002** - Legacy embedding model\n\n### OpenAI Usage Example\n\n```go\nimport (\n \"github.com/firebase/genkit/go/plugins/compat_oai/openai\"\n \"github.com/firebase/genkit/go/plugins/compat_oai\"\n)\n\n// Initialize OpenAI plugin\noai := &openai.OpenAI{APIKey: \"YOUR_API_KEY\"}\ng, err := genkit.Init(ctx, genkit.WithPlugins(oai))\n\n// Use GPT-4o for general tasks\nmodel := oai.Model(g, \"gpt-4o\")\nresp, err := genkit.Generate(ctx, g, \n ai.WithModel(model), \n ai.WithPrompt(\"Explain quantum computing.\"),\n)\n\n// Use embeddings\nembedder := oai.Embedder(g, \"text-embedding-3-large\")\nembeds, err := ai.Embed(ctx, embedder, ai.WithDocs(\"Hello, world!\"))\n```\n\n## Anthropic Provider\n\n### Configuration\n\n```go\nimport \"github.com/firebase/genkit/go/plugins/compat_oai/anthropic\"\n```\n\n```go\ng, err := genkit.Init(context.Background(), genkit.WithPlugins(&anthropic.Anthropic{\n Opts: []option.RequestOption{\n option.WithAPIKey(\"YOUR_ANTHROPIC_API_KEY\"),\n },\n}))\n```\n\n### Supported Models\n\n- **claude-3-7-sonnet-20250219** - Latest Claude 3.7 Sonnet with advanced capabilities\n- **claude-3-5-haiku-20241022** - Fast and efficient Claude 3.5 Haiku\n- **claude-3-5-sonnet-20240620** - Balanced Claude 3.5 Sonnet\n- **claude-3-opus-20240229** - Most capable Claude 3 model\n- **claude-3-haiku-20240307** - Fastest Claude 3 model\n\n### Anthropic Usage Example\n\n```go\nimport (\n \"github.com/firebase/genkit/go/plugins/compat_oai/anthropic\"\n \"github.com/openai/openai-go/option\"\n)\n\n// Initialize Anthropic plugin\nclaude := &anthropic.Anthropic{\n Opts: []option.RequestOption{\n option.WithAPIKey(\"YOUR_ANTHROPIC_API_KEY\"),\n },\n}\ng, err := genkit.Init(ctx, genkit.WithPlugins(claude))\n\n// Use Claude for tasks requiring reasoning\nmodel := claude.Model(g, \"claude-3-7-sonnet-20250219\")\nresp, err := genkit.Generate(ctx, g,\n ai.WithModel(model),\n ai.WithPrompt(\"Analyze this complex problem step by step.\"),\n)\n```\n\n## Using Multiple Providers\n\nYou can use both providers in the same application:\n\n```go\nimport (\n \"github.com/firebase/genkit/go/plugins/compat_oai/openai\"\n \"github.com/firebase/genkit/go/plugins/compat_oai/anthropic\"\n)\n\noai := &openai.OpenAI{APIKey: \"YOUR_OPENAI_KEY\"}\nclaude := &anthropic.Anthropic{\n Opts: []option.RequestOption{\n option.WithAPIKey(\"YOUR_ANTHROPIC_KEY\"),\n },\n}\n\ng, err := genkit.Init(ctx, genkit.WithPlugins(oai, claude))\n\n// Use OpenAI for embeddings and tool-heavy tasks\nopenaiModel := oai.Model(g, \"gpt-4o\")\nembedder := oai.Embedder(g, \"text-embedding-3-large\")\n\n// Use Anthropic for reasoning and analysis\nclaudeModel := claude.Model(g, \"claude-3-7-sonnet-20250219\")\n```\n\n## Advanced Features\n\n### Tool Calling\n\nOpenAI models support tool calling:\n\n```go\n// Define a tool\nweatherTool := genkit.DefineTool(g, \"get_weather\", \"Get current weather\",\n func(ctx *ai.ToolContext, input struct{City string}) (string, error) {\n return fmt.Sprintf(\"It's sunny in %s\", input.City), nil\n })\n\n// Use with GPT models (tools not supported on Claude via OpenAI API)\nmodel := oai.Model(g, \"gpt-4o\")\nresp, err := genkit.Generate(ctx, g,\n ai.WithModel(model),\n ai.WithPrompt(\"What's the weather like in San Francisco?\"),\n ai.WithTools(weatherTool),\n)\n```\n\n### Multimodal Support\n\nBoth providers support vision capabilities:\n\n```go\n// Works with GPT-4o and Claude models\nresp, err := genkit.Generate(ctx, g,\n ai.WithModel(model),\n ai.WithMessages([]*ai.Message{\n ai.NewUserMessage(\n ai.WithTextPart(\"What do you see in this image?\"),\n ai.WithMediaPart(\"image/jpeg\", imageData),\n ),\n }),\n)\n```\n\n### Streaming\n\nBoth providers support streaming responses:\n\n```go\nresp, err := genkit.Generate(ctx, g,\n ai.WithModel(model),\n ai.WithPrompt(\"Write a long explanation.\"),\n ai.WithStreaming(func(ctx context.Context, chunk *ai.ModelResponseChunk) error {\n for _, content := range chunk.Content {\n fmt.Print(content.Text)\n }\n return nil\n }),\n)\n```\n\n## Configuration\n\n### Common Configuration\n\nBoth providers support OpenAI-compatible configuration:\n\n```go\nimport \"github.com/firebase/genkit/go/plugins/compat_oai\"\n\nconfig := &compat_oai.OpenAIConfig{\n Temperature: 0.7,\n MaxOutputTokens: 1000,\n TopP: 0.9,\n StopSequences: []string{\"END\"},\n}\n\nresp, err := genkit.Generate(ctx, g,\n ai.WithModel(model),\n ai.WithPrompt(\"Your prompt here\"),\n ai.WithConfig(config),\n)\n```\n\n### Advanced Options\n\n```go\nimport \"github.com/openai/openai-go/option\"\n\n// Custom base URL for OpenAI-compatible services\nopts := []option.RequestOption{\n option.WithAPIKey(\"YOUR_API_KEY\"),\n option.WithBaseURL(\"https://your-custom-endpoint.com/v1\"),\n option.WithOrganization(\"your-org-id\"),\n option.WithHeader(\"Custom-Header\", \"value\"),\n}\n```\n\n", + "text": "# OpenAI-Compatible API plugin\n\nThe OpenAI-Compatible API plugin (`compat_oai`) provides a unified interface for accessing multiple AI providers that implement OpenAI's API specification. This includes OpenAI, Anthropic, and other compatible services.\n\n## Overview\n\nThe `compat_oai` package serves as a foundation for building plugins that work with OpenAI-compatible APIs. It includes:\n\n- **Base Implementation**: Common functionality for OpenAI-compatible APIs\n- **OpenAI Plugin**: Direct access to OpenAI's models and embeddings\n- **Anthropic Plugin**: Access to Claude models through OpenAI-compatible endpoints\n- **Extensible Framework**: Build custom plugins for other compatible providers\n\n## Prerequisites\n\nDepending on which provider you use, you'll need:\n\n- **OpenAI**: API key from [OpenAI API Keys page](https://platform.openai.com/api-keys)\n- **Anthropic**: API key from [Anthropic Console](https://console.anthropic.com/)\n- **Other providers**: API keys from the respective services\n\n## OpenAI Provider\n\n### Configuration\n\n```go\nimport \"github.com/firebase/genkit/go/plugins/compat_oai/openai\"\n```\n\n```go\ng, err := genkit.Init(context.Background(), genkit.WithPlugins(&openai.OpenAI{\n APIKey: \"YOUR_OPENAI_API_KEY\", // or set OPENAI_API_KEY env var\n}))\n```\n\n### Supported Models\n\n#### Latest Models\n- **gpt-4.1** - Latest GPT-4.1 with multimodal support\n- **gpt-4.1-mini** - Faster, cost-effective GPT-4.1 variant\n- **gpt-4.1-nano** - Ultra-efficient GPT-4.1 variant\n- **gpt-4.5-preview** - Preview of GPT-4.5 with advanced capabilities\n\n#### Production Models\n- **gpt-4o** - Advanced GPT-4 with vision and tool support\n- **gpt-4o-mini** - Fast and cost-effective GPT-4o variant\n- **gpt-4-turbo** - High-performance GPT-4 with large context window\n\n#### Reasoning Models\n- **o3-mini** - Latest compact reasoning model\n- **o1** - Advanced reasoning model for complex problems\n- **o1-mini** - Compact reasoning model\n- **o1-preview** - Preview reasoning model\n\n#### Legacy Models\n- **gpt-4** - Original GPT-4 model\n- **gpt-3.5-turbo** - Fast and efficient language model\n\n### Embedding Models\n- **text-embedding-3-large** - Most capable embedding model\n- **text-embedding-3-small** - Fast and efficient embedding model \n- **text-embedding-ada-002** - Legacy embedding model\n\n### OpenAI Usage Example\n\n```go\nimport (\n \"github.com/firebase/genkit/go/plugins/compat_oai/openai\"\n \"github.com/firebase/genkit/go/plugins/compat_oai\"\n)\n\n// Initialize OpenAI plugin\noai := &openai.OpenAI{APIKey: \"YOUR_API_KEY\"}\ng, err := genkit.Init(ctx, genkit.WithPlugins(oai))\n\n// Use GPT-4o for general tasks\nmodel := oai.Model(g, \"gpt-4o\")\nresp, err := genkit.Generate(ctx, g, \n ai.WithModel(model), \n ai.WithPrompt(\"Explain quantum computing.\"),\n)\n\n// Use embeddings\nembedder := oai.Embedder(g, \"text-embedding-3-large\")\nembeds, err := ai.Embed(ctx, embedder, ai.WithDocs(\"Hello, world!\"))\n```\n\n## Anthropic Provider\n\n### Configuration\n\n```go\nimport \"github.com/firebase/genkit/go/plugins/compat_oai/anthropic\"\n```\n\n```go\ng, err := genkit.Init(context.Background(), genkit.WithPlugins(&anthropic.Anthropic{\n Opts: []option.RequestOption{\n option.WithAPIKey(\"YOUR_ANTHROPIC_API_KEY\"),\n },\n}))\n```\n\n### Supported Models\n\n- **claude-3-7-sonnet-20250219** - Latest Claude 3.7 Sonnet with advanced capabilities\n- **claude-3-5-haiku-20241022** - Fast and efficient Claude 3.5 Haiku\n- **claude-3-5-sonnet-20240620** - Balanced Claude 3.5 Sonnet\n- **claude-3-opus-20240229** - Most capable Claude 3 model\n- **claude-3-haiku-20240307** - Fastest Claude 3 model\n\n### Anthropic Usage Example\n\n```go\nimport (\n \"github.com/firebase/genkit/go/plugins/compat_oai/anthropic\"\n \"github.com/openai/openai-go/option\"\n)\n\n// Initialize Anthropic plugin\nclaude := &anthropic.Anthropic{\n Opts: []option.RequestOption{\n option.WithAPIKey(\"YOUR_ANTHROPIC_API_KEY\"),\n },\n}\ng, err := genkit.Init(ctx, genkit.WithPlugins(claude))\n\n// Use Claude for tasks requiring reasoning\nmodel := claude.Model(g, \"claude-3-7-sonnet-20250219\")\nresp, err := genkit.Generate(ctx, g,\n ai.WithModel(model),\n ai.WithPrompt(\"Analyze this complex problem step by step.\"),\n)\n```\n\n## Using Multiple Providers\n\nYou can use both providers in the same application:\n\n```go\nimport (\n \"github.com/firebase/genkit/go/plugins/compat_oai/openai\"\n \"github.com/firebase/genkit/go/plugins/compat_oai/anthropic\"\n)\n\noai := &openai.OpenAI{APIKey: \"YOUR_OPENAI_KEY\"}\nclaude := &anthropic.Anthropic{\n Opts: []option.RequestOption{\n option.WithAPIKey(\"YOUR_ANTHROPIC_KEY\"),\n },\n}\n\ng, err := genkit.Init(ctx, genkit.WithPlugins(oai, claude))\n\n// Use OpenAI for embeddings and tool-heavy tasks\nopenaiModel := oai.Model(g, \"gpt-4o\")\nembedder := oai.Embedder(g, \"text-embedding-3-large\")\n\n// Use Anthropic for reasoning and analysis\nclaudeModel := claude.Model(g, \"claude-3-7-sonnet-20250219\")\n```\n\n## Advanced Features\n\n### Tool Calling\n\nOpenAI models support tool calling:\n\n```go\n// Define a tool\nweatherTool := genkit.DefineTool(g, \"get_weather\", \"Get current weather\",\n func(ctx *ai.ToolContext, input struct{City string}) (string, error) {\n return fmt.Sprintf(\"It's sunny in %s\", input.City), nil\n })\n\n// Use with GPT models (tools not supported on Claude via OpenAI API)\nmodel := oai.Model(g, \"gpt-4o\")\nresp, err := genkit.Generate(ctx, g,\n ai.WithModel(model),\n ai.WithPrompt(\"What's the weather like in San Francisco?\"),\n ai.WithTools(weatherTool),\n)\n```\n\n### Multimodal Support\n\nBoth providers support vision capabilities:\n\n```go\n// Works with GPT-4o and Claude models\nresp, err := genkit.Generate(ctx, g,\n ai.WithModel(model),\n ai.WithMessages([]*ai.Message{\n ai.NewUserMessage(\n ai.WithTextPart(\"What do you see in this image?\"),\n ai.WithMediaPart(\"image/jpeg\", imageData),\n ),\n }),\n)\n```\n\n### Streaming\n\nBoth providers support streaming responses:\n\n```go\nresp, err := genkit.Generate(ctx, g,\n ai.WithModel(model),\n ai.WithPrompt(\"Write a long explanation.\"),\n ai.WithStreaming(func(ctx context.Context, chunk *ai.ModelResponseChunk) error {\n for _, content := range chunk.Content {\n fmt.Print(content.Text)\n }\n return nil\n }),\n)\n```\n\n## Configuration\n\n### Common Configuration\n\nBoth providers support OpenAI-compatible configuration:\n\n```go\nimport \"github.com/firebase/genkit/go/plugins/compat_oai\"\n\nconfig := &compat_oai.OpenAIConfig{\n Temperature: 0.7,\n MaxOutputTokens: 1000,\n TopP: 0.9,\n StopSequences: []string{\"END\"},\n}\n\nresp, err := genkit.Generate(ctx, g,\n ai.WithModel(model),\n ai.WithPrompt(\"Your prompt here\"),\n ai.WithConfig(config),\n)\n```\n\n### Advanced Options\n\n```go\nimport \"github.com/openai/openai-go/option\"\n\n// Custom base URL for OpenAI-compatible services\nopts := []option.RequestOption{\n option.WithAPIKey(\"YOUR_API_KEY\"),\n option.WithBaseURL(\"https://your-custom-endpoint.com/v1\"),\n option.WithOrganization(\"your-org-id\"),\n option.WithHeader(\"Custom-Header\", \"value\"),\n}\n```\n\n", "title": "OpenAI-Compatible API plugin", - "lang": "go" + "description": "Learn how to use the Genkit OpenAI-Compatible plugin for Go to access OpenAI, Anthropic, and other OpenAI-compatible APIs.", + "lang": "go", + "headers": "## Overview\n## Prerequisites\n## OpenAI Provider\n### Configuration\n### Supported Models\n#### Latest Models\n#### Production Models\n#### Reasoning Models\n#### Legacy Models\n### Embedding Models\n### OpenAI Usage Example\n## Anthropic Provider\n### Configuration\n### Supported Models\n### Anthropic Usage Example\n## Using Multiple Providers\n## Advanced Features\n### Tool Calling\n### Multimodal Support\n### Streaming\n## Configuration\n### Common Configuration\n### Advanced Options\n" }, "go/plugins/pgvector.md": { - "text": "You can use PostgreSQL and `pgvector` as your retriever implementation. Use the\nfollowing examples as a starting point and modify it to work with your database\nschema.\n\nWe use [database/sql](https://pkg.go.dev/database/sql) to connect to the Postgres server, but you may use another client library of your choice.\n\n```go\nfunc defineRetriever(g *genkit.Genkit, db *sql.DB, embedder ai.Embedder) ai.Retriever {\n\tf := func(ctx context.Context, req *ai.RetrieverRequest) (*ai.RetrieverResponse, error) {\n\t\teres, err := ai.Embed(ctx, embedder, ai.WithDocs(req.Query))\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\trows, err := db.QueryContext(ctx, `\n\t\t\tSELECT episode_id, season_number, chunk as content\n\t\t\tFROM embeddings\n\t\t\tWHERE show_id = $1\n\t\t \tORDER BY embedding <#> $2\n\t\t \tLIMIT 2`,\n\t\t\treq.Options, pgv.NewVector(eres.Embeddings[0].Embedding))\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tdefer rows.Close()\n\n\t\tres := &ai.RetrieverResponse{}\n\t\tfor rows.Next() {\n\t\t\tvar eid, sn int\n\t\t\tvar content string\n\t\t\tif err := rows.Scan(&eid, &sn, &content); err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tmeta := map[string]any{\n\t\t\t\t\"episode_id\": eid,\n\t\t\t\t\"season_number\": sn,\n\t\t\t}\n\t\t\tdoc := &ai.Document{\n\t\t\t\tContent: []*ai.Part{ai.NewTextPart(content)},\n\t\t\t\tMetadata: meta,\n\t\t\t}\n\t\t\tres.Documents = append(res.Documents, doc)\n\t\t}\n\t\tif err := rows.Err(); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn res, nil\n\t}\n\treturn genkit.DefineRetriever(g, provider, \"shows\", f)\n}\n```\n\nAnd here's how to use the retriever in a flow:\n\n```go\nretriever := defineRetriever(g, db, embedder)\n\ntype input struct {\n\tQuestion string\n\tShow string\n}\n\ngenkit.DefineFlow(g, \"askQuestion\", func(ctx context.Context, in input) (string, error) {\n\tres, err := ai.Retrieve(ctx, retriever,\n\t\tai.WithConfig(in.Show),\n\t\tai.WithTextDocs(in.Question))\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tfor _, doc := range res.Documents {\n\t\tfmt.Printf(\"%+v %q\\n\", doc.Metadata, doc.Content[0].Text)\n\t}\n\t// Use documents in RAG prompts.\n\treturn \"\", nil\n})\n```\n", + "text": "# pgvector retriever template\n\nYou can use PostgreSQL and `pgvector` as your retriever implementation. Use the\nfollowing examples as a starting point and modify it to work with your database\nschema.\n\nWe use [database/sql](https://pkg.go.dev/database/sql) to connect to the Postgres server, but you may use another client library of your choice.\n\n```go\nfunc defineRetriever(g *genkit.Genkit, db *sql.DB, embedder ai.Embedder) ai.Retriever {\n\tf := func(ctx context.Context, req *ai.RetrieverRequest) (*ai.RetrieverResponse, error) {\n\t\teres, err := ai.Embed(ctx, embedder, ai.WithDocs(req.Query))\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\trows, err := db.QueryContext(ctx, `\n\t\t\tSELECT episode_id, season_number, chunk as content\n\t\t\tFROM embeddings\n\t\t\tWHERE show_id = $1\n\t\t \tORDER BY embedding <#> $2\n\t\t \tLIMIT 2`,\n\t\t\treq.Options, pgv.NewVector(eres.Embeddings[0].Embedding))\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tdefer rows.Close()\n\n\t\tres := &ai.RetrieverResponse{}\n\t\tfor rows.Next() {\n\t\t\tvar eid, sn int\n\t\t\tvar content string\n\t\t\tif err := rows.Scan(&eid, &sn, &content); err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tmeta := map[string]any{\n\t\t\t\t\"episode_id\": eid,\n\t\t\t\t\"season_number\": sn,\n\t\t\t}\n\t\t\tdoc := &ai.Document{\n\t\t\t\tContent: []*ai.Part{ai.NewTextPart(content)},\n\t\t\t\tMetadata: meta,\n\t\t\t}\n\t\t\tres.Documents = append(res.Documents, doc)\n\t\t}\n\t\tif err := rows.Err(); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn res, nil\n\t}\n\treturn genkit.DefineRetriever(g, provider, \"shows\", f)\n}\n```\n\nAnd here's how to use the retriever in a flow:\n\n```go\nretriever := defineRetriever(g, db, embedder)\n\ntype input struct {\n\tQuestion string\n\tShow string\n}\n\ngenkit.DefineFlow(g, \"askQuestion\", func(ctx context.Context, in input) (string, error) {\n\tres, err := ai.Retrieve(ctx, retriever,\n\t\tai.WithConfig(in.Show),\n\t\tai.WithTextDocs(in.Question))\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tfor _, doc := range res.Documents {\n\t\tfmt.Printf(\"%+v %q\\n\", doc.Metadata, doc.Content[0].Text)\n\t}\n\t// Use documents in RAG prompts.\n\treturn \"\", nil\n})\n```\n", "title": "pgvector retriever template", - "lang": "go" + "description": "Learn how to use PostgreSQL and pgvector as a retriever implementation in Genkit Go.", + "lang": "go", + "headers": "" }, "go/plugins/pinecone.md": { - "text": "The Pinecone plugin provides indexer and retriever implementatons that use the\n[Pinecone](https://www.pinecone.io/) cloud vector database.\n\n## Configuration\n\nTo use this plugin, import the `pinecone` package and call `pinecone.Init()`:\n\n```go\nimport \"github.com/firebase/genkit/go/plugins/pinecone\"\n```\n\n```go\nif err := (&pinecone.Pinecone{}).Init(ctx, g); err != nil {\n\treturn err\n}\n```\n\nThe plugin requires your Pinecone API key.\nConfigure the plugin to use your API key by doing one of the following:\n\n- Set the `PINECONE_API_KEY` environment variable to your API key.\n\n- Specify the API key when you initialize the plugin:\n\n ```go\n if err := (&pinecone.Pinecone{APIKey: pineconeAPIKey}).Init(ctx, g); err != nil {\n\treturn err\n }\n ```\n\n However, don't embed your API key directly in code! Use this feature only\n in conjunction with a service like Cloud Secret Manager or similar.\n\n## Usage\n\nTo add documents to a Pinecone index, first create an index definition that\nspecifies the name of the index and the embedding model you're using:\n\n```go\nmenuIndexer, err := pinecone.DefineIndexer(ctx, g, pinecone.Config{\n\tIndexID: \"menu_data\", // Your Pinecone index\n\tEmbedder: googlegenai.GoogleAIEmbedder(g, \"text-embedding-004\"), // Embedding model of your choice\n})\nif err != nil {\n\treturn err\n}\n```\n\nYou can also optionally specify the key that Pinecone uses for document data\n(`_content`, by default).\n\nThen, call the index's `Index()` method, passing it a list of the documents you\nwant to add:\n\n```go\nif err := ai.Index(\n\tctx,\n\tmenuIndexer,\n\tai.WithDocs(docChunks...)); err != nil {\n\treturn err\n}\n```\n\nSimilarly, to retrieve documents from an index, first create a retriever\ndefinition:\n\n```go\nmenuRetriever, err := pinecone.DefineRetriever(ctx, g, pinecone.Config{\n\tIndexID: \"menu_data\", // Your Pinecone index\n\tEmbedder: googlegenai.GoogleAIEmbedder(g, \"text-embedding-004\"), // Embedding model of your choice\n})\nif err != nil {\n\treturn err\n}\n```\n\nThen, call the retriever's `Retrieve()` method, passing it a text query:\n\n```go\nresp, err := menuRetriever.Retrieve(ctx, &ai.RetrieverRequest{\n\tQuery: ai.DocumentFromText(userInput, nil),\n\tOptions: nil,\n})\nif err != nil {\n\treturn err\n}\nmenuInfo := resp.Documents\n```\n\nSee the [Retrieval-augmented generation](/go/docs/rag) page for a general\ndiscussion on using indexers and retrievers for RAG.\n", + "text": "# Pinecone plugin\n\nThe Pinecone plugin provides retriever implementatons that use the\n[Pinecone](https://www.pinecone.io/) cloud vector database.\n\n## Configuration\n\nTo use this plugin, import the `pinecone` package and call `pinecone.Init()`:\n\n```go\nimport \"github.com/firebase/genkit/go/plugins/pinecone\"\n```\n\n```go\nif err := (&pinecone.Pinecone{}).Init(ctx, g); err != nil {\n\treturn err\n}\n```\n\nThe plugin requires your Pinecone API key.\nConfigure the plugin to use your API key by doing one of the following:\n\n- Set the `PINECONE_API_KEY` environment variable to your API key.\n\n- Specify the API key when you initialize the plugin:\n\n ```go\n if err := (&pinecone.Pinecone{APIKey: pineconeAPIKey}).Init(ctx, g); err != nil {\n\treturn err\n }\n ```\n\n However, don't embed your API key directly in code! Use this feature only\n in conjunction with a service like Cloud Secret Manager or similar.\n\n## Usage\n\nIndex your documents in pinecone. An example of indexing is provided within the Pinecone plugin as shown below. This functionality should be customized by the user according to their use case.\n```go\nerr = pinecone.Index(ctx, docChunks, ds, \"\")\nif err != nil {\n\treturn err\n}\n```\n\nTo retrieve documents from an index, first create a retriever\ndefinition:\n\n```go\nmenuRetriever, err := pinecone.DefineRetriever(ctx, g, pinecone.Config{\n\tIndexID: \"menu_data\", // Your Pinecone index\n\tEmbedder: googlegenai.GoogleAIEmbedder(g, \"text-embedding-004\"), // Embedding model of your choice\n})\nif err != nil {\n\treturn err\n}\n```\n\nThen, call the retriever's `Retrieve()` method, passing it a text query:\n\n```go\nresp, err := menuRetriever.Retrieve(ctx, &ai.RetrieverRequest{\n\tQuery: ai.DocumentFromText(userInput, nil),\n\tOptions: nil,\n})\nif err != nil {\n\treturn err\n}\nmenuInfo := resp.Documents\n```\n\nSee the [Retrieval-augmented generation](/go/docs/rag) page for a general\ndiscussion on using retrievers for RAG.\n", "title": "Pinecone plugin", - "lang": "go" + "description": "Learn how to configure and use the Genkit Pinecone plugin for Go to integrate with the Pinecone cloud vector database.", + "lang": "go", + "headers": "## Configuration\n## Usage\n" }, - "go/plugins/third-party-plugins/index.mdx": { - "text": "import { Card, CardGrid } from '@astrojs/starlight/components';\nimport { Icon } from '@astrojs/starlight/components';\n\nThis page lists third-party plugins for Genkit that are built and maintained by Firebase or\nour partners.\n\n\n \n The Pinecone plugin provides indexer and retriever implementations that use the{' '}\n \n Pinecone\n \n {' '}\n cloud vector database.\n
\n View plugin info\n \n \n The Ollama plugin provides interfaces to any of the local LLMs supported by\n \n Ollama\n \n \n
\n
\n View plugin info\n \n \n The pgvector template is an example PostgreSQL and pgvector retriever implementation. You can use the provided\n examples as a starting point and modify them to work with your database schema.\n
\n View template\n \n\n", + "go/plugins/third-party-plugins/index.md": { + "text": "# Third-party plugins by Firebase and partners\n\nThis page lists third-party plugins for Genkit that are built and maintained by Firebase or\nour partners.\n\n\n \n The Pinecone plugin provides retriever implementations that use the{' '}\n \n Pinecone\n \n {' '}\n cloud vector database.\n
\n View plugin info\n \n \n The Ollama plugin provides interfaces to any of the local LLMs supported by\n \n Ollama\n \n \n
\n
\n View plugin info\n \n \n The pgvector template is an example PostgreSQL and pgvector retriever implementation. You can use the provided\n examples as a starting point and modify them to work with your database schema.\n
\n View template\n \n \n The AlloyDB plugin provides retriever implementations that use the{' '}\n \n AlloyDB for PostgreSQL\n \n {' '}\n cloud vector database.\n
\n View plugin info\n \n \n The Cloud SQL for PostgreSQL plugin provides retriever implementations that use the{' '}\n \n Cloud SQL for PostgreSQL\n \n {' '}\n cloud vector database.\n
\n View plugin info\n \n", "title": "Third-party plugins by Firebase and partners", - "lang": "go" + "description": "This page lists third-party plugins for Genkit that are built and maintained by Firebase or our partners.", + "lang": "go", + "headers": "" }, "python/cloud-run.md": { - "text": "You can easily deploy your Genkit app to Cloud Run.\n\nFor prerequisites and basic scaffolding see [Cloud Run - Python quickstart](https://cloud.google.com/run/docs/quickstarts/build-and-deploy/deploy-python-service#before-you-begin) documentation.\n\nOnce you have a simple Cloud Run app set up and ready to go, update the `requirements.txt` to add Genkit libraries. In this example we'll be using the Google GenAI plugin.\n\n```text title=\"requirements.txt\"\ngenkit\ngenkit-plugin-google-genai\n```\n\nUpdate you app code to use Genkit.\n\n```python\nimport os\n\nfrom flask import Flask\n\nfrom genkit.ai import Genkit\nfrom genkit.plugins.flask import genkit_flask_handler\nfrom genkit.plugins.google_genai import (\n GoogleGenai,\n google_genai_name,\n)\n\nai = Genkit(\n plugins=[GoogleGenai()],\n model=google_genai_name('gemini-2.5-flash'),\n)\n\napp = Flask(__name__)\n\n@app.post('/joke')\n@genkit_flask_handler(ai)\n@ai.flow()\nasync def joke(name: str, ctx):\n return await ai.generate(\n on_chunk=ctx.send_chunk,\n prompt=f'tell a medium sized joke about {name}',\n )\n\nif __name__ == \"__main__\":\n app.run(debug=True, host=\"0.0.0.0\", port=int(os.environ.get(\"PORT\", 8080)))\n```\n\nThen proceeed with Cloud Run [deployment](https://cloud.google.com/run/docs/quickstarts/build-and-deploy/deploy-python-service#deploy) instructions.\n", + "text": "# Deploy with Cloud Run\n\nYou can easily deploy your Genkit app to Cloud Run.\n\nFor prerequisites and basic scaffolding see [Cloud Run - Python quickstart](https://cloud.google.com/run/docs/quickstarts/build-and-deploy/deploy-python-service#before-you-begin) documentation.\n\nOnce you have a simple Cloud Run app set up and ready to go, update the `requirements.txt` to add Genkit libraries. In this example we'll be using the Google GenAI plugin.\n\n```text title=\"requirements.txt\"\ngenkit\ngenkit-plugin-google-genai\n```\n\nUpdate you app code to use Genkit.\n\n```python\nimport os\n\nfrom flask import Flask\n\nfrom genkit.ai import Genkit\nfrom genkit.plugins.flask import genkit_flask_handler\nfrom genkit.plugins.google_genai import (\n GoogleGenai,\n google_genai_name,\n)\n\nai = Genkit(\n plugins=[GoogleGenai()],\n model=google_genai_name('gemini-2.5-flash'),\n)\n\napp = Flask(__name__)\n\n@app.post('/joke')\n@genkit_flask_handler(ai)\n@ai.flow()\nasync def joke(name: str, ctx):\n return await ai.generate(\n on_chunk=ctx.send_chunk,\n prompt=f'tell a medium sized joke about {name}',\n )\n\nif __name__ == \"__main__\":\n app.run(debug=True, host=\"0.0.0.0\", port=int(os.environ.get(\"PORT\", 8080)))\n```\n\nThen proceeed with Cloud Run [deployment](https://cloud.google.com/run/docs/quickstarts/build-and-deploy/deploy-python-service#deploy) instructions.\n", "title": "Deploy with Cloud Run", - "lang": "python" + "description": "Learn how to deploy your Genkit Python app to Cloud Run.", + "lang": "python", + "headers": "" }, "python/flask.md": { - "text": "Prerequisites: make sure you have everything installed from the [Get Started](/python/docs/get-started/) guide.\n\n1. Install Genkit Flask plugin\n\n ```bash\n pip install git+https://github.com/firebase/genkit#subdirectory=py/plugins/flask\n ```\n\n Or create a `requirements.txt` file\n\n ```text title=\"requirements.txt\"\n genkit-plugin-flask @ git+https://github.com/firebase/genkit#subdirectory=py/plugins/google-genai\n ```\n\n2. Create `main.py` file:\n\n ```python title=\"main.py\"\n from flask import Flask\n\n from genkit.ai import Genkit\n from genkit.plugins.flask import genkit_flask_handler\n from genkit.plugins.google_genai import (\n GoogleGenai,\n google_genai_name,\n )\n\n ai = Genkit(\n plugins=[GoogleGenai()],\n model=google_genai_name('gemini-2.5-flash'),\n )\n\n app = Flask(__name__)\n\n\n @app.post('/joke')\n @genkit_flask_handler(ai)\n @ai.flow()\n async def joke(name: str, ctx):\n return await ai.generate(\n on_chunk=ctx.send_chunk,\n prompt=f'tell a medium sized joke about {name}',\n )\n ```\n\n3. Run the app:\n\n ```bash\n flask --app main.py run\n ```\n\n Or with Dev UI:\n\n ```bash\n genkit start -- flask --app main.py run\n ```\n\n You can invoke the flow via HTTP:\n\n ```bash\n curl -X POST http://127.0.0.1:5000/joke -d '{\"data\": \"banana\"}' -H 'content-Type: application/json' -H 'Accept: text/event-stream'\n ```\n\n or you can use [Genkit client library](https://js.api.genkit.dev/modules/genkit.beta_client.html).\n\n## Authorization and custom context\n\nYou can do custom authorization and custom context parsing by passing a `ContextProvider` implementation.\n\n```python\nfrom genkit.types import GenkitError\n\n# Assume parse_request_header is defined elsewhere\n# def parse_request_header(auth_header):\n# # Example implementation: Replace with your actual logic\n# if auth_header and auth_header.startswith('Bearer '):\n# token = auth_header.split(' ')[1]\n# # Validate token and return username, or None/raise error\n# if token == \"valid-token\":\n# return \"testuser\"\n# return None\n\nasync def my_context_provider(request):\n # This function needs access to the request object from Flask\n # The exact way to get headers might depend on how genkit_flask_handler passes the request\n auth_header = request.headers.get('authorization')\n username = parse_request_header(auth_header) # Call the (assumed) function\n return {'username': username}\n\n@app.post('/say_hi')\n@genkit_flask_handler(ai, context_provider=my_context_provider)\n@ai.flow()\nasync def say_hi(name: str, ctx):\n if not ctx.context.get('username'):\n raise GenkitError(status='UNAUTHENTICATED', message='user not provided')\n\n return await ai.generate(\n on_chunk=ctx.send_chunk,\n prompt=f'say hi to {ctx.context.get(\"username\")}',\n )\n```\n\n`parse_request_header` can be your custom authorization header parsing/validation.\n", + "text": "# Deploy with Flask\n\nPrerequisites: make sure you have everything installed from the [Get Started](/python/docs/get-started/) guide.\n\n1. Install Genkit Flask plugin\n\n ```bash\n pip install git+https://github.com/firebase/genkit#subdirectory=py/plugins/flask\n ```\n\n Or create a `requirements.txt` file\n\n ```text title=\"requirements.txt\"\n genkit-plugin-flask @ git+https://github.com/firebase/genkit#subdirectory=py/plugins/google-genai\n ```\n\n2. Create `main.py` file:\n\n ```python title=\"main.py\"\n from flask import Flask\n\n from genkit.ai import Genkit\n from genkit.plugins.flask import genkit_flask_handler\n from genkit.plugins.google_genai import (\n GoogleGenai,\n google_genai_name,\n )\n\n ai = Genkit(\n plugins=[GoogleGenai()],\n model=google_genai_name('gemini-2.5-flash'),\n )\n\n app = Flask(__name__)\n\n\n @app.post('/joke')\n @genkit_flask_handler(ai)\n @ai.flow()\n async def joke(name: str, ctx):\n return await ai.generate(\n on_chunk=ctx.send_chunk,\n prompt=f'tell a medium sized joke about {name}',\n )\n ```\n\n3. Run the app:\n\n ```bash\n flask --app main.py run\n ```\n\n Or with Dev UI:\n\n ```bash\n genkit start -- flask --app main.py run\n ```\n\n You can invoke the flow via HTTP:\n\n ```bash\n curl -X POST http://127.0.0.1:5000/joke -d '{\"data\": \"banana\"}' -H 'content-Type: application/json' -H 'Accept: text/event-stream'\n ```\n\n or you can use [Genkit client library](https://js.api.genkit.dev/modules/genkit.beta_client.html).\n\n## Authorization and custom context\n\nYou can do custom authorization and custom context parsing by passing a `ContextProvider` implementation.\n\n```python\nfrom genkit.types import GenkitError\n\n# Assume parse_request_header is defined elsewhere\n# def parse_request_header(auth_header):\n# # Example implementation: Replace with your actual logic\n# if auth_header and auth_header.startswith('Bearer '):\n# token = auth_header.split(' ')[1]\n# # Validate token and return username, or None/raise error\n# if token == \"valid-token\":\n# return \"testuser\"\n# return None\n\nasync def my_context_provider(request):\n # This function needs access to the request object from Flask\n # The exact way to get headers might depend on how genkit_flask_handler passes the request\n auth_header = request.headers.get('authorization')\n username = parse_request_header(auth_header) # Call the (assumed) function\n return {'username': username}\n\n@app.post('/say_hi')\n@genkit_flask_handler(ai, context_provider=my_context_provider)\n@ai.flow()\nasync def say_hi(name: str, ctx):\n if not ctx.context.get('username'):\n raise GenkitError(status='UNAUTHENTICATED', message='user not provided')\n\n return await ai.generate(\n on_chunk=ctx.send_chunk,\n prompt=f'say hi to {ctx.context.get(\"username\")}',\n )\n```\n\n`parse_request_header` can be your custom authorization header parsing/validation.\n", "title": "Deploy with Flask", - "lang": "python" + "description": "Learn how to build a Flask application using Genkit for Python.", + "lang": "python", + "headers": "## Authorization and custom context\n# Assume parse_request_header is defined elsewhere\n# def parse_request_header(auth_header):\n# # Example implementation: Replace with your actual logic\n# if auth_header and auth_header.startswith('Bearer '):\n# token = auth_header.split(' ')[1]\n# # Validate token and return username, or None/raise error\n# if token == \"valid-token\":\n# return \"testuser\"\n# return None\n" }, - "python/get-started.mdx": { - "text": "import { Tabs, TabItem } from '@astrojs/starlight/components';\n\nThe Genkit libraries for Python are now available for preview! Because the\nPython libraries are currently in Alpha, you might see API and functional\nchanges as development progresses. We recommend using it only for prototyping\nand exploration.\n\nIf you discover issues with the libraries or this documentation please report\nthem in our [GitHub repository](https://github.com/firebase/genkit/).\n\nThis guide shows you how to get started with Genkit in a Python app.\n\n## Requirements\n\n- Python 3.10 or later. See [Download and\n install](https://www.python.org/downloads/) in the official Python docs.\n\n- Node.js 20 or later (for the Genkit CLI and UI). See the below for a brief\n guide on installing Node.\n\n## Create and explore a sample project\n\n1. Create a new project directory:\n\n ```bash\n mkdir genkit-intro && cd genkit-intro\n ```\n\n2. (recommended) Create a Python virtual environment:\n\n ```bash\n python3 -m venv .\n ```\n\n (activate if necessary, depending on the environment)\n\n ```bash\n source bin/activate # for bash\n ```\n\n3. Install dependencies\n\n ```bash\n pip3 install genkit\n pip3 install genkit-plugin-google-genai\n ```\n\n Or create a `requirements.txt` file\n\n ```text title=\"requirements.txt\"\n genkit\n genkit-plugin-google-genai\n ```\n\n and run:\n\n ```bash\n pip3 install -r requirements.txt\n ```\n\n4. Configure your model API key\n\n The simplest way to get started is with Google AI Gemini API. Make sure it's\n [available in your region](https://ai.google.dev/available_regions).\n\n [Generate an API key](https://aistudio.google.com/app/apikey) for the\n Gemini API using Google AI Studio. Then, set the `GEMINI_API_KEY`\n environment variable to your key:\n\n ```bash\n export GEMINI_API_KEY=\n ```\n\n5. Create `main.py` file:\n\n ```python title=\"main.py\"\n import json\n from pydantic import BaseModel, Field\n from genkit.ai import Genkit\n from genkit.plugins.google_genai import GoogleAI\n\n ai = Genkit(\n plugins=[GoogleAI()],\n model='googleai/gemini-2.5-flash',\n )\n\n class RpgCharacter(BaseModel):\n name: str = Field(description='name of the character')\n back_story: str = Field(description='back story')\n abilities: list[str] = Field(description='list of abilities (3-4)')\n\n @ai.flow()\n async def generate_character(name: str):\n result = await ai.generate(\n prompt=f'generate an RPG character named {name}',\n output_schema=RpgCharacter,\n )\n return result.output\n\n async def main() -> None:\n print(json.dumps(await generate_character('Goblorb'), indent=2))\n\n ai.run_main(main())\n ```\n\n6. Run your app. Genkit apps are just regular Python applications. Run them\n however you normally run your app.\n\n ```bash\n python3 main.py\n ```\n\n7. Inspect your app with the Genkit Dev UI\n\n See instructions for installing the Genkit CLI (which includes the Dev UI)\n below.\n\n To inspect your app with Genkit Dev UI run with `genkit start -- `\n command. E.g.:\n\n ```bash\n genkit start -- python3 main.py\n ```\n\n The command will print the Dev UI URL. E.g.:\n\n ```\n Genkit Developer UI: http://localhost:4000\n ```\n\n## Install Genkit CLI\n\n1. If you don't already have Node 20 or newer on your system, install it now.\n\n Recommendation: The [`nvm`](https://github.com/nvm-sh/nvm) and\n [`nvm-windows`](https://github.com/coreybutler/nvm-windows) tools are a\n convenient way to install specific versions of Node if it's not already\n installed on your system. These tools install Node on a per-user basis, so\n you don't need to make system-wide changes.\n\n To install `nvm`:\n\n \n\n \n\n Run the following command:\n\n ```bash\n curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash\n ```\n\n \n\n \n\n Download and run the installer as described in the [nvm-windows\n docs](https://github.com/coreybutler/nvm-windows?tab=readme-ov-file#install-nvm-windows).\n\n Then, to install Node and `npm`, open a new shell and run the following\n command:\n\n ```bash\n nvm install 20\n ```\n\n \n\n \n\n2. Install the Genkit CLI by running the following command:\n\n ```bash\n npm install -g genkit-cli\n ```\n\n This command installs the Genkit CLI into your Node installation directory so\n that it can be used outside of a Node project.\n", + "python/get-started.md": { + "text": "# Get Started with Python (alpha)\n\nThe Genkit libraries for Python are now available for preview! Because the\nPython libraries are currently in Alpha, you might see API and functional\nchanges as development progresses. We recommend using it only for prototyping\nand exploration.\n\nIf you discover issues with the libraries or this documentation please report\nthem in our [GitHub repository](https://github.com/firebase/genkit/).\n\nThis guide shows you how to get started with Genkit in a Python app.\n\n## Requirements\n\n- Python 3.10 or later. See [Download and\n install](https://www.python.org/downloads/) in the official Python docs.\n\n- Node.js 20 or later (for the Genkit CLI and UI). See the below for a brief\n guide on installing Node.\n\n## Create and explore a sample project\n\n1. Create a new project directory:\n\n ```bash\n mkdir genkit-intro && cd genkit-intro\n ```\n\n2. (recommended) Create a Python virtual environment:\n\n ```bash\n python3 -m venv .\n ```\n\n (activate if necessary, depending on the environment)\n\n ```bash\n source bin/activate # for bash\n ```\n\n3. Install dependencies\n\n ```bash\n pip3 install genkit\n pip3 install genkit-plugin-google-genai\n ```\n\n Or create a `requirements.txt` file\n\n ```text title=\"requirements.txt\"\n genkit\n genkit-plugin-google-genai\n ```\n\n and run:\n\n ```bash\n pip3 install -r requirements.txt\n ```\n\n4. Configure your model API key\n\n The simplest way to get started is with Google AI Gemini API. Make sure it's\n [available in your region](https://ai.google.dev/available_regions).\n\n [Generate an API key](https://aistudio.google.com/app/apikey) for the\n Gemini API using Google AI Studio. Then, set the `GEMINI_API_KEY`\n environment variable to your key:\n\n ```bash\n export GEMINI_API_KEY=\n ```\n\n5. Create `main.py` file:\n\n ```python title=\"main.py\"\n import json\n from pydantic import BaseModel, Field\n from genkit.ai import Genkit\n from genkit.plugins.google_genai import GoogleAI\n\n ai = Genkit(\n plugins=[GoogleAI()],\n model='googleai/gemini-2.5-flash',\n )\n\n class RpgCharacter(BaseModel):\n name: str = Field(description='name of the character')\n back_story: str = Field(description='back story')\n abilities: list[str] = Field(description='list of abilities (3-4)')\n\n @ai.flow()\n async def generate_character(name: str):\n result = await ai.generate(\n prompt=f'generate an RPG character named {name}',\n output_schema=RpgCharacter,\n )\n return result.output\n\n async def main() -> None:\n print(json.dumps(await generate_character('Goblorb'), indent=2))\n\n ai.run_main(main())\n ```\n\n6. Run your app. Genkit apps are just regular Python applications. Run them\n however you normally run your app.\n\n ```bash\n python3 main.py\n ```\n\n7. Inspect your app with the Genkit Dev UI\n\n See instructions for installing the Genkit CLI (which includes the Dev UI)\n below.\n\n To inspect your app with Genkit Dev UI run with `genkit start -- `\n command. E.g.:\n\n ```bash\n genkit start -- python3 main.py\n ```\n\n The command will print the Dev UI URL. E.g.:\n\n ```\n Genkit Developer UI: http://localhost:4000\n ```\n\n## Install Genkit CLI\n\n1. If you don't already have Node 20 or newer on your system, install it now.\n\n Recommendation: The [`nvm`](https://github.com/nvm-sh/nvm) and\n [`nvm-windows`](https://github.com/coreybutler/nvm-windows) tools are a\n convenient way to install specific versions of Node if it's not already\n installed on your system. These tools install Node on a per-user basis, so\n you don't need to make system-wide changes.\n\n To install `nvm`:\n\n \n\n \n\n Run the following command:\n\n ```bash\n curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash\n ```\n\n \n\n \n\n Download and run the installer as described in the [nvm-windows\n docs](https://github.com/coreybutler/nvm-windows?tab=readme-ov-file#install-nvm-windows).\n\n Then, to install Node and `npm`, open a new shell and run the following\n command:\n\n ```bash\n nvm install 20\n ```\n\n \n\n \n\n2. Install the Genkit CLI by running the following command:\n\n ```bash\n npm install -g genkit-cli\n ```\n\n This command installs the Genkit CLI into your Node installation directory so\n that it can be used outside of a Node project.", "title": "Get Started with Python (alpha)", - "lang": "python" + "description": "Get started with Genkit using Python (alpha).", + "lang": "python", + "headers": "## Requirements\n## Create and explore a sample project\n## Install Genkit CLI\n" }, "python/reference/flows.md": { - "text": "The core of your app's AI features are generative model requests, but it's rare\nthat you can simply take user input, pass it to the model, and display the model\noutput back to the user. Usually, there are pre- and post-processing steps that\nmust accompany the model call. For example:\n\n- Retrieving contextual information to send with the model call\n- Retrieving the history of the user's current session, for example in a chat\n app\n- Using one model to reformat the user input in a way that's suitable to pass\n to another model\n- Evaluating the \"safety\" of a model's output before presenting it to the user\n- Combining the output of several models\n\nEvery step of this workflow must work together for any AI-related task to\nsucceed.\n\nIn Genkit, you represent this tightly-linked logic using a construction called a\nflow. Flows are written just like functions, using ordinary Python code, but\nthey add additional capabilities intended to ease the development of AI\nfeatures:\n\n- **Type safety**: Input and output schemas defined using\n [Pydantic Models](https://docs.pydantic.dev/latest/concepts/models/), which\n provides both static and runtime type checking\n- **Streaming**: Flows support streaming of data, such as parital LLM responses,\n or any custom serializable objects.\n- **Integration with developer UI**: Debug flows independently of your\n application code using the developer UI. In the developer UI, you can run\n flows and view traces for each step of the flow.\n- **Simplified deployment**: Deploy flows directly as web API endpoints, using\n Cloud Run or any platform that can host a web app.\n\nUnlike similar features in other frameworks, Genkit's flows are lightweight and\nunobtrusive, and don't force your app to conform to any specific abstraction.\nAll of the flow's logic is written in standard Python, and code inside a\nflow doesn't need to be flow-aware.\n\n## Defining and calling flows\n\nIn its simplest form, a flow just wraps a function. The following example wraps\na function that calls `generate()`:\n\n```python\n@ai.flow()\nasync def menu_suggestion_flow(theme: str):\n response = await ai.generate(\n prompt=f'Invent a menu item for a {theme} themed restaurant.',\n )\n return response.text\n```\n\nJust by wrapping your `generate()` calls like this, you add some functionality:\ndoing so lets you run the flow from the Genkit CLI and from the developer UI,\nand is a requirement for several of Genkit's features, including deployment and\nobservability (later sections discuss these topics).\n\n### Input and output schemas\n\nOne of the most important advantages Genkit flows have over directly calling a\nmodel API is type safety of both inputs and outputs. When defining flows, you\ncan define schemas for them using Pydantic.\n\nHere's a refinement of the last example, which defines a flow that takes a\nstring as input and outputs an object:\n\n```python\nfrom pydantic import BaseModel\n\nclass MenuItemSchema(BaseModel):\n dishname: str\n description: str\n\n@ai.flow()\nasync def menu_suggestion_flow(theme: str) -> MenuItemSchema:\n response = await ai.generate(\n prompt=f'Invent a menu item for a {theme} themed restaurant.',\n output_schema=MenuItemSchema,\n )\n return response.output\n```\n\nNote that the schema of a flow does not necessarily have to line up with the\nschema of the `generate()` calls within the flow (in fact, a flow might not even\ncontain `generate()` calls). Here's a variation of the example that passes a\nschema to `generate()`, but uses the structured output to format a simple\nstring, which the flow returns.\n\n```python\n@ai.flow()\nasync def menu_suggestion_flow(theme: str) -> str: # Changed return type annotation\n response = await ai.generate(\n prompt=f'Invent a menu item for a {theme} themed restaurant.',\n output_schema=MenuItemSchema,\n )\n output: MenuItemSchema = response.output\n return f'**{output.dishname}**: {output.description}'\n```\n\n### Calling flows\n\nOnce you've defined a flow, you can call it from your Python code as a regular function. The argument to the flow must conform to the input schema, if you defined one.\n\n```python\nresponse = await menu_suggestion_flow('bistro')\n```\n\nIf you defined an output schema, the flow response will conform to it. For\nexample, if you set the output schema to `MenuItemSchema`, the flow output will\ncontain its properties.\n\n## Streaming flows\n\nFlows support streaming using an interface similar to `generate_stream()`'s streaming\ninterface. Streaming is useful when your flow generates a large amount of\noutput, because you can present the output to the user as it's being generated,\nwhich improves the perceived responsiveness of your app. As a familiar example,\nchat-based LLM interfaces often stream their responses to the user as they are\ngenerated.\n\nHere's an example of a flow that supports streaming:\n\n```python\n@ai.flow()\nasync def menu_suggestion_flow(theme: str, ctx):\n stream, response = ai.generate_stream(\n prompt=f'Invent a menu item for a {theme} themed restaurant.',\n )\n\n async for chunk in stream:\n ctx.send_chunk(chunk.text)\n\n return {\n 'theme': theme,\n 'menu_item': (await response).text,\n }\n\n```\n\nThe second parameter to your flow definition is called \"side channel\". It\nprovides features such as request context and the `send_chunk` callback.\nThe `send_chunk` callback takes a single parameter. Whenever data becomes\navailable within your flow, send the data to the output stream by calling\nthis function.\n\nIn the above example, the values streamed by the flow are directly coupled to\nthe values streamed by the `generate_stream()` call inside the flow. Although this is\noften the case, it doesn't have to be: you can output values to the stream using\nthe callback as often as is useful for your flow.\n\n### Calling streaming flows\n\nStreaming flows are also callable, but they immediately return a response object\nrather than a promise. Flow's `stream` method returns the stream async iterable,\nwhich you can iterate over the streaming output of the flow as it's generated.\n\n```python\nstream, response = menu_suggestion_flow.stream('bistro')\nasync for chunk in stream:\n print(chunk)\n```\n\nYou can also get the complete output of the flow, as you can with a\nnon-streaming flow. The final response is a future that you can `await` on.\n\n```python\nprint(await response)\n```\n\nNote that the streaming output of a flow might not be the same type as the\ncomplete output.\n\n## Debugging flows\n\nOne of the advantages of encapsulating AI logic within a flow is that you can\ntest and debug the flow independently from your app using the Genkit developer\nUI.\n\nTo start the developer UI, run the following commands from your project\ndirectory:\n\n```bash\ngenkit start -- python app.py\n```\n\nUpdate `python app.py` to match the way you normally run your app.\n\nFrom the **Run** tab of developer UI, you can run any of the flows defined in\nyour project:\n\nAfter you've run a flow, you can inspect a trace of the flow invocation by\neither clicking **View trace** or looking on the **Inspect** tab.\n\nIn the trace viewer, you can see details about the execution of the entire flow,\nas well as details for each of the individual steps within the flow.\n\n## Deploying flows\n\nYou can deploy your flows directly as web API endpoints, ready for you to call\nfrom your app clients. Deployment is discussed in detail on several other pages,\nbut this section gives brief overviews of your deployment options.\n\nFor information on deploying to specific platforms, see\n[Deploy with Cloud Run](/python/docs/cloud-run/) and\n[Deploy with Flask](/python/docs/flask/).\n", + "text": "# Defining AI workflows with Flows\n\nThe core of your app's AI features are generative model requests, but it's rare\nthat you can simply take user input, pass it to the model, and display the model\noutput back to the user. Usually, there are pre- and post-processing steps that\nmust accompany the model call. For example:\n\n- Retrieving contextual information to send with the model call\n- Retrieving the history of the user's current session, for example in a chat\n app\n- Using one model to reformat the user input in a way that's suitable to pass\n to another model\n- Evaluating the \"safety\" of a model's output before presenting it to the user\n- Combining the output of several models\n\nEvery step of this workflow must work together for any AI-related task to\nsucceed.\n\nIn Genkit, you represent this tightly-linked logic using a construction called a\nflow. Flows are written just like functions, using ordinary Python code, but\nthey add additional capabilities intended to ease the development of AI\nfeatures:\n\n- **Type safety**: Input and output schemas defined using\n [Pydantic Models](https://docs.pydantic.dev/latest/concepts/models/), which\n provides both static and runtime type checking\n- **Streaming**: Flows support streaming of data, such as parital LLM responses,\n or any custom serializable objects.\n- **Integration with developer UI**: Debug flows independently of your\n application code using the developer UI. In the developer UI, you can run\n flows and view traces for each step of the flow.\n- **Simplified deployment**: Deploy flows directly as web API endpoints, using\n Cloud Run or any platform that can host a web app.\n\nUnlike similar features in other frameworks, Genkit's flows are lightweight and\nunobtrusive, and don't force your app to conform to any specific abstraction.\nAll of the flow's logic is written in standard Python, and code inside a\nflow doesn't need to be flow-aware.\n\n## Defining and calling flows\n\nIn its simplest form, a flow just wraps a function. The following example wraps\na function that calls `generate()`:\n\n```python\n@ai.flow()\nasync def menu_suggestion_flow(theme: str):\n response = await ai.generate(\n prompt=f'Invent a menu item for a {theme} themed restaurant.',\n )\n return response.text\n```\n\nJust by wrapping your `generate()` calls like this, you add some functionality:\ndoing so lets you run the flow from the Genkit CLI and from the developer UI,\nand is a requirement for several of Genkit's features, including deployment and\nobservability (later sections discuss these topics).\n\n### Input and output schemas\n\nOne of the most important advantages Genkit flows have over directly calling a\nmodel API is type safety of both inputs and outputs. When defining flows, you\ncan define schemas for them using Pydantic.\n\nHere's a refinement of the last example, which defines a flow that takes a\nstring as input and outputs an object:\n\n```python\nfrom pydantic import BaseModel\n\nclass MenuItemSchema(BaseModel):\n dishname: str\n description: str\n\n@ai.flow()\nasync def menu_suggestion_flow(theme: str) -> MenuItemSchema:\n response = await ai.generate(\n prompt=f'Invent a menu item for a {theme} themed restaurant.',\n output_schema=MenuItemSchema,\n )\n return response.output\n```\n\nNote that the schema of a flow does not necessarily have to line up with the\nschema of the `generate()` calls within the flow (in fact, a flow might not even\ncontain `generate()` calls). Here's a variation of the example that passes a\nschema to `generate()`, but uses the structured output to format a simple\nstring, which the flow returns.\n\n```python\n@ai.flow()\nasync def menu_suggestion_flow(theme: str) -> str: # Changed return type annotation\n response = await ai.generate(\n prompt=f'Invent a menu item for a {theme} themed restaurant.',\n output_schema=MenuItemSchema,\n )\n output: MenuItemSchema = response.output\n return f'**{output.dishname}**: {output.description}'\n```\n\n### Calling flows\n\nOnce you've defined a flow, you can call it from your Python code as a regular function. The argument to the flow must conform to the input schema, if you defined one.\n\n```python\nresponse = await menu_suggestion_flow('bistro')\n```\n\nIf you defined an output schema, the flow response will conform to it. For\nexample, if you set the output schema to `MenuItemSchema`, the flow output will\ncontain its properties.\n\n## Streaming flows\n\nFlows support streaming using an interface similar to `generate_stream()`'s streaming\ninterface. Streaming is useful when your flow generates a large amount of\noutput, because you can present the output to the user as it's being generated,\nwhich improves the perceived responsiveness of your app. As a familiar example,\nchat-based LLM interfaces often stream their responses to the user as they are\ngenerated.\n\nHere's an example of a flow that supports streaming:\n\n```python\n@ai.flow()\nasync def menu_suggestion_flow(theme: str, ctx):\n stream, response = ai.generate_stream(\n prompt=f'Invent a menu item for a {theme} themed restaurant.',\n )\n\n async for chunk in stream:\n ctx.send_chunk(chunk.text)\n\n return {\n 'theme': theme,\n 'menu_item': (await response).text,\n }\n\n```\n\nThe second parameter to your flow definition is called \"side channel\". It\nprovides features such as request context and the `send_chunk` callback.\nThe `send_chunk` callback takes a single parameter. Whenever data becomes\navailable within your flow, send the data to the output stream by calling\nthis function.\n\nIn the above example, the values streamed by the flow are directly coupled to\nthe values streamed by the `generate_stream()` call inside the flow. Although this is\noften the case, it doesn't have to be: you can output values to the stream using\nthe callback as often as is useful for your flow.\n\n### Calling streaming flows\n\nStreaming flows are also callable, but they immediately return a response object\nrather than a promise. Flow's `stream` method returns the stream async iterable,\nwhich you can iterate over the streaming output of the flow as it's generated.\n\n```python\nstream, response = menu_suggestion_flow.stream('bistro')\nasync for chunk in stream:\n print(chunk)\n```\n\nYou can also get the complete output of the flow, as you can with a\nnon-streaming flow. The final response is a future that you can `await` on.\n\n```python\nprint(await response)\n```\n\nNote that the streaming output of a flow might not be the same type as the\ncomplete output.\n\n## Debugging flows\n\nOne of the advantages of encapsulating AI logic within a flow is that you can\ntest and debug the flow independently from your app using the Genkit developer\nUI.\n\nTo start the developer UI, run the following commands from your project\ndirectory:\n\n```bash\ngenkit start -- python app.py\n```\n\nUpdate `python app.py` to match the way you normally run your app.\n\nFrom the **Run** tab of developer UI, you can run any of the flows defined in\nyour project:\n\nAfter you've run a flow, you can inspect a trace of the flow invocation by\neither clicking **View trace** or looking on the **Inspect** tab.\n\nIn the trace viewer, you can see details about the execution of the entire flow,\nas well as details for each of the individual steps within the flow.\n\n## Deploying flows\n\nYou can deploy your flows directly as web API endpoints, ready for you to call\nfrom your app clients. Deployment is discussed in detail on several other pages,\nbut this section gives brief overviews of your deployment options.\n\nFor information on deploying to specific platforms, see\n[Deploy with Cloud Run](/python/docs/cloud-run/) and\n[Deploy with Flask](/python/docs/flask/).\n", "title": "Defining AI workflows with Flows", - "lang": "python" + "description": "Learn how to define, call, stream, debug, and deploy AI workflows using Genkit Flows in Python.", + "lang": "python", + "headers": "## Defining and calling flows\n### Input and output schemas\n### Calling flows\n## Streaming flows\n### Calling streaming flows\n## Debugging flows\n## Deploying flows\n" }, "python/reference/interrupts.md": { - "text": "import ExampleLink from \"@/examples/ExampleLink.astro\";\n\n_Interrupts_ are a special kind of [tool](/python/docs/reference/tools/) that can pause the\nLLM generation-and-tool-calling loop to return control back to you. When\nyou're ready, you can then _resume_ generation by sending _replies_ that the LLM\nprocesses for further generation.\n\nThe most common uses for interrupts fall into a few categories:\n\n- **Human-in-the-Loop:** Enabling the user of an interactive AI\n to clarify needed information or confirm the LLM's action\n before it is completed, providing a measure of safety and confidence.\n- **Async Processing:** Starting an asynchronous task that can only be\n completed out-of-band, such as sending an approval notification to\n a human reviewer or kicking off a long-running background process.\n- **Exit from an Autonomous Task:** Providing the model a way\n to mark a task as complete, in a workflow that might iterate through\n a long series of tool calls.\n\n## Before you begin\n\nAll of the examples documented here assume that you have already set up a\nproject with Genkit dependencies installed. If you want to run the code\nexamples on this page, first complete the steps in the\n[Get started](/python/docs/get-started/) guide.\n\nBefore diving too deeply, you should also be familiar with the following\nconcepts:\n\n- [Generating content](/python/docs/reference/models/) with AI models.\n- Genkit's system for [defining input and output schemas](/python/docs/reference/flows/).\n- General methods of [tool-calling](/python/docs/reference/tools/).\n\n## Overview of interrupts\n\nAt a high level, this is what an interrupt looks like when\ninteracting with an LLM:\n\n1. The calling application prompts the LLM with a request. The prompt includes\n a list of tools, including at least one for an interrupt that the LLM\n can use to generate a response.\n2. The LLM either generates either a complete response or a tool call request\n in a specific format. To the LLM, an interrupt call looks like any\n other tool call.\n3. If the LLM calls an interrupting tool,\n the Genkit library automatically pauses generation rather than immediately\n passing responses back to the model for additional processing.\n4. The developer checks whether an interrupt call is made, and performs whatever\n task is needed to collect the information needed for the interrupt response.\n5. The developer resumes generation by passing an interrupt response to the\n model. This action triggers a return to Step 2.\n\n## Define manual-response interrupts\n\nThe most common kind of interrupt allows the LLM to request clarification from\nthe user, for example by asking a multiple-choice question.\n\nFor this use case, use the Genkit instance's `tool()` decorator:\n\n```python\nfrom pydantic import BaseModel, Field\n\nclass Questions(BaseModel):\n choices: list[str] = Field(description='the choices to display to the user')\n allow_other: bool = Field(description='when true, allow write-ins')\n\n\n@ai.tool()\ndef ask_question(input: Questions, ctx) -> str:\n \"\"\"Use this to ask the user a clarifying question\"\"\"\n ctx.interrupt()\n```\n\nNote that the `outputSchema` of an interrupt corresponds to the response data\nyou will provide as opposed to something that will be automatically populated\nby a tool function.\n\n### Use interrupts\n\nInterrupts are passed into the `tools` array when generating content, just like\nother types of tools. You can pass both normal tools and interrupts to the\nsame `generate` call:\n\n```python\ninterrupted_response = await ai.generate(\n prompt='Ask me a movie trivia question.',\n tools=['ask_question'],\n)\n```\n\nGenkit immediately returns a response on receipt of an interrupt tool call.\n\n### Respond to interrupts\n\nIf you've passed one or more interrupts to your generate call, you\nneed to check the response for interrupts so that you can handle them:\n\n```python\n# You can check the 'finish_reason' attribute of the response\nif interrupted_response.finish_reason == 'interrupted':\n print(\"Generation interrupted.\")\n\n# Or you can check if any interrupt requests are on the response\nif interrupted_response.interrupts and len(interrupted_response.interrupts) > 0:\n print(f\"Interrupts found: {len(interrupted_response.interrupts)}\")\n```\n\nResponding to an interrupt is done using the `tool_responses` option on a subsequent\n`generate` call, making sure to pass in the existing history. There's a `tool_response`\nhelper function to help you construct the response.\n\nOnce resumed, the model re-enters the generation loop, including tool\nexecution, until either it completes or another interrupt is triggered:\n\n```python\nfrom genkit.ai import tool_response # Assuming tool_response is imported\n\nresponse = await ai.generate(\n messages=interrupted_response.messages,\n tool_responses=[tool_response(interrupted_response.interrupts[0], 'b')],\n tools=['ask_question'],\n)\n```\n", + "text": "# Tool Interrupts\n\nimport ExampleLink from \"@/examples/ExampleLink.astro\";\n\n_Interrupts_ are a special kind of [tool](/python/docs/reference/tools/) that can pause the\nLLM generation-and-tool-calling loop to return control back to you. When\nyou're ready, you can then _resume_ generation by sending _replies_ that the LLM\nprocesses for further generation.\n\nThe most common uses for interrupts fall into a few categories:\n\n- **Human-in-the-Loop:** Enabling the user of an interactive AI\n to clarify needed information or confirm the LLM's action\n before it is completed, providing a measure of safety and confidence.\n- **Async Processing:** Starting an asynchronous task that can only be\n completed out-of-band, such as sending an approval notification to\n a human reviewer or kicking off a long-running background process.\n- **Exit from an Autonomous Task:** Providing the model a way\n to mark a task as complete, in a workflow that might iterate through\n a long series of tool calls.\n\n## Before you begin\n\nAll of the examples documented here assume that you have already set up a\nproject with Genkit dependencies installed. If you want to run the code\nexamples on this page, first complete the steps in the\n[Get started](/python/docs/get-started/) guide.\n\nBefore diving too deeply, you should also be familiar with the following\nconcepts:\n\n- [Generating content](/python/docs/reference/models/) with AI models.\n- Genkit's system for [defining input and output schemas](/python/docs/reference/flows/).\n- General methods of [tool-calling](/python/docs/reference/tools/).\n\n## Overview of interrupts\n\nAt a high level, this is what an interrupt looks like when\ninteracting with an LLM:\n\n1. The calling application prompts the LLM with a request. The prompt includes\n a list of tools, including at least one for an interrupt that the LLM\n can use to generate a response.\n2. The LLM generates either a complete response or a tool call request\n in a specific format. To the LLM, an interrupt call looks like any\n other tool call.\n3. If the LLM calls an interrupting tool,\n the Genkit library automatically pauses generation rather than immediately\n passing responses back to the model for additional processing.\n4. The developer checks whether an interrupt call is made, and performs whatever\n task is needed to collect the information needed for the interrupt response.\n5. The developer resumes generation by passing an interrupt response to the\n model. This action triggers a return to Step 2.\n\n## Define manual-response interrupts\n\nThe most common kind of interrupt allows the LLM to request clarification from\nthe user, for example by asking a multiple-choice question.\n\nFor this use case, use the Genkit instance's `tool()` decorator:\n\n```python\nfrom pydantic import BaseModel, Field\n\nclass Questions(BaseModel):\n choices: list[str] = Field(description='the choices to display to the user')\n allow_other: bool = Field(description='when true, allow write-ins')\n\n\n@ai.tool()\ndef ask_question(input: Questions, ctx) -> str:\n \"\"\"Use this to ask the user a clarifying question\"\"\"\n ctx.interrupt()\n```\n\nNote that the `outputSchema` of an interrupt corresponds to the response data\nyou will provide as opposed to something that will be automatically populated\nby a tool function.\n\n### Use interrupts\n\nInterrupts are passed into the `tools` array when generating content, just like\nother types of tools. You can pass both normal tools and interrupts to the\nsame `generate` call:\n\n```python\ninterrupted_response = await ai.generate(\n prompt='Ask me a movie trivia question.',\n tools=['ask_question'],\n)\n```\n\nGenkit immediately returns a response on receipt of an interrupt tool call.\n\n### Respond to interrupts\n\nIf you've passed one or more interrupts to your generate call, you\nneed to check the response for interrupts so that you can handle them:\n\n```python\n# You can check the 'finish_reason' attribute of the response\nif interrupted_response.finish_reason == 'interrupted':\n print(\"Generation interrupted.\")\n\n# Or you can check if any interrupt requests are on the response\nif interrupted_response.interrupts and len(interrupted_response.interrupts) > 0:\n print(f\"Interrupts found: {len(interrupted_response.interrupts)}\")\n```\n\nResponding to an interrupt is done using the `tool_responses` option on a subsequent\n`generate` call, making sure to pass in the existing history. There's a `tool_response`\nhelper function to help you construct the response.\n\nOnce resumed, the model re-enters the generation loop, including tool\nexecution, until either it completes or another interrupt is triggered:\n\n```python\nfrom genkit.ai import tool_response # Assuming tool_response is imported\n\nresponse = await ai.generate(\n messages=interrupted_response.messages,\n tool_responses=[tool_response(interrupted_response.interrupts[0], 'b')],\n tools=['ask_question'],\n)\n```\n", "title": "Tool Interrupts", - "lang": "python" + "description": "Learn how to use interrupts to pause and resume LLM generation loops in Genkit Python.", + "lang": "python", + "headers": "## Before you begin\n## Overview of interrupts\n## Define manual-response interrupts\n### Use interrupts\n### Respond to interrupts\n# You can check the 'finish_reason' attribute of the response\n# Or you can check if any interrupt requests are on the response\n" }, "python/reference/models.md": { - "text": "At the heart of generative AI are AI _models_. Currently, the two most prominent\nexamples of generative models are large language models (LLMs) and image\ngeneration models. These models take input, called a _prompt_ (most commonly\ntext, an image, or a combination of both), and from it produce as output text,\nan image, or even audio or video.\n\nThe output of these models can be surprisingly convincing: LLMs generate text\nthat appears as though it could have been written by a human being, and image\ngeneration models can produce images that are very close to real photographs or\nartwork created by humans.\n\nIn addition, LLMs have proven capable of tasks beyond simple text generation:\n\n- Writing computer programs\n- Planning subtasks that are required to complete a larger task\n- Organizing unorganized data\n- Understanding and extracting information data from a corpus of text\n- Following and performing automated activities based on a text description of\n the activity\n\nThere are many models available to you, from several different providers. Each\nmodel has its own strengths and weaknesses and one model might excel at one task\nbut perform less well at others. Apps making use of generative AI can often\nbenefit from using multiple different models depending on the task at hand.\n\nAs an app developer, you typically don't interact with generative AI\nmodels directly, but rather through services available as web APIs.\nAlthough these services often have similar functionality, they all provide them\nthrough different and incompatible APIs. If you want to make use of multiple\nmodel services, you have to use each of their proprietary SDKs, potentially\nincompatible with each other. And if you want to upgrade from one model to the\nnewest and most capable one, you might have to build that integration all over\nagain.\n\nGenkit addresses this challenge by providing a single interface that abstracts\naway the details of accessing potentially any generative AI model service, with\nseveral pre-built implementations already available. Building your AI-powered\napp around Genkit simplifies the process of making your first generative AI call\nand makes it equally easy to combine multiple models or swap one model for\nanother as new models emerge.\n\n### Loading and configuring model plugins\n\nBefore you can use Genkit to start generating content, you need to load and\nconfigure a model plugin. If you're coming from the Getting Started guide,\nyou've already done this. Otherwise, see the [Get started](/python/docs/get-started/)\nguide or the individual plugin's documentation and follow the steps there before\ncontinuing.\n\n### The generate() method\n\nIn Genkit, the primary interface through which you interact with generative AI\nmodels is the `generate()` method.\n\nThe simplest `generate()` call specifies the model you want to use and a text\nprompt:\n\n```python\nimport asyncio\nfrom genkit.ai import Genkit\nfrom genkit.plugins.google_genai import GoogleGenai\n\nai = Genkit(\n plugins=[GoogleGenai()],\n model='googleai/gemini-2.5-flash',\n)\n\nasync def main() -> None:\n result = await ai.generate(\n prompt='Invent a menu item for a pirate themed restaurant.',\n )\n print(result.text)\n\nai.run_main(main())\n```\n\nWhen you run this brief example it will print out the output of the `generate()`\nall, which will usually be Markdown text as in the following example:\n\n```md\n## The Blackheart's Bounty\n\n**A hearty stew of slow-cooked beef, spiced with rum and molasses, served in a\nhollowed-out cannonball with a side of crusty bread and a dollop of tangy\npineapple salsa.**\n\n**Description:** This dish is a tribute to the hearty meals enjoyed by pirates\non the high seas. The beef is tender and flavorful, infused with the warm spices\nof rum and molasses. The pineapple salsa adds a touch of sweetness and acidity,\nbalancing the richness of the stew. The cannonball serving vessel adds a fun and\nthematic touch, making this dish a perfect choice for any pirate-themed\nadventure.\n```\n\nRun the script again and you'll get a different output.\n\nThe preceding code sample sent the generation request to the default model,\nwhich you specified when you configured the Genkit instance.\n\nYou can also specify a model for a single `generate()` call:\n\n```python\nresult = await ai.generate(\n prompt='Invent a menu item for a pirate themed restaurant.',\n model='googleai/gemini-2.0-pro',\n)\n```\n\nA model string identifier looks like `providerid/modelid`, where the provider ID\n(in this case, `google_genai`) identifies the plugin, and the model ID is a\nplugin-specific string identifier for a specific version of a model.\n\nThese examples also illustrate an important point: when you use\n`generate()` to make generative AI model calls, changing the model you want to\nuse is simply a matter of passing a different value to the model parameter. By\nusing `generate()` instead of the native model SDKs, you give yourself the\nflexibility to more easily use several different models in your app and change\nmodels in the future.\n\nSo far you have only seen examples of the simplest `generate()` calls. However,\n`generate()` also provides an interface for more advanced interactions with\ngenerative models, which you will see in the sections that follow.\n\n### System prompts\n\nSome models support providing a _system prompt_, which gives the model\ninstructions as to how you want it to respond to messages from the user. You can\nuse the system prompt to specify a persona you want the model to adopt, the tone\nof its responses, the format of its responses, and so on.\n\nIf the model you're using supports system prompts, you can provide one with the\n`system` parameter:\n\n```python\nresult = await ai.generate(\n system='You are a food industry marketing consultant.',\n prompt='Invent a menu item for a pirate themed restaurant.',\n)\n```\n\n### Model parameters\n\nThe `generate()` function takes a `config` parameter, through which you can\nspecify optional settings that control how the model generates content:\n\n```python\nresult = await ai.generate(\n prompt='Invent a menu item for a pirate themed restaurant.',\n config={\n 'max_output_tokens': 400,\n 'stop_sequences': ['', ''],\n 'temperature': 1.2,\n 'top_p': 0.4,\n 'top_k': 50,\n },\n)\n```\n\nThe exact parameters that are supported depend on the individual model and model\nAPI. However, the parameters in the previous example are common to almost every\nmodel. The following is an explanation of these parameters:\n\n### Structured output\n\nWhen using generative AI as a component in your application, you often want\noutput in a format other than plain text. Even if you're just generating content\nto display to the user, you can benefit from structured output simply for the\npurpose of presenting it more attractively to the user. But for more advanced\napplications of generative AI, such as programmatic use of the model's output,\nor feeding the output of one model into another, structured output is a must.\n\nIn Genkit, you can request structured output from a model by specifying a schema\nwhen you call `generate()`:\n\n```python\nfrom pydantic import BaseModel\n\nclass MenuItemSchema(BaseModel):\n name: str\n description: str\n calories: int\n allergens: list[str]\n\nresult = await ai.generate(\n prompt='Invent a menu item for a pirate themed restaurant.',\n output_schema=MenuItemSchema,\n)\n```\n\nModel output schemas are specified using the [Pydantic Models](https://docs.pydantic.dev/latest/concepts/models/). In addition to a schema definition language, Pydantic also provides runtime\ntype checking, which bridges the gap between static Python types and the\nunpredictable output of generative AI models. Pydantic lets you write code that can\nrely on the fact that a successful generate call will always return output that\nconforms to your Python types.\n\nWhen you specify a schema in `generate()`, Genkit does several things behind the\nscenes:\n\n- Augments the prompt with additional guidance about the desired output format.\n This also has the side effect of specifying to the model what content exactly\n you want to generate (for example, not only suggest a menu item but also\n generate a description, a list of allergens, and so on).\n- Parses the model output into a Pydantic object.\n- Verifies that the output conforms with the schema.\n\nTo get structured output from a successful generate call, use the response\nobject's `output` property:\n\n```python\noutput = response.output\n```\n\n### Streaming\n\nWhen generating large amounts of text, you can improve the experience for your\nusers by presenting the output as it's generated—streaming the output. A\nfamiliar example of streaming in action can be seen in most LLM chat apps: users\ncan read the model's response to their message as it's being generated, which\nimproves the perceived responsiveness of the application and enhances the\nillusion of chatting with an intelligent counterpart.\n\nIn Genkit, you can stream output using the `generateStream()` method. Its\nsyntax is similar to the `generate()` method:\n\n```python\nstream, response = ai.generate_stream(\n prompt='Suggest a complete menu for a pirate themed restaurant.',\n)\n```\n\nThe response object has a `stream` property, which you can use to iterate over\nthe streaming output of the request as it's generated:\n\n```python\nasync for chunk in stream:\n print(chunk.text)\n```\n\nYou can also get the complete output of the request, as you can with a\nnon-streaming request:\n\n```python\ncomplete_text = (await response).text\n```\n\nStreaming also works with structured output:\n\n```python\nfrom pydantic import BaseModel\n\nclass MenuItemSchema(BaseModel):\n name: str\n description: str\n calories: int\n allergens: list[str]\n\nclass MenuSchema(BaseModel):\n starters: list[MenuItemSchema]\n mains: list[MenuItemSchema]\n desserts: list[MenuItemSchema]\n\nstream, response = ai.generate_stream(\n prompt='Invent a menu item for a pirate themed restaurant.',\n output_schema=MenuSchema,\n)\n\nasync for chunk in stream:\n print(chunk.output)\n\nprint((await response).output)\n```\n\nStreaming structured output works a little differently from streaming text: the\n`output` property of a response chunk is an object constructed from the\naccumulation of the chunks that have been produced so far, rather than an object\nrepresenting a single chunk (which might not be valid on its own). **Every chunk\nof structured output in a sense supersedes the chunk that came before it**.\n\nFor example, here's what the first five outputs from the prior example might\nlook like:\n\n```json\nnull\n\n{ \"starters\": [ {} ] }\n\n{\n \"starters\": [ { \"name\": \"Captain's Treasure Chest\", \"description\": \"A\" } ]\n}\n\n{\n \"starters\": [\n {\n \"name\": \"Captain's Treasure Chest\",\n \"description\": \"A mix of spiced nuts, olives, and marinated cheese served in a treasure chest.\",\n \"calories\": 350\n }\n ]\n}\n\n{\n \"starters\": [\n {\n \"name\": \"Captain's Treasure Chest\",\n \"description\": \"A mix of spiced nuts, olives, and marinated cheese served in a treasure chest.\",\n \"calories\": 350,\n \"allergens\": []\n },\n { \"name\": \"Shipwreck Salad\", \"description\": \"Fresh\" }\n ]\n}\n```\n\n### Multimodal input\n\nThe examples you've seen so far have used text strings as model prompts. While\nthis remains the most common way to prompt generative AI models, many models can\nalso accept other media as prompts. Media prompts are most often used in\nconjunction with text prompts that instruct the model to perform some operation\non the media, such as to caption an image or transcribe an audio recording.\n\nThe ability to accept media input and the types of media you can use are\ncompletely dependent on the model and its API. For example, the Gemini 1.5\nseries of models can accept images, video, and audio as prompts.\n\nTo provide a media prompt to a model that supports it, instead of passing a\nsimple text prompt to `generate`, pass an array consisting of a media part and a\ntext part:\n\n```python\nfrom genkit.ai import Part # Import Part\n\nresult = await ai.generate(\n prompt=[\n Part(media={'url': 'https://example.com/photo.jpg'}),\n Part(text='Compose a poem about this image.'),\n ],\n)\n```\n\nIn the above example, you specified an image using a publicly-accessible HTTPS\nURL. You can also pass media data directly by encoding it as a data URL. For\nexample:\n\n```python\nimport base64\nfrom genkit.ai import Part # Import Part\n\n# Assume read_file is defined elsewhere to read image bytes\n# def read_file(path):\n# with open(path, 'rb') as f:\n# return f.read()\n\nimage_bytes = read_file('image.jpg')\nbase64_encoded_image = base64.b64encode(image_bytes).decode('utf-8') # Decode bytes to string\n\nresult = await ai.generate(\n prompt=[\n Part(media={'url': f'data:image/jpeg;base64,{base64_encoded_image}'}),\n Part(text='Compose a poem about this image.'),\n ],\n)\n```\n\nAll models that support media input support both data URLs and HTTPS URLs. Some\nmodel plugins add support for other media sources. For example, the Vertex AI\nplugin also lets you use Cloud Storage (`gs://`) URLs.\n\n\n\n### Generating media\n\nSo far, most of the examples on this page have dealt with generating text using\nLLMs. However, Genkit can also be used with image generation models. Using\n`generate()` with an image generation model is similar to using an LLM. For\nexample, to generate an image using the Imagen model:\n\n```python\n# TODO: Add example for image generation\n```\n\n\n", + "text": "# Generating content with AI models\n\nAt the heart of generative AI are AI _models_. Currently, the two most prominent\nexamples of generative models are large language models (LLMs) and image\ngeneration models. These models take input, called a _prompt_ (most commonly\ntext, an image, or a combination of both), and from it produce as output text,\nan image, or even audio or video.\n\nThe output of these models can be surprisingly convincing: LLMs generate text\nthat appears as though it could have been written by a human being, and image\ngeneration models can produce images that are very close to real photographs or\nartwork created by humans.\n\nIn addition, LLMs have proven capable of tasks beyond simple text generation:\n\n- Writing computer programs\n- Planning subtasks that are required to complete a larger task\n- Organizing unorganized data\n- Understanding and extracting information data from a corpus of text\n- Following and performing automated activities based on a text description of\n the activity\n\nThere are many models available to you, from several different providers. Each\nmodel has its own strengths and weaknesses and one model might excel at one task\nbut perform less well at others. Apps making use of generative AI can often\nbenefit from using multiple different models depending on the task at hand.\n\nAs an app developer, you typically don't interact with generative AI\nmodels directly, but rather through services available as web APIs.\nAlthough these services often have similar functionality, they all provide them\nthrough different and incompatible APIs. If you want to make use of multiple\nmodel services, you have to use each of their proprietary SDKs, potentially\nincompatible with each other. And if you want to upgrade from one model to the\nnewest and most capable one, you might have to build that integration all over\nagain.\n\nGenkit addresses this challenge by providing a single interface that abstracts\naway the details of accessing potentially any generative AI model service, with\nseveral pre-built implementations already available. Building your AI-powered\napp around Genkit simplifies the process of making your first generative AI call\nand makes it equally easy to combine multiple models or swap one model for\nanother as new models emerge.\n\n### Loading and configuring model plugins\n\nBefore you can use Genkit to start generating content, you need to load and\nconfigure a model plugin. If you're coming from the Getting Started guide,\nyou've already done this. Otherwise, see the [Get started](/python/docs/get-started/)\nguide or the individual plugin's documentation and follow the steps there before\ncontinuing.\n\n### The generate() method\n\nIn Genkit, the primary interface through which you interact with generative AI\nmodels is the `generate()` method.\n\nThe simplest `generate()` call specifies the model you want to use and a text\nprompt:\n\n```python\nimport asyncio\nfrom genkit.ai import Genkit\nfrom genkit.plugins.google_genai import GoogleGenai\n\nai = Genkit(\n plugins=[GoogleGenai()],\n model='googleai/gemini-2.5-flash',\n)\n\nasync def main() -> None:\n result = await ai.generate(\n prompt='Invent a menu item for a pirate themed restaurant.',\n )\n print(result.text)\n\nai.run_main(main())\n```\n\nWhen you run this brief example it will print out the output of the `generate()`\nall, which will usually be Markdown text as in the following example:\n\n```md\n## The Blackheart's Bounty\n\n**A hearty stew of slow-cooked beef, spiced with rum and molasses, served in a\nhollowed-out cannonball with a side of crusty bread and a dollop of tangy\npineapple salsa.**\n\n**Description:** This dish is a tribute to the hearty meals enjoyed by pirates\non the high seas. The beef is tender and flavorful, infused with the warm spices\nof rum and molasses. The pineapple salsa adds a touch of sweetness and acidity,\nbalancing the richness of the stew. The cannonball serving vessel adds a fun and\nthematic touch, making this dish a perfect choice for any pirate-themed\nadventure.\n```\n\nRun the script again and you'll get a different output.\n\nThe preceding code sample sent the generation request to the default model,\nwhich you specified when you configured the Genkit instance.\n\nYou can also specify a model for a single `generate()` call:\n\n```python\nresult = await ai.generate(\n prompt='Invent a menu item for a pirate themed restaurant.',\n model='googleai/gemini-2.0-pro',\n)\n```\n\nA model string identifier looks like `providerid/modelid`, where the provider ID\n(in this case, `google_genai`) identifies the plugin, and the model ID is a\nplugin-specific string identifier for a specific version of a model.\n\nThese examples also illustrate an important point: when you use\n`generate()` to make generative AI model calls, changing the model you want to\nuse is simply a matter of passing a different value to the model parameter. By\nusing `generate()` instead of the native model SDKs, you give yourself the\nflexibility to more easily use several different models in your app and change\nmodels in the future.\n\nSo far you have only seen examples of the simplest `generate()` calls. However,\n`generate()` also provides an interface for more advanced interactions with\ngenerative models, which you will see in the sections that follow.\n\n### System prompts\n\nSome models support providing a _system prompt_, which gives the model\ninstructions as to how you want it to respond to messages from the user. You can\nuse the system prompt to specify a persona you want the model to adopt, the tone\nof its responses, the format of its responses, and so on.\n\nIf the model you're using supports system prompts, you can provide one with the\n`system` parameter:\n\n```python\nresult = await ai.generate(\n system='You are a food industry marketing consultant.',\n prompt='Invent a menu item for a pirate themed restaurant.',\n)\n```\n\n### Model parameters\n\nThe `generate()` function takes a `config` parameter, through which you can\nspecify optional settings that control how the model generates content:\n\n```python\nresult = await ai.generate(\n prompt='Invent a menu item for a pirate themed restaurant.',\n config={\n 'max_output_tokens': 400,\n 'stop_sequences': ['', ''],\n 'temperature': 1.2,\n 'top_p': 0.4,\n 'top_k': 50,\n },\n)\n```\n\nThe exact parameters that are supported depend on the individual model and model\nAPI. However, the parameters in the previous example are common to almost every\nmodel. The following is an explanation of these parameters:\n\n### Structured output\n\nWhen using generative AI as a component in your application, you often want\noutput in a format other than plain text. Even if you're just generating content\nto display to the user, you can benefit from structured output simply for the\npurpose of presenting it more attractively to the user. But for more advanced\napplications of generative AI, such as programmatic use of the model's output,\nor feeding the output of one model into another, structured output is a must.\n\nIn Genkit, you can request structured output from a model by specifying a schema\nwhen you call `generate()`:\n\n```python\nfrom pydantic import BaseModel\n\nclass MenuItemSchema(BaseModel):\n name: str\n description: str\n calories: int\n allergens: list[str]\n\nresult = await ai.generate(\n prompt='Invent a menu item for a pirate themed restaurant.',\n output_schema=MenuItemSchema,\n)\n```\n\nModel output schemas are specified using the [Pydantic Models](https://docs.pydantic.dev/latest/concepts/models/). In addition to a schema definition language, Pydantic also provides runtime\ntype checking, which bridges the gap between static Python types and the\nunpredictable output of generative AI models. Pydantic lets you write code that can\nrely on the fact that a successful generate call will always return output that\nconforms to your Python types.\n\nWhen you specify a schema in `generate()`, Genkit does several things behind the\nscenes:\n\n- Augments the prompt with additional guidance about the desired output format.\n This also has the side effect of specifying to the model what content exactly\n you want to generate (for example, not only suggest a menu item but also\n generate a description, a list of allergens, and so on).\n- Parses the model output into a Pydantic object.\n- Verifies that the output conforms with the schema.\n\nTo get structured output from a successful generate call, use the response\nobject's `output` property:\n\n```python\noutput = response.output\n```\n\n### Streaming\n\nWhen generating large amounts of text, you can improve the experience for your\nusers by presenting the output as it's generated—streaming the output. A\nfamiliar example of streaming in action can be seen in most LLM chat apps: users\ncan read the model's response to their message as it's being generated, which\nimproves the perceived responsiveness of the application and enhances the\nillusion of chatting with an intelligent counterpart.\n\nIn Genkit, you can stream output using the `generateStream()` method. Its\nsyntax is similar to the `generate()` method:\n\n```python\nstream, response = ai.generate_stream(\n prompt='Suggest a complete menu for a pirate themed restaurant.',\n)\n```\n\nThe response object has a `stream` property, which you can use to iterate over\nthe streaming output of the request as it's generated:\n\n```python\nasync for chunk in stream:\n print(chunk.text)\n```\n\nYou can also get the complete output of the request, as you can with a\nnon-streaming request:\n\n```python\ncomplete_text = (await response).text\n```\n\nStreaming also works with structured output:\n\n```python\nfrom pydantic import BaseModel\n\nclass MenuItemSchema(BaseModel):\n name: str\n description: str\n calories: int\n allergens: list[str]\n\nclass MenuSchema(BaseModel):\n starters: list[MenuItemSchema]\n mains: list[MenuItemSchema]\n desserts: list[MenuItemSchema]\n\nstream, response = ai.generate_stream(\n prompt='Invent a menu item for a pirate themed restaurant.',\n output_schema=MenuSchema,\n)\n\nasync for chunk in stream:\n print(chunk.output)\n\nprint((await response).output)\n```\n\nStreaming structured output works a little differently from streaming text: the\n`output` property of a response chunk is an object constructed from the\naccumulation of the chunks that have been produced so far, rather than an object\nrepresenting a single chunk (which might not be valid on its own). **Every chunk\nof structured output in a sense supersedes the chunk that came before it**.\n\nFor example, here's what the first five outputs from the prior example might\nlook like:\n\n```json\nnull\n\n{ \"starters\": [ {} ] }\n\n{\n \"starters\": [ { \"name\": \"Captain's Treasure Chest\", \"description\": \"A\" } ]\n}\n\n{\n \"starters\": [\n {\n \"name\": \"Captain's Treasure Chest\",\n \"description\": \"A mix of spiced nuts, olives, and marinated cheese served in a treasure chest.\",\n \"calories\": 350\n }\n ]\n}\n\n{\n \"starters\": [\n {\n \"name\": \"Captain's Treasure Chest\",\n \"description\": \"A mix of spiced nuts, olives, and marinated cheese served in a treasure chest.\",\n \"calories\": 350,\n \"allergens\": []\n },\n { \"name\": \"Shipwreck Salad\", \"description\": \"Fresh\" }\n ]\n}\n```\n\n### Multimodal input\n\nThe examples you've seen so far have used text strings as model prompts. While\nthis remains the most common way to prompt generative AI models, many models can\nalso accept other media as prompts. Media prompts are most often used in\nconjunction with text prompts that instruct the model to perform some operation\non the media, such as to caption an image or transcribe an audio recording.\n\nThe ability to accept media input and the types of media you can use are\ncompletely dependent on the model and its API. For example, the Gemini 1.5\nseries of models can accept images, video, and audio as prompts.\n\nTo provide a media prompt to a model that supports it, instead of passing a\nsimple text prompt to `generate`, pass an array consisting of a media part and a\ntext part:\n\n```python\nfrom genkit.ai import Part # Import Part\n\nresult = await ai.generate(\n prompt=[\n Part(media={'url': 'https://example.com/photo.jpg'}),\n Part(text='Compose a poem about this image.'),\n ],\n)\n```\n\nIn the above example, you specified an image using a publicly-accessible HTTPS\nURL. You can also pass media data directly by encoding it as a data URL. For\nexample:\n\n```python\nimport base64\nfrom genkit.ai import Part # Import Part\n\n# Assume read_file is defined elsewhere to read image bytes\n# def read_file(path):\n# with open(path, 'rb') as f:\n# return f.read()\n\nimage_bytes = read_file('image.jpg')\nbase64_encoded_image = base64.b64encode(image_bytes).decode('utf-8') # Decode bytes to string\n\nresult = await ai.generate(\n prompt=[\n Part(media={'url': f'data:image/jpeg;base64,{base64_encoded_image}'}),\n Part(text='Compose a poem about this image.'),\n ],\n)\n```\n\nAll models that support media input support both data URLs and HTTPS URLs. Some\nmodel plugins add support for other media sources. For example, the Vertex AI\nplugin also lets you use Cloud Storage (`gs://`) URLs.\n\n\n\n### Generating media\n\nSo far, most of the examples on this page have dealt with generating text using\nLLMs. However, Genkit can also be used with image generation models. Using\n`generate()` with an image generation model is similar to using an LLM. For\nexample, to generate an image using the Imagen model:\n\n```python\n# TODO: Add example for image generation\n```\n\n\n", "title": "Generating content with AI models", - "lang": "python" + "description": "Learn how to use Genkit to generate content with various AI models in Python, including text, structured output, streaming, and multimodal input.", + "lang": "python", + "headers": "### Loading and configuring model plugins\n### The generate() method\n## The Blackheart's Bounty\n### System prompts\n### Model parameters\n### Structured output\n### Streaming\n### Multimodal input\n# Assume read_file is defined elsewhere to read image bytes\n# def read_file(path):\n# with open(path, 'rb') as f:\n# return f.read()\n### Generating media\n# TODO: Add example for image generation\n" }, "python/reference/rag.md": { - "text": "Genkit provides abstractions that help you build retrieval-augmented\ngeneration (RAG) flows, as well as plugins that provide integrations with\nrelated tools.\n\n## What is RAG?\n\nRetrieval-augmented generation is a technique used to incorporate external\nsources of information into an LLM’s responses. It's important to be able to do\nso because, while LLMs are typically trained on a broad body of material,\npractical use of LLMs often requires specific domain knowledge (for example, you\nmight want to use an LLM to answer customers' questions about your company’s\nproducts).\n\nOne solution is to fine-tune the model using more specific data. However, this\ncan be expensive both in terms of compute cost and in terms of the effort needed\nto prepare adequate training data.\n\nIn contrast, RAG works by incorporating external data sources into a prompt at\nthe time it's passed to the model. For example, you could imagine the prompt,\n\"What is Bart's relationship to Lisa?\" might be expanded (\"augmented\") by\nprepending some relevant information, resulting in the prompt, \"Homer and\nMarge's children are named Bart, Lisa, and Maggie. What is Bart's relationship\nto Lisa?\"\n\nThis approach has several advantages:\n\n- It can be more cost-effective because you don't have to retrain the model.\n- You can continuously update your data source and the LLM can immediately\n make use of the updated information.\n- You now have the potential to cite references in your LLM's responses.\n\nOn the other hand, using RAG naturally means longer prompts, and some LLM API\nservices charge for each input token you send. Ultimately, you must evaluate the\ncost tradeoffs for your applications.\n\nRAG is a very broad area and there are many different techniques used to achieve\nthe best quality RAG. The core Genkit framework offers three main abstractions\nto help you do RAG:\n\n- **Embedders**: transforms documents into a vector representation\n- **Retrievers**: retrieve documents from an \"index\", given a query.\n\nThese definitions are broad on purpose because Genkit is un-opinionated about\nwhat an \"index\" is or how exactly documents are retrieved from it. Genkit only\nprovides a `Document` format and everything else is defined by the retriever or\nindexer implementation provider.\n\n### Embedders\n\nAn embedder is a function that takes content (text, images, audio, etc.) and\ncreates a numeric vector that encodes the semantic meaning of the original\ncontent. As mentioned above, embedders are leveraged as part of the process of\nindexing, however, they can also be used independently to create embeddings\nwithout an index.\n\n### Retrievers\n\nA retriever is a concept that encapsulates logic related to any kind of document\nretrieval. The most popular retrieval cases typically include retrieval from\nvector stores, however, in Genkit a retriever can be any function that returns\ndata.\n\nTo create a retriever, you can use one of the provided implementations or create\nyour own.\n\n## Defining a RAG Flow\n\nThe following examples show how you could ingest a collection of restaurant menu\nPDF documents into a vector database and retrieve them for use in a flow that\ndetermines what food items are available. Note that indexing is outside the scope\nof Genkit and you should use the SDKs/APIs provided by the vector store you are using.\n\nThe following example shows how you might use a retriever in a RAG flow. Like\nthe retriever example, this example uses Firestore Vector Store.\n\n```python\nfrom genkit.ai import Genkit, Document\nfrom genkit.plugins.google_genai import (\n VertexAI,\n vertexai_name,\n)\nfrom genkit.plugins.firebase.firestore import FirestoreVectorStore, DistanceMeasure\n\nai = Genkit(\n plugins=[\n VertexAI(),\n FirestoreVectorStore(\n name='my_firestore_retriever',\n collection='mycollection',\n vector_field='embedding',\n content_field='text',\n embedder=EMBEDDING_MODEL,\n distance_measure=DistanceMeasure.EUCLIDEAN,\n firestore_client=firestore_client,\n ),\n ],\n)\n\n@ai.flow()\nasync def qa_flow(query: str):\n docs = await ai.retrieve(\n query=Document.from_text(query),\n retriever='firestore/my_firestore_retriever'\n )\n response = await ai.generate(prompt=query, docs=docs)\n return response.text\n```\n\n#### Run the retriever flow\n\n```python\nresult = await qa_flow('Recommend a dessert from the menu while avoiding dairy and nuts')\nprint(result)\n```\n\nThe output for this command should contain a response from the model, grounded\nin the indexed `menu.pdf` file.\n\n## Write your own retrievers\n\nIt's also possible to create your own retriever. This is useful if your\ndocuments are managed in a document store that is not supported in Genkit (eg:\nMySQL, Google Drive, etc.). The Genkit SDK provides flexible methods that let\nyou provide custom code for fetching documents. You can also define custom\nretrievers that build on top of existing retrievers in Genkit and apply advanced\nRAG techniques (such as reranking or prompt extensions) on top.\n\n```python\nfrom genkit.types import (\n RetrieverRequest,\n RetrieverResponse,\n Document,\n ActionRunContext\n)\n\nasync def my_retriever(request: RetrieverRequest, ctx: ActionRunContext):\n \"\"\"Example of a retriever.\n\n Args:\n request: The request to the retriever.\n ctx: The context of the retriever.\n \"\"\"\n return RetrieverResponse(documents=[Document.from_text('Hello'), Document.from_text('World')])\n\n\nai.define_retriever(name='my_retriever', fn=my_retriever)\n```\n\nThen you'll be able to use your retriever with `ai.retrieve`:\n\n```python\ndocs = await ai.retrieve(\n query=Document.from_text(query),\n retriever='my_retriever'\n)\n```\n", + "text": "# Retrieval-Augmented Generation (RAG)\n\nGenkit provides abstractions that help you build retrieval-augmented\ngeneration (RAG) flows, as well as plugins that provide integrations with\nrelated tools.\n\n## What is RAG?\n\nRetrieval-augmented generation is a technique used to incorporate external\nsources of information into an LLM’s responses. It's important to be able to do\nso because, while LLMs are typically trained on a broad body of material,\npractical use of LLMs often requires specific domain knowledge (for example, you\nmight want to use an LLM to answer customers' questions about your company’s\nproducts).\n\nOne solution is to fine-tune the model using more specific data. However, this\ncan be expensive both in terms of compute cost and in terms of the effort needed\nto prepare adequate training data.\n\nIn contrast, RAG works by incorporating external data sources into a prompt at\nthe time it's passed to the model. For example, you could imagine the prompt,\n\"What is Bart's relationship to Lisa?\" might be expanded (\"augmented\") by\nprepending some relevant information, resulting in the prompt, \"Homer and\nMarge's children are named Bart, Lisa, and Maggie. What is Bart's relationship\nto Lisa?\"\n\nThis approach has several advantages:\n\n- It can be more cost-effective because you don't have to retrain the model.\n- You can continuously update your data source and the LLM can immediately\n make use of the updated information.\n- You now have the potential to cite references in your LLM's responses.\n\nOn the other hand, using RAG naturally means longer prompts, and some LLM API\nservices charge for each input token you send. Ultimately, you must evaluate the\ncost tradeoffs for your applications.\n\nRAG is a very broad area and there are many different techniques used to achieve\nthe best quality RAG. The core Genkit framework offers three main abstractions\nto help you do RAG:\n\n- **Embedders**: transforms documents into a vector representation\n- **Retrievers**: retrieve documents from an \"index\", given a query.\n\nThese definitions are broad on purpose because Genkit is un-opinionated about\nwhat an \"index\" is or how exactly documents are retrieved from it. Genkit only\nprovides a `Document` format and everything else is defined by the retriever or\nindexer implementation provider.\n\n### Embedders\n\nAn embedder is a function that takes content (text, images, audio, etc.) and\ncreates a numeric vector that encodes the semantic meaning of the original\ncontent. As mentioned above, embedders are leveraged as part of the process of\nindexing, however, they can also be used independently to create embeddings\nwithout an index.\n\n### Retrievers\n\nA retriever is a concept that encapsulates logic related to any kind of document\nretrieval. The most popular retrieval cases typically include retrieval from\nvector stores, however, in Genkit a retriever can be any function that returns\ndata.\n\nTo create a retriever, you can use one of the provided implementations or create\nyour own.\n\n## Defining a RAG Flow\n\nThe following examples show how you could ingest a collection of restaurant menu\nPDF documents into a vector database and retrieve them for use in a flow that\ndetermines what food items are available. Note that indexing is outside the scope\nof Genkit and you should use the SDKs/APIs provided by the vector store you are using.\n\nThe following example shows how you might use a retriever in a RAG flow. Like\nthe retriever example, this example uses Firestore Vector Store.\n\n```python\nfrom genkit.ai import Genkit, Document\nfrom genkit.plugins.google_genai import (\n VertexAI,\n vertexai_name,\n)\nfrom genkit.plugins.firebase.firestore import FirestoreVectorStore, DistanceMeasure\n\nai = Genkit(\n plugins=[\n VertexAI(),\n FirestoreVectorStore(\n name='my_firestore_retriever',\n collection='mycollection',\n vector_field='embedding',\n content_field='text',\n embedder=EMBEDDING_MODEL,\n distance_measure=DistanceMeasure.EUCLIDEAN,\n firestore_client=firestore_client,\n ),\n ],\n)\n\n@ai.flow()\nasync def qa_flow(query: str):\n docs = await ai.retrieve(\n query=Document.from_text(query),\n retriever='firestore/my_firestore_retriever'\n )\n response = await ai.generate(prompt=query, docs=docs)\n return response.text\n```\n\n#### Run the retriever flow\n\n```python\nresult = await qa_flow('Recommend a dessert from the menu while avoiding dairy and nuts')\nprint(result)\n```\n\nThe output for this command should contain a response from the model, grounded\nin the indexed `menu.pdf` file.\n\n## Write your own retrievers\n\nIt's also possible to create your own retriever. This is useful if your\ndocuments are managed in a document store that is not supported in Genkit (eg:\nMySQL, Google Drive, etc.). The Genkit SDK provides flexible methods that let\nyou provide custom code for fetching documents. You can also define custom\nretrievers that build on top of existing retrievers in Genkit and apply advanced\nRAG techniques (such as reranking or prompt extensions) on top.\n\n```python\nfrom genkit.types import (\n RetrieverRequest,\n RetrieverResponse,\n Document,\n ActionRunContext\n)\n\nasync def my_retriever(request: RetrieverRequest, ctx: ActionRunContext):\n \"\"\"Example of a retriever.\n\n Args:\n request: The request to the retriever.\n ctx: The context of the retriever.\n \"\"\"\n return RetrieverResponse(documents=[Document.from_text('Hello'), Document.from_text('World')])\n\n\nai.define_retriever(name='my_retriever', fn=my_retriever)\n```\n\nThen you'll be able to use your retriever with `ai.retrieve`:\n\n```python\ndocs = await ai.retrieve(\n query=Document.from_text(query),\n retriever='my_retriever'\n)\n```\n", "title": "Retrieval-Augmented Generation (RAG)", - "lang": "python" + "description": "Learn how to build Retrieval-Augmented Generation (RAG) flows in Genkit Python using embedders and retrievers.", + "lang": "python", + "headers": "## What is RAG?\n### Embedders\n### Retrievers\n## Defining a RAG Flow\n#### Run the retriever flow\n## Write your own retrievers\n" }, "python/reference/tools.md": { - "text": "_Tool calling_, also known as _function calling_, is a structured way to give\nLLMs the ability to make requests back to the application that called it. You\ndefine the tools you want to make available to the model, and the model will\nmake tool requests to your app as necessary to fulfill the prompts you give it.\n\nThe use cases of tool calling generally fall into a few themes:\n\n**Giving an LLM access to information it wasn't trained with**\n\n- Frequently changing information, such as a stock price or the current\n weather.\n- Information specific to your app domain, such as product information or user\n profiles.\n\nNote the overlap with retrieval augmented generation (RAG), which is also\na way to let an LLM integrate factual information into its generations. RAG is a\nheavier solution that is most suited when you have a large amount of information\nor the information that's most relevant to a prompt is ambiguous. On the other\nhand, if retrieving the information the LLM needs is a simple function call or\ndatabase lookup, tool calling is more appropriate.\n\n**Introducing a degree of determinism into an LLM workflow**\n\n- Performing calculations that the LLM cannot reliably complete itself.\n- Forcing an LLM to generate verbatim text under certain circumstances, such\n as when responding to a question about an app's terms of service.\n\n**Performing an action when initiated by an LLM**\n\n- Turning on and off lights in an LLM-powered home assistant\n- Reserving table reservations in an LLM-powered restaurant agent\n\n## Before you begin\n\nIf you want to run the code examples on this page, first complete the steps in\nthe [Get started](/python/docs/get-started/) guide. All of the examples assume that you\nhave already set up a project with Genkit dependencies installed.\n\nThis page discusses one of the advanced features of Genkit model abstraction, so\nbefore you dive too deeply, you should be familiar with the content on the\n[Generating content with AI models](/python/docs/reference/models/) page. You should also be familiar\nwith Genkit's system for defining input and output schemas, which is discussed\non the [Flows](/python/docs/reference/flows/) page.\n\n## Overview of tool calling\n\nAt a high level, this is what a typical tool-calling interaction with an LLM\nlooks like:\n\n1. The calling application prompts the LLM with a request and also includes in\n the prompt a list of tools the LLM can use to generate a response.\n2. The LLM either generates a complete response or generates a tool call request\n in a specific format.\n3. If the caller receives a complete response, the request is fulfilled and the\n interaction ends; but if the caller receives a tool call, it performs\n whatever logic is appropriate and sends a new request to the LLM containing\n the original prompt or some variation of it as well as the result of the tool\n call.\n4. The LLM handles the new prompt as in Step 2.\n\nFor this to work, several requirements must be met:\n\n- The model must be trained to make tool requests when it's needed to complete\n a prompt. Most of the larger models provided through web APIs, such as\n Gemini and Claude, can do this, but smaller and more specialized models\n often cannot. Genkit will throw an error if you try to provide tools to a\n model that doesn't support it.\n- The calling application must provide tool definitions to the model in the\n format it expects.\n- The calling application must prompt the model to generate tool calling\n requests in the format the application expects.\n\n## Tool calling with Genkit\n\nGenkit provides a single interface for tool calling with models that support it.\nEach model plugin ensures that the last two of the above criteria are met, and\nthe Genkit instance's `generate()` function automatically carries out the tool\ncalling loop described earlier.\n\n### Model support\n\nTool calling support depends on the model, the model API, and the Genkit plugin.\nConsult the relevant documentation to determine if tool calling is likely to be\nsupported. In addition:\n\n- Genkit will throw an error if you try to provide tools to a model that\n doesn't support it.\n- If the plugin exports model references, the `info.supports.tools` property\n will indicate if it supports tool calling.\n\n### Defining tools\n\nUse the Genkit instance's `tool()` decorator to write tool definitions:\n\n```python\nfrom pydantic import BaseModel, Field\nfrom genkit.ai import Genkit\nfrom genkit.plugins.google_genai import GoogleGenai\n\nai = Genkit(\n plugins=[GoogleGenai()],\n model='googleai/gemini-2.5-flash',\n)\n\nclass WeatherInput(BaseModel):\n location: str = Field(description='The location to get the current weather for')\n\n\n@ai.tool()\ndef get_weather(input: WeatherInput) -> str:\n \"\"\"Gets the current weather in a given location\"\"\"\n # Replace with actual weather fetching logic\n return f'The current weather in {input.location} is 63°F and sunny.'\n```\n\nThe syntax here looks just like the `flow()` syntax; however `description`\nparameter is required. When writing a tool definition, take special care\nwith the wording and descriptiveness of these parameters. They are vital\nfor the LLM to make effective use of the available tools.\n\n### Using tools\n\nInclude defined tools in your prompts to generate content.\n\n```python\nresult = await ai.generate(\n prompt='What is the weather in Baltimore?',\n tools=['get_weather'],\n)\n```\n\nGenkit will automatically handle the tool call if the LLM needs to use the\n`get_weather` tool to answer the prompt.\n\n### Pause the tool loop by using interrupts\n\nBy default, Genkit repeatedly calls the LLM until every tool call has been\nresolved. You can conditionally pause execution in situations where you want\nto, for example:\n\n- Ask the user a question or display UI.\n- Confirm a potentially risky action with the user.\n- Request out-of-band approval for an action.\n\n**Interrupts** are special tools that can halt the loop and return control\nto your code so that you can handle more advanced scenarios. Visit the\n[interrupts guide](/python/docs/reference/interrupts/) to learn how to use them.\n\n### Explicitly handling tool calls\n\nIf you want full control over this tool-calling loop, for example to\napply more complicated logic, set the `return_tool_requests` parameter to `True`.\nNow it's your responsibility to ensure all of the tool requests are fulfilled:\n\n```python\nllm_response = await ai.generate(\n prompt='What is the weather in Baltimore?',\n tools=['get_weather'],\n return_tool_requests=True,\n)\n\ntool_request_parts = llm_response.tool_requests\n\nif len(tool_request_parts) == 0:\n print(llm_response.text)\nelse:\n for part in tool_request_parts:\n await handle_tool(part.name, part.input)\n```\n", + "text": "# Tool (Function) Calling\n\n_Tool calling_, also known as _function calling_, is a structured way to give\nLLMs the ability to make requests back to the application that called it. You\ndefine the tools you want to make available to the model, and the model will\nmake tool requests to your app as necessary to fulfill the prompts you give it.\n\nThe use cases of tool calling generally fall into a few themes:\n\n**Giving an LLM access to information it wasn't trained with**\n\n- Frequently changing information, such as a stock price or the current\n weather.\n- Information specific to your app domain, such as product information or user\n profiles.\n\nNote the overlap with retrieval augmented generation (RAG), which is also\na way to let an LLM integrate factual information into its generations. RAG is a\nheavier solution that is most suited when you have a large amount of information\nor the information that's most relevant to a prompt is ambiguous. On the other\nhand, if retrieving the information the LLM needs is a simple function call or\ndatabase lookup, tool calling is more appropriate.\n\n**Introducing a degree of determinism into an LLM workflow**\n\n- Performing calculations that the LLM cannot reliably complete itself.\n- Forcing an LLM to generate verbatim text under certain circumstances, such\n as when responding to a question about an app's terms of service.\n\n**Performing an action when initiated by an LLM**\n\n- Turning on and off lights in an LLM-powered home assistant\n- Reserving table reservations in an LLM-powered restaurant agent\n\n## Before you begin\n\nIf you want to run the code examples on this page, first complete the steps in\nthe [Get started](/python/docs/get-started/) guide. All of the examples assume that you\nhave already set up a project with Genkit dependencies installed.\n\nThis page discusses one of the advanced features of Genkit model abstraction, so\nbefore you dive too deeply, you should be familiar with the content on the\n[Generating content with AI models](/python/docs/reference/models/) page. You should also be familiar\nwith Genkit's system for defining input and output schemas, which is discussed\non the [Flows](/python/docs/reference/flows/) page.\n\n## Overview of tool calling\n\nAt a high level, this is what a typical tool-calling interaction with an LLM\nlooks like:\n\n1. The calling application prompts the LLM with a request and also includes in\n the prompt a list of tools the LLM can use to generate a response.\n2. The LLM either generates a complete response or generates a tool call request\n in a specific format.\n3. If the caller receives a complete response, the request is fulfilled and the\n interaction ends; but if the caller receives a tool call, it performs\n whatever logic is appropriate and sends a new request to the LLM containing\n the original prompt or some variation of it as well as the result of the tool\n call.\n4. The LLM handles the new prompt as in Step 2.\n\nFor this to work, several requirements must be met:\n\n- The model must be trained to make tool requests when it's needed to complete\n a prompt. Most of the larger models provided through web APIs, such as\n Gemini and Claude, can do this, but smaller and more specialized models\n often cannot. Genkit will throw an error if you try to provide tools to a\n model that doesn't support it.\n- The calling application must provide tool definitions to the model in the\n format it expects.\n- The calling application must prompt the model to generate tool calling\n requests in the format the application expects.\n\n## Tool calling with Genkit\n\nGenkit provides a single interface for tool calling with models that support it.\nEach model plugin ensures that the last two of the above criteria are met, and\nthe Genkit instance's `generate()` function automatically carries out the tool\ncalling loop described earlier.\n\n### Model support\n\nTool calling support depends on the model, the model API, and the Genkit plugin.\nConsult the relevant documentation to determine if tool calling is likely to be\nsupported. In addition:\n\n- Genkit will throw an error if you try to provide tools to a model that\n doesn't support it.\n- If the plugin exports model references, the `info.supports.tools` property\n will indicate if it supports tool calling.\n\n### Defining tools\n\nUse the Genkit instance's `tool()` decorator to write tool definitions:\n\n```python\nfrom pydantic import BaseModel, Field\nfrom genkit.ai import Genkit\nfrom genkit.plugins.google_genai import GoogleGenai\n\nai = Genkit(\n plugins=[GoogleGenai()],\n model='googleai/gemini-2.5-flash',\n)\n\nclass WeatherInput(BaseModel):\n location: str = Field(description='The location to get the current weather for')\n\n\n@ai.tool()\ndef get_weather(input: WeatherInput) -> str:\n \"\"\"Gets the current weather in a given location\"\"\"\n # Replace with actual weather fetching logic\n return f'The current weather in {input.location} is 63°F and sunny.'\n```\n\nThe syntax here looks just like the `flow()` syntax; however `description`\nparameter is required. When writing a tool definition, take special care\nwith the wording and descriptiveness of these parameters. They are vital\nfor the LLM to make effective use of the available tools.\n\n### Using tools\n\nInclude defined tools in your prompts to generate content.\n\n```python\nresult = await ai.generate(\n prompt='What is the weather in Baltimore?',\n tools=['get_weather'],\n)\n```\n\nGenkit will automatically handle the tool call if the LLM needs to use the\n`get_weather` tool to answer the prompt.\n\n### Pause the tool loop by using interrupts\n\nBy default, Genkit repeatedly calls the LLM until every tool call has been\nresolved. You can conditionally pause execution in situations where you want\nto, for example:\n\n- Ask the user a question or display UI.\n- Confirm a potentially risky action with the user.\n- Request out-of-band approval for an action.\n\n**Interrupts** are special tools that can halt the loop and return control\nto your code so that you can handle more advanced scenarios. Visit the\n[interrupts guide](/python/docs/reference/interrupts/) to learn how to use them.\n\n### Explicitly handling tool calls\n\nIf you want full control over this tool-calling loop, for example to\napply more complicated logic, set the `return_tool_requests` parameter to `True`.\nNow it's your responsibility to ensure all of the tool requests are fulfilled:\n\n```python\nllm_response = await ai.generate(\n prompt='What is the weather in Baltimore?',\n tools=['get_weather'],\n return_tool_requests=True,\n)\n\ntool_request_parts = llm_response.tool_requests\n\nif len(tool_request_parts) == 0:\n print(llm_response.text)\nelse:\n for part in tool_request_parts:\n await handle_tool(part.name, part.input)\n```\n", "title": "Tool (Function) Calling", - "lang": "python" + "description": "Learn how to use tool calling (function calling) with Genkit Python to give LLMs access to external information and actions.", + "lang": "python", + "headers": "## Before you begin\n## Overview of tool calling\n## Tool calling with Genkit\n### Model support\n### Defining tools\n### Using tools\n### Pause the tool loop by using interrupts\n### Explicitly handling tool calls\n" }, "python/reference/plugins/dev-local-vectorstore.md": { - "text": "# Dev Local Vector Store\n\nThe Dev Local Vector Store plugin provides a local, file-based vector store for development and testing purposes. It is not intended for production use.\n\n## Installation\n\n```bash\npip3 install genkit-plugin-dev-local-vectorstore\n```\n\n## Configuration\n\nTo use this plugin, specify it when you initialize Genkit:\n\n```python\nfrom genkit.ai import Genkit\nfrom genkit.plugins.dev_local_vectorstore import DevLocalVectorStore\nfrom genkit.plugins.google_genai import VertexAI # Assuming VertexAI is used for embedder\n\nai = Genkit(\n plugins=[\n VertexAI(), # Ensure the embedder's plugin is loaded\n DevLocalVectorStore(\n name='my_vectorstore',\n embedder='vertexai/text-embedding-004', # Example embedder\n ),\n ],\n # Define a default model if needed\n # model='vertexai/gemini-1.5-flash',\n)\n```\n\n### Configuration Options\n\n- **name** (str): A unique name for this vector store instance. This is used as the `retriever` argument to `ai.retrieve`.\n- **embedder** (str): The name of the embedding model to use. Must match a configured embedder in your Genkit project.\n- **embedder_options** (dict, optional): Options to pass to the embedder.\n\n## Usage\n\n### Indexing Documents\n\nThe Dev Local Vector Store automatically creates indexes. To populate with data you must call the static method `.index(name, documents)`:\n\n```python\nfrom genkit.ai import Genkit\nfrom genkit.plugins.dev_local_vectorstore import DevLocalVectorStore\nfrom genkit.plugins.google_genai import VertexAI # Assuming VertexAI is used for embedder\nfrom genkit.types import Document\n\n# Assuming 'ai' is configured as shown in the Configuration section\n# ai = Genkit(...)\n\ndata_list = [\n 'This is the first document.',\n 'This is the second document.',\n 'This is the third document.',\n \"This is the fourth document.\",\n]\n\ngenkit_docs = [Document.from_text(text=item) for item in data_list]\n# Ensure the vector store name matches the one in the Genkit config\nawait DevLocalVectorStore.index('my_vectorstore', genkit_docs)\n```\n\n### Retrieving Documents\n\nUse `ai.retrieve` and pass the store name configured in the DevLocalVectorStore constructor.\n\n```python\nfrom genkit.types import Document\n# Assuming 'ai' is configured as shown in the Configuration section\n# ai = Genkit(...)\n\ndocs = await ai.retrieve(\n query=Document.from_text('search query'),\n retriever='my_vectorstore', # Matches the 'name' in DevLocalVectorStore config\n)\n# print(docs) # Process the retrieved documents\n```\n", + "text": "# Dev Local Vector Store Plugin\n\n# Dev Local Vector Store\n\nThe Dev Local Vector Store plugin provides a local, file-based vector store for development and testing purposes. It is not intended for production use.\n\n## Installation\n\n```bash\npip3 install genkit-plugin-dev-local-vectorstore\n```\n\n## Configuration\n\nTo use this plugin, specify it when you initialize Genkit:\n\n```python\nfrom genkit.ai import Genkit\nfrom genkit.plugins.dev_local_vectorstore import DevLocalVectorStore\nfrom genkit.plugins.google_genai import VertexAI # Assuming VertexAI is used for embedder\n\nai = Genkit(\n plugins=[\n VertexAI(), # Ensure the embedder's plugin is loaded\n DevLocalVectorStore(\n name='my_vectorstore',\n embedder='vertexai/text-embedding-004', # Example embedder\n ),\n ],\n # Define a default model if needed\n # model='vertexai/gemini-1.5-flash',\n)\n```\n\n### Configuration Options\n\n- **name** (str): A unique name for this vector store instance. This is used as the `retriever` argument to `ai.retrieve`.\n- **embedder** (str): The name of the embedding model to use. Must match a configured embedder in your Genkit project.\n- **embedder_options** (dict, optional): Options to pass to the embedder.\n\n## Usage\n\n### Indexing Documents\n\nThe Dev Local Vector Store automatically creates indexes. To populate with data you must call the static method `.index(name, documents)`:\n\n```python\nfrom genkit.ai import Genkit\nfrom genkit.plugins.dev_local_vectorstore import DevLocalVectorStore\nfrom genkit.plugins.google_genai import VertexAI # Assuming VertexAI is used for embedder\nfrom genkit.types import Document\n\n# Assuming 'ai' is configured as shown in the Configuration section\n# ai = Genkit(...)\n\ndata_list = [\n 'This is the first document.',\n 'This is the second document.',\n 'This is the third document.',\n \"This is the fourth document.\",\n]\n\ngenkit_docs = [Document.from_text(text=item) for item in data_list]\n# Ensure the vector store name matches the one in the Genkit config\nawait DevLocalVectorStore.index('my_vectorstore', genkit_docs)\n```\n\n### Retrieving Documents\n\nUse `ai.retrieve` and pass the store name configured in the DevLocalVectorStore constructor.\n\n```python\nfrom genkit.types import Document\n# Assuming 'ai' is configured as shown in the Configuration section\n# ai = Genkit(...)\n\ndocs = await ai.retrieve(\n query=Document.from_text('search query'),\n retriever='my_vectorstore', # Matches the 'name' in DevLocalVectorStore config\n)\n# print(docs) # Process the retrieved documents\n```\n", "title": "Dev Local Vector Store Plugin", - "lang": "python" + "description": "Learn how to use the Dev Local Vector Store plugin for local development and testing in Genkit Python.", + "lang": "python", + "headers": "# Dev Local Vector Store\n## Installation\n## Configuration\n### Configuration Options\n## Usage\n### Indexing Documents\n# Assuming 'ai' is configured as shown in the Configuration section\n# ai = Genkit(...)\n# Ensure the vector store name matches the one in the Genkit config\n### Retrieving Documents\n# Assuming 'ai' is configured as shown in the Configuration section\n# ai = Genkit(...)\n# print(docs) # Process the retrieved documents\n" }, "python/reference/plugins/firestore.md": { - "text": "# Firestore Vector Store\n\nThe Firestore plugin provides retriever implementations that use Google Cloud\nFirestore as a vector store.\n\n## Installation\n\n```bash\npip3 install genkit-plugin-firebase\n```\n\n## Prerequisites\n\n- A Firebase project with Cloud Firestore enabled.\n- The `genkit` package installed.\n- `gcloud` CLI for managing credentials and Firestore indexes.\n\n## Configuration\n\nTo use this plugin, specify it when you initialize Genkit:\n\n```python\nfrom genkit.ai import Genkit\nfrom genkit.plugins.firebase.firestore import FirestoreVectorStore\nfrom genkit.plugins.google_genai import VertexAI # Assuming VertexAI provides the embedder\nfrom google.cloud import firestore\n\n# Ensure you have authenticated with gcloud and set the project\nfirestore_client = firestore.Client()\n\nai = Genkit(\n plugins=[\n VertexAI(), # Ensure the embedder's plugin is loaded\n FirestoreVectorStore(\n name='my_firestore_retriever',\n collection='my_collection', # Replace with your collection name\n vector_field='embedding',\n content_field='text',\n embedder='vertexai/text-embedding-004', # Example embedder\n firestore_client=firestore_client,\n ),\n ]\n # Define a default model if needed\n # model='vertexai/gemini-1.5-flash',\n)\n```\n\n### Configuration Options\n\n- **name** (str): A unique name for this retriever instance.\n- **collection** (str): The name of the Firestore collection to query.\n- **vector_field** (str): The name of the field in the Firestore documents that contains the vector embedding.\n- **content_field** (str): The name of the field in the Firestore documents that contains the text content.\n- **embedder** (str): The name of the embedding model to use. Must match a configured embedder in your Genkit project.\n- **firestore_client**: A `google.cloud.firestore.Client` object that will be used for all queries to the vectorstore.\n\n## Usage\n\n1. **Create a Firestore Client**:\n\n ```python\n from google.cloud import firestore\n # Ensure you have authenticated with gcloud and set the project\n firestore_client = firestore.Client()\n ```\n\n2. **Define a Firestore Retriever**:\n\n ```python\n from genkit.ai import Genkit\n from genkit.plugins.firebase.firestore import FirestoreVectorStore\n from genkit.plugins.google_genai import VertexAI # Assuming VertexAI provides the embedder\n from google.cloud import firestore\n\n # Assuming firestore_client is already created\n # firestore_client = firestore.Client()\n\n ai = Genkit(\n plugins=[\n VertexAI(), # Ensure the embedder's plugin is loaded\n FirestoreVectorStore(\n name='my_firestore_retriever',\n collection='my_collection', # Replace with your collection name\n vector_field='embedding',\n content_field='text',\n embedder='vertexai/text-embedding-004', # Example embedder\n firestore_client=firestore_client,\n ),\n ]\n # Define a default model if needed\n # model='vertexai/gemini-1.5-flash',\n )\n ```\n\n3. **Retrieve Documents**:\n\n ```python\n from genkit.ai import Document # Import Document\n # Assuming 'ai' is configured as above\n\n async def retrieve_documents():\n # Note: ai.retrieve expects a Document object for the query\n query_doc = Document.from_text(\"What are the main topics?\")\n return await ai.retrieve(\n query=query_doc,\n retriever='my_firestore_retriever', # Matches the 'name' in FirestoreVectorStore config\n )\n\n # Example of calling the async function\n # import asyncio\n # retrieved_docs = asyncio.run(retrieve_documents())\n # print(retrieved_docs)\n ```\n\n## Populating the Index\n\nBefore you can retrieve documents, you need to populate your Firestore collection with data and their corresponding vector embeddings. Here's how you can do it:\n\n1. **Prepare your Data**: Organize your data into documents. Each document should have at least two fields: a `text` field containing the content you want to retrieve, and an `embedding` field that holds the vector embedding of the content. You can add any other metadata as well.\n\n2. **Generate Embeddings**: Use the same embedding model configured in your `FirestoreVectorStore` to generate vector embeddings for your text content. The `ai.embed()` method can be used.\n\n3. **Upload Documents to Firestore**: Use the Firestore client to upload the documents with their embeddings to the specified collection.\n\nHere's an example of how to index data:\n\n```python\nfrom genkit.ai import Document, Genkit # Import Genkit and Document\nfrom genkit.types import TextPart\nfrom google.cloud import firestore # Import firestore\n\n# Assuming 'ai' is configured with VertexAI and FirestoreVectorStore plugins\n# Assuming 'firestore_client' is an initialized firestore.Client() instance\n\nasync def index_documents(documents: list[str], collection_name: str):\n \"\"\"Indexes the documents in Firestore.\"\"\"\n genkit_documents = [Document(content=[TextPart(text=doc)]) for doc in documents]\n # Ensure the embedder name matches the one configured in Genkit\n embed_response = await ai.embed(embedder='vertexai/text-embedding-004', content=genkit_documents) # Use 'content' parameter\n embeddings = [emb.embedding for emb in embed_response.embeddings]\n\n for i, document_text in enumerate(documents):\n doc_id = f'doc-{i + 1}'\n embedding = embeddings[i]\n\n doc_ref = firestore_client.collection(collection_name).document(doc_id)\n result = doc_ref.set({\n 'text': document_text,\n 'embedding': embedding, # Ensure this field name matches 'vector_field' in config\n 'metadata': f'metadata for doc {i + 1}',\n })\n print(f\"Indexed document {doc_id}\") # Optional: print progress\n\n# Example Usage\n# documents = [\n# \"This is document one.\",\n# \"This is document two.\",\n# \"This is document three.\",\n# ]\n# import asyncio\n# asyncio.run(index_documents(documents, 'my_collection')) # Replace 'my_collection' with your actual collection name\n```\n\n## Creating a Firestore Index\n\nTo enable vector similarity search you will need to configure the index in your Firestore database. Use the following command:\n\n```bash\ngcloud firestore indexes composite create \\\n --project= \\\n --collection-group= \\\n --query-scope=COLLECTION \\\n --field-config=vector-config='{\"dimension\":,\"flat\": {}}',field-path=\n```\n\n- Replace `` with the ID of your Firebase project.\n- Replace `` with the name of your Firestore collection (e.g., `my_collection`).\n- Replace `` with the correct dimension for your embedding model. Common values are:\n - `768` for `text-embedding-004` (Vertex AI)\n- Replace `` with the name of the field containing vector embeddings (e.g., `embedding`).\n", + "text": "# Firestore Vector Store Plugin\n\n# Firestore Vector Store\n\nThe Firestore plugin provides retriever implementations that use Google Cloud\nFirestore as a vector store.\n\n## Installation\n\n```bash\npip3 install genkit-plugin-firebase\n```\n\n## Prerequisites\n\n- A Firebase project with Cloud Firestore enabled.\n- The `genkit` package installed.\n- `gcloud` CLI for managing credentials and Firestore indexes.\n\n## Configuration\n\nTo use this plugin, specify it when you initialize Genkit:\n\n```python\nfrom genkit.ai import Genkit\nfrom genkit.plugins.firebase.firestore import FirestoreVectorStore\nfrom genkit.plugins.google_genai import VertexAI # Assuming VertexAI provides the embedder\nfrom google.cloud import firestore\n\n# Ensure you have authenticated with gcloud and set the project\nfirestore_client = firestore.Client()\n\nai = Genkit(\n plugins=[\n VertexAI(), # Ensure the embedder's plugin is loaded\n FirestoreVectorStore(\n name='my_firestore_retriever',\n collection='my_collection', # Replace with your collection name\n vector_field='embedding',\n content_field='text',\n embedder='vertexai/text-embedding-004', # Example embedder\n firestore_client=firestore_client,\n ),\n ]\n # Define a default model if needed\n # model='vertexai/gemini-1.5-flash',\n)\n```\n\n### Configuration Options\n\n- **name** (str): A unique name for this retriever instance.\n- **collection** (str): The name of the Firestore collection to query.\n- **vector_field** (str): The name of the field in the Firestore documents that contains the vector embedding.\n- **content_field** (str): The name of the field in the Firestore documents that contains the text content.\n- **embedder** (str): The name of the embedding model to use. Must match a configured embedder in your Genkit project.\n- **firestore_client**: A `google.cloud.firestore.Client` object that will be used for all queries to the vectorstore.\n\n## Usage\n\n1. **Create a Firestore Client**:\n\n ```python\n from google.cloud import firestore\n # Ensure you have authenticated with gcloud and set the project\n firestore_client = firestore.Client()\n ```\n\n2. **Define a Firestore Retriever**:\n\n ```python\n from genkit.ai import Genkit\n from genkit.plugins.firebase.firestore import FirestoreVectorStore\n from genkit.plugins.google_genai import VertexAI # Assuming VertexAI provides the embedder\n from google.cloud import firestore\n\n # Assuming firestore_client is already created\n # firestore_client = firestore.Client()\n\n ai = Genkit(\n plugins=[\n VertexAI(), # Ensure the embedder's plugin is loaded\n FirestoreVectorStore(\n name='my_firestore_retriever',\n collection='my_collection', # Replace with your collection name\n vector_field='embedding',\n content_field='text',\n embedder='vertexai/text-embedding-004', # Example embedder\n firestore_client=firestore_client,\n ),\n ]\n # Define a default model if needed\n # model='vertexai/gemini-1.5-flash',\n )\n ```\n\n3. **Retrieve Documents**:\n\n ```python\n from genkit.ai import Document # Import Document\n # Assuming 'ai' is configured as above\n\n async def retrieve_documents():\n # Note: ai.retrieve expects a Document object for the query\n query_doc = Document.from_text(\"What are the main topics?\")\n return await ai.retrieve(\n query=query_doc,\n retriever='my_firestore_retriever', # Matches the 'name' in FirestoreVectorStore config\n )\n\n # Example of calling the async function\n # import asyncio\n # retrieved_docs = asyncio.run(retrieve_documents())\n # print(retrieved_docs)\n ```\n\n## Populating the Index\n\nBefore you can retrieve documents, you need to populate your Firestore collection with data and their corresponding vector embeddings. Here's how you can do it:\n\n1. **Prepare your Data**: Organize your data into documents. Each document should have at least two fields: a `text` field containing the content you want to retrieve, and an `embedding` field that holds the vector embedding of the content. You can add any other metadata as well.\n\n2. **Generate Embeddings**: Use the same embedding model configured in your `FirestoreVectorStore` to generate vector embeddings for your text content. The `ai.embed()` method can be used.\n\n3. **Upload Documents to Firestore**: Use the Firestore client to upload the documents with their embeddings to the specified collection.\n\nHere's an example of how to index data:\n\n```python\nfrom genkit.ai import Document, Genkit # Import Genkit and Document\nfrom genkit.types import TextPart\nfrom google.cloud import firestore # Import firestore\n\n# Assuming 'ai' is configured with VertexAI and FirestoreVectorStore plugins\n# Assuming 'firestore_client' is an initialized firestore.Client() instance\n\nasync def index_documents(documents: list[str], collection_name: str):\n \"\"\"Indexes the documents in Firestore.\"\"\"\n genkit_documents = [Document(content=[TextPart(text=doc)]) for doc in documents]\n # Ensure the embedder name matches the one configured in Genkit\n embed_response = await ai.embed(embedder='vertexai/text-embedding-004', content=genkit_documents) # Use 'content' parameter\n embeddings = [emb.embedding for emb in embed_response.embeddings]\n\n for i, document_text in enumerate(documents):\n doc_id = f'doc-{i + 1}'\n embedding = embeddings[i]\n\n doc_ref = firestore_client.collection(collection_name).document(doc_id)\n result = doc_ref.set({\n 'text': document_text,\n 'embedding': embedding, # Ensure this field name matches 'vector_field' in config\n 'metadata': f'metadata for doc {i + 1}',\n })\n print(f\"Indexed document {doc_id}\") # Optional: print progress\n\n# Example Usage\n# documents = [\n# \"This is document one.\",\n# \"This is document two.\",\n# \"This is document three.\",\n# ]\n# import asyncio\n# asyncio.run(index_documents(documents, 'my_collection')) # Replace 'my_collection' with your actual collection name\n```\n\n## Creating a Firestore Index\n\nTo enable vector similarity search you will need to configure the index in your Firestore database. Use the following command:\n\n```bash\ngcloud firestore indexes composite create \\\n --project= \\\n --collection-group= \\\n --query-scope=COLLECTION \\\n --field-config=vector-config='{\"dimension\":,\"flat\": {}}',field-path=\n```\n\n- Replace `` with the ID of your Firebase project.\n- Replace `` with the name of your Firestore collection (e.g., `my_collection`).\n- Replace `` with the correct dimension for your embedding model. Common values are:\n - `768` for `text-embedding-004` (Vertex AI)\n- Replace `` with the name of the field containing vector embeddings (e.g., `embedding`).\n", "title": "Firestore Vector Store Plugin", - "lang": "python" + "description": "Learn how to use the Firestore Vector Store plugin with Genkit Python to leverage Google Cloud Firestore for RAG.", + "lang": "python", + "headers": "# Firestore Vector Store\n## Installation\n## Prerequisites\n## Configuration\n# Ensure you have authenticated with gcloud and set the project\n### Configuration Options\n## Usage\n## Populating the Index\n# Assuming 'ai' is configured with VertexAI and FirestoreVectorStore plugins\n# Assuming 'firestore_client' is an initialized firestore.Client() instance\n# Example Usage\n# documents = [\n# \"This is document one.\",\n# \"This is document two.\",\n# \"This is document three.\",\n# ]\n# import asyncio\n# asyncio.run(index_documents(documents, 'my_collection')) # Replace 'my_collection' with your actual collection name\n## Creating a Firestore Index\n" }, "python/reference/plugins/google-genai.md": { - "text": "# Google Gen AI\n\nThe `genkit-plugin-google-genai` package provides two plugins for accessing Google's generative AI models:\n\n1. `GoogleAI`: For accessing models via the Google Gemini API (requires an API key).\n2. `VertexAI`: For accessing models via the Gemini API within Google Cloud Vertex AI (uses standard Google Cloud authentication).\n\n## Installation\n\n```bash\npip3 install genkit-plugin-google-genai\n```\n\n## Configuration\n\n### Google Gemini API (`GoogleAI`)\n\nTo use the Google Gemini API, you need an API key.\n\n```python\nfrom genkit.ai import Genkit\nfrom genkit.plugins.google_genai import GoogleAI\n\nai = Genkit(\n plugins=[GoogleAI()],\n model='googleai/gemini-2.5-flash', \n)\n```\n\nYou will need to set GEMINI_API_KEY environment variable or you can provide the API Key directly:\n\n```python\nai = Genkit(\n plugins=[GoogleAI(api_key='...')]\n)\n```\n\n### Gemini API in Vertex AI (`VertexAI`)\n\nTo use models via Vertex AI, ensure you have authenticated with Google Cloud (e.g., via `gcloud auth application-default login`).\n\n```python\nfrom genkit.ai import Genkit\nfrom genkit.plugins.google_genai import VertexAI\n\nai = Genkit(\n plugins=[VertexAI()],\n model='vertexai/gemini-2.5-flash', # optional\n)\n```\n\nYou can specify the `location` and `project` ID, among other configuration options available in the `VertexAI` constructor.\n\n```python\nai = Genkit(\n plugins=[VertexAI(\n location='us-east1',\n project='my-project-id',\n )],\n)\n```", + "text": "# Google GenAI Plugin\n\n# Google Gen AI\n\nThe `genkit-plugin-google-genai` package provides two plugins for accessing Google's generative AI models:\n\n1. `GoogleAI`: For accessing models via the Google Gemini API (requires an API key).\n2. `VertexAI`: For accessing models via the Gemini API within Google Cloud Vertex AI (uses standard Google Cloud authentication).\n\n## Installation\n\n```bash\npip3 install genkit-plugin-google-genai\n```\n\n## Configuration\n\n### Google Gemini API (`GoogleAI`)\n\nTo use the Google Gemini API, you need an API key.\n\n```python\nfrom genkit.ai import Genkit\nfrom genkit.plugins.google_genai import GoogleAI\n\nai = Genkit(\n plugins=[GoogleAI()],\n model='googleai/gemini-2.5-flash', \n)\n```\n\nYou will need to set GEMINI_API_KEY environment variable or you can provide the API Key directly:\n\n```python\nai = Genkit(\n plugins=[GoogleAI(api_key='...')]\n)\n```\n\n### Gemini API in Vertex AI (`VertexAI`)\n\nTo use models via Vertex AI, ensure you have authenticated with Google Cloud (e.g., via `gcloud auth application-default login`).\n\n```python\nfrom genkit.ai import Genkit\nfrom genkit.plugins.google_genai import VertexAI\n\nai = Genkit(\n plugins=[VertexAI()],\n model='vertexai/gemini-2.5-flash', # optional\n)\n```\n\nYou can specify the `location` and `project` ID, among other configuration options available in the `VertexAI` constructor.\n\n```python\nai = Genkit(\n plugins=[VertexAI(\n location='us-east1',\n project='my-project-id',\n )],\n)\n```", "title": "Google GenAI Plugin", - "lang": "python" + "description": "Learn how to configure and use the Google GenAI plugin for Genkit Python, providing access to Google Gemini API and Vertex AI models.", + "lang": "python", + "headers": "# Google Gen AI\n## Installation\n## Configuration\n### Google Gemini API (`GoogleAI`)\n### Gemini API in Vertex AI (`VertexAI`)\n" }, "python/reference/plugins/ollama.md": { - "text": "# Ollama\n\nThe `genkit-plugin-ollama` package provides integration with [Ollama](https://ollama.com/), allowing you to run various open-source large language models and embedding models locally.\n\n## Installation\n\n```bash\npip3 install genkit-plugin-ollama\n```\n\nYou will need to download and install Ollama separately: [https://ollama.com/download](https://ollama.com/download)\n\nUse the Ollama CLI to pull the models you would like to use. For example:\n\n```bash\nollama pull gemma2 # Example model\nollama pull nomic-embed-text # Example embedder\n```\n\n## Configuration\n\nConfigure the Ollama plugin in your Genkit initialization, specifying the models and embedders you have pulled and wish to use.\n\n```python\nfrom genkit.ai import Genkit\nfrom genkit.plugins.ollama import Ollama, ModelDefinition, EmbeddingModelDefinition\n\nai = Genkit(\n plugins=[\n Ollama(\n models=[\n ModelDefinition(name='gemma2'), # Match the model pulled via ollama CLI\n # Add other models as needed\n # ModelDefinition(name='mistral'),\n ],\n embedders=[\n EmbeddingModelDefinition(\n name='nomic-embed-text', # Match the embedder pulled via ollama CLI\n # Specify dimensions if known/required by your use case\n # dimensions=768, # Example dimension\n )\n ],\n # Optional: Specify Ollama server address if not default (http://127.0.0.1:11434)\n # address=\"http://custom-ollama-host:11434\"\n )\n ],\n)\n```\n\nThen use Ollama models and embedders by specifying the `ollama/` prefix followed by the model/embedder name defined in the configuration:\n\n```python\nfrom genkit.ai import Document # Import Document\n# Assuming 'ai' is configured as above\n\nasync def run_ollama():\n generate_response = await ai.generate(\n prompt='Tell me a short story about a space cat.',\n model='ollama/gemma2', # Use the configured model name\n )\n print(\"Generated Text:\", generate_response.text)\n\n embedding_response = await ai.embed(\n embedder='ollama/nomic-embed-text', # Use the configured embedder name\n content=[Document.from_text('This is text to embed.')], # Pass content as a list of Documents\n )\n print(\"Embedding:\", embedding_response.embeddings[0].embedding) # Access the embedding vector\n\n# Example of running the async function\n# import asyncio\n# asyncio.run(run_ollama())\n```\n", + "text": "# Ollama Plugin\n\n# Ollama\n\nThe `genkit-plugin-ollama` package provides integration with [Ollama](https://ollama.com/), allowing you to run various open-source large language models and embedding models locally.\n\n## Installation\n\n```bash\npip3 install genkit-plugin-ollama\n```\n\nYou will need to download and install Ollama separately: [https://ollama.com/download](https://ollama.com/download)\n\nUse the Ollama CLI to pull the models you would like to use. For example:\n\n```bash\nollama pull gemma2 # Example model\nollama pull nomic-embed-text # Example embedder\n```\n\n## Configuration\n\nConfigure the Ollama plugin in your Genkit initialization, specifying the models and embedders you have pulled and wish to use.\n\n```python\nfrom genkit.ai import Genkit\nfrom genkit.plugins.ollama import Ollama, ModelDefinition, EmbeddingModelDefinition\n\nai = Genkit(\n plugins=[\n Ollama(\n models=[\n ModelDefinition(name='gemma2'), # Match the model pulled via ollama CLI\n # Add other models as needed\n # ModelDefinition(name='mistral'),\n ],\n embedders=[\n EmbeddingModelDefinition(\n name='nomic-embed-text', # Match the embedder pulled via ollama CLI\n # Specify dimensions if known/required by your use case\n # dimensions=768, # Example dimension\n )\n ],\n # Optional: Specify Ollama server address if not default (http://127.0.0.1:11434)\n # address=\"http://custom-ollama-host:11434\"\n )\n ],\n)\n```\n\nThen use Ollama models and embedders by specifying the `ollama/` prefix followed by the model/embedder name defined in the configuration:\n\n```python\nfrom genkit.ai import Document # Import Document\n# Assuming 'ai' is configured as above\n\nasync def run_ollama():\n generate_response = await ai.generate(\n prompt='Tell me a short story about a space cat.',\n model='ollama/gemma2', # Use the configured model name\n )\n print(\"Generated Text:\", generate_response.text)\n\n embedding_response = await ai.embed(\n embedder='ollama/nomic-embed-text', # Use the configured embedder name\n content=[Document.from_text('This is text to embed.')], # Pass content as a list of Documents\n )\n print(\"Embedding:\", embedding_response.embeddings[0].embedding) # Access the embedding vector\n\n# Example of running the async function\n# import asyncio\n# asyncio.run(run_ollama())\n```\n", "title": "Ollama Plugin", - "lang": "python" + "description": "Learn how to configure and use the Ollama plugin for Genkit Python to run local LLMs and embedding models.", + "lang": "python", + "headers": "# Ollama\n## Installation\n## Configuration\n# Assuming 'ai' is configured as above\n# Example of running the async function\n# import asyncio\n# asyncio.run(run_ollama())\n" + }, + "releases.md": { + "title": "Release Notes", + "text": "---\ntitle: Release Notes\n---\n\n# Release Notes\n\n## [Genkit JS 1.15.5](https://github.com/firebase/genkit/releases/tag/genkit%401.15.5) - 7/25/2025\n\n## What's Changed\r\n* fix(js/plugins/pinecone): Handle missing docMetadata by @ssbushi in https://github.com/firebase/genkit/pull/3272\r\n* fix(js/plugins/googleai): fixed aggregation of thought parts by @pavelgj in https://github.com/firebase/genkit/pull/3289\r\n\r\n**Full Changelog**: https://github.com/firebase/genkit/compare/genkitx-pinecone@1.15.2...genkit@1.15.5\n\n## [Genkit JS 1.15.2](https://github.com/firebase/genkit/releases/tag/genkit%401.15.2) - 7/21/2025\n\n## What's Changed\r\n* fix(js/plugins/compat-oai): Fix for ESM by @ssbushi in https://github.com/firebase/genkit/pull/3255\r\n\r\n**Full Changelog**: https://github.com/firebase/genkit/compare/genkit@1.15.1...genkit@1.15.2\n\n## [Genkit JS 1.15.1](https://github.com/firebase/genkit/releases/tag/genkit%401.15.1) - 7/18/2025\n\n## New plugins\r\n - [@genkit-ai/compat-oai](https://www.npmjs.com/package/@genkit-ai/compat-oai) - plugin for [OpenAI API compatible](https://genkit.dev/docs/plugins/compat-oai/) providers: [OpenAI](https://genkit.dev/docs/plugins/openai/), [DeepSeek](https://genkit.dev/docs/plugins/deepseek/), [xAI](https://genkit.dev/docs/plugins/xai/)\r\n - [genkitx-cloud-sql-pg](https://www.npmjs.com/package/genkitx-cloud-sql-pg) - Cloud SQL for PostgreSQL plugin. See [docs](https://genkit.dev/docs/plugins/cloud-sql-pg/).\r\n - [@genkit-ai/mcp](https://www.npmjs.com/package/@genkit-ai/mcp) - plugin for building MCP clients and servers. See [docs](https://genkit.dev/docs/plugins/mcp/).\r\n\r\n## What's Changed\r\n* feat(js): added id and metadata to executable prompts by @pavelgj in https://github.com/firebase/genkit/pull/3084\r\n* feat(cli): compile CI to a binary with bun by @cabljac in https://github.com/firebase/genkit/pull/3101\r\n* fix(js/plugins/\\{googleai,vertexai\\}): fixed handling of empty object schema (used by MCP) by @pavelgj in https://github.com/firebase/genkit/pull/3215\r\n* fix(js/core): add port to runtime id for uniqueness by @MichaelDoyle in https://github.com/firebase/genkit/pull/3182\r\n* fix(js/ai): remove obsolete streaming callback ALS and correctly passthrought streaming callback from action by @pavelgj in https://github.com/firebase/genkit/pull/3243\r\n* refactor(js): more obsolete streaming callback ALS cleanup by @pavelgj in https://github.com/firebase/genkit/pull/3244\r\n\r\n\r\n**Full Changelog**: https://github.com/firebase/genkit/compare/genkit@1.14.1...genkit@1.15.1\n\n## [Genkit JS 1.14.1](https://github.com/firebase/genkit/releases/tag/genkit%401.14.1) - 7/10/2025\n\n## What's Changed\r\n* fix(js): Remove numberOfVideos config parameter for Veo by @shrutip90 in https://github.com/firebase/genkit/pull/3156\r\n* fix(js/core): add port number to runtime file name by @MichaelDoyle in https://github.com/firebase/genkit/pull/3167\r\n* fix(js/core): correctly passthrough telemetry labels when streaming by @pavelgj in https://github.com/firebase/genkit/pull/3173\r\n* fix(cli): fixed analytics acknowledgement by @pavelgj in https://github.com/firebase/genkit/pull/3170\r\n\r\n## New Contributors\r\n* @schlich made their first contribution in https://github.com/firebase/genkit/pull/3146\r\n* @leixiaotian1 made their first contribution in https://github.com/firebase/genkit/pull/3144\r\n\r\n**Full Changelog**: https://github.com/firebase/genkit/compare/genkit@1.14.0...genkit@1.14.1\n\n## [Genkit Go v0.6.2](https://github.com/firebase/genkit/releases/tag/go/v0.6.2) - 7/2/2025\n\n## What's Changed\r\n* fix(go/plugins/googlegenai): Update gemini models by @leixiaotian1 in https://github.com/firebase/genkit/pull/3144\r\n* fix(go): fixed incomplete dynamic action resolution by @apascal07 in https://github.com/firebase/genkit/pull/3152\r\n\r\n## New Contributors\r\n* @leixiaotian1 made their first contribution in https://github.com/firebase/genkit/pull/3144\r\n\r\n**Full Changelog**: https://github.com/firebase/genkit/compare/go/v0.6.1...go/v0.6.2\n\n## [Genkit Go v0.6.1](https://github.com/firebase/genkit/releases/tag/go/v0.6.1) - 6/30/2025\n\n## What's Changed\r\n* fix(go/plugins): update gemini models and modelgarden provider by @hugoaguirre in https://github.com/firebase/genkit/pull/3103\r\n* feat(go): plugin implementation for Google AlloyDB for Postgresql by @davidnastasi in https://github.com/firebase/genkit/pull/3010\r\n* feat(go): plugin implementation for Google Cloud SQL for PostgreSQL by @davidnastasi in https://github.com/firebase/genkit/pull/3009\r\n* feat(go): Support streamable http in MCP Plugin by @huangjeff5 in https://github.com/firebase/genkit/pull/3135\r\n* fix(go): GoogleAI Plugin should let Gemini handle generativelanguage.googleapis.com URLs directly by @huangjeff5 in https://github.com/firebase/genkit/pull/3128\r\n* feat(go/plugins/compat_oai/openai): add dynamic model resolution by @hugoaguirre in https://github.com/firebase/genkit/pull/3095\r\n\r\n**Full Changelog**: https://github.com/firebase/genkit/compare/go/v0.6.0...go/v0.6.1\n\n## [Genkit JS 1.14.0](https://github.com/firebase/genkit/releases/tag/genkit%401.14.0) - 6/27/2025\n\n## What's Changed\r\n* fix(js/plugins/google-cloud): Write input and output logs for generate step by @bryanatkinson in https://github.com/firebase/genkit/pull/3107\r\n* fix(js): process spans in isolation to improve accuracy of telemetry by @andrewbrook in https://github.com/firebase/genkit/pull/3113\r\n* feat(js): Enable Firebase Genkit Monitoring via an environment variable by @schnecle in https://github.com/firebase/genkit/pull/3043\r\n* feat(js): indexer and retriever for Cloud SQL by @tanyasingh1902 in https://github.com/firebase/genkit/pull/2997\r\n* fix(js): Limits on GoogleAI Gemini model config by @shrutip90 in https://github.com/firebase/genkit/pull/3089\r\n* fix(js): Fix limits on vertexai and ollama config by @shrutip90 in https://github.com/firebase/genkit/pull/3129\r\n* feat(cli): added experimental mcp server to genkit start command by @pavelgj in https://github.com/firebase/genkit/pull/3132\r\n* feat(js/ai): added support for dynamically resolvable resources by @pavelgj in https://github.com/firebase/genkit/pull/3133\r\n* feat(js/plugins/express): use connection close/timeout as abort signal for the action by @pavelgj in https://github.com/firebase/genkit/pull/3138\r\n* feat(js/plugins/next): use request signal as abort signal for the action by @pavelgj in https://github.com/firebase/genkit/pull/3137\r\n* feat(js/client): added abort signal to runFlow/streamFlow client functions by @pavelgj in https://github.com/firebase/genkit/pull/3139\r\n\r\n## New Contributors\r\n* @davidnastasi made their first contribution in https://github.com/firebase/genkit/pull/3010\r\n* @tanyasingh1902 made their first contribution in https://github.com/firebase/genkit/pull/2997\r\n* @katopz made their first contribution in https://github.com/firebase/genkit/pull/3134\r\n* @forivall made their first contribution in https://github.com/firebase/genkit/pull/3118\r\n\r\n**Full Changelog**: https://github.com/firebase/genkit/compare/genkit@1.13.0...genkit@1.14.0\n\n## [Genkit JS 1.13.0](https://github.com/firebase/genkit/releases/tag/genkit%401.13.0) - 6/19/2025\n\n## Highlights \r\n\r\nAdded support for Veo 2 and Imagen 3 models for `googleai` plugin. See code samples:\r\n - [Veo 2](https://github.com/firebase/genkit/blob/31b18b04f0bd8823b749cf3c44ff7e10c77285fa/samples/js-gemini/src/index.ts#L261)\r\n - [Imagen 3](https://github.com/firebase/genkit/blob/31b18b04f0bd8823b749cf3c44ff7e10c77285fa/samples/js-gemini/src/index.ts#L186)\r\n\r\n## What's Changed\r\n* feat(js/plugins/googleai): added support for imagen and veo models to googleai plugin by @pavelgj in https://github.com/firebase/genkit/pull/3024\r\n* feat(js): added dynamicTool factory function that does not depend on genkit instance by @pavelgj in https://github.com/firebase/genkit/pull/3026\r\n* feat(js): added abort signal to actions/flows by @pavelgj in https://github.com/firebase/genkit/pull/3090\r\n* fix(js/plugins/google-cloud): Update metric attribute length truncation to match Trace attribute max length. by @bryanatkinson in https://github.com/firebase/genkit/pull/3050\r\n* fix(js/plugins/google-cloud): Remove metrics and dimensions unused by the Firebase Genkit Monitoring dashboard by @bryanatkinson in https://github.com/firebase/genkit/pull/3075\r\n* fix(js/plugins/google-cloud): Run all tests and refactor to only initialize telemetry once by @bryanatkinson in https://github.com/firebase/genkit/pull/3076\r\n* fix(js/plugins/google-cloud): Fix feature name parsing to include \"dotprompt\" type by @andrewbrook in https://github.com/firebase/genkit/pull/3094\r\n* fix(js/plugins/google-cloud): Add missing role field to structured input and output logs from GCP plugin by @bryanatkinson in https://github.com/firebase/genkit/pull/3093\r\n* fix(js): Add config for 2.5 Pro TTS model by @shrutip90 in https://github.com/firebase/genkit/pull/3041\r\n\r\n\r\n**Full Changelog**: https://github.com/firebase/genkit/compare/genkit@1.12.0...genkit@1.13.0\n\n## [Genkit Go v0.6.0](https://github.com/firebase/genkit/releases/tag/go/v0.6.0) - 6/13/2025\n\n## What's Changed\r\n* feat(go/genkit): Remove indexers for localvec by @rutujaD1999 in https://github.com/firebase/genkit/pull/2862\r\n* fix(go/plugins/googlegenai): runtime panic if no candidates returned by @Dasio in https://github.com/firebase/genkit/pull/2925\r\n* ci(go): configure the go.yaml checks to ensure go code is properly formatted by @yesudeep in https://github.com/firebase/genkit/pull/2900\r\n* feat(go/plugins/googlegenai): Add `reasoning` parts by @hugoaguirre in https://github.com/firebase/genkit/pull/2952\r\n* fix(go/plugins/googlegenai): Tool calls with streaming by @quinlanjager in https://github.com/firebase/genkit/pull/2980\r\n* feat(go): added support for dynamic models by @apascal07 in https://github.com/firebase/genkit/pull/2920\r\n* feat(go/dotprompt): Add support to MediaParts by @hugoaguirre in https://github.com/firebase/genkit/pull/3006\r\n* feat(go/plugins/googlegenai): add signature thought to reasoning parts by @hugoaguirre in https://github.com/firebase/genkit/pull/3007\r\n* feat(go): add DefineToolWithInputSchema to support custom JSON schemas by @MyeongKim in https://github.com/firebase/genkit/pull/2922\r\n* feat(go): added resume/restart for interruptible tools by @apascal07 in https://github.com/firebase/genkit/pull/3035\r\n* fix(go/plugins/googlegenai): Skip nil part by @Dasio in https://github.com/firebase/genkit/pull/3034\r\n* fix(go): fixed broken build by @apascal07 in https://github.com/firebase/genkit/pull/3039\r\n* feat(go): Add MCP Client by @huangjeff5 in https://github.com/firebase/genkit/pull/2959\r\n* fix(go/plugins/googlegenai): distinct text parts from thoughts by @hugoaguirre in https://github.com/firebase/genkit/pull/3048\r\n* feat(go): added dynamic tools support by @apascal07 in https://github.com/firebase/genkit/pull/3025\r\n* fix(go/ai)!: removed options related to indexing by @apascal07 in https://github.com/firebase/genkit/pull/3063\r\n* feat(go/plugins/googlegenai): Add dynamic model registration by @hugoaguirre in https://github.com/firebase/genkit/pull/2931\r\n* chore(go/plugins/vertexai/modelgarden): add claude opus and sonnet 4 models by @hugoaguirre in https://github.com/firebase/genkit/pull/3066\r\n* fix(go/genkit): Update dotprompt version by @rutujaD1999 in https://github.com/firebase/genkit/pull/3068\r\n* feat(go/plugins/googlegenai)!: use `go-genai` SDK configuration by @hugoaguirre in https://github.com/firebase/genkit/pull/2958\r\n* feat(go/plugins/googlegenai): add `image-generation` models by @hugoaguirre in https://github.com/firebase/genkit/pull/2903\r\n\r\n## New Contributors\r\n* @Dasio made their first contribution in https://github.com/firebase/genkit/pull/2925\r\n* @quinlanjager made their first contribution in https://github.com/firebase/genkit/pull/2980\r\n* @MyeongKim made their first contribution in https://github.com/firebase/genkit/pull/2922\r\n\r\n**Full Changelog**: https://github.com/firebase/genkit/compare/go/v0.5.4...go/v0.6.0\n\n## [Genkit JS 1.12.0](https://github.com/firebase/genkit/releases/tag/genkit%401.12.0) - 6/7/2025\n\n## What's Changed\r\n* fix(js/plugins/googleai): fixed systemRole support for dynamic models by @ifielker in https://github.com/firebase/genkit/pull/3018\r\n* feat(js): Support speech configuration for Gemini TTS models by @shrutip90 in https://github.com/firebase/genkit/pull/3016\r\n* fix: add span for prompt template rendering by @MichaelDoyle in https://github.com/firebase/genkit/pull/2165\r\n* fix(js/plugins/vertexai): fixed dynamic embedder resolution by @pavelgj in https://github.com/firebase/genkit/pull/3038\r\n\r\n## New Contributors\r\n* @MyeongKim made their first contribution in https://github.com/firebase/genkit/pull/2922\r\n\r\n**Full Changelog**: https://github.com/firebase/genkit/compare/genkit@1.11.1...genkit@1.12.0\n\n## [Genkit JS 1.11.1](https://github.com/firebase/genkit/releases/tag/genkit%401.11.1) - 6/2/2025\n\n## What's Changed\r\n* fix(js/plugins/googleai): fixed streamed response aggregation by @pavelgj in https://github.com/firebase/genkit/pull/3013\r\n* fix(js/plugins/googleai): Handle Gemini API thought signatures. by @mbleigh in https://github.com/firebase/genkit/pull/3014\r\n* fix(js/plugins/vertexai): Handle Gemini API thought signatures by @pavelgj in https://github.com/firebase/genkit/pull/3015\r\n\r\n**Full Changelog**: https://github.com/firebase/genkit/compare/genkit@1.11.0...genkit@1.11.1\n\n## [Genkit JS 1.11.0](https://github.com/firebase/genkit/releases/tag/genkit%401.11.0) - 5/30/2025\n\n## What's Changed\r\n* fix(js/core/flows): allow setting metadata on flows by @pavelgj in https://github.com/firebase/genkit/pull/2953\r\n* feat(js/ai/evaluators): Support batching for JS evaluator & CLI by @ssbushi in https://github.com/firebase/genkit/pull/2977\r\n* samples(js): added test app for gemini text-to-speach by @pavelgj in https://github.com/firebase/genkit/pull/2974\r\n* fix(js/plugins/vertexai): remove logging of models in listModels by @NimJay in https://github.com/firebase/genkit/pull/2985\r\n* fix(js/plugins/\\{googleai,vertexai\\}): correctly passthrough custom tool definitions by @pavelgj in https://github.com/firebase/genkit/pull/2995\r\n* fix(js/ai): missing types for uuid module (#3000) by @yesudeep in https://github.com/firebase/genkit/pull/3001\r\n* fix(js/plugins/vertexai): use googleSearch tool (instead of googleSearchRetrieval) for newer models by @pavelgj in https://github.com/firebase/genkit/pull/3005\r\n* fix: fix up invalid tsconfig shipped on npm by @gioboa in https://github.com/firebase/genkit/pull/2973\r\n* fix(cli): fixed telemetry server saving spans that get filtered by @pavelgj in https://github.com/firebase/genkit/pull/2979\r\n\r\n## New Contributors\r\n* @NimJay made their first contribution in https://github.com/firebase/genkit/pull/2985\r\n* @gioboa made their first contribution in https://github.com/firebase/genkit/pull/2973\r\n\r\n**Full Changelog**: https://github.com/firebase/genkit/compare/genkit@1.10.0...genkit@1.11.0\n\n## [genkit-python@0.4.0](https://github.com/firebase/genkit/releases/tag/genkit-python%400.4.0) - 5/26/2025\n\n## What's Changed\r\n* feat(py): Add list models for dev UI Gemini by @AbeJLazaro in https://github.com/firebase/genkit/pull/2949\r\n* fix(types): Resolve type errors and improve type annotations by @hendrixmar in https://github.com/firebase/genkit/pull/2808\r\n* feat(py): Dynamic model registration for Gemini by @AbeJLazaro in https://github.com/firebase/genkit/pull/2908 https://github.com/firebase/genkit/pull/2917\r\n* feat(py): Add vertex-ai vector search plugin by @AbeJLazaro in https://github.com/firebase/genkit/pull/2871\r\n\r\n## New Contributors\r\n* @hendrixmar made their first contribution in https://github.com/firebase/genkit/pull/2808\r\n\r\n**Full Changelog**: https://github.com/firebase/genkit/compare/genkit@1.10.0...genkit-python@0.4.0\n\n## [Genkit JS 1.10.0](https://github.com/firebase/genkit/releases/tag/genkit%401.10.0) - 5/21/2025\n\n## What's Changed\r\n* chore(js/plugins/googleai): added google search options to gemini config schema by @pavelgj in https://github.com/firebase/genkit/pull/2938\r\n* feat(js): added reasoning part type and implemented for googleai and vertexai by @pavelgj in https://github.com/firebase/genkit/pull/2945\r\n\r\n\r\n### Dev UI changes:\r\n* fix: update genkit title font weight by @tonybaroneee\r\n* fix: wrong background color in flow runner by @hugomurillomtz\r\n* feat: make trace drawer collapse when clicking chevron by @hugomurillomtz\r\n* fix: trace details overflow by @hugomurillomtz\r\n* Update runners to use json-viewer by @MichaelDoyle\r\n* set container-border-radius to 8px by @MichaelDoyle\r\n* feat: update flow runner to use json-viewer by @MichaelDoyle\r\n* feat: Add toggles for custom option objects to explicitly set/unset the option by @shrutip90\r\n* fix: Allow zeroes for temperature in dotprompt files by @shrutip90\r\n* feat: update span tree layout by @tonybaroneee\r\n* fix: Set custom option object toggles during patch/reset config by @shrutip90\r\n\r\n**Full Changelog**: https://github.com/firebase/genkit/compare/genkit@1.9.0...genkit@1.10.0\n\n## [Genkit JS 1.9.0](https://github.com/firebase/genkit/releases/tag/genkit%401.9.0) - 5/14/2025\n\n## What's Changed\r\n* feat(\\{go,js,py\\}): Add missing `UsageMetadata` fields by @hugoaguirre in https://github.com/firebase/genkit/pull/2860\r\n* chore(js/plugins/\\{googleai,vertexai\\}): stop pre-registering gemini 1.0 by @pavelgj in https://github.com/firebase/genkit/pull/2910\r\n* fix(js): added missing exports and typedoc config for `genkit/context` by @pavelgj in https://github.com/firebase/genkit/pull/2909\r\n\r\n**Full Changelog**: https://github.com/firebase/genkit/compare/genkit@1.8.0...genkit@1.9.0\n\n## [Genkit Go v0.5.4](https://github.com/firebase/genkit/releases/tag/go/v0.5.4) - 5/13/2025\n\n## What's Changed\r\n* chore(go/plugins/googlegenai): expose `gemini-2.5-pro-preview-05-06` by @hugoaguirre in https://github.com/firebase/genkit/pull/2870\r\n* feat(go/plugins): add generic OpenAI plugin by @kekoawong in https://github.com/firebase/genkit/pull/2829\r\n* feat(\\{go,js,py\\}): Add missing `UsageMetadata` fields by @hugoaguirre in https://github.com/firebase/genkit/pull/2860\r\n* fix(go/genkit): update YAML library from go-yaml/yaml to goccy/go-yaml by @ihan211 in https://github.com/firebase/genkit/pull/2841\r\n* fix(go): run modernize happy by @zchee in https://github.com/firebase/genkit/pull/2897\r\n* fix(go): unify indirect require section in go.mod by @zchee in https://github.com/firebase/genkit/pull/2898\r\n* feat(go/plugins/googlegenai): add thinking support by @hugoaguirre in https://github.com/firebase/genkit/pull/2846\r\n* bug(go): propagate telemetry labels from Reflection API to tracing by @sahdev77 in https://github.com/firebase/genkit/pull/2881\r\n\r\n## New Contributors\r\n* @devchas made their first contribution in https://github.com/firebase/genkit/pull/2646\r\n* @ihan211 made their first contribution in https://github.com/firebase/genkit/pull/2841\r\n* @zchee made their first contribution in https://github.com/firebase/genkit/pull/2897\r\n\r\n**Full Changelog**: https://github.com/firebase/genkit/compare/go/v0.5.3...go/v0.5.4\n\n## [Genkit JS 1.8.0](https://github.com/firebase/genkit/releases/tag/genkit%401.8.0) - 5/5/2025\n\n## What's Changed\r\n* refactor(js/plugins/vertexai): Dynamic model support for Gemini models by @ifielker in https://github.com/firebase/genkit/pull/2824\r\n* feat(js/core): allow defining a \"list actions\" fn for plugins by @pavelgj in https://github.com/firebase/genkit/pull/2830\r\n* refactor(js/plugins/vertexai): refactored tests for dynamic models by @ifielker in https://github.com/firebase/genkit/pull/2836\r\n* feat(js/plugins/\\{googleai,vertexai\\}): implemented dynamic model listing for googleai, and new model lookup API for googleai & vertexai by @pavelgj in https://github.com/firebase/genkit/pull/2839\r\n* fix(js/prompts): fixed .prompt input handling on prompt render by @pavelgj in https://github.com/firebase/genkit/pull/2843\r\n* feat(js/plugins/ollama): implemented dynamic model resolver and list actions for ollama plugin by @pavelgj in https://github.com/firebase/genkit/pull/2831\r\n* feat('js/plugins/vertexai'): dynamically resolve imagen models and try to list models from the API by @pavelgj in https://github.com/firebase/genkit/pull/2847\r\n* feat(js/plugins/\\{googleai,vertexai\\}): pasthrough unknown config options directly to the underlying API by @pavelgj in https://github.com/firebase/genkit/pull/2848\r\n* feat(js/ai): added dynamic tools that can be dynamically defined at runtime by @pavelgj in https://github.com/firebase/genkit/pull/2854\r\n\r\n\r\n**Full Changelog**: https://github.com/firebase/genkit/compare/genkit@1.7.0...genkit@1.8.0\n\n## [Genkit Go v0.5.3](https://github.com/firebase/genkit/releases/tag/go/v0.5.3) - 5/1/2025\n\n## What's Changed\r\n* fix(go/plugins/googlegenai): `ai.NewMediaPart` support to inline and hosted data by @hugoaguirre in https://github.com/firebase/genkit/pull/2805\r\n* chore(go/plugins/googlegenai): bump `go-genai` sdk to v1.1.0 by @hugoaguirre in https://github.com/firebase/genkit/pull/2802\r\n* fix(go/plugins/googlegenai): ignore response mime type if tools are present by @hugoaguirre in https://github.com/firebase/genkit/pull/2794\r\n* feat(go): Add ollama vision support by @huangjeff5 in https://github.com/firebase/genkit/pull/2795\r\n* fix(go/plugins/googlegenai): update test cases with new go-genai types by @hugoaguirre in https://github.com/firebase/genkit/pull/2819\r\n* fix(go): fix `ModelResponse.ToolRequests()` return type by @apascal07 in https://github.com/firebase/genkit/pull/2823\r\n* fix(go/genkit): remove colons from runtime filename by @hugoaguirre in https://github.com/firebase/genkit/pull/2817\r\n* chore(go): update all dependencies to their latest versions by @hugoaguirre in https://github.com/firebase/genkit/pull/2825\r\n* fix(go/plugins/googlegenai): use `boolean` instead of `bool` in JSON schemas by @hugoaguirre in https://github.com/firebase/genkit/pull/2842\r\n* feat(go): Add tool support for ollama models by @huangjeff5 in https://github.com/firebase/genkit/pull/2796\r\n* feat(go): use `GenkitError` instead of regular `error` by @sahdev77 in https://github.com/firebase/genkit/pull/2643\r\n\r\n\r\n**Full Changelog**: https://github.com/firebase/genkit/compare/go/v0.5.2...go/v0.5.3\n\n## [genkit-python@0.3.2](https://github.com/firebase/genkit/releases/tag/genkit-python%400.3.2) - 4/29/2025\n\n## What's Changed\r\n* feat(py/plugins/google-genai, py/plugins/vertex-ai): add gemini-2.5-flash-preview-04-17 model @nozomi-koborinai\r\n in https://github.com/firebase/genkit/pull/2784\r\n* feat(python): Integration of OTEL for GCP @AbeJLazaro in https://github.com/firebase/genkit/pull/2791\r\n* release: python genkit 0.3.2 by @AbeJLazaro in https://github.com/firebase/genkit/pull/2840\r\n\r\n\r\n**Full Changelog**: https://github.com/firebase/genkit/compare/genkit@1.7.0...genkit-python@0.3.2\n\n## [genkit-python@0.3.1](https://github.com/firebase/genkit/releases/tag/genkit-python%400.3.1) - 4/29/2025\n\n## What's Changed\r\n* Initial semi productive version of Genkit package and plugins\r\n\r\n\r\n**Full Changelog**: https://github.com/firebase/genkit/compare/genkit@1.6.0...genkit-python@0.3.1\n\n## [Genkit JS 1.7.0](https://github.com/firebase/genkit/releases/tag/genkit%401.7.0) - 4/25/2025\n\n## What's Changed\r\n* feat(js/core): added trace information to action and flow contexts by @pavelgj in https://github.com/firebase/genkit/pull/2806\r\n* feat(js): Dynamic action resolver support by @ifielker in https://github.com/firebase/genkit/pull/2807\r\n\r\n\r\n**Full Changelog**: https://github.com/firebase/genkit/compare/genkit@1.6.2...genkit@1.7.0\n\n## [Genkit Go v0.5.2](https://github.com/firebase/genkit/releases/tag/go/v0.5.2) - 4/18/2025\n\n## What's Changed\r\n* feat(go/genkit): Add action context example by @rutujaD1999 in https://github.com/firebase/genkit/pull/2746\r\n* feat(go/ai): added array and enum format support by @dysrama in https://github.com/firebase/genkit/pull/2745\r\n* fix(go/ai): pass ptr. of value for generate data in JSON (#2756) by @suapapa in https://github.com/firebase/genkit/pull/2757\r\n* bug(go): refactored Firebase plugin to adhere to plugin interface by @sahdev77 in https://github.com/firebase/genkit/pull/2753\r\n* fix(go/genkit): Fix input from context by @rutujaD1999 in https://github.com/firebase/genkit/pull/2764\r\n* feat(go/plugins/googlegenai): add image generation native support by @hugoaguirre in https://github.com/firebase/genkit/pull/2630\r\n* feat(go/plugins/vertexai): Add `modelgarden` with `anthropic` support by @hugoaguirre in https://github.com/firebase/genkit/pull/2749\r\n* fix(go/genkit): fix server response formatting and headers by @apascal07 in https://github.com/firebase/genkit/pull/2774\r\n* feat(\\{js,go\\}): Add `gemini-2.5-flash-preview-04-17` to Vertex AI by @apascal07 in https://github.com/firebase/genkit/pull/2782\r\n\r\n**Full Changelog**: https://github.com/firebase/genkit/compare/go/v0.5.1...go/v0.5.2\n\n## [Genkit JS 1.6.2](https://github.com/firebase/genkit/releases/tag/genkit%401.6.2) - 4/18/2025\n\n## What's Changed\r\n* fix(js/plugins/vertexai): better error handling for empty response for imagen models by @pavelgj in https://github.com/firebase/genkit/pull/2747\r\n* fix(js/ai): properly set prompt meta for variants by @MichaelDoyle in https://github.com/firebase/genkit/pull/2754\r\n* fix(js/ai/formats): default to json if jsonSchema present by @MichaelDoyle in https://github.com/firebase/genkit/pull/2762\r\n* chore(\\{js,go\\}): Add `gemini-2.5-flash-preview-04-17` to Vertex AI by @apascal07 in https://github.com/firebase/genkit/pull/2782\r\n* fix(js/plugins/\\{googleai,vertexai\\}): fixed gemini-2.5-flash-preview-04-17 refs and init by @pavelgj in https://github.com/firebase/genkit/pull/2786\r\n\n\n## [Genkit Go v0.5.1](https://github.com/firebase/genkit/releases/tag/go/v0.5.1) - 4/9/2025\n\n## What's Changed\r\n* Updated internal version.\n\n## [Genkit CLI 1.6.1](https://github.com/firebase/genkit/releases/tag/genkit-cli%401.6.1) - 4/9/2025\n\n## What's Changed\r\n\r\nfix: Go lang prompt counts to show up in the Dev UI nav bar by @ssbushi\n\n## [Genkit Go v0.5.0](https://github.com/firebase/genkit/releases/tag/go/v0.5.0) - 4/8/2025\n\n## What's Changed\r\n* feat(go): Added prompts as executable prompts by @apascal07 in https://github.com/firebase/genkit/pull/2535\r\n* fix(go/genkit): use controlled flushing on JSON responses by @hugoaguirre in https://github.com/firebase/genkit/pull/2530\r\n* feat(go): Call `DefineGenerateAction` in `Genkit.Init` by @apascal07 in https://github.com/firebase/genkit/pull/2575\r\n* refactor(go/plugins): keep gemini code in `googlegenai` by @hugoaguirre in https://github.com/firebase/genkit/pull/2576\r\n* feat(go): Add customConfig field to ModelInfo by @huangjeff5 in https://github.com/firebase/genkit/pull/2537\r\n* fix(go): remove legacy prompts from go by @ssbushi in https://github.com/firebase/genkit/pull/2598\r\n* feat(go/ai): Use `dotprompt` package for rendering prompt templates by @rutujaD1999 in https://github.com/firebase/genkit/pull/2526\r\n* chore(go): bump `go-genai` SDK to v0.7.0 by @hugoaguirre in https://github.com/firebase/genkit/pull/2620\r\n* feat(go): Added `ModelArg` interface and `ModelRef` by @huangjeff5 in https://github.com/firebase/genkit/pull/2487\r\n* fix(go): add tool to prompt metadata by @ssbushi in https://github.com/firebase/genkit/pull/2622\r\n* feat(go/genkit): Added support for formatters and custom instructions by @dysrama in https://github.com/firebase/genkit/pull/2417\r\n* feat(go/genkit): Exposed `DefineFormat` in `Genkit`. by @apascal07 in https://github.com/firebase/genkit/pull/2647\r\n* fix(go/plugins): Unpacked common fields in `GeminiConfig` and fixed output config by @apascal07 in https://github.com/firebase/genkit/pull/2649\r\n* chore(go): expose model `gemini-2.5-pro-preview-03-25` by @hugoaguirre in https://github.com/firebase/genkit/pull/2651\r\n* fix(go): minor bug fixes by @hugoaguirre in https://github.com/firebase/genkit/pull/2654\r\n* fix(go): Fixed prompt fields not propagating to execute for loaded prompts. by @apascal07 in https://github.com/firebase/genkit/pull/2655\r\n* fix(go): Fix Issues in using genkit with ollama plugin. by @suapapa in https://github.com/firebase/genkit/pull/2681\r\n* feat(go/plugins/googlegenai): add constrained generation by @hugoaguirre in https://github.com/firebase/genkit/pull/2467\r\n* fix(go): metadata and status enum fixes by @ssbushi in https://github.com/firebase/genkit/pull/2698\r\n* fix(go): Fixed nil pointer in Google AI plugin. by @apascal07 in https://github.com/firebase/genkit/pull/2700\r\n* fix(go/py/tooling): More small fixes related to evals by @ssbushi in https://github.com/firebase/genkit/pull/2703\r\n* refactor(go)!: Updated all primitives to unified options. by @apascal07 in https://github.com/firebase/genkit/pull/2550\r\n* bug(go)!: remove `local` provider from tools and prompts by @hugoaguirre in https://github.com/firebase/genkit/pull/2713\r\n* fix(go): update samples to use `GeminiConfig` by @hugoaguirre in https://github.com/firebase/genkit/pull/2726\r\n* fix(go): use `GeminiConfig` in cache live tests by @hugoaguirre in https://github.com/firebase/genkit/pull/2730\r\n\r\n## New Contributors\r\n* @suapapa made their first contribution in https://github.com/firebase/genkit/pull/2681\r\n\r\n**Full Changelog**: https://github.com/firebase/genkit/compare/go/v0.3.0...go/v0.5.0\n\n## [Genkit JS and CLI 1.6.0](https://github.com/firebase/genkit/releases/tag/genkit%401.6.0) - 4/8/2025\n\n## What's Changed\r\n\r\n* chore(js/plugins/vertexai): add gemini-2.5-pro-exp-03-25 model by @nozomi-koborinai in https://github.com/firebase/genkit/pull/2666\r\n\r\n### CLI/Dev UI changes\r\n\r\n* feat(dev ui): prompt support for golang by @ssbushi \r\n* feat(dev ui): show duration and token count in traces table by @hugomurillomtz\r\n\r\n## New Contributors\r\n* @nozomi-koborinai made their first contribution in https://github.com/firebase/genkit/pull/2666\r\n\r\n**Full Changelog**: https://github.com/firebase/genkit/compare/genkit@1.5.0...genkit@1.6.0\n\n## [Genkit JS 1.5.0](https://github.com/firebase/genkit/releases/tag/genkit%401.5.0) - 4/5/2025\n\n## What's Changed\r\n* fix: make customOptions consistent in action metadata by @ssbushi in https://github.com/firebase/genkit/pull/2465\r\n* feat(evals): add `status` enum for evaluation scores by @ssbushi in https://github.com/firebase/genkit/pull/2169\r\n* chore(js): Update Dotprompt to v1.1.0. Fixes #2039 by @mbleigh in https://github.com/firebase/genkit/pull/2573\r\n* feat(js/plugins/firebase): Data Connect tools (beta), serverApp context. by @mbleigh in https://github.com/firebase/genkit/pull/2572\r\n* fix(js/plugins/express): Check request.body is defined before reading data from it by @ifielker in https://github.com/firebase/genkit/pull/2600\r\n* chore: Add descriptions for fields in model config schemas by @shrutip90 in https://github.com/firebase/genkit/pull/2627\r\n* chore: expose model `gemini-2.5-pro-preview-03-25` by @hugoaguirre in https://github.com/firebase/genkit/pull/2651\r\n\r\n## New Contributors\r\n* @AbeJLazaro made their first contribution in https://github.com/firebase/genkit/pull/2466\r\n* @mukezhz made their first contribution in https://github.com/firebase/genkit/pull/2439\r\n\r\n**Full Changelog**: https://github.com/firebase/genkit/compare/genkit@1.4.0...genkit@1.5.0\n\n## [Genkit Go v0.3.0](https://github.com/firebase/genkit/releases/tag/go/v0.3.0) - 3/28/2025\n\n## What's Changed\r\n- breaking(go): remove multiple candidates support by @dysrama in https://github.com/firebase/genkit/pull/1307\r\n- fix(go): readd old result body for runAction, skip go test by @dysrama in https://github.com/firebase/genkit/pull/1430\r\n- fix(go): set log level to debug in golang by @falonso81 in https://github.com/firebase/genkit/pull/1236\r\n- breaking(go): Refactored into instance-based Genkit and Registry. by @apascal07 in https://github.com/firebase/genkit/pull/1459\r\n- fix(go): Fix TestActionTracing test case by @hugoaguirre in https://github.com/firebase/genkit/pull/1595\r\n- breaking(go): update flow streaming protocol to SSE by @hugoaguirre in https://github.com/firebase/genkit/pull/1316\r\n- breaking(go): upgrade ergonomics of prompt definition and calling by @dysrama in https://github.com/firebase/genkit/pull/1447\r\n- feature(go): add support for multiple model versions by @hugoaguirre in https://github.com/firebase/genkit/pull/1575\r\n- feat(go): add gemini-2.0 models to vertexAI and googleAI plugins by @hugoaguirre in https://github.com/firebase/genkit/pull/1876\r\n- feat(go): Added /util/generate + filled feature gaps in Generate API. by @apascal07 in https://github.com/firebase/genkit/pull/1818\r\n- fix(go/plugins/googleai): convertTools err propagate and schema type integer by @decibelcooper in https://github.com/firebase/genkit/pull/1911\r\n- feat(go/plugins/firebase): add retriever, init methods by @cabljac in https://github.com/firebase/genkit/pull/1470\r\n- fix(go): fix go test cases by @hugoaguirre in https://github.com/firebase/genkit/pull/1964\r\n- docs(contribution): update contribution guide to include GO by @kekoawong in https://github.com/firebase/genkit/pull/2038\r\n- refactor(go): Refactored Flow, context providers, and server definition. by @apascal07 in https://github.com/firebase/genkit/pull/1956\r\n- fix(go): Fixed nil options on Generate() calls in samples. by @apascal07 in https://github.com/firebase/genkit/pull/2187\r\n- fix(go): bin/fmt to format go code #2193 by @yesudeep in https://github.com/firebase/genkit/pull/2194\r\n- refactor(go): Removed exposure to internal action interface. by @apascal07 in https://github.com/firebase/genkit/pull/2174\r\n- feat(go): Added generate-level middleware. by @apascal07 in https://github.com/firebase/genkit/pull/1949\r\n- fix(go): Added JSON schema hack for []any types. by @apascal07 in https://github.com/firebase/genkit/pull/2207\r\n- fix(bin/fmt): issue with finding the go path when formatting by @yesudeep in https://github.com/firebase/genkit/pull/2225\r\n- fix(go): Fixed e2e tests. by @apascal07 in https://github.com/firebase/genkit/pull/2228\r\n- fix(go): Fixed retrievers data types (Dev UI mismatch). by @apascal07 in https://github.com/firebase/genkit/pull/2242\r\n- feat(go/plugins/googleai): Respect ToolChoice setting by @kwjw in https://github.com/firebase/genkit/pull/2238\r\n- refactor(go): Moved model version validation to middleware. by @apascal07 in https://github.com/firebase/genkit/pull/2233\r\n- breaking(go): refactored prompts API #1491 by @dysrama in https://github.com/firebase/genkit/pull/1980\r\n- fix(go): Fixed stream callback naming. by @apascal07 in https://github.com/firebase/genkit/pull/2251\r\n- fix(go): check validation for json part on text that generated by LLM (#2265) by @jcooky in https://github.com/firebase/genkit/pull/2273\r\n- feat(go/plugins): Use go-genai SDK in VertexAI and GoogleAI plugins by @hugoaguirre in https://github.com/firebase/genkit/pull/2259\r\n- refactor(go/ai): Refactored Generate API with unified options. by @apascal07 in https://github.com/firebase/genkit/pull/2320\r\n- refactor(go/ai): Refactored Prompt API with unified options. by @apascal07 in https://github.com/firebase/genkit/pull/2256\r\n- fix(\\{go,cli\\}): relax time event shema by @pavelgj in https://github.com/firebase/genkit/pull/2370\r\n- fix(go/samples/rag): Fixed rag sample missing template. by @apascal07 in https://github.com/firebase/genkit/pull/2372\r\n- feat(go): Add evaluation primitives by @ssbushi in https://github.com/firebase/genkit/pull/2227\r\n- fix(go): Fixed Go schema generator. by @huangjeff5 in https://github.com/firebase/genkit/pull/2367\r\n- fix(go): Fixed duplicate Score. by @apascal07 in https://github.com/firebase/genkit/pull/2389\r\n- feat(go/plugins): add context-caching to VertexAI and GoogleAI plugins by @hugoaguirre in https://github.com/firebase/genkit/pull/2266\r\n- fix(go): Added simulated system prompt support by @sahdev77 in https://github.com/firebase/genkit/pull/2399\r\n- fix(go): Made WithTools accept a ToolRef interface by @apascal07 in https://github.com/firebase/genkit/pull/2403\r\n- feat(go): add batch evaluator by @ssbushi in https://github.com/firebase/genkit/pull/2388\r\n- refactor(go): Added better naming for generated types. by @apascal07 in https://github.com/firebase/genkit/pull/2409\r\n- fix(go): Return error message if go code is not re-generated from updated schema by @huangjeff5 in https://github.com/firebase/genkit/pull/2424\r\n- refactor(go): Added genkit.WithPlugins option with unified plugin interface by @apascal07 in https://github.com/firebase/genkit/pull/2423\r\n- feat(go): Added simulated document-based context support by @sahdev77 in https://github.com/firebase/genkit/pull/2415\r\n- feat(go): Added media download middleware (enabled for Google GenAI) by @sahdev77 in https://github.com/firebase/genkit/pull/2402\r\n- fix(go): use stable versions of go: 1.23+ by @yesudeep in https://github.com/firebase/genkit/pull/2455\r\n- feat(go/genkit): Added loading of dotprompt files on Genkit init by @rutujaD1999 in https://github.com/firebase/genkit/pull/2413\r\n- feat(go/plugins)!: Added unified googlegenai plugin by @hugoaguirre in https://github.com/firebase/genkit/pull/2450\r\n- feat(go/plugins/googlegenai): add gemini-2.5-pro-exp-03-25 model by @hugoaguirre in https://github.com/firebase/genkit/pull/2464\r\n- fix(go): Fixed error propagation in Reflection API. by @apascal07 in https://github.com/firebase/genkit/pull/2410\r\n- feat(go/plugins): Add genkitEval plugin by @ssbushi in https://github.com/firebase/genkit/pull/2436\r\n- feat(go/plugins/googlegenai): Added data part support by @hugoaguirre in https://github.com/firebase/genkit/pull/2469\r\n- docs(go): Updated package docs and made various fixes along the way by @apascal07 in https://github.com/firebase/genkit/pull/2474\r\n- fix(go/samples): samples maintentance by @hugoaguirre in https://github.com/firebase/genkit/pull/2480\r\n- fix(go): Fixed default input for prompts. by @apascal07 in https://github.com/firebase/genkit/pull/2491\r\n- docs(go/genkit): Updated genkit package docs and increment version by @apascal07 in https://github.com/firebase/genkit/pull/2499\r\n\r\n## New Contributors\r\n* @dysrama made their first contribution in https://github.com/firebase/genkit/pull/1307\r\n* @hugoaguirre made their first contribution in https://github.com/firebase/genkit/pull/1595\r\n* @yesudeep made their first contribution in https://github.com/firebase/genkit/pull/1682\r\n* @decibelcooper made their first contribution in https://github.com/firebase/genkit/pull/1911\r\n* @kekoawong made their first contribution in https://github.com/firebase/genkit/pull/2038\r\n* @kwjw made their first contribution in https://github.com/firebase/genkit/pull/2238\r\n* @sahdev77 made their first contribution in https://github.com/firebase/genkit/pull/2399\r\n* @rutujaD1999 made their first contribution in https://github.com/firebase/genkit/pull/2413\r\n\r\n**Full Changelog**: https://github.com/firebase/genkit/compare/go/v0.2.1...go/v0.3.0\n\n## [Genkit JS 1.4.0](https://github.com/firebase/genkit/releases/tag/genkit%401.4.0) - 3/26/2025\n\n## What's Changed\r\n* fix(cli): default to current folder when looking for project root by @pavelgj in https://github.com/firebase/genkit/pull/2400\r\n* chore(js/plugins/googleai): expose gemini-2.5-pro-exp-03-25 (`gemini25ProExp0325`) by @pavelgj in https://github.com/firebase/genkit/pull/2461\r\n* feat(js/plugins/mcp): Adds websocket support, fixes tool schema issue. by @mbleigh in https://github.com/firebase/genkit/pull/2478\r\n\r\n**Full Changelog**: https://github.com/firebase/genkit/compare/genkit@1.3.0...genkit@1.4.0\n", + "lang": "md", + "headers": "## Release Notes" } } \ No newline at end of file diff --git a/src/content.config.ts b/src/content.config.ts index d9ee8c9d..86ea1d1e 100644 --- a/src/content.config.ts +++ b/src/content.config.ts @@ -1,7 +1,6 @@ import { defineCollection } from 'astro:content'; -import { docsLoader } from '@astrojs/starlight/loaders'; import { docsSchema } from '@astrojs/starlight/schema'; export const collections = { - docs: defineCollection({ loader: docsLoader(), schema: docsSchema() }), + 'docs': defineCollection({ schema: docsSchema() }), }; diff --git a/src/pages/changelog.astro b/src/pages/changelog.astro new file mode 100644 index 00000000..9d330693 --- /dev/null +++ b/src/pages/changelog.astro @@ -0,0 +1,28 @@ +--- +import StarlightPage from '@astrojs/starlight/components/StarlightPage.astro'; +import { marked } from 'marked'; + +interface Release { + name: string; + html_url: string; + published_at: string; + body: string; +} + +const response = await fetch('https://api.github.com/repos/firebase/genkit/releases'); +const releases: Release[] = await response.json(); +--- + + + + diff --git a/src/pages/releases/[slug].astro b/src/pages/releases/[slug].astro new file mode 100644 index 00000000..866a08af --- /dev/null +++ b/src/pages/releases/[slug].astro @@ -0,0 +1,32 @@ +--- +import StarlightPage from '@astrojs/starlight/components/StarlightPage.astro'; +import { marked } from 'marked'; + +export async function getStaticPaths() { + const response = await fetch('https://api.github.com/repos/firebase/genkit/releases'); + const releases: any[] = await response.json(); + + return releases.map((release) => ({ + params: { slug: release.name.replace(/ /g, '_') }, + props: { release }, + })); +} + +const { release } = Astro.props; +--- + + +
+

{release.name}

({new Date(release.published_at).toLocaleDateString()}) +
+
+ + diff --git a/src/sidebar.ts b/src/sidebar.ts index afb57a26..028413cd 100644 --- a/src/sidebar.ts +++ b/src/sidebar.ts @@ -2,6 +2,7 @@ const JS_SIDEBAR = [ { label: "Get started", slug: "docs/get-started" }, { label: "Developer tools", slug: "docs/devtools" }, { label: "MCP Server", slug: "docs/mcp-server" }, + { label: "Changelog", link: "/changelog" }, { label: "Tutorials", items: [ diff --git a/src/tailwind.css b/src/tailwind.css index 8717f6d6..7240775e 100644 --- a/src/tailwind.css +++ b/src/tailwind.css @@ -218,4 +218,19 @@ h1 { [data-theme="light"] .sidebar-content a[aria-current="page"]:hover { color: var(--sl-color-accent); -} \ No newline at end of file +} + +.release-notes { + white-space: pre-wrap; + word-break: break-word; +} + +.no-underline { + text-decoration: none; +} + +.flex-center { + display: flex; + align-items: center; + gap: 0.5rem; +} diff --git a/tsconfig.json b/tsconfig.json index 990abae7..2cfdbe45 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -4,8 +4,11 @@ "exclude": ["dist"], "compilerOptions": { "baseUrl": ".", + "strictNullChecks": true, "paths": { "@/*": ["src/*"] - } + }, + "jsx": "react-jsx", + "jsxImportSource": "astro" } }