test(nextjs): Add AI provider orchestrion instrumentations to e2e app (#22550)

Adds e2e coverage for `openai`, `anthropic-ai` and `google-genai`
orchestrion instrumentations in the `nextjs-16-orchestrion` app. A
shared `node:http` mock server stands in for the three provider APIs so
the real SDK clients emit gen_ai spans without live credentials. These
libraries are bundled (internalized) and instrumented by the orchestrion
build-time loader.

Closes #22506

---------

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: chargome

dev-packages/e2e-tests/test-applications/nextjs-16-orchestrion/ai-mock-server.mjs

@@ -0,0 +1,93 @@
+import { createServer } from 'node:http';
+
+// A single mock server standing in for the OpenAI, Anthropic and Google GenAI HTTP APIs, so the real
+// SDK clients emit gen_ai spans without any live credentials. Response bodies mirror the mock servers
+// in the node-integration tests (suites/tracing/{openai,anthropic,google-genai}). Uses raw `node:http`
+// (not express) so the mock doesn't itself get instrumented.
+
+function readJson(req) {
+  return new Promise(resolve => {
+    let body = '';
+    req.on('data', chunk => (body += chunk));
+    req.on('end', () => {
+      try {
+        resolve(JSON.parse(body || '{}'));
+      } catch {
+        resolve({});
+      }
+    });
+  });
+}
+
+function sendJson(res, status, obj) {
+  res.writeHead(status, { 'Content-Type': 'application/json' });
+  res.end(JSON.stringify(obj));
+}
+
+let serverPromise;
+
+/** Lazily starts the shared mock server and resolves to its port. */
+export function getMockAiPort() {
+  serverPromise ??= new Promise(resolve => {
+    const server = createServer(async (req, res) => {
+      const url = req.url || '';
+
+      // OpenAI: chat completions
+      if (req.method === 'POST' && url.endsWith('/openai/chat/completions')) {
+        const { model } = await readJson(req);
+        sendJson(res, 200, {
+          id: 'chatcmpl-mock123',
+          object: 'chat.completion',
+          created: 1677652288,
+          model,
+          choices: [
+            { index: 0, message: { role: 'assistant', content: 'Hello from OpenAI mock!' }, finish_reason: 'stop' },
+          ],
+          usage: { prompt_tokens: 10, completion_tokens: 15, total_tokens: 25 },
+        });
+        return;
+      }
+
+      // Anthropic: messages
+      if (req.method === 'POST' && url.endsWith('/anthropic/v1/messages')) {
+        const { model } = await readJson(req);
+        sendJson(res, 200, {
+          id: 'msg_mock123',
+          type: 'message',
+          model,
+          role: 'assistant',
+          content: [{ type: 'text', text: 'Hello from Anthropic mock!' }],
+          stop_reason: 'end_turn',
+          stop_sequence: null,
+          usage: { input_tokens: 10, output_tokens: 15 },
+        });
+        return;
+      }
+
+      // Google GenAI: generateContent (the model name is embedded in the path before `:generateContent`).
+      // Plain string checks avoid the polynomial-backtracking risk of a `.+` regex on the URL.
+      if (req.method === 'POST' && url.startsWith('/v1beta/models/') && url.endsWith(':generateContent')) {
+        await readJson(req);
+        sendJson(res, 200, {
+          candidates: [
+            {
+              content: { parts: [{ text: 'Mock response from Google GenAI!' }], role: 'model' },
+              finishReason: 'stop',
+              index: 0,
+            },
+          ],
+          usageMetadata: { promptTokenCount: 8, candidatesTokenCount: 12, totalTokenCount: 20 },
+        });
+        return;
+      }
+
+      res.writeHead(404).end();
+    });
+
+    server.listen(0, () => {
+      resolve(server.address().port);
+    });
+  });
+
+  return serverPromise;
+}

dev-packages/e2e-tests/test-applications/nextjs-16-orchestrion/app/api/anthropic/route.ts

@@ -0,0 +1,21 @@
+import Anthropic from '@anthropic-ai/sdk';
+import { NextResponse } from 'next/server';
+import { getMockAiPort } from '../../../ai-mock-server.mjs';
+
+export const dynamic = 'force-dynamic';
+
+export async function GET() {
+  const port = await getMockAiPort();
+  const client = new Anthropic({
+    apiKey: 'mock-api-key',
+    baseURL: `http://localhost:${port}/anthropic`,
+  });
+
+  await client.messages.create({
+    model: 'claude-3-haiku-20240307',
+    max_tokens: 100,
+    messages: [{ role: 'user', content: 'What is the capital of France?' }],
+  });
+
+  return NextResponse.json({ status: 'ok' });
+}

dev-packages/e2e-tests/test-applications/nextjs-16-orchestrion/app/api/google-genai/route.ts

@@ -0,0 +1,21 @@
+import { GoogleGenAI } from '@google/genai';
+import { NextResponse } from 'next/server';
+import { getMockAiPort } from '../../../ai-mock-server.mjs';
+
+export const dynamic = 'force-dynamic';
+
+export async function GET() {
+  const port = await getMockAiPort();
+  const client = new GoogleGenAI({
+    apiKey: 'mock-api-key',
+    httpOptions: { baseUrl: `http://localhost:${port}` },
+  });
+
+  await client.models.generateContent({
+    model: 'gemini-1.5-flash',
+    config: { temperature: 0.7, topP: 0.9, maxOutputTokens: 100 },
+    contents: [{ role: 'user', parts: [{ text: 'What is the capital of France?' }] }],
+  });
+
+  return NextResponse.json({ status: 'ok' });
+}

dev-packages/e2e-tests/test-applications/nextjs-16-orchestrion/app/api/openai/route.ts

@@ -0,0 +1,20 @@
+import { NextResponse } from 'next/server';
+import OpenAI from 'openai';
+import { getMockAiPort } from '../../../ai-mock-server.mjs';
+
+export const dynamic = 'force-dynamic';
+
+export async function GET() {
+  const port = await getMockAiPort();
+  const client = new OpenAI({
+    baseURL: `http://localhost:${port}/openai`,
+    apiKey: 'mock-api-key',
+  });
+
+  await client.chat.completions.create({
+    model: 'gpt-3.5-turbo',
+    messages: [{ role: 'user', content: 'What is the capital of France?' }],
+  });
+
+  return NextResponse.json({ status: 'ok' });
+}

dev-packages/e2e-tests/test-applications/nextjs-16-orchestrion/package.json

@@ -16,6 +16,8 @@
   },
   "//": "Pin `ioredis` to 5.10.1 and `mysql2` to 3.19.1: both are the last versions before the driver publishes its own native diagnostics channels; orchestrion's configs cover `ioredis <5.11.0` and `mysql2 <3.20.0`.",
   "dependencies": {
+    "@anthropic-ai/sdk": "0.63.0",
+    "@google/genai": "^1.20.0",
     "@sentry/core": "file:../../packed/sentry-core-packed.tgz",
     "@sentry/nextjs": "file:../../packed/sentry-nextjs-packed.tgz",
     "dataloader": "2.2.2",
@@ -28,6 +30,7 @@
     "mysql": "^2.18.1",
     "mysql2": "3.19.1",
     "next": "16.2.10",
+    "openai": "5.18.1",
     "pg": "^8.13.1",
     "postgres": "^3.4.7",
     "react": "19.1.0",

dev-packages/e2e-tests/test-applications/nextjs-16-orchestrion/tests/anthropic.test.ts

@@ -0,0 +1,23 @@
+import { expect, test } from '@playwright/test';
+import { waitForStreamedSpans } from '@sentry-internal/test-utils';
+
+// gen_ai spans are emitted as a separate span-v2 envelope item (not inline on the transaction), so we
+// assert on the streamed spans. Attribute values are wrapped as `{ value, type }` in the v2 format.
+test('Instruments anthropic-ai automatically via orchestrion', async ({ baseURL }) => {
+  const spansPromise = waitForStreamedSpans('nextjs-16-orchestrion', spans =>
+    spans.some(span => span.attributes['sentry.origin']?.value === 'auto.ai.orchestrion.anthropic'),
+  );
+
+  await fetch(`${baseURL}/api/anthropic`);
+
+  const spans = await spansPromise;
+
+  const chatSpan = spans.find(span => span.name === 'chat claude-3-haiku-20240307');
+  expect(chatSpan).toBeDefined();
+  expect(chatSpan?.attributes['sentry.op']?.value).toBe('gen_ai.chat');
+  expect(chatSpan?.attributes['sentry.origin']?.value).toBe('auto.ai.orchestrion.anthropic');
+  expect(chatSpan?.attributes['gen_ai.system']?.value).toBe('anthropic');
+  expect(chatSpan?.attributes['gen_ai.request.model']?.value).toBe('claude-3-haiku-20240307');
+  expect(chatSpan?.attributes['gen_ai.usage.input_tokens']?.value).toBe(10);
+  expect(chatSpan?.attributes['gen_ai.usage.output_tokens']?.value).toBe(15);
+});

dev-packages/e2e-tests/test-applications/nextjs-16-orchestrion/tests/google-genai.test.ts

@@ -0,0 +1,24 @@
+import { expect, test } from '@playwright/test';
+import { waitForStreamedSpans } from '@sentry-internal/test-utils';
+
+// gen_ai spans are emitted as a separate span-v2 envelope item (not inline on the transaction), so we
+// assert on the streamed spans. Attribute values are wrapped as `{ value, type }` in the v2 format.
+test('Instruments google-genai automatically via orchestrion', async ({ baseURL }) => {
+  const spansPromise = waitForStreamedSpans('nextjs-16-orchestrion', spans =>
+    spans.some(span => span.attributes['sentry.origin']?.value === 'auto.ai.orchestrion.google_genai'),
+  );
+
+  await fetch(`${baseURL}/api/google-genai`);
+
+  const spans = await spansPromise;
+
+  const generateSpan = spans.find(span => span.name === 'generate_content gemini-1.5-flash');
+  expect(generateSpan).toBeDefined();
+  expect(generateSpan?.attributes['sentry.op']?.value).toBe('gen_ai.generate_content');
+  expect(generateSpan?.attributes['sentry.origin']?.value).toBe('auto.ai.orchestrion.google_genai');
+  expect(generateSpan?.attributes['gen_ai.system']?.value).toBe('google_genai');
+  expect(generateSpan?.attributes['gen_ai.request.model']?.value).toBe('gemini-1.5-flash');
+  expect(generateSpan?.attributes['gen_ai.usage.input_tokens']?.value).toBe(8);
+  expect(generateSpan?.attributes['gen_ai.usage.output_tokens']?.value).toBe(12);
+  expect(generateSpan?.attributes['gen_ai.usage.total_tokens']?.value).toBe(20);
+});

dev-packages/e2e-tests/test-applications/nextjs-16-orchestrion/tests/openai.test.ts

@@ -0,0 +1,24 @@
+import { expect, test } from '@playwright/test';
+import { waitForStreamedSpans } from '@sentry-internal/test-utils';
+
+// gen_ai spans are emitted as a separate span-v2 envelope item (not inline on the transaction), so we
+// assert on the streamed spans. Attribute values are wrapped as `{ value, type }` in the v2 format.
+test('Instruments openai automatically via orchestrion', async ({ baseURL }) => {
+  const spansPromise = waitForStreamedSpans('nextjs-16-orchestrion', spans =>
+    spans.some(span => span.attributes['sentry.origin']?.value === 'auto.ai.orchestrion.openai'),
+  );
+
+  await fetch(`${baseURL}/api/openai`);
+
+  const spans = await spansPromise;
+
+  const chatSpan = spans.find(span => span.name === 'chat gpt-3.5-turbo');
+  expect(chatSpan).toBeDefined();
+  expect(chatSpan?.attributes['sentry.op']?.value).toBe('gen_ai.chat');
+  expect(chatSpan?.attributes['sentry.origin']?.value).toBe('auto.ai.orchestrion.openai');
+  expect(chatSpan?.attributes['gen_ai.system']?.value).toBe('openai');
+  expect(chatSpan?.attributes['gen_ai.request.model']?.value).toBe('gpt-3.5-turbo');
+  expect(chatSpan?.attributes['gen_ai.usage.input_tokens']?.value).toBe(10);
+  expect(chatSpan?.attributes['gen_ai.usage.output_tokens']?.value).toBe(15);
+  expect(chatSpan?.attributes['gen_ai.usage.total_tokens']?.value).toBe(25);
+});