feat(ts-sdk): expose llmProvider for bring-your-own-provider injection

sdks/typescript/src/batch/evaluator.ts

@@ -116,6 +116,7 @@ export class BatchEvaluator {
         maxRetries: this.config.maxRetries,
         telemetry: this.config.telemetry,
         modelOverride: this.config.modelOverride,
+        llmProvider: this.config.llmProvider,
       });
 
       this.evaluatorInstances.set(id, evaluator);

sdks/typescript/src/batch/types.ts

@@ -2,6 +2,7 @@
  * Batch evaluation types
  */
 import type { TelemetryOptions, ModelOverride } from '../evaluators/base.js';
+import type { LLMProvider } from '../providers/index.js';
 
 /**
  * Input row from CSV
@@ -87,4 +88,11 @@ export interface BatchConfig {
   telemetry?: boolean | TelemetryOptions;
   bypassRowLimit?: boolean;
   modelOverride?: ModelOverride;
+
+  /**
+   * Bring your own LLM provider for every evaluator in the batch.
+   * See {@link BaseEvaluatorConfig.llmProvider}. When set, no API keys are
+   * required; mutually exclusive with modelOverride (setting both throws).
+   */
+  llmProvider?: LLMProvider;
 }

sdks/typescript/src/evaluators/base.ts

@@ -85,6 +85,22 @@ export interface BaseEvaluatorConfig {
    */
   modelOverride?: ModelOverride;
 
+  /**
+   * Bring your own LLM provider.
+   *
+   * Inject any object implementing {@link LLMProvider} and the evaluator routes
+   * every LLM call through it, skipping the built-in OpenAI/Google/Anthropic
+   * API-key adapters entirely. Use this to run the evaluators on a provider you
+   * already have wired — Google Vertex AI, Amazon Bedrock, an AI-SDK gateway,
+   * or an eval framework's model system — with no separate API keys.
+   *
+   * When set, API-key validation is skipped (the injected provider manages its
+   * own model, auth, and retries). It is mutually exclusive with
+   * {@link BaseEvaluatorConfig.modelOverride}: setting both throws
+   * `ConfigurationError`. Ambient API keys, if present, are simply unused.
+   */
+  llmProvider?: LLMProvider;
+
   /**
    * Maximum number of retries for failed API calls (default: 2)
    * Set to 0 to disable retries.
@@ -161,6 +177,7 @@ export abstract class BaseEvaluator {
     googleApiKey?: string;
     openaiApiKey?: string;
     anthropicApiKey?: string;
+    llmProvider?: LLMProvider;
   };
 
   /**
@@ -192,11 +209,29 @@ export abstract class BaseEvaluator {
     // Initialize logger
     this.logger = createLogger(config.logger, config.logLevel ?? LogLevel.WARN);
 
-    // Validate modelOverride shape before key checks
-    this.validateModelOverride(config);
+    // An injected llmProvider replaces the built-in adapters entirely.
+    // Treat "provided" as `!== undefined` so a falsy-but-present value (e.g.
+    // `null` from an untyped JS caller) fails fast with an llmProvider error
+    // rather than falling through to a misleading "missing API key" error.
+    if (config.llmProvider !== undefined) {
+      this.validateLlmProvider(config.llmProvider);
+
+      // modelOverride is a contradictory directive — it configures a built-in
+      // adapter that llmProvider bypasses. Fail fast rather than silently
+      // ignoring one. (API keys, by contrast, are ambient and simply unused.)
+      if (config.modelOverride) {
+        throw new ConfigurationError(
+          'Cannot set both llmProvider and modelOverride: llmProvider replaces the built-in provider entirely, so modelOverride does not apply. Set one or the other.'
+        );
+      }
+      // It manages its own model and auth, so API-key and override checks do not apply.
+    } else {
+      // Validate modelOverride shape before key checks
+      this.validateModelOverride(config);
 
-    // Validate required API keys based on metadata
-    this.validateApiKeys(config);
+      // Validate required API keys based on metadata
+      this.validateApiKeys(config);
+    }
 
     // Normalize telemetry config
     const telemetryConfig = this.normalizeTelemetryConfig(config.telemetry);
@@ -209,6 +244,7 @@ export abstract class BaseEvaluator {
       googleApiKey: config.googleApiKey,
       openaiApiKey: config.openaiApiKey,
       anthropicApiKey: config.anthropicApiKey,
+      llmProvider: config.llmProvider,
     };
 
     if (config.modelOverride) {
@@ -244,6 +280,30 @@ export abstract class BaseEvaluator {
     return meta;
   }
 
+  /**
+   * Validate that an injected llmProvider implements the LLMProvider contract:
+   * a string `label` and `generateStructured` / `generateText` methods.
+   * @throws {ConfigurationError} If the provider is missing required members
+   */
+  private validateLlmProvider(provider: unknown): void {
+    if (provider === null || typeof provider !== 'object') {
+      throw new ConfigurationError(
+        'llmProvider must be an object implementing the LLMProvider interface; received ' +
+          (provider === null ? 'null' : typeof provider) + '.'
+      );
+    }
+    const p = provider as Partial<LLMProvider>;
+    if (
+      typeof p.label !== 'string' ||
+      typeof p.generateStructured !== 'function' ||
+      typeof p.generateText !== 'function'
+    ) {
+      throw new ConfigurationError(
+        'llmProvider must implement the LLMProvider interface: a string `label` and `generateStructured` / `generateText` methods.'
+      );
+    }
+  }
+
   /**
    * Validate modelOverride shape: provider must be a known Provider value and
    * model must be a non-empty string.
@@ -408,15 +468,19 @@ export abstract class BaseEvaluator {
   }
 
   /**
-   * Create an LLM provider, honouring modelOverride if set.
-   * When override is active, the key for the override provider is resolved
-   * from the matching top-level config field (e.g. anthropicApiKey for Anthropic).
+   * Create an LLM provider, honouring llmProvider then modelOverride if set.
+   * An injected llmProvider wins outright and is returned as-is. Otherwise, when
+   * override is active, the key for the override provider is resolved from the
+   * matching top-level config field (e.g. anthropicApiKey for Anthropic).
    */
   protected createConfiguredProvider(
     defaultType: Provider,
     defaultModel: string,
     defaultApiKey: string | undefined,
   ): LLMProvider {
+    if (this.config.llmProvider) {
+      return this.config.llmProvider;
+    }
     const override = this.config.modelOverride;
     if (override) {
       const apiKeyFor: Record<Provider, string | undefined> = {

sdks/typescript/tests/unit/batch/llm-provider.test.ts

@@ -0,0 +1,94 @@
+import { describe, it, expect, vi } from 'vitest';
+import { getAvailableGroups, BatchEvaluator } from '../../../src/batch/index.js';
+import type { BatchInput } from '../../../src/batch/index.js';
+import type { LLMProvider } from '../../../src/providers/base.js';
+
+/**
+ * Unit tests for `BatchConfig.llmProvider` (bring-your-own-provider at the batch
+ * level). Asserts that the injected provider is plumbed to every evaluator in a
+ * group and that a batch can run with no API keys — the batch equivalent of the
+ * single-evaluator coverage in tests/unit/evaluators/llm-provider.test.ts.
+ */
+
+const GROUP = getAvailableGroups().find((g) => g.id === 'text-complexity')!;
+
+function makeFakeProvider(label = 'vertex:gemini-2.5-pro') {
+  const generateStructured = vi.fn().mockResolvedValue({
+    data: {
+      grade: '6-8',
+      alternative_grade: '4-5',
+      scaffolding_needed: 'Pre-teach vocabulary',
+      reasoning: 'Middle-school-appropriate prose.',
+    },
+    model: label,
+    usage: { inputTokens: 100, outputTokens: 50 },
+    latencyMs: 10,
+  });
+  const provider: LLMProvider = {
+    label,
+    generateStructured,
+    generateText: vi
+      .fn()
+      .mockResolvedValue({ text: '', usage: { inputTokens: 0, outputTokens: 0 }, latencyMs: 0 }),
+  };
+  return { provider, generateStructured };
+}
+
+function makeInputs(count: number): BatchInput[] {
+  return Array.from({ length: count }, (_, i) => ({
+    text:
+      'The mitochondria is the powerhouse of the cell. It produces energy through a process ' +
+      'called cellular respiration in eukaryotic organisms.',
+    grade: '6-8',
+    rowIndex: i + 2,
+    originalRow: { text: 'sample', grade: '6-8' },
+  }));
+}
+
+/** Read the private evaluator instances the batch lazily constructs. */
+function instancesOf(
+  batch: BatchEvaluator
+): Map<string, { config?: { llmProvider?: LLMProvider } }> {
+  return (
+    batch as unknown as {
+      evaluatorInstances: Map<string, { config?: { llmProvider?: LLMProvider } }>;
+    }
+  ).evaluatorInstances;
+}
+
+describe('BatchConfig.llmProvider — bring-your-own-provider', () => {
+  it('constructs and runs with no API keys when llmProvider is set', async () => {
+    const { provider, generateStructured } = makeFakeProvider();
+    const batch = new BatchEvaluator({ llmProvider: provider, telemetry: false });
+
+    const { summary } = await batch.evaluate(makeInputs(1), GROUP.id);
+
+    // One task per evaluator in the group, all attempted (no missing-key crash).
+    expect(summary.totalTasks).toBe(GROUP.evaluatorIds.length);
+    expect(generateStructured).toHaveBeenCalled();
+  });
+
+  it('plumbs the injected provider into every evaluator in the group', async () => {
+    const { provider } = makeFakeProvider();
+    const batch = new BatchEvaluator({ llmProvider: provider, telemetry: false });
+
+    await batch.evaluate(makeInputs(1), GROUP.id);
+
+    const instances = instancesOf(batch);
+    expect(instances.size).toBe(GROUP.evaluatorIds.length);
+    for (const id of GROUP.evaluatorIds) {
+      expect(instances.get(id)?.config?.llmProvider).toBe(provider);
+    }
+  });
+
+  it('never reports a missing-API-key error when llmProvider is set', async () => {
+    const { provider } = makeFakeProvider();
+    const batch = new BatchEvaluator({ llmProvider: provider, telemetry: false });
+
+    const { results } = await batch.evaluate(makeInputs(1), GROUP.id);
+
+    for (const r of results) {
+      expect(r.error ?? '').not.toMatch(/API key/i);
+    }
+  });
+});

sdks/typescript/tests/unit/evaluators/llm-provider.test.ts

@@ -0,0 +1,120 @@
+import { describe, it, expect, vi } from 'vitest';
+import { GradeLevelAppropriatenessEvaluator } from '../../../src/evaluators/grade-level-appropriateness.js';
+import { Provider } from '../../../src/evaluators/base.js';
+import { ConfigurationError } from '../../../src/errors.js';
+import type { LLMProvider } from '../../../src/providers/base.js';
+
+/**
+ * Unit tests for the `llmProvider` config option (bring-your-own-provider).
+ *
+ * Mirrors the Python SDK's `llm_provider` injection: implement the provider
+ * interface, inject the instance, and the evaluator routes every LLM call
+ * through it — no built-in API keys, no createProvider factory.
+ *
+ * These tests deliberately do NOT mock `createProvider`: a correctly injected
+ * `llmProvider` must bypass it entirely, so the fake provider below is the only
+ * model path exercised.
+ */
+
+const GLA_DATA = {
+  grade: '6-8',
+  alternative_grade: '4-5',
+  scaffolding_needed: 'Pre-teach vocabulary; use diagrams',
+  reasoning: 'Middle-school-appropriate science prose.',
+};
+
+const SAMPLE_TEXT =
+  'The mitochondria is the powerhouse of the cell. It produces energy through a process ' +
+  'called cellular respiration in eukaryotic organisms.';
+
+function makeFakeProvider(label = 'vertex:gemini-2.5-pro') {
+  const generateStructured = vi.fn().mockResolvedValue({
+    data: GLA_DATA,
+    model: label,
+    usage: { inputTokens: 100, outputTokens: 50 },
+    latencyMs: 10,
+  });
+  const provider: LLMProvider = {
+    label,
+    generateStructured,
+    generateText: vi.fn().mockResolvedValue({
+      text: '',
+      usage: { inputTokens: 0, outputTokens: 0 },
+      latencyMs: 0,
+    }),
+  };
+  return { provider, generateStructured };
+}
+
+describe('llmProvider — bring-your-own-provider', () => {
+  it('constructs with no API key when llmProvider is set', () => {
+    const { provider } = makeFakeProvider();
+    expect(
+      () => new GradeLevelAppropriatenessEvaluator({ llmProvider: provider, telemetry: false })
+    ).not.toThrow();
+  });
+
+  it('routes evaluation through the injected provider', async () => {
+    const { provider, generateStructured } = makeFakeProvider();
+    const evaluator = new GradeLevelAppropriatenessEvaluator({
+      llmProvider: provider,
+      telemetry: false,
+    });
+
+    const result = await evaluator.evaluate(SAMPLE_TEXT);
+
+    expect(generateStructured).toHaveBeenCalledTimes(1);
+    expect(result.score).toBe('6-8');
+    expect(result.metadata.model).toBe('vertex:gemini-2.5-pro');
+    // The injected provider is the one in use, not a createProvider-built one.
+    // @ts-expect-error accessing private property for testing
+    expect(evaluator.provider).toBe(provider);
+  });
+
+  it('tolerates ambient API keys (they are simply unused)', () => {
+    const { provider } = makeFakeProvider();
+    expect(
+      () =>
+        new GradeLevelAppropriatenessEvaluator({
+          llmProvider: provider,
+          googleApiKey: 'ambient-key-from-env',
+          telemetry: false,
+        })
+    ).not.toThrow();
+  });
+
+  it('throws when both llmProvider and modelOverride are set', () => {
+    const { provider } = makeFakeProvider();
+    expect(
+      () =>
+        new GradeLevelAppropriatenessEvaluator({
+          llmProvider: provider,
+          modelOverride: { provider: Provider.OpenAI, model: 'gpt-4o' },
+          telemetry: false,
+        })
+    ).toThrow(ConfigurationError);
+  });
+
+  describe('rejects a malformed llmProvider at construction', () => {
+    const cases: Array<[string, unknown]> = [
+      ['null', null],
+      ['a primitive', 'not-a-provider'],
+      ['empty object', {}],
+      ['missing generateText', { label: 'x', generateStructured: vi.fn() }],
+      ['missing generateStructured', { label: 'x', generateText: vi.fn() }],
+      ['non-string label', { label: 5, generateStructured: vi.fn(), generateText: vi.fn() }],
+      ['method is not a function', { label: 'x', generateStructured: 'nope', generateText: vi.fn() }],
+    ];
+
+    it.each(cases)('throws ConfigurationError — %s', (_name, bad) => {
+      expect(
+        () =>
+          new GradeLevelAppropriatenessEvaluator({
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            llmProvider: bad as any,
+            telemetry: false,
+          })
+      ).toThrow(ConfigurationError);
+    });
+  });
+});