diff --git a/README.md b/README.md
index 93928b18ec..9bd09e9bf3 100644
--- a/README.md
+++ b/README.md
@@ -14,7 +14,7 @@ projects.
 - [@appland/validate](https://github.com/getappmap/appmap-js/tree/main/packages/validate)
 
 Looking for the AppMap client for JavaScript? Find it here:
-https://github.com/getappmap/appmap-agent-js
+https://github.com/getappmap/appmap-node
 
 ## Development
 
diff --git a/packages/cli/package.json b/packages/cli/package.json
index 96ba21b2ab..f8eacfa87c 100644
--- a/packages/cli/package.json
+++ b/packages/cli/package.json
@@ -38,7 +38,6 @@
     "@eslint/js": "~8.57.0",
     "@octokit/types": "^11.1.0",
     "@swc/core": "^1.3.76",
-    "@types/better-sqlite3": "^7.6.11",
     "@types/cors": "^2.8.17",
     "@types/fs-extra": "^9.0.13",
     "@types/gitconfiglocal": "^2.0.1",
@@ -103,7 +102,6 @@
     "applicationinsights": "^2.1.4",
     "async": "^3.2.4",
     "axios": "^0.27.2",
-    "better-sqlite3": "^11.5.0",
     "body-parser": "^1.20.2",
     "boxen": "^5.0.1",
     "chalk": "^4.1.2",
@@ -131,6 +129,7 @@
     "lunr": "^2.3.9",
     "minimatch": "^5.1.2",
     "moo": "^0.5.1",
+    "node-sqlite3-wasm": "^0.8.34",
     "open": "^8.2.1",
     "openapi-diff": "^0.23.6",
     "openapi-types": "^12.1.3",
@@ -167,9 +166,7 @@
       "resources",
       "built/appmap.html",
       "built/sequenceDiagram.html",
-      "built/docs",
-      "../../node_modules/better-sqlite3/build/Release/better_sqlite3.node",
-      "node_modules/better-sqlite3/build/Release/better_sqlite3.node"
+      "built/docs"
     ],
     "outputPath": "dist"
   },
diff --git a/packages/cli/src/cmds/index/rpc.ts b/packages/cli/src/cmds/index/rpc.ts
index 5a71f7c6d0..8c605f67f5 100644
--- a/packages/cli/src/cmds/index/rpc.ts
+++ b/packages/cli/src/cmds/index/rpc.ts
@@ -81,6 +81,13 @@ export function rpcMethods(navie: INavieProvider, codeEditor?: string): RpcHandl
 
 export const handler = async (argv: HandlerArguments) => {
   verbose(argv.verbose);
+  const observer = new PerformanceObserver((list) => {
+    for (const entry of list.getEntries()) {
+      const duration = entry.duration;
+      console.log(`${entry.name}: ${duration.toFixed(0)} ms`);
+    }
+  });
+  observer.observe({ type: 'measure' });
 
   const navie = buildNavieProvider(argv);
   let codeEditor: string | undefined = argv.codeEditor;
diff --git a/packages/cli/src/cmds/record/action/guessTestCommands.ts b/packages/cli/src/cmds/record/action/guessTestCommands.ts
index 8321574223..b0d76c1fbd 100644
--- a/packages/cli/src/cmds/record/action/guessTestCommands.ts
+++ b/packages/cli/src/cmds/record/action/guessTestCommands.ts
@@ -34,14 +34,14 @@ const TestCommands: TestCommandGuess[] = [
   {
     paths: ['package.json', 'node_modules/mocha'],
     command: {
-      command: 'npx @appland/appmap-agent-js --recorder=mocha -- npx mocha',
+      command: 'npx appmap-node npx mocha',
       env: {},
     },
   },
   {
     paths: ['package.json', 'node_modules/@jest'],
     command: {
-      command: 'npx @appland/appmap-agent-js --recorder=jest -- npx jest',
+      command: 'npx appmap-node npx jest',
       env: {},
     },
   },
diff --git a/packages/cli/src/cmds/record/state/agentProcessNotRunning.ts b/packages/cli/src/cmds/record/state/agentProcessNotRunning.ts
index 872f6adaeb..1e06bd17fb 100644
--- a/packages/cli/src/cmds/record/state/agentProcessNotRunning.ts
+++ b/packages/cli/src/cmds/record/state/agentProcessNotRunning.ts
@@ -24,7 +24,7 @@ export default async function agentProcessNotRunning(context: RecordContext): Pr
   Django:     https://appland.com/docs/reference/appmap-python.html#django
   Flask:      https://appland.com/docs/reference/appmap-python.html#flask
   Java:       https://appland.com/docs/reference/appmap-java.html#remote-recording
-  JavaScript: https://appland.com/docs/reference/appmap-agent-js.html#remote-recording
+  JavaScript: https://appmap.io/docs/reference/appmap-node.html#remote-recording
   `);
 
   await configureHostAndPort(context);
diff --git a/packages/cli/src/cmds/search/search.ts b/packages/cli/src/cmds/search/search.ts
index c7370a40f0..5f7f640582 100644
--- a/packages/cli/src/cmds/search/search.ts
+++ b/packages/cli/src/cmds/search/search.ts
@@ -1,5 +1,5 @@
+import sqlite3 from 'node-sqlite3-wasm';
 import yargs from 'yargs';
-import sqlite3 from 'better-sqlite3';
 import assert from 'assert';
 import { readFileSync } from 'fs';
 import { writeFile } from 'fs/promises';
@@ -186,7 +186,7 @@ export const handler = async (argv: ArgumentTypes) => {
     };
 
     const index = await buildIndexInTempDir('appmaps', async (indexFile) => {
-      const db = new sqlite3(indexFile);
+      const db = new sqlite3.Database(indexFile);
       const fileIndex = new FileIndex(db);
       await buildAppMapIndex(fileIndex, [process.cwd()]);
       return fileIndex;
diff --git a/packages/cli/src/rpc/explain/collect-location-context.ts b/packages/cli/src/rpc/explain/collect-location-context.ts
index d205e76a1d..a5cb1c008d 100644
--- a/packages/cli/src/rpc/explain/collect-location-context.ts
+++ b/packages/cli/src/rpc/explain/collect-location-context.ts
@@ -140,14 +140,22 @@ async function directoryContextItem(
   };
 }
 
-async function* listDirectory(path: string, depth: number): AsyncGenerator<string> {
+const MAX_SUBENTRIES = 100;
+
+export async function* listDirectory(path: string, depth: number): AsyncGenerator<string> {
   const entries = await readdir(path, { withFileTypes: true });
   for (const entry of entries) {
     const entryPath = join(path, entry.name);
     if (entry.isDirectory()) {
       if (depth > 0) {
-        yield `${entry.name}/`;
-        for await (const subentry of listDirectory(entryPath, depth - 1)) yield `\t${subentry}`;
+        const subentries: string[] = [];
+        for await (const subentry of listDirectory(entryPath, depth - 1)) subentries.push(subentry);
+        if (subentries.length <= MAX_SUBENTRIES) {
+          yield `${entry.name}/`;
+          for (const subentry of subentries) yield `\t${subentry}`;
+        } else {
+          yield `${entry.name}/ (${subentries.length} entries)`;
+        }
       } else yield `${entry.name}/ (${(await readdir(entryPath)).length} entries)`;
     } else if (entry.isFile()) {
       yield entry.name;
diff --git a/packages/cli/src/rpc/explain/collect-search-context.ts b/packages/cli/src/rpc/explain/collect-search-context.ts
index 71827c69a5..e7f57404f0 100644
--- a/packages/cli/src/rpc/explain/collect-search-context.ts
+++ b/packages/cli/src/rpc/explain/collect-search-context.ts
@@ -1,4 +1,4 @@
-import { log } from 'console';
+import makeDebug from 'debug';
 
 import { ContextV2, applyContext } from '@appland/navie';
 import { SearchRpc } from '@appland/rpc';
@@ -10,6 +10,8 @@ import { searchProjectFiles } from './index/project-file-index';
 import { buildProjectFileSnippetIndex } from './index/project-file-snippet-index';
 import { generateSessionId } from '@appland/search';
 
+const debug = makeDebug('appmap:cli:search-context');
+
 export type SearchContextRequest = {
   appmaps?: string[];
   excludePatterns?: RegExp[];
@@ -69,7 +71,7 @@ export default async function collectSearchContext(
       type: 'appmap',
     };
 
-    log(`[search-context] Matched ${selectedAppMaps.results.length} AppMaps.`);
+    debug(`Matched ${selectedAppMaps.results.length} AppMaps.`);
   }
 
   const fileSearchResults = await searchProjectFiles(
@@ -95,23 +97,23 @@ export default async function collectSearchContext(
   try {
     let charCount = 0;
     let maxEventsPerDiagram = 5;
-    log(`[search-context] Requested char limit: ${charLimit}`);
+    debug(`Requested char limit: ${charLimit}`);
     for (;;) {
-      log(`[search-context] Collecting context with ${maxEventsPerDiagram} events per diagram.`);
+      debug(`Collecting context with ${maxEventsPerDiagram} events per diagram.`);
 
       contextCandidate = await snippetIndex.search(maxEventsPerDiagram, charLimit, request);
       const appliedContext = applyContext(contextCandidate.context, charLimit);
       const appliedContextSize = appliedContext.reduce((acc, item) => acc + item.content.length, 0);
       contextCandidate.context = appliedContext;
       contextCandidate.contextSize = appliedContextSize;
-      log(`[search-context] Collected an estimated ${appliedContextSize} characters.`);
+      debug(`Collected an estimated ${appliedContextSize} characters.`);
 
       if (appliedContextSize === charCount || appliedContextSize > charLimit) {
         break;
       }
       charCount = appliedContextSize;
       maxEventsPerDiagram = Math.ceil(maxEventsPerDiagram * 1.5);
-      log(`[search-context] Increasing max events per diagram to ${maxEventsPerDiagram}.`);
+      debug(`Increasing max events per diagram to ${maxEventsPerDiagram}.`);
     }
   } finally {
     snippetIndex.close();
diff --git a/packages/cli/src/rpc/explain/index-files.ts b/packages/cli/src/rpc/explain/index-files.ts
index 2543e5eac2..1838dc41cd 100644
--- a/packages/cli/src/rpc/explain/index-files.ts
+++ b/packages/cli/src/rpc/explain/index-files.ts
@@ -1,4 +1,4 @@
-import sqlite3 from 'better-sqlite3';
+import type sqlite3 from 'node-sqlite3-wasm';
 
 import {
   buildFileIndex,
diff --git a/packages/cli/src/rpc/explain/index-snippets.ts b/packages/cli/src/rpc/explain/index-snippets.ts
index ceb2d58f01..da744e965a 100644
--- a/packages/cli/src/rpc/explain/index-snippets.ts
+++ b/packages/cli/src/rpc/explain/index-snippets.ts
@@ -1,3 +1,5 @@
+import sqlite3 from 'node-sqlite3-wasm';
+
 import {
   buildSnippetIndex,
   FileSearchResult,
@@ -6,7 +8,6 @@ import {
   readFileSafe,
   SnippetIndex,
 } from '@appland/search';
-import sqlite3 from 'better-sqlite3';
 
 export default async function indexSnippets(
   db: sqlite3.Database,
diff --git a/packages/cli/src/rpc/explain/index/appmap-file-index.ts b/packages/cli/src/rpc/explain/index/appmap-file-index.ts
index aaf6aa1bd6..bc152e3c09 100644
--- a/packages/cli/src/rpc/explain/index/appmap-file-index.ts
+++ b/packages/cli/src/rpc/explain/index/appmap-file-index.ts
@@ -1,4 +1,4 @@
-import sqlite3 from 'better-sqlite3';
+import sqlite3 from 'node-sqlite3-wasm';
 
 import { FileIndex, SessionId } from '@appland/search';
 
@@ -10,7 +10,7 @@ export async function buildAppMapFileIndex(
   appmapDirectories: string[]
 ): Promise<CloseableIndex<FileIndex>> {
   return await buildIndexInTempDir<FileIndex>('appmaps', async (indexFile) => {
-    const db = new sqlite3(indexFile);
+    const db = new sqlite3.Database(indexFile);
     const fileIndex = new FileIndex(db);
     await buildAppMapIndex(fileIndex, appmapDirectories);
     return fileIndex;
diff --git a/packages/cli/src/rpc/explain/index/project-file-index.ts b/packages/cli/src/rpc/explain/index/project-file-index.ts
index c9b44c2b65..235a0e2d84 100644
--- a/packages/cli/src/rpc/explain/index/project-file-index.ts
+++ b/packages/cli/src/rpc/explain/index/project-file-index.ts
@@ -1,5 +1,5 @@
-import sqlite3 from 'better-sqlite3';
 import makeDebug from 'debug';
+import sqlite3 from 'node-sqlite3-wasm';
 
 import {
   buildFileIndex,
@@ -71,7 +71,7 @@ export async function buildProjectFileIndex(
   excludePatterns: RegExp[] | undefined
 ): Promise<CloseableIndex<FileIndex>> {
   return await buildIndexInTempDir('files', async (indexFile) => {
-    const db = new sqlite3(indexFile);
+    const db = new sqlite3.Database(indexFile);
     return await indexFiles(db, sourceDirectories, includePatterns, excludePatterns);
   });
 }
diff --git a/packages/cli/src/rpc/explain/index/project-file-snippet-index.ts b/packages/cli/src/rpc/explain/index/project-file-snippet-index.ts
index 8b97cdaac1..9fdfbc5d89 100644
--- a/packages/cli/src/rpc/explain/index/project-file-snippet-index.ts
+++ b/packages/cli/src/rpc/explain/index/project-file-snippet-index.ts
@@ -1,4 +1,5 @@
-import sqlite3 from 'better-sqlite3';
+import sqlite3 from 'node-sqlite3-wasm';
+
 import { FileSearchResult, SessionId, SnippetIndex } from '@appland/search';
 
 import buildIndexInTempDir, { CloseableIndex } from './build-index-in-temp-dir';
@@ -76,7 +77,7 @@ export async function buildProjectFileSnippetIndex(
   fileSearchResults: FileSearchResult[]
 ): Promise<ProjectFileSnippetIndex> {
   const snippetIndex = await buildIndexInTempDir('snippets', async (indexFile) => {
-    const db = new sqlite3(indexFile);
+    const db = new sqlite3.Database(indexFile);
     return await indexSnippets(db, fileSearchResults);
   });
 
diff --git a/packages/cli/src/rpc/search/search.ts b/packages/cli/src/rpc/search/search.ts
index cbeb63c1e3..7fe32dfcbb 100644
--- a/packages/cli/src/rpc/search/search.ts
+++ b/packages/cli/src/rpc/search/search.ts
@@ -1,5 +1,7 @@
 import { isAbsolute, join } from 'path';
-import sqlite3 from 'better-sqlite3';
+
+import sqlite3 from 'node-sqlite3-wasm';
+
 import { FileIndex, generateSessionId } from '@appland/search';
 import { SearchRpc } from '@appland/rpc';
 
@@ -62,7 +64,7 @@ export async function handler(
     // Search across all AppMaps, creating a map from AppMap id to AppMapSearchResult
     const maxResults = options.maxDiagrams || options.maxResults || DEFAULT_MAX_DIAGRAMS;
     const index = await buildIndexInTempDir('appmaps', async (indexFile) => {
-      const db = new sqlite3(indexFile);
+      const db = new sqlite3.Database(indexFile);
       const fileIndex = new FileIndex(db);
       await buildAppMapIndex(
         fileIndex,
diff --git a/packages/cli/tests/unit/rpc/explain/collect-location-context.spec.ts b/packages/cli/tests/unit/rpc/explain/collect-location-context.spec.ts
index c3e12c6614..f2a72d6bdd 100644
--- a/packages/cli/tests/unit/rpc/explain/collect-location-context.spec.ts
+++ b/packages/cli/tests/unit/rpc/explain/collect-location-context.spec.ts
@@ -5,7 +5,9 @@ import { isBinaryFile } from '@appland/search';
 
 import Location from '../../../../src/rpc/explain/location';
 import ContentRestrictions from '../../../../src/rpc/explain/ContentRestrictions';
-import collectLocationContext from '../../../../src/rpc/explain/collect-location-context';
+import collectLocationContext, {
+  listDirectory,
+} from '../../../../src/rpc/explain/collect-location-context';
 
 jest.mock('node:fs/promises');
 jest.mock('@appland/search');
@@ -18,6 +20,39 @@ describe('collectLocationContext', () => {
     ContentRestrictions.instance.reset();
   });
 
+  describe('listDirectory', () => {
+    const mockReaddir = jest.spyOn(fs, 'readdir');
+
+    afterEach(() => {
+      jest.resetAllMocks();
+    });
+
+    it('lists files and directories up to the specified depth', async () => {
+      mockReaddir.mockResolvedValueOnce([mockDirent('file1.js'), mockDirent('dir1', true)]);
+      mockReaddir.mockResolvedValueOnce([mockDirent('file2.js'), mockDirent('file3.js')]);
+
+      const result: string[] = [];
+      for await (const entry of listDirectory('/test', 1)) {
+        result.push(entry);
+      }
+
+      expect(result).toEqual(['file1.js', 'dir1/', '\tfile2.js', '\tfile3.js']);
+    });
+
+    it('limits the number of subentries', async () => {
+      const subentries = Array.from({ length: 101 }, (_, i) => mockDirent(`file${i}.js`));
+      mockReaddir.mockResolvedValueOnce([mockDirent('dir1', true)]);
+      mockReaddir.mockResolvedValueOnce(subentries);
+
+      const result: string[] = [];
+      for await (const entry of listDirectory('/test', 1)) {
+        result.push(entry);
+      }
+
+      expect(result).toEqual(['dir1/ (101 entries)']);
+    });
+  });
+
   describe('with empty locations', () => {
     it('handles empty locations', async () => {
       const result = await collectLocationContext(sourceDirectories, []);
@@ -151,7 +186,9 @@ describe('collectLocationContext', () => {
 
     it('handles directory listings', async () => {
       stat.mockResolvedValue({ isDirectory: () => true, isFile: () => false } as Stats);
-      jest.spyOn(fs, 'readdir').mockResolvedValue(['file1.js', 'file2.js'].map(mockDirent));
+      jest
+        .spyOn(fs, 'readdir')
+        .mockResolvedValue(['file1.js', 'file2.js'].map((p) => mockDirent(p)));
 
       const result = await collectLocationContext(sourceDirectories, [Location.parse('/src:0')]);
       expect(result).toEqual([
@@ -208,11 +245,11 @@ describe('collectLocationContext', () => {
   });
 });
 
-function mockDirent(name: string): Dirent {
+function mockDirent(name: string, isDirectory = false): Dirent {
   return {
     name,
     isFile: () => true,
-    isDirectory: () => false,
+    isDirectory: () => isDirectory,
     isBlockDevice: () => false,
     isCharacterDevice: () => false,
     isSymbolicLink: () => false,
diff --git a/packages/components/src/lib/documentation.js b/packages/components/src/lib/documentation.js
index 2ebd86250f..110aa310ef 100644
--- a/packages/components/src/lib/documentation.js
+++ b/packages/components/src/lib/documentation.js
@@ -2,7 +2,7 @@ const documentationUrls = {
   ruby: 'https://appmap.io/docs/reference/appmap-ruby.html',
   python: 'https://appmap.io/docs/reference/appmap-python.html',
   java: 'https://appmap.io/docs/reference/appmap-java.html',
-  javascript: 'https://appmap.io/docs/reference/appmap-agent-js.html',
+  javascript: 'https://appmap.io/docs/reference/appmap-node.html',
 };
 
 export function getAgentDocumentationUrl(language) {
diff --git a/packages/navie/package.json b/packages/navie/package.json
index 0dabe15f02..6c1edf84b1 100644
--- a/packages/navie/package.json
+++ b/packages/navie/package.json
@@ -33,6 +33,7 @@
     "eslint-plugin-promise": "^6.0.1",
     "eslint-plugin-unicorn": "^39.0.0",
     "jest": "^29.7.0",
+    "prettier": "^3.3.3",
     "ts-jest": "^29.2.5",
     "ts-node": "^10.4.0",
     "tsc": "^2.0.3",
@@ -44,6 +45,7 @@
     "@langchain/google-vertexai-web": "^0.1.0",
     "@langchain/openai": "^0.2.7",
     "fast-xml-parser": "^4.4.0",
+    "js-tiktoken": "^1.0.18",
     "js-yaml": "^4.1.0",
     "jsdom": "^16.6.0",
     "langchain": "^0.2.16",
diff --git a/packages/navie/src/commands/explain-command.ts b/packages/navie/src/commands/explain-command.ts
index cf80b245c1..5208f66326 100644
--- a/packages/navie/src/commands/explain-command.ts
+++ b/packages/navie/src/commands/explain-command.ts
@@ -42,6 +42,8 @@ export default class ExplainCommand implements Command {
   ) {}
 
   async *execute(request: CommandRequest, chatHistory?: ChatHistory): AsyncIterable<string> {
+    performance.mark('start');
+
     const { question: baseQuestion, codeSelection } = request;
 
     const classifyEnabled = request.userOptions.isEnabled('classify', true);
@@ -49,6 +51,7 @@ export default class ExplainCommand implements Command {
 
     let contextLabelsFn: Promise<ContextV2.ContextLabel[]> | undefined;
 
+    performance.mark('classifyStart');
     if (classifyEnabled)
       contextLabelsFn = this.classifierService
         .classifyQuestion(baseQuestion, chatHistory)
@@ -56,6 +59,7 @@ export default class ExplainCommand implements Command {
           warn(`Error classifying question: ${err}`);
           return [];
         });
+    void contextLabelsFn?.then(() => performance.measure('classify', 'classifyStart'));
 
     let projectInfo: ProjectInfo[] = [];
     if (isProjectInfoEnabled(request.userOptions)) {
@@ -134,11 +138,13 @@ export default class ExplainCommand implements Command {
 
     if (projectInfo) this.projectInfoService.promptProjectInfo(isArchitecture, projectInfo);
 
+    performance.mark('agentStart');
     const agentResponse = await mode.perform(agentOptions, tokensAvailable);
     if (agentResponse) {
       yield agentResponse.response;
       if (agentResponse.abort) return;
     }
+    performance.measure('agent preparation', 'agentStart');
 
     if (codeSelection) this.codeSelectionService.addSystemPrompt();
 
@@ -165,7 +171,9 @@ export default class ExplainCommand implements Command {
     mode.applyQuestionPrompt(question);
 
     if (isGathererEnabled(request.userOptions, agentMode, contextLabels)) {
+      performance.mark('gathererStart');
       yield* this.gatherAdditionalInformation();
+      performance.measure('gatherer', 'gathererStart');
     }
 
     const { messages } = this.interactionHistory.buildState();
@@ -177,9 +185,15 @@ export default class ExplainCommand implements Command {
       )
     );
 
+    performance.mark('completionStart');
+    console.log('completion start');
     const response = this.completionService.complete(messages, { temperature: mode.temperature });
     if (mode.filter) yield* mode.filter(response);
     else yield* response;
+    console.log('completion end');
+    performance.measure('completion', 'completionStart');
+
+    performance.measure('total', 'start');
   }
 
   private async *gatherAdditionalInformation(maxSteps = 10) {
diff --git a/packages/navie/src/lib/max-tokens.ts b/packages/navie/src/lib/max-tokens.ts
new file mode 100644
index 0000000000..29abaccaa3
--- /dev/null
+++ b/packages/navie/src/lib/max-tokens.ts
@@ -0,0 +1,60 @@
+const MAX_TOKENS: Record<string, number> = {
+  // OpenAI
+  'gpt-3.5-turbo': 16_385,
+  'gpt-3.5-turbo-0125': 16_385,
+  'gpt-3.5-turbo-1106': 16_385,
+  'gpt-3.5-turbo-instruct': 4_096,
+  'gpt-4': 8_192,
+  'gpt-4-0314': 8_192,
+  'gpt-4-0613': 8_192,
+  'gpt-4-0125-preview': 128_000,
+  'gpt-4-1106-preview': 128_000,
+  'gpt-4-turbo-preview': 128_000,
+  'gpt-4-turbo-2024-04-09': 128_000,
+  'gpt-4-turbo': 128_000,
+  'gpt-4o-mini': 128_000,
+  'gpt-4o-mini-2024-07-18': 128_000,
+  'gpt-4o': 128_000,
+  'gpt-4o-2024-05-13': 128_000,
+  'gpt-4o-2024-08-06': 128_000,
+  'chatgpt-4o-latest': 128_000,
+  'o1-preview': 128_000,
+  'o1-preview-2024-09-12': 128_000,
+  'o1-mini': 128_000,
+  'o1-mini-2024-09-12': 128_000,
+
+  // Claude
+  // Note that these token limits are subject to change depending on server load.
+  'claude-instant-1.2': 100_000,
+  'claude-2.0': 100_000,
+  'claude-2.1': 200_000,
+  'claude-3-5-sonnet-20240620': 200_000,
+  'claude-3-opus-20240229': 200_000,
+  'claude-3-sonnet-20240229': 200_000,
+  'claude-3-haiku-20240307': 200_000,
+
+  // Gemini
+  'gemini-pro': 32_760,
+  'gemini-1.0-pro': 32_760,
+  'gemini-1.0-pro-latest': 32_760,
+  'gemini-1.0-pro-001': 32_760,
+  'gemini-1.0-pro-vision': 16_384,
+  'gemini-1.5-pro': 2_097_152,
+  'gemini-1.5-pro-latest': 2_097_152,
+  'gemini-1.5-pro-001': 2_097_152,
+  'gemini-1.5-pro-002': 2_097_152,
+  'gemini-1.5-pro-exp-0827': 2_097_152,
+  'gemini-1.5-flash': 1_048_576,
+  'gemini-1.5-flash-latest': 1_048_576,
+  'gemini-1.5-flash-001': 1_048_576,
+  'gemini-1.5-flash-002': 1_048_576,
+  'gemini-1.5-flash-8b-exp-0924': 1_048_576,
+  'gemini-1.5-flash-8b-exp-0827': 1_048_576,
+  'gemini-1.5-flash-exp-0827': 1_048_576,
+};
+
+const DEFAULT_MAX_TOKENS = 32_768;
+
+export default function maxTokens(model: string): number {
+  return MAX_TOKENS[model] ?? DEFAULT_MAX_TOKENS;
+}
diff --git a/packages/navie/src/lib/parse-options.ts b/packages/navie/src/lib/parse-options.ts
index 130d428b59..faa8a1295a 100644
--- a/packages/navie/src/lib/parse-options.ts
+++ b/packages/navie/src/lib/parse-options.ts
@@ -1,7 +1,5 @@
-import { warn } from 'console';
 import { ContextV2 } from '../context';
 
-/* eslint-disable no-continue */
 export class UserOptions {
   constructor(private options: Map<string, string | boolean>) {}
 
@@ -102,7 +100,6 @@ export default function parseOptions(question: string): {
       }
     }
 
-    warn(`User option ${key}=${value.toString()}`);
     options.set(key.toLowerCase(), value);
   }
 
diff --git a/packages/navie/src/lib/truncate-messages.ts b/packages/navie/src/lib/truncate-messages.ts
new file mode 100644
index 0000000000..5781b26b82
--- /dev/null
+++ b/packages/navie/src/lib/truncate-messages.ts
@@ -0,0 +1,33 @@
+import type Message from '../message';
+
+export default function truncateMessages(
+  messages: Message[],
+  currentTokens: number,
+  maxTokens: number
+): Message[] {
+  if (currentTokens <= maxTokens) return [...messages];
+  let largestMessageIndex = -1;
+  let totalContentLength = 0;
+  messages.forEach((message, index) => {
+    totalContentLength += message.content.length;
+
+    const maxLength = messages[largestMessageIndex]?.content?.length ?? -1;
+    if (message.role === 'user' && message.content.length > maxLength) {
+      largestMessageIndex = index;
+    }
+  });
+
+  if (largestMessageIndex === -1) return [...messages];
+
+  const largestMessage = messages[largestMessageIndex];
+  const reductionRatio = 1 - maxTokens / currentTokens;
+  const charsToRemove = Math.ceil(reductionRatio * totalContentLength);
+  const updatedMessage = { ...messages[largestMessageIndex] };
+  updatedMessage.content = largestMessage.content.slice(0, -charsToRemove);
+
+  return [
+    ...messages.slice(0, largestMessageIndex),
+    updatedMessage,
+    ...messages.slice(largestMessageIndex + 1),
+  ];
+}
diff --git a/packages/navie/src/services/anthropic-completion-service.ts b/packages/navie/src/services/anthropic-completion-service.ts
index ebb45625d8..404e737998 100644
--- a/packages/navie/src/services/anthropic-completion-service.ts
+++ b/packages/navie/src/services/anthropic-completion-service.ts
@@ -4,6 +4,7 @@ import { isNativeError } from 'node:util/types';
 import { ChatAnthropic } from '@langchain/anthropic';
 import { z } from 'zod';
 
+import maxTokens from '../lib/max-tokens';
 import Message from '../message';
 import CompletionService, {
   Completion,
@@ -15,7 +16,7 @@ import CompletionService, {
   Usage,
 } from './completion-service';
 import Trajectory from '../lib/trajectory';
-import MessageTokenReducerService from './message-token-reducer-service';
+import truncateMessages from '../lib/truncate-messages';
 
 /*
   Generated on https://docs.anthropic.com/en/docs/about-claude/models with
@@ -109,12 +110,13 @@ export default class AnthropicCompletionService implements CompletionService {
   constructor(
     public readonly modelName: string,
     public readonly temperature: number,
-    private trajectory: Trajectory,
-    private readonly messageTokenReducerService: MessageTokenReducerService
+    private trajectory: Trajectory
   ) {
     this.model = this.buildModel({ temperature });
+    this.tokenLimit = maxTokens(modelName);
   }
   model: ChatAnthropic;
+  tokenLimit: number;
 
   // Construct a model with non-default options. There doesn't seem to be a way to configure
   // the model parameters at invocation time like with OpenAI.
@@ -208,12 +210,11 @@ export default class AnthropicCompletionService implements CompletionService {
               apiError.error.type === 'invalid_request_error' &&
               apiError.error.message.includes('prompt is too long')
             ) {
-              warn('Context length exceeded. Reducing token count and retrying.');
-              sentMessages = await this.messageTokenReducerService.reduceMessageTokens(
-                sentMessages,
-                this.modelName,
-                { message: apiError.error.message }
-              );
+              const { message } = apiError.error;
+              const count = await this.countTokens(sentMessages, message);
+              const max = this.ratchetMaxTokens(count, message);
+              warn(`Reducing tokens from ${count} to ${max}`);
+              sentMessages = truncateMessages(sentMessages, count, max);
               shouldRetry = true;
             }
 
@@ -242,6 +243,30 @@ export default class AnthropicCompletionService implements CompletionService {
     warn(usage.toString());
     return usage;
   }
+
+  async countTokens(messages: Message[], errorMessage?: string): Promise<number> {
+    if (errorMessage) {
+      const countMatch = /prompt is too long: (\d+) tokens > \d+ maximum/.exec(errorMessage);
+      if (countMatch) return parseInt(countMatch[1], 10);
+    }
+
+    const counts = await Promise.all(messages.map((m) => this.model.getNumTokens(m.content)));
+    return counts.reduce((a, b) => a + b, 0);
+  }
+
+  private ratchetMaxTokens(overflowedCount: number, errorMessage?: string): number {
+    let maxTokens = Math.min(this.tokenLimit, overflowedCount * 0.9);
+
+    if (errorMessage) {
+      const countMatch = /prompt is too long: \d+ tokens > (\d+) maximum/.exec(errorMessage);
+      if (countMatch) maxTokens = Math.min(parseInt(countMatch[1], 10), maxTokens);
+      if (maxTokens === this.tokenLimit)
+        // something is wrong, add a margin
+        maxTokens *= 0.9;
+    }
+
+    return (this.tokenLimit = Math.floor(maxTokens));
+  }
 }
 
 function errorMessage(err: unknown): string {
diff --git a/packages/navie/src/services/completion-service-factory.ts b/packages/navie/src/services/completion-service-factory.ts
index 13395147e5..500d4bc424 100644
--- a/packages/navie/src/services/completion-service-factory.ts
+++ b/packages/navie/src/services/completion-service-factory.ts
@@ -5,7 +5,6 @@ import OpenAICompletionService from './openai-completion-service';
 import AnthropicCompletionService from './anthropic-completion-service';
 import CompletionService from './completion-service';
 import Trajectory from '../lib/trajectory';
-import MessageTokenReducerService from './message-token-reducer-service';
 
 interface Options {
   modelName: string;
@@ -45,7 +44,6 @@ export default function createCompletionService({
   trajectory,
   backend = determineCompletionBackend(),
 }: Options): CompletionService {
-  const messageTokenReducerService = new MessageTokenReducerService();
   warn(`Using completion service ${backend}`);
-  return new BACKENDS[backend](modelName, temperature, trajectory, messageTokenReducerService);
+  return new BACKENDS[backend](modelName, temperature, trajectory);
 }
diff --git a/packages/navie/src/services/message-token-reducer-service/index.ts b/packages/navie/src/services/message-token-reducer-service/index.ts
deleted file mode 100644
index b73f392fed..0000000000
--- a/packages/navie/src/services/message-token-reducer-service/index.ts
+++ /dev/null
@@ -1,92 +0,0 @@
-import ClaudeTokenCountErrorParser from './token-count-strategies/claude-token-count-error-parser';
-import OpenAiTokenCountErrorParser from './token-count-strategies/openai-token-count-error-parser';
-import ClaudeTokenLimitErrorParser from './token-limit-identifiers/claude-token-limit-error-parser';
-import OpenAiTokenLimitErrorParser from './token-limit-identifiers/openai-token-limit-error-parser';
-import MessageTruncationStrategy from './token-reduction-strategies/message-truncation-strategy';
-import EstimateTokenCount from './token-count-strategies/estimate-token-count';
-import LookupMaxTokens from './token-limit-identifiers/lookup-max-tokens';
-import type Message from '../../message';
-import type {
-  ApiError,
-  TokenCountStrategy,
-  TokenLimitIdentifier,
-  TokenReductionStrategy,
-} from './interfaces';
-import { CHARACTERS_PER_TOKEN } from '../../message';
-
-class TokenError extends Error {
-  constructor(message: string) {
-    super(message);
-  }
-}
-
-export default class MessageTokenReducerService {
-  private static readonly tokenLimitStrategies: TokenLimitIdentifier[] = [
-    /* 
-      Use errors before the lookup table. Context limits may be dynamic.
-      E.g., OpenAI may update a model pointer or Claude may throttle context limits
-      during peak usage (https://support.anthropic.com/en/articles/8241175-how-do-i-increase-my-usage-limits).
-    */
-    new OpenAiTokenLimitErrorParser(),
-    new ClaudeTokenLimitErrorParser(),
-    new LookupMaxTokens(),
-  ];
-
-  private static readonly tokenCountStrategies: TokenCountStrategy[] = [
-    new OpenAiTokenCountErrorParser(),
-    new ClaudeTokenCountErrorParser(),
-    new EstimateTokenCount(CHARACTERS_PER_TOKEN),
-  ];
-
-  private static readonly tokenReductionStrategies: TokenReductionStrategy[] = [
-    new MessageTruncationStrategy('user'),
-  ];
-
-  constructor(private readonly defaultTokenLimit = 32_768) {}
-
-  private getTokenLimit(model: string, apiError?: ApiError): number | undefined {
-    for (const strategy of MessageTokenReducerService.tokenLimitStrategies) {
-      const tokenLimit = strategy.getTokenLimit(model, apiError);
-      if (tokenLimit !== undefined) return tokenLimit;
-    }
-  }
-
-  private countTokens(messages: Message[], model: string, apiError?: ApiError): number | undefined {
-    for (const strategy of MessageTokenReducerService.tokenCountStrategies) {
-      const tokenCount = strategy.countTokens(messages, model, apiError);
-      if (tokenCount !== undefined) return tokenCount;
-    }
-  }
-
-  async reduceMessageTokens(
-    messages: Message[],
-    model: string,
-    apiError?: ApiError
-  ): Promise<Message[]> {
-    let tokenLimit = this.getTokenLimit(model, apiError);
-    if (!tokenLimit) {
-      // We can either throw an error or guess a relatively reasonable token limit.
-      // For now, we'll assume the default token limit.
-      tokenLimit = this.defaultTokenLimit;
-    }
-
-    let errorConsumed = false;
-    let result = [...messages];
-
-    for (const strategy of MessageTokenReducerService.tokenReductionStrategies) {
-      const tokenCount = this.countTokens(result, model, errorConsumed ? undefined : apiError);
-      if (tokenCount === undefined) {
-        throw new TokenError('Unable to count tokens');
-      }
-
-      if (tokenCount > tokenLimit) {
-        result = await strategy.reduceTokens(result, tokenCount, tokenLimit);
-        errorConsumed = true;
-      } else {
-        break;
-      }
-    }
-
-    return result;
-  }
-}
diff --git a/packages/navie/src/services/message-token-reducer-service/interfaces.ts b/packages/navie/src/services/message-token-reducer-service/interfaces.ts
deleted file mode 100644
index d3a7b2ad12..0000000000
--- a/packages/navie/src/services/message-token-reducer-service/interfaces.ts
+++ /dev/null
@@ -1,21 +0,0 @@
-import Message from '../../message';
-
-export interface ApiError {
-  message: string;
-}
-
-export interface TokenLimitIdentifier {
-  getTokenLimit(model: string, apiError?: ApiError): number | undefined;
-}
-
-export interface TokenCountStrategy {
-  countTokens(message: Message[], model: string, apiError?: ApiError): number | undefined;
-}
-
-export interface TokenReductionStrategy {
-  reduceTokens(
-    messages: Message[],
-    currentTokens: number,
-    maxTokens: number
-  ): Promise<Message[]> | Message[];
-}
diff --git a/packages/navie/src/services/message-token-reducer-service/token-count-strategies/claude-token-count-error-parser.ts b/packages/navie/src/services/message-token-reducer-service/token-count-strategies/claude-token-count-error-parser.ts
deleted file mode 100644
index f5d921ab2d..0000000000
--- a/packages/navie/src/services/message-token-reducer-service/token-count-strategies/claude-token-count-error-parser.ts
+++ /dev/null
@@ -1,13 +0,0 @@
-import Message from '../../../message';
-import { ApiError, TokenCountStrategy } from '../interfaces';
-
-export default class ClaudeTokenCountErrorParser implements TokenCountStrategy {
-  countTokens(_messages: Message[], _model: string, apiError?: ApiError): number | undefined {
-    if (!apiError) return undefined;
-
-    const match = /prompt is too long: (\d+) tokens > \d+ maximum/g.exec(apiError.message);
-    if (!match) return undefined;
-
-    return parseInt(match[1], 10) || undefined;
-  }
-}
diff --git a/packages/navie/src/services/message-token-reducer-service/token-count-strategies/estimate-token-count.ts b/packages/navie/src/services/message-token-reducer-service/token-count-strategies/estimate-token-count.ts
deleted file mode 100644
index 6639557167..0000000000
--- a/packages/navie/src/services/message-token-reducer-service/token-count-strategies/estimate-token-count.ts
+++ /dev/null
@@ -1,10 +0,0 @@
-import Message from '../../../message';
-import { TokenCountStrategy } from '../interfaces';
-
-export default class EstimateTokenCount implements TokenCountStrategy {
-  constructor(private readonly charactersPerToken = 3) {}
-
-  countTokens(messages: Message[]): number | undefined {
-    return messages.reduce((acc, { content }) => acc + content.length, 0) / this.charactersPerToken;
-  }
-}
diff --git a/packages/navie/src/services/message-token-reducer-service/token-count-strategies/openai-token-count-error-parser.ts b/packages/navie/src/services/message-token-reducer-service/token-count-strategies/openai-token-count-error-parser.ts
deleted file mode 100644
index 9f55371773..0000000000
--- a/packages/navie/src/services/message-token-reducer-service/token-count-strategies/openai-token-count-error-parser.ts
+++ /dev/null
@@ -1,13 +0,0 @@
-import Message from '../../../message';
-import { ApiError, TokenCountStrategy } from '../interfaces';
-
-export default class OpenAiTokenCountErrorParser implements TokenCountStrategy {
-  countTokens(_messages: Message[], _model: string, apiError?: ApiError): number | undefined {
-    if (!apiError) return undefined;
-
-    const match = /However, your messages resulted in (\d+) tokens./g.exec(apiError.message);
-    if (!match) return undefined;
-
-    return parseInt(match[1], 10) || undefined;
-  }
-}
diff --git a/packages/navie/src/services/message-token-reducer-service/token-limit-identifiers/claude-token-limit-error-parser.ts b/packages/navie/src/services/message-token-reducer-service/token-limit-identifiers/claude-token-limit-error-parser.ts
deleted file mode 100644
index 6505749331..0000000000
--- a/packages/navie/src/services/message-token-reducer-service/token-limit-identifiers/claude-token-limit-error-parser.ts
+++ /dev/null
@@ -1,12 +0,0 @@
-import { ApiError, TokenLimitIdentifier } from '../interfaces';
-
-export default class ClaudeTokenLimitErrorParser implements TokenLimitIdentifier {
-  getTokenLimit(_model: string, apiError?: ApiError): number | undefined {
-    if (!apiError) return undefined;
-
-    const match = /prompt is too long: \d+ tokens > (\d+) maximum/g.exec(apiError.message);
-    if (!match) return undefined;
-
-    return parseInt(match[1], 10) || undefined;
-  }
-}
diff --git a/packages/navie/src/services/message-token-reducer-service/token-limit-identifiers/lookup-max-tokens.ts b/packages/navie/src/services/message-token-reducer-service/token-limit-identifiers/lookup-max-tokens.ts
deleted file mode 100644
index 59f7c42157..0000000000
--- a/packages/navie/src/services/message-token-reducer-service/token-limit-identifiers/lookup-max-tokens.ts
+++ /dev/null
@@ -1,62 +0,0 @@
-import { TokenLimitIdentifier } from '../interfaces';
-
-export default class LookupMaxTokens implements TokenLimitIdentifier {
-  private static readonly MAX_TOKENS: Record<string, number> = {
-    // OpenAI
-    'gpt-3.5-turbo': 16_385,
-    'gpt-3.5-turbo-0125': 16_385,
-    'gpt-3.5-turbo-1106': 16_385,
-    'gpt-3.5-turbo-instruct': 4_096,
-    'gpt-4': 8_192,
-    'gpt-4-0314': 8_192,
-    'gpt-4-0613': 8_192,
-    'gpt-4-0125-preview': 128_000,
-    'gpt-4-1106-preview': 128_000,
-    'gpt-4-turbo-preview': 128_000,
-    'gpt-4-turbo-2024-04-09': 128_000,
-    'gpt-4-turbo': 128_000,
-    'gpt-4o-mini': 128_000,
-    'gpt-4o-mini-2024-07-18': 128_000,
-    'gpt-4o': 128_000,
-    'gpt-4o-2024-05-13': 128_000,
-    'gpt-4o-2024-08-06': 128_000,
-    'chatgpt-4o-latest': 128_000,
-    'o1-preview': 128_000,
-    'o1-preview-2024-09-12': 128_000,
-    'o1-mini': 128_000,
-    'o1-mini-2024-09-12': 128_000,
-
-    // Claude
-    // Note that these token limits are subject to change depending on server load.
-    'claude-instant-1.2': 100_000,
-    'claude-2.0': 100_000,
-    'claude-2.1': 200_000,
-    'claude-3-5-sonnet-20240620': 200_000,
-    'claude-3-opus-20240229': 200_000,
-    'claude-3-sonnet-20240229': 200_000,
-    'claude-3-haiku-20240307': 200_000,
-
-    // Gemini
-    'gemini-pro': 32_760,
-    'gemini-1.0-pro': 32_760,
-    'gemini-1.0-pro-latest': 32_760,
-    'gemini-1.0-pro-001': 32_760,
-    'gemini-1.0-pro-vision': 16_384,
-    'gemini-1.5-pro': 2_097_152,
-    'gemini-1.5-pro-latest': 2_097_152,
-    'gemini-1.5-pro-001': 2_097_152,
-    'gemini-1.5-pro-002': 2_097_152,
-    'gemini-1.5-pro-exp-0827': 2_097_152,
-    'gemini-1.5-flash': 1_048_576,
-    'gemini-1.5-flash-latest': 1_048_576,
-    'gemini-1.5-flash-001': 1_048_576,
-    'gemini-1.5-flash-002': 1_048_576,
-    'gemini-1.5-flash-8b-exp-0924': 1_048_576,
-    'gemini-1.5-flash-8b-exp-0827': 1_048_576,
-    'gemini-1.5-flash-exp-0827': 1_048_576,
-  };
-
-  getTokenLimit(model: string): number | undefined {
-    return LookupMaxTokens.MAX_TOKENS[model];
-  }
-}
diff --git a/packages/navie/src/services/message-token-reducer-service/token-limit-identifiers/openai-token-limit-error-parser.ts b/packages/navie/src/services/message-token-reducer-service/token-limit-identifiers/openai-token-limit-error-parser.ts
deleted file mode 100644
index 3e269e5a3a..0000000000
--- a/packages/navie/src/services/message-token-reducer-service/token-limit-identifiers/openai-token-limit-error-parser.ts
+++ /dev/null
@@ -1,12 +0,0 @@
-import { ApiError, TokenLimitIdentifier } from '../interfaces';
-
-export default class OpenAiTokenLimitErrorParser implements TokenLimitIdentifier {
-  getTokenLimit(_model: string, apiError?: ApiError): number | undefined {
-    if (!apiError) return undefined;
-
-    const match = /This model's maximum context length is (\d+)/g.exec(apiError.message);
-    if (!match) return undefined;
-
-    return parseInt(match[1], 10) || undefined;
-  }
-}
diff --git a/packages/navie/src/services/message-token-reducer-service/token-reduction-strategies/message-truncation-strategy.ts b/packages/navie/src/services/message-token-reducer-service/token-reduction-strategies/message-truncation-strategy.ts
deleted file mode 100644
index 61b33ccc09..0000000000
--- a/packages/navie/src/services/message-token-reducer-service/token-reduction-strategies/message-truncation-strategy.ts
+++ /dev/null
@@ -1,33 +0,0 @@
-import Message from '../../../message';
-import { TokenReductionStrategy } from '../interfaces';
-
-export default class MessageTruncationStrategy implements TokenReductionStrategy {
-  constructor(private readonly role: string) {}
-
-  reduceTokens(messages: Message[], currentTokens: number, maxTokens: number): Message[] {
-    let largestMessageIndex = -1;
-    let totalContentLength = 0;
-    messages.forEach((message, index) => {
-      totalContentLength += message.content.length;
-
-      const maxLength = messages[largestMessageIndex]?.content?.length ?? -1;
-      if (message.role === this.role && message.content.length > maxLength) {
-        largestMessageIndex = index;
-      }
-    });
-
-    if (largestMessageIndex === -1) return [...messages];
-
-    const largestMessage = messages[largestMessageIndex];
-    const reductionRatio = 1 - maxTokens / currentTokens;
-    const charsToRemove = Math.ceil(reductionRatio * totalContentLength);
-    const updatedMessage = { ...messages[largestMessageIndex] };
-    updatedMessage.content = largestMessage.content.slice(0, -charsToRemove);
-
-    return [
-      ...messages.slice(0, largestMessageIndex),
-      updatedMessage,
-      ...messages.slice(largestMessageIndex + 1),
-    ];
-  }
-}
diff --git a/packages/navie/src/services/openai-completion-service.ts b/packages/navie/src/services/openai-completion-service.ts
index 3525ff4ce5..ec315ecc0b 100644
--- a/packages/navie/src/services/openai-completion-service.ts
+++ b/packages/navie/src/services/openai-completion-service.ts
@@ -1,11 +1,11 @@
 import { isNativeError } from 'node:util/types';
 
+import { getModelNameForTiktoken } from '@langchain/core/language_models/base';
 import { ChatOpenAI } from '@langchain/openai';
 import type { ChatCompletion, ChatCompletionChunk } from 'openai/resources/index';
 import { z } from 'zod';
 import { zodResponseFormat } from 'openai/helpers/zod';
 import { warn } from 'console';
-import Message from '../message';
 import CompletionService, {
   Completion,
   CompletionRetries,
@@ -15,12 +15,21 @@ import CompletionService, {
   Usage,
 } from './completion-service';
 import Trajectory from '../lib/trajectory';
+import Message, { CHARACTERS_PER_TOKEN } from '../message';
 import { APIError } from 'openai';
-import MessageTokenReducerService from './message-token-reducer-service';
 import { findObject, tryParseJson } from '../lib/parse-json';
 import trimFences from '../lib/trim-fences';
 import { performance } from 'node:perf_hooks';
 
+import maxTokens from '../lib/max-tokens';
+import truncateMessages from '../lib/truncate-messages';
+
+// For some reason this doesn't work as import
+// eslint-disable-next-line @typescript-eslint/no-require-imports
+const { getEncodingNameForModel } = require('js-tiktoken/lite') as {
+  getEncodingNameForModel: (x: string) => string;
+};
+
 /*
   Generated on https://openai.com/api/pricing/ with
   Object.fromEntries(
@@ -177,8 +186,7 @@ export default class OpenAICompletionService implements CompletionService {
   constructor(
     public readonly modelName: string,
     public readonly temperature: number,
-    private trajectory: Trajectory,
-    private readonly messageTokenReducerService: MessageTokenReducerService
+    private trajectory: Trajectory
   ) {
     this.model = new ChatOpenAI({
       modelName: this.modelName,
@@ -186,8 +194,17 @@ export default class OpenAICompletionService implements CompletionService {
       streaming: true,
       onFailedAttempt,
     });
+    try {
+      getEncodingNameForModel(getModelNameForTiktoken(modelName));
+    } catch {
+      warn(`Unknown model ${modelName}, using estimated token count`);
+      this.model.getNumTokens = (c) =>
+        Promise.resolve(typeof c === 'string' ? c.length / CHARACTERS_PER_TOKEN : 0);
+    }
+    this.tokenLimit = maxTokens(modelName);
   }
   model: ChatOpenAI;
+  tokenLimit: number;
 
   get miniModelName(): string {
     const miniModel = process.env.APPMAP_NAVIE_MINI_MODEL;
@@ -309,6 +326,12 @@ export default class OpenAICompletionService implements CompletionService {
     const isO1 = this.modelName.startsWith('o1-');
     const usage = new Usage(COST_PER_M_TOKEN[model]);
     let sentMessages: Message[] = mergeSystemMessages(messages);
+    const tokenCount = await this.countTokens(sentMessages);
+    if (tokenCount > this.tokenLimit) {
+      warn(`Token count (${tokenCount}) exceeds limit (${this.tokenLimit}), truncating messages.`);
+      sentMessages = truncateMessages(sentMessages, tokenCount, this.tokenLimit);
+      warn(`New token count: ${await this.countTokens(sentMessages)}`);
+    }
 
     if (isO1) {
       // O1 does not support system messages, so prepend the system message to the newest user message.
@@ -377,12 +400,11 @@ export default class OpenAICompletionService implements CompletionService {
         if (!(error instanceof APIError)) throw error; // only retry on server errors
         if (tokens.length || attempt === CompletionRetries - 1) throw error; // Rethrow if tokens were yielded or max attempts reached
         if (error.type === 'invalid_request_error' && error.code === 'context_length_exceeded') {
-          warn('Context length exceeded. Reducing token count and retrying.');
-          sentMessages = await this.messageTokenReducerService.reduceMessageTokens(
-            sentMessages,
-            model,
-            { message: error.message }
-          );
+          const count = await this.countTokens(sentMessages, error);
+          const max = this.ratchetMaxTokens(count, error);
+          warn(`Reducing tokens from ${count} to ${max}`);
+          sentMessages = truncateMessages(sentMessages, count, max);
+          continue;
         } else if (error.type !== 'server_error') throw error; // only retry on server errors
         await new Promise(
           (resolve) => setTimeout(resolve, CompletionRetryDelay * Math.pow(2, attempt)) // Exponential backoff
@@ -395,6 +417,30 @@ export default class OpenAICompletionService implements CompletionService {
     warn(usage.toString());
     return usage;
   }
+
+  private async countTokens(messages: Message[], error?: Error): Promise<number> {
+    if (error) {
+      const countMatch = /However, your messages resulted in (\d+) tokens./.exec(error.message);
+      if (countMatch) return parseInt(countMatch[1], 10);
+    }
+
+    const counts = await Promise.all(messages.map((m) => this.model.getNumTokens(m.content)));
+    return counts.reduce((a, b) => a + b, 0);
+  }
+
+  private ratchetMaxTokens(overflowedCount: number, error?: Error): number {
+    let maxTokens = Math.min(this.tokenLimit, overflowedCount * 0.9);
+
+    if (error) {
+      const countMatch = /This model's maximum context length is (\d+)/.exec(error.message);
+      if (countMatch) maxTokens = Math.min(parseInt(countMatch[1], 10), maxTokens);
+      if (maxTokens === this.tokenLimit)
+        // something is wrong, add a margin
+        maxTokens *= 0.9;
+    }
+
+    return (this.tokenLimit = Math.floor(maxTokens));
+  }
 }
 
 /**
diff --git a/packages/navie/test/lib/truncate-messages.spec.ts b/packages/navie/test/lib/truncate-messages.spec.ts
new file mode 100644
index 0000000000..f62948f827
--- /dev/null
+++ b/packages/navie/test/lib/truncate-messages.spec.ts
@@ -0,0 +1,67 @@
+import truncateMessages from '../../src/lib/truncate-messages';
+import type Message from '../../src/message';
+
+describe('truncateMessages', () => {
+  it('returns original messages when no truncation needed', () => {
+    const messages: Message[] = [
+      { role: 'user', content: 'hello' },
+      { role: 'assistant', content: 'hi' },
+    ];
+    const result = truncateMessages(messages, 100, 200);
+    expect(result).toEqual(messages);
+    expect(result).not.toBe(messages); // Should be a new array
+  });
+
+  it('returns original messages when no user messages present', () => {
+    const messages: Message[] = [
+      { role: 'assistant', content: 'hello' },
+      { role: 'system', content: 'hi' },
+    ];
+    const result = truncateMessages(messages, 150, 100);
+    expect(result).toEqual(messages);
+  });
+
+  it('truncates the largest user message correctly', () => {
+    const messages: Message[] = [
+      { role: 'user', content: 'small' },
+      { role: 'assistant', content: 'medium message' },
+      { role: 'user', content: 'this is the largest message' },
+    ];
+    const totalLength = messages.reduce((sum, msg) => sum + msg.content.length, 0);
+    const result = truncateMessages(messages, 200, 150);
+
+    expect(result[2].content.length).toBeLessThan(messages[2].content.length);
+    expect(result[0]).toEqual(messages[0]);
+    expect(result[1]).toEqual(messages[1]);
+  });
+
+  it('handles empty message array', () => {
+    const messages: Message[] = [];
+    const result = truncateMessages(messages, 100, 50);
+    expect(result).toEqual([]);
+    expect(result).not.toBe(messages);
+  });
+
+  it('handles single user message', () => {
+    const messages: Message[] = [{ role: 'user', content: 'single message here' }];
+    const result = truncateMessages(messages, 100, 50);
+    expect(result[0].content.length).toBeLessThan(messages[0].content.length);
+  });
+
+  it('handles single large user message', () => {
+    const messages: Message[] = [{ role: 'user', content: 'single message here' }];
+    const result = truncateMessages(messages, 92421, 32768);
+    expect(result[0].content.length).toBeLessThan(messages[0].content.length);
+  });
+
+  it('truncates correctly with multiple same-length messages', () => {
+    const messages: Message[] = [
+      { role: 'user', content: '1234567890' },
+      { role: 'assistant', content: 'abcdefghij' },
+      { role: 'user', content: '1234567890' },
+    ];
+    const result = truncateMessages(messages, 100, 50);
+    expect(result[0].content.length).toBeLessThan(messages[0].content.length);
+    expect(result[2]).toEqual(messages[2]);
+  });
+});
diff --git a/packages/navie/test/services/anthropic-completion-service.spec.ts b/packages/navie/test/services/anthropic-completion-service.spec.ts
index 064c26dd16..3ecc523f28 100644
--- a/packages/navie/test/services/anthropic-completion-service.spec.ts
+++ b/packages/navie/test/services/anthropic-completion-service.spec.ts
@@ -6,11 +6,12 @@ import { AIMessageChunk } from '@langchain/core/messages';
 import { z } from 'zod';
 import { RunnableBinding } from '@langchain/core/runnables';
 import Trajectory, { TrajectoryEvent } from '../../src/lib/trajectory';
-import MessageTokenReducerService from '../../src/services/message-token-reducer-service';
+import * as truncater from '../../src/lib/truncate-messages';
+
+jest.mock('../../src/lib/truncate-messages');
 
 describe('AnthropicCompletionService', () => {
   let interactionHistory: InteractionHistory;
-  let messageTokenReducerService: MessageTokenReducerService;
   let trajectory: Trajectory;
   let service: AnthropicCompletionService;
   const modelName = 'anthropic-model';
@@ -21,18 +22,14 @@ describe('AnthropicCompletionService', () => {
   beforeEach(() => {
     process.env['ANTHROPIC_API_KEY'] = 'test-api-key';
     interactionHistory = new InteractionHistory();
-    messageTokenReducerService = new MessageTokenReducerService();
     trajectory = new Trajectory();
-    service = new AnthropicCompletionService(
-      modelName,
-      temperature,
-      trajectory,
-      messageTokenReducerService
-    );
+    service = new AnthropicCompletionService(modelName, temperature, trajectory);
+    jest.useFakeTimers();
   });
 
   afterEach(() => {
     process.env = originalEnv;
+    jest.useRealTimers();
   });
 
   it('has the correct model name', () => {
@@ -155,9 +152,7 @@ describe('AnthropicCompletionService', () => {
         });
         throw error;
       });
-      const reduceMessageTokens = jest
-        .spyOn(messageTokenReducerService, 'reduceMessageTokens')
-        .mockResolvedValue([]);
+      const truncateMessages = jest.spyOn(truncater, 'default').mockReturnValue([]);
       const completion = service.complete([]);
       const consumePromise = (async () => {
         // eslint-disable-next-line @typescript-eslint/no-unused-vars
@@ -171,24 +166,13 @@ describe('AnthropicCompletionService', () => {
           const err = e as Error;
           expect(err).toBeInstanceOf(Error);
           expect(err.message).toContain('prompt is too long');
-          expect(reduceMessageTokens).toHaveBeenCalledTimes(4);
+          expect(truncateMessages).toHaveBeenCalledTimes(4);
           expect(stream).toHaveBeenCalledTimes(5);
           /* eslint-enable jest/no-conditional-expect */
         }
       })();
 
-      const delays = [1000, 2000, 4000, 8000];
-
-      for (const delay of delays) {
-        // Yield to the event loop to allow another attempt to be made
-        // eslint-disable-next-line no-await-in-loop
-        await Promise.resolve();
-
-        // Another yield because the call to `reduceMessageTokens` is async
-        await Promise.resolve();
-
-        jest.advanceTimersByTime(delay);
-      }
+      await jest.runAllTimersAsync();
 
       await consumePromise;
     });
diff --git a/packages/navie/test/services/message-token-reducer-service/index.spec.ts b/packages/navie/test/services/message-token-reducer-service/index.spec.ts
deleted file mode 100644
index e7132a30a9..0000000000
--- a/packages/navie/test/services/message-token-reducer-service/index.spec.ts
+++ /dev/null
@@ -1,97 +0,0 @@
-import MessageTokenReducerService from '../../../src/services/message-token-reducer-service';
-import Message from '../../../src/message';
-
-const countCharacters = (messages: Message[]) =>
-  messages.reduce((acc, { content }) => acc + content.length, 0);
-
-const repeatCharacters = (str: string, numChars: number) => {
-  const repeats = Math.ceil(numChars / str.length);
-  const repeatedStr = str.repeat(repeats);
-  return repeatedStr.slice(0, numChars);
-};
-
-describe('MessageTokenReducerService', () => {
-  let messages: Message[];
-  let service: MessageTokenReducerService;
-
-  beforeEach(() => {
-    messages = [
-      { role: 'system', content: 'You are a helpful assistant.' },
-      { role: 'user', content: 'Hello' },
-      { role: 'assistant', content: 'Greetings! How can I assist you today?' },
-      { role: 'user', content: 'Let me know about the weather.' },
-    ];
-    service = new MessageTokenReducerService();
-  });
-
-  describe('OpenAI', () => {
-    describe('with an error', () => {
-      it('reduces tokens as expected', async () => {
-        const numCharacters = countCharacters(messages);
-        const apiError = {
-          message: `This model's maximum context length is ${
-            numCharacters - ` weather.`.length
-          } tokens. However, your messages resulted in ${numCharacters} tokens. Please reduce the length of the messages.`,
-        };
-        const result = await service.reduceMessageTokens(messages, 'gpt-4o', apiError);
-        expect(result).toStrictEqual([
-          { role: 'system', content: 'You are a helpful assistant.' },
-          { role: 'user', content: 'Hello' },
-          { role: 'assistant', content: 'Greetings! How can I assist you today?' },
-          { role: 'user', content: 'Let me know about the' },
-        ]);
-      });
-    });
-  });
-
-  it('returns the original messages if the token limit is not exceeded', async () => {
-    const result = await service.reduceMessageTokens(messages, 'gpt-4o');
-    expect(result).toStrictEqual(messages);
-  });
-
-  it('trims the messages if the token limit is exceeded', async () => {
-    const maxTokens = 8_192;
-    const charactersPerToken = 3;
-
-    // The last user message will now overflow the token limit
-    messages[messages.length - 1].content = 'abc'.repeat(maxTokens);
-
-    const contentLength = messages.slice(0, 3).reduce((acc, m) => acc + m.content.length, 0);
-    const result = await service.reduceMessageTokens(messages, 'gpt-4');
-    const newContentLength = countCharacters(result);
-    expect(newContentLength).toBe(maxTokens * 3);
-    expect(result).toStrictEqual([
-      { role: 'system', content: 'You are a helpful assistant.' },
-      { role: 'user', content: 'Hello' },
-      { role: 'assistant', content: 'Greetings! How can I assist you today?' },
-      {
-        role: 'user',
-        content: repeatCharacters('abc', maxTokens * charactersPerToken - contentLength),
-      },
-    ]);
-  });
-
-  describe('Claude', () => {
-    describe('with an error', () => {
-      it('reduces tokens as expected', async () => {
-        const numCharacters = countCharacters(messages);
-        const apiError = {
-          message: `prompt is too long: ${numCharacters} tokens > ${
-            numCharacters - ` weather.`.length
-          } maximum`,
-        };
-        const result = await service.reduceMessageTokens(
-          messages,
-          'claude-3-5-sonnet-20240620',
-          apiError
-        );
-        expect(result).toStrictEqual([
-          { role: 'system', content: 'You are a helpful assistant.' },
-          { role: 'user', content: 'Hello' },
-          { role: 'assistant', content: 'Greetings! How can I assist you today?' },
-          { role: 'user', content: 'Let me know about the' },
-        ]);
-      });
-    });
-  });
-});
diff --git a/packages/navie/test/services/message-token-reducer-service/token-count-strategies/claude-token-count-error-parser.spec.ts b/packages/navie/test/services/message-token-reducer-service/token-count-strategies/claude-token-count-error-parser.spec.ts
deleted file mode 100644
index 59eddb41eb..0000000000
--- a/packages/navie/test/services/message-token-reducer-service/token-count-strategies/claude-token-count-error-parser.spec.ts
+++ /dev/null
@@ -1,30 +0,0 @@
-import { Message } from '../../../../src';
-import ClaudeTokenCountErrorParser from '../../../../src/services/message-token-reducer-service/token-count-strategies/claude-token-count-error-parser';
-
-describe('ClaudeTokenCountErrorParser', () => {
-  let parser: ClaudeTokenCountErrorParser;
-  const model = 'gpt-4o-2024-08-06';
-  const messages: Message[] = [];
-
-  beforeEach(() => {
-    parser = new ClaudeTokenCountErrorParser();
-  });
-
-  it('returns undefined if there is no error', () => {
-    const result = parser.countTokens(messages, model);
-    expect(result).toBeUndefined();
-  });
-
-  it('returns undefined if the error does not match the pattern', () => {
-    const result = parser.countTokens(messages, model, { message: 'This is not an error' });
-    expect(result).toBeUndefined();
-  });
-
-  it('returns the token limit if the error matches the pattern', () => {
-    const tokenLimit = 16_385;
-    const result = parser.countTokens(messages, model, {
-      message: `prompt is too long: ${tokenLimit + 1} tokens > ${tokenLimit} maximum`,
-    });
-    expect(result).toBe(tokenLimit + 1);
-  });
-});
diff --git a/packages/navie/test/services/message-token-reducer-service/token-count-strategies/estimate-token-count.spec.ts b/packages/navie/test/services/message-token-reducer-service/token-count-strategies/estimate-token-count.spec.ts
deleted file mode 100644
index f1c17a8e34..0000000000
--- a/packages/navie/test/services/message-token-reducer-service/token-count-strategies/estimate-token-count.spec.ts
+++ /dev/null
@@ -1,21 +0,0 @@
-import EstimateTokenCount from '../../../../src/services/message-token-reducer-service/token-count-strategies/estimate-token-count';
-
-describe('EstimateTokenCount', () => {
-  it('returns zero if there are no messages', () => {
-    const estimate = new EstimateTokenCount();
-    const result = estimate.countTokens([]);
-    expect(result).toBe(0);
-  });
-
-  it('returns the number of characters divided by the characters per token', () => {
-    const estimate = new EstimateTokenCount(2);
-    const result = estimate.countTokens([{ content: 'Four', role: 'user' }]);
-    expect(result).toBe(2);
-  });
-
-  it('defaults to 3 characters per token', () => {
-    const estimate = new EstimateTokenCount();
-    const result = estimate.countTokens([{ content: 'abcabc', role: 'user' }]);
-    expect(result).toBe(2);
-  });
-});
diff --git a/packages/navie/test/services/message-token-reducer-service/token-count-strategies/openai-token-count-error-parser.spec.ts b/packages/navie/test/services/message-token-reducer-service/token-count-strategies/openai-token-count-error-parser.spec.ts
deleted file mode 100644
index a96a56dfdc..0000000000
--- a/packages/navie/test/services/message-token-reducer-service/token-count-strategies/openai-token-count-error-parser.spec.ts
+++ /dev/null
@@ -1,32 +0,0 @@
-import type { Message } from '../../../../src';
-import OpenAiTokenCountErrorParser from '../../../../src/services/message-token-reducer-service/token-count-strategies/openai-token-count-error-parser';
-
-describe('OpenAiTokenCountErrorParser', () => {
-  let parser: OpenAiTokenCountErrorParser;
-  const model = 'gpt-4o-2024-08-06';
-  const messages: Message[] = [];
-
-  beforeEach(() => {
-    parser = new OpenAiTokenCountErrorParser();
-  });
-
-  it('returns undefined if there is no error', () => {
-    const result = parser.countTokens(messages, model);
-    expect(result).toBeUndefined();
-  });
-
-  it('returns undefined if the error does not match the pattern', () => {
-    const result = parser.countTokens(messages, model, { message: 'This is not an error' });
-    expect(result).toBeUndefined();
-  });
-
-  it('returns the token limit if the error matches the pattern', () => {
-    const tokenLimit = 16_385;
-    const result = parser.countTokens(messages, model, {
-      message: `This model's maximum context length is ${tokenLimit} tokens. However, your messages resulted in ${
-        tokenLimit + 1
-      } tokens. Please reduce the length of the messages.`,
-    });
-    expect(result).toBe(tokenLimit + 1);
-  });
-});
diff --git a/packages/navie/test/services/message-token-reducer-service/token-limit-identifiers/claude-token-limit-error-parser.spec.ts b/packages/navie/test/services/message-token-reducer-service/token-limit-identifiers/claude-token-limit-error-parser.spec.ts
deleted file mode 100644
index 3673eb32a3..0000000000
--- a/packages/navie/test/services/message-token-reducer-service/token-limit-identifiers/claude-token-limit-error-parser.spec.ts
+++ /dev/null
@@ -1,28 +0,0 @@
-import ClaudeTokenLimitErrorParser from '../../../../src/services/message-token-reducer-service/token-limit-identifiers/claude-token-limit-error-parser';
-
-describe('ClaudeTokenLimitErrorParser', () => {
-  let parser: ClaudeTokenLimitErrorParser;
-  const model = 'claude-3-5-sonnet-20240620';
-
-  beforeEach(() => {
-    parser = new ClaudeTokenLimitErrorParser();
-  });
-
-  it('returns undefined if there is no error', () => {
-    const result = parser.getTokenLimit(model);
-    expect(result).toBeUndefined();
-  });
-
-  it('returns undefined if the error does not match the pattern', () => {
-    const result = parser.getTokenLimit(model, { message: 'This is not an error' });
-    expect(result).toBeUndefined();
-  });
-
-  it('returns the token limit if the error matches the pattern', () => {
-    const tokenLimit = 16_385;
-    const result = parser.getTokenLimit(model, {
-      message: `prompt is too long: ${tokenLimit + 1} tokens > ${tokenLimit} maximum`,
-    });
-    expect(result).toBe(tokenLimit);
-  });
-});
diff --git a/packages/navie/test/services/message-token-reducer-service/token-limit-identifiers/lookup-max-tokens.spec.ts b/packages/navie/test/services/message-token-reducer-service/token-limit-identifiers/lookup-max-tokens.spec.ts
deleted file mode 100644
index 5212e7d2fd..0000000000
--- a/packages/navie/test/services/message-token-reducer-service/token-limit-identifiers/lookup-max-tokens.spec.ts
+++ /dev/null
@@ -1,19 +0,0 @@
-import LookupMaxTokens from '../../../../src/services/message-token-reducer-service/token-limit-identifiers/lookup-max-tokens';
-
-describe('LookupMaxTokens', () => {
-  let service: LookupMaxTokens;
-
-  beforeEach(() => {
-    service = new LookupMaxTokens();
-  });
-
-  it('returns undefined if the model is unknown', () => {
-    const result = service.getTokenLimit('unknown-model');
-    expect(result).toBeUndefined();
-  });
-
-  it('returns the token limit for the given model', () => {
-    const result = service.getTokenLimit('gpt-4o-2024-08-06');
-    expect(result).toBe(128_000);
-  });
-});
diff --git a/packages/navie/test/services/message-token-reducer-service/token-limit-identifiers/openai-token-limit-error-parser.spec.ts b/packages/navie/test/services/message-token-reducer-service/token-limit-identifiers/openai-token-limit-error-parser.spec.ts
deleted file mode 100644
index 70a03d221a..0000000000
--- a/packages/navie/test/services/message-token-reducer-service/token-limit-identifiers/openai-token-limit-error-parser.spec.ts
+++ /dev/null
@@ -1,29 +0,0 @@
-import OpenAiTokenLimitErrorParser from '../../../../src/services/message-token-reducer-service/token-limit-identifiers/openai-token-limit-error-parser';
-
-describe('OpenAiTokenLimitErrorParser', () => {
-  let parser: OpenAiTokenLimitErrorParser;
-
-  beforeEach(() => {
-    parser = new OpenAiTokenLimitErrorParser();
-  });
-
-  it('returns undefined if there is no error', () => {
-    const result = parser.getTokenLimit('gpt-4o');
-    expect(result).toBeUndefined();
-  });
-
-  it('returns undefined if the error does not match the pattern', () => {
-    const result = parser.getTokenLimit('gpt-4o', { message: 'This is not an error' });
-    expect(result).toBeUndefined();
-  });
-
-  it('returns the token limit if the error matches the pattern', () => {
-    const tokenLimit = 16_385;
-    const result = parser.getTokenLimit('gpt-4o', {
-      message: `This model's maximum context length is ${tokenLimit} tokens. However, your messages resulted in ${
-        tokenLimit + 1
-      } tokens. Please reduce the length of the messages.`,
-    });
-    expect(result).toBe(tokenLimit);
-  });
-});
diff --git a/packages/navie/test/services/message-token-reducer-service/token-reduction-strategies/message-truncation-strategy.spec.ts b/packages/navie/test/services/message-token-reducer-service/token-reduction-strategies/message-truncation-strategy.spec.ts
deleted file mode 100644
index 2516a814f9..0000000000
--- a/packages/navie/test/services/message-token-reducer-service/token-reduction-strategies/message-truncation-strategy.spec.ts
+++ /dev/null
@@ -1,48 +0,0 @@
-import { Message } from '../../../../src';
-import MessageTruncationStrategy from '../../../../src/services/message-token-reducer-service/token-reduction-strategies/message-truncation-strategy';
-
-describe('MessageTruncationStrategy', () => {
-  let strategy: MessageTruncationStrategy;
-
-  beforeEach(() => {
-    strategy = new MessageTruncationStrategy('user');
-  });
-
-  it('returns the original messages if there are no messages', () => {
-    const messages: Message[] = [];
-    const result = strategy.reduceTokens(messages, 0, 0);
-    expect(result).toEqual(messages);
-  });
-
-  it('trims the largest message by the given parameters', () => {
-    const messages: Message[] = [
-      { content: 'Hello world!', role: 'system' },
-      { content: 'This is the largest message.', role: 'user' },
-      { content: 'Short message.', role: 'assistant' },
-    ];
-    const numCharacters = messages.reduce((acc, { content }) => acc + content.length, 0);
-    const result = strategy.reduceTokens(messages, numCharacters, numCharacters - 8);
-    const expected = [
-      { content: 'Hello world!', role: 'system' },
-      { content: 'This is the largest ', role: 'user' },
-      { content: 'Short message.', role: 'assistant' },
-    ];
-    expect(result).toEqual(expected);
-  });
-
-  it('only trims messages matching the role given to the constructor', () => {
-    const messages: Message[] = [
-      { content: 'This is the largest message.', role: 'system' },
-      { content: 'Hello world!', role: 'user' },
-      { content: 'Short message.', role: 'assistant' },
-    ];
-    const numCharacters = messages.reduce((acc, { content }) => acc + content.length, 0);
-    const result = strategy.reduceTokens(messages, numCharacters, numCharacters - 6);
-    const expected = [
-      { content: 'This is the largest message.', role: 'system' },
-      { content: 'Hello', role: 'user' },
-      { content: 'Short message.', role: 'assistant' },
-    ];
-    expect(result).toEqual(expected);
-  });
-});
diff --git a/packages/navie/test/services/openai-completion-service.spec.ts b/packages/navie/test/services/openai-completion-service.spec.ts
index 161038cb07..62db8aa4c1 100644
--- a/packages/navie/test/services/openai-completion-service.spec.ts
+++ b/packages/navie/test/services/openai-completion-service.spec.ts
@@ -7,13 +7,14 @@ import { z } from 'zod';
 import Message from '../../src/message';
 import { PromptType } from '../../src/prompt';
 import { APIError } from 'openai';
-import MessageTokenReducerService from '../../src/services/message-token-reducer-service';
 
+import * as truncater from '../../src/lib/truncate-messages';
+
+jest.mock('../../src/lib/truncate-messages');
 jest.mock('@langchain/openai');
 
 describe('OpenAICompletionService', () => {
   let interactionHistory: InteractionHistory;
-  let messageTokenReducerService: MessageTokenReducerService;
   let service: OpenAICompletionService;
   let trajectory: Trajectory;
   const modelName = 'the-model-name';
@@ -21,14 +22,8 @@ describe('OpenAICompletionService', () => {
 
   beforeEach(() => {
     interactionHistory = new InteractionHistory();
-    messageTokenReducerService = new MessageTokenReducerService();
     trajectory = new Trajectory();
-    service = new OpenAICompletionService(
-      modelName,
-      temperature,
-      trajectory,
-      messageTokenReducerService
-    );
+    service = new OpenAICompletionService(modelName, temperature, trajectory);
   });
 
   describe('when the completion service is created', () => {
@@ -127,9 +122,7 @@ describe('OpenAICompletionService', () => {
       });
       mockCompletion('Hello');
 
-      const reduceMessageTokens = jest
-        .spyOn(messageTokenReducerService, 'reduceMessageTokens')
-        .mockResolvedValue([]);
+      const truncateMessages = jest.spyOn(truncater, 'default').mockReturnValue([]);
 
       const messages = [{ role: 'user', content: 'Hello' }] as const;
       const result = [];
@@ -139,7 +132,7 @@ describe('OpenAICompletionService', () => {
 
       expect(result).toEqual(['Hello']);
       expect(completionWithRetry).toHaveBeenCalledTimes(2);
-      expect(reduceMessageTokens).toHaveBeenCalledTimes(1);
+      expect(truncateMessages).toHaveBeenCalledTimes(1);
     });
 
     describe('with a custom temperature', () => {
diff --git a/packages/scanner/package.json b/packages/scanner/package.json
index 4ba6bc483e..b2a184296e 100644
--- a/packages/scanner/package.json
+++ b/packages/scanner/package.json
@@ -27,7 +27,6 @@
   "author": "AppLand, Inc.",
   "license": "Commons Clause + MIT",
   "devDependencies": {
-    "@appland/appmap-agent-js": "^13.9.0",
     "@semantic-release/changelog": "^6.0.1",
     "@semantic-release/git": "^10.0.1",
     "@types/async": "^3.2.12",
diff --git a/packages/search/package.json b/packages/search/package.json
index cd5fe365ee..7a2c4c2913 100644
--- a/packages/search/package.json
+++ b/packages/search/package.json
@@ -21,7 +21,6 @@
   "author": "AppLand, Inc",
   "license": "Commons Clause + MIT",
   "devDependencies": {
-    "@types/better-sqlite3": "^7.6.11",
     "@types/jest": "^29.5.4",
     "@types/node": "^16",
     "eslint": "^9",
@@ -39,8 +38,8 @@
     "typescript-eslint": "^8.11.0"
   },
   "dependencies": {
-    "better-sqlite3": "^11.5.0",
     "isbinaryfile": "^5.0.4",
+    "node-sqlite3-wasm": "^0.8.34",
     "yargs": "^17.7.2"
   }
 }
diff --git a/packages/search/src/cli.ts b/packages/search/src/cli.ts
index 0d3fb45dd8..14597e40c7 100644
--- a/packages/search/src/cli.ts
+++ b/packages/search/src/cli.ts
@@ -1,7 +1,7 @@
 import yargs from 'yargs';
 import { hideBin } from 'yargs/helpers';
-import sqlite3 from 'better-sqlite3';
 import makeDebug from 'debug';
+import sqlite3 from 'node-sqlite3-wasm';
 
 import { fileTokens } from './tokenize';
 import FileIndex from './file-index';
@@ -62,7 +62,7 @@ const cli = yargs(hideBin(process.argv))
         return !filterRE.test(path);
       };
 
-      const db = new sqlite3(':memory:');
+      const db = new sqlite3.Database(':memory:');
       const fileIndex = new FileIndex(db);
       const sessionId = generateSessionId();
 
diff --git a/packages/search/src/file-index.ts b/packages/search/src/file-index.ts
index 9f0c63cf14..1b9253c256 100644
--- a/packages/search/src/file-index.ts
+++ b/packages/search/src/file-index.ts
@@ -1,4 +1,5 @@
-import sqlite3 from 'better-sqlite3';
+import type sqlite3 from 'node-sqlite3-wasm';
+
 import { SessionId } from './session-id';
 
 const CREATE_TABLE_SQL = `CREATE VIRTUAL TABLE file_content USING fts5(
@@ -74,14 +75,14 @@ export type FileSearchResult = {
 export default class FileIndex {
   #insert: sqlite3.Statement;
   #updateBoost: sqlite3.Statement;
-  #deleteSession: sqlite3.Statement<string>;
-  #search: sqlite3.Statement<[string, string, number]>;
+  #deleteSession: sqlite3.Statement;
+  #search: sqlite3.Statement;
 
   constructor(public database: sqlite3.Database) {
     this.database.exec(CREATE_TABLE_SQL);
     this.database.exec(CREATE_BOOST_TABLE_SQL);
-    this.database.pragma('journal_mode = OFF');
-    this.database.pragma('synchronous = OFF');
+    this.database.exec('PRAGMA journal_mode = OFF');
+    this.database.exec('PRAGMA synchronous = OFF');
     this.#insert = this.database.prepare(INSERT_SQL);
     this.#updateBoost = this.database.prepare(UPDATE_BOOST_SQL);
     this.#deleteSession = this.database.prepare(DELETE_SESSION_SQL);
@@ -89,7 +90,7 @@ export default class FileIndex {
   }
 
   indexFile(directory: string, filePath: string, symbols: string, words: string): void {
-    this.#insert.run(directory, filePath, symbols, words);
+    this.#insert.run([directory, filePath, symbols, words]);
   }
 
   /**
@@ -99,7 +100,7 @@ export default class FileIndex {
    * @param boostFactor - The factor by which to boost the file's relevance.
    */
   boostFile(sessionId: SessionId, filePath: string, boostFactor: number): void {
-    this.#updateBoost.run(sessionId, filePath, boostFactor);
+    this.#updateBoost.run([sessionId, filePath, boostFactor]);
   }
 
   /**
@@ -118,7 +119,7 @@ export default class FileIndex {
    * @returns An array of search results with directory, file path, and score.
    */
   search(sessionId: SessionId, query: string, limit = 10): FileSearchResult[] {
-    const rows = this.#search.all(sessionId, query, limit) as FileIndexRow[];
+    const rows = this.#search.all([sessionId, query, limit]) as FileIndexRow[];
     return rows.map((row) => ({
       directory: row.directory,
       filePath: row.file_path,
diff --git a/packages/search/src/git.ts b/packages/search/src/git.ts
index 79c4cfbe53..091af3325f 100644
--- a/packages/search/src/git.ts
+++ b/packages/search/src/git.ts
@@ -1,4 +1,4 @@
-import { exec as execCallback, spawn } from 'child_process';
+import { exec as execCallback, spawn } from 'node:child_process';
 import { PathLike } from 'fs';
 import { promisify } from 'util';
 
diff --git a/packages/search/src/snippet-index.ts b/packages/search/src/snippet-index.ts
index bc13e33676..40f9f0d82b 100644
--- a/packages/search/src/snippet-index.ts
+++ b/packages/search/src/snippet-index.ts
@@ -1,5 +1,7 @@
 import assert from 'assert';
-import sqlite3 from 'better-sqlite3';
+
+import type sqlite3 from 'node-sqlite3-wasm';
+
 import { SessionId } from './session-id';
 
 const CREATE_SNIPPET_CONTENT_TABLE_SQL = `CREATE VIRTUAL TABLE snippet_content USING fts5(
@@ -115,14 +117,14 @@ type SnippetSearchRow = {
 export default class SnippetIndex {
   #insertSnippet: sqlite3.Statement;
   #updateSnippetBoost: sqlite3.Statement;
-  #deleteSession: sqlite3.Statement<[string]>;
-  #searchSnippet: sqlite3.Statement<[string, string, number]>;
+  #deleteSession: sqlite3.Statement;
+  #searchSnippet: sqlite3.Statement;
 
   constructor(public database: sqlite3.Database) {
     this.database.exec(CREATE_SNIPPET_CONTENT_TABLE_SQL);
     this.database.exec(CREATE_SNIPPET_BOOST_TABLE_SQL);
-    this.database.pragma('journal_mode = OFF');
-    this.database.pragma('synchronous = OFF');
+    this.database.exec('PRAGMA journal_mode = OFF');
+    this.database.exec('PRAGMA synchronous = OFF');
     this.#insertSnippet = this.database.prepare(INSERT_SNIPPET_SQL);
     this.#deleteSession = this.database.prepare(DELETE_SESSION_SQL);
     this.#updateSnippetBoost = this.database.prepare(UPDATE_SNIPPET_BOOST_SQL);
@@ -152,7 +154,7 @@ export default class SnippetIndex {
     words: string,
     content: string
   ): void {
-    this.#insertSnippet.run(encodeSnippetId(snippetId), directory, symbols, words, content);
+    this.#insertSnippet.run([encodeSnippetId(snippetId), directory, symbols, words, content]);
   }
 
   /**
@@ -162,11 +164,11 @@ export default class SnippetIndex {
    * @param boostFactor - The factor by which to boost the snippet's relevance.
    */
   boostSnippet(sessionId: SessionId, snippetId: SnippetId, boostFactor: number): void {
-    this.#updateSnippetBoost.run(sessionId, encodeSnippetId(snippetId), boostFactor);
+    this.#updateSnippetBoost.run([sessionId, encodeSnippetId(snippetId), boostFactor]);
   }
 
   searchSnippets(sessionId: SessionId, query: string, limit = 10): SnippetSearchResult[] {
-    const rows = this.#searchSnippet.all(sessionId, query, limit) as SnippetSearchRow[];
+    const rows = this.#searchSnippet.all([sessionId, query, limit]) as SnippetSearchRow[];
     return rows.map((row) => ({
       directory: row.directory,
       snippetId: parseSnippetId(row.snippet_id),
diff --git a/packages/search/test/file-index.spec.ts b/packages/search/test/file-index.spec.ts
index cff7369b1c..2b6664610c 100644
--- a/packages/search/test/file-index.spec.ts
+++ b/packages/search/test/file-index.spec.ts
@@ -1,5 +1,7 @@
 import { strict as assert } from 'assert';
-import sqlite3 from 'better-sqlite3';
+
+import sqlite3 from 'node-sqlite3-wasm';
+
 import FileIndex, { FileSearchResult } from '../src/file-index';
 import { generateSessionId, SessionId } from '../src/session-id';
 
@@ -16,7 +18,7 @@ describe('FileIndex', () => {
   const directory = 'src';
 
   beforeEach(() => {
-    db = new sqlite3(':memory:');
+    db = new sqlite3.Database(':memory:');
     index = new FileIndex(db);
     sessionId = generateSessionId();
   });
diff --git a/packages/search/test/snippet-index.spec.ts b/packages/search/test/snippet-index.spec.ts
index 352d6a2098..71bd102f42 100644
--- a/packages/search/test/snippet-index.spec.ts
+++ b/packages/search/test/snippet-index.spec.ts
@@ -1,5 +1,6 @@
 import { strict as assert } from 'assert';
-import sqlite3 from 'better-sqlite3';
+
+import sqlite3 from 'node-sqlite3-wasm';
 
 import SnippetIndex, {
   fileChunkSnippetId,
@@ -26,7 +27,7 @@ describe('SnippetIndex', () => {
   const snippet4: SnippetId = { type: 'code-snippet', id: 'test4.txt:31' };
 
   beforeEach(() => {
-    db = new sqlite3(':memory:');
+    db = new sqlite3.Database(':memory:');
     index = new SnippetIndex(db);
     sessionId = generateSessionId();
   });
diff --git a/packages/sequence-diagram/package.json b/packages/sequence-diagram/package.json
index 433bb9b8a1..ae41601b75 100644
--- a/packages/sequence-diagram/package.json
+++ b/packages/sequence-diagram/package.json
@@ -29,7 +29,6 @@
     "access": "public"
   },
   "devDependencies": {
-    "@appland/appmap-agent-js": "^13.9.0",
     "@types/diff": "^5.0.2",
     "@types/jest": "^29.4.1",
     "@typescript-eslint/eslint-plugin": "^5.51.0",
diff --git a/yarn.lock b/yarn.lock
index 83184c190e..c3a3f8a73e 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -88,42 +88,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@appland/appmap-agent-js@npm:^13.9.0":
-  version: 13.9.0
-  resolution: "@appland/appmap-agent-js@npm:13.9.0"
-  dependencies:
-    "@appland/appmap-validate": ^2.3.0
-    "@babel/parser": ^7.15.8
-    ajv: ^8.6.3
-    ajv-error-tree: 0.0.5
-    astring: ^1.8.4
-    chalk: ^5.0.1
-    glob: ^8.0.3
-    htmlparser2: ^8.0.1
-    klaw-sync: ^6.0.0
-    minimatch: ^5.1.0
-    minimist: ^1.2.7
-    net-socket-messaging: ^0.1.6
-    posix-socket: ^1.1.3
-    posix-socket-messaging: 0.0.1
-    prompts: ^2.4.2
-    semver: ^7.3.5
-    treeify: ^1.1.0
-    vlq: ^2.0.4
-    ws: ^8.13.0
-    yaml: ^2.1.1
-  dependenciesMeta:
-    posix-socket:
-      optional: true
-    posix-socket-messaging:
-      optional: true
-  bin:
-    appmap-agent-js: bin/appmap-agent-js.cjs
-  checksum: f8e8ffd90cb4f37c6ab5e22931a4ff9cbaa4a5eb0c3f112ce46b763ae59361bc031f963f83bee41043e633af4c260e27648230a50b3c1d7b37544bc9f56f6056
-  languageName: node
-  linkType: hard
-
-"@appland/appmap-validate@^2.3.0, @appland/appmap-validate@workspace:packages/validate":
+"@appland/appmap-validate@workspace:packages/validate":
   version: 0.0.0-use.local
   resolution: "@appland/appmap-validate@workspace:packages/validate"
   dependencies:
@@ -160,7 +125,6 @@ __metadata:
     "@octokit/types": ^11.1.0
     "@sidvind/better-ajv-errors": ^0.9.1
     "@swc/core": ^1.3.76
-    "@types/better-sqlite3": ^7.6.11
     "@types/cors": ^2.8.17
     "@types/fs-extra": ^9.0.13
     "@types/gitconfiglocal": ^2.0.1
@@ -182,7 +146,6 @@ __metadata:
     applicationinsights: ^2.1.4
     async: ^3.2.4
     axios: ^0.27.2
-    better-sqlite3: ^11.5.0
     body-parser: ^1.20.2
     boxen: ^5.0.1
     chalk: ^4.1.2
@@ -221,6 +184,7 @@ __metadata:
     minimatch: ^5.1.2
     moo: ^0.5.1
     node-fetch: 2.6.7
+    node-sqlite3-wasm: ^0.8.34
     open: ^8.2.1
     openapi-diff: ^0.23.6
     openapi-types: ^12.1.3
@@ -475,12 +439,14 @@ __metadata:
     eslint-plugin-unicorn: ^39.0.0
     fast-xml-parser: ^4.4.0
     jest: ^29.7.0
+    js-tiktoken: ^1.0.18
     js-yaml: ^4.1.0
     jsdom: ^16.6.0
     langchain: ^0.2.16
     mermaid: ^9
     openai: ^4.56.0
     p-retry: 4
+    prettier: ^3.3.3
     ts-jest: ^29.2.5
     ts-node: ^10.4.0
     tsc: ^2.0.3
@@ -523,7 +489,6 @@ __metadata:
   version: 0.0.0-use.local
   resolution: "@appland/scanner@workspace:packages/scanner"
   dependencies:
-    "@appland/appmap-agent-js": ^13.9.0
     "@appland/client": ^1.12.0
     "@appland/models": "workspace:^2.10.0"
     "@appland/openapi": "workspace:^1.7.0"
@@ -592,10 +557,8 @@ __metadata:
   version: 0.0.0-use.local
   resolution: "@appland/search@workspace:packages/search"
   dependencies:
-    "@types/better-sqlite3": ^7.6.11
     "@types/jest": ^29.5.4
     "@types/node": ^16
-    better-sqlite3: ^11.5.0
     eslint: ^9
     eslint-config-prettier: ^9
     eslint-plugin-eslint-comments: ^3.2.0
@@ -605,6 +568,7 @@ __metadata:
     eslint-plugin-promise: ^7.1.0
     isbinaryfile: ^5.0.4
     jest: ^29.7.0
+    node-sqlite3-wasm: ^0.8.34
     prettier: ^3.3.3
     ts-jest: ^29.2.5
     tsc: ^2.0.4
@@ -620,7 +584,6 @@ __metadata:
   version: 0.0.0-use.local
   resolution: "@appland/sequence-diagram@workspace:packages/sequence-diagram"
   dependencies:
-    "@appland/appmap-agent-js": ^13.9.0
     "@appland/models": "workspace:^2.10.0"
     "@appland/openapi": "workspace:^1.7.0"
     "@datastructures-js/priority-queue": ^6.1.3
@@ -2067,7 +2030,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@babel/parser@npm:^7.1.0, @babel/parser@npm:^7.14.7, @babel/parser@npm:^7.15.8, @babel/parser@npm:^7.16.4, @babel/parser@npm:^7.16.7, @babel/parser@npm:^7.17.0, @babel/parser@npm:^7.18.10, @babel/parser@npm:^7.20.5, @babel/parser@npm:^7.20.7, @babel/parser@npm:^7.21.0, @babel/parser@npm:^7.21.2, @babel/parser@npm:^7.4.3, @babel/parser@npm:^7.6.0, @babel/parser@npm:^7.7.0, @babel/parser@npm:^7.9.6":
+"@babel/parser@npm:^7.1.0, @babel/parser@npm:^7.14.7, @babel/parser@npm:^7.16.4, @babel/parser@npm:^7.16.7, @babel/parser@npm:^7.17.0, @babel/parser@npm:^7.18.10, @babel/parser@npm:^7.20.5, @babel/parser@npm:^7.20.7, @babel/parser@npm:^7.21.0, @babel/parser@npm:^7.21.2, @babel/parser@npm:^7.4.3, @babel/parser@npm:^7.6.0, @babel/parser@npm:^7.7.0, @babel/parser@npm:^7.9.6":
   version: 7.21.2
   resolution: "@babel/parser@npm:7.21.2"
   bin:
@@ -10670,15 +10633,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@types/better-sqlite3@npm:^7.6.11":
-  version: 7.6.11
-  resolution: "@types/better-sqlite3@npm:7.6.11"
-  dependencies:
-    "@types/node": "*"
-  checksum: 981740c78f97961bf1d8e78ed8bc841792cb25695afd9d920769d5b890d40ddf3724ace8a4db1adbf634c2f05c6efe3491ce9bf66818bacd93629ea060493546
-  languageName: node
-  linkType: hard
-
 "@types/body-parser@npm:*":
   version: 1.19.2
   resolution: "@types/body-parser@npm:1.19.2"
@@ -14174,7 +14128,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"ajv-error-tree@npm:0.0.5, ajv-error-tree@npm:^0.0.5":
+"ajv-error-tree@npm:^0.0.5":
   version: 0.0.5
   resolution: "ajv-error-tree@npm:0.0.5"
   checksum: b54060ab7dd075b6c5b86de899d39970fa714ece2f5918a1353d9f16bfd514618b01cb89281185ba98097b079764e25042b7e89cd8da4c18ab392207e19c7a61
@@ -15033,15 +14987,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"astring@npm:^1.8.4":
-  version: 1.8.4
-  resolution: "astring@npm:1.8.4"
-  bin:
-    astring: bin/astring
-  checksum: bc0b98087350c4a0c8a510d491d648cf8b299ec904629d5e0f5ae8d2ccc515cd27475327bb9729c7e92f4a4873adcd05cef15379d0f6f7293f1320319f0d24f0
-  languageName: node
-  linkType: hard
-
 "astring@npm:^1.8.6":
   version: 1.8.6
   resolution: "astring@npm:1.8.6"
@@ -15901,17 +15846,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"better-sqlite3@npm:^11.5.0":
-  version: 11.5.0
-  resolution: "better-sqlite3@npm:11.5.0"
-  dependencies:
-    bindings: ^1.5.0
-    node-gyp: latest
-    prebuild-install: ^7.1.1
-  checksum: 37acef8d4272ad57fe211aa5a2e177f95443eafb794c7db6c2d0dae0a4fe4c2ef508b8b970d82f1d9744d009677d82067279f45e27d4155a5e68a578f5c5c051
-  languageName: node
-  linkType: hard
-
 "bfj@npm:^6.1.1":
   version: 6.1.2
   resolution: "bfj@npm:6.1.2"
@@ -16959,13 +16893,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"chalk@npm:^5.0.1":
-  version: 5.2.0
-  resolution: "chalk@npm:5.2.0"
-  checksum: 03d8060277de6cf2fd567dc25fcf770593eb5bb85f460ce443e49255a30ff1242edd0c90a06a03803b0466ff0687a939b41db1757bec987113e83de89a003caa
-  languageName: node
-  linkType: hard
-
 "char-regex@npm:^1.0.2":
   version: 1.0.2
   resolution: "char-regex@npm:1.0.2"
@@ -20113,17 +20040,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"dom-serializer@npm:^2.0.0":
-  version: 2.0.0
-  resolution: "dom-serializer@npm:2.0.0"
-  dependencies:
-    domelementtype: ^2.3.0
-    domhandler: ^5.0.2
-    entities: ^4.2.0
-  checksum: cd1810544fd8cdfbd51fa2c0c1128ec3a13ba92f14e61b7650b5de421b88205fd2e3f0cc6ace82f13334114addb90ed1c2f23074a51770a8e9c1273acbc7f3e6
-  languageName: node
-  linkType: hard
-
 "dom-to-svg@npm:^0.12.2":
   version: 0.12.2
   resolution: "dom-to-svg@npm:0.12.2"
@@ -20163,13 +20079,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"domelementtype@npm:^2.3.0":
-  version: 2.3.0
-  resolution: "domelementtype@npm:2.3.0"
-  checksum: ee837a318ff702622f383409d1f5b25dd1024b692ef64d3096ff702e26339f8e345820f29a68bcdcea8cfee3531776b3382651232fbeae95612d6f0a75efb4f6
-  languageName: node
-  linkType: hard
-
 "domexception@npm:^1.0.1":
   version: 1.0.1
   resolution: "domexception@npm:1.0.1"
@@ -20197,15 +20106,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"domhandler@npm:^5.0.2, domhandler@npm:^5.0.3":
-  version: 5.0.3
-  resolution: "domhandler@npm:5.0.3"
-  dependencies:
-    domelementtype: ^2.3.0
-  checksum: 0f58f4a6af63e6f3a4320aa446d28b5790a009018707bce2859dcb1d21144c7876482b5188395a188dfa974238c019e0a1e610d2fc269a12b2c192ea2b0b131c
-  languageName: node
-  linkType: hard
-
 "dompurify@npm:2.4.3":
   version: 2.4.3
   resolution: "dompurify@npm:2.4.3"
@@ -20248,17 +20148,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"domutils@npm:^3.0.1":
-  version: 3.1.0
-  resolution: "domutils@npm:3.1.0"
-  dependencies:
-    dom-serializer: ^2.0.0
-    domelementtype: ^2.3.0
-    domhandler: ^5.0.3
-  checksum: e5757456ddd173caa411cfc02c2bb64133c65546d2c4081381a3bafc8a57411a41eed70494551aa58030be9e58574fcc489828bebd673863d39924fb4878f416
-  languageName: node
-  linkType: hard
-
 "dot-case@npm:^3.0.4":
   version: 3.0.4
   resolution: "dot-case@npm:3.0.4"
@@ -20647,13 +20536,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"entities@npm:^4.2.0, entities@npm:^4.4.0":
-  version: 4.5.0
-  resolution: "entities@npm:4.5.0"
-  checksum: 853f8ebd5b425d350bffa97dd6958143179a5938352ccae092c62d1267c4e392a039be1bae7d51b6e4ffad25f51f9617531fedf5237f15df302ccfb452cbf2d7
-  languageName: node
-  linkType: hard
-
 "env-ci@npm:^5.0.0":
   version: 5.5.0
   resolution: "env-ci@npm:5.5.0"
@@ -25221,18 +25103,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"htmlparser2@npm:^8.0.1":
-  version: 8.0.2
-  resolution: "htmlparser2@npm:8.0.2"
-  dependencies:
-    domelementtype: ^2.3.0
-    domhandler: ^5.0.3
-    domutils: ^3.0.1
-    entities: ^4.4.0
-  checksum: 29167a0f9282f181da8a6d0311b76820c8a59bc9e3c87009e21968264c2987d2723d6fde5a964d4b7b6cba663fca96ffb373c06d8223a85f52a6089ced942700
-  languageName: node
-  linkType: hard
-
 "http-browserify@npm:^1.7.0":
   version: 1.7.0
   resolution: "http-browserify@npm:1.7.0"
@@ -28738,12 +28608,12 @@ __metadata:
   languageName: node
   linkType: hard
 
-"js-tiktoken@npm:^1.0.12":
-  version: 1.0.14
-  resolution: "js-tiktoken@npm:1.0.14"
+"js-tiktoken@npm:^1.0.12, js-tiktoken@npm:^1.0.18":
+  version: 1.0.18
+  resolution: "js-tiktoken@npm:1.0.18"
   dependencies:
     base64-js: ^1.5.1
-  checksum: 0feb0f4186221f9db4cf7224313e83b8b869460f63c40634b47946d8312c2e05a66a2fd256ecd577f913563dce05f6d52fa9e3eba6a5e9270f82a4630a77fc2c
+  checksum: ee86872b8ef77c72952905e9c321e00a4005883e9807a7ed83539b1bfd625e01000e640b18071f5072a2d4e423e66d29f8239eb3b7d743d8d5203ec8e83b78d3
   languageName: node
   linkType: hard
 
@@ -29430,15 +29300,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"klaw-sync@npm:^6.0.0":
-  version: 6.0.0
-  resolution: "klaw-sync@npm:6.0.0"
-  dependencies:
-    graceful-fs: ^4.1.11
-  checksum: 0da397f8961313c3ef8f79fb63af9002cde5a8fb2aeb1a37351feff0dd6006129c790400c3f5c3b4e757bedcabb13d21ec0a5eaef5a593d59515d4f2c291e475
-  languageName: node
-  linkType: hard
-
 "klaw@npm:^1.0.0":
   version: 1.3.1
   resolution: "klaw@npm:1.3.1"
@@ -31775,7 +31636,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"minimist@npm:^1.1.1, minimist@npm:^1.2.0, minimist@npm:^1.2.3, minimist@npm:^1.2.5, minimist@npm:^1.2.6, minimist@npm:^1.2.7, minimist@npm:^1.2.8":
+"minimist@npm:^1.1.1, minimist@npm:^1.2.0, minimist@npm:^1.2.3, minimist@npm:^1.2.5, minimist@npm:^1.2.6, minimist@npm:^1.2.8":
   version: 1.2.8
   resolution: "minimist@npm:1.2.8"
   checksum: 75a6d645fb122dad29c06a7597bddea977258957ed88d7a6df59b5cd3fe4a527e253e9bbf2e783e4b73657f9098b96a5fe96ab8a113655d4109108577ecf85b0
@@ -32275,13 +32136,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"net-socket-messaging@npm:^0.1.6":
-  version: 0.1.6
-  resolution: "net-socket-messaging@npm:0.1.6"
-  checksum: de185dfe32c25750874fbc1e4ed5dfcfb12d8fa80f27dd167b84a0ed2bb9acc75f0d7433e9c2c7dad1c2659f6c43632a510b1aef381dc3e4f7e5defeb41d946b
-  languageName: node
-  linkType: hard
-
 "netstats@npm:0.0.6":
   version: 0.0.6
   resolution: "netstats@npm:0.0.6"
@@ -32507,14 +32361,14 @@ __metadata:
   languageName: node
   linkType: hard
 
-"node-gyp@npm:^8.4.0, node-gyp@npm:latest":
-  version: 8.4.1
-  resolution: "node-gyp@npm:8.4.1"
+"node-gyp@npm:^9.0.0":
+  version: 9.0.0
+  resolution: "node-gyp@npm:9.0.0"
   dependencies:
     env-paths: ^2.2.0
     glob: ^7.1.4
     graceful-fs: ^4.2.6
-    make-fetch-happen: ^9.1.0
+    make-fetch-happen: ^10.0.3
     nopt: ^5.0.0
     npmlog: ^6.0.0
     rimraf: ^3.0.2
@@ -32523,19 +32377,19 @@ __metadata:
     which: ^2.0.2
   bin:
     node-gyp: bin/node-gyp.js
-  checksum: 341710b5da39d3660e6a886b37e210d33f8282047405c2e62c277bcc744c7552c5b8b972ebc3a7d5c2813794e60cc48c3ebd142c46d6e0321db4db6c92dd0355
+  checksum: 4d8ef8860f7e4f4d86c91db3f519d26ed5cc23b48fe54543e2afd86162b4acbd14f21de42a5db344525efb69a991e021b96a68c70c6e2d5f4a5cb770793da6d3
   languageName: node
   linkType: hard
 
-"node-gyp@npm:^9.0.0":
-  version: 9.0.0
-  resolution: "node-gyp@npm:9.0.0"
+"node-gyp@npm:^9.1.0":
+  version: 9.3.1
+  resolution: "node-gyp@npm:9.3.1"
   dependencies:
     env-paths: ^2.2.0
     glob: ^7.1.4
     graceful-fs: ^4.2.6
     make-fetch-happen: ^10.0.3
-    nopt: ^5.0.0
+    nopt: ^6.0.0
     npmlog: ^6.0.0
     rimraf: ^3.0.2
     semver: ^7.3.5
@@ -32543,19 +32397,19 @@ __metadata:
     which: ^2.0.2
   bin:
     node-gyp: bin/node-gyp.js
-  checksum: 4d8ef8860f7e4f4d86c91db3f519d26ed5cc23b48fe54543e2afd86162b4acbd14f21de42a5db344525efb69a991e021b96a68c70c6e2d5f4a5cb770793da6d3
+  checksum: b860e9976fa645ca0789c69e25387401b4396b93c8375489b5151a6c55cf2640a3b6183c212b38625ef7c508994930b72198338e3d09b9d7ade5acc4aaf51ea7
   languageName: node
   linkType: hard
 
-"node-gyp@npm:^9.1.0":
-  version: 9.3.1
-  resolution: "node-gyp@npm:9.3.1"
+"node-gyp@npm:latest":
+  version: 8.4.1
+  resolution: "node-gyp@npm:8.4.1"
   dependencies:
     env-paths: ^2.2.0
     glob: ^7.1.4
     graceful-fs: ^4.2.6
-    make-fetch-happen: ^10.0.3
-    nopt: ^6.0.0
+    make-fetch-happen: ^9.1.0
+    nopt: ^5.0.0
     npmlog: ^6.0.0
     rimraf: ^3.0.2
     semver: ^7.3.5
@@ -32563,7 +32417,7 @@ __metadata:
     which: ^2.0.2
   bin:
     node-gyp: bin/node-gyp.js
-  checksum: b860e9976fa645ca0789c69e25387401b4396b93c8375489b5151a6c55cf2640a3b6183c212b38625ef7c508994930b72198338e3d09b9d7ade5acc4aaf51ea7
+  checksum: 341710b5da39d3660e6a886b37e210d33f8282047405c2e62c277bcc744c7552c5b8b972ebc3a7d5c2813794e60cc48c3ebd142c46d6e0321db4db6c92dd0355
   languageName: node
   linkType: hard
 
@@ -32664,6 +32518,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"node-sqlite3-wasm@npm:^0.8.34":
+  version: 0.8.34
+  resolution: "node-sqlite3-wasm@npm:0.8.34"
+  checksum: c6c9b2c75d714badaa41ac66a9ed6a30a718252d9259268bb96d899308aa9024c8766a0995c10e5225e7c57cc8c1483ef9279c3b6e97ddd45b6dba3af3e6e551
+  languageName: node
+  linkType: hard
+
 "non-layered-tidy-tree-layout@npm:^2.0.2":
   version: 2.0.2
   resolution: "non-layered-tidy-tree-layout@npm:2.0.2"
@@ -34691,24 +34552,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"posix-socket-messaging@npm:0.0.1":
-  version: 0.0.1
-  resolution: "posix-socket-messaging@npm:0.0.1"
-  dependencies:
-    posix-socket: ^1.0.1
-  checksum: d6877cf8c1dbd8e55b69352e795b1944a3a7325b21465a15334bb142a61863da8fa07c3ea39cdd04fb27b9bdf861589ac63ab97a16efc436475babdd54f4d8a1
-  languageName: node
-  linkType: hard
-
-"posix-socket@npm:^1.0.1, posix-socket@npm:^1.1.3":
-  version: 1.1.4
-  resolution: "posix-socket@npm:1.1.4"
-  dependencies:
-    node-gyp: ^8.4.0
-  conditions: (os=linux | os=darwin)
-  languageName: node
-  linkType: hard
-
 "possible-typed-array-names@npm:^1.0.0":
   version: 1.0.0
   resolution: "possible-typed-array-names@npm:1.0.0"
@@ -35376,28 +35219,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"prebuild-install@npm:^7.1.1":
-  version: 7.1.2
-  resolution: "prebuild-install@npm:7.1.2"
-  dependencies:
-    detect-libc: ^2.0.0
-    expand-template: ^2.0.3
-    github-from-package: 0.0.0
-    minimist: ^1.2.3
-    mkdirp-classic: ^0.5.3
-    napi-build-utils: ^1.0.1
-    node-abi: ^3.3.0
-    pump: ^3.0.0
-    rc: ^1.2.7
-    simple-get: ^4.0.0
-    tar-fs: ^2.0.0
-    tunnel-agent: ^0.6.0
-  bin:
-    prebuild-install: bin.js
-  checksum: 543dadf8c60e004ae9529e6013ca0cbeac8ef38b5f5ba5518cb0b622fe7f8758b34e4b5cb1a791db3cdc9d2281766302df6088bd1a225f206925d6fee17d6c5c
-  languageName: node
-  linkType: hard
-
 "prelude-ls@npm:^1.2.1":
   version: 1.2.1
   resolution: "prelude-ls@npm:1.2.1"
@@ -35687,7 +35508,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"prompts@npm:^2.0.1, prompts@npm:^2.4.0, prompts@npm:^2.4.2":
+"prompts@npm:^2.0.1, prompts@npm:^2.4.0":
   version: 2.4.2
   resolution: "prompts@npm:2.4.2"
   dependencies:
@@ -42063,13 +41884,6 @@ typescript@~4.4.3:
   languageName: node
   linkType: hard
 
-"vlq@npm:^2.0.4":
-  version: 2.0.4
-  resolution: "vlq@npm:2.0.4"
-  checksum: b2ed0d3a5423f34bba98a18250f8b13a96eebff9c8f9427fa9cd78065d31f35641f6fd659c5642253b79532000a37aec0582abac95d1ef4af2cd0c96a716f1b6
-  languageName: node
-  linkType: hard
-
 "vm-browserify@npm:^1.0.1":
   version: 1.1.2
   resolution: "vm-browserify@npm:1.1.2"
@@ -43174,21 +42988,6 @@ typescript@~4.4.3:
   languageName: node
   linkType: hard
 
-"ws@npm:^8.13.0":
-  version: 8.13.0
-  resolution: "ws@npm:8.13.0"
-  peerDependencies:
-    bufferutil: ^4.0.1
-    utf-8-validate: ">=5.0.2"
-  peerDependenciesMeta:
-    bufferutil:
-      optional: true
-    utf-8-validate:
-      optional: true
-  checksum: 53e991bbf928faf5dc6efac9b8eb9ab6497c69feeb94f963d648b7a3530a720b19ec2e0ec037344257e05a4f35bd9ad04d9de6f289615ffb133282031b18c61c
-  languageName: node
-  linkType: hard
-
 "ws@npm:^8.2.3":
   version: 8.5.0
   resolution: "ws@npm:8.5.0"
@@ -43298,13 +43097,6 @@ typescript@~4.4.3:
   languageName: node
   linkType: hard
 
-"yaml@npm:^2.1.1":
-  version: 2.3.1
-  resolution: "yaml@npm:2.3.1"
-  checksum: 2c7bc9a7cd4c9f40d3b0b0a98e370781b68b8b7c4515720869aced2b00d92f5da1762b4ffa947f9e795d6cd6b19f410bd4d15fdd38aca7bd96df59bd9486fb54
-  languageName: node
-  linkType: hard
-
 "yaml@npm:^2.2.1":
   version: 2.4.0
   resolution: "yaml@npm:2.4.0"