Skip to content

Feat/locomo accuracy improvements #55

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
kaghni wants to merge 9 commits into from
Closed

Feat/locomo accuracy improvements #55

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
a1140a5
feat: cross-table grep enrichment for improved retrieval accuracy
kaghni Apr 16, 2026
97ac6d0
fix: index.md generation works with any table structure
kaghni Apr 16, 2026
40ed163
feat: auto-convert unsupported memory commands to file reads
kaghni Apr 16, 2026
c3f2764
feat: return full wiki summaries in grep results
kaghni Apr 16, 2026
c4929b9
feat: smart multi-word OR grep for broader search coverage
kaghni Apr 16, 2026
c325033
feat: include session date in grep output for temporal reasoning
kaghni Apr 16, 2026
02521d2
fix: address PR review feedback — security and consistency
kaghni Apr 17, 2026
367eb63
chore: address remaining PR review comment + rebuild bundles
kaghni Apr 17, 2026
7ebc267
fix: narrow auto-read conversion to clean interpreter reads only
kaghni Apr 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 69 additions & 8 deletions claude-code/bundle/pre-tool-use.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -425,7 +425,14 @@ async function handleGrepDirect(api, table, sessionsTable, params) {
const hasRegexMeta = !fixedString && /[.*+?^${}()|[\]\\]/.test(pattern);
let rows = [];
if (!hasRegexMeta) {
const contentFilter = ` AND summary ${likeOp} '%${escapedLike}%'`;
const words = pattern.split(/\s+/).filter((w) => w.length > 2);
let contentFilter;
if (words.length > 1) {
const wordFilters = words.slice(0, 4).map((w) => `summary ${likeOp} '%${sqlLike(w)}%'`);
contentFilter = ` AND (${wordFilters.join(" OR ")})`;
} else {
contentFilter = ` AND summary ${likeOp} '%${escapedLike}%'`;
}
try {
rows = await api.query(`SELECT path, summary AS content FROM "${table}" WHERE 1=1${pathFilter}${contentFilter} LIMIT 100`);
} catch {
Expand All @@ -438,6 +445,29 @@ async function handleGrepDirect(api, table, sessionsTable, params) {
rows = [];
}
}
const output = [];
if (!hasRegexMeta) {
const memoryTable = table.endsWith("_sessions") ? table.replace(/_sessions$/, "_memory") : sessionsTable !== table ? sessionsTable : null;
if (memoryTable && memoryTable !== table) {
try {
const words2 = pattern.split(/\s+/).filter((w) => w.length > 2);
const contentFilter = words2.length > 1 ? ` AND (${words2.slice(0, 4).map((w) => `summary ${likeOp} '%${sqlLike(w)}%'`).join(" OR ")})` : ` AND summary ${likeOp} '%${escapedLike}%'`;
const summaryRows = await api.query(`SELECT path, summary AS content FROM "${memoryTable}" WHERE 1=1${contentFilter} LIMIT 20`);
if (summaryRows.length > 0) {
for (const sr of summaryRows) {
const sp = sr["path"];
const sc = sr["content"];
if (sc) {
output.push(`=== ${sp} ===`);
output.push(sc);
output.push("");
}
}
}
} catch {
}
}
}
let reStr = fixedString ? pattern.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") : pattern;
if (wordMatch)
reStr = `\\b${reStr}\\b`;
Expand All @@ -447,13 +477,14 @@ async function handleGrepDirect(api, table, sessionsTable, params) {
} catch {
re = new RegExp(pattern.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), ignoreCase ? "i" : "");
}
const output = [];
const multi = rows.length > 1;
for (const row of rows) {
const p = row["path"];
const text = row["content"];
if (!text)
continue;
const dateMatch = text.match(/"date_time"\s*:\s*"([^"]+)"/);
let sessionDate = dateMatch ? `[${dateMatch[1]}] ` : "";
const lines = text.split("\n");
const matched = [];
for (let i = 0; i < lines.length; i++) {
Expand All @@ -464,7 +495,9 @@ async function handleGrepDirect(api, table, sessionsTable, params) {
}
const prefix = multi ? `${p}:` : "";
const ln = lineNumber ? `${i + 1}:` : "";
matched.push(`${prefix}${ln}${lines[i]}`);
matched.push(`${prefix}${sessionDate}${ln}${lines[i]}`);
if (sessionDate)
sessionDate = "";
}
}
if (!filesOnly) {
Expand Down Expand Up @@ -690,6 +723,27 @@ async function main() {
const toolPath = input.tool_input.file_path ?? input.tool_input.path ?? "";
if (!shellCmd && (touchesMemory(cmd) || touchesMemory(toolPath))) {
const guidance = "[RETRY REQUIRED] The command you tried is not available for ~/.deeplake/memory/. This virtual filesystem only supports bash builtins: cat, ls, grep, echo, jq, head, tail, sed, awk, wc, sort, find, etc. python, python3, node, and curl are NOT available. You MUST rewrite your command using only the bash tools listed above and try again. For example, to parse JSON use: cat file.json | jq '.key'. To count keys: cat file.json | jq 'keys | length'.";
const isReadLike = /^(?:python3?|node|deno|bun|ruby|perl)\b/.test(cmd.trim());
const hasShellMeta = /[$`;|&<>()\\]/.test(cmd);
if (isReadLike && !hasShellMeta) {
const pathMatch = cmd.match(/~\/\.deeplake\/memory\/[\w./_-]+/) || toolPath.match(/~\/\.deeplake\/memory\/[\w./_-]+/);
const memPath = pathMatch ? pathMatch[0] : "";
const cleanPath = memPath ? rewritePaths(memPath) : "";
if (cleanPath && !cleanPath.endsWith("/")) {
log3(`unsupported command on file, converting to read: ${cleanPath}`);
console.log(JSON.stringify({
hookSpecificOutput: {
hookEventName: "PreToolUse",
permissionDecision: "allow",
updatedInput: {
command: `cat '${cleanPath.replace(/'/g, "'\\''")}'`,
description: "[DeepLake] converted unsupported command to file read"
}
}
}));
return;
}
}
log3(`unsupported command, returning guidance: ${cmd}`);
console.log(JSON.stringify({
hookSpecificOutput: {
Expand Down Expand Up @@ -787,14 +841,21 @@ async function main() {
if (rows.length > 0 && rows[0]["summary"]) {
content = rows[0]["summary"];
} else if (virtualPath === "/index.md") {
const idxRows = await api.query(`SELECT path, project, description, creation_date FROM "${table}" WHERE path LIKE '/summaries/%' ORDER BY creation_date DESC`);
const lines = ["# Memory Index", "", `${idxRows.length} sessions:`, ""];
const memTable = table.endsWith("_sessions") ? table.replace(/_sessions$/, "_memory") : table;
let idxRows = [];
try {
idxRows = await api.query(`SELECT path, description, creation_date FROM "${memTable}" ORDER BY path LIMIT 500`);
} catch {
}
if (idxRows.length === 0) {
idxRows = await api.query(`SELECT path, description, creation_date FROM "${table}" ORDER BY path LIMIT 500`);
}
const lines = ["# Memory Index", "", `${idxRows.length} entries:`, ""];
for (const r of idxRows) {
const p = r["path"];
const proj = r["project"] || "";
const desc = (r["description"] || "").slice(0, 120);
const desc = (r["description"] || "").slice(0, 100);
const date = (r["creation_date"] || "").slice(0, 10);
lines.push(`- [${p}](${p}) ${date} ${proj ? `[${proj}]` : ""} ${desc}`);
lines.push(`- [${p}](${p}) ${date} ${desc}`);
}
content = lines.join("\n");
}
Expand Down
39 changes: 36 additions & 3 deletions codex/bundle/pre-tool-use.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -426,7 +426,14 @@ async function handleGrepDirect(api, table, sessionsTable, params) {
const hasRegexMeta = !fixedString && /[.*+?^${}()|[\]\\]/.test(pattern);
let rows = [];
if (!hasRegexMeta) {
const contentFilter = ` AND summary ${likeOp} '%${escapedLike}%'`;
const words = pattern.split(/\s+/).filter((w) => w.length > 2);
let contentFilter;
if (words.length > 1) {
const wordFilters = words.slice(0, 4).map((w) => `summary ${likeOp} '%${sqlLike(w)}%'`);
contentFilter = ` AND (${wordFilters.join(" OR ")})`;
} else {
contentFilter = ` AND summary ${likeOp} '%${escapedLike}%'`;
}
try {
rows = await api.query(`SELECT path, summary AS content FROM "${table}" WHERE 1=1${pathFilter}${contentFilter} LIMIT 100`);
} catch {
Expand All @@ -439,6 +446,29 @@ async function handleGrepDirect(api, table, sessionsTable, params) {
rows = [];
}
}
const output = [];
if (!hasRegexMeta) {
const memoryTable = table.endsWith("_sessions") ? table.replace(/_sessions$/, "_memory") : sessionsTable !== table ? sessionsTable : null;
if (memoryTable && memoryTable !== table) {
try {
const words2 = pattern.split(/\s+/).filter((w) => w.length > 2);
const contentFilter = words2.length > 1 ? ` AND (${words2.slice(0, 4).map((w) => `summary ${likeOp} '%${sqlLike(w)}%'`).join(" OR ")})` : ` AND summary ${likeOp} '%${escapedLike}%'`;
const summaryRows = await api.query(`SELECT path, summary AS content FROM "${memoryTable}" WHERE 1=1${contentFilter} LIMIT 20`);
if (summaryRows.length > 0) {
for (const sr of summaryRows) {
const sp = sr["path"];
const sc = sr["content"];
if (sc) {
output.push(`=== ${sp} ===`);
output.push(sc);
output.push("");
}
}
}
} catch {
}
}
}
let reStr = fixedString ? pattern.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") : pattern;
if (wordMatch)
reStr = `\\b${reStr}\\b`;
Expand All @@ -448,13 +478,14 @@ async function handleGrepDirect(api, table, sessionsTable, params) {
} catch {
re = new RegExp(pattern.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), ignoreCase ? "i" : "");
}
const output = [];
const multi = rows.length > 1;
for (const row of rows) {
const p = row["path"];
const text = row["content"];
if (!text)
continue;
const dateMatch = text.match(/"date_time"\s*:\s*"([^"]+)"/);
let sessionDate = dateMatch ? `[${dateMatch[1]}] ` : "";
const lines = text.split("\n");
const matched = [];
for (let i = 0; i < lines.length; i++) {
Expand All @@ -465,7 +496,9 @@ async function handleGrepDirect(api, table, sessionsTable, params) {
}
const prefix = multi ? `${p}:` : "";
const ln = lineNumber ? `${i + 1}:` : "";
matched.push(`${prefix}${ln}${lines[i]}`);
matched.push(`${prefix}${sessionDate}${ln}${lines[i]}`);
if (sessionDate)
sessionDate = "";
}
}
if (!filesOnly) {
Expand Down
57 changes: 50 additions & 7 deletions src/hooks/grep-direct.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,25 +121,65 @@ export async function handleGrepDirect(
// Strategy: BM25 first (ranked, fast with index), LIKE fallback if BM25 fails.
let rows: Record<string, unknown>[] = [];

// Search primary table — for multi-word patterns, use OR to find any word
if (!hasRegexMeta) {
// BM25 ranked search disabled — CREATE INDEX causes oid errors on fresh tables.
// See bm25-oid-bug.sh. Using LIKE until Deeplake fixes the oid invalidation.
// When re-enabling, uncomment the BM25 block and make LIKE the fallback.
const contentFilter = ` AND summary ${likeOp} '%${escapedLike}%'`;
const words = pattern.split(/\s+/).filter((w: string) => w.length > 2);
let contentFilter: string;
if (words.length > 1) {
// Multi-word: search for any word (OR) to cast a wider net
const wordFilters = words.slice(0, 4).map((w: string) =>
`summary ${likeOp} '%${sqlLike(w)}%'`
);
contentFilter = ` AND (${wordFilters.join(" OR ")})`;
} else {
contentFilter = ` AND summary ${likeOp} '%${escapedLike}%'`;
}
try {
rows = await api.query(
`SELECT path, summary AS content FROM "${table}" WHERE 1=1${pathFilter}${contentFilter} LIMIT 100`,
);
} catch { rows = []; }
} else {
// Regex pattern — fetch all files under path, filter in-memory
try {
rows = await api.query(
`SELECT path, summary AS content FROM "${table}" WHERE 1=1${pathFilter} LIMIT 100`,
);
} catch { rows = []; }
}

const output: string[] = [];
// Cross-table enrichment: search the companion memory/summaries table
// for structured wiki-style context. Convention: if table is X_sessions
// or X, companion is X_memory. Summaries are prepended for priority.
if (!hasRegexMeta) {
const memoryTable = table.endsWith("_sessions")
? table.replace(/_sessions$/, "_memory")
: (sessionsTable !== table ? sessionsTable : null);
if (memoryTable && memoryTable !== table) {
try {
const words2 = pattern.split(/\s+/).filter((w: string) => w.length > 2);
const contentFilter = words2.length > 1
? ` AND (${words2.slice(0, 4).map((w: string) => `summary ${likeOp} '%${sqlLike(w)}%'`).join(" OR ")})`
: ` AND summary ${likeOp} '%${escapedLike}%'`;
const summaryRows = await api.query(
`SELECT path, summary AS content FROM "${memoryTable}" WHERE 1=1${contentFilter} LIMIT 20`,
);
if (summaryRows.length > 0) {
// Output full summaries directly (compact and structured)
for (const sr of summaryRows) {
const sp = sr["path"] as string;
const sc = sr["content"] as string;
if (sc) {
output.push(`=== ${sp} ===`);
output.push(sc);
output.push("");
}
}
}
} catch { /* best-effort — table may not exist */ }
}
}

// ── regex refinement ──
let reStr = fixedString
? pattern.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")
Expand All @@ -149,13 +189,15 @@ export async function handleGrepDirect(
try { re = new RegExp(reStr, ignoreCase ? "i" : ""); }
catch { re = new RegExp(pattern.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), ignoreCase ? "i" : ""); }

const output: string[] = [];
const multi = rows.length > 1;

for (const row of rows) {
const p = row["path"] as string;
const text = row["content"] as string;
if (!text) continue;
// Extract date from session JSON for temporal context
const dateMatch = text.match(/"date_time"\s*:\s*"([^"]+)"/);
let sessionDate = dateMatch ? `[${dateMatch[1]}] ` : "";

const lines = text.split("\n");
const matched: string[] = [];
Expand All @@ -165,7 +207,8 @@ export async function handleGrepDirect(
if (filesOnly) { output.push(p); break; }
const prefix = multi ? `${p}:` : "";
const ln = lineNumber ? `${i + 1}:` : "";
matched.push(`${prefix}${ln}${lines[i]}`);
matched.push(`${prefix}${sessionDate}${ln}${lines[i]}`);
Copy link
Copy Markdown

@claude claude Bot Apr 16, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The @ts-ignore is unnecessary — sessionDate && (sessionDate = "") is valid TypeScript; the suppressor could hide a real type error on the next line. Use a plain if instead.

Suggested change
matched.push(`${prefix}${sessionDate}${ln}${lines[i]}`);
if (sessionDate) sessionDate = "";

if (sessionDate) sessionDate = "";
}
}

Expand Down
54 changes: 46 additions & 8 deletions src/hooks/pre-tool-use.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,34 @@ async function main(): Promise<void> {
"python, python3, node, and curl are NOT available. " +
"You MUST rewrite your command using only the bash tools listed above and try again. " +
"For example, to parse JSON use: cat file.json | jq '.key'. To count keys: cat file.json | jq 'keys | length'.";
// Only auto-convert when the user is clearly trying to READ a memory
// file with an unsupported interpreter (python, node, ruby, perl).
// curl/wget and anything with shell metacharacters fall through to the
// RETRY guidance below — converting them would hide actual intent.
const isReadLike = /^(?:python3?|node|deno|bun|ruby|perl)\b/.test(cmd.trim());
const hasShellMeta = /[$`;|&<>()\\]/.test(cmd);
if (isReadLike && !hasShellMeta) {
const pathMatch = cmd.match(/~\/\.deeplake\/memory\/[\w./_-]+/)
|| toolPath.match(/~\/\.deeplake\/memory\/[\w./_-]+/);
const memPath = pathMatch ? pathMatch[0] : "";
const cleanPath = memPath ? rewritePaths(memPath) : "";
if (cleanPath && !cleanPath.endsWith("/")) {
log(`unsupported command on file, converting to read: ${cleanPath}`);
console.log(JSON.stringify({
hookSpecificOutput: {
hookEventName: "PreToolUse",
permissionDecision: "allow",
updatedInput: {
command: `cat '${cleanPath.replace(/'/g, "'\\''")}'`,
description: "[DeepLake] converted unsupported command to file read",
},
},
}));
return;
}
}


log(`unsupported command, returning guidance: ${cmd}`);
console.log(JSON.stringify({
hookSpecificOutput: {
Expand Down Expand Up @@ -289,17 +317,27 @@ async function main(): Promise<void> {
if (rows.length > 0 && rows[0]["summary"]) {
content = rows[0]["summary"] as string;
} else if (virtualPath === "/index.md") {
// Virtual index — generate from metadata
const idxRows = await api.query(
`SELECT path, project, description, creation_date FROM "${table}" WHERE path LIKE '/summaries/%' ORDER BY creation_date DESC`
);
const lines = ["# Memory Index", "", `${idxRows.length} sessions:`, ""];
// Virtual index — generate from all entries in memory table
// Try companion memory table first (has descriptions), fall back to primary
const memTable = table.endsWith("_sessions")
? table.replace(/_sessions$/, "_memory") : table;
let idxRows: Record<string, unknown>[] = [];
try {
idxRows = await api.query(
`SELECT path, description, creation_date FROM "${memTable}" ORDER BY path LIMIT 500`
);
} catch { /* companion table may not exist */ }
if (idxRows.length === 0) {
idxRows = await api.query(
`SELECT path, description, creation_date FROM "${table}" ORDER BY path LIMIT 500`
);
}
const lines = ["# Memory Index", "", `${idxRows.length} entries:`, ""];
for (const r of idxRows) {
const p = r["path"] as string;
const proj = r["project"] as string || "";
const desc = (r["description"] as string || "").slice(0, 120);
const desc = (r["description"] as string || "").slice(0, 100);
const date = (r["creation_date"] as string || "").slice(0, 10);
lines.push(`- [${p}](${p}) ${date} ${proj ? `[${proj}]` : ""} ${desc}`);
lines.push(`- [${p}](${p}) ${date} ${desc}`);
}
content = lines.join("\n");
}
Expand Down
Loading