diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 38c2ea0..85efda7 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -10,13 +10,6 @@ "homepage": "https://github.com/bitflight-devops/hallucination-detector", "repository": "https://github.com/bitflight-devops/hallucination-detector", "license": "MIT", - "keywords": [ - "hallucination", - "speculation", - "causality", - "verification", - "stop-hook", - "quality" - ], + "keywords": ["hallucination", "speculation", "causality", "verification", "stop-hook", "quality"], "commands": "./commands/" } diff --git a/.cursor-plugin/plugin.json b/.cursor-plugin/plugin.json index 38c2ea0..85efda7 100644 --- a/.cursor-plugin/plugin.json +++ b/.cursor-plugin/plugin.json @@ -10,13 +10,6 @@ "homepage": "https://github.com/bitflight-devops/hallucination-detector", "repository": "https://github.com/bitflight-devops/hallucination-detector", "license": "MIT", - "keywords": [ - "hallucination", - "speculation", - "causality", - "verification", - "stop-hook", - "quality" - ], + "keywords": ["hallucination", "speculation", "causality", "verification", "stop-hook", "quality"], "commands": "./commands/" } diff --git a/package-lock.json b/package-lock.json index 395b98b..d2b6f25 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "hallucination-detector", - "version": "1.15.3", + "version": "1.17.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "hallucination-detector", - "version": "1.15.3", + "version": "1.17.0", "license": "MIT", "devDependencies": { "@biomejs/biome": "^2.4.4", @@ -28,7 +28,7 @@ "yaml": "^2.8.2" }, "engines": { - "node": ">=22.5.0" + "node": ">=22.6.0" } }, "node_modules/@actions/core": { diff --git a/package.json b/package.json index bf3a35c..6894857 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ }, "homepage": "https://github.com/bitflight-devops/hallucination-detector#readme", "engines": { - "node": ">=22.5.0" + "node": ">=22.6.0" }, "devDependencies": { "@biomejs/biome": "^2.4.4", diff --git a/scripts/hallucination-annotate.cjs b/scripts/hallucination-annotate.cjs index 7065028..e9986c0 100644 --- a/scripts/hallucination-annotate.cjs +++ b/scripts/hallucination-annotate.cjs @@ -14,7 +14,7 @@ 'use strict'; const fs = require('node:fs'); -const { DEFAULT_WEIGHTS } = require('./hallucination-config.cjs'); +const { DEFAULT_WEIGHTS } = require('./hallucination-config-defaults.cjs'); // --------------------------------------------------------------------------- // Argument parsing diff --git a/scripts/hallucination-audit-stop.cjs b/scripts/hallucination-audit-stop.cjs index 79073c5..8e309c6 100644 --- a/scripts/hallucination-audit-stop.cjs +++ b/scripts/hallucination-audit-stop.cjs @@ -41,17 +41,33 @@ try { } const { - loadConfig, - loadWeights, + safeLoadConfig, + safeLoadWeights, DEFAULT_WEIGHTS, DEFAULT_THRESHOLDS, DEFAULT_CONFIDENCE_WEIGHTS, - isValidCategoryThreshold, -} = require('./hallucination-config.cjs'); -const { - validateClaimStructure, - CLAIM_LABEL_ALTERNATION, -} = require('./hallucination-claim-structure.cjs'); +} = require('./hallucination-config-safe.cjs'); + +let _isValidCategoryThreshold = null; +try { + ({ + isValidCategoryThreshold: _isValidCategoryThreshold, + } = require('./hallucination-config-validate.cjs')); +} catch { + /* validation unavailable — per-category thresholds will use global defaults */ +} +const isValidCategoryThreshold = _isValidCategoryThreshold || (() => false); + +let _claim_structure = null; +try { + _claim_structure = require('./hallucination-claim-structure.cjs'); +} catch { + /* claim structure analysis unavailable */ +} +const validateClaimStructure = + _claim_structure?.validateClaimStructure || + (() => ({ structured: false, valid: true, claims: [], errors: [] })); +const CLAIM_LABEL_ALTERNATION = _claim_structure?.CLAIM_LABEL_ALTERNATION || ''; // ============================================================================= // Telemetry @@ -2315,7 +2331,7 @@ function getLabelForScore(score, thresholds) { return 'HALLUCINATED'; } -// loadWeights and loadConfig imported from ./hallucination-config.cjs +// safeLoadWeights and safeLoadConfig imported from ./hallucination-config-safe.cjs /** * Score every sentence in a block of text. @@ -2424,9 +2440,10 @@ function emitJson(obj) { * @returns {string} */ // Built from CLAIM_LABEL_ALTERNATION so adding a new label requires one edit. -const LABELED_CLAIM_LINE_RE = new RegExp( - `^\\s*-?\\s*(?:\\[(?:${CLAIM_LABEL_ALTERNATION})\\])+\\[c\\d+\\].*`, -); +// Falls back to a never-match regex when the claim-structure module is unavailable. +const LABELED_CLAIM_LINE_RE = CLAIM_LABEL_ALTERNATION + ? new RegExp(`^\\s*-?\\s*(?:\\[(?:${CLAIM_LABEL_ALTERNATION})\\])+\\[c\\d+\\].*`) + : /(?!)/; const METADATA_LINE_RE = /^\s+(?:Evidence|Basis|Missing|Contradicted by):\s*/i; function stripLabeledClaimLines(text) { @@ -2733,7 +2750,7 @@ function main() { // returns defaults — that is acceptable. const assistantMeta = getLastAssistantMeta(entries); - const config = loadConfig(); + const config = safeLoadConfig(); const maxBlocks = config.maxBlocksPerSession ?? 2; // Template validation: check for observation template blocks before structural @@ -3171,9 +3188,9 @@ module.exports = { scoreSentence, aggregateWeightedScore, getLabelForScore, - // Re-exported from hallucination-config.cjs for backward compatibility - loadWeights, - loadConfig, + // Re-exported from hallucination-config-safe.cjs for backward compatibility + loadWeights: safeLoadWeights, + loadConfig: safeLoadConfig, scoreText, DEFAULT_WEIGHTS, DEFAULT_THRESHOLDS, diff --git a/scripts/hallucination-config-defaults.cjs b/scripts/hallucination-config-defaults.cjs new file mode 100644 index 0000000..77b6cc7 --- /dev/null +++ b/scripts/hallucination-config-defaults.cjs @@ -0,0 +1,96 @@ +#!/usr/bin/env node +/** + * Default configuration constants for hallucination-detector hooks. + * Zero dependencies — Node.js built-ins only. + */ + +'use strict'; + +/** + * Default weights for each detection category. + * Weights are relative severity signals; `aggregateWeightedScore` normalizes + * by their sum so aggregate scores always remain in [0, 1] regardless of + * whether the weights themselves sum to 1.0. + */ +const DEFAULT_WEIGHTS = { + speculation_language: 0.25, + causality_language: 0.3, + pseudo_quantification: 0.15, + completeness_claim: 0.2, + // fabricated_source: reserved for future implementation (issue #18) + evaluative_design_claim: 0.4, + internal_contradiction: 0.35, + unsupported_absence: 0.7, + ungrounded_behavioral_assertion: 0.5, +}; + +/** + * Default score thresholds for three-tier label classification. + * - uncertain: scores >= this value are labelled UNCERTAIN (not GROUNDED) + * - hallucinated: scores > this value are labelled HALLUCINATED + */ +const DEFAULT_THRESHOLDS = { + uncertain: 0.3, + hallucinated: 0.6, +}; + +/** + * Default weights for the four confidence-score components. + * These control how much each factor contributes to the per-match + * confidence integer in [0, 100]. + * + * - patternStrength: contribution of the pattern's inherent severity + * - evidenceProximity: contribution of evidence markers near the match + * - categoryStacking: bonus when multiple categories fire in the same sentence + * - contextDensity: bonus when multiple matches cluster within 200 chars + */ +const DEFAULT_CONFIDENCE_WEIGHTS = { + patternStrength: 0.4, + evidenceProximity: 0.25, + categoryStacking: 0.2, + contextDensity: 0.15, +}; + +/** + * Default full configuration object. + */ +const DEFAULT_CONFIG = { + weights: DEFAULT_WEIGHTS, + thresholds: DEFAULT_THRESHOLDS, + introspect: false, + introspectOutputPath: null, + // Shadow mode: log would-block events without actually blocking. + dryRun: false, + // Global settings + severity: 'error', + maxTriggersPerResponse: 20, + maxBlocksPerSession: null, + outputFormat: 'text', + debug: false, + // Per-category settings (keyed by category name) + categories: {}, + // Filtering settings + ignorePatterns: [], + ignoreBlocks: [], + evidenceMarkers: [], + allowlist: [], + // Response settings + responseTemplates: {}, + includeContext: true, + contextLines: 2, + // Session-type gating + warnOnly: false, // log telemetry but never emit a block to stdout + ignoreCategories: [], // category names skipped entirely (still written to telemetry with was_ignored=1) + blockSubagents: false, // block when hook_event_name is SubagentStop + blockUserSessions: true, // block when hook_event_name is Stop (user-facing session) + // Confidence scoring + confidenceWeights: DEFAULT_CONFIDENCE_WEIGHTS, + reportingThreshold: 50, // minimum confidence [0,100] for a match to appear in block reason text +}; + +module.exports = { + DEFAULT_WEIGHTS, + DEFAULT_THRESHOLDS, + DEFAULT_CONFIDENCE_WEIGHTS, + DEFAULT_CONFIG, +}; diff --git a/scripts/hallucination-config-merge.cjs b/scripts/hallucination-config-merge.cjs new file mode 100644 index 0000000..99645da --- /dev/null +++ b/scripts/hallucination-config-merge.cjs @@ -0,0 +1,100 @@ +#!/usr/bin/env node +/** + * Deep freeze and deep merge utilities for hallucination-detector config objects. + * Zero dependencies — Node.js built-ins only. + */ + +'use strict'; + +/** + * Recursively freeze an object and all nested plain objects / arrays. + * @param {*} obj + * @returns {*} The frozen value. + */ +function deepFreeze(obj) { + if (obj === null || typeof obj !== 'object') return obj; + for (const val of Object.values(obj)) { + deepFreeze(val); + } + return Object.freeze(obj); +} + +/** + * Deep-merge two config objects. Rules: + * - Plain objects are merged recursively. + * - `categories..customPatterns` arrays are concatenated unless the override + * has `replacePatterns: true` for that category. + * - All other arrays are replaced by the override value. + * - Scalar values are replaced by the override value. + * + * Neither argument is mutated; a new object is returned. + * + * @param {object} base - Lower-priority config. + * @param {object} override - Higher-priority config (wins on conflict). + * @returns {object} Merged config. + */ +function mergeConfig(base, override) { + if (!override || typeof override !== 'object' || Array.isArray(override)) return base; + if (!base || typeof base !== 'object' || Array.isArray(base)) return override; + + const result = { ...base }; + + for (const key of Object.keys(override)) { + if (key === '__proto__' || key === 'constructor' || key === 'prototype') continue; + const overVal = override[key]; + const baseVal = base[key]; + + if (key === 'categories') { + // Merge categories map, with special customPatterns concatenation logic. + const baseCats = + typeof baseVal === 'object' && baseVal !== null && !Array.isArray(baseVal) ? baseVal : {}; + const overCats = + typeof overVal === 'object' && overVal !== null && !Array.isArray(overVal) ? overVal : {}; + const merged = { ...baseCats }; + for (const catName of Object.keys(overCats)) { + if (catName === '__proto__' || catName === 'constructor' || catName === 'prototype') + continue; + const baseCat = baseCats[catName] || {}; + const overCat = overCats[catName]; + if (typeof overCat !== 'object' || overCat === null) { + merged[catName] = overCat; + continue; + } + // Extract customPatterns and replacePatterns before spreading to handle + // replacePatterns:true correctly even when overCat.customPatterns is absent. + const { customPatterns: basePatterns, ...baseCatRest } = baseCat; + const { customPatterns: overPatterns, replacePatterns, ...overCatRest } = overCat; + const mergedCat = { ...baseCatRest, ...overCatRest }; + if (replacePatterns) { + mergedCat.customPatterns = overPatterns !== undefined ? overPatterns : []; + mergedCat.replacePatterns = true; + } else if (Array.isArray(basePatterns) && Array.isArray(overPatterns)) { + mergedCat.customPatterns = [...basePatterns, ...overPatterns]; + } else if (Array.isArray(basePatterns)) { + mergedCat.customPatterns = basePatterns; + } else if (Array.isArray(overPatterns)) { + mergedCat.customPatterns = overPatterns; + } + merged[catName] = mergedCat; + } + result[key] = merged; + } else if ( + typeof overVal === 'object' && + overVal !== null && + !Array.isArray(overVal) && + typeof baseVal === 'object' && + baseVal !== null && + !Array.isArray(baseVal) + ) { + // Both are plain objects — recurse. + result[key] = mergeConfig(baseVal, overVal); + } else { + // Scalar, array, or null — override wins. + result[key] = overVal; + } + } + + return result; +} + +module.exports = { mergeConfig, deepFreeze }; diff --git a/scripts/hallucination-config-safe.cjs b/scripts/hallucination-config-safe.cjs new file mode 100644 index 0000000..ffd6b2c --- /dev/null +++ b/scripts/hallucination-config-safe.cjs @@ -0,0 +1,79 @@ +#!/usr/bin/env node +/** + * Safe configuration loader — guarantees a valid config on any failure. + * + * Top-level requires are limited to pure-data modules that cannot throw. + * The full config loader is required lazily inside function bodies so that + * a syntax error or runtime failure in the loader chain never crashes a hook. + * + * Zero dependencies — Node.js built-ins only. + */ + +'use strict'; + +const { + DEFAULT_WEIGHTS, + DEFAULT_THRESHOLDS, + DEFAULT_CONFIDENCE_WEIGHTS, + DEFAULT_CONFIG, +} = require('./hallucination-config-defaults.cjs'); + +/** + * Load config with guaranteed fallback to defaults. + * Never throws — returns a frozen DEFAULT_CONFIG on any failure. + * @param {object} [opts] - Options forwarded to loadConfig. + * @returns {object} Frozen config object. + */ +function safeLoadConfig(opts) { + try { + const { loadConfig } = require('./hallucination-config.cjs'); + return loadConfig(opts); + } catch (err) { + process.stderr.write( + `[hallucination-detector] Config load error: ${err?.message ? err.message : String(err)}; using default config\n`, + ); + let deepFreeze = (x) => x; + try { + ({ deepFreeze } = require('./hallucination-config-merge.cjs')); + } catch { + /* fallback: no-op identity */ + } + return deepFreeze({ + ...DEFAULT_CONFIG, + weights: { ...DEFAULT_WEIGHTS }, + thresholds: { ...DEFAULT_THRESHOLDS }, + confidenceWeights: { ...DEFAULT_CONFIDENCE_WEIGHTS }, + categories: {}, + ignorePatterns: [], + ignoreBlocks: [], + evidenceMarkers: [], + allowlist: [], + responseTemplates: {}, + }); + } +} + +/** + * Load weights with guaranteed fallback. + * @returns {object} Validated weights map. + */ +function safeLoadWeights() { + try { + const { loadWeights } = require('./hallucination-config.cjs'); + return loadWeights(); + } catch (err) { + process.stderr.write( + `[hallucination-detector] Config load error: ${err?.message ? err.message : String(err)}; using default weights\n`, + ); + return { ...DEFAULT_WEIGHTS }; + } +} + +module.exports = { + safeLoadConfig, + safeLoadWeights, + DEFAULT_WEIGHTS, + DEFAULT_THRESHOLDS, + DEFAULT_CONFIDENCE_WEIGHTS, + DEFAULT_CONFIG, +}; diff --git a/scripts/hallucination-config-toml.cjs b/scripts/hallucination-config-toml.cjs new file mode 100644 index 0000000..0d090af --- /dev/null +++ b/scripts/hallucination-config-toml.cjs @@ -0,0 +1,179 @@ +#!/usr/bin/env node +/** + * Minimal TOML parser for hallucination-detector hooks. + * Handles the subset needed for pyproject.toml sections. + * Supports: simple key-value pairs (string, number, boolean), section headers + * ([section] / [section.sub]), and single-line arrays of strings or inline tables. + * Zero dependencies — Node.js built-ins only. + */ + +'use strict'; + +/** + * Split `content` on `sep` at depth 0, respecting nested brackets and quoted strings. + * @param {string} content + * @param {string} sep - Single separator character. + * @returns {string[]} + */ +function splitTopLevel(content, sep) { + const parts = []; + let depth = 0; + let inStr = false; + let strChar = ''; + let start = 0; + for (let i = 0; i < content.length; i++) { + const ch = content[i]; + if (inStr) { + if (ch === '\\' && i + 1 < content.length) { + i++; + continue; + } + if (ch === strChar) inStr = false; + } else if (ch === '"' || ch === "'") { + inStr = true; + strChar = ch; + } else if (ch === '[' || ch === '{') { + depth++; + } else if (ch === ']' || ch === '}') { + depth--; + } else if (ch === sep && depth === 0) { + parts.push(content.slice(start, i)); + start = i + 1; + } + } + parts.push(content.slice(start)); + return parts; +} + +/** + * Strip a `#` inline comment from a TOML value string, respecting quoted strings. + * @param {string} valStr + * @returns {string} + */ +function stripTomlInlineComment(valStr) { + let inStr = false; + let strChar = ''; + for (let i = 0; i < valStr.length; i++) { + const ch = valStr[i]; + if (inStr) { + if (ch === '\\' && i + 1 < valStr.length) { + i++; + continue; + } + if (ch === strChar) inStr = false; + } else if (ch === '"' || ch === "'") { + inStr = true; + strChar = ch; + } else if (ch === '#') { + return valStr.slice(0, i).trim(); + } + } + return valStr.trim(); +} + +/** + * Parse a single TOML value string into a JS value. + * @param {string} valStr + * @returns {*} + */ +function parseTomlValue(valStr) { + const s = stripTomlInlineComment(valStr); + if (!s) return null; + // Quoted string + if (s.startsWith('"') && s.endsWith('"')) { + // Double-quoted: apply escape decoding in a single pass to avoid ordering issues. + // e.g. "\\n" → literal backslash + n (not newline). + return s.slice(1, -1).replace(/\\(\\|n|t|")/g, (_, c) => { + if (c === '\\') return '\\'; + if (c === 'n') return '\n'; + if (c === 't') return '\t'; + return '"'; + }); + } + if (s.startsWith("'") && s.endsWith("'")) { + // Single-quoted (literal): no escape processing + return s.slice(1, -1); + } + if (s === 'true') return true; + if (s === 'false') return false; + if (/^-?\d+(\.\d+)?$/.test(s)) return Number(s); + // Array + if (s.startsWith('[') && s.endsWith(']')) { + const inner = s.slice(1, -1).trim(); + if (!inner) return []; + return splitTopLevel(inner, ',') + .map((item) => item.trim()) + .filter((item) => item !== '') + .map((item) => parseTomlValue(item)); + } + // Inline table + if (s.startsWith('{') && s.endsWith('}')) { + return parseTomlInlineTable(s.slice(1, -1)); + } + return s; +} + +/** + * Parse a TOML inline table body (content between `{` and `}`). + * @param {string} content + * @returns {object} + */ +function parseTomlInlineTable(content) { + const table = Object.create(null); + if (!content.trim()) return table; + for (const pair of splitTopLevel(content.trim(), ',')) { + const p = pair.trim(); + const eqIdx = p.indexOf('='); + if (eqIdx === -1) continue; + const k = p.slice(0, eqIdx).trim(); + const v = p.slice(eqIdx + 1).trim(); + if (k) table[k] = parseTomlValue(v); + } + return table; +} + +/** + * Parse a TOML source string into a plain JS object. + * Only handles the subset needed for `[tool.hallucination-detector]` sections. + * + * @param {string} source - TOML source text. + * @returns {object} + */ +function parseToml(source) { + const result = Object.create(null); + let current = result; + + for (const rawLine of source.split('\n')) { + const line = rawLine.trim(); + if (!line || line.startsWith('#')) continue; + + // Section header: [key] or [key.subkey] (not array tables [[...]]) + if (line.startsWith('[') && !line.startsWith('[[')) { + const end = line.indexOf(']'); + if (end === -1) continue; + const sectionStr = line.slice(1, end).trim(); + // Split on '.' to get nested path (bare keys may contain hyphens) + const parts = sectionStr.split('.').map((p) => p.trim()); + current = result; + for (const part of parts) { + if (typeof current[part] !== 'object' || current[part] === null) { + current[part] = Object.create(null); + } + current = current[part]; + } + continue; + } + + // Key-value pair + const eqIdx = line.indexOf('='); + if (eqIdx === -1) continue; + const key = line.slice(0, eqIdx).trim(); + const valStr = line.slice(eqIdx + 1).trim(); + if (!key) continue; + current[key] = parseTomlValue(valStr); + } + + return result; +} + +module.exports = { parseToml }; diff --git a/scripts/hallucination-config-validate.cjs b/scripts/hallucination-config-validate.cjs new file mode 100644 index 0000000..753d03c --- /dev/null +++ b/scripts/hallucination-config-validate.cjs @@ -0,0 +1,262 @@ +#!/usr/bin/env node +/** + * Schema validation for hallucination-detector configuration objects. + * Zero dependencies — Node.js built-ins only. + */ + +'use strict'; + +const { + DEFAULT_WEIGHTS, + DEFAULT_CONFIDENCE_WEIGHTS, + DEFAULT_THRESHOLDS, +} = require('./hallucination-config-defaults.cjs'); + +const VALID_SEVERITIES = new Set(['error', 'warning', 'info']); +const VALID_OUTPUT_FORMATS = new Set(['text', 'json', 'jsonl']); + +/** + * Returns true when `t` is a valid thresholds object: a non-null plain object + * with both `uncertain` and `hallucinated` as finite numbers in [0,1] and + * `uncertain <= hallucinated`. + * + * @param {*} t - Value to check. + * @returns {boolean} + */ +function isValidThresholds(t) { + return ( + t !== null && + typeof t === 'object' && + !Array.isArray(t) && + typeof t.uncertain === 'number' && + Number.isFinite(t.uncertain) && + t.uncertain >= 0 && + t.uncertain <= 1 && + typeof t.hallucinated === 'number' && + Number.isFinite(t.hallucinated) && + t.hallucinated >= 0 && + t.hallucinated <= 1 && + t.uncertain <= t.hallucinated + ); +} + +/** + * Returns true when `value` is a valid per-category threshold pair: a non-null + * plain object with both `uncertain` and `hallucinated` as finite numbers in + * [0,1] and `uncertain <= hallucinated`. Delegates to `isValidThresholds` + * because the shapes are identical. + * + * @param {*} value - Value to check. + * @returns {boolean} + */ +function isValidCategoryThreshold(value) { + return isValidThresholds(value); +} + +/** + * Validate a raw config object loaded from a source, logging warnings to stderr + * for invalid field values and deleting them so they fall back to defaults during + * the merge step. Mutates the provided object in place. + * + * @param {object} obj - Raw config object to validate. + * @param {string} source - Human-readable source label used in warning messages. + * @returns {object} The (mutated) object. + */ +function validateConfig(obj, source) { + if (!obj || typeof obj !== 'object') return {}; + const src = source || 'unknown source'; + + /** + * Emit a validation warning to stderr. + * @param {string} field + * @param {*} val + * @param {*} def + */ + function warn(field, val, def) { + process.stderr.write( + `[hallucination-detector] Invalid ${field} "${val}" from ${src}; using default ${JSON.stringify(def)}\n`, + ); + } + + if ('severity' in obj && !VALID_SEVERITIES.has(obj.severity)) { + warn('severity', obj.severity, 'error'); + delete obj.severity; + } + if ('outputFormat' in obj && !VALID_OUTPUT_FORMATS.has(obj.outputFormat)) { + warn('outputFormat', obj.outputFormat, 'text'); + delete obj.outputFormat; + } + if ('maxTriggersPerResponse' in obj) { + if (!Number.isInteger(obj.maxTriggersPerResponse) || obj.maxTriggersPerResponse < 0) { + warn('maxTriggersPerResponse', obj.maxTriggersPerResponse, 20); + delete obj.maxTriggersPerResponse; + } + } + if ('maxBlocksPerSession' in obj && obj.maxBlocksPerSession !== null) { + if (!Number.isInteger(obj.maxBlocksPerSession) || obj.maxBlocksPerSession < 0) { + warn('maxBlocksPerSession', obj.maxBlocksPerSession, null); + delete obj.maxBlocksPerSession; + } + } + if ('debug' in obj && typeof obj.debug !== 'boolean') { + warn('debug', obj.debug, false); + delete obj.debug; + } + if ('introspect' in obj && typeof obj.introspect !== 'boolean') { + warn('introspect', obj.introspect, false); + delete obj.introspect; + } + if ('dryRun' in obj && typeof obj.dryRun !== 'boolean') { + warn('dryRun', obj.dryRun, false); + delete obj.dryRun; + } + if ('warnOnly' in obj && typeof obj.warnOnly !== 'boolean') { + warn('warnOnly', obj.warnOnly, false); + delete obj.warnOnly; + } + if ('blockSubagents' in obj && typeof obj.blockSubagents !== 'boolean') { + warn('blockSubagents', obj.blockSubagents, false); + delete obj.blockSubagents; + } + if ('blockUserSessions' in obj && typeof obj.blockUserSessions !== 'boolean') { + warn('blockUserSessions', obj.blockUserSessions, true); + delete obj.blockUserSessions; + } + if ('ignoreCategories' in obj) { + if (!Array.isArray(obj.ignoreCategories)) { + warn('ignoreCategories', obj.ignoreCategories, []); + delete obj.ignoreCategories; + } + } + if ('includeContext' in obj && typeof obj.includeContext !== 'boolean') { + warn('includeContext', obj.includeContext, true); + delete obj.includeContext; + } + if ('contextLines' in obj) { + if (!Number.isInteger(obj.contextLines) || obj.contextLines < 0) { + warn('contextLines', obj.contextLines, 2); + delete obj.contextLines; + } + } + // weights: object with numeric values + if ('weights' in obj) { + if (typeof obj.weights !== 'object' || obj.weights === null || Array.isArray(obj.weights)) { + warn('weights', obj.weights, DEFAULT_WEIGHTS); + delete obj.weights; + } else { + for (const key of Object.keys(obj.weights)) { + const val = obj.weights[key]; + if (typeof val !== 'number' || !Number.isFinite(val)) { + const defaultVal = key in DEFAULT_WEIGHTS ? DEFAULT_WEIGHTS[key] : undefined; + warn('weights.' + key, val, defaultVal); + delete obj.weights[key]; + } + } + } + } + // thresholds: { uncertain, hallucinated } both numbers in [0,1], uncertain <= hallucinated + if ('thresholds' in obj) { + if (!isValidThresholds(obj.thresholds)) { + warn('thresholds', JSON.stringify(obj.thresholds), DEFAULT_THRESHOLDS); + delete obj.thresholds; + } + } + // reportingThreshold: finite number in [0, 100] + if ('reportingThreshold' in obj) { + if ( + !Number.isFinite(obj.reportingThreshold) || + obj.reportingThreshold < 0 || + obj.reportingThreshold > 100 + ) { + warn('reportingThreshold', obj.reportingThreshold, 50); + delete obj.reportingThreshold; + } + } + // confidenceWeights: plain object; each of 4 recognized keys must be finite number in [0,1]; + // unknown keys are preserved with a warning. + if ('confidenceWeights' in obj) { + if ( + typeof obj.confidenceWeights !== 'object' || + obj.confidenceWeights === null || + Array.isArray(obj.confidenceWeights) + ) { + process.stderr.write( + `[hallucination-detector] Invalid confidenceWeights value from ${src}; must be a plain object. Using default\n`, + ); + delete obj.confidenceWeights; + } else { + const KNOWN_CONFIDENCE_KEYS = new Set([ + 'patternStrength', + 'evidenceProximity', + 'categoryStacking', + 'contextDensity', + ]); + for (const key of Object.keys(obj.confidenceWeights)) { + if (!KNOWN_CONFIDENCE_KEYS.has(key)) { + process.stderr.write( + `[hallucination-detector] Unknown confidenceWeights key "${key}" from ${src}; preserved for future use\n`, + ); + continue; + } + const val = obj.confidenceWeights[key]; + if (!Number.isFinite(val) || val < 0 || val > 1) { + process.stderr.write( + `[hallucination-detector] Invalid confidenceWeights.${key} "${val}" from ${src}; using default ${DEFAULT_CONFIDENCE_WEIGHTS[key]}\n`, + ); + delete obj.confidenceWeights[key]; + } + } + } + } + // categories: per-category overrides — validate threshold pairs when present. + // Unknown category names are preserved with a warning (they may be user-defined + // or from a future version). Invalid threshold fields are deleted so they fall + // back to global thresholds; other category fields (enabled, customPatterns, + // replacePatterns) are always preserved. + if ('categories' in obj) { + if ( + typeof obj.categories !== 'object' || + obj.categories === null || + Array.isArray(obj.categories) + ) { + process.stderr.write( + `[hallucination-detector] Invalid categories value from ${src}; must be a plain object. Using default {}\n`, + ); + delete obj.categories; + } else { + const VALID_CATEGORIES = new Set(Object.keys(DEFAULT_WEIGHTS)); + for (const catName of Object.keys(obj.categories)) { + if (!VALID_CATEGORIES.has(catName)) { + process.stderr.write( + `[hallucination-detector] Unknown category name "${catName}" from ${src}; entry preserved but may have no effect\n`, + ); + } + const catEntry = obj.categories[catName]; + if (catEntry === null || typeof catEntry !== 'object' || Array.isArray(catEntry)) { + continue; + } + const hasUncertain = 'uncertain' in catEntry; + const hasHallucinated = 'hallucinated' in catEntry; + if (hasUncertain || hasHallucinated) { + // Only validate when at least one threshold field is present. + // Both fields must be present and valid together. + if ( + !isValidCategoryThreshold({ + uncertain: catEntry.uncertain, + hallucinated: catEntry.hallucinated, + }) + ) { + process.stderr.write( + `[hallucination-detector] Invalid threshold pair for category "${catName}" from ${src} (uncertain=${catEntry.uncertain}, hallucinated=${catEntry.hallucinated}); threshold fields removed, other category fields preserved\n`, + ); + delete catEntry.uncertain; + delete catEntry.hallucinated; + } + } + } + } + } + return obj; +} + +module.exports = { validateConfig, isValidThresholds, isValidCategoryThreshold }; diff --git a/scripts/hallucination-config.cjs b/scripts/hallucination-config.cjs index 3dd0414..d6c3489 100644 --- a/scripts/hallucination-config.cjs +++ b/scripts/hallucination-config.cjs @@ -21,586 +21,22 @@ const fs = require('node:fs'); const os = require('node:os'); const path = require('node:path'); -/** - * Default weights for each detection category. - * Weights are relative severity signals; `aggregateWeightedScore` normalizes - * by their sum so aggregate scores always remain in [0, 1] regardless of - * whether the weights themselves sum to 1.0. - */ -const DEFAULT_WEIGHTS = { - speculation_language: 0.25, - causality_language: 0.3, - pseudo_quantification: 0.15, - completeness_claim: 0.2, - // fabricated_source: reserved for future implementation (issue #18) - evaluative_design_claim: 0.4, - internal_contradiction: 0.35, - unsupported_absence: 0.7, - ungrounded_behavioral_assertion: 0.5, -}; - -/** - * Default score thresholds for three-tier label classification. - * - uncertain: scores >= this value are labelled UNCERTAIN (not GROUNDED) - * - hallucinated: scores > this value are labelled HALLUCINATED - */ -const DEFAULT_THRESHOLDS = { - uncertain: 0.3, - hallucinated: 0.6, -}; - -/** - * Default weights for the four confidence-score components. - * These control how much each factor contributes to the per-match - * confidence integer in [0, 100]. - * - * - patternStrength: contribution of the pattern's inherent severity - * - evidenceProximity: contribution of evidence markers near the match - * - categoryStacking: bonus when multiple categories fire in the same sentence - * - contextDensity: bonus when multiple matches cluster within 200 chars - */ -const DEFAULT_CONFIDENCE_WEIGHTS = { - patternStrength: 0.4, - evidenceProximity: 0.25, - categoryStacking: 0.2, - contextDensity: 0.15, -}; - -/** - * Default full configuration object. - */ -const DEFAULT_CONFIG = { - weights: DEFAULT_WEIGHTS, - thresholds: DEFAULT_THRESHOLDS, - introspect: false, - introspectOutputPath: null, - // Shadow mode: log would-block events without actually blocking. - dryRun: false, - // Global settings - severity: 'error', - maxTriggersPerResponse: 20, - maxBlocksPerSession: null, - outputFormat: 'text', - debug: false, - // Per-category settings (keyed by category name) - categories: {}, - // Filtering settings - ignorePatterns: [], - ignoreBlocks: [], - evidenceMarkers: [], - allowlist: [], - // Response settings - responseTemplates: {}, - includeContext: true, - contextLines: 2, - // Session-type gating - warnOnly: false, // log telemetry but never emit a block to stdout - ignoreCategories: [], // category names skipped entirely (still written to telemetry with was_ignored=1) - blockSubagents: false, // block when hook_event_name is SubagentStop - blockUserSessions: true, // block when hook_event_name is Stop (user-facing session) - // Confidence scoring - confidenceWeights: DEFAULT_CONFIDENCE_WEIGHTS, - reportingThreshold: 50, // minimum confidence [0,100] for a match to appear in block reason text -}; - -// ============================================================================ -// Minimal TOML parser — handles the subset needed for pyproject.toml sections. -// Supports: simple key-value pairs (string, number, boolean), section headers -// ([section] / [section.sub]), and single-line arrays of strings or inline tables. -// ============================================================================ - -/** - * Split `content` on `sep` at depth 0, respecting nested brackets and quoted strings. - * @param {string} content - * @param {string} sep - Single separator character. - * @returns {string[]} - */ -function splitTopLevel(content, sep) { - const parts = []; - let depth = 0; - let inStr = false; - let strChar = ''; - let start = 0; - for (let i = 0; i < content.length; i++) { - const ch = content[i]; - if (inStr) { - if (ch === '\\' && i + 1 < content.length) { - i++; - continue; - } - if (ch === strChar) inStr = false; - } else if (ch === '"' || ch === "'") { - inStr = true; - strChar = ch; - } else if (ch === '[' || ch === '{') { - depth++; - } else if (ch === ']' || ch === '}') { - depth--; - } else if (ch === sep && depth === 0) { - parts.push(content.slice(start, i)); - start = i + 1; - } - } - parts.push(content.slice(start)); - return parts; -} - -/** - * Strip a `#` inline comment from a TOML value string, respecting quoted strings. - * @param {string} valStr - * @returns {string} - */ -function stripTomlInlineComment(valStr) { - let inStr = false; - let strChar = ''; - for (let i = 0; i < valStr.length; i++) { - const ch = valStr[i]; - if (inStr) { - if (ch === '\\' && i + 1 < valStr.length) { - i++; - continue; - } - if (ch === strChar) inStr = false; - } else if (ch === '"' || ch === "'") { - inStr = true; - strChar = ch; - } else if (ch === '#') { - return valStr.slice(0, i).trim(); - } - } - return valStr.trim(); -} - -/** - * Parse a single TOML value string into a JS value. - * @param {string} valStr - * @returns {*} - */ -function parseTomlValue(valStr) { - const s = stripTomlInlineComment(valStr); - if (!s) return null; - // Quoted string - if ((s.startsWith('"') && s.endsWith('"')) || (s.startsWith("'") && s.endsWith("'"))) { - return s - .slice(1, -1) - .replace(/\\n/g, '\n') - .replace(/\\t/g, '\t') - .replace(/\\\\/g, '\\') - .replace(/\\"/g, '"'); - } - if (s === 'true') return true; - if (s === 'false') return false; - if (/^-?\d+(\.\d+)?$/.test(s)) return Number(s); - // Array - if (s.startsWith('[') && s.endsWith(']')) { - const inner = s.slice(1, -1).trim(); - if (!inner) return []; - return splitTopLevel(inner, ',') - .map((item) => item.trim()) - .filter((item) => item !== '') - .map((item) => parseTomlValue(item)); - } - // Inline table - if (s.startsWith('{') && s.endsWith('}')) { - return parseTomlInlineTable(s.slice(1, -1)); - } - return s; -} - -/** - * Parse a TOML inline table body (content between `{` and `}`). - * @param {string} content - * @returns {object} - */ -function parseTomlInlineTable(content) { - const table = {}; - if (!content.trim()) return table; - for (const pair of splitTopLevel(content.trim(), ',')) { - const p = pair.trim(); - const eqIdx = p.indexOf('='); - if (eqIdx === -1) continue; - const k = p.slice(0, eqIdx).trim(); - const v = p.slice(eqIdx + 1).trim(); - if (k) table[k] = parseTomlValue(v); - } - return table; -} - -/** - * Parse a TOML source string into a plain JS object. - * Only handles the subset needed for `[tool.hallucination-detector]` sections. - * - * @param {string} source - TOML source text. - * @returns {object} - */ -function parseToml(source) { - const result = {}; - let current = result; - - for (const rawLine of source.split('\n')) { - const line = rawLine.trim(); - if (!line || line.startsWith('#')) continue; - - // Section header: [key] or [key.subkey] (not array tables [[...]]) - if (line.startsWith('[') && !line.startsWith('[[')) { - const end = line.indexOf(']'); - if (end === -1) continue; - const sectionStr = line.slice(1, end).trim(); - // Split on '.' to get nested path (bare keys may contain hyphens) - const parts = sectionStr.split('.').map((p) => p.trim()); - current = result; - for (const part of parts) { - if (typeof current[part] !== 'object' || current[part] === null) { - current[part] = {}; - } - current = current[part]; - } - continue; - } - - // Key-value pair - const eqIdx = line.indexOf('='); - if (eqIdx === -1) continue; - const key = line.slice(0, eqIdx).trim(); - const valStr = line.slice(eqIdx + 1).trim(); - if (!key) continue; - current[key] = parseTomlValue(valStr); - } - - return result; -} - -// ============================================================================ -// Deep freeze -// ============================================================================ - -/** - * Recursively freeze an object and all nested plain objects / arrays. - * @param {*} obj - * @returns {*} The frozen value. - */ -function deepFreeze(obj) { - if (obj === null || typeof obj !== 'object') return obj; - for (const val of Object.values(obj)) { - deepFreeze(val); - } - return Object.freeze(obj); -} - -// ============================================================================ -// Schema validation -// ============================================================================ - -const VALID_SEVERITIES = new Set(['error', 'warning', 'info']); -const VALID_OUTPUT_FORMATS = new Set(['text', 'json', 'jsonl']); - -/** - * Returns true when `t` is a valid thresholds object: a non-null plain object - * with both `uncertain` and `hallucinated` as finite numbers in [0,1] and - * `uncertain <= hallucinated`. - * - * @param {*} t - Value to check. - * @returns {boolean} - */ -function isValidThresholds(t) { - return ( - t !== null && - typeof t === 'object' && - !Array.isArray(t) && - typeof t.uncertain === 'number' && - Number.isFinite(t.uncertain) && - t.uncertain >= 0 && - t.uncertain <= 1 && - typeof t.hallucinated === 'number' && - Number.isFinite(t.hallucinated) && - t.hallucinated >= 0 && - t.hallucinated <= 1 && - t.uncertain <= t.hallucinated - ); -} - -/** - * Returns true when `value` is a valid per-category threshold pair: a non-null - * plain object with both `uncertain` and `hallucinated` as finite numbers in - * [0,1] and `uncertain <= hallucinated`. Delegates to `isValidThresholds` - * because the shapes are identical. - * - * @param {*} value - Value to check. - * @returns {boolean} - */ -function isValidCategoryThreshold(value) { - return isValidThresholds(value); -} - -/** - * Validate a raw config object loaded from a source, logging warnings to stderr - * for invalid field values and deleting them so they fall back to defaults during - * the merge step. Mutates the provided object in place. - * - * @param {object} obj - Raw config object to validate. - * @param {string} source - Human-readable source label used in warning messages. - * @returns {object} The (mutated) object. - */ -function validateConfig(obj, source) { - if (!obj || typeof obj !== 'object') return {}; - const src = source || 'unknown source'; - - /** - * Emit a validation warning to stderr. - * @param {string} field - * @param {*} val - * @param {*} def - */ - function warn(field, val, def) { - process.stderr.write( - `[hallucination-detector] Invalid ${field} "${val}" from ${src}; using default ${JSON.stringify(def)}\n`, - ); - } - - if ('severity' in obj && !VALID_SEVERITIES.has(obj.severity)) { - warn('severity', obj.severity, 'error'); - delete obj.severity; - } - if ('outputFormat' in obj && !VALID_OUTPUT_FORMATS.has(obj.outputFormat)) { - warn('outputFormat', obj.outputFormat, 'text'); - delete obj.outputFormat; - } - if ('maxTriggersPerResponse' in obj) { - if (!Number.isInteger(obj.maxTriggersPerResponse) || obj.maxTriggersPerResponse < 0) { - warn('maxTriggersPerResponse', obj.maxTriggersPerResponse, 20); - delete obj.maxTriggersPerResponse; - } - } - if ('maxBlocksPerSession' in obj && obj.maxBlocksPerSession !== null) { - if (!Number.isInteger(obj.maxBlocksPerSession) || obj.maxBlocksPerSession < 0) { - warn('maxBlocksPerSession', obj.maxBlocksPerSession, null); - delete obj.maxBlocksPerSession; - } - } - if ('debug' in obj && typeof obj.debug !== 'boolean') { - warn('debug', obj.debug, false); - delete obj.debug; - } - if ('introspect' in obj && typeof obj.introspect !== 'boolean') { - warn('introspect', obj.introspect, false); - delete obj.introspect; - } - if ('dryRun' in obj && typeof obj.dryRun !== 'boolean') { - warn('dryRun', obj.dryRun, false); - delete obj.dryRun; - } - if ('warnOnly' in obj && typeof obj.warnOnly !== 'boolean') { - warn('warnOnly', obj.warnOnly, false); - delete obj.warnOnly; - } - if ('blockSubagents' in obj && typeof obj.blockSubagents !== 'boolean') { - warn('blockSubagents', obj.blockSubagents, false); - delete obj.blockSubagents; - } - if ('blockUserSessions' in obj && typeof obj.blockUserSessions !== 'boolean') { - warn('blockUserSessions', obj.blockUserSessions, true); - delete obj.blockUserSessions; - } - if ('ignoreCategories' in obj) { - if (!Array.isArray(obj.ignoreCategories)) { - warn('ignoreCategories', obj.ignoreCategories, []); - delete obj.ignoreCategories; - } - } - if ('includeContext' in obj && typeof obj.includeContext !== 'boolean') { - warn('includeContext', obj.includeContext, true); - delete obj.includeContext; - } - if ('contextLines' in obj) { - if (!Number.isInteger(obj.contextLines) || obj.contextLines < 0) { - warn('contextLines', obj.contextLines, 2); - delete obj.contextLines; - } - } - // weights: object with numeric values - if ('weights' in obj) { - if (typeof obj.weights !== 'object' || obj.weights === null || Array.isArray(obj.weights)) { - warn('weights', obj.weights, DEFAULT_WEIGHTS); - delete obj.weights; - } - } - // thresholds: { uncertain, hallucinated } both numbers in [0,1], uncertain <= hallucinated - if ('thresholds' in obj) { - if (!isValidThresholds(obj.thresholds)) { - warn('thresholds', JSON.stringify(obj.thresholds), DEFAULT_THRESHOLDS); - delete obj.thresholds; - } - } - // reportingThreshold: finite number in [0, 100] - if ('reportingThreshold' in obj) { - if ( - !Number.isFinite(obj.reportingThreshold) || - obj.reportingThreshold < 0 || - obj.reportingThreshold > 100 - ) { - warn('reportingThreshold', obj.reportingThreshold, 50); - delete obj.reportingThreshold; - } - } - // confidenceWeights: plain object; each of 4 recognized keys must be finite number in [0,1]; - // unknown keys are preserved with a warning. - if ('confidenceWeights' in obj) { - if ( - typeof obj.confidenceWeights !== 'object' || - obj.confidenceWeights === null || - Array.isArray(obj.confidenceWeights) - ) { - process.stderr.write( - `[hallucination-detector] Invalid confidenceWeights value from ${src}; must be a plain object. Using default\n`, - ); - delete obj.confidenceWeights; - } else { - const KNOWN_CONFIDENCE_KEYS = new Set([ - 'patternStrength', - 'evidenceProximity', - 'categoryStacking', - 'contextDensity', - ]); - for (const key of Object.keys(obj.confidenceWeights)) { - if (!KNOWN_CONFIDENCE_KEYS.has(key)) { - process.stderr.write( - `[hallucination-detector] Unknown confidenceWeights key "${key}" from ${src}; preserved for future use\n`, - ); - continue; - } - const val = obj.confidenceWeights[key]; - if (!Number.isFinite(val) || val < 0 || val > 1) { - process.stderr.write( - `[hallucination-detector] Invalid confidenceWeights.${key} "${val}" from ${src}; using default ${DEFAULT_CONFIDENCE_WEIGHTS[key]}\n`, - ); - delete obj.confidenceWeights[key]; - } - } - } - } - // categories: per-category overrides — validate threshold pairs when present. - // Unknown category names are preserved with a warning (they may be user-defined - // or from a future version). Invalid threshold fields are deleted so they fall - // back to global thresholds; other category fields (enabled, customPatterns, - // replacePatterns) are always preserved. - if ('categories' in obj) { - if ( - typeof obj.categories !== 'object' || - obj.categories === null || - Array.isArray(obj.categories) - ) { - process.stderr.write( - `[hallucination-detector] Invalid categories value from ${src}; must be a plain object. Using default {}\n`, - ); - delete obj.categories; - } else { - const VALID_CATEGORIES = new Set(Object.keys(DEFAULT_WEIGHTS)); - for (const catName of Object.keys(obj.categories)) { - if (!VALID_CATEGORIES.has(catName)) { - process.stderr.write( - `[hallucination-detector] Unknown category name "${catName}" from ${src}; entry preserved but may have no effect\n`, - ); - } - const catEntry = obj.categories[catName]; - if (catEntry === null || typeof catEntry !== 'object' || Array.isArray(catEntry)) { - continue; - } - const hasUncertain = 'uncertain' in catEntry; - const hasHallucinated = 'hallucinated' in catEntry; - if (hasUncertain || hasHallucinated) { - // Only validate when at least one threshold field is present. - // Both fields must be present and valid together. - if ( - !isValidCategoryThreshold({ - uncertain: catEntry.uncertain, - hallucinated: catEntry.hallucinated, - }) - ) { - process.stderr.write( - `[hallucination-detector] Invalid threshold pair for category "${catName}" from ${src} (uncertain=${catEntry.uncertain}, hallucinated=${catEntry.hallucinated}); threshold fields removed, other category fields preserved\n`, - ); - delete catEntry.uncertain; - delete catEntry.hallucinated; - } - } - } - } - } - return obj; -} +const { + DEFAULT_WEIGHTS, + DEFAULT_THRESHOLDS, + DEFAULT_CONFIDENCE_WEIGHTS, + DEFAULT_CONFIG, +} = require('./hallucination-config-defaults.cjs'); -// ============================================================================ -// Deep merge -// ============================================================================ +const { parseToml } = require('./hallucination-config-toml.cjs'); -/** - * Deep-merge two config objects. Rules: - * - Plain objects are merged recursively. - * - `categories..customPatterns` arrays are concatenated unless the override - * has `replacePatterns: true` for that category. - * - All other arrays are replaced by the override value. - * - Scalar values are replaced by the override value. - * - * Neither argument is mutated; a new object is returned. - * - * @param {object} base - Lower-priority config. - * @param {object} override - Higher-priority config (wins on conflict). - * @returns {object} Merged config. - */ -function mergeConfig(base, override) { - if (!override || typeof override !== 'object' || Array.isArray(override)) return base; - if (!base || typeof base !== 'object' || Array.isArray(base)) return override; - - const result = { ...base }; - - for (const key of Object.keys(override)) { - const overVal = override[key]; - const baseVal = base[key]; - - if (key === 'categories') { - // Merge categories map, with special customPatterns concatenation logic. - const baseCats = - typeof baseVal === 'object' && baseVal !== null && !Array.isArray(baseVal) ? baseVal : {}; - const overCats = - typeof overVal === 'object' && overVal !== null && !Array.isArray(overVal) ? overVal : {}; - const merged = { ...baseCats }; - for (const catName of Object.keys(overCats)) { - const baseCat = baseCats[catName] || {}; - const overCat = overCats[catName]; - if (typeof overCat !== 'object' || overCat === null) { - merged[catName] = overCat; - continue; - } - const mergedCat = { ...baseCat, ...overCat }; - // Concatenate customPatterns unless replacePatterns is true in the override. - if ( - !overCat.replacePatterns && - Array.isArray(baseCat.customPatterns) && - Array.isArray(overCat.customPatterns) - ) { - mergedCat.customPatterns = [...baseCat.customPatterns, ...overCat.customPatterns]; - } - merged[catName] = mergedCat; - } - result[key] = merged; - } else if ( - typeof overVal === 'object' && - overVal !== null && - !Array.isArray(overVal) && - typeof baseVal === 'object' && - baseVal !== null && - !Array.isArray(baseVal) - ) { - // Both are plain objects — recurse. - result[key] = mergeConfig(baseVal, overVal); - } else { - // Scalar, array, or null — override wins. - result[key] = overVal; - } - } +const { mergeConfig, deepFreeze } = require('./hallucination-config-merge.cjs'); - return result; -} +const { + validateConfig, + isValidThresholds, + isValidCategoryThreshold, +} = require('./hallucination-config-validate.cjs'); // ============================================================================ // Source loaders diff --git a/scripts/hallucination-framing-session-start.cjs b/scripts/hallucination-framing-session-start.cjs index 2a8c455..3c0796f 100644 --- a/scripts/hallucination-framing-session-start.cjs +++ b/scripts/hallucination-framing-session-start.cjs @@ -23,8 +23,7 @@ const fs = require('node:fs'); const os = require('node:os'); const path = require('node:path'); - -const { loadConfig } = require('./hallucination-config.cjs'); +const { safeLoadConfig } = require('./hallucination-config-safe.cjs'); // Base enforcement framing — language discipline rules and structured claim format. const _FRAMING_TEXT_BASE = `# Hallucination Prevention — Behavioral Framing @@ -199,16 +198,12 @@ function main() { } let framingText = FRAMING_TEXT; - try { - const config = loadConfig(); - if (config.introspect) { - const logPath = - config.introspectOutputPath || - path.join(os.tmpdir(), 'hallucination-detector-introspect.jsonl'); - framingText = buildIntrospectFramingText(logPath); - } - } catch { - // loadConfig failure is non-fatal — fall through to default enforcement framing + const config = safeLoadConfig(); + if (config.introspect) { + const logPath = + config.introspectOutputPath || + path.join(os.tmpdir(), 'hallucination-detector-introspect.jsonl'); + framingText = buildIntrospectFramingText(logPath); } emitSessionStartContext(framingText); diff --git a/tests/hallucination-config-defaults.test.cjs b/tests/hallucination-config-defaults.test.cjs new file mode 100644 index 0000000..499b9f9 --- /dev/null +++ b/tests/hallucination-config-defaults.test.cjs @@ -0,0 +1,180 @@ +'use strict'; + +const { + DEFAULT_WEIGHTS, + DEFAULT_THRESHOLDS, + DEFAULT_CONFIDENCE_WEIGHTS, + DEFAULT_CONFIG, +} = require('../scripts/hallucination-config-defaults.cjs'); + +describe('hallucination-config-defaults', () => { + describe('exports', () => { + it('exports all four constants', () => { + expect(DEFAULT_WEIGHTS).toBeDefined(); + expect(DEFAULT_THRESHOLDS).toBeDefined(); + expect(DEFAULT_CONFIDENCE_WEIGHTS).toBeDefined(); + expect(DEFAULT_CONFIG).toBeDefined(); + }); + }); + + describe('DEFAULT_WEIGHTS', () => { + it('is a plain object', () => { + expect(typeof DEFAULT_WEIGHTS).toBe('object'); + expect(Array.isArray(DEFAULT_WEIGHTS)).toBe(false); + expect(DEFAULT_WEIGHTS).not.toBeNull(); + }); + + it('has the required category keys', () => { + expect('speculation_language' in DEFAULT_WEIGHTS).toBe(true); + expect('causality_language' in DEFAULT_WEIGHTS).toBe(true); + expect('pseudo_quantification' in DEFAULT_WEIGHTS).toBe(true); + expect('completeness_claim' in DEFAULT_WEIGHTS).toBe(true); + expect('evaluative_design_claim' in DEFAULT_WEIGHTS).toBe(true); + expect('internal_contradiction' in DEFAULT_WEIGHTS).toBe(true); + expect('unsupported_absence' in DEFAULT_WEIGHTS).toBe(true); + expect('ungrounded_behavioral_assertion' in DEFAULT_WEIGHTS).toBe(true); + }); + + it('all values are finite positive numbers', () => { + for (const [key, val] of Object.entries(DEFAULT_WEIGHTS)) { + expect(typeof val, `DEFAULT_WEIGHTS.${key} type`).toBe('number'); + expect(Number.isFinite(val), `DEFAULT_WEIGHTS.${key} is finite`).toBe(true); + expect(val > 0, `DEFAULT_WEIGHTS.${key} > 0`).toBe(true); + } + }); + + it('has specific expected values', () => { + expect(DEFAULT_WEIGHTS.speculation_language).toBe(0.25); + expect(DEFAULT_WEIGHTS.causality_language).toBe(0.3); + expect(DEFAULT_WEIGHTS.pseudo_quantification).toBe(0.15); + expect(DEFAULT_WEIGHTS.completeness_claim).toBe(0.2); + expect(DEFAULT_WEIGHTS.evaluative_design_claim).toBe(0.4); + expect(DEFAULT_WEIGHTS.internal_contradiction).toBe(0.35); + expect(DEFAULT_WEIGHTS.unsupported_absence).toBe(0.7); + expect(DEFAULT_WEIGHTS.ungrounded_behavioral_assertion).toBe(0.5); + }); + }); + + describe('DEFAULT_THRESHOLDS', () => { + it('is a plain object', () => { + expect(typeof DEFAULT_THRESHOLDS).toBe('object'); + expect(Array.isArray(DEFAULT_THRESHOLDS)).toBe(false); + expect(DEFAULT_THRESHOLDS).not.toBeNull(); + }); + + it('has uncertain and hallucinated keys', () => { + expect('uncertain' in DEFAULT_THRESHOLDS).toBe(true); + expect('hallucinated' in DEFAULT_THRESHOLDS).toBe(true); + }); + + it('both values are numbers in [0,1]', () => { + expect(typeof DEFAULT_THRESHOLDS.uncertain).toBe('number'); + expect(DEFAULT_THRESHOLDS.uncertain).toBeGreaterThanOrEqual(0); + expect(DEFAULT_THRESHOLDS.uncertain).toBeLessThanOrEqual(1); + expect(typeof DEFAULT_THRESHOLDS.hallucinated).toBe('number'); + expect(DEFAULT_THRESHOLDS.hallucinated).toBeGreaterThanOrEqual(0); + expect(DEFAULT_THRESHOLDS.hallucinated).toBeLessThanOrEqual(1); + }); + + it('uncertain <= hallucinated', () => { + expect(DEFAULT_THRESHOLDS.uncertain).toBeLessThanOrEqual(DEFAULT_THRESHOLDS.hallucinated); + }); + + it('has expected values', () => { + expect(DEFAULT_THRESHOLDS.uncertain).toBe(0.3); + expect(DEFAULT_THRESHOLDS.hallucinated).toBe(0.6); + }); + }); + + describe('DEFAULT_CONFIDENCE_WEIGHTS', () => { + it('is a plain object', () => { + expect(typeof DEFAULT_CONFIDENCE_WEIGHTS).toBe('object'); + expect(Array.isArray(DEFAULT_CONFIDENCE_WEIGHTS)).toBe(false); + expect(DEFAULT_CONFIDENCE_WEIGHTS).not.toBeNull(); + }); + + it('has exactly 4 keys', () => { + expect(Object.keys(DEFAULT_CONFIDENCE_WEIGHTS)).toHaveLength(4); + }); + + it('has patternStrength, evidenceProximity, categoryStacking, contextDensity', () => { + expect('patternStrength' in DEFAULT_CONFIDENCE_WEIGHTS).toBe(true); + expect('evidenceProximity' in DEFAULT_CONFIDENCE_WEIGHTS).toBe(true); + expect('categoryStacking' in DEFAULT_CONFIDENCE_WEIGHTS).toBe(true); + expect('contextDensity' in DEFAULT_CONFIDENCE_WEIGHTS).toBe(true); + }); + + it('all values are finite numbers', () => { + for (const [key, val] of Object.entries(DEFAULT_CONFIDENCE_WEIGHTS)) { + expect(typeof val, `DEFAULT_CONFIDENCE_WEIGHTS.${key} type`).toBe('number'); + expect(Number.isFinite(val), `DEFAULT_CONFIDENCE_WEIGHTS.${key} is finite`).toBe(true); + } + }); + + it('has expected values', () => { + expect(DEFAULT_CONFIDENCE_WEIGHTS.patternStrength).toBe(0.4); + expect(DEFAULT_CONFIDENCE_WEIGHTS.evidenceProximity).toBe(0.25); + expect(DEFAULT_CONFIDENCE_WEIGHTS.categoryStacking).toBe(0.2); + expect(DEFAULT_CONFIDENCE_WEIGHTS.contextDensity).toBe(0.15); + }); + }); + + describe('DEFAULT_CONFIG', () => { + it('references DEFAULT_WEIGHTS by identity', () => { + expect(DEFAULT_CONFIG.weights).toBe(DEFAULT_WEIGHTS); + }); + + it('references DEFAULT_THRESHOLDS by identity', () => { + expect(DEFAULT_CONFIG.thresholds).toBe(DEFAULT_THRESHOLDS); + }); + + it('references DEFAULT_CONFIDENCE_WEIGHTS by identity', () => { + expect(DEFAULT_CONFIG.confidenceWeights).toBe(DEFAULT_CONFIDENCE_WEIGHTS); + }); + + it('has boolean flags with correct types', () => { + expect(typeof DEFAULT_CONFIG.introspect).toBe('boolean'); + expect(typeof DEFAULT_CONFIG.dryRun).toBe('boolean'); + expect(typeof DEFAULT_CONFIG.debug).toBe('boolean'); + expect(typeof DEFAULT_CONFIG.warnOnly).toBe('boolean'); + expect(typeof DEFAULT_CONFIG.blockSubagents).toBe('boolean'); + expect(typeof DEFAULT_CONFIG.blockUserSessions).toBe('boolean'); + expect(typeof DEFAULT_CONFIG.includeContext).toBe('boolean'); + }); + + it('has expected boolean defaults', () => { + expect(DEFAULT_CONFIG.introspect).toBe(false); + expect(DEFAULT_CONFIG.dryRun).toBe(false); + expect(DEFAULT_CONFIG.debug).toBe(false); + expect(DEFAULT_CONFIG.warnOnly).toBe(false); + expect(DEFAULT_CONFIG.blockSubagents).toBe(false); + expect(DEFAULT_CONFIG.blockUserSessions).toBe(true); + expect(DEFAULT_CONFIG.includeContext).toBe(true); + }); + + it('has string fields with correct types', () => { + expect(typeof DEFAULT_CONFIG.severity).toBe('string'); + expect(typeof DEFAULT_CONFIG.outputFormat).toBe('string'); + }); + + it('has numeric fields with correct types', () => { + expect(typeof DEFAULT_CONFIG.maxTriggersPerResponse).toBe('number'); + expect(typeof DEFAULT_CONFIG.reportingThreshold).toBe('number'); + expect(typeof DEFAULT_CONFIG.contextLines).toBe('number'); + }); + + it('has array fields as empty arrays', () => { + expect(Array.isArray(DEFAULT_CONFIG.ignorePatterns)).toBe(true); + expect(Array.isArray(DEFAULT_CONFIG.ignoreBlocks)).toBe(true); + expect(Array.isArray(DEFAULT_CONFIG.evidenceMarkers)).toBe(true); + expect(Array.isArray(DEFAULT_CONFIG.allowlist)).toBe(true); + expect(Array.isArray(DEFAULT_CONFIG.ignoreCategories)).toBe(true); + }); + + it('has categories as empty object', () => { + expect(typeof DEFAULT_CONFIG.categories).toBe('object'); + expect(DEFAULT_CONFIG.categories).not.toBeNull(); + expect(Object.keys(DEFAULT_CONFIG.categories)).toHaveLength(0); + }); + }); +}); diff --git a/tests/hallucination-config-merge.test.cjs b/tests/hallucination-config-merge.test.cjs new file mode 100644 index 0000000..c22052f --- /dev/null +++ b/tests/hallucination-config-merge.test.cjs @@ -0,0 +1,214 @@ +'use strict'; + +const { mergeConfig, deepFreeze } = require('../scripts/hallucination-config-merge.cjs'); + +describe('mergeConfig', () => { + describe('basic scalar merging', () => { + it('returns base when override is null', () => { + const base = { a: 1 }; + expect(mergeConfig(base, null)).toBe(base); + }); + + it('returns base when override is undefined', () => { + const base = { a: 1 }; + expect(mergeConfig(base, undefined)).toBe(base); + }); + + it('returns override when base is null', () => { + const override = { a: 1 }; + expect(mergeConfig(null, override)).toBe(override); + }); + + it('returns override when base is an array', () => { + const override = { a: 1 }; + expect(mergeConfig([1, 2], override)).toBe(override); + }); + + it('returns base when override is an array', () => { + const base = { a: 1 }; + expect(mergeConfig(base, [1, 2])).toBe(base); + }); + + it('override scalar wins over base scalar', () => { + const result = mergeConfig({ x: 1 }, { x: 2 }); + expect(result.x).toBe(2); + }); + + it('base scalar is kept when not in override', () => { + const result = mergeConfig({ x: 1, y: 2 }, { x: 99 }); + expect(result.y).toBe(2); + }); + + it('does not mutate base', () => { + const base = { x: 1 }; + mergeConfig(base, { x: 2 }); + expect(base.x).toBe(1); + }); + + it('does not mutate override', () => { + const override = { x: 2 }; + mergeConfig({ x: 1 }, override); + expect(override.x).toBe(2); + }); + }); + + describe('nested object merging', () => { + it('recursively merges nested plain objects', () => { + const base = { a: { x: 1, y: 2 } }; + const override = { a: { y: 99, z: 3 } }; + const result = mergeConfig(base, override); + expect(result.a).toEqual({ x: 1, y: 99, z: 3 }); + }); + + it('override array replaces base array', () => { + const base = { items: [1, 2] }; + const override = { items: [3, 4, 5] }; + const result = mergeConfig(base, override); + expect(result.items).toEqual([3, 4, 5]); + }); + + it('override null replaces base object', () => { + const base = { meta: { a: 1 } }; + const override = { meta: null }; + const result = mergeConfig(base, override); + expect(result.meta).toBeNull(); + }); + }); + + describe('categories merging', () => { + it('categories from override are added to base', () => { + const base = { categories: {} }; + const override = { categories: { speculation_language: { enabled: false } } }; + const result = mergeConfig(base, override); + expect(result.categories.speculation_language).toEqual({ enabled: false }); + }); + + it('customPatterns are concatenated by default', () => { + const base = { + categories: { + speculation_language: { customPatterns: ['pat1'] }, + }, + }; + const override = { + categories: { + speculation_language: { customPatterns: ['pat2'] }, + }, + }; + const result = mergeConfig(base, override); + expect(result.categories.speculation_language.customPatterns).toEqual(['pat1', 'pat2']); + }); + + it('customPatterns are replaced when replacePatterns is true', () => { + const base = { + categories: { + speculation_language: { customPatterns: ['pat1', 'pat2'] }, + }, + }; + const override = { + categories: { + speculation_language: { customPatterns: ['new'], replacePatterns: true }, + }, + }; + const result = mergeConfig(base, override); + expect(result.categories.speculation_language.customPatterns).toEqual(['new']); + }); + + it('category scalar fields are overridden', () => { + const base = { + categories: { + speculation_language: { enabled: true, weight: 0.5 }, + }, + }; + const override = { + categories: { + speculation_language: { weight: 0.8 }, + }, + }; + const result = mergeConfig(base, override); + expect(result.categories.speculation_language.weight).toBe(0.8); + expect(result.categories.speculation_language.enabled).toBe(true); + }); + + it('preserves base categories not in override', () => { + const base = { + categories: { + cat_a: { enabled: true }, + cat_b: { enabled: false }, + }, + }; + const override = { + categories: { + cat_a: { enabled: false }, + }, + }; + const result = mergeConfig(base, override); + expect(result.categories.cat_b).toEqual({ enabled: false }); + }); + + it('non-object category override replaces base category', () => { + const base = { + categories: { + cat_a: { enabled: true }, + }, + }; + const override = { + categories: { + cat_a: null, + }, + }; + const result = mergeConfig(base, override); + expect(result.categories.cat_a).toBeNull(); + }); + }); +}); + +describe('deepFreeze', () => { + it('freezes a flat object', () => { + const obj = { a: 1, b: 2 }; + const frozen = deepFreeze(obj); + expect(Object.isFrozen(frozen)).toBe(true); + }); + + it('returns the same reference', () => { + const obj = { a: 1 }; + expect(deepFreeze(obj)).toBe(obj); + }); + + it('freezes nested objects', () => { + const obj = { a: { b: { c: 3 } } }; + deepFreeze(obj); + expect(Object.isFrozen(obj.a)).toBe(true); + expect(Object.isFrozen(obj.a.b)).toBe(true); + }); + + it('freezes nested arrays', () => { + const obj = { items: [1, 2, 3] }; + deepFreeze(obj); + expect(Object.isFrozen(obj.items)).toBe(true); + }); + + it('handles null without throwing', () => { + expect(() => deepFreeze(null)).not.toThrow(); + expect(deepFreeze(null)).toBeNull(); + }); + + it('handles primitive values without throwing', () => { + expect(deepFreeze(42)).toBe(42); + expect(deepFreeze('hello')).toBe('hello'); + expect(deepFreeze(true)).toBe(true); + }); + + it('prevents property assignment after freezing (strict mode)', () => { + const obj = deepFreeze({ x: 1 }); + expect(() => { + obj.x = 99; + }).toThrow(); + }); + + it('prevents nested property assignment after freezing (strict mode)', () => { + const obj = deepFreeze({ nested: { y: 2 } }); + expect(() => { + obj.nested.y = 99; + }).toThrow(); + }); +}); diff --git a/tests/hallucination-config-safe.test.cjs b/tests/hallucination-config-safe.test.cjs new file mode 100644 index 0000000..1308e27 --- /dev/null +++ b/tests/hallucination-config-safe.test.cjs @@ -0,0 +1,230 @@ +'use strict'; + +describe('hallucination-config-safe', () => { + beforeEach(() => { + vi.doUnmock('../scripts/hallucination-config.cjs'); + vi.resetModules(); + }); + + describe('constant re-exports', () => { + it('exports DEFAULT_WEIGHTS matching hallucination-config-defaults', () => { + const { DEFAULT_WEIGHTS } = require('../scripts/hallucination-config-safe.cjs'); + const { DEFAULT_WEIGHTS: EXPECTED } = require('../scripts/hallucination-config-defaults.cjs'); + expect(DEFAULT_WEIGHTS).toEqual(EXPECTED); + }); + + it('exports DEFAULT_THRESHOLDS matching hallucination-config-defaults', () => { + const { DEFAULT_THRESHOLDS } = require('../scripts/hallucination-config-safe.cjs'); + const { + DEFAULT_THRESHOLDS: EXPECTED, + } = require('../scripts/hallucination-config-defaults.cjs'); + expect(DEFAULT_THRESHOLDS).toEqual(EXPECTED); + }); + + it('exports DEFAULT_CONFIDENCE_WEIGHTS matching hallucination-config-defaults', () => { + const { DEFAULT_CONFIDENCE_WEIGHTS } = require('../scripts/hallucination-config-safe.cjs'); + const { + DEFAULT_CONFIDENCE_WEIGHTS: EXPECTED, + } = require('../scripts/hallucination-config-defaults.cjs'); + expect(DEFAULT_CONFIDENCE_WEIGHTS).toEqual(EXPECTED); + }); + + it('exports DEFAULT_CONFIG matching hallucination-config-defaults', () => { + const { DEFAULT_CONFIG } = require('../scripts/hallucination-config-safe.cjs'); + const { DEFAULT_CONFIG: EXPECTED } = require('../scripts/hallucination-config-defaults.cjs'); + expect(DEFAULT_CONFIG).toEqual(EXPECTED); + }); + }); + + describe('safeLoadConfig — loader works', () => { + it('returns a defined object', () => { + const { safeLoadConfig } = require('../scripts/hallucination-config-safe.cjs'); + const config = safeLoadConfig(); + expect(config).toBeDefined(); + expect(typeof config).toBe('object'); + expect(config).not.toBeNull(); + }); + + it('returned config has weights key', () => { + const { safeLoadConfig } = require('../scripts/hallucination-config-safe.cjs'); + const config = safeLoadConfig(); + expect(config.weights).toBeDefined(); + expect(typeof config.weights).toBe('object'); + }); + + it('returned config has thresholds key', () => { + const { safeLoadConfig } = require('../scripts/hallucination-config-safe.cjs'); + const config = safeLoadConfig(); + expect(config.thresholds).toBeDefined(); + expect(typeof config.thresholds.uncertain).toBe('number'); + expect(typeof config.thresholds.hallucinated).toBe('number'); + }); + + it('returned config has introspect key', () => { + const { safeLoadConfig } = require('../scripts/hallucination-config-safe.cjs'); + const config = safeLoadConfig(); + expect(typeof config.introspect).toBe('boolean'); + }); + + it('returned config is frozen', () => { + const { safeLoadConfig } = require('../scripts/hallucination-config-safe.cjs'); + const config = safeLoadConfig(); + expect(Object.isFrozen(config)).toBe(true); + }); + + it('forwards opts._homeDir to loadConfig', () => { + const { safeLoadConfig } = require('../scripts/hallucination-config-safe.cjs'); + // Passing a non-existent home dir — still returns a valid frozen config + const config = safeLoadConfig({ _homeDir: '/tmp/no-such-home-for-test-xyz' }); + expect(config).toBeDefined(); + expect(Object.isFrozen(config)).toBe(true); + }); + }); + + describe('safeLoadConfig — loader throws', () => { + it('returns a defined object when loader throws', () => { + vi.doMock('../scripts/hallucination-config.cjs', () => { + throw new Error('Simulated loader failure'); + }); + const { safeLoadConfig } = require('../scripts/hallucination-config-safe.cjs'); + const config = safeLoadConfig(); + expect(config).toBeDefined(); + expect(typeof config).toBe('object'); + }); + + it('returned fallback config has weights matching DEFAULT_WEIGHTS', () => { + vi.doMock('../scripts/hallucination-config.cjs', () => { + throw new Error('Simulated loader failure'); + }); + const { + safeLoadConfig, + DEFAULT_WEIGHTS, + } = require('../scripts/hallucination-config-safe.cjs'); + const config = safeLoadConfig(); + expect(config.weights).toEqual(DEFAULT_WEIGHTS); + }); + + it('returned fallback config has thresholds matching DEFAULT_THRESHOLDS', () => { + vi.doMock('../scripts/hallucination-config.cjs', () => { + throw new Error('Simulated loader failure'); + }); + const { + safeLoadConfig, + DEFAULT_THRESHOLDS, + } = require('../scripts/hallucination-config-safe.cjs'); + const config = safeLoadConfig(); + expect(config.thresholds).toEqual(DEFAULT_THRESHOLDS); + }); + + it('returned fallback config has confidenceWeights matching DEFAULT_CONFIDENCE_WEIGHTS', () => { + vi.doMock('../scripts/hallucination-config.cjs', () => { + throw new Error('Simulated loader failure'); + }); + const { + safeLoadConfig, + DEFAULT_CONFIDENCE_WEIGHTS, + } = require('../scripts/hallucination-config-safe.cjs'); + const config = safeLoadConfig(); + expect(config.confidenceWeights).toEqual(DEFAULT_CONFIDENCE_WEIGHTS); + }); + + it('returned fallback config is frozen', () => { + vi.doMock('../scripts/hallucination-config.cjs', () => { + throw new Error('Simulated loader failure'); + }); + const { safeLoadConfig } = require('../scripts/hallucination-config-safe.cjs'); + const config = safeLoadConfig(); + expect(Object.isFrozen(config)).toBe(true); + }); + + it('returned fallback config nested weights are frozen', () => { + vi.doMock('../scripts/hallucination-config.cjs', () => { + throw new Error('Simulated loader failure'); + }); + const { safeLoadConfig } = require('../scripts/hallucination-config-safe.cjs'); + const config = safeLoadConfig(); + expect(Object.isFrozen(config.weights)).toBe(true); + }); + + it('returned fallback config has categories as empty frozen object', () => { + vi.doMock('../scripts/hallucination-config.cjs', () => { + throw new Error('Simulated loader failure'); + }); + const { safeLoadConfig } = require('../scripts/hallucination-config-safe.cjs'); + const config = safeLoadConfig(); + expect(config.categories).toEqual({}); + expect(Object.isFrozen(config.categories)).toBe(true); + }); + + it('returned fallback config has ignorePatterns as empty frozen array', () => { + vi.doMock('../scripts/hallucination-config.cjs', () => { + throw new Error('Simulated loader failure'); + }); + const { safeLoadConfig } = require('../scripts/hallucination-config-safe.cjs'); + const config = safeLoadConfig(); + expect(Array.isArray(config.ignorePatterns)).toBe(true); + expect(config.ignorePatterns).toHaveLength(0); + }); + + it('safeLoadConfig does not throw regardless of the error type', () => { + vi.doMock('../scripts/hallucination-config.cjs', () => { + throw new TypeError('Unexpected token'); + }); + const { safeLoadConfig } = require('../scripts/hallucination-config-safe.cjs'); + expect(() => safeLoadConfig()).not.toThrow(); + }); + }); + + describe('safeLoadWeights — loader works', () => { + it('returns a plain object', () => { + const { safeLoadWeights } = require('../scripts/hallucination-config-safe.cjs'); + const weights = safeLoadWeights(); + expect(typeof weights).toBe('object'); + expect(weights).not.toBeNull(); + expect(Array.isArray(weights)).toBe(false); + }); + + it('returned weights include expected category keys', () => { + const { safeLoadWeights } = require('../scripts/hallucination-config-safe.cjs'); + const weights = safeLoadWeights(); + expect('speculation_language' in weights).toBe(true); + expect('causality_language' in weights).toBe(true); + expect('pseudo_quantification' in weights).toBe(true); + }); + }); + + describe('safeLoadWeights — loader throws', () => { + it('returns an object matching DEFAULT_WEIGHTS when loader throws', () => { + vi.doMock('../scripts/hallucination-config.cjs', () => { + throw new Error('Simulated loader failure'); + }); + const { + safeLoadWeights, + DEFAULT_WEIGHTS, + } = require('../scripts/hallucination-config-safe.cjs'); + const weights = safeLoadWeights(); + expect(weights).toEqual(DEFAULT_WEIGHTS); + }); + + it('returned fallback weights is not the same reference as DEFAULT_WEIGHTS', () => { + vi.doMock('../scripts/hallucination-config.cjs', () => { + throw new Error('Simulated loader failure'); + }); + const { + safeLoadWeights, + DEFAULT_WEIGHTS, + } = require('../scripts/hallucination-config-safe.cjs'); + const weights = safeLoadWeights(); + // safeLoadWeights returns a spread copy, not the frozen constant itself + expect(weights).not.toBe(DEFAULT_WEIGHTS); + }); + + it('safeLoadWeights does not throw regardless of the error type', () => { + vi.doMock('../scripts/hallucination-config.cjs', () => { + throw new RangeError('Out of range'); + }); + const { safeLoadWeights } = require('../scripts/hallucination-config-safe.cjs'); + expect(() => safeLoadWeights()).not.toThrow(); + }); + }); +}); diff --git a/tests/hallucination-config-toml.test.cjs b/tests/hallucination-config-toml.test.cjs new file mode 100644 index 0000000..ac0b3a4 --- /dev/null +++ b/tests/hallucination-config-toml.test.cjs @@ -0,0 +1,207 @@ +'use strict'; + +const { parseToml } = require('../scripts/hallucination-config-toml.cjs'); + +describe('parseToml', () => { + describe('empty input', () => { + it('empty string returns empty object', () => { + expect(parseToml('')).toEqual({}); + }); + + it('whitespace-only string returns empty object', () => { + expect(parseToml(' \n \n ')).toEqual({}); + }); + }); + + describe('comments', () => { + it('full-line comments are skipped', () => { + const src = ` +# This is a comment +# Another comment +`; + expect(parseToml(src)).toEqual({}); + }); + + it('inline comments are stripped from values', () => { + const src = 'key = "hello" # comment here'; + expect(parseToml(src)).toEqual({ key: 'hello' }); + }); + + it('inline comment after number', () => { + const src = 'count = 42 # the answer'; + expect(parseToml(src)).toEqual({ count: 42 }); + }); + }); + + describe('scalar key-value pairs', () => { + it('parses string values', () => { + expect(parseToml('name = "Alice"')).toEqual({ name: 'Alice' }); + }); + + it('parses single-quoted string values', () => { + expect(parseToml("mode = 'strict'")).toEqual({ mode: 'strict' }); + }); + + it('parses integer values', () => { + expect(parseToml('count = 10')).toEqual({ count: 10 }); + }); + + it('parses float values', () => { + expect(parseToml('ratio = 0.75')).toEqual({ ratio: 0.75 }); + }); + + it('parses negative numbers', () => { + expect(parseToml('offset = -5')).toEqual({ offset: -5 }); + }); + + it('parses boolean true', () => { + expect(parseToml('enabled = true')).toEqual({ enabled: true }); + }); + + it('parses boolean false', () => { + expect(parseToml('verbose = false')).toEqual({ verbose: false }); + }); + + it('ignores lines without an equals sign', () => { + const src = 'not a valid line\nkey = "value"'; + expect(parseToml(src)).toEqual({ key: 'value' }); + }); + }); + + describe('section headers', () => { + it('[section] creates a nested object', () => { + const src = ` +[tool] +name = "test" +`; + expect(parseToml(src)).toEqual({ tool: { name: 'test' } }); + }); + + it('[section.sub] creates doubly-nested object', () => { + const src = ` +[tool.myapp] +debug = true +`; + expect(parseToml(src)).toEqual({ tool: { myapp: { debug: true } } }); + }); + + it('multiple section headers', () => { + const src = ` +[a] +x = 1 + +[b] +y = 2 +`; + expect(parseToml(src)).toEqual({ a: { x: 1 }, b: { y: 2 } }); + }); + + it('top-level keys before any section header', () => { + const src = ` +version = "1.0" + +[meta] +author = "Alice" +`; + expect(parseToml(src)).toEqual({ version: '1.0', meta: { author: 'Alice' } }); + }); + + it('deeply dotted section header [a.b.c]', () => { + const src = ` +[a.b.c] +leaf = 99 +`; + expect(parseToml(src)).toEqual({ a: { b: { c: { leaf: 99 } } } }); + }); + + it('section with hyphenated name', () => { + const src = ` +[hallucination-detector] +enabled = true +`; + expect(parseToml(src)).toEqual({ 'hallucination-detector': { enabled: true } }); + }); + }); + + describe('arrays', () => { + it('parses empty array', () => { + expect(parseToml('items = []')).toEqual({ items: [] }); + }); + + it('parses array of numbers', () => { + expect(parseToml('nums = [1, 2, 3]')).toEqual({ nums: [1, 2, 3] }); + }); + + it('parses array of strings', () => { + expect(parseToml('tags = ["a", "b", "c"]')).toEqual({ tags: ['a', 'b', 'c'] }); + }); + + it('parses array of booleans', () => { + expect(parseToml('flags = [true, false, true]')).toEqual({ flags: [true, false, true] }); + }); + }); + + describe('inline tables', () => { + it('parses inline table with string values', () => { + const src = 'config = {key = "value", mode = "strict"}'; + expect(parseToml(src)).toEqual({ config: { key: 'value', mode: 'strict' } }); + }); + + it('parses inline table with number values', () => { + const src = 'thresholds = {uncertain = 0.3, hallucinated = 0.6}'; + expect(parseToml(src)).toEqual({ thresholds: { uncertain: 0.3, hallucinated: 0.6 } }); + }); + + it('parses empty inline table', () => { + const src = 'opts = {}'; + expect(parseToml(src)).toEqual({ opts: {} }); + }); + }); + + describe('quoted string escape sequences', () => { + it('parses \\n as newline', () => { + const src = 'msg = "line1\\nline2"'; + const result = parseToml(src); + expect(result.msg).toBe('line1\nline2'); + }); + + it('parses \\t as tab', () => { + const src = 'msg = "col1\\tcol2"'; + const result = parseToml(src); + expect(result.msg).toBe('col1\tcol2'); + }); + + it('parses \\\\ as backslash', () => { + const src = 'path = "C:\\\\Users"'; + const result = parseToml(src); + expect(result.path).toBe('C:\\Users'); + }); + + it('parses \\" as double-quote', () => { + const src = 'msg = "say \\"hi\\""'; + const result = parseToml(src); + expect(result.msg).toBe('say "hi"'); + }); + }); + + describe('real-world hallucination-detector config section', () => { + it('parses a realistic [tool.hallucination-detector] block', () => { + const src = ` +[tool.hallucination-detector] +severity = "warning" +maxTriggersPerResponse = 10 +debug = false +outputFormat = "json" +ignoreCategories = ["completeness_claim"] +`; + const result = parseToml(src); + expect(result.tool['hallucination-detector']).toEqual({ + severity: 'warning', + maxTriggersPerResponse: 10, + debug: false, + outputFormat: 'json', + ignoreCategories: ['completeness_claim'], + }); + }); + }); +}); diff --git a/tests/hallucination-config-validate.test.cjs b/tests/hallucination-config-validate.test.cjs new file mode 100644 index 0000000..f691829 --- /dev/null +++ b/tests/hallucination-config-validate.test.cjs @@ -0,0 +1,402 @@ +'use strict'; + +const { + validateConfig, + isValidThresholds, + isValidCategoryThreshold, +} = require('../scripts/hallucination-config-validate.cjs'); + +describe('isValidThresholds', () => { + it('returns true for valid thresholds', () => { + expect(isValidThresholds({ uncertain: 0.3, hallucinated: 0.6 })).toBe(true); + }); + + it('returns true when uncertain equals hallucinated', () => { + expect(isValidThresholds({ uncertain: 0.5, hallucinated: 0.5 })).toBe(true); + }); + + it('returns true for boundary values 0 and 1', () => { + expect(isValidThresholds({ uncertain: 0, hallucinated: 1 })).toBe(true); + }); + + it('returns false for null', () => { + expect(isValidThresholds(null)).toBe(false); + }); + + it('returns false for undefined', () => { + expect(isValidThresholds(undefined)).toBe(false); + }); + + it('returns false for a string', () => { + expect(isValidThresholds('0.3')).toBe(false); + }); + + it('returns false for an array', () => { + expect(isValidThresholds([0.3, 0.6])).toBe(false); + }); + + it('returns false when uncertain is missing', () => { + expect(isValidThresholds({ hallucinated: 0.6 })).toBe(false); + }); + + it('returns false when hallucinated is missing', () => { + expect(isValidThresholds({ uncertain: 0.3 })).toBe(false); + }); + + it('returns false when uncertain is a string', () => { + expect(isValidThresholds({ uncertain: '0.3', hallucinated: 0.6 })).toBe(false); + }); + + it('returns false when uncertain is NaN', () => { + expect(isValidThresholds({ uncertain: NaN, hallucinated: 0.6 })).toBe(false); + }); + + it('returns false when uncertain is Infinity', () => { + expect(isValidThresholds({ uncertain: Infinity, hallucinated: 0.6 })).toBe(false); + }); + + it('returns false when uncertain < 0', () => { + expect(isValidThresholds({ uncertain: -0.1, hallucinated: 0.6 })).toBe(false); + }); + + it('returns false when uncertain > 1', () => { + expect(isValidThresholds({ uncertain: 1.1, hallucinated: 0.6 })).toBe(false); + }); + + it('returns false when hallucinated < 0', () => { + expect(isValidThresholds({ uncertain: 0.3, hallucinated: -0.1 })).toBe(false); + }); + + it('returns false when hallucinated > 1', () => { + expect(isValidThresholds({ uncertain: 0.3, hallucinated: 1.5 })).toBe(false); + }); + + it('returns false when uncertain > hallucinated (inverted)', () => { + expect(isValidThresholds({ uncertain: 0.8, hallucinated: 0.3 })).toBe(false); + }); +}); + +describe('isValidCategoryThreshold', () => { + it('delegates to isValidThresholds — valid input returns true', () => { + expect(isValidCategoryThreshold({ uncertain: 0.2, hallucinated: 0.7 })).toBe(true); + }); + + it('delegates to isValidThresholds — invalid input returns false', () => { + expect(isValidCategoryThreshold(null)).toBe(false); + expect(isValidCategoryThreshold({ uncertain: 0.9, hallucinated: 0.1 })).toBe(false); + }); +}); + +describe('validateConfig', () => { + describe('input guards', () => { + it('returns empty object for null input', () => { + expect(validateConfig(null, 'test')).toEqual({}); + }); + + it('returns empty object for undefined input', () => { + expect(validateConfig(undefined, 'test')).toEqual({}); + }); + + it('returns empty object for non-object input', () => { + expect(validateConfig('string', 'test')).toEqual({}); + }); + + it('does not throw when source is omitted', () => { + expect(() => validateConfig({ severity: 'error' })).not.toThrow(); + }); + }); + + describe('severity validation', () => { + it('preserves valid severity "error"', () => { + const obj = { severity: 'error' }; + validateConfig(obj, 'test'); + expect(obj.severity).toBe('error'); + }); + + it('preserves valid severity "warning"', () => { + const obj = { severity: 'warning' }; + validateConfig(obj, 'test'); + expect(obj.severity).toBe('warning'); + }); + + it('preserves valid severity "info"', () => { + const obj = { severity: 'info' }; + validateConfig(obj, 'test'); + expect(obj.severity).toBe('info'); + }); + + it('removes invalid severity value', () => { + const obj = { severity: 'critical' }; + validateConfig(obj, 'test'); + expect('severity' in obj).toBe(false); + }); + }); + + describe('outputFormat validation', () => { + it('preserves valid outputFormat "text"', () => { + const obj = { outputFormat: 'text' }; + validateConfig(obj, 'test'); + expect(obj.outputFormat).toBe('text'); + }); + + it('preserves valid outputFormat "json"', () => { + const obj = { outputFormat: 'json' }; + validateConfig(obj, 'test'); + expect(obj.outputFormat).toBe('json'); + }); + + it('preserves valid outputFormat "jsonl"', () => { + const obj = { outputFormat: 'jsonl' }; + validateConfig(obj, 'test'); + expect(obj.outputFormat).toBe('jsonl'); + }); + + it('removes invalid outputFormat value', () => { + const obj = { outputFormat: 'xml' }; + validateConfig(obj, 'test'); + expect('outputFormat' in obj).toBe(false); + }); + }); + + describe('numeric field validation', () => { + it('preserves valid maxTriggersPerResponse', () => { + const obj = { maxTriggersPerResponse: 10 }; + validateConfig(obj, 'test'); + expect(obj.maxTriggersPerResponse).toBe(10); + }); + + it('removes non-integer maxTriggersPerResponse', () => { + const obj = { maxTriggersPerResponse: 1.5 }; + validateConfig(obj, 'test'); + expect('maxTriggersPerResponse' in obj).toBe(false); + }); + + it('removes negative maxTriggersPerResponse', () => { + const obj = { maxTriggersPerResponse: -1 }; + validateConfig(obj, 'test'); + expect('maxTriggersPerResponse' in obj).toBe(false); + }); + + it('preserves maxBlocksPerSession = null', () => { + const obj = { maxBlocksPerSession: null }; + validateConfig(obj, 'test'); + expect(obj.maxBlocksPerSession).toBeNull(); + }); + + it('removes non-integer maxBlocksPerSession', () => { + const obj = { maxBlocksPerSession: 2.5 }; + validateConfig(obj, 'test'); + expect('maxBlocksPerSession' in obj).toBe(false); + }); + + it('preserves valid contextLines', () => { + const obj = { contextLines: 3 }; + validateConfig(obj, 'test'); + expect(obj.contextLines).toBe(3); + }); + + it('removes negative contextLines', () => { + const obj = { contextLines: -1 }; + validateConfig(obj, 'test'); + expect('contextLines' in obj).toBe(false); + }); + + it('preserves valid reportingThreshold', () => { + const obj = { reportingThreshold: 75 }; + validateConfig(obj, 'test'); + expect(obj.reportingThreshold).toBe(75); + }); + + it('removes out-of-range reportingThreshold', () => { + const obj = { reportingThreshold: 150 }; + validateConfig(obj, 'test'); + expect('reportingThreshold' in obj).toBe(false); + }); + }); + + describe('boolean field validation', () => { + it('preserves boolean debug = true', () => { + const obj = { debug: true }; + validateConfig(obj, 'test'); + expect(obj.debug).toBe(true); + }); + + it('removes non-boolean debug', () => { + const obj = { debug: 'yes' }; + validateConfig(obj, 'test'); + expect('debug' in obj).toBe(false); + }); + + it('removes non-boolean introspect', () => { + const obj = { introspect: 1 }; + validateConfig(obj, 'test'); + expect('introspect' in obj).toBe(false); + }); + + it('removes non-boolean dryRun', () => { + const obj = { dryRun: 'true' }; + validateConfig(obj, 'test'); + expect('dryRun' in obj).toBe(false); + }); + + it('removes non-boolean warnOnly', () => { + const obj = { warnOnly: 0 }; + validateConfig(obj, 'test'); + expect('warnOnly' in obj).toBe(false); + }); + + it('removes non-boolean blockSubagents', () => { + const obj = { blockSubagents: null }; + validateConfig(obj, 'test'); + expect('blockSubagents' in obj).toBe(false); + }); + + it('removes non-boolean blockUserSessions', () => { + const obj = { blockUserSessions: 'yes' }; + validateConfig(obj, 'test'); + expect('blockUserSessions' in obj).toBe(false); + }); + + it('removes non-boolean includeContext', () => { + const obj = { includeContext: 1 }; + validateConfig(obj, 'test'); + expect('includeContext' in obj).toBe(false); + }); + }); + + describe('ignoreCategories validation', () => { + it('preserves valid array', () => { + const obj = { ignoreCategories: ['speculation_language'] }; + validateConfig(obj, 'test'); + expect(obj.ignoreCategories).toEqual(['speculation_language']); + }); + + it('removes non-array ignoreCategories', () => { + const obj = { ignoreCategories: 'speculation_language' }; + validateConfig(obj, 'test'); + expect('ignoreCategories' in obj).toBe(false); + }); + }); + + describe('weights validation', () => { + it('preserves valid weights object', () => { + const obj = { weights: { speculation_language: 0.5 } }; + validateConfig(obj, 'test'); + expect(obj.weights).toEqual({ speculation_language: 0.5 }); + }); + + it('removes non-object weights', () => { + const obj = { weights: 'invalid' }; + validateConfig(obj, 'test'); + expect('weights' in obj).toBe(false); + }); + + it('removes array weights', () => { + const obj = { weights: [0.5] }; + validateConfig(obj, 'test'); + expect('weights' in obj).toBe(false); + }); + }); + + describe('thresholds validation', () => { + it('preserves valid thresholds', () => { + const obj = { thresholds: { uncertain: 0.3, hallucinated: 0.6 } }; + validateConfig(obj, 'test'); + expect(obj.thresholds).toEqual({ uncertain: 0.3, hallucinated: 0.6 }); + }); + + it('removes invalid thresholds', () => { + const obj = { thresholds: { uncertain: 0.9, hallucinated: 0.1 } }; + validateConfig(obj, 'test'); + expect('thresholds' in obj).toBe(false); + }); + }); + + describe('confidenceWeights validation', () => { + it('preserves valid confidenceWeights object', () => { + const obj = { + confidenceWeights: { + patternStrength: 0.4, + evidenceProximity: 0.25, + categoryStacking: 0.2, + contextDensity: 0.15, + }, + }; + validateConfig(obj, 'test'); + expect(obj.confidenceWeights).toBeDefined(); + expect(obj.confidenceWeights.patternStrength).toBe(0.4); + }); + + it('removes confidenceWeights when it is not a plain object', () => { + const obj = { confidenceWeights: 'invalid' }; + validateConfig(obj, 'test'); + expect('confidenceWeights' in obj).toBe(false); + }); + + it('removes individual invalid confidence weight keys', () => { + const obj = { confidenceWeights: { patternStrength: 1.5, evidenceProximity: 0.25 } }; + validateConfig(obj, 'test'); + expect('patternStrength' in obj.confidenceWeights).toBe(false); + expect(obj.confidenceWeights.evidenceProximity).toBe(0.25); + }); + }); + + describe('categories validation', () => { + it('preserves valid categories object', () => { + const obj = { + categories: { + speculation_language: { enabled: false }, + }, + }; + validateConfig(obj, 'test'); + expect(obj.categories.speculation_language).toEqual({ enabled: false }); + }); + + it('removes non-object categories', () => { + const obj = { categories: 'invalid' }; + validateConfig(obj, 'test'); + expect('categories' in obj).toBe(false); + }); + + it('preserves categories with valid threshold pairs', () => { + const obj = { + categories: { + speculation_language: { uncertain: 0.2, hallucinated: 0.5 }, + }, + }; + validateConfig(obj, 'test'); + expect(obj.categories.speculation_language.uncertain).toBe(0.2); + expect(obj.categories.speculation_language.hallucinated).toBe(0.5); + }); + + it('removes threshold fields but preserves other fields on invalid threshold pair', () => { + const obj = { + categories: { + speculation_language: { + uncertain: 0.9, + hallucinated: 0.1, + enabled: false, + }, + }, + }; + validateConfig(obj, 'test'); + expect('uncertain' in obj.categories.speculation_language).toBe(false); + expect('hallucinated' in obj.categories.speculation_language).toBe(false); + expect(obj.categories.speculation_language.enabled).toBe(false); + }); + }); + + describe('valid fields preservation', () => { + it('returns the mutated input object', () => { + const obj = { severity: 'error' }; + const result = validateConfig(obj, 'test'); + expect(result).toBe(obj); + }); + + it('preserves fields not validated by validateConfig', () => { + const obj = { severity: 'error', customField: 'kept' }; + validateConfig(obj, 'test'); + expect(obj.customField).toBe('kept'); + }); + }); +}); diff --git a/vitest.config.cjs b/vitest.config.cjs index 0330eed..aa9b594 100644 --- a/vitest.config.cjs +++ b/vitest.config.cjs @@ -16,6 +16,11 @@ module.exports = defineConfig({ 'scripts/hallucination-claim-structure.cjs', 'scripts/hallucination-memory-gate.cjs', 'scripts/hallucination-config.cjs', + 'scripts/hallucination-config-defaults.cjs', + 'scripts/hallucination-config-toml.cjs', + 'scripts/hallucination-config-merge.cjs', + 'scripts/hallucination-config-validate.cjs', + 'scripts/hallucination-config-safe.cjs', 'scripts/hallucination-annotate.cjs', '.claude/scripts/lib/story-helpers.cjs', ],