feat(backend): AGI tutor RAG pipeline over course content (#406)

backend/.env.example

@@ -168,3 +168,20 @@ GLOSSARY_TECH_ID=your-tech-glossary-id
 GLOSSARY_SUBTITLE_ID=your-subtitle-glossary-id
 TRANSLATION_CACHE_TTL=2592000
 TRANSLATION_QUALITY_THRESHOLD=0.85
+
+# =============================================================================
+# AGI Tutor RAG Pipeline Configuration (Issue #406)
+# =============================================================================
+# Vector store mode: "qdrant" (default) or "memory" (tests / no-Docker dev)
+RAG_VECTOR_STORE=qdrant
+QDRANT_URL=http://localhost:6333
+QDRANT_COLLECTION=aethermint_course_content
+RAG_EMBEDDING_DIM=768
+RAG_TOP_K=5
+RAG_CONFIDENCE_THRESHOLD=0.55
+# Content indexing worker poll interval (ms)
+RAG_INDEX_INTERVAL_MS=300000
+# Optional LLM for answer generation. When unset, a deterministic
+# extractive generator grounded in the retrieved passages is used.
+OPENAI_API_KEY=
+RAG_LLM_MODEL=gpt-4o-mini

backend/src/__tests__/ragPipeline.test.ts

@@ -0,0 +1,257 @@
+/**
+ * Tests for the AGI tutor RAG pipeline (Issue #406).
+ *
+ * All tests run against the in-memory vector store so they are
+ * deterministic and require no external services.
+ */
+
+import {
+  LocalHashEmbedder,
+  cosineSimilarity,
+  MemoryVectorStore,
+  SeedCourseContentProvider,
+  RagPipeline,
+  faithfulnessScore,
+  citationCoverage,
+  computeGroundingMetrics,
+  OpenAiAnswerGenerator,
+} from '../services/tutor';
+
+describe('LocalHashEmbedder', () => {
+  const embedder = new LocalHashEmbedder(256);
+
+  it('produces fixed-dimension, unit-norm vectors', () => {
+    const vector = embedder.embed('smart contracts run on a blockchain');
+    expect(vector).toHaveLength(256);
+    const norm = Math.sqrt(vector.reduce((sum, v) => sum + v * v, 0));
+    expect(norm).toBeCloseTo(1, 5);
+  });
+
+  it('is deterministic for the same input', () => {
+    expect(embedder.embed('what is a smart contract?')).toEqual(
+      embedder.embed('what is a smart contract?')
+    );
+  });
+
+  it('ranks related text above unrelated text', () => {
+    const query = embedder.embed('what is a smart contract?');
+    const related = embedder.embed(
+      'A smart contract is a program that runs on a blockchain.'
+    );
+    const unrelated = embedder.embed('the quick brown fox jumps over the lazy dog');
+
+    expect(cosineSimilarity(query, related)).toBeGreaterThan(
+      cosineSimilarity(query, unrelated)
+    );
+  });
+});
+
+describe('MemoryVectorStore', () => {
+  const store = new MemoryVectorStore({ collection: 'test' });
+  const embedder = new LocalHashEmbedder(256);
+
+  beforeEach(async () => {
+    await store.clear();
+  });
+
+  it('upserts, counts, and searches by similarity', async () => {
+    const chunks = [
+      {
+        id: 'c1',
+        courseId: 'course_a',
+        courseTitle: 'Course A',
+        moduleId: 'm1',
+        moduleTitle: 'Module 1',
+        lessonId: 'l1',
+        lessonTitle: 'Smart Contracts',
+        title: 'Smart Contracts',
+        content: 'A smart contract is a program that runs on a blockchain.',
+        contentType: 'lesson' as const,
+      },
+      {
+        id: 'c2',
+        courseId: 'course_a',
+        courseTitle: 'Course A',
+        moduleId: 'm2',
+        moduleTitle: 'Module 2',
+        lessonId: 'l2',
+        lessonTitle: 'Graph Theory',
+        title: 'Graph Theory',
+        content: 'A graph is a collection of vertices connected by edges.',
+        contentType: 'lesson' as const,
+      },
+    ];
+    const vectors = chunks.map((c) => embedder.embed(c.content));
+
+    await store.upsert(chunks, vectors);
+    expect(await store.count()).toBe(2);
+
+    const results = await store.search(embedder.embed('smart contracts'), 1);
+    expect(results).toHaveLength(1);
+    expect(results[0].chunk.id).toBe('c1');
+    expect(results[0].score).toBeGreaterThan(0);
+    expect(results[0].excerpt.length).toBeGreaterThan(0);
+  });
+
+  it('replaces existing chunks on re-upsert', async () => {
+    const chunk = {
+      id: 'c1',
+      courseId: 'course_a',
+      courseTitle: 'Course A',
+      moduleId: 'm1',
+      moduleTitle: 'Module 1',
+      title: 'Lesson',
+      content: 'version one',
+      contentType: 'lesson' as const,
+    };
+    await store.upsert([chunk], [embedder.embed(chunk.content)]);
+    await store.upsert([{ ...chunk, content: 'version two' }], [embedder.embed('version two')]);
+    expect(await store.count()).toBe(1);
+  });
+});
+
+describe('Grounding metrics', () => {
+  const passage = (content: string) => ({
+    chunk: {
+      id: 'p1',
+      courseId: 'course_a',
+      courseTitle: 'Course A',
+      moduleId: 'm1',
+      moduleTitle: 'Module 1',
+      title: 'Lesson',
+      content,
+      contentType: 'lesson' as const,
+    },
+    score: 0.8,
+    excerpt: content.slice(0, 100),
+  });
+
+  it('scores grounded answers as faithful', () => {
+    const source =
+      'Consensus is the process by which participants in a blockchain network agree on the state of the ledger. Proof of Work requires miners to solve a computationally expensive puzzle.';
+    const answer =
+      'Consensus is the process by which participants agree on the state of the ledger [1]. Proof of Work requires solving a computationally expensive puzzle [1].';
+    const metrics = computeGroundingMetrics('what is consensus?', answer, [
+      passage(source),
+    ]);
+    expect(metrics.faithfulnessScore).toBeGreaterThan(0.9);
+    expect(metrics.citationCoverage).toBe(1);
+    expect(metrics.grounded).toBe(true);
+  });
+
+  it('scores answers unrelated to the context as unfaithful', () => {
+    const source =
+      'Consensus is the process by which participants in a blockchain network agree on the state of the ledger.';
+    const answer = 'The capital of France is Paris, a city famous for its art.';
+    const metrics = computeGroundingMetrics('what is the capital?', answer, [
+      passage(source),
+    ]);
+    expect(metrics.faithfulnessScore).toBeLessThan(0.5);
+    expect(metrics.grounded).toBe(false);
+  });
+
+  it('citation coverage reflects inline markers', () => {
+    expect(citationCoverage('First claim [1]. Second claim [2].')).toBe(1);
+    expect(citationCoverage('First claim. Second claim.')).toBe(0);
+  });
+
+  it('faithfulness handles empty inputs', () => {
+    expect(faithfulnessScore('', [passage('some content')])).toBe(0);
+  });
+});
+
+describe('RagPipeline (in-memory)', () => {
+  const buildPipeline = () =>
+    new RagPipeline({
+      store: new MemoryVectorStore({ collection: 'test' }),
+      provider: new SeedCourseContentProvider(),
+      embedder: new LocalHashEmbedder(768),
+    });
+
+  it('indexes seed course content into the store', async () => {
+    const pipeline = buildPipeline();
+    const result = await pipeline.indexContent();
+
+    expect(result.indexed).toBeGreaterThan(0);
+    expect(result.chunkCount).toBe(result.indexed);
+
+    const status = await pipeline.getStatus();
+    expect(status.chunkCount).toBe(result.indexed);
+    expect(status.lastIndexedAt).not.toBeNull();
+  });
+
+  it('skips re-indexing when content is unchanged', async () => {
+    const pipeline = buildPipeline();
+    await pipeline.indexContent();
+    const second = await pipeline.indexContent();
+
+    expect(second.indexed).toBe(0);
+    expect(second.skipped).toBeGreaterThan(0);
+  });
+
+  it('answers a question grounded in course material with citations', async () => {
+    const pipeline = buildPipeline();
+    await pipeline.indexContent();
+
+    const answer = await pipeline.answer('what is a smart contract?');
+
+    expect(answer.fallback).toBe(false);
+    expect(answer.grounded).toBe(true);
+    expect(answer.answer).not.toContain("don't know");
+    expect(answer.citations.length).toBeGreaterThan(0);
+    expect(answer.citations[0]).toMatchObject({
+      courseId: 'course_blockchain_fundamentals',
+    });
+    expect(answer.sources.length).toBeGreaterThan(0);
+    expect(answer.metrics.confidence).toBeGreaterThanOrEqual(answer.metrics.threshold);
+    expect(answer.metrics.faithfulnessScore).toBeGreaterThan(0.5);
+    expect(answer.metrics.citationCoverage).toBeGreaterThan(0);
+  });
+
+  it('falls back to a safe "I don\'t know" when confidence is low', async () => {
+    const pipeline = buildPipeline();
+    await pipeline.indexContent();
+
+    const answer = await pipeline.answer(
+      'what is the population of the city of Atlantis?'
+    );
+
+    expect(answer.fallback).toBe(true);
+    expect(answer.grounded).toBe(false);
+    expect(answer.answer).toContain("don't know");
+    expect(answer.citations).toHaveLength(0);
+    expect(answer.sources).toHaveLength(0);
+  });
+
+  it('answers without indexing (empty store) via fallback', async () => {
+    const pipeline = buildPipeline();
+    const answer = await pipeline.answer('what is a smart contract?');
+
+    expect(answer.fallback).toBe(true);
+    expect(answer.answer).toContain("don't know");
+  });
+});
+
+describe('OpenAiAnswerGenerator', () => {
+  it('is unavailable when no API key is configured', () => {
+    const previous = process.env.OPENAI_API_KEY;
+    delete process.env.OPENAI_API_KEY;
+    const generator = new OpenAiAnswerGenerator();
+    expect(generator.available).toBe(false);
+    if (previous !== undefined) {
+      process.env.OPENAI_API_KEY = previous;
+    }
+  });
+
+  it('throws when asked to generate without an API key', async () => {
+    const previous = process.env.OPENAI_API_KEY;
+    delete process.env.OPENAI_API_KEY;
+    const generator = new OpenAiAnswerGenerator();
+    await expect(generator.generate('question', [])).rejects.toThrow(
+      'OPENAI_API_KEY'
+    );
+    if (previous !== undefined) {
+      process.env.OPENAI_API_KEY = previous;
+    }
+  });
+});

backend/src/controllers/agiTutorController.ts

@@ -4,6 +4,7 @@ import { UniversalKnowledgeService } from '../services/universalKnowledgeService
 import { StudentAdaptationService } from '../services/studentAdaptationService';
 import { EmotionalIntelligenceService } from '../services/emotionalIntelligenceService';
 import { CrossDomainIntegrationService } from '../services/crossDomainIntegrationService';
+import { ragPipeline } from '../services/tutor';
 import logger from '../utils/logger';
 
 export class AGITutorController {
@@ -289,4 +290,76 @@ export class AGITutorController {
       });
     }
   }
+
+  /**
+   * Answer a question using the RAG pipeline over indexed course content
+   */
+  async askRagQuestion(req: Request, res: Response) {
+    try {
+      const { question, topK } = req.body;
+
+      if (!question || typeof question !== 'string' || question.trim() === '') {
+        res.status(400).json({
+          success: false,
+          error: 'question is required'
+        });
+        return;
+      }
+
+      const answer = await ragPipeline.answer(question, {
+        topK: typeof topK === 'number' && topK > 0 ? topK : undefined
+      });
+
+      res.json({
+        success: true,
+        data: answer
+      });
+    } catch (error) {
+      logger.error('Error answering RAG question:', error);
+      res.status(500).json({
+        success: false,
+        error: 'Failed to answer question'
+      });
+    }
+  }
+
+  /**
+   * Trigger indexing of course content into the vector store
+   */
+  async triggerRagIndexing(req: Request, res: Response) {
+    try {
+      const result = await ragPipeline.indexContent();
+
+      res.json({
+        success: true,
+        data: result
+      });
+    } catch (error) {
+      logger.error('Error indexing course content:', error);
+      res.status(500).json({
+        success: false,
+        error: 'Failed to index course content'
+      });
+    }
+  }
+
+  /**
+   * Get RAG pipeline indexing status
+   */
+  async getRagStatus(req: Request, res: Response) {
+    try {
+      const status = await ragPipeline.getStatus();
+
+      res.json({
+        success: true,
+        data: status
+      });
+    } catch (error) {
+      logger.error('Error getting RAG status:', error);
+      res.status(500).json({
+        success: false,
+        error: 'Failed to get RAG status'
+      });
+    }
+  }
 }

backend/src/index.ts

@@ -27,22 +27,24 @@
 import mongoose from 'mongoose';
 import { MigrationRunner, createPool } from './utils/migrate';
 import * as path from 'path';
 // @ts-ignore
 import SecureRealtimeCommunication from './services/secureRealtimeCommunication';
 import { swaggerSpec } from './config/swagger';
 import { openApiSpec } from './docs/openapi';
 import { Migrator } from './utils/migrate';
 
 // @ts-ignore
 import * as transactionQueue from './services/transactionQueue';
 // @ts-ignore
 import * as transactionProcessor from './workers/transactionProcessor';
 // @ts-ignore
 import * as transactionEvents from './events/transactionEvents';
 
 // Bridge relayer monitor watch job — Issue #423
 import { bridgeMonitorJob } from './workers/bridgeMonitorJob';
 import { processQuestionGenerationJob } from './workers/questionGenJob';
+// Course content indexing worker — Issue #406 (AGI tutor RAG pipeline)
+import { startIndexingJob, stopIndexingJob } from './workers/indexingJob';
 import questionGeneratorService from './services/questionGen/questionGenerator';
 
 // Background job queue — Issue #258
@@ -60,9 +62,9 @@
   securityHeadersMiddleware
 } from './middleware/security';
 import { detectSuspiciousPatterns } from './middleware/sanitizer';
 // @ts-ignore - CommonJS module without type declarations
 import { validateFileUpload } from './middleware/sanitizeMiddleware';
 // @ts-ignore
 import { tieredRateLimiter, transactionLimiter } from './middleware/rateLimiter';
 import { rateLimits } from './middleware/rateLimit';
 import { idempotency } from './middleware/idempotency';
@@ -91,7 +93,7 @@
 };
 
 // Import routes
 // @ts-ignore
 const quizRoutes = loadRoute('./routes/quizRoutes');
 // @ts-ignore
 const questionGenerationRoutes = loadRoute('./routes/questionGen');
@@ -543,6 +545,11 @@
       await bridgeMonitorJob.start();
     }
 
+    // Start the course content indexing worker for the AGI tutor RAG
+    // pipeline (Issue #406). Fails open when the vector store is
+    // unreachable so single-node deployments keep working.
+    startIndexingJob();
+
     // Initialise the presence & availability system (Issue #405). Fails open
     // when Redis is unreachable so single-node deployments keep working.
     await presenceService.initialize();
@@ -665,6 +672,7 @@
       { name: 'transaction-processor', run: () => typeof (transactionProcessor as any).stop === 'function' && (transactionProcessor as any).stop() },
       { name: 'transaction-events', run: () => typeof (transactionEvents as any).stopListening === 'function' && (transactionEvents as any).stopListening() },
       { name: 'presence', run: () => presenceService.destroy() },
+      { name: 'indexing-job', run: () => stopIndexingJob() },
       { name: 'job-queue', run: async () => { try { const jq = getJobQueue(); await jq.destroy(); } catch { /* queue may not be initialised */ } } },
       {
         name: 'redis',