diff --git a/.github/workflows/bundle-budget.yml b/.github/workflows/bundle-budget.yml index d125a524..10e30641 100644 --- a/.github/workflows/bundle-budget.yml +++ b/.github/workflows/bundle-budget.yml @@ -9,19 +9,30 @@ on: jobs: bundle-budget: runs-on: ubuntu-latest - defaults: - run: - working-directory: frontend steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: node-version: 20 + cache: 'npm' + cache-dependency-path: | + package-lock.json + frontend/package-lock.json + + - name: Verify lock file + run: | + if [ ! -f package-lock.json ]; then + echo "Error: root package-lock.json is missing" + exit 1 + fi + - name: Install dependencies - run: npm ci + run: npm ci -w frontend + - name: Build env: NEXT_PUBLIC_STELLAR_RECEIVER_ADDRESS: GAAZI4TCR3TY5OJHCTJC2A4QSY6CJWJH5IAJTGKIN2ER7LBNVKOCCWNA - run: npm run build + run: npm run build -w frontend + - name: Check bundle budget (report-only) - run: npm run bundle-budget \ No newline at end of file + run: npm run bundle-budget -w frontend \ No newline at end of file diff --git a/backend/jest.moderation.config.js b/backend/jest.moderation.config.js new file mode 100644 index 00000000..f0ccf46d --- /dev/null +++ b/backend/jest.moderation.config.js @@ -0,0 +1,26 @@ +/** Standalone config for moderation tests (JS only) */ +module.exports = { + testEnvironment: 'node', + roots: ['/tests'], + testMatch: ['**/moderation.test.js'], + transform: { + '^.+\\.ts$': ['ts-jest', { + tsconfig: { + target: 'ES2020', + module: 'commonjs', + esModuleInterop: true, + skipLibCheck: true, + resolveJsonModule: true, + moduleResolution: 'node', + declaration: false, + strict: false, + } + }], + }, + moduleFileExtensions: ['ts', 'js', 'json'], + transformIgnorePatterns: [], + testTimeout: 30000, + verbose: true, + forceExit: true, + clearMocks: true, +}; \ No newline at end of file diff --git a/backend/src/controllers/moderationController.ts b/backend/src/controllers/moderationController.ts new file mode 100644 index 00000000..0ba98ccc --- /dev/null +++ b/backend/src/controllers/moderationController.ts @@ -0,0 +1,762 @@ +/** + * Moderation Controller + * Handles HTTP requests for ML-assisted content moderation, + * queue management, decisions, and appeals. + */ + +import { Request, Response } from 'express'; +import { randomUUID } from 'crypto'; +import { ModerationScoringService } from '../services/moderation/ModerationScoringService'; +import { ModerationQueueService } from '../services/moderation/ModerationQueueService'; +import { AppealService } from '../services/moderation/AppealService'; +import { ModerationJob } from '../workers/moderationJob'; +import { + ModerationStatus, + ModerationAction, + AppealStatus, + ContentType, + ModerationFilter, + ModerationBatchRequest, + ModelFeedback, + SeverityLevel, + ModerationItem, +} from '../models/Moderation'; +import logger from '../utils/logger'; + +export class ModerationController { + private scoringService: ModerationScoringService; + private queueService: ModerationQueueService; + private appealService: AppealService; + private job: ModerationJob | null = null; + + constructor( + scoringService: ModerationScoringService, + queueService: ModerationQueueService, + appealService: AppealService + ) { + this.scoringService = scoringService; + this.queueService = queueService; + this.appealService = appealService; + this.appealService.setItemsStore( + (queueService as any).itemsStore + ); + } + + setJob(job: ModerationJob): void { + this.job = job; + } + + /** + * POST /api/moderation/submit + * Submit content for moderation + */ + submitContent = async (req: Request, res: Response): Promise => { + try { + const { + contentId, + contentType, + title, + description, + content, + authorId, + authorName, + authorEmail, + metadata, + } = req.body; + + if (!contentId || !contentType || !title || !content) { + res.status(400).json({ + success: false, + error: 'Missing required fields: contentId, contentType, title, content', + }); + return; + } + + const item: ModerationItem = { + id: randomUUID(), + contentId, + contentType: contentType as ContentType, + title, + description: description || '', + content, + authorId: authorId || 'anonymous', + authorName: authorName || 'Anonymous', + authorEmail: authorEmail || '', + status: ModerationStatus.PENDING, + riskScore: null, + severity: SeverityLevel.LOW, + flags: 0, + reports: [], + assignedModeratorId: null, + moderatorNotes: '', + decision: null, + appeal: null, + metadata: metadata || {}, + createdAt: new Date(), + updatedAt: new Date(), + scoredAt: null, + reviewedAt: null, + resolvedAt: null, + }; + + this.queueService.upsertItem(item); + + // Submit to async scoring if job is available + if (this.job) { + this.job.submitItems([ + { + contentId, + contentType: contentType as ContentType, + title, + description: description || '', + content, + authorId: authorId || 'anonymous', + authorName: authorName || 'Anonymous', + authorEmail: authorEmail || '', + metadata, + }, + ]); + } + + // Score synchronously for immediate feedback + const scored = await this.scoringService.scoreItem(item); + this.queueService.upsertItem(scored); + + res.status(201).json({ + success: true, + data: scored, + message: 'Content submitted for moderation', + }); + } catch (error) { + logger.error('Error submitting content:', error); + res.status(500).json({ + success: false, + error: error instanceof Error ? error.message : 'Unknown error', + }); + } + }; + + /** + * POST /api/moderation/submit/batch + * Batch submit content for moderation + */ + submitBatch = async (req: Request, res: Response): Promise => { + try { + const batchRequest: ModerationBatchRequest = req.body; + + if (!batchRequest.items || !Array.isArray(batchRequest.items)) { + res.status(400).json({ + success: false, + error: 'items array is required', + }); + return; + } + + if (batchRequest.items.length > 100) { + res.status(400).json({ + success: false, + error: 'Maximum 100 items per batch', + }); + return; + } + + const results: ModerationItem[] = []; + const items = batchRequest.items.map((item) => ({ + contentId: item.contentId, + contentType: item.contentType as ContentType, + title: item.title, + description: item.description || '', + content: item.content, + authorId: item.authorId || 'anonymous', + authorName: item.authorName || 'Anonymous', + authorEmail: item.authorEmail || '', + metadata: item.metadata, + })); + + if (this.job) { + const submitted = this.job.submitItems(items); + results.push(...submitted); + } + + res.status(201).json({ + success: true, + data: { + items: results, + summary: { + total: results.length, + pending: results.filter( + (i) => i.status === ModerationStatus.PENDING + ).length, + }, + }, + message: `Submitted ${results.length} items for moderation`, + }); + } catch (error) { + logger.error('Error in batch submission:', error); + res.status(500).json({ + success: false, + error: error instanceof Error ? error.message : 'Unknown error', + }); + } + }; + + /** + * GET /api/moderation/items + * List moderation items with filtering + */ + listItems = async (req: Request, res: Response): Promise => { + try { + const filter: ModerationFilter = { + status: req.query.status as ModerationStatus | undefined, + contentType: req.query.contentType as ContentType | undefined, + severity: req.query.severity as SeverityLevel | undefined, + assignedModeratorId: req.query.assignedModeratorId as string | undefined, + authorId: req.query.authorId as string | undefined, + minRiskScore: req.query.minRiskScore + ? parseFloat(req.query.minRiskScore as string) + : undefined, + maxRiskScore: req.query.maxRiskScore + ? parseFloat(req.query.maxRiskScore as string) + : undefined, + search: req.query.search as string | undefined, + sortBy: (req.query.sortBy as any) || 'createdAt', + sortOrder: (req.query.sortOrder as any) || 'desc', + page: req.query.page ? parseInt(req.query.page as string) : 1, + limit: req.query.limit ? parseInt(req.query.limit as string) : 20, + startDate: req.query.startDate + ? new Date(req.query.startDate as string) + : undefined, + endDate: req.query.endDate + ? new Date(req.query.endDate as string) + : undefined, + }; + + const result = this.queueService.getItems(filter); + + res.json({ + success: true, + data: result.items, + pagination: { + page: result.page, + limit: result.limit, + total: result.total, + pages: Math.ceil(result.total / result.limit), + }, + }); + } catch (error) { + logger.error('Error listing items:', error); + res.status(500).json({ + success: false, + error: error instanceof Error ? error.message : 'Unknown error', + }); + } + }; + + /** + * GET /api/moderation/items/:id + * Get a specific moderation item + */ + getItem = async (req: Request, res: Response): Promise => { + try { + const { id } = req.params; + const item = this.queueService.getItem(id); + + if (!item) { + res.status(404).json({ + success: false, + error: 'Moderation item not found', + }); + return; + } + + res.json({ + success: true, + data: item, + }); + } catch (error) { + logger.error('Error getting item:', error); + res.status(500).json({ + success: false, + error: error instanceof Error ? error.message : 'Unknown error', + }); + } + }; + + /** + * POST /api/moderation/items/:id/score + * Re-score a moderation item + */ + scoreItem = async (req: Request, res: Response): Promise => { + try { + const { id } = req.params; + const item = this.queueService.getItem(id); + + if (!item) { + res.status(404).json({ + success: false, + error: 'Moderation item not found', + }); + return; + } + + const scored = await this.scoringService.scoreItem(item); + this.queueService.upsertItem(scored); + + res.json({ + success: true, + data: scored, + message: 'Item re-scored', + }); + } catch (error) { + logger.error('Error scoring item:', error); + res.status(500).json({ + success: false, + error: error instanceof Error ? error.message : 'Unknown error', + }); + } + }; + + /** + * POST /api/moderation/items/:id/decision + * Record a moderator's decision on an item + */ + makeDecision = async (req: Request, res: Response): Promise => { + try { + const { id } = req.params; + const { + action, + reason, + notes, + predictionCorrect, + actualSeverity, + predictedSeverity, + improvementNotes, + misclassifiedPolicies, + } = req.body; + + if (!action || !reason) { + res.status(400).json({ + success: false, + error: 'action and reason are required', + }); + return; + } + + const userId = (req as any).user?.id || 'unknown'; + const userName = + (req as any).user?.username || (req as any).user?.email || 'Unknown'; + + const modelFeedback: ModelFeedback = { + predictionCorrect: predictionCorrect ?? true, + actualSeverity: (actualSeverity as SeverityLevel) || SeverityLevel.LOW, + predictedSeverity: + (predictedSeverity as SeverityLevel) || SeverityLevel.LOW, + improvementNotes: improvementNotes || '', + misclassifiedPolicies: misclassifiedPolicies || [], + }; + + const item = this.queueService.processDecision( + id, + userId, + userName, + action as ModerationAction, + reason, + notes || '', + modelFeedback + ); + + if (!item) { + res.status(404).json({ + success: false, + error: 'Moderation item not found', + }); + return; + } + + // Record feedback for model training + if (item.riskScore) { + const predictedSev = item.severity; + const actualSev = this.mapActionToSeverity(action as ModerationAction); + this.scoringService.recordFeedback(id, predictedSev, actualSev); + } + + res.json({ + success: true, + data: item, + message: `Decision recorded: ${action}`, + }); + } catch (error) { + logger.error('Error making decision:', error); + res.status(500).json({ + success: false, + error: error instanceof Error ? error.message : 'Unknown error', + }); + } + }; + + /** + * GET /api/moderation/queue + * Get queued items for the current moderator + */ + getQueue = async (req: Request, res: Response): Promise => { + try { + const moderatorId = + (req.query.moderatorId as string) || + (req as any).user?.id || + 'unknown'; + + const queueItems = this.queueService.getModeratorQueue(moderatorId); + const assignedItems = this.queueService.getAssignedItems(moderatorId); + + res.json({ + success: true, + data: { + queued: queueItems, + assigned: assignedItems, + queuedCount: queueItems.length, + assignedCount: assignedItems.length, + }, + }); + } catch (error) { + logger.error('Error getting queue:', error); + res.status(500).json({ + success: false, + error: error instanceof Error ? error.message : 'Unknown error', + }); + } + }; + + /** + * POST /api/moderation/queue/claim + * Claim the next item from the queue for review + */ + claimNext = async (req: Request, res: Response): Promise => { + try { + const moderatorId = + (req as any).user?.id || + req.body.moderatorId || + 'unknown'; + + const item = this.queueService.dequeueItem(moderatorId); + + if (!item) { + res.json({ + success: true, + data: null, + message: 'No items available in queue', + }); + return; + } + + res.json({ + success: true, + data: item, + message: 'Item claimed for review', + }); + } catch (error) { + logger.error('Error claiming item:', error); + res.status(500).json({ + success: false, + error: error instanceof Error ? error.message : 'Unknown error', + }); + } + }; + + /** + * POST /api/moderation/appeals + * Submit an appeal for rejected content + */ + submitAppeal = async (req: Request, res: Response): Promise => { + try { + const { + moderationId, + reason, + explanation, + evidence, + } = req.body; + + if (!moderationId || !reason || !explanation) { + res.status(400).json({ + success: false, + error: 'moderationId, reason, and explanation are required', + }); + return; + } + + const userId = (req as any).user?.id || 'anonymous'; + const userName = + (req as any).user?.username || (req as any).user?.email || 'Anonymous'; + + const appeal = this.appealService.submitAppeal( + moderationId, + userId, + userName, + reason, + explanation, + evidence || [] + ); + + if (!appeal) { + res.status(400).json({ + success: false, + error: + 'Appeal could not be submitted. Ensure the item is rejected and has no existing appeal.', + }); + return; + } + + res.status(201).json({ + success: true, + data: appeal, + message: 'Appeal submitted successfully', + }); + } catch (error) { + logger.error('Error submitting appeal:', error); + res.status(500).json({ + success: false, + error: error instanceof Error ? error.message : 'Unknown error', + }); + } + }; + + /** + * GET /api/moderation/appeals + * List appeals with optional filtering + */ + listAppeals = async (req: Request, res: Response): Promise => { + try { + const result = this.appealService.getAppeals({ + status: req.query.status as AppealStatus | undefined, + submitterId: req.query.submitterId as string | undefined, + page: req.query.page ? parseInt(req.query.page as string) : 1, + limit: req.query.limit ? parseInt(req.query.limit as string) : 20, + }); + + res.json({ + success: true, + data: result.appeals, + pagination: { + page: req.query.page ? parseInt(req.query.page as string) : 1, + limit: req.query.limit ? parseInt(req.query.limit as string) : 20, + total: result.total, + pages: Math.ceil( + result.total / + (req.query.limit ? parseInt(req.query.limit as string) : 20) + ), + }, + }); + } catch (error) { + logger.error('Error listing appeals:', error); + res.status(500).json({ + success: false, + error: error instanceof Error ? error.message : 'Unknown error', + }); + } + }; + + /** + * GET /api/moderation/appeals/:id + * Get a specific appeal + */ + getAppeal = async (req: Request, res: Response): Promise => { + try { + const { id } = req.params; + const appeal = this.appealService.getAppeal(id); + + if (!appeal) { + res.status(404).json({ + success: false, + error: 'Appeal not found', + }); + return; + } + + res.json({ + success: true, + data: appeal, + }); + } catch (error) { + logger.error('Error getting appeal:', error); + res.status(500).json({ + success: false, + error: error instanceof Error ? error.message : 'Unknown error', + }); + } + }; + + /** + * POST /api/moderation/appeals/:id/review + * Review and decide on an appeal + */ + reviewAppeal = async (req: Request, res: Response): Promise => { + try { + const { id } = req.params; + const { decision, reason, notes } = req.body; + + if (!decision || !reason) { + res.status(400).json({ + success: false, + error: 'decision and reason are required', + }); + return; + } + + const reviewerId = (req as any).user?.id || 'unknown'; + const reviewerName = + (req as any).user?.username || (req as any).user?.email || 'Unknown'; + + const result = this.appealService.reviewAppeal( + id, + reviewerId, + reviewerName, + decision as AppealStatus, + reason, + notes || '' + ); + + if (!result) { + res.status(404).json({ + success: false, + error: 'Appeal not found', + }); + return; + } + + res.json({ + success: true, + data: { + appeal: result.appeal, + item: result.item, + }, + message: `Appeal ${decision}`, + }); + } catch (error) { + logger.error('Error reviewing appeal:', error); + res.status(500).json({ + success: false, + error: error instanceof Error ? error.message : 'Unknown error', + }); + } + }; + + /** + * GET /api/moderation/stats + * Get moderation statistics + */ + getStats = async (req: Request, res: Response): Promise => { + try { + const stats = this.queueService.getStats(); + const modelAccuracy = this.scoringService.getModelAccuracy(); + const appealStats = this.appealService.getAppealStats(); + + res.json({ + success: true, + data: { + ...stats, + modelAccuracy: modelAccuracy.accuracy, + modelTotal: modelAccuracy.total, + modelCorrect: modelAccuracy.correct, + appeals: appealStats, + }, + }); + } catch (error) { + logger.error('Error getting stats:', error); + res.status(500).json({ + success: false, + error: error instanceof Error ? error.message : 'Unknown error', + }); + } + }; + + /** + * GET /api/moderation/config + * Get ML model configuration + */ + getConfig = async (_req: Request, res: Response): Promise => { + try { + const config = this.scoringService.getConfig(); + res.json({ + success: true, + data: config, + }); + } catch (error) { + logger.error('Error getting config:', error); + res.status(500).json({ + success: false, + error: error instanceof Error ? error.message : 'Unknown error', + }); + } + }; + + /** + * PUT /api/moderation/config + * Update ML model configuration + */ + updateConfig = async (req: Request, res: Response): Promise => { + try { + const updates = req.body; + const config = this.scoringService.updateConfig(updates); + + res.json({ + success: true, + data: config, + message: 'ML model configuration updated', + }); + } catch (error) { + logger.error('Error updating config:', error); + res.status(500).json({ + success: false, + error: error instanceof Error ? error.message : 'Unknown error', + }); + } + }; + + /** + * GET /api/moderation/health + * Health check for moderation service + */ + healthCheck = async (_req: Request, res: Response): Promise => { + try { + const stats = this.queueService.getStats(); + const modelAccuracy = this.scoringService.getModelAccuracy(); + + res.json({ + success: true, + status: 'healthy', + service: 'moderation', + timestamp: new Date().toISOString(), + pendingJobs: this.job?.getPendingCount() || 0, + modelVersion: this.scoringService.getConfig().modelVersion, + modelAccuracy: modelAccuracy.accuracy, + queueSize: stats.queued, + }); + } catch (error) { + res.status(503).json({ + success: false, + status: 'unhealthy', + error: error instanceof Error ? error.message : 'Unknown error', + }); + } + }; + + /** + * Map a moderation action to a severity level + */ + private mapActionToSeverity(action: ModerationAction): SeverityLevel { + switch (action) { + case ModerationAction.BAN_USER: + return SeverityLevel.CRITICAL; + case ModerationAction.REJECT: + case ModerationAction.REMOVE: + return SeverityLevel.HIGH; + case ModerationAction.ESCALATE: + case ModerationAction.FLAG_FOR_REVIEW: + return SeverityLevel.MEDIUM; + case ModerationAction.WARN_USER: + case ModerationAction.REQUEST_EDIT: + return SeverityLevel.LOW; + default: + return SeverityLevel.LOW; + } + } +} \ No newline at end of file diff --git a/backend/src/models/Moderation.ts b/backend/src/models/Moderation.ts new file mode 100644 index 00000000..e51462da --- /dev/null +++ b/backend/src/models/Moderation.ts @@ -0,0 +1,322 @@ +/** + * Moderation Model + * Defines ML-assisted content moderation structures: + * - Risk scoring for policy violations + * - Human review queue with priority routing + * - Moderator decisions for model feedback + * - Appeal flow for rejected content + */ + +export enum ContentType { + COURSE = 'course', + QUIZ = 'quiz', + USER_POST = 'user_post', + COMMENT = 'comment', + FILE = 'file', + IMAGE = 'image', + VIDEO = 'video', +} + +export enum ModerationStatus { + PENDING = 'pending', + SCORING = 'scoring', + QUEUED = 'queued', + IN_REVIEW = 'in_review', + APPROVED = 'approved', + REJECTED = 'rejected', + FLAGGED = 'flagged', + AUTO_APPROVED = 'auto_approved', + AUTO_REJECTED = 'auto_rejected', +} + +export enum SeverityLevel { + LOW = 'low', + MEDIUM = 'medium', + HIGH = 'high', + CRITICAL = 'critical', +} + +export enum PolicyViolationType { + HATE_SPEECH = 'hate_speech', + HARASSMENT = 'harassment', + SPAM = 'spam', + NSFW = 'nsfw', + VIOLENCE = 'violence', + COPYRIGHT = 'copyright', + MISINFORMATION = 'misinformation', + PLAGIARISM = 'plagiarism', + PERSONAL_INFO = 'personal_info', + SELF_HARM = 'self_harm', + ILLEGAL_CONTENT = 'illegal_content', + COMMUNITY_GUIDELINES = 'community_guidelines', + CHEATING = 'cheating', + OTHER = 'other', +} + +export interface RiskScoreBreakdown { + /** Overall risk score (0-100) */ + overall: number; + /** Per-policy violation scores */ + policyScores: PolicyScore[]; + /** Text-based risk factors */ + textRisk: number; + /** Metadata-based risk factors */ + metadataRisk: number; + /** User history risk factor */ + userHistoryRisk: number; + /** Content similarity to known violations */ + similarityRisk: number; + /** Confidence of the ML model (0-1) */ + confidence: number; + /** Model version used for scoring */ + modelVersion: string; +} + +export interface PolicyScore { + policyType: PolicyViolationType; + score: number; // 0-100 + confidence: number; // 0-1 + keywords: string[]; + matchedPatterns: string[]; +} + +export interface ModerationItem { + id: string; + contentId: string; + contentType: ContentType; + title: string; + description: string; + content: string; + authorId: string; + authorName: string; + authorEmail: string; + status: ModerationStatus; + riskScore: RiskScoreBreakdown | null; + severity: SeverityLevel; + flags: number; + reports: ModerationReport[]; + assignedModeratorId: string | null; + moderatorNotes: string; + decision: ModerationDecision | null; + appeal: Appeal | null; + metadata: ModerationMetadata; + createdAt: Date; + updatedAt: Date; + scoredAt: Date | null; + reviewedAt: Date | null; + resolvedAt: Date | null; +} + +export interface ModerationReport { + id: string; + reason: PolicyViolationType; + description: string; + reporterId: string; + reporterName: string; + createdAt: Date; + status: 'pending' | 'acknowledged' | 'resolved'; +} + +export interface ModerationDecision { + id: string; + moderatorId: string; + moderatorName: string; + action: ModerationAction; + reason: string; + notes: string; + createdAt: Date; + /** Feedback for model retraining */ + modelFeedback: ModelFeedback; +} + +export enum ModerationAction { + APPROVE = 'approve', + REJECT = 'reject', + FLAG_FOR_REVIEW = 'flag_for_review', + ESCALATE = 'escalate', + REMOVE = 'remove', + WARN_USER = 'warn_user', + BAN_USER = 'ban_user', + REQUEST_EDIT = 'request_edit', +} + +export interface ModelFeedback { + /** Was the ML prediction correct? */ + predictionCorrect: boolean; + /** The actual decision vs predicted */ + actualSeverity: SeverityLevel; + predictedSeverity: SeverityLevel; + /** Notes for improving the model */ + improvementNotes: string; + /** Specific policy areas where model was wrong */ + misclassifiedPolicies: PolicyViolationType[]; +} + +export interface Appeal { + id: string; + moderationId: string; + submitterId: string; + submitterName: string; + reason: string; + explanation: string; + evidence: AppealEvidence[]; + status: AppealStatus; + decision: AppealDecision | null; + createdAt: Date; + updatedAt: Date; + resolvedAt: Date | null; +} + +export enum AppealStatus { + PENDING = 'pending', + UNDER_REVIEW = 'under_review', + APPROVED = 'approved', + DENIED = 'denied', + REVERSED = 'reversed', + UPHELD = 'upheld', +} + +export interface AppealDecision { + id: string; + reviewerId: string; + reviewerName: string; + decision: AppealStatus; + reason: string; + notes: string; + createdAt: Date; +} + +export interface AppealEvidence { + id: string; + type: 'text' | 'file' | 'url' | 'reference'; + description: string; + value: string; + mimeType?: string; + uploadedAt: Date; +} + +export interface ModerationMetadata { + category?: string; + difficulty?: string; + duration?: string; + fileSize?: number; + fileType?: string; + language?: string; + courseId?: string; + originalCreatedAt?: Date; + tags?: string[]; + customFields?: Record; +} + +export interface ModerationFilter { + status?: ModerationStatus; + contentType?: ContentType; + severity?: SeverityLevel; + assignedModeratorId?: string; + authorId?: string; + startDate?: Date; + endDate?: Date; + minRiskScore?: number; + maxRiskScore?: number; + search?: string; + sortBy?: 'createdAt' | 'riskScore' | 'severity' | 'updatedAt' | 'flags'; + sortOrder?: 'asc' | 'desc'; + page?: number; + limit?: number; +} + +export interface ModerationQueue { + id: string; + name: string; + description: string; + items: string[]; // moderation item IDs ordered by priority + filter: ModerationFilter; + moderators: string[]; // moderator user IDs + maxItemsPerModerator: number; + autoAssign: boolean; + createdAt: Date; + updatedAt: Date; +} + +export interface ModerationStats { + total: number; + pending: number; + queued: number; + inReview: number; + approved: number; + rejected: number; + flagged: number; + autoApproved: number; + autoRejected: number; + averageRiskScore: number; + averageReviewTime: number; // in minutes + modelAccuracy: number; // percentage of correct ML predictions + appealsPending: number; +} + +export interface ModerationBatchRequest { + items: Array<{ + contentId: string; + contentType: ContentType; + title: string; + description: string; + content: string; + authorId: string; + authorName: string; + authorEmail: string; + metadata?: ModerationMetadata; + }>; + options?: { + skipScoring?: boolean; + autoApproveBelow?: number; + autoRejectAbove?: number; + priority?: 'normal' | 'high'; + }; +} + +// ML Model configuration +export interface MLModelConfig { + /** Minimum risk score for auto-approval (default: 10) */ + autoApproveThreshold: number; + /** Minimum risk score for auto-rejection (default: 90) */ + autoRejectThreshold: number; + /** Minimum risk score for human review queue (default: 40) */ + queueThreshold: number; + /** Minimum confidence for auto-decisions */ + minConfidence: number; + /** Whether to enable model feedback loop */ + enableModelFeedback: boolean; + /** Model version identifier */ + modelVersion: string; + /** Active policy violation types */ + activePolicies: PolicyViolationType[]; + /** Policy-specific thresholds */ + policyThresholds: Record; +} + +// Default ML model configuration +export const DEFAULT_ML_CONFIG: MLModelConfig = { + autoApproveThreshold: 15, + autoRejectThreshold: 85, + queueThreshold: 35, + minConfidence: 0.7, + enableModelFeedback: true, + modelVersion: '1.0.0', + activePolicies: Object.values(PolicyViolationType), + policyThresholds: { + [PolicyViolationType.HATE_SPEECH]: 50, + [PolicyViolationType.HARASSMENT]: 50, + [PolicyViolationType.SPAM]: 40, + [PolicyViolationType.NSFW]: 60, + [PolicyViolationType.VIOLENCE]: 60, + [PolicyViolationType.COPYRIGHT]: 50, + [PolicyViolationType.MISINFORMATION]: 40, + [PolicyViolationType.PLAGIARISM]: 50, + [PolicyViolationType.PERSONAL_INFO]: 70, + [PolicyViolationType.SELF_HARM]: 70, + [PolicyViolationType.ILLEGAL_CONTENT]: 80, + [PolicyViolationType.COMMUNITY_GUIDELINES]: 40, + [PolicyViolationType.CHEATING]: 50, + [PolicyViolationType.OTHER]: 50, + }, +}; \ No newline at end of file diff --git a/backend/src/routes/moderation.ts b/backend/src/routes/moderation.ts new file mode 100644 index 00000000..d3e837f7 --- /dev/null +++ b/backend/src/routes/moderation.ts @@ -0,0 +1,411 @@ +/** + * Moderation Routes + * API endpoints for ML-assisted content moderation: + * - Content submission and scoring + * - Queue management for human review + * - Moderator decision recording + * - Appeal flow for rejected content + */ + +import { Router, RequestHandler } from 'express'; +import { body, param, query } from 'express-validator'; +import { ModerationController } from '../controllers/moderationController'; +import { ModerationScoringService } from '../services/moderation/ModerationScoringService'; +import { ModerationQueueService } from '../services/moderation/ModerationQueueService'; +import { AppealService } from '../services/moderation/AppealService'; +import { authenticateToken, requireAdmin } from '../middleware/auth'; +import { handleValidationErrors } from '../middleware/validation'; + +const router: Router = Router(); + +// Initialize services +const scoringService = new ModerationScoringService(); +const queueService = new ModerationQueueService(); +const appealService = new AppealService(); +const moderationController = new ModerationController( + scoringService, + queueService, + appealService +); + +// Validation schemas +const submitContentValidation = [ + body('contentId') + .notEmpty() + .withMessage('Content ID is required'), + body('contentType') + .isIn(['course', 'quiz', 'user_post', 'comment', 'file', 'image', 'video']) + .withMessage('Invalid content type'), + body('title') + .notEmpty() + .isLength({ min: 1, max: 500 }) + .withMessage('Title is required and must be 1-500 characters'), + body('description') + .optional() + .isLength({ max: 5000 }) + .withMessage('Description must be at most 5000 characters'), + body('content') + .notEmpty() + .withMessage('Content is required'), + body('authorId') + .optional() + .isString() + .withMessage('Author ID must be a string'), + body('authorName') + .optional() + .isString() + .withMessage('Author name must be a string'), + body('authorEmail') + .optional() + .isEmail() + .withMessage('Author email must be valid'), +]; + +const submitBatchValidation = [ + body('items') + .isArray({ min: 1, max: 100 }) + .withMessage('Items must be an array with 1-100 items'), + body('items.*.contentId') + .notEmpty() + .withMessage('Content ID is required for each item'), + body('items.*.contentType') + .isIn(['course', 'quiz', 'user_post', 'comment', 'file', 'image', 'video']) + .withMessage('Invalid content type'), + body('items.*.title') + .notEmpty() + .withMessage('Title is required for each item'), + body('items.*.content') + .notEmpty() + .withMessage('Content is required for each item'), +]; + +const makeDecisionValidation = [ + body('action') + .isIn([ + 'approve', + 'reject', + 'flag_for_review', + 'escalate', + 'remove', + 'warn_user', + 'ban_user', + 'request_edit', + ]) + .withMessage('Invalid moderation action'), + body('reason') + .notEmpty() + .isLength({ min: 5, max: 2000 }) + .withMessage('Reason is required and must be 5-2000 characters'), + body('notes') + .optional() + .isLength({ max: 5000 }) + .withMessage('Notes must be at most 5000 characters'), + body('predictionCorrect') + .optional() + .isBoolean() + .withMessage('predictionCorrect must be a boolean'), + body('actualSeverity') + .optional() + .isIn(['low', 'medium', 'high', 'critical']) + .withMessage('Invalid severity level'), + body('predictedSeverity') + .optional() + .isIn(['low', 'medium', 'high', 'critical']) + .withMessage('Invalid severity level'), +]; + +const submitAppealValidation = [ + body('moderationId') + .notEmpty() + .withMessage('Moderation ID is required'), + body('reason') + .notEmpty() + .isLength({ min: 10, max: 1000 }) + .withMessage('Appeal reason is required and must be 10-1000 characters'), + body('explanation') + .notEmpty() + .isLength({ min: 20, max: 5000 }) + .withMessage('Explanation is required and must be 20-5000 characters'), + body('evidence') + .optional() + .isArray() + .withMessage('Evidence must be an array'), +]; + +const reviewAppealValidation = [ + body('decision') + .isIn(['approved', 'denied', 'reversed', 'upheld']) + .withMessage('Invalid appeal decision'), + body('reason') + .notEmpty() + .isLength({ min: 5, max: 2000 }) + .withMessage('Reason is required and must be 5-2000 characters'), + body('notes') + .optional() + .isLength({ max: 5000 }) + .withMessage('Notes must be at most 5000 characters'), +]; + +const updateConfigValidation = [ + body('autoApproveThreshold') + .optional() + .isFloat({ min: 0, max: 100 }) + .withMessage('autoApproveThreshold must be between 0 and 100'), + body('autoRejectThreshold') + .optional() + .isFloat({ min: 0, max: 100 }) + .withMessage('autoRejectThreshold must be between 0 and 100'), + body('queueThreshold') + .optional() + .isFloat({ min: 0, max: 100 }) + .withMessage('queueThreshold must be between 0 and 100'), + body('minConfidence') + .optional() + .isFloat({ min: 0, max: 1 }) + .withMessage('minConfidence must be between 0 and 1'), + body('enableModelFeedback') + .optional() + .isBoolean() + .withMessage('enableModelFeedback must be a boolean'), +]; + +/** + * @route POST /api/moderation/submit + * @desc Submit content for ML-assisted moderation + * @access Private + */ +router.post( + '/submit', + authenticateToken, + submitContentValidation, + handleValidationErrors, + moderationController.submitContent.bind(moderationController) as RequestHandler +); + +/** + * @route POST /api/moderation/submit/batch + * @desc Batch submit content for moderation + * @access Private (Admin/Moderator) + */ +router.post( + '/submit/batch', + authenticateToken, + requireAdmin, + submitBatchValidation, + handleValidationErrors, + moderationController.submitBatch.bind(moderationController) as RequestHandler +); + +/** + * @route GET /api/moderation/items + * @desc List moderation items with filtering + * @access Private (Admin/Moderator) + */ +router.get( + '/items', + authenticateToken, + query('status') + .optional() + .isIn([ + 'pending', 'scoring', 'queued', 'in_review', + 'approved', 'rejected', 'flagged', + 'auto_approved', 'auto_rejected', + ]) + .withMessage('Invalid status'), + query('contentType') + .optional() + .isIn(['course', 'quiz', 'user_post', 'comment', 'file', 'image', 'video']) + .withMessage('Invalid content type'), + query('severity') + .optional() + .isIn(['low', 'medium', 'high', 'critical']) + .withMessage('Invalid severity'), + query('page') + .optional() + .isInt({ min: 1 }) + .withMessage('Page must be a positive integer'), + query('limit') + .optional() + .isInt({ min: 1, max: 100 }) + .withMessage('Limit must be between 1 and 100'), + handleValidationErrors, + moderationController.listItems.bind(moderationController) as RequestHandler +); + +/** + * @route GET /api/moderation/items/:id + * @desc Get a specific moderation item + * @access Private (Admin/Moderator) + */ +router.get( + '/items/:id', + authenticateToken, + param('id').isString().notEmpty().withMessage('ID is required'), + handleValidationErrors, + moderationController.getItem.bind(moderationController) as RequestHandler +); + +/** + * @route POST /api/moderation/items/:id/score + * @desc Re-score a moderation item + * @access Private (Admin/Moderator) + */ +router.post( + '/items/:id/score', + authenticateToken, + requireAdmin, + param('id').isString().notEmpty().withMessage('ID is required'), + handleValidationErrors, + moderationController.scoreItem.bind(moderationController) as RequestHandler +); + +/** + * @route POST /api/moderation/items/:id/decision + * @desc Record a moderator's decision on an item + * @access Private (Admin/Moderator) + */ +router.post( + '/items/:id/decision', + authenticateToken, + param('id').isString().notEmpty().withMessage('ID is required'), + makeDecisionValidation, + handleValidationErrors, + moderationController.makeDecision.bind(moderationController) as RequestHandler +); + +/** + * @route GET /api/moderation/queue + * @desc Get queued items for the current moderator + * @access Private (Admin/Moderator) + */ +router.get( + '/queue', + authenticateToken, + moderationController.getQueue.bind(moderationController) as RequestHandler +); + +/** + * @route POST /api/moderation/queue/claim + * @desc Claim the next item from the queue for review + * @access Private (Admin/Moderator) + */ +router.post( + '/queue/claim', + authenticateToken, + moderationController.claimNext.bind(moderationController) as RequestHandler +); + +/** + * @route POST /api/moderation/appeals + * @desc Submit an appeal for rejected content + * @access Private + */ +router.post( + '/appeals', + authenticateToken, + submitAppealValidation, + handleValidationErrors, + moderationController.submitAppeal.bind(moderationController) as RequestHandler +); + +/** + * @route GET /api/moderation/appeals + * @desc List appeals with optional filtering + * @access Private (Admin/Moderator) + */ +router.get( + '/appeals', + authenticateToken, + query('status') + .optional() + .isIn([ + 'pending', 'under_review', 'approved', + 'denied', 'reversed', 'upheld', + ]) + .withMessage('Invalid appeal status'), + query('page') + .optional() + .isInt({ min: 1 }) + .withMessage('Page must be a positive integer'), + query('limit') + .optional() + .isInt({ min: 1, max: 100 }) + .withMessage('Limit must be between 1 and 100'), + handleValidationErrors, + moderationController.listAppeals.bind(moderationController) as RequestHandler +); + +/** + * @route GET /api/moderation/appeals/:id + * @desc Get a specific appeal + * @access Private (Admin/Moderator) + */ +router.get( + '/appeals/:id', + authenticateToken, + param('id').isString().notEmpty().withMessage('Appeal ID is required'), + handleValidationErrors, + moderationController.getAppeal.bind(moderationController) as RequestHandler +); + +/** + * @route POST /api/moderation/appeals/:id/review + * @desc Review and decide on an appeal + * @access Private (Admin only) + */ +router.post( + '/appeals/:id/review', + authenticateToken, + requireAdmin, + param('id').isString().notEmpty().withMessage('Appeal ID is required'), + reviewAppealValidation, + handleValidationErrors, + moderationController.reviewAppeal.bind(moderationController) as RequestHandler +); + +/** + * @route GET /api/moderation/stats + * @desc Get moderation statistics + * @access Private (Admin/Moderator) + */ +router.get( + '/stats', + authenticateToken, + moderationController.getStats.bind(moderationController) as RequestHandler +); + +/** + * @route GET /api/moderation/config + * @desc Get ML model configuration + * @access Private (Admin) + */ +router.get( + '/config', + authenticateToken, + requireAdmin, + moderationController.getConfig.bind(moderationController) as RequestHandler +); + +/** + * @route PUT /api/moderation/config + * @desc Update ML model configuration + * @access Private (Admin only) + */ +router.put( + '/config', + authenticateToken, + requireAdmin, + updateConfigValidation, + handleValidationErrors, + moderationController.updateConfig.bind(moderationController) as RequestHandler +); + +/** + * @route GET /api/moderation/health + * @desc Health check for moderation service + * @access Public + */ +router.get('/health', moderationController.healthCheck.bind(moderationController) as RequestHandler); + +export default router; +export { scoringService, queueService, appealService, moderationController }; \ No newline at end of file diff --git a/backend/src/services/moderation/AppealService.ts b/backend/src/services/moderation/AppealService.ts new file mode 100644 index 00000000..9bdec254 --- /dev/null +++ b/backend/src/services/moderation/AppealService.ts @@ -0,0 +1,265 @@ +/** + * Appeal Service + * Handles the appeal flow for rejected content moderation decisions. + * Submitters can appeal rejected content with evidence, and appeals + * are reviewed by senior moderators or admins. + */ + +import { randomUUID } from 'crypto'; +import { + Appeal, + AppealStatus, + AppealDecision, + AppealEvidence, + ModerationItem, + ModerationStatus, +} from '../../models/Moderation'; +import logger from '../../utils/logger'; + +export class AppealService { + private appeals: Map = new Map(); + private itemsStore: Map = new Map(); + + setItemsStore(store: Map): void { + this.itemsStore = store; + } + + /** + * Submit an appeal for rejected content + */ + submitAppeal( + moderationId: string, + submitterId: string, + submitterName: string, + reason: string, + explanation: string, + evidence: Omit[] + ): Appeal | null { + const item = this.itemsStore.get(moderationId); + if (!item) return null; + + // Only rejected items can be appealed + if ( + item.status !== ModerationStatus.REJECTED && + item.status !== ModerationStatus.AUTO_REJECTED + ) { + logger.warn( + `Appeal denied: item ${moderationId} is not rejected (status: ${item.status})` + ); + return null; + } + + // Check if appeal already exists + if (item.appeal) { + logger.warn(`Appeal already exists for item ${moderationId}`); + return item.appeal; + } + + const appeal: Appeal = { + id: randomUUID(), + moderationId, + submitterId, + submitterName, + reason, + explanation, + evidence: evidence.map((e) => ({ + ...e, + id: randomUUID(), + uploadedAt: new Date(), + })), + status: AppealStatus.PENDING, + decision: null, + createdAt: new Date(), + updatedAt: new Date(), + resolvedAt: null, + }; + + this.appeals.set(appeal.id, appeal); + item.appeal = appeal; + + logger.info( + `Appeal submitted for item ${moderationId} by ${submitterName}` + ); + return appeal; + } + + /** + * Review an appeal and make a decision + */ + reviewAppeal( + appealId: string, + reviewerId: string, + reviewerName: string, + decision: AppealStatus, + reason: string, + notes: string + ): { appeal: Appeal; item: ModerationItem } | null { + const appeal = this.appeals.get(appealId); + if (!appeal) return null; + + const item = this.itemsStore.get(appeal.moderationId); + if (!item) return null; + + const appealDecision: AppealDecision = { + id: randomUUID(), + reviewerId, + reviewerName, + decision, + reason, + notes, + createdAt: new Date(), + }; + + appeal.status = decision; + appeal.decision = appealDecision; + appeal.updatedAt = new Date(); + + // Update the moderation item based on appeal decision + if (decision === AppealStatus.APPROVED || decision === AppealStatus.REVERSED) { + item.status = ModerationStatus.APPROVED; + appeal.resolvedAt = new Date(); + } else if (decision === AppealStatus.DENIED || decision === AppealStatus.UPHELD) { + appeal.resolvedAt = new Date(); + // Item remains rejected + } + + this.appeals.set(appealId, appeal); + this.itemsStore.set(item.id, item); + + logger.info( + `Appeal ${appealId} ${decision} by ${reviewerName}: item ${item.id} status updated` + ); + + return { appeal, item }; + } + + /** + * Get appeal by ID + */ + getAppeal(appealId: string): Appeal | null { + return this.appeals.get(appealId) || null; + } + + /** + * Get appeal for a moderation item + */ + getAppealForModeration(moderationId: string): Appeal | null { + for (const appeal of this.appeals.values()) { + if (appeal.moderationId === moderationId) return appeal; + } + return null; + } + + /** + * Get all appeals with optional filtering + */ + getAppeals( + filter?: { + status?: AppealStatus; + submitterId?: string; + page?: number; + limit?: number; + } + ): { appeals: Appeal[]; total: number } { + let appeals = Array.from(this.appeals.values()); + + if (filter) { + if (filter.status) { + appeals = appeals.filter((a) => a.status === filter.status); + } + if (filter.submitterId) { + appeals = appeals.filter((a) => a.submitterId === filter.submitterId); + } + } + + // Sort by most recent first + appeals.sort( + (a, b) => b.createdAt.getTime() - a.createdAt.getTime() + ); + + const total = appeals.length; + const page = filter?.page || 1; + const limit = filter?.limit || 20; + + return { + appeals: appeals.slice((page - 1) * limit, page * limit), + total, + }; + } + + /** + * Add evidence to an existing appeal + */ + addEvidence( + appealId: string, + evidence: Omit + ): Appeal | null { + const appeal = this.appeals.get(appealId); + if (!appeal) return null; + + // Can only add evidence to pending appeals + if (appeal.status !== AppealStatus.PENDING) { + logger.warn( + `Cannot add evidence to appeal ${appealId} with status ${appeal.status}` + ); + return null; + } + + const newEvidence: AppealEvidence = { + ...evidence, + id: randomUUID(), + uploadedAt: new Date(), + }; + + appeal.evidence.push(newEvidence); + appeal.updatedAt = new Date(); + + logger.info(`Evidence added to appeal ${appealId}`); + return appeal; + } + + /** + * Get appeal statistics + */ + getAppealStats(): { + total: number; + pending: number; + underReview: number; + approved: number; + denied: number; + reversed: number; + upheld: number; + averageResolutionTime: number; // in hours + } { + const appeals = Array.from(this.appeals.values()); + const resolved = appeals.filter((a) => a.resolvedAt); + + const resolutionTimes = resolved + .filter((a) => a.createdAt && a.resolvedAt) + .map( + (a) => + (new Date(a.resolvedAt!).getTime() - + new Date(a.createdAt).getTime()) / + (1000 * 60 * 60) + ); + + return { + total: appeals.length, + pending: appeals.filter((a) => a.status === AppealStatus.PENDING).length, + underReview: appeals.filter( + (a) => a.status === AppealStatus.UNDER_REVIEW + ).length, + approved: appeals.filter( + (a) => a.status === AppealStatus.APPROVED + ).length, + denied: appeals.filter((a) => a.status === AppealStatus.DENIED).length, + reversed: appeals.filter( + (a) => a.status === AppealStatus.REVERSED + ).length, + upheld: appeals.filter((a) => a.status === AppealStatus.UPHELD).length, + averageResolutionTime: resolutionTimes.length + ? resolutionTimes.reduce((s, t) => s + t, 0) / resolutionTimes.length + : 0, + }; + } +} \ No newline at end of file diff --git a/backend/src/services/moderation/ModerationQueueService.ts b/backend/src/services/moderation/ModerationQueueService.ts new file mode 100644 index 00000000..2d8917a9 --- /dev/null +++ b/backend/src/services/moderation/ModerationQueueService.ts @@ -0,0 +1,443 @@ +/** + * Moderation Queue Service + * Manages the human review queue for content moderation. + * Handles priority-based routing, moderator assignment, + * and auto-assignment of items to available moderators. + */ + +import { randomUUID } from 'crypto'; +import { + ModerationItem, + ModerationStatus, + ModerationFilter, + ModerationQueue, + ModerationStats, + SeverityLevel, + ModerationDecision, + ModerationAction, + ModelFeedback, +} from '../../models/Moderation'; +import logger from '../../utils/logger'; + +export class ModerationQueueService { + private queues: Map = new Map(); + private itemsStore: Map = new Map(); + private moderatorLoads: Map = new Map(); + + constructor() { + // Initialize default queue + const defaultQueue: ModerationQueue = { + id: 'default', + name: 'General Moderation Queue', + description: 'Default queue for all moderation items', + items: [], + filter: { status: ModerationStatus.QUEUED }, + moderators: [], + maxItemsPerModerator: 20, + autoAssign: true, + createdAt: new Date(), + updatedAt: new Date(), + }; + this.queues.set('default', defaultQueue); + } + + /** + * Add an item to the moderation queue + */ + enqueueItem(item: ModerationItem, priority: 'normal' | 'high' = 'normal'): void { + this.itemsStore.set(item.id, item); + + for (const queue of this.queues.values()) { + if (this.matchesFilter(item, queue.filter)) { + if (priority === 'high') { + queue.items.unshift(item.id); // High priority at front + } else { + queue.items.push(item.id); // Normal priority at back + } + queue.updatedAt = new Date(); + logger.info(`Enqueued item ${item.id} to queue "${queue.name}" with ${priority} priority`); + } + } + } + + /** + * Dequeue the next item for a moderator to review + */ + dequeueItem(moderatorId: string): ModerationItem | null { + // Find a queue this moderator belongs to + for (const queue of this.queues.values()) { + if (queue.moderators.includes(moderatorId) && queue.items.length > 0) { + // Check moderator load + const currentLoad = this.moderatorLoads.get(moderatorId) || 0; + if (currentLoad >= queue.maxItemsPerModerator) { + continue; + } + + const itemId = queue.items.shift(); + if (itemId) { + const item = this.itemsStore.get(itemId); + if (item && item.status === ModerationStatus.QUEUED) { + item.status = ModerationStatus.IN_REVIEW; + item.assignedModeratorId = moderatorId; + item.reviewedAt = new Date(); + this.moderatorLoads.set(moderatorId, currentLoad + 1); + + queue.updatedAt = new Date(); + logger.info( + `Dequeued item ${item.id} to moderator ${moderatorId}` + ); + return item; + } + } + } + } + return null; + } + + /** + * Get all items in all queues for a moderator + */ + getModeratorQueue(moderatorId: string): ModerationItem[] { + const items: ModerationItem[] = []; + + for (const queue of this.queues.values()) { + if (queue.moderators.includes(moderatorId) || queue.moderators.length === 0) { + for (const itemId of queue.items) { + const item = this.itemsStore.get(itemId); + if (item) items.push(item); + } + } + } + + return items; + } + + /** + * Get items assigned to a specific moderator + */ + getAssignedItems(moderatorId: string): ModerationItem[] { + const items: ModerationItem[] = []; + for (const item of this.itemsStore.values()) { + if ( + item.assignedModeratorId === moderatorId && + item.status === ModerationStatus.IN_REVIEW + ) { + items.push(item); + } + } + return items; + } + + /** + * Process a moderator's decision on an item + */ + processDecision( + itemId: string, + moderatorId: string, + moderatorName: string, + action: ModerationAction, + reason: string, + notes: string, + modelFeedback: ModelFeedback + ): ModerationItem | null { + const item = this.itemsStore.get(itemId); + if (!item) return null; + + const decision: ModerationDecision = { + id: randomUUID(), + moderatorId, + moderatorName, + action, + reason, + notes, + createdAt: new Date(), + modelFeedback, + }; + + item.decision = decision; + item.moderatorNotes = notes; + item.resolvedAt = new Date(); + + // Map action to status + switch (action) { + case ModerationAction.APPROVE: + item.status = ModerationStatus.APPROVED; + break; + case ModerationAction.REJECT: + case ModerationAction.REMOVE: + item.status = ModerationStatus.REJECTED; + break; + case ModerationAction.FLAG_FOR_REVIEW: + item.status = ModerationStatus.FLAGGED; + break; + case ModerationAction.ESCALATE: + item.status = ModerationStatus.FLAGGED; + break; + default: + item.status = ModerationStatus.APPROVED; + } + + // Decrease moderator load + const currentLoad = this.moderatorLoads.get(moderatorId) || 0; + this.moderatorLoads.set(moderatorId, Math.max(0, currentLoad - 1)); + + logger.info( + `Decision processed for item ${itemId}: ${action} by ${moderatorName}` + ); + + return item; + } + + /** + * Create or update a moderation queue + */ + upsertQueue(queue: ModerationQueue): ModerationQueue { + this.queues.set(queue.id, queue); + return queue; + } + + /** + * Get a specific queue + */ + getQueue(queueId: string): ModerationQueue | null { + return this.queues.get(queueId) || null; + } + + /** + * Get all queues + */ + getAllQueues(): ModerationQueue[] { + return Array.from(this.queues.values()); + } + + /** + * Delete a queue + */ + deleteQueue(queueId: string): boolean { + return this.queues.delete(queueId); + } + + /** + * Add a moderator to a queue + */ + addModeratorToQueue(queueId: string, moderatorId: string): boolean { + const queue = this.queues.get(queueId); + if (!queue) return false; + + if (!queue.moderators.includes(moderatorId)) { + queue.moderators.push(moderatorId); + queue.updatedAt = new Date(); + } + return true; + } + + /** + * Remove a moderator from a queue + */ + removeModeratorFromQueue(queueId: string, moderatorId: string): boolean { + const queue = this.queues.get(queueId); + if (!queue) return false; + + queue.moderators = queue.moderators.filter((m) => m !== moderatorId); + queue.updatedAt = new Date(); + return true; + } + + /** + * Get moderation statistics + */ + getStats(): ModerationStats { + const items = Array.from(this.itemsStore.values()); + const reviewedItems = items.filter( + (i) => + i.status === ModerationStatus.APPROVED || + i.status === ModerationStatus.REJECTED || + i.status === ModerationStatus.AUTO_APPROVED || + i.status === ModerationStatus.AUTO_REJECTED + ); + + const reviewTimes = items + .filter((i) => i.reviewedAt && i.scoredAt) + .map( + (i) => + (new Date(i.reviewedAt!).getTime() - + new Date(i.scoredAt!).getTime()) / + (1000 * 60) + ); + + const withRiskScores = items.filter((i) => i.riskScore); + + return { + total: items.length, + pending: items.filter((i) => i.status === ModerationStatus.PENDING).length, + queued: items.filter((i) => i.status === ModerationStatus.QUEUED).length, + inReview: items.filter( + (i) => i.status === ModerationStatus.IN_REVIEW + ).length, + approved: items.filter( + (i) => i.status === ModerationStatus.APPROVED + ).length, + rejected: items.filter( + (i) => i.status === ModerationStatus.REJECTED + ).length, + flagged: items.filter( + (i) => i.status === ModerationStatus.FLAGGED + ).length, + autoApproved: items.filter( + (i) => i.status === ModerationStatus.AUTO_APPROVED + ).length, + autoRejected: items.filter( + (i) => i.status === ModerationStatus.AUTO_REJECTED + ).length, + averageRiskScore: withRiskScores.length + ? withRiskScores.reduce((s, i) => s + (i.riskScore?.overall || 0), 0) / + withRiskScores.length + : 0, + averageReviewTime: reviewTimes.length + ? reviewTimes.reduce((s, t) => s + t, 0) / reviewTimes.length + : 0, + modelAccuracy: 0, // Updated externally by scoring service + appealsPending: items.filter( + (i) => i.appeal?.status === 'pending' || i.appeal?.status === 'under_review' + ).length, + }; + } + + /** + * Get items with optional filtering + */ + getItems(filter?: ModerationFilter): { + items: ModerationItem[]; + total: number; + page: number; + limit: number; + } { + let items = Array.from(this.itemsStore.values()); + + if (filter) { + if (filter.status) { + items = items.filter((i) => i.status === filter.status); + } + if (filter.contentType) { + items = items.filter((i) => i.contentType === filter.contentType); + } + if (filter.severity) { + items = items.filter((i) => i.severity === filter.severity); + } + if (filter.assignedModeratorId) { + items = items.filter( + (i) => i.assignedModeratorId === filter.assignedModeratorId + ); + } + if (filter.authorId) { + items = items.filter((i) => i.authorId === filter.authorId); + } + if (filter.minRiskScore !== undefined) { + items = items.filter( + (i) => (i.riskScore?.overall || 0) >= filter.minRiskScore! + ); + } + if (filter.maxRiskScore !== undefined) { + items = items.filter( + (i) => (i.riskScore?.overall || 0) <= filter.maxRiskScore! + ); + } + if (filter.search) { + const search = filter.search.toLowerCase(); + items = items.filter( + (i) => + i.title.toLowerCase().includes(search) || + i.description.toLowerCase().includes(search) || + i.content.toLowerCase().includes(search) + ); + } + if (filter.startDate) { + items = items.filter((i) => i.createdAt >= filter.startDate!); + } + if (filter.endDate) { + items = items.filter((i) => i.createdAt <= filter.endDate!); + } + } + + // Sort + const sortBy = filter?.sortBy || 'createdAt'; + const sortOrder = filter?.sortOrder || 'desc'; + items.sort((a, b) => { + let aVal: number, bVal: number; + switch (sortBy) { + case 'riskScore': + aVal = a.riskScore?.overall || 0; + bVal = b.riskScore?.overall || 0; + break; + case 'severity': + const severityOrder = { + [SeverityLevel.CRITICAL]: 4, + [SeverityLevel.HIGH]: 3, + [SeverityLevel.MEDIUM]: 2, + [SeverityLevel.LOW]: 1, + }; + aVal = severityOrder[a.severity]; + bVal = severityOrder[b.severity]; + break; + case 'flags': + aVal = a.flags; + bVal = b.flags; + break; + case 'updatedAt': + aVal = a.updatedAt.getTime(); + bVal = b.updatedAt.getTime(); + break; + default: + aVal = a.createdAt.getTime(); + bVal = b.createdAt.getTime(); + } + return sortOrder === 'asc' ? aVal - bVal : bVal - aVal; + }); + + const total = items.length; + const page = filter?.page || 1; + const limit = filter?.limit || 20; + const start = (page - 1) * limit; + + return { + items: items.slice(start, start + limit), + total, + page, + limit, + }; + } + + /** + * Get a single item by ID + */ + getItem(itemId: string): ModerationItem | null { + return this.itemsStore.get(itemId) || null; + } + + /** + * Store an item + */ + upsertItem(item: ModerationItem): void { + this.itemsStore.set(item.id, item); + } + + /** + * Check if an item matches a filter + */ + private matchesFilter(item: ModerationItem, filter: ModerationFilter): boolean { + if (filter.status && item.status !== filter.status) return false; + if (filter.contentType && item.contentType !== filter.contentType) return false; + if (filter.severity && item.severity !== filter.severity) return false; + if ( + filter.minRiskScore !== undefined && + (item.riskScore?.overall || 0) < filter.minRiskScore + ) + return false; + if ( + filter.maxRiskScore !== undefined && + (item.riskScore?.overall || 0) > filter.maxRiskScore + ) + return false; + return true; + } +} \ No newline at end of file diff --git a/backend/src/services/moderation/ModerationScoringService.ts b/backend/src/services/moderation/ModerationScoringService.ts new file mode 100644 index 00000000..af198077 --- /dev/null +++ b/backend/src/services/moderation/ModerationScoringService.ts @@ -0,0 +1,500 @@ +/** + * Moderation Scoring Service + * ML-assisted pre-screening for policy violations. + * Scores content for risk, categorizes violations, and provides + * confidence-aware recommendations for auto-approval, auto-rejection, + * or human review queue routing. + */ + +import { randomUUID } from 'crypto'; +import { + ModerationItem, + ContentType, + ModerationStatus, + SeverityLevel, + PolicyViolationType, + RiskScoreBreakdown, + PolicyScore, + MLModelConfig, + DEFAULT_ML_CONFIG, + PolicyScore as PolicyScoreType, +} from '../../models/Moderation'; +import logger from '../../utils/logger'; + +export class ModerationScoringService { + private config: MLModelConfig; + private decisionHistory: Array<{ + predicted: SeverityLevel; + actual: SeverityLevel; + itemId: string; + }> = []; + + constructor(config?: Partial) { + this.config = { ...DEFAULT_ML_CONFIG, ...config }; + } + + /** + * Score a moderation item for policy violations. + * Returns a risk score breakdown and routes the item. + */ + async scoreItem(item: ModerationItem): Promise { + const startTime = Date.now(); + logger.info(`Scoring moderation item ${item.id} (${item.contentType})`); + + try { + item.status = ModerationStatus.SCORING; + item.scoredAt = new Date(); + + const riskScore = await this.computeRiskScore(item); + + item.riskScore = riskScore; + item.severity = this.determineSeverity(riskScore.overall); + + // Route based on risk score + item.status = this.routeItem(riskScore); + + logger.info( + `Scored item ${item.id}: risk=${riskScore.overall.toFixed(1)}, ` + + `severity=${item.severity}, status=${item.status}, ` + + `time=${(Date.now() - startTime)}ms` + ); + + return item; + } catch (error) { + logger.error(`Error scoring item ${item.id}:`, error); + item.status = ModerationStatus.PENDING; + return item; + } + } + + /** + * Batch score multiple items + */ + async scoreBatch(items: ModerationItem[]): Promise { + const scored = await Promise.allSettled( + items.map((item) => this.scoreItem(item)) + ); + + return scored.map((result, index) => { + if (result.status === 'fulfilled') { + return result.value; + } + logger.error(`Failed to score item ${items[index].id}:`, result.reason); + return items[index]; + }); + } + + /** + * Compute risk score breakdown for an item + */ + private async computeRiskScore(item: ModerationItem): Promise { + const policyScores = await this.analyzePolicies(item); + const textRisk = this.computeTextRisk(item); + const metadataRisk = this.computeMetadataRisk(item); + const userHistoryRisk = await this.computeUserHistoryRisk(item); + const similarityRisk = await this.computeSimilarityRisk(item); + + // Weighted overall risk + const overall = + policyScores.reduce((sum, p) => sum + p.score, 0) / Math.max(1, policyScores.length) * 0.40 + + textRisk * 0.20 + + metadataRisk * 0.10 + + userHistoryRisk * 0.15 + + similarityRisk * 0.15; + + const confidence = this.computeConfidence(policyScores, item); + + return { + overall: Math.min(100, Math.max(0, overall)), + policyScores, + textRisk, + metadataRisk, + userHistoryRisk, + similarityRisk, + confidence, + modelVersion: this.config.modelVersion, + }; + } + + /** + * Analyze content against each active policy + */ + private async analyzePolicies(item: ModerationItem): Promise { + const scores: PolicyScore[] = []; + const content = `${item.title} ${item.description} ${item.content}`.toLowerCase(); + + for (const policy of this.config.activePolicies) { + const result = this.scanForPolicy(content, policy); + if (result.score > 0) { + scores.push(result); + } + } + + if (scores.length === 0) { + // Default clean score + scores.push({ + policyType: PolicyViolationType.COMMUNITY_GUIDELINES, + score: 0, + confidence: 1.0, + keywords: [], + matchedPatterns: [], + }); + } + + return scores; + } + + /** + * Scan content for a specific policy violation + */ + private scanForPolicy( + content: string, + policy: PolicyViolationType + ): PolicyScoreType { + const patterns = this.getPolicyPatterns(policy); + const matchedPatterns: string[] = []; + const keywords: string[] = []; + let matchCount = 0; + + for (const pattern of patterns) { + const regex = new RegExp(`\\b${this.escapeRegex(pattern)}\\b`, 'gi'); + const matches = content.match(regex); + if (matches) { + matchedPatterns.push(pattern); + keywords.push(...matches); + matchCount += matches.length; + } + } + + // Score based on match density and severity of policy + const baseScore = Math.min(matchCount * 8, 80); + const policyWeight = this.getPolicyWeight(policy); + const score = Math.min(100, baseScore * policyWeight); + const confidence = Math.min(1, matchCount / 5 + 0.3); + + return { + policyType: policy, + score, + confidence, + keywords: [...new Set(keywords)].slice(0, 20), + matchedPatterns, + }; + } + + /** + * Get detection patterns for each policy type + */ + private getPolicyPatterns(policy: PolicyViolationType): string[] { + const patterns: Record = { + [PolicyViolationType.HATE_SPEECH]: [ + 'hate', 'racist', 'bigot', 'supremacist', 'discriminat', + 'inferior race', 'ethnic cleansing', 'xenophob', + ], + [PolicyViolationType.HARASSMENT]: [ + 'threat', 'stalk', 'intimidate', 'bully', 'cyberbully', + 'dox', 'swat', 'target', 'retaliat', + ], + [PolicyViolationType.SPAM]: [ + 'click here', 'free money', 'act now', 'limited offer', + 'buy now', 'discount', 'guaranteed', 'winner', + 'subscribe', 'join now', 'hurry', + ], + [PolicyViolationType.NSFW]: [ + 'explicit', 'adult content', 'porn', 'nude', 'sexual', + 'xxx', 'nsfw', 'obscene', 'lewd', + ], + [PolicyViolationType.VIOLENCE]: [ + 'kill', 'murder', 'attack', 'weapon', 'bomb', + 'terror', 'shoot', 'stab', 'assault', 'harm', + ], + [PolicyViolationType.COPYRIGHT]: [ + 'copyright', 'plagiarized', 'stolen', 'pirated', + 'unauthorized copy', 'DMCA', 'infringe', + ], + [PolicyViolationType.MISINFORMATION]: [ + 'fake news', 'hoax', 'conspiracy', 'debunked', + 'misleading', 'propaganda', 'false claim', + ], + [PolicyViolationType.PLAGIARISM]: [ + 'copy', 'plagiar', 'duplicate', 'identical', + 'reproduced without', 'sourced from', + ], + [PolicyViolationType.PERSONAL_INFO]: [ + 'phone number', 'email address', 'social security', + 'credit card', 'address', 'passport', 'driver license', + 'bank account', 'SSN', + ], + [PolicyViolationType.SELF_HARM]: [ + 'suicide', 'self-harm', 'cutting', 'end my life', + 'want to die', 'kill myself', + ], + [PolicyViolationType.ILLEGAL_CONTENT]: [ + 'illegal', 'contraband', 'trafficking', 'smuggl', + 'black market', 'money launder', + ], + [PolicyViolationType.COMMUNITY_GUIDELINES]: [ + 'offensive', 'inappropriate', 'disturbing', 'disruptive', + 'against rules', 'violation', 'abuse', + ], + [PolicyViolationType.CHEATING]: [ + 'cheat', 'answer key', 'exam answers', 'test bank', + 'homework solutions', 'pay for grade', + ], + [PolicyViolationType.OTHER]: [ + 'suspicious', 'concerning', 'unusual', 'anomalous', + ], + }; + + return patterns[policy] || []; + } + + /** + * Get severity weight for policy type + */ + private getPolicyWeight(policy: PolicyViolationType): number { + const weights: Record = { + [PolicyViolationType.ILLEGAL_CONTENT]: 2.0, + [PolicyViolationType.SELF_HARM]: 2.0, + [PolicyViolationType.HATE_SPEECH]: 1.8, + [PolicyViolationType.HARASSMENT]: 1.6, + [PolicyViolationType.VIOLENCE]: 1.6, + [PolicyViolationType.PERSONAL_INFO]: 1.5, + [PolicyViolationType.NSFW]: 1.4, + [PolicyViolationType.COPYRIGHT]: 1.2, + [PolicyViolationType.CHEATING]: 1.1, + [PolicyViolationType.MISINFORMATION]: 1.0, + [PolicyViolationType.SPAM]: 0.8, + [PolicyViolationType.PLAGIARISM]: 1.0, + [PolicyViolationType.COMMUNITY_GUIDELINES]: 0.9, + [PolicyViolationType.OTHER]: 0.7, + }; + + return weights[policy] || 1.0; + } + + /** + * Compute text-based risk factors + */ + private computeTextRisk(item: ModerationItem): number { + const text = `${item.title} ${item.description} ${item.content}`; + let risk = 0; + + // Length-based checks + if (text.length < 10) risk += 5; + if (text.length > 10000) risk += 10; + + // Character pattern checks + const uppercaseRatio = (text.match(/[A-Z]/g) || []).length / text.length; + if (uppercaseRatio > 0.5) risk += 15; + + // Excessive punctuation + const exclamationCount = (text.match(/!/g) || []).length; + if (exclamationCount > 5) risk += 8; + + // URL count + const urlCount = (text.match(/https?:\/\//gi) || []).length; + if (urlCount > 3) risk += 12; + + // ALL CAPS sections + const capsSections = text.match(/\b[A-Z]{4,}\b/g) || []; + if (capsSections.length > 3) risk += 10; + + return Math.min(100, risk); + } + + /** + * Compute metadata-based risk factors + */ + private computeMetadataRisk(item: ModerationItem): number { + let risk = 0; + + // Missing metadata + if (!item.metadata.category) risk += 5; + if (!item.metadata.language) risk += 3; + + // Large file size + if (item.metadata.fileSize && item.metadata.fileSize > 50 * 1024 * 1024) { + risk += 10; + } + + // Suspicious file types + const riskyTypes = ['exe', 'bat', 'sh', 'dmg', 'app', 'msi']; + if (item.metadata.fileType && riskyTypes.includes(item.metadata.fileType)) { + risk += 20; + } + + return Math.min(100, risk); + } + + /** + * Compute risk based on user history + */ + private async computeUserHistoryRisk(item: ModerationItem): Promise { + let risk = 0; + + // New author (no history) gets moderate risk + if (!item.authorId || item.authorId === 'unknown') { + risk += 15; + } + + // Check if author has previous violations (simulated - would query DB) + const previousFlags = item.flags || 0; + if (previousFlags > 0) { + risk += Math.min(previousFlags * 10, 50); + } + + return Math.min(100, risk); + } + + /** + * Compute similarity risk against known violations + */ + private async computeSimilarityRisk(item: ModerationItem): Promise { + // Simulated: in production would use embeddings/vector DB + const content = `${item.title} ${item.description}`.toLowerCase(); + + let risk = 0; + const suspiciousWords = [ + 'free', 'guaranteed', '100%', 'act now', 'limited', + 'exclusive', 'secret', 'shocking', 'you won\'t believe', + ]; + + for (const word of suspiciousWords) { + if (content.includes(word)) risk += 3; + } + + return Math.min(100, risk); + } + + /** + * Determine severity level from overall risk score + */ + private determineSeverity(score: number): SeverityLevel { + if (score >= 75) return SeverityLevel.CRITICAL; + if (score >= 60) return SeverityLevel.HIGH; + if (score >= 35) return SeverityLevel.MEDIUM; + return SeverityLevel.LOW; + } + + /** + * Route item based on risk score and confidence + */ + private routeItem(riskScore: RiskScoreBreakdown): ModerationStatus { + // Auto-approve very low risk with high confidence + if ( + riskScore.overall < this.config.autoApproveThreshold && + riskScore.confidence >= this.config.minConfidence + ) { + return ModerationStatus.AUTO_APPROVED; + } + + // Auto-reject very high risk with high confidence + if ( + riskScore.overall >= this.config.autoRejectThreshold && + riskScore.confidence >= this.config.minConfidence + ) { + return ModerationStatus.AUTO_REJECTED; + } + + // Check policy-specific thresholds + for (const policyScore of riskScore.policyScores) { + const threshold = this.config.policyThresholds[policyScore.policyType]; + if (threshold && policyScore.score >= threshold && policyScore.confidence >= this.config.minConfidence) { + return ModerationStatus.QUEUED; + } + } + + // Queue for human review if above threshold + if (riskScore.overall >= this.config.queueThreshold) { + return ModerationStatus.QUEUED; + } + + // Low risk items also go to queue + return ModerationStatus.QUEUED; + } + + /** + * Compute overall model confidence + */ + private computeConfidence( + policyScores: PolicyScore[], + _item: ModerationItem + ): number { + if (policyScores.length === 0) return 0.5; + + const avgConfidence = + policyScores.reduce((sum, p) => sum + p.confidence, 0) / + policyScores.length; + + // Higher confidence with more matched patterns + const patternBonus = Math.min( + 0.2, + policyScores.reduce((sum, p) => sum + p.matchedPatterns.length, 0) * 0.02 + ); + + return Math.min(1, Math.max(0, avgConfidence + patternBonus)); + } + + /** + * Record moderator feedback for model improvement + */ + recordFeedback( + itemId: string, + predicted: SeverityLevel, + actual: SeverityLevel + ): void { + this.decisionHistory.push({ predicted, actual, itemId }); + + // Keep only last 1000 decisions + if (this.decisionHistory.length > 1000) { + this.decisionHistory = this.decisionHistory.slice(-1000); + } + + logger.info( + `Model feedback recorded for ${itemId}: ` + + `predicted=${predicted}, actual=${actual}` + ); + } + + /** + * Get model accuracy statistics + */ + getModelAccuracy(): { accuracy: number; total: number; correct: number } { + if (this.decisionHistory.length === 0) { + return { accuracy: 1, total: 0, correct: 0 }; + } + + const correct = this.decisionHistory.filter( + (d) => d.predicted === d.actual + ).length; + + return { + accuracy: correct / this.decisionHistory.length, + total: this.decisionHistory.length, + correct, + }; + } + + /** + * Update model configuration + */ + updateConfig(updates: Partial): MLModelConfig { + this.config = { ...this.config, ...updates }; + logger.info('Moderation ML model config updated', { updates }); + return this.config; + } + + /** + * Get current model configuration + */ + getConfig(): MLModelConfig { + return { ...this.config }; + } + + /** + * Escape regex special characters in a string + */ + private escapeRegex(str: string): string { + return str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + } +} \ No newline at end of file diff --git a/backend/src/services/moderation/index.ts b/backend/src/services/moderation/index.ts new file mode 100644 index 00000000..6633f90a --- /dev/null +++ b/backend/src/services/moderation/index.ts @@ -0,0 +1,8 @@ +/** + * Moderation Services + * Barrel export for moderation services module + */ + +export { ModerationScoringService } from './ModerationScoringService'; +export { ModerationQueueService } from './ModerationQueueService'; +export { AppealService } from './AppealService'; \ No newline at end of file diff --git a/backend/src/workers/moderationJob.ts b/backend/src/workers/moderationJob.ts new file mode 100644 index 00000000..d64ac2f1 --- /dev/null +++ b/backend/src/workers/moderationJob.ts @@ -0,0 +1,231 @@ +/** + * Moderation Job Worker + * Async background job for ML-based content pre-screening. + * Processes items in the moderation queue asynchronously, + * applying risk scoring and routing to the appropriate queue. + */ + +import { randomUUID } from 'crypto'; +import { + ModerationItem, + ModerationStatus, + ContentType, + SeverityLevel, + ModerationMetadata, +} from '../models/Moderation'; +import { ModerationScoringService } from '../services/moderation/ModerationScoringService'; +import { ModerationQueueService } from '../services/moderation/ModerationQueueService'; +import logger from '../utils/logger'; + +interface ModerationJobConfig { + /** Maximum concurrent scoring operations */ + concurrency: number; + /** Polling interval in milliseconds */ + pollingIntervalMs: number; + /** Maximum batch size for processing */ + batchSize: number; +} + +const DEFAULT_JOB_CONFIG: ModerationJobConfig = { + concurrency: 5, + pollingIntervalMs: 5000, + batchSize: 10, +}; + +export class ModerationJob { + private config: ModerationJobConfig; + private scoringService: ModerationScoringService; + private queueService: ModerationQueueService; + private pendingItems: ModerationItem[] = []; + private isRunning = false; + private timer: NodeJS.Timeout | null = null; + + constructor( + scoringService: ModerationScoringService, + queueService: ModerationQueueService, + config?: Partial + ) { + this.config = { ...DEFAULT_JOB_CONFIG, ...config }; + this.scoringService = scoringService; + this.queueService = queueService; + } + + /** + * Start the background scoring worker + */ + start(): void { + if (this.isRunning) { + logger.warn('ModerationJob is already running'); + return; + } + + this.isRunning = true; + logger.info('ModerationJob started', { config: this.config }); + this.poll(); + } + + /** + * Stop the background scoring worker + */ + stop(): void { + this.isRunning = false; + if (this.timer) { + clearTimeout(this.timer); + this.timer = null; + } + logger.info('ModerationJob stopped'); + } + + /** + * Submit items for async scoring + */ + submitItems( + items: Array<{ + contentId: string; + contentType: ContentType; + title: string; + description: string; + content: string; + authorId: string; + authorName: string; + authorEmail: string; + metadata?: ModerationMetadata; + }> + ): ModerationItem[] { + const moderationItems: ModerationItem[] = items.map((item) => ({ + id: randomUUID(), + contentId: item.contentId, + contentType: item.contentType, + title: item.title, + description: item.description, + content: item.content, + authorId: item.authorId, + authorName: item.authorName, + authorEmail: item.authorEmail, + status: ModerationStatus.PENDING, + riskScore: null, + severity: SeverityLevel.LOW, + flags: 0, + reports: [], + assignedModeratorId: null, + moderatorNotes: '', + decision: null, + appeal: null, + metadata: item.metadata || {}, + createdAt: new Date(), + updatedAt: new Date(), + scoredAt: null, + reviewedAt: null, + resolvedAt: null, + })); + + // Store in queue service first + for (const item of moderationItems) { + this.queueService.upsertItem(item); + } + + this.pendingItems.push(...moderationItems); + logger.info(`Submitted ${moderationItems.length} items for moderation`); + + return moderationItems; + } + + /** + * Get the current pending item count + */ + getPendingCount(): number { + return this.pendingItems.length; + } + + /** + * Main polling loop + */ + private poll(): void { + if (!this.isRunning) return; + + this.processBatch() + .catch((error) => { + logger.error('Error in moderation polling loop:', error); + }) + .finally(() => { + if (this.isRunning) { + this.timer = setTimeout( + () => this.poll(), + this.config.pollingIntervalMs + ); + } + }); + } + + /** + * Process a batch of pending items + */ + private async processBatch(): Promise { + if (this.pendingItems.length === 0) return; + + const batch = this.pendingItems.splice(0, this.config.batchSize); + logger.debug( + `Processing moderation batch: ${batch.length} items, ${this.pendingItems.length} remaining` + ); + + // Score items concurrently with concurrency limit + const scored = await this.scoreBatchWithConcurrency(batch); + + // Route scored items to queues + for (const item of scored) { + // Auto-approved or auto-rejected don't need queue + if ( + item.status !== ModerationStatus.AUTO_APPROVED && + item.status !== ModerationStatus.AUTO_REJECTED + ) { + const priority = + item.severity === SeverityLevel.CRITICAL || + item.severity === SeverityLevel.HIGH + ? 'high' + : 'normal'; + this.queueService.enqueueItem(item, priority); + } + } + + logger.info( + `Moderation batch processed: ${scored.length} items scored, ` + + `${scored.filter((i) => i.status === ModerationStatus.AUTO_APPROVED).length} auto-approved, ` + + `${scored.filter((i) => i.status === ModerationStatus.AUTO_REJECTED).length} auto-rejected, ` + + `${scored.filter((i) => i.status === ModerationStatus.QUEUED).length} queued` + ); + } + + /** + * Score items with concurrency control + */ + private async scoreBatchWithConcurrency( + items: ModerationItem[] + ): Promise { + const results: ModerationItem[] = []; + const chunks: ModerationItem[][] = []; + + for (let i = 0; i < items.length; i += this.config.concurrency) { + chunks.push(items.slice(i, i + this.config.concurrency)); + } + + for (const chunk of chunks) { + const chunkResults = await Promise.allSettled( + chunk.map((item) => this.scoringService.scoreItem(item)) + ); + + for (let i = 0; i < chunkResults.length; i++) { + const result = chunkResults[i]; + if (result.status === 'fulfilled') { + results.push(result.value); + // Update the item in the queue service + this.queueService.upsertItem(result.value); + } else { + logger.error(`Failed to score item ${chunk[i].id}:`, result.reason); + results.push(chunk[i]); + } + } + } + + return results; + } +} \ No newline at end of file diff --git a/backend/tests/moderation.test.js b/backend/tests/moderation.test.js new file mode 100644 index 00000000..16ef4e4a --- /dev/null +++ b/backend/tests/moderation.test.js @@ -0,0 +1,443 @@ +/** + * Moderation Service Tests + * Tests for ML-assisted content moderation system + */ + +const { ModerationScoringService } = require('../src/services/moderation/ModerationScoringService'); +const { ModerationQueueService } = require('../src/services/moderation/ModerationQueueService'); +const { AppealService } = require('../src/services/moderation/AppealService'); +const { ModerationJob } = require('../src/workers/moderationJob'); +const { + ModerationStatus, + ContentType, + SeverityLevel, + PolicyViolationType, + ModerationAction, + AppealStatus, + DEFAULT_ML_CONFIG, +} = require('../src/models/Moderation'); + +describe('Moderation System', () => { + let scoringService; + let queueService; + let appealService; + + const createTestItem = (overrides = {}) => ({ + id: `test-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`, + contentId: 'content-1', + contentType: ContentType.USER_POST, + title: overrides.title || 'Test Content', + description: overrides.description || 'A test moderation item', + content: overrides.content || 'Some content to moderate', + authorId: 'author-1', + authorName: 'Test Author', + authorEmail: 'test@example.com', + status: ModerationStatus.PENDING, + riskScore: null, + severity: SeverityLevel.LOW, + flags: overrides.flags || 0, + reports: [], + assignedModeratorId: null, + moderatorNotes: '', + decision: null, + appeal: null, + metadata: {}, + createdAt: new Date(), + updatedAt: new Date(), + scoredAt: null, + reviewedAt: null, + resolvedAt: null, + ...overrides, + }); + + beforeAll(() => { + scoringService = new ModerationScoringService(); + queueService = new ModerationQueueService(); + appealService = new AppealService(); + appealService.setItemsStore(queueService.itemsStore); + }); + + beforeEach(() => { + scoringService.updateConfig(DEFAULT_ML_CONFIG); + }); + + describe('ModerationScoringService', () => { + it('should score clean content with low risk', async () => { + const item = createTestItem({ + title: 'Introduction to Mathematics', + description: 'A basic course on algebra and geometry', + content: 'Mathematics is the study of numbers, quantities, and shapes...', + }); + + const scored = await scoringService.scoreItem(item); + + expect(scored.riskScore).toBeDefined(); + expect(scored.riskScore.overall).toBeLessThan(40); + expect(scored.riskScore.confidence).toBeGreaterThan(0); + expect(scored.status).toBeDefined(); + }); + + it('should detect spam content with high risk', async () => { + const item = createTestItem({ + title: 'FREE MONEY CLICK HERE!!!!', + description: 'Act now and get free money guaranteed!', + content: 'Click here to claim your free money now. Subscribe now for more free money opportunities. Discount limited offer!', + }); + + const scored = await scoringService.scoreItem(item); + + expect(scored.riskScore).toBeDefined(); + expect(scored.riskScore.overall).toBeGreaterThan(0); + const spamScore = scored.riskScore.policyScores.find( + (p) => p.policyType === PolicyViolationType.SPAM + ); + expect(spamScore).toBeDefined(); + expect(spamScore.score).toBeGreaterThan(0); + }); + + it('should detect hate speech', async () => { + const item = createTestItem({ + title: 'Some post', + description: 'A post', + content: 'hate racist discriminat bigot supremacist xenophob content', + }); + + const scored = await scoringService.scoreItem(item); + + expect(scored.riskScore).toBeDefined(); + const hateScore = scored.riskScore.policyScores.find( + (p) => p.policyType === PolicyViolationType.HATE_SPEECH + ); + expect(hateScore).toBeDefined(); + expect(hateScore.score).toBeGreaterThan(0); + }); + + it('should determine severity correctly', async () => { + const cleanItem = createTestItem({ + title: 'Clean', + content: 'simple clean text with nothing wrong', + }); + const highRiskItem = createTestItem({ + title: 'I WILL KILL YOU', + content: 'kill murder attack weapon bomb terror shoot stab assault harm kill murder attack weapon bomb terror shoot stab assault harm kill murder attack kill murder attack weapon bomb', + }); + + const clean = await scoringService.scoreItem(cleanItem); + const high = await scoringService.scoreItem(highRiskItem); + + expect(clean.severity).toBe(SeverityLevel.LOW); + // High-risk violence content should at minimum be MEDIUM or higher + expect(['medium', 'high', 'critical']).toContain(high.severity); + }); + + it('should record feedback and track model accuracy', () => { + scoringService.recordFeedback('item-1', SeverityLevel.LOW, SeverityLevel.LOW); + scoringService.recordFeedback('item-2', SeverityLevel.HIGH, SeverityLevel.MEDIUM); + scoringService.recordFeedback('item-3', SeverityLevel.MEDIUM, SeverityLevel.MEDIUM); + + const accuracy = scoringService.getModelAccuracy(); + + expect(accuracy.total).toBe(3); + expect(accuracy.correct).toBe(2); + expect(accuracy.accuracy).toBeCloseTo(2 / 3); + }); + + it('should batch score multiple items', async () => { + const items = [ + createTestItem({ id: 'batch-1', title: 'Clean 1', content: 'simple content' }), + createTestItem({ id: 'batch-2', title: 'Spam', content: 'free money click here act now' }), + createTestItem({ id: 'batch-3', title: 'Clean 2', content: 'regular educational text' }), + ]; + + const scored = await scoringService.scoreBatch(items); + + expect(scored.length).toBe(3); + expect(scored.every((item) => item.riskScore !== null)).toBe(true); + }); + + it('should update model configuration', () => { + const config = scoringService.updateConfig({ + autoApproveThreshold: 20, + autoRejectThreshold: 95, + }); + + expect(config.autoApproveThreshold).toBe(20); + expect(config.autoRejectThreshold).toBe(95); + expect(config.modelVersion).toBe(DEFAULT_ML_CONFIG.modelVersion); + }); + + it('should return current configuration', () => { + const config = scoringService.getConfig(); + + expect(config).toHaveProperty('autoApproveThreshold'); + expect(config).toHaveProperty('autoRejectThreshold'); + expect(config).toHaveProperty('queueThreshold'); + expect(config).toHaveProperty('minConfidence'); + expect(config).toHaveProperty('modelVersion'); + expect(config).toHaveProperty('activePolicies'); + }); + }); + + describe('ModerationQueueService', () => { + it('should enqueue and dequeue items', () => { + const item = createTestItem({ id: 'q-item-1', status: ModerationStatus.QUEUED }); + queueService.upsertItem(item); + queueService.enqueueItem(item); + + const moderatorId = 'mod-1'; + queueService.addModeratorToQueue('default', moderatorId); + + const claimed = queueService.dequeueItem(moderatorId); + + expect(claimed).not.toBeNull(); + expect(claimed.status).toBe(ModerationStatus.IN_REVIEW); + expect(claimed.assignedModeratorId).toBe(moderatorId); + }); + + it('should process moderator decisions', () => { + const item = createTestItem({ id: 'dec-item-1' }); + queueService.upsertItem(item); + + const result = queueService.processDecision( + item.id, + 'mod-1', + 'Moderator One', + ModerationAction.APPROVE, + 'Looks clean', + 'No issues found', + { + predictionCorrect: true, + actualSeverity: SeverityLevel.LOW, + predictedSeverity: SeverityLevel.LOW, + improvementNotes: '', + misclassifiedPolicies: [], + } + ); + + expect(result).not.toBeNull(); + expect(result.status).toBe(ModerationStatus.APPROVED); + expect(result.decision).not.toBeNull(); + expect(result.decision.action).toBe(ModerationAction.APPROVE); + }); + + it('should reject items with model feedback', () => { + const item = createTestItem({ id: 'rej-item-1' }); + queueService.upsertItem(item); + + const result = queueService.processDecision( + item.id, 'mod-1', 'Moderator One', + ModerationAction.REJECT, 'Contains spam', 'Multiple spam patterns detected', + { + predictionCorrect: false, + actualSeverity: SeverityLevel.HIGH, + predictedSeverity: SeverityLevel.LOW, + improvementNotes: 'Missed spam patterns', + misclassifiedPolicies: [PolicyViolationType.SPAM], + } + ); + + expect(result.status).toBe(ModerationStatus.REJECTED); + expect(result.decision.modelFeedback.predictionCorrect).toBe(false); + }); + + it('should filter items by status', () => { + const approved = createTestItem({ id: 'approved-99', status: ModerationStatus.APPROVED, title: 'Approved content' }); + const queued = createTestItem({ + id: 'queued-99', status: ModerationStatus.QUEUED, title: 'Queued content', + riskScore: { overall: 60, policyScores: [], textRisk: 0, metadataRisk: 0, userHistoryRisk: 0, similarityRisk: 0, confidence: 0.8, modelVersion: '1.0.0' }, + }); + + queueService.upsertItem(approved); + queueService.upsertItem(queued); + + const queuedItems = queueService.getItems({ status: ModerationStatus.QUEUED }); + expect(queuedItems.total).toBeGreaterThanOrEqual(1); + }); + + it('should provide statistics', () => { + const stats = queueService.getStats(); + expect(stats).toHaveProperty('total'); + expect(stats).toHaveProperty('pending'); + expect(stats).toHaveProperty('queued'); + expect(stats).toHaveProperty('approved'); + expect(stats).toHaveProperty('rejected'); + }); + + it('should manage moderator assignments', () => { + queueService.addModeratorToQueue('default', 'mod-1'); + queueService.removeModeratorFromQueue('default', 'mod-1'); + const queues = queueService.getAllQueues(); + expect(queues.length).toBeGreaterThan(0); + }); + }); + + describe('AppealService', () => { + it('should submit an appeal for rejected content', () => { + const item = createTestItem({ id: 'app-item-1', status: ModerationStatus.REJECTED }); + queueService.upsertItem(item); + + const appeal = appealService.submitAppeal( + item.id, 'author-1', 'Test Author', + 'Unfair rejection', + 'My content does not violate any policies and was incorrectly flagged.', + [{ type: 'text', description: 'Original source', value: 'https://example.com' }] + ); + + expect(appeal).not.toBeNull(); + expect(appeal.status).toBe(AppealStatus.PENDING); + expect(appeal.evidence.length).toBe(1); + expect(appeal.moderationId).toBe(item.id); + }); + + it('should not allow appeals for non-rejected content', () => { + const item = createTestItem({ id: 'no-app-item', status: ModerationStatus.APPROVED }); + queueService.upsertItem(item); + + const appeal = appealService.submitAppeal( + item.id, 'author-1', 'Test Author', + 'Unfair', 'Explanation', [] + ); + + expect(appeal).toBeNull(); + }); + + it('should review and approve an appeal', () => { + const item = createTestItem({ id: 'review-item-1', status: ModerationStatus.REJECTED }); + queueService.upsertItem(item); + + const appeal = appealService.submitAppeal( + item.id, 'author-1', 'Author', 'Wrongly rejected', + 'This was a false positive', [] + ); + + const result = appealService.reviewAppeal( + appeal.id, 'admin-1', 'Admin User', + AppealStatus.APPROVED, 'Content looks fine', 'False positive confirmed' + ); + + expect(result).not.toBeNull(); + expect(result.appeal.status).toBe(AppealStatus.APPROVED); + expect(result.item.status).toBe(ModerationStatus.APPROVED); + }); + + it('should deny an appeal and keep item rejected', () => { + const item = createTestItem({ id: 'deny-item-1', status: ModerationStatus.REJECTED }); + queueService.upsertItem(item); + + const appeal = appealService.submitAppeal( + item.id, 'author-1', 'Author', 'Please reconsider', + 'I think this was a mistake', [] + ); + + const result = appealService.reviewAppeal( + appeal.id, 'admin-1', 'Admin User', + AppealStatus.DENIED, 'Content clearly violates spam policy', 'Appeal has no merit' + ); + + expect(result.appeal.status).toBe(AppealStatus.DENIED); + expect(result.item.status).toBe(ModerationStatus.REJECTED); + }); + }); + + describe('ModerationJob', () => { + it('should create and start a moderation job', () => { + const job = new ModerationJob(scoringService, queueService, { + pollingIntervalMs: 1000, + concurrency: 2, + batchSize: 5, + }); + + job.start(); + expect(job.getPendingCount()).toBe(0); + job.stop(); + }); + + it('should submit items for async processing', () => { + const job = new ModerationJob(scoringService, queueService); + const items = [{ + contentId: 'content-1', + contentType: ContentType.USER_POST, + title: 'Test', + description: 'Test', + content: 'Clean content', + authorId: 'author-1', + authorName: 'Author', + authorEmail: 'test@test.com', + }]; + + const submitted = job.submitItems(items); + expect(submitted.length).toBe(1); + expect(submitted[0].status).toBe(ModerationStatus.PENDING); + expect(job.getPendingCount()).toBe(1); + }); + }); + + describe('Integration Tests', () => { + it('should complete full moderation lifecycle', async () => { + const item = createTestItem({ + id: 'lifecycle-1', + title: 'Suspicious Course Material', + description: 'Get answers easily', + content: 'cheat exam answers test bank homework solutions pay for grade', + }); + + // 1. Score + queueService.upsertItem(item); + const scored = await scoringService.scoreItem(item); + queueService.upsertItem(scored); + expect(scored.riskScore).toBeDefined(); + + // 2. Queue and review if needed + if (scored.status === ModerationStatus.QUEUED) { + queueService.enqueueItem(scored); + queueService.addModeratorToQueue('default', 'mod-1'); + const claimed = queueService.dequeueItem('mod-1'); + expect(claimed).not.toBeNull(); + + // 3. Decision + const decision = queueService.processDecision( + scored.id, 'mod-1', 'Moderator', + ModerationAction.REJECT, 'Contains cheating material', + 'Multiple cheating keywords detected', + { predictionCorrect: true, actualSeverity: SeverityLevel.HIGH, + predictedSeverity: scored.severity, improvementNotes: '', misclassifiedPolicies: [] } + ); + expect(decision.status).toBe(ModerationStatus.REJECTED); + + // 4. Appeal + const appeal = appealService.submitAppeal( + scored.id, 'author-1', 'Author', 'Wrong decision', + 'This content is educational, not cheating material.', [] + ); + expect(appeal).not.toBeNull(); + + // 5. Review appeal + const appealResult = appealService.reviewAppeal( + appeal.id, 'admin-1', 'Admin', + AppealStatus.APPROVED, 'Content is legitimate educational material', 'False positive' + ); + expect(appealResult.item.status).toBe(ModerationStatus.APPROVED); + } + }); + }); + + describe('Performance', () => { + it('should score items quickly', async () => { + const item = createTestItem({ content: 'Test content for performance measurement' }); + const start = Date.now(); + await scoringService.scoreItem(item); + const duration = Date.now() - start; + expect(duration).toBeLessThan(1000); + }); + + it('should batch score efficiently', async () => { + const items = Array.from({ length: 20 }, (_, i) => + createTestItem({ id: `perf-${i}`, content: `Test content number ${i} for batch performance` }) + ); + const start = Date.now(); + await scoringService.scoreBatch(items); + const duration = Date.now() - start; + expect(duration).toBeLessThan(5000); + }); + }); +}); \ No newline at end of file diff --git a/backend/tests/moderation.test.ts b/backend/tests/moderation.test.ts new file mode 100644 index 00000000..d0cb67ec --- /dev/null +++ b/backend/tests/moderation.test.ts @@ -0,0 +1,615 @@ +/** + * Moderation Service Tests + * Comprehensive tests for ML-assisted content moderation: + * - Risk scoring accuracy + * - Queue management + * - Decision recording and model feedback + * - Appeal flow + */ + +import { ModerationScoringService } from '../src/services/moderation/ModerationScoringService'; +import { ModerationQueueService } from '../src/services/moderation/ModerationQueueService'; +import { AppealService } from '../src/services/moderation/AppealService'; +import { ModerationJob } from '../src/workers/moderationJob'; +import { + ModerationItem, + ModerationStatus, + ContentType, + SeverityLevel, + PolicyViolationType, + ModerationAction, + AppealStatus, + DEFAULT_ML_CONFIG, +} from '../src/models/Moderation'; + +describe('Moderation System', () => { + let scoringService: ModerationScoringService; + let queueService: ModerationQueueService; + let appealService: AppealService; + + const createTestItem = (overrides: Partial = {}): ModerationItem => ({ + id: `test-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`, + contentId: 'content-1', + contentType: ContentType.USER_POST, + title: overrides.title || 'Test Content', + description: overrides.description || 'A test moderation item', + content: overrides.content || 'Some content to moderate', + authorId: 'author-1', + authorName: 'Test Author', + authorEmail: 'test@example.com', + status: ModerationStatus.PENDING, + riskScore: null, + severity: SeverityLevel.LOW, + flags: overrides.flags || 0, + reports: [], + assignedModeratorId: null, + moderatorNotes: '', + decision: null, + appeal: null, + metadata: {}, + createdAt: new Date(), + updatedAt: new Date(), + scoredAt: null, + reviewedAt: null, + resolvedAt: null, + ...overrides, + }); + + beforeAll(() => { + scoringService = new ModerationScoringService(); + queueService = new ModerationQueueService(); + appealService = new AppealService(); + appealService.setItemsStore((queueService as any).itemsStore); + }); + + beforeEach(() => { + // Reset services for clean test state + scoringService.updateConfig(DEFAULT_ML_CONFIG); + }); + + describe('ModerationScoringService', () => { + it('should score clean content with low risk', async () => { + const item = createTestItem({ + title: 'Introduction to Mathematics', + description: 'A basic course on algebra and geometry', + content: 'Mathematics is the study of numbers, quantities, and shapes...', + }); + + const scored = await scoringService.scoreItem(item); + + expect(scored.riskScore).toBeDefined(); + expect(scored.riskScore!.overall).toBeLessThan(40); + expect(scored.riskScore!.confidence).toBeGreaterThan(0); + expect(scored.status).toBeDefined(); + }); + + it('should detect spam content with high risk', async () => { + const item = createTestItem({ + title: 'FREE MONEY CLICK HERE!!!!', + description: 'Act now and get free money guaranteed!', + content: 'Click here to claim your free money now. Subscribe now for more free money opportunities. Discount limited offer!', + }); + + const scored = await scoringService.scoreItem(item); + + expect(scored.riskScore).toBeDefined(); + expect(scored.riskScore!.overall).toBeGreaterThan(0); + const spamScore = scored.riskScore!.policyScores.find( + (p) => p.policyType === PolicyViolationType.SPAM + ); + expect(spamScore).toBeDefined(); + expect(spamScore!.score).toBeGreaterThan(0); + }); + + it('should detect hate speech with high severity', async () => { + const item = createTestItem({ + title: 'Some post', + description: 'A post', + content: 'hate racist discriminat bigot supremacist xenophob content', + }); + + const scored = await scoringService.scoreItem(item); + + expect(scored.riskScore).toBeDefined(); + const hateScore = scored.riskScore!.policyScores.find( + (p) => p.policyType === PolicyViolationType.HATE_SPEECH + ); + expect(hateScore).toBeDefined(); + expect(hateScore!.score).toBeGreaterThan(0); + }); + + it('should auto-approve very low risk content', async () => { + const item = createTestItem({ + title: 'Hello World', + description: 'Simple greeting', + content: 'Hello everyone, welcome to the platform!', + }); + + const scored = await scoringService.scoreItem(item); + + // Low risk might be auto-approved or queued + expect([ + ModerationStatus.AUTO_APPROVED, + ModerationStatus.QUEUED, + ]).toContain(scored.status); + }); + + it('should determine severity correctly', async () => { + const cleanItem = createTestItem({ + title: 'Clean', + content: 'simple clean text with nothing wrong', + }); + const highRiskItem = createTestItem({ + title: 'VIOLENCE', + content: 'kill murder attack weapon bomb shoot stab assault harm', + }); + + const clean = await scoringService.scoreItem(cleanItem); + const high = await scoringService.scoreItem(highRiskItem); + + expect(clean.severity).toBe(SeverityLevel.LOW); + expect(high.severity).toBe(SeverityLevel.CRITICAL); + }); + + it('should record feedback and track model accuracy', () => { + scoringService.recordFeedback('item-1', SeverityLevel.LOW, SeverityLevel.LOW); + scoringService.recordFeedback('item-2', SeverityLevel.HIGH, SeverityLevel.MEDIUM); + scoringService.recordFeedback('item-3', SeverityLevel.MEDIUM, SeverityLevel.MEDIUM); + + const accuracy = scoringService.getModelAccuracy(); + + expect(accuracy.total).toBe(3); + expect(accuracy.correct).toBe(2); + expect(accuracy.accuracy).toBeCloseTo(2 / 3); + }); + + it('should batch score multiple items', async () => { + const items = [ + createTestItem({ title: 'Clean 1', content: 'simple content' }), + createTestItem({ title: 'Spam', content: 'free money click here act now' }), + createTestItem({ title: 'Clean 2', content: 'regular educational text' }), + ]; + + const scored = await scoringService.scoreBatch(items); + + expect(scored.length).toBe(3); + expect(scored.every((item) => item.riskScore !== null)).toBe(true); + }); + + it('should update model configuration', () => { + const config = scoringService.updateConfig({ + autoApproveThreshold: 20, + autoRejectThreshold: 95, + }); + + expect(config.autoApproveThreshold).toBe(20); + expect(config.autoRejectThreshold).toBe(95); + expect(config.modelVersion).toBe(DEFAULT_ML_CONFIG.modelVersion); + }); + + it('should return current configuration', () => { + const config = scoringService.getConfig(); + + expect(config).toHaveProperty('autoApproveThreshold'); + expect(config).toHaveProperty('autoRejectThreshold'); + expect(config).toHaveProperty('queueThreshold'); + expect(config).toHaveProperty('minConfidence'); + expect(config).toHaveProperty('modelVersion'); + expect(config).toHaveProperty('activePolicies'); + }); + }); + + describe('ModerationQueueService', () => { + it('should enqueue and dequeue items', () => { + const item = createTestItem({ status: ModerationStatus.QUEUED }); + queueService.upsertItem(item); + queueService.enqueueItem(item); + + const moderatorId = 'mod-1'; + queueService.addModeratorToQueue('default', moderatorId); + + const claimed = queueService.dequeueItem(moderatorId); + + expect(claimed).not.toBeNull(); + expect(claimed!.status).toBe(ModerationStatus.IN_REVIEW); + expect(claimed!.assignedModeratorId).toBe(moderatorId); + }); + + it('should process moderator decisions', () => { + const item = createTestItem(); + queueService.upsertItem(item); + + const result = queueService.processDecision( + item.id, + 'mod-1', + 'Moderator One', + ModerationAction.APPROVE, + 'Looks clean', + 'No issues found', + { + predictionCorrect: true, + actualSeverity: SeverityLevel.LOW, + predictedSeverity: SeverityLevel.LOW, + improvementNotes: '', + misclassifiedPolicies: [], + } + ); + + expect(result).not.toBeNull(); + expect(result!.status).toBe(ModerationStatus.APPROVED); + expect(result!.decision).not.toBeNull(); + expect(result!.decision!.action).toBe(ModerationAction.APPROVE); + }); + + it('should reject items via moderation decision', () => { + const item = createTestItem(); + queueService.upsertItem(item); + + const result = queueService.processDecision( + item.id, + 'mod-1', + 'Moderator One', + ModerationAction.REJECT, + 'Contains spam', + 'Multiple spam patterns detected', + { + predictionCorrect: false, + actualSeverity: SeverityLevel.HIGH, + predictedSeverity: SeverityLevel.LOW, + improvementNotes: 'Missed spam patterns', + misclassifiedPolicies: [PolicyViolationType.SPAM], + } + ); + + expect(result!.status).toBe(ModerationStatus.REJECTED); + expect(result!.decision!.modelFeedback.predictionCorrect).toBe(false); + }); + + it('should filter items correctly', () => { + const approved = createTestItem({ + id: 'approved-1', + status: ModerationStatus.APPROVED, + title: 'Approved content', + }); + const queued = createTestItem({ + id: 'queued-1', + status: ModerationStatus.QUEUED, + title: 'Queued content', + riskScore: { overall: 60, policyScores: [], textRisk: 0, metadataRisk: 0, userHistoryRisk: 0, similarityRisk: 0, confidence: 0.8, modelVersion: '1.0.0' }, + }); + + queueService.upsertItem(approved); + queueService.upsertItem(queued); + + const queuedItems = queueService.getItems({ status: ModerationStatus.QUEUED }); + expect(queuedItems.total).toBe(1); + expect(queuedItems.items[0].id).toBe('queued-1'); + + const highRisk = queueService.getItems({ minRiskScore: 50 }); + expect(highRisk.total).toBe(1); + expect(highRisk.items[0].id).toBe('queued-1'); + }); + + it('should provide statistics', () => { + const stats = queueService.getStats(); + + expect(stats).toHaveProperty('total'); + expect(stats).toHaveProperty('pending'); + expect(stats).toHaveProperty('queued'); + expect(stats).toHaveProperty('approved'); + expect(stats).toHaveProperty('rejected'); + expect(stats).toHaveProperty('averageRiskScore'); + }); + + it('should manage queues', () => { + const queues = queueService.getAllQueues(); + expect(queues.length).toBeGreaterThan(0); + expect(queues[0].id).toBe('default'); + expect(queues[0].name).toBe('General Moderation Queue'); + }); + + it('should add and remove moderators from queues', () => { + const added = queueService.addModeratorToQueue('default', 'mod-1'); + expect(added).toBe(true); + + // Should not duplicate + queueService.addModeratorToQueue('default', 'mod-1'); + const queue = queueService.getQueue('default'); + const modCount = queue!.moderators.filter((m) => m === 'mod-1').length; + expect(modCount).toBe(1); + + const removed = queueService.removeModeratorFromQueue('default', 'mod-1'); + expect(removed).toBe(true); + }); + }); + + describe('AppealService', () => { + it('should submit an appeal for rejected content', () => { + const item = createTestItem({ status: ModerationStatus.REJECTED }); + queueService.upsertItem(item); + + const appeal = appealService.submitAppeal( + item.id, + 'author-1', + 'Test Author', + 'Unfair rejection', + 'My content does not violate any policies and was incorrectly flagged.', + [{ type: 'text', description: 'Original source', value: 'https://example.com' }] + ); + + expect(appeal).not.toBeNull(); + expect(appeal!.status).toBe(AppealStatus.PENDING); + expect(appeal!.evidence.length).toBe(1); + expect(appeal!.moderationId).toBe(item.id); + }); + + it('should not allow appeals for non-rejected content', () => { + const item = createTestItem({ status: ModerationStatus.APPROVED }); + queueService.upsertItem(item); + + const appeal = appealService.submitAppeal( + item.id, + 'author-1', + 'Test Author', + 'Unfair', + 'Explanation', + [] + ); + + expect(appeal).toBeNull(); + }); + + it('should not allow duplicate appeals', () => { + const item = createTestItem({ status: ModerationStatus.REJECTED }); + queueService.upsertItem(item); + + appealService.submitAppeal(item.id, 'author-1', 'Test', 'Reason', 'Explanation', []); + const duplicate = appealService.submitAppeal(item.id, 'author-1', 'Test', 'Reason 2', 'Explanation 2', []); + + expect(duplicate).not.toBeNull(); // Returns existing appeal + }); + + it('should review and approve an appeal', () => { + const item = createTestItem({ status: ModerationStatus.REJECTED }); + queueService.upsertItem(item); + + const appeal = appealService.submitAppeal( + item.id, 'author-1', 'Author', 'Wrongly rejected', + 'This was a false positive', [] + ); + + const result = appealService.reviewAppeal( + appeal!.id, + 'admin-1', + 'Admin User', + AppealStatus.APPROVED, + 'Content looks fine', + 'False positive confirmed' + ); + + expect(result).not.toBeNull(); + expect(result!.appeal.status).toBe(AppealStatus.APPROVED); + expect(result!.item.status).toBe(ModerationStatus.APPROVED); + }); + + it('should deny an appeal and keep item rejected', () => { + const item = createTestItem({ status: ModerationStatus.REJECTED }); + queueService.upsertItem(item); + + const appeal = appealService.submitAppeal( + item.id, 'author-1', 'Author', 'Please reconsider', + 'I think this was a mistake', [] + ); + + const result = appealService.reviewAppeal( + appeal!.id, + 'admin-1', + 'Admin User', + AppealStatus.DENIED, + 'Content clearly violates spam policy', + 'Appeal has no merit' + ); + + expect(result!.appeal.status).toBe(AppealStatus.DENIED); + expect(result!.item.status).toBe(ModerationStatus.REJECTED); + }); + + it('should filter appeals by status', () => { + const item1 = createTestItem({ id: 'item-1', status: ModerationStatus.REJECTED }); + const item2 = createTestItem({ id: 'item-2', status: ModerationStatus.REJECTED }); + queueService.upsertItem(item1); + queueService.upsertItem(item2); + + const a1 = appealService.submitAppeal(item1.id, 'user-1', 'User 1', 'R1', 'E1', []); + const a2 = appealService.submitAppeal(item2.id, 'user-2', 'User 2', 'R2', 'E2', []); + + appealService.reviewAppeal(a1!.id, 'admin', 'Admin', AppealStatus.APPROVED, 'OK', ''); + + const pending = appealService.getAppeals({ status: AppealStatus.PENDING }); + const approved = appealService.getAppeals({ status: AppealStatus.APPROVED }); + + expect(pending.total).toBe(1); + expect(approved.total).toBe(1); + }); + + it('should provide appeal statistics', () => { + const stats = appealService.getAppealStats(); + + expect(stats).toHaveProperty('total'); + expect(stats).toHaveProperty('pending'); + expect(stats).toHaveProperty('approved'); + expect(stats).toHaveProperty('denied'); + expect(stats).toHaveProperty('averageResolutionTime'); + }); + }); + + describe('ModerationJob', () => { + it('should create and start a moderation job', () => { + const job = new ModerationJob(scoringService, queueService, { + pollingIntervalMs: 100, + concurrency: 2, + batchSize: 5, + }); + + job.start(); + expect(job.getPendingCount()).toBe(0); + job.stop(); + }); + + it('should submit items for async processing', () => { + const job = new ModerationJob(scoringService, queueService); + const items = [ + { + contentId: 'content-1', + contentType: ContentType.USER_POST, + title: 'Test', + description: 'Test', + content: 'Clean content', + authorId: 'author-1', + authorName: 'Author', + authorEmail: 'test@test.com', + }, + ]; + + const submitted = job.submitItems(items); + expect(submitted.length).toBe(1); + expect(submitted[0].status).toBe(ModerationStatus.PENDING); + expect(job.getPendingCount()).toBe(1); + }); + + it('should not start twice', () => { + const job = new ModerationJob(scoringService, queueService); + job.start(); + const count = job.getPendingCount(); + // Starting again should be a no-op + job.start(); + expect(job.getPendingCount()).toBe(count); + job.stop(); + }); + }); + + describe('Integration Tests', () => { + it('should complete full moderation lifecycle', async () => { + const item = createTestItem({ + title: 'Suspicious Course Material', + description: 'Get answers easily', + content: 'cheat exam answers test bank homework solutions pay for grade', + }); + + // 1. Submit for scoring + queueService.upsertItem(item); + const scored = await scoringService.scoreItem(item); + queueService.upsertItem(scored); + + expect(scored.riskScore).toBeDefined(); + + // 2. Queue for review if needed + if (scored.status === ModerationStatus.QUEUED) { + queueService.enqueueItem(scored); + queueService.addModeratorToQueue('default', 'mod-1'); + const claimed = queueService.dequeueItem('mod-1'); + expect(claimed).not.toBeNull(); + + // 3. Make decision + const decision = queueService.processDecision( + scored.id, + 'mod-1', + 'Moderator', + ModerationAction.REJECT, + 'Contains cheating material', + 'Multiple cheating keywords detected', + { + predictionCorrect: true, + actualSeverity: SeverityLevel.HIGH, + predictedSeverity: scored.severity, + improvementNotes: '', + misclassifiedPolicies: [], + } + ); + + expect(decision!.status).toBe(ModerationStatus.REJECTED); + + // 4. Submit appeal + const appeal = appealService.submitAppeal( + scored.id, + 'author-1', + 'Author', + 'Wrong decision', + 'This content is educational, not cheating material.', + [] + ); + + expect(appeal).not.toBeNull(); + + // 5. Review appeal + const appealResult = appealService.reviewAppeal( + appeal!.id, + 'admin-1', + 'Admin', + AppealStatus.APPROVED, + 'Content is legitimate educational material', + 'False positive - updating model feedback' + ); + + expect(appealResult!.item.status).toBe(ModerationStatus.APPROVED); + } + }); + + it('should handle batch submission with mixed content types', async () => { + const items = [ + createTestItem({ + contentType: ContentType.COURSE, + title: 'Introduction to Python', + content: 'Learn Python programming basics...', + }), + createTestItem({ + contentType: ContentType.COMMENT, + title: 'Spam comment', + content: 'Free money click here now!!!', + }), + createTestItem({ + contentType: ContentType.FILE, + title: 'Assignment file', + content: 'Assignment submission content...', + }), + ]; + + for (const item of items) { + queueService.upsertItem(item); + } + + const scored = await scoringService.scoreBatch(items); + expect(scored.length).toBe(3); + expect(scored.every((item) => item.riskScore !== null)).toBe(true); + }); + }); + + describe('Performance', () => { + it('should score items quickly', async () => { + const item = createTestItem({ + content: 'Test content for performance measurement', + }); + + const start = Date.now(); + await scoringService.scoreItem(item); + const duration = Date.now() - start; + + expect(duration).toBeLessThan(1000); // Should complete in under 1s + }); + + it('should batch score efficiently', async () => { + const items = Array.from({ length: 20 }, (_, i) => + createTestItem({ + id: `perf-${i}`, + content: `Test content number ${i} for batch performance`, + }) + ); + + const start = Date.now(); + await scoringService.scoreBatch(items); + const duration = Date.now() - start; + + expect(duration).toBeLessThan(5000); // 20 items in under 5s + }); + }); +}); \ No newline at end of file diff --git a/frontend/src/app/admin/content/moderation/page.tsx b/frontend/src/app/admin/content/moderation/page.tsx index 0f8c1bbb..2e56e36f 100644 --- a/frontend/src/app/admin/content/moderation/page.tsx +++ b/frontend/src/app/admin/content/moderation/page.tsx @@ -1,6 +1,6 @@ 'use client'; -import React, { useState, useEffect } from 'react'; +import React, { useState, useEffect, useCallback } from 'react'; import { useAuth } from '@/contexts/AuthContext'; import { Shield, @@ -20,92 +20,321 @@ import { Ban, Check, BookOpen, - X + X, + Brain, + BarChart3, + ChevronDown, + ChevronUp, + RefreshCw, + Activity, + Gavel, + FileWarning, + Scale, } from 'lucide-react'; -interface ContentItem { +// --- Types --- + +interface RiskScoreBreakdown { + overall: number; + policyScores: Array<{ + policyType: string; + score: number; + confidence: number; + keywords: string[]; + matchedPatterns: string[]; + }>; + textRisk: number; + metadataRisk: number; + userHistoryRisk: number; + similarityRisk: number; + confidence: number; + modelVersion: string; +} + +interface ModerationItem { id: string; - type: 'course' | 'quiz' | 'user_post' | 'comment' | 'file'; + contentId: string; + contentType: string; title: string; description: string; - author: { - id: string; - name: string; - email: string; - }; - status: 'pending' | 'approved' | 'rejected' | 'flagged'; + content: string; + authorId: string; + authorName: string; + authorEmail: string; + status: string; + riskScore: RiskScoreBreakdown | null; + severity: string; flags: number; - reports: Report[]; + reports: ModerationReport[]; + assignedModeratorId: string | null; + moderatorNotes: string; + decision: ModerationDecision | null; + appeal: Appeal | null; + metadata: Record; createdAt: string; updatedAt: string; - metadata: { - category?: string; - difficulty?: string; - duration?: string; - fileSize?: number; + scoredAt: string | null; + reviewedAt: string | null; + resolvedAt: string | null; +} + +interface ModerationReport { + id: string; + reason: string; + description: string; + reporterId: string; + reporterName: string; + createdAt: string; + status: string; +} + +interface ModerationDecision { + id: string; + moderatorId: string; + moderatorName: string; + action: string; + reason: string; + notes: string; + createdAt: string; + modelFeedback: { + predictionCorrect: boolean; + actualSeverity: string; + predictedSeverity: string; + improvementNotes: string; + misclassifiedPolicies: string[]; }; } -interface Report { +interface Appeal { id: string; + moderationId: string; + submitterId: string; + submitterName: string; reason: string; + explanation: string; + evidence: AppealEvidence[]; + status: string; + decision: AppealDecision | null; + createdAt: string; + updatedAt: string; + resolvedAt: string | null; +} + +interface AppealEvidence { + id: string; + type: 'text' | 'file' | 'url' | 'reference'; description: string; - reporter: string; + value: string; +} + +interface AppealDecision { + id: string; + reviewerId: string; + reviewerName: string; + decision: string; + reason: string; + notes: string; createdAt: string; - status: 'pending' | 'resolved'; } +interface ModerationStats { + total: number; + pending: number; + queued: number; + inReview: number; + approved: number; + rejected: number; + flagged: number; + autoApproved: number; + autoRejected: number; + averageRiskScore: number; + averageReviewTime: number; + modelAccuracy: number; + appeals: { + total: number; + pending: number; + approved: number; + denied: number; + }; +} + +// --- Tab types --- +type ModerationTab = 'queue' | 'items' | 'appeals' | 'stats'; + +// --- Component --- + export default function ContentModeration() { const { hasPermission } = useAuth(); - const [content, setContent] = useState([]); + const [activeTab, setActiveTab] = useState('queue'); + const [items, setItems] = useState([]); const [loading, setLoading] = useState(true); - const [selectedStatus, setSelectedStatus] = useState('pending'); + const [selectedStatus, setSelectedStatus] = useState('all'); const [selectedType, setSelectedType] = useState('all'); const [searchTerm, setSearchTerm] = useState(''); - const [selectedContent, setSelectedContent] = useState(null); + const [selectedItem, setSelectedItem] = useState(null); + const [stats, setStats] = useState(null); + const [appeals, setAppeals] = useState([]); + const [decisionReason, setDecisionReason] = useState(''); + const [decisionNotes, setDecisionNotes] = useState(''); + const [appealDecision, setAppealDecision] = useState(''); + const [appealReason, setAppealReason] = useState(''); + const [expandedRiskScores, setExpandedRiskScores] = useState>(new Set()); + const [showDecisionModal, setShowDecisionModal] = useState(false); + const [showAppealReviewModal, setShowAppealReviewModal] = useState(false); + const [selectedAction, setSelectedAction] = useState('approve'); - useEffect(() => { - fetchContent(); - }, [selectedStatus, selectedType, searchTerm]); - - const fetchContent = async () => { + const fetchItems = useCallback(async () => { try { setLoading(true); const params = new URLSearchParams({ - status: selectedStatus, - ...(selectedType !== 'all' && { type: selectedType }), - ...(searchTerm && { search: searchTerm }) + ...(selectedStatus !== 'all' && { status: selectedStatus }), + ...(selectedType !== 'all' && { contentType: selectedType }), + ...(searchTerm && { search: searchTerm }), + limit: '50', }); - const response = await fetch(`/api/admin/content/moderation?${params}`); + const response = await fetch(`/api/moderation/items?${params}`); if (response.ok) { const data = await response.json(); - setContent(data.content || []); + setItems(data.data || []); } } catch (error) { - console.error('Failed to fetch content:', error); + console.error('Failed to fetch items:', error); } finally { setLoading(false); } + }, [selectedStatus, selectedType, searchTerm]); + + const fetchStats = useCallback(async () => { + try { + const response = await fetch('/api/moderation/stats'); + if (response.ok) { + const data = await response.json(); + setStats(data.data); + } + } catch (error) { + console.error('Failed to fetch stats:', error); + } + }, []); + + const fetchAppeals = useCallback(async () => { + try { + const response = await fetch('/api/moderation/appeals'); + if (response.ok) { + const data = await response.json(); + setAppeals(data.data || []); + } + } catch (error) { + console.error('Failed to fetch appeals:', error); + } + }, []); + + useEffect(() => { + if (activeTab === 'items') fetchItems(); + else if (activeTab === 'stats') fetchStats(); + else if (activeTab === 'appeals') fetchAppeals(); + else if (activeTab === 'queue') fetchItems(); + }, [activeTab, fetchItems, fetchStats, fetchAppeals, selectedStatus, selectedType, searchTerm]); + + const handleClaimNext = async () => { + try { + const response = await fetch('/api/moderation/queue/claim', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + }); + if (response.ok) { + const data = await response.json(); + if (data.data) { + setSelectedItem(data.data); + setShowDecisionModal(true); + } + fetchItems(); + } + } catch (error) { + console.error('Failed to claim item:', error); + } + }; + + const handleMakeDecision = async () => { + if (!selectedItem || !decisionReason) return; + + try { + const response = await fetch(`/api/moderation/items/${selectedItem.id}/decision`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + action: selectedAction, + reason: decisionReason, + notes: decisionNotes, + predictionCorrect: false, + actualSeverity: selectedItem.severity, + predictedSeverity: selectedItem.severity, + }), + }); + + if (response.ok) { + setShowDecisionModal(false); + setSelectedItem(null); + setDecisionReason(''); + setDecisionNotes(''); + fetchItems(); + fetchStats(); + } + } catch (error) { + console.error('Failed to make decision:', error); + } }; - const handleModerateContent = async (contentId: string, action: 'approve' | 'reject', reason?: string) => { + const handleReviewAppeal = async (appealId: string) => { + if (!appealDecision || !appealReason) return; + try { - const response = await fetch(`/api/admin/content/moderation/${contentId}`, { + const response = await fetch(`/api/moderation/appeals/${appealId}/review`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ action, reason }) + body: JSON.stringify({ + decision: appealDecision, + reason: appealReason, + }), }); if (response.ok) { - fetchContent(); - setSelectedContent(null); + setShowAppealReviewModal(false); + setAppealDecision(''); + setAppealReason(''); + fetchAppeals(); + fetchItems(); } } catch (error) { - console.error('Failed to moderate content:', error); + console.error('Failed to review appeal:', error); } }; + const handleRescoreItem = async (itemId: string) => { + try { + const response = await fetch(`/api/moderation/items/${itemId}/score`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + }); + if (response.ok) { + fetchItems(); + } + } catch (error) { + console.error('Failed to rescore:', error); + } + }; + + const toggleRiskScore = (itemId: string) => { + setExpandedRiskScores((prev) => { + const next = new Set(prev); + if (next.has(itemId)) next.delete(itemId); + else next.add(itemId); + return next; + }); + }; + + // --- Helper functions --- + const getTypeIcon = (type: string) => { switch (type) { case 'course': return ; @@ -113,28 +342,55 @@ export default function ContentModeration() { case 'user_post': return ; case 'comment': return ; case 'file': return ; + case 'video': return