diff --git a/doc/alarms.md b/doc/alarms.md index 8a9df3540a..5d098c76e2 100644 --- a/doc/alarms.md +++ b/doc/alarms.md @@ -121,7 +121,14 @@ config always has the last word. ### Repeat throttling -The chat transport won't repost the same alarm id within +Two levels, and they stack. + +The client itself backs off before a repeat is reported at all: every +occurrence is reported up to a small burst, then one per interval. A fault +recurring hundreds of times a second still counts every occurrence, but it +does not write a log line or build an alert payload for each one. + +Below that, the chat transport won't repost the same alarm id within `repeatThrottleMs` (default 15 minutes). The first occurrence always posts, and the next one that gets through reports how many piled up in between — so a hot loop reads as one message with a count, not a wall of them. diff --git a/src/backend/clients/alarm/AlarmClient.test.ts b/src/backend/clients/alarm/AlarmClient.test.ts index 82bdf4ac42..75c92272b3 100644 --- a/src/backend/clients/alarm/AlarmClient.test.ts +++ b/src/backend/clients/alarm/AlarmClient.test.ts @@ -375,18 +375,18 @@ describe('AlarmClient alarm registry', () => { const client = makeClient(); const seen = capture(client); - for (let i = 0; i < 50; i++) { + for (let i = 0; i < 500; i++) { client.create('hot', `occurrence ${i}`, { i }); } const alarm = client.get('hot')!; - expect(alarm.count).toBe(50); + expect(alarm.count).toBe(500); expect(alarm.occurrences).toHaveLength(20); expect(alarm.timestamps).toHaveLength(20); // The window kept is the most recent one, not the oldest. - expect(alarm.occurrences[19].message).toBe('occurrence 49'); + expect(alarm.occurrences[19].message).toBe('occurrence 499'); // Trimming history must not rewind what the transports are told. - expect(seen[49]).toMatchObject({ repeatCount: 50, isRepeat: true }); + expect(seen.at(-1)).toMatchObject({ repeatCount: 500, isRepeat: true }); }); it('names anonymous handlers by their registration order', () => { @@ -519,3 +519,53 @@ describe('AlarmClient known-error rules', () => { expect(chat[0].severity).toBe('info'); }); }); + +describe('AlarmClient repeat reporting', () => { + beforeEach(() => { + vi.spyOn(console, 'error').mockImplementation(() => {}); + vi.spyOn(console, 'warn').mockImplementation(() => {}); + vi.spyOn(console, 'log').mockImplementation(() => {}); + pdEvent.mockClear(); + }); + + it('reports every occurrence up to the burst, then backs off', () => { + const client = makeClient(); + const seen = capture(client); + + for (let i = 0; i < 12; i++) client.create('flap', 'same fault'); + + expect(seen).toHaveLength(10); + expect(seen[9].repeatCount).toBe(10); + }); + + it('reports again once the interval is reached', () => { + const client = makeClient(); + const seen = capture(client); + + for (let i = 0; i < 500; i++) client.create('flap', 'same fault'); + + expect(seen).toHaveLength(11); + expect(seen[10].repeatCount).toBe(500); + }); + + it('keeps a fast-repeating fault out of the log after the burst', () => { + const client = makeClient(); + capture(client); + const warn = vi.mocked(console.warn); + warn.mockClear(); + + for (let i = 0; i < 100; i++) client.create('flap', 'same fault'); + + // Occurrences 2-10 only; the first is an ACTIVE line on console.error. + expect(warn.mock.calls).toHaveLength(9); + }); + + it('skips the payload entirely when no transport accepts the severity', () => { + const client = makeClient(); + const seen = capture(client, 'critical'); + + client.create('quiet', 'nobody is listening', {}, 'info'); + + expect(seen).toHaveLength(0); + }); +}); diff --git a/src/backend/clients/alarm/AlarmClient.ts b/src/backend/clients/alarm/AlarmClient.ts index 179c3afbde..7b4f949c0e 100644 --- a/src/backend/clients/alarm/AlarmClient.ts +++ b/src/backend/clients/alarm/AlarmClient.ts @@ -69,6 +69,15 @@ const FALLBACK_SEVERITY: PagerSeverity = 'critical'; * and `count` keeps that. */ const OCCURRENCE_HISTORY_LIMIT = 20; + +// A fast-recurring fault repeats far faster than it is useful to report, and +// every report allocates a log line and a payload. Report every occurrence up +// to the burst, then one per interval; `count` still tracks all of them. +const REPEAT_REPORT_BURST = 10; +const REPEAT_REPORT_INTERVAL = 500; + +const shouldReportRepeat = (count: number): boolean => + count <= REPEAT_REPORT_BURST || count % REPEAT_REPORT_INTERVAL === 0; /** Keeps `info` alarms out of the paging system unless config says otherwise. */ const DEFAULT_PAGERDUTY_MIN_SEVERITY: PagerSeverity = 'warning'; /** Slack's ceiling once a pager exists: chat gets what doesn't page. */ @@ -456,6 +465,8 @@ export class AlarmClient extends PuterClient { private handleRepeat(alarm: Alarm): void { this.applyKnownErrors(alarm); + if (!shouldReportRepeat(alarm.count)) return; + console.warn( `[alarm] REPEAT ${displayId(alarm)} :: ${alarm.message} (${alarm.count})`, ); @@ -483,6 +494,18 @@ export class AlarmClient extends PuterClient { ); } + private hasHandlerFor(severity: PagerSeverity): boolean { + for (const { minSeverity, maxSeverity } of this.alertHandlers) { + if ( + meetsMinSeverity(severity, minSeverity) && + withinMaxSeverity(severity, maxSeverity) + ) { + return true; + } + } + return false; + } + private dispatchAlert(alarm: Alarm): void { const resolved = this.resolveSeverity(alarm); if (resolved === 'mute') { @@ -494,6 +517,10 @@ export class AlarmClient extends PuterClient { } alarm.severity = resolved; + // Build nothing for a severity no transport accepts — the payload + // (cleaned fields plus the stack, twice) is the expensive part. + if (!this.hasHandlerFor(resolved)) return; + const fieldsClean = cleanFields(alarm.fields); const repeatCount = alarm.count;