Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion doc/alarms.md
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,14 @@ config always has the last word.

### Repeat throttling

The chat transport won't repost the same alarm id within
Two levels, and they stack.

The client itself backs off before a repeat is reported at all: every
occurrence is reported up to a small burst, then one per interval. A fault
recurring hundreds of times a second still counts every occurrence, but it
does not write a log line or build an alert payload for each one.

Below that, the chat transport won't repost the same alarm id within
`repeatThrottleMs` (default 15 minutes). The first occurrence always posts,
and the next one that gets through reports how many piled up in between —
so a hot loop reads as one message with a count, not a wall of them.
58 changes: 54 additions & 4 deletions src/backend/clients/alarm/AlarmClient.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -375,18 +375,18 @@ describe('AlarmClient alarm registry', () => {
const client = makeClient();
const seen = capture(client);

for (let i = 0; i < 50; i++) {
for (let i = 0; i < 500; i++) {
client.create('hot', `occurrence ${i}`, { i });
}

const alarm = client.get('hot')!;
expect(alarm.count).toBe(50);
expect(alarm.count).toBe(500);
expect(alarm.occurrences).toHaveLength(20);
expect(alarm.timestamps).toHaveLength(20);
// The window kept is the most recent one, not the oldest.
expect(alarm.occurrences[19].message).toBe('occurrence 49');
expect(alarm.occurrences[19].message).toBe('occurrence 499');
// Trimming history must not rewind what the transports are told.
expect(seen[49]).toMatchObject({ repeatCount: 50, isRepeat: true });
expect(seen.at(-1)).toMatchObject({ repeatCount: 500, isRepeat: true });
});

it('names anonymous handlers by their registration order', () => {
Expand Down Expand Up @@ -519,3 +519,53 @@ describe('AlarmClient known-error rules', () => {
expect(chat[0].severity).toBe('info');
});
});

describe('AlarmClient repeat reporting', () => {
beforeEach(() => {
vi.spyOn(console, 'error').mockImplementation(() => {});
vi.spyOn(console, 'warn').mockImplementation(() => {});
vi.spyOn(console, 'log').mockImplementation(() => {});
pdEvent.mockClear();
});

it('reports every occurrence up to the burst, then backs off', () => {
const client = makeClient();
const seen = capture(client);

for (let i = 0; i < 12; i++) client.create('flap', 'same fault');

expect(seen).toHaveLength(10);
expect(seen[9].repeatCount).toBe(10);
});

it('reports again once the interval is reached', () => {
const client = makeClient();
const seen = capture(client);

for (let i = 0; i < 500; i++) client.create('flap', 'same fault');

expect(seen).toHaveLength(11);
expect(seen[10].repeatCount).toBe(500);
});

it('keeps a fast-repeating fault out of the log after the burst', () => {
const client = makeClient();
capture(client);
const warn = vi.mocked(console.warn);
warn.mockClear();

for (let i = 0; i < 100; i++) client.create('flap', 'same fault');

// Occurrences 2-10 only; the first is an ACTIVE line on console.error.
expect(warn.mock.calls).toHaveLength(9);
});

it('skips the payload entirely when no transport accepts the severity', () => {
const client = makeClient();
const seen = capture(client, 'critical');

client.create('quiet', 'nobody is listening', {}, 'info');

expect(seen).toHaveLength(0);
});
});
27 changes: 27 additions & 0 deletions src/backend/clients/alarm/AlarmClient.ts
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,15 @@ const FALLBACK_SEVERITY: PagerSeverity = 'critical';
* and `count` keeps that.
*/
const OCCURRENCE_HISTORY_LIMIT = 20;

// A fast-recurring fault repeats far faster than it is useful to report, and
// every report allocates a log line and a payload. Report every occurrence up
// to the burst, then one per interval; `count` still tracks all of them.
const REPEAT_REPORT_BURST = 10;
const REPEAT_REPORT_INTERVAL = 500;

const shouldReportRepeat = (count: number): boolean =>
count <= REPEAT_REPORT_BURST || count % REPEAT_REPORT_INTERVAL === 0;
/** Keeps `info` alarms out of the paging system unless config says otherwise. */
const DEFAULT_PAGERDUTY_MIN_SEVERITY: PagerSeverity = 'warning';
/** Slack's ceiling once a pager exists: chat gets what doesn't page. */
Expand Down Expand Up @@ -456,6 +465,8 @@ export class AlarmClient extends PuterClient {
private handleRepeat(alarm: Alarm): void {
this.applyKnownErrors(alarm);

if (!shouldReportRepeat(alarm.count)) return;

console.warn(
`[alarm] REPEAT ${displayId(alarm)} :: ${alarm.message} (${alarm.count})`,
);
Expand Down Expand Up @@ -483,6 +494,18 @@ export class AlarmClient extends PuterClient {
);
}

private hasHandlerFor(severity: PagerSeverity): boolean {
for (const { minSeverity, maxSeverity } of this.alertHandlers) {
if (
meetsMinSeverity(severity, minSeverity) &&
withinMaxSeverity(severity, maxSeverity)
) {
return true;
}
}
return false;
}

private dispatchAlert(alarm: Alarm): void {
const resolved = this.resolveSeverity(alarm);
if (resolved === 'mute') {
Expand All @@ -494,6 +517,10 @@ export class AlarmClient extends PuterClient {
}
alarm.severity = resolved;

// Build nothing for a severity no transport accepts — the payload
// (cleaned fields plus the stack, twice) is the expensive part.
if (!this.hasHandlerFor(resolved)) return;

const fieldsClean = cleanFields(alarm.fields);
const repeatCount = alarm.count;

Expand Down
Loading