Scaffold dd0c/alert: ingestion, correlation engine, HMAC validation, tests
- Webhook ingestion: HMAC validation for Datadog/PagerDuty/OpsGenie with 5-min timestamp freshness - Payload normalizers: canonical alert schema with severity mapping per provider - Correlation engine: time-window grouping, late-alert attachment (2x window), FakeClock for testing - InMemoryWindowStore for unit tests - Tests: 12 HMAC validation cases, 5 normalizer cases, 7 correlation engine cases - PostgreSQL schema with RLS: tenants, incidents, alerts, webhook_secrets, notification_configs - Free tier enforcement columns (alert_count_month, reset_at) - Fly.io config, Dockerfile, Gitea Actions CI
This commit is contained in:
106
products/03-alert-intelligence/tests/unit/correlation.test.ts
Normal file
106
products/03-alert-intelligence/tests/unit/correlation.test.ts
Normal file
@@ -0,0 +1,106 @@
|
||||
import { describe, it, expect, beforeEach } from 'vitest';
|
||||
import {
|
||||
CorrelationEngine,
|
||||
InMemoryWindowStore,
|
||||
FakeClock,
|
||||
} from '../../src/correlation/engine.js';
|
||||
import type { CanonicalAlert } from '../../src/ingestion/webhook.js';
|
||||
|
||||
function makeAlert(overrides: Partial<CanonicalAlert> = {}): CanonicalAlert {
|
||||
return {
|
||||
sourceProvider: 'datadog',
|
||||
sourceId: `alert-${Math.random().toString(36).slice(2)}`,
|
||||
fingerprint: 'cpu-high',
|
||||
title: 'CPU High',
|
||||
severity: 'high',
|
||||
status: 'firing',
|
||||
service: 'auth',
|
||||
tags: {},
|
||||
rawPayload: {},
|
||||
timestamp: Date.now(),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe('CorrelationEngine', () => {
|
||||
let store: InMemoryWindowStore;
|
||||
let clock: FakeClock;
|
||||
let engine: CorrelationEngine;
|
||||
|
||||
beforeEach(() => {
|
||||
store = new InMemoryWindowStore();
|
||||
clock = new FakeClock(1000000);
|
||||
engine = new CorrelationEngine(store, clock, 5 * 60 * 1000); // 5min window
|
||||
});
|
||||
|
||||
it('creates a new window for first alert', async () => {
|
||||
const result = await engine.process('t1', makeAlert());
|
||||
expect(result.action).toBe('window_updated');
|
||||
expect(result.alertCount).toBe(1);
|
||||
});
|
||||
|
||||
it('groups alerts with same fingerprint into one window', async () => {
|
||||
await engine.process('t1', makeAlert({ fingerprint: 'cpu-high' }));
|
||||
const result = await engine.process('t1', makeAlert({ fingerprint: 'cpu-high' }));
|
||||
expect(result.action).toBe('window_updated');
|
||||
expect(result.alertCount).toBe(2);
|
||||
});
|
||||
|
||||
it('keeps different fingerprints in separate windows', async () => {
|
||||
await engine.process('t1', makeAlert({ fingerprint: 'cpu-high' }));
|
||||
const result = await engine.process('t1', makeAlert({ fingerprint: 'disk-full' }));
|
||||
expect(result.action).toBe('window_updated');
|
||||
expect(result.alertCount).toBe(1); // New window, only 1 alert
|
||||
});
|
||||
|
||||
it('late alert within 2x window attaches to existing incident', async () => {
|
||||
// Alert 1 at T=0
|
||||
await engine.process('t1', makeAlert());
|
||||
|
||||
// Flush at T=5min (window closes, incident created)
|
||||
clock.advanceBy(5 * 60 * 1000);
|
||||
const shipped = await engine.flushWindows('t1');
|
||||
expect(shipped).toHaveLength(1);
|
||||
expect(shipped[0].incidentId).toBeTruthy();
|
||||
|
||||
// Late alert at T=6min (within 2x window = 10min)
|
||||
clock.advanceBy(1 * 60 * 1000);
|
||||
const result = await engine.process('t1', makeAlert());
|
||||
expect(result.action).toBe('attached_to_existing');
|
||||
expect(result.incidentId).toBe(shipped[0].incidentId);
|
||||
});
|
||||
|
||||
it('very late alert (>2x window) creates new incident', async () => {
|
||||
await engine.process('t1', makeAlert());
|
||||
|
||||
clock.advanceBy(5 * 60 * 1000);
|
||||
await engine.flushWindows('t1');
|
||||
|
||||
// 15 minutes later (3x window)
|
||||
clock.advanceBy(10 * 60 * 1000);
|
||||
const result = await engine.process('t1', makeAlert());
|
||||
expect(result.action).toBe('new_incident');
|
||||
});
|
||||
|
||||
it('flushWindows ships expired windows', async () => {
|
||||
await engine.process('t1', makeAlert({ fingerprint: 'a' }));
|
||||
await engine.process('t1', makeAlert({ fingerprint: 'b' }));
|
||||
|
||||
clock.advanceBy(6 * 60 * 1000); // Past 5min window
|
||||
const shipped = await engine.flushWindows('t1');
|
||||
expect(shipped).toHaveLength(2);
|
||||
expect(shipped[0].incidentId).toBeTruthy();
|
||||
expect(shipped[1].incidentId).toBeTruthy();
|
||||
});
|
||||
|
||||
it('does not re-ship already shipped windows', async () => {
|
||||
await engine.process('t1', makeAlert());
|
||||
clock.advanceBy(6 * 60 * 1000);
|
||||
|
||||
const first = await engine.flushWindows('t1');
|
||||
expect(first).toHaveLength(1);
|
||||
|
||||
const second = await engine.flushWindows('t1');
|
||||
expect(second).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
138
products/03-alert-intelligence/tests/unit/ingestion.test.ts
Normal file
138
products/03-alert-intelligence/tests/unit/ingestion.test.ts
Normal file
@@ -0,0 +1,138 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import {
|
||||
validateDatadogHmac,
|
||||
validatePagerdutyHmac,
|
||||
validateOpsgenieHmac,
|
||||
normalizeDatadog,
|
||||
normalizePagerduty,
|
||||
normalizeOpsgenie,
|
||||
} from '../../src/ingestion/webhook.js';
|
||||
import crypto from 'node:crypto';
|
||||
|
||||
const SECRET = 'test-webhook-secret';
|
||||
|
||||
describe('HMAC Validation', () => {
|
||||
describe('Datadog', () => {
|
||||
it('accepts valid signature with fresh timestamp', () => {
|
||||
const body = '{"alert_id":"123"}';
|
||||
const ts = Math.floor(Date.now() / 1000).toString();
|
||||
const sig = crypto.createHmac('sha256', SECRET).update(ts + body).digest('hex');
|
||||
|
||||
const result = validateDatadogHmac(body, sig, ts, SECRET);
|
||||
expect(result.valid).toBe(true);
|
||||
});
|
||||
|
||||
it('rejects stale timestamp (>5 minutes)', () => {
|
||||
const body = '{"alert_id":"123"}';
|
||||
const ts = (Math.floor(Date.now() / 1000) - 301).toString();
|
||||
const sig = crypto.createHmac('sha256', SECRET).update(ts + body).digest('hex');
|
||||
|
||||
const result = validateDatadogHmac(body, sig, ts, SECRET);
|
||||
expect(result.valid).toBe(false);
|
||||
expect(result.error).toContain('stale timestamp');
|
||||
});
|
||||
|
||||
it('rejects missing signature', () => {
|
||||
const result = validateDatadogHmac('{}', undefined, '123', SECRET);
|
||||
expect(result.valid).toBe(false);
|
||||
});
|
||||
|
||||
it('rejects invalid signature', () => {
|
||||
const ts = Math.floor(Date.now() / 1000).toString();
|
||||
const result = validateDatadogHmac('{}', 'bad-sig', ts, SECRET);
|
||||
expect(result.valid).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('PagerDuty', () => {
|
||||
it('rejects missing signature', () => {
|
||||
const result = validatePagerdutyHmac('{}', undefined, SECRET);
|
||||
expect(result.valid).toBe(false);
|
||||
});
|
||||
|
||||
it('rejects malformed signature', () => {
|
||||
const result = validatePagerdutyHmac('{}', 'garbage', SECRET);
|
||||
expect(result.valid).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('OpsGenie', () => {
|
||||
it('rejects stale timestamp from payload body', () => {
|
||||
const staleTs = Date.now() - 6 * 60 * 1000; // 6 minutes ago
|
||||
const body = JSON.stringify({ alert: { alertId: '1' }, timestamp: staleTs });
|
||||
const sig = crypto.createHmac('sha256', SECRET).update(body).digest('hex');
|
||||
|
||||
const result = validateOpsgenieHmac(body, sig, SECRET);
|
||||
expect(result.valid).toBe(false);
|
||||
expect(result.error).toContain('stale timestamp');
|
||||
});
|
||||
|
||||
it('accepts fresh payload', () => {
|
||||
const body = JSON.stringify({ alert: { alertId: '1' }, timestamp: Date.now() });
|
||||
const sig = crypto.createHmac('sha256', SECRET).update(body).digest('hex');
|
||||
|
||||
const result = validateOpsgenieHmac(body, sig, SECRET);
|
||||
expect(result.valid).toBe(true);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('Payload Normalizers', () => {
|
||||
it('normalizes Datadog payload', () => {
|
||||
const alert = normalizeDatadog({
|
||||
alert_id: 'dd-123',
|
||||
title: 'High CPU',
|
||||
priority: 'P1',
|
||||
alert_transition: 'Triggered',
|
||||
tags: { service: 'auth', env: 'prod' },
|
||||
date_happened: 1709251200,
|
||||
});
|
||||
expect(alert.sourceProvider).toBe('datadog');
|
||||
expect(alert.severity).toBe('critical');
|
||||
expect(alert.status).toBe('firing');
|
||||
expect(alert.service).toBe('auth');
|
||||
});
|
||||
|
||||
it('normalizes PagerDuty payload', () => {
|
||||
const alert = normalizePagerduty({
|
||||
event: {
|
||||
event_type: 'incident.triggered',
|
||||
data: {
|
||||
id: 'pd-456',
|
||||
title: 'Disk Full',
|
||||
urgency: 'high',
|
||||
service: { name: 'storage' },
|
||||
created_at: '2026-03-01T00:00:00Z',
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(alert.sourceProvider).toBe('pagerduty');
|
||||
expect(alert.severity).toBe('critical');
|
||||
expect(alert.service).toBe('storage');
|
||||
});
|
||||
|
||||
it('normalizes OpsGenie payload', () => {
|
||||
const alert = normalizeOpsgenie({
|
||||
action: 'Create',
|
||||
alert: {
|
||||
alertId: 'og-789',
|
||||
message: 'Memory Leak',
|
||||
priority: 'P2',
|
||||
tags: ['service:api'],
|
||||
},
|
||||
});
|
||||
expect(alert.sourceProvider).toBe('opsgenie');
|
||||
expect(alert.severity).toBe('high');
|
||||
expect(alert.status).toBe('firing');
|
||||
});
|
||||
|
||||
it('maps Datadog Recovered to resolved', () => {
|
||||
const alert = normalizeDatadog({ alert_transition: 'Recovered' });
|
||||
expect(alert.status).toBe('resolved');
|
||||
});
|
||||
|
||||
it('maps OpsGenie Close to resolved', () => {
|
||||
const alert = normalizeOpsgenie({ action: 'Close', alert: {} });
|
||||
expect(alert.status).toBe('resolved');
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user