Scaffold dd0c/cost: Welford baseline, anomaly scorer, governance engine, tests

- Welford online algorithm for running mean/stddev baselines
- Anomaly scorer: z-score → 0-100 mapping, property-based tests (10K runs, fast-check)
- Governance engine: 14-day auto-promotion with FP rate gate, injectable Clock
- Panic mode: defaults to active (safe) when Redis unreachable
- Tests: 12 scorer cases (incl 2x 10K property-based), 9 governance cases, 3 panic mode cases
- PostgreSQL schema with RLS: baselines (optimistic locking), anomalies, remediation_actions
- Fly.io config, Dockerfile
This commit is contained in:
2026-03-01 02:52:53 +00:00
parent 23db74b306
commit 6f692fc5ef
8 changed files with 540 additions and 0 deletions

View File

@@ -0,0 +1,72 @@
import { describe, it, expect, beforeEach } from 'vitest';
import { GovernanceEngine, FakeClock } from '../../src/governance/engine.js';
describe('GovernanceEngine', () => {
let clock: FakeClock;
let engine: GovernanceEngine;
beforeEach(() => {
clock = new FakeClock(new Date('2026-03-01').getTime());
engine = new GovernanceEngine(clock);
});
it('does not promote at day 13', () => {
const result = engine.evaluatePromotion('t1', { fpRate: 0.05, daysInCurrentMode: 13 });
expect(result.promoted).toBe(false);
});
it('promotes at day 15 with low FP rate', () => {
const result = engine.evaluatePromotion('t1', { fpRate: 0.05, daysInCurrentMode: 15 });
expect(result.promoted).toBe(true);
expect(result.newMode).toBe('audit');
});
it('does not promote at day 15 with high FP rate', () => {
const result = engine.evaluatePromotion('t1', { fpRate: 0.15, daysInCurrentMode: 15 });
expect(result.promoted).toBe(false);
expect(result.reason).toContain('false-positive rate');
});
it('promotes at exactly day 14 with 0% FP rate', () => {
const result = engine.evaluatePromotion('t1', { fpRate: 0, daysInCurrentMode: 14 });
expect(result.promoted).toBe(true);
});
it('does not promote at exactly 10% FP rate', () => {
const result = engine.evaluatePromotion('t1', { fpRate: 0.10, daysInCurrentMode: 20 });
expect(result.promoted).toBe(true); // 10% is the threshold, not exceeded
});
it('does not promote at 10.1% FP rate', () => {
const result = engine.evaluatePromotion('t1', { fpRate: 0.101, daysInCurrentMode: 20 });
expect(result.promoted).toBe(false);
});
});
describe('Panic Mode Redis Failure', () => {
let engine: GovernanceEngine;
beforeEach(() => {
engine = new GovernanceEngine();
});
it('defaults to panic=active when Redis is unreachable', async () => {
const fakeRedis = {
get: async () => { throw new Error('Connection refused'); },
};
const result = await engine.checkPanicMode('t1', fakeRedis);
expect(result).toBe(true); // Safe default
});
it('returns false when panic is not set', async () => {
const fakeRedis = { get: async () => null };
const result = await engine.checkPanicMode('t1', fakeRedis);
expect(result).toBe(false);
});
it('returns true when panic is active', async () => {
const fakeRedis = { get: async () => '1' };
const result = await engine.checkPanicMode('t1', fakeRedis);
expect(result).toBe(true);
});
});

View File

@@ -0,0 +1,113 @@
import { describe, it, expect } from 'vitest';
import fc from 'fast-check';
import { WelfordBaseline, scoreAnomaly, shouldAlert } from '../../src/detection/scorer.js';
describe('WelfordBaseline', () => {
it('computes correct mean for simple series', () => {
const b = new WelfordBaseline();
[10, 20, 30].forEach(v => b.update(v));
expect(b.mean).toBeCloseTo(20, 5);
expect(b.count).toBe(3);
});
it('computes correct stddev', () => {
const b = new WelfordBaseline();
[2, 4, 4, 4, 5, 5, 7, 9].forEach(v => b.update(v));
expect(b.stddev).toBeCloseTo(2, 0);
});
it('serializes and deserializes correctly', () => {
const b = new WelfordBaseline();
[1, 2, 3, 4, 5].forEach(v => b.update(v));
const json = b.toJSON();
const restored = WelfordBaseline.fromJSON(json);
expect(restored.mean).toBeCloseTo(b.mean, 10);
expect(restored.stddev).toBeCloseTo(b.stddev, 10);
expect(restored.count).toBe(b.count);
});
it('handles single observation', () => {
const b = new WelfordBaseline();
b.update(42);
expect(b.mean).toBe(42);
expect(b.stddev).toBe(0);
});
});
describe('scoreAnomaly', () => {
it('returns 0 for cost at mean', () => {
expect(scoreAnomaly({ cost: 10, mean: 10, stddev: 2 })).toBe(0);
});
it('returns 25 for 1 stddev above', () => {
expect(scoreAnomaly({ cost: 12, mean: 10, stddev: 2 })).toBe(25);
});
it('returns 50 for 2 stddev above', () => {
expect(scoreAnomaly({ cost: 14, mean: 10, stddev: 2 })).toBe(50);
});
it('returns 100 for 4+ stddev above', () => {
expect(scoreAnomaly({ cost: 20, mean: 10, stddev: 2 })).toBe(100);
});
it('returns 0 for cost below mean', () => {
expect(scoreAnomaly({ cost: 5, mean: 10, stddev: 2 })).toBe(0);
});
it('returns 0 for zero baseline', () => {
expect(scoreAnomaly({ cost: 5, mean: 0, stddev: 0 })).toBe(0);
});
it('returns 100 for zero stddev with cost above mean', () => {
expect(scoreAnomaly({ cost: 15, mean: 10, stddev: 0 })).toBe(100);
});
// Property-based: score always 0-100 (BMad must-have: 10K runs)
it('score is always between 0 and 100 (10K runs)', () => {
fc.assert(
fc.property(
fc.record({
cost: fc.float({ min: 0, max: 10000, noNaN: true }),
mean: fc.float({ min: 0, max: 10000, noNaN: true }),
stddev: fc.float({ min: 0, max: 1000, noNaN: true }),
}),
(input) => {
const score = scoreAnomaly(input);
return score >= 0 && score <= 100;
}
),
{ numRuns: 10000, seed: 42 }
);
});
// Property-based: monotonically increasing (BMad must-have: 10K runs)
it('score monotonically increases as cost increases (10K runs)', () => {
fc.assert(
fc.property(
fc.float({ min: 0, max: 100, noNaN: true }),
fc.float({ min: 0, max: 100, noNaN: true }),
fc.float({ min: 0.01, max: 50, noNaN: true }),
(costA, costB, stddev) => {
const baseline = { mean: 5.0, stddev };
const scoreA = scoreAnomaly({ cost: Math.min(costA, costB), ...baseline });
const scoreB = scoreAnomaly({ cost: Math.max(costA, costB), ...baseline });
return scoreB >= scoreA;
}
),
{ numRuns: 10000, seed: 42 }
);
});
});
describe('shouldAlert', () => {
it('alerts at default threshold (50)', () => {
expect(shouldAlert(50)).toBe(true);
expect(shouldAlert(49.99)).toBe(false);
});
it('respects custom threshold', () => {
expect(shouldAlert(30, 25)).toBe(true);
expect(shouldAlert(20, 25)).toBe(false);
});
});