Security hardening: auth encapsulation, pool restriction, rate limiting, invites, async webhooks

Phase 1 (Security Critical): - Auth plugin encapsulation: replaced global addHook with Fastify plugin scope - Removed startsWith URL matching; public routes registered outside auth scope - JWT verify now enforces algorithms: ['HS256'] (prevents algorithm confusion) - Raw pool no longer exported from db.ts; systemQuery() + getPoolForAuth() instead - withTenant() remains primary tenant-scoped query path Phase 2 (Infrastructure): - docker-compose.yml: all secrets via env var substitution (${VAR:-default}) - Per-service Postgres users (dd0c_drift, dd0c_alert, etc.) in docker-init-db.sh - .env.example with all configurable secrets - build-push.sh uses $REGISTRY_PASSWORD instead of hardcoded - .gitignore excludes .env files - @fastify/rate-limit: 100 req/min global, 5/min login, 3/min signup - CORS_ORIGIN default changed from '*' to 'http://localhost:5173' Phase 3 (Product): - Team invite flow: tenant_invites table, POST /invite, GET /invites, DELETE /invites/:id - Signup accepts optional invite_token to join existing tenant - Async webhook ingestion (P3): LPUSH to Redis, BRPOP worker, dead-letter queue Console: - All 5 product modules wired: drift, alert, portal, cost, run - PageHeader accepts children prop - 71 modules, 70KB gzipped production build All 6 projects compile clean (tsc --noEmit).
2026-03-02 23:53:55 +00:00
parent be3f37cfdd
commit eb953cdea5
67 changed files with 4461 additions and 410 deletions
--- a/products/03-alert-intelligence/src/api/webhooks.ts
+++ b/products/03-alert-intelligence/src/api/webhooks.ts
@@ -4,15 +4,13 @@ import {
  validateDatadogHmac,
  validatePagerdutyHmac,
  validateOpsgenieHmac,
-  normalizeDatadog,
-  normalizePagerduty,
-  normalizeOpsgenie,
-  type CanonicalAlert,
 } from '../ingestion/webhook.js';
-import { withTenant } from '../data/db.js';
+import { redis } from '../data/redis.js';

 const logger = pino({ name: 'api-webhooks' });

+const REDIS_QUEUE = 'dd0c:webhooks:incoming';
+
 export function registerWebhookRoutes(app: FastifyInstance) {
  // Datadog webhook
  app.post('/webhooks/datadog/:tenantSlug', async (req, reply) => {
@@ -34,8 +32,12 @@ export function registerWebhookRoutes(app: FastifyInstance) {
      return reply.status(401).send({ error: hmac.error });
    }

-    const alert = normalizeDatadog(body);
-    await ingestAlert(secret.tenantId, alert);
+    await redis.lpush(REDIS_QUEUE, JSON.stringify({
+      provider: 'datadog',
+      tenantId: secret.tenantId,
+      payload: body,
+      receivedAt: Date.now(),
+    }));
    return reply.status(202).send({ status: 'accepted' });
  });

@@ -58,8 +60,12 @@ export function registerWebhookRoutes(app: FastifyInstance) {
      return reply.status(401).send({ error: hmac.error });
    }

-    const alert = normalizePagerduty(body);
-    await ingestAlert(secret.tenantId, alert);
+    await redis.lpush(REDIS_QUEUE, JSON.stringify({
+      provider: 'pagerduty',
+      tenantId: secret.tenantId,
+      payload: body,
+      receivedAt: Date.now(),
+    }));
    return reply.status(202).send({ status: 'accepted' });
  });

@@ -82,8 +88,12 @@ export function registerWebhookRoutes(app: FastifyInstance) {
      return reply.status(401).send({ error: hmac.error });
    }

-    const alert = normalizeOpsgenie(body);
-    await ingestAlert(secret.tenantId, alert);
+    await redis.lpush(REDIS_QUEUE, JSON.stringify({
+      provider: 'opsgenie',
+      tenantId: secret.tenantId,
+      payload: body,
+      receivedAt: Date.now(),
+    }));
    return reply.status(202).send({ status: 'accepted' });
  });

@@ -100,41 +110,19 @@ export function registerWebhookRoutes(app: FastifyInstance) {
      return reply.status(401).send({ error: 'Invalid token' });
    }

-    const alert = normalizeGrafana(body);
-    await ingestAlert(secret.tenantId, alert);
+    await redis.lpush(REDIS_QUEUE, JSON.stringify({
+      provider: 'grafana',
+      tenantId: secret.tenantId,
+      payload: body,
+      receivedAt: Date.now(),
+    }));
    return reply.status(202).send({ status: 'accepted' });
  });
 }

-function normalizeGrafana(payload: any): CanonicalAlert {
-  const alert = payload.alerts?.[0] ?? payload;
-  return {
-    sourceProvider: 'grafana' as any,
-    sourceId: alert.fingerprint ?? crypto.randomUUID(),
-    fingerprint: alert.fingerprint ?? '',
-    title: alert.labels?.alertname ?? payload.title ?? 'Grafana Alert',
-    severity: mapGrafanaSeverity(alert.labels?.severity),
-    status: alert.status === 'resolved' ? 'resolved' : 'firing',
-    service: alert.labels?.service,
-    environment: alert.labels?.env,
-    tags: alert.labels ?? {},
-    rawPayload: payload,
-    timestamp: alert.startsAt ? new Date(alert.startsAt).getTime() : Date.now(),
-  };
-}
-
-function mapGrafanaSeverity(s: string | undefined): CanonicalAlert['severity'] {
-  switch (s) {
-    case 'critical': return 'critical';
-    case 'warning': return 'high';
-    case 'info': return 'info';
-    default: return 'medium';
-  }
-}
-
 async function getWebhookSecret(tenantSlug: string, provider: string): Promise<{ tenantId: string; secret: string } | null> {
-  const { pool } = await import('../data/db.js');
-  const result = await pool.query(
+  const { systemQuery } = await import('../data/db.js');
+  const result = await systemQuery(
    `SELECT ws.secret, t.id as tenant_id
     FROM webhook_secrets ws
     JOIN tenants t ON ws.tenant_id = t.id
@@ -144,51 +132,3 @@ async function getWebhookSecret(tenantSlug: string, provider: string): Promise<{
  if (!result.rows[0]) return null;
  return { tenantId: result.rows[0].tenant_id, secret: result.rows[0].secret };
 }
-
-async function ingestAlert(tenantId: string, alert: CanonicalAlert): Promise<void> {
-  await withTenant(tenantId, async (client) => {
-    // Persist raw alert
-    const alertResult = await client.query(
-      `INSERT INTO alerts (tenant_id, source_provider, source_id, fingerprint, title, severity, status, service, environment, tags, raw_payload)
-       VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
-       RETURNING id`,
-      [tenantId, alert.sourceProvider, alert.sourceId, alert.fingerprint, alert.title, alert.severity, alert.status, alert.service, alert.environment, JSON.stringify(alert.tags), JSON.stringify(alert.rawPayload)],
-    );
-    const alertId = alertResult.rows[0].id;
-
-    // Check for existing open incident with same fingerprint
-    const existing = await client.query(
-      `SELECT id, alert_count FROM incidents
-       WHERE fingerprint = $1 AND status IN ('open', 'acknowledged')
-       ORDER BY created_at DESC LIMIT 1`,
-      [alert.fingerprint],
-    );
-
-    if (existing.rows[0]) {
-      // Attach to existing incident
-      await client.query('UPDATE alerts SET incident_id = $1 WHERE id = $2', [existing.rows[0].id, alertId]);
-      await client.query(
-        `UPDATE incidents SET alert_count = alert_count + 1, last_alert_at = now() WHERE id = $1`,
-        [existing.rows[0].id],
-      );
-    } else if (alert.status === 'firing') {
-      // Create new incident
-      const incident = await client.query(
-        `INSERT INTO incidents (tenant_id, incident_key, fingerprint, service, title, severity, alert_count, first_alert_at, last_alert_at)
-         VALUES ($1, $2, $3, $4, $5, $6, 1, now(), now())
-         RETURNING id`,
-        [tenantId, `inc_${crypto.randomUUID().slice(0, 8)}`, alert.fingerprint, alert.service ?? 'unknown', alert.title, alert.severity],
-      );
-      await client.query('UPDATE alerts SET incident_id = $1 WHERE id = $2', [incident.rows[0].id, alertId]);
-    }
-
-    // Auto-resolve if alert status is resolved
-    if (alert.status === 'resolved') {
-      await client.query(
-        `UPDATE incidents SET status = 'resolved', resolved_at = now()
-         WHERE fingerprint = $1 AND status IN ('open', 'acknowledged')`,
-        [alert.fingerprint],
-      );
-    }
-  });
-}
--- a/products/03-alert-intelligence/src/auth/middleware.ts
+++ b/products/03-alert-intelligence/src/auth/middleware.ts
@@ -15,21 +15,11 @@ export interface AuthPayload {
 }

 /**
- * JWT auth middleware. Extracts tenant context from Bearer token.
- * Also supports API key auth via `X-API-Key` header (dd0c_ prefix).
+ * Returns an onRequest hook that validates JWT or API key auth.
+ * No URL matching — only register this hook inside a protected plugin scope.
 */
-export function registerAuth(app: FastifyInstance, jwtSecret: string, pool: Pool) {
-  app.decorateRequest('tenantId', '');
-  app.decorateRequest('userId', '');
-  app.decorateRequest('userRole', 'viewer');
-
-  app.addHook('onRequest', async (req: FastifyRequest, reply: FastifyReply) => {
-    if (req.url === '/health' || req.url === '/version') return;
-    if (req.url.startsWith('/webhooks/')) return;
-    if (req.url.startsWith('/slack/')) return;
-    const path = req.url.split('?')[0];
-    if (path === '/api/v1/auth/login' || path === '/api/v1/auth/signup') return;
-
+export function authHook(jwtSecret: string, pool: Pool) {
+  return async (req: FastifyRequest, reply: FastifyReply) => {
    const apiKey = req.headers['x-api-key'] as string | undefined;
    const authHeader = req.headers['authorization'];

@@ -38,7 +28,7 @@ export function registerAuth(app: FastifyInstance, jwtSecret: string, pool: Pool
        return reply.status(401).send({ error: 'Invalid API key format' });
      }

-      const prefix = apiKey.slice(0, 13); // dd0c_ + 8 hex chars
+      const prefix = apiKey.slice(0, 13);
      const keyHash = crypto.createHash('sha256').update(apiKey).digest('hex');

      const result = await pool.query(
@@ -61,7 +51,7 @@ export function registerAuth(app: FastifyInstance, jwtSecret: string, pool: Pool
    if (authHeader?.startsWith('Bearer ')) {
      const token = authHeader.slice(7);
      try {
-        const payload = jwt.verify(token, jwtSecret) as AuthPayload;
+        const payload = jwt.verify(token, jwtSecret, { algorithms: ['HS256'] }) as AuthPayload;
        (req as any).tenantId = payload.tenantId;
        (req as any).userId = payload.userId;
        (req as any).userRole = payload.role;
@@ -72,7 +62,17 @@ export function registerAuth(app: FastifyInstance, jwtSecret: string, pool: Pool
    }

    return reply.status(401).send({ error: 'Missing authentication' });
-  });
+  };
+}
+
+/**
+ * Decorate the Fastify request with auth properties.
+ * Call this once on the root app instance before registering any routes.
+ */
+export function decorateAuth(app: FastifyInstance) {
+  app.decorateRequest('tenantId', '');
+  app.decorateRequest('userId', '');
+  app.decorateRequest('userRole', 'viewer');
 }

 export function requireRole(req: FastifyRequest, reply: FastifyReply, minRole: AuthPayload['role']): boolean {
@@ -123,11 +123,21 @@ const loginSchema = z.object({
 const signupSchema = z.object({
  email: z.string().email(),
  password: z.string().min(8),
-  tenant_name: z.string().min(1).max(100),
+  tenant_name: z.string().min(1).max(100).optional(),
+  invite_token: z.string().optional(),
+}).refine(
+  (data) => data.invite_token || data.tenant_name,
+  { message: 'Either tenant_name or invite_token is required', path: ['tenant_name'] },
+);
+
+const inviteSchema = z.object({
+  email: z.string().email(),
+  role: z.enum(['admin', 'member', 'viewer']).default('member'),
 });

+/** Public auth routes — login/signup. No auth required. */
 export function registerAuthRoutes(app: FastifyInstance, jwtSecret: string, pool: Pool) {
-  app.post('/api/v1/auth/login', async (req, reply) => {
+  app.post('/api/v1/auth/login', { config: { rateLimit: { max: 5, timeWindow: '1 minute' } } }, async (req, reply) => {
    const body = loginSchema.parse(req.body);

    const result = await pool.query(
@@ -152,29 +162,64 @@ export function registerAuthRoutes(app: FastifyInstance, jwtSecret: string, pool
    return { token, expires_in: '24h' };
  });

-  app.post('/api/v1/auth/signup', async (req, reply) => {
+  app.post('/api/v1/auth/signup', { config: { rateLimit: { max: 3, timeWindow: '1 minute' } } }, async (req, reply) => {
    const body = signupSchema.parse(req.body);

-    // Check if email already exists
    const existing = await pool.query('SELECT id FROM users WHERE email = $1', [body.email]);
    if (existing.rows[0]) return reply.status(409).send({ error: 'Email already registered' });

    const passwordHash = await hashPassword(body.password);
-    const slug = body.tenant_name.toLowerCase().replace(/[^a-z0-9]+/g, '-').slice(0, 42) + '-' + crypto.randomBytes(3).toString('hex');

    const client = await pool.connect();
    try {
      await client.query('BEGIN');

-      const tenant = await client.query(
-        `INSERT INTO tenants (name, slug) VALUES ($1, $2) RETURNING id`,
-        [body.tenant_name, slug],
-      );
-      const tenantId = tenant.rows[0].id;
+      let tenantId: string;
+      let role: string;
+
+      if (body.invite_token) {
+        const invite = await client.query(
+          `SELECT id, tenant_id, role, expires_at, accepted_at FROM tenant_invites WHERE token = $1`,
+          [body.invite_token],
+        );
+        if (!invite.rows[0]) {
+          await client.query('ROLLBACK');
+          return reply.status(400).send({ error: 'Invalid invite token' });
+        }
+        const inv = invite.rows[0];
+        if (inv.accepted_at) {
+          await client.query('ROLLBACK');
+          return reply.status(400).send({ error: 'Invite already accepted' });
+        }
+        if (new Date(inv.expires_at) < new Date()) {
+          await client.query('ROLLBACK');
+          return reply.status(400).send({ error: 'Invite expired' });
+        }
+
+        tenantId = inv.tenant_id;
+        role = inv.role;
+
+        await client.query(
+          `UPDATE tenant_invites SET accepted_at = NOW() WHERE id = $1`,
+          [inv.id],
+        );
+      } else {
+        if (!body.tenant_name) {
+          await client.query('ROLLBACK');
+          return reply.status(400).send({ error: 'tenant_name is required for new signups' });
+        }
+        const slug = body.tenant_name.toLowerCase().replace(/[^a-z0-9]+/g, '-').slice(0, 42) + '-' + crypto.randomBytes(3).toString('hex');
+        const tenant = await client.query(
+          `INSERT INTO tenants (name, slug) VALUES ($1, $2) RETURNING id`,
+          [body.tenant_name, slug],
+        );
+        tenantId = tenant.rows[0].id;
+        role = 'owner';
+      }

      const user = await client.query(
-        `INSERT INTO users (tenant_id, email, password_hash, role) VALUES ($1, $2, $3, 'owner') RETURNING id`,
-        [tenantId, body.email, passwordHash],
+        `INSERT INTO users (tenant_id, email, password_hash, role) VALUES ($1, $2, $3, $4) RETURNING id`,
+        [tenantId, body.email, passwordHash, role],
      );

      await client.query('COMMIT');
@@ -183,7 +228,7 @@ export function registerAuthRoutes(app: FastifyInstance, jwtSecret: string, pool
        tenantId,
        userId: user.rows[0].id,
        email: body.email,
-        role: 'owner',
+        role: role as AuthPayload['role'],
      }, jwtSecret);

      return reply.status(201).send({ token, tenant_id: tenantId, expires_in: '24h' });
@@ -194,7 +239,10 @@ export function registerAuthRoutes(app: FastifyInstance, jwtSecret: string, pool
      client.release();
    }
  });
+}

+/** Protected auth routes — me, api-keys. Must be registered inside an auth-protected plugin scope. */
+export function registerProtectedAuthRoutes(app: FastifyInstance, jwtSecret: string, pool: Pool) {
  app.get('/api/v1/auth/me', async (req, reply) => {
    return {
      tenant_id: (req as any).tenantId,
@@ -203,7 +251,6 @@ export function registerAuthRoutes(app: FastifyInstance, jwtSecret: string, pool
    };
  });

-  // Generate API key
  app.post('/api/v1/auth/api-keys', async (req, reply) => {
    const tenantId = (req as any).tenantId;
    const userId = (req as any).userId;
@@ -219,7 +266,53 @@ export function registerAuthRoutes(app: FastifyInstance, jwtSecret: string, pool
      [tenantId, userId, prefix, keyHash],
    );

-    // Return the raw key ONCE — it's never stored or retrievable again
    return reply.status(201).send({ api_key: rawKey, prefix });
  });
+
+  // --- Invite endpoints ---
+
+  app.post('/api/v1/auth/invite', async (req, reply) => {
+    if (!requireRole(req, reply, 'admin')) return;
+    const tenantId = (req as any).tenantId;
+    const userId = (req as any).userId;
+    const body = inviteSchema.parse(req.body);
+
+    const token = crypto.randomBytes(32).toString('hex');
+    const result = await pool.query(
+      `INSERT INTO tenant_invites (tenant_id, email, role, token, invited_by)
+       VALUES ($1, $2, $3, $4, $5)
+       RETURNING expires_at`,
+      [tenantId, body.email, body.role, token, userId],
+    );
+
+    return reply.status(201).send({ invite_token: token, expires_at: result.rows[0].expires_at });
+  });
+
+  app.get('/api/v1/auth/invites', async (req, reply) => {
+    if (!requireRole(req, reply, 'admin')) return;
+    const tenantId = (req as any).tenantId;
+
+    const result = await pool.query(
+      `SELECT id, email, role, expires_at, created_at FROM tenant_invites
+       WHERE tenant_id = $1 AND accepted_at IS NULL AND expires_at > NOW()
+       ORDER BY created_at DESC`,
+      [tenantId],
+    );
+
+    return { invites: result.rows };
+  });
+
+  app.delete('/api/v1/auth/invites/:id', async (req, reply) => {
+    if (!requireRole(req, reply, 'admin')) return;
+    const tenantId = (req as any).tenantId;
+    const { id } = req.params as { id: string };
+
+    const result = await pool.query(
+      `DELETE FROM tenant_invites WHERE id = $1 AND tenant_id = $2 AND accepted_at IS NULL RETURNING id`,
+      [id, tenantId],
+    );
+
+    if (!result.rows[0]) return reply.status(404).send({ error: 'Invite not found' });
+    return { deleted: true };
+  });
 }
--- a/products/03-alert-intelligence/src/config/index.ts
+++ b/products/03-alert-intelligence/src/config/index.ts
@@ -5,7 +5,7 @@ const envSchema = z.object({
  DATABASE_URL: z.string().default('postgresql://localhost:5432/dd0c_alert'),
  REDIS_URL: z.string().default('redis://localhost:6379'),
  JWT_SECRET: z.string().min(32).default('dev-secret-change-me-in-production!!'),
-  CORS_ORIGIN: z.string().default('*'),
+  CORS_ORIGIN: z.string().default('http://localhost:5173'),
  LOG_LEVEL: z.string().default('info'),
 });

--- a/products/03-alert-intelligence/src/data/db.ts
+++ b/products/03-alert-intelligence/src/data/db.ts
@@ -4,12 +4,11 @@ import { config } from '../config/index.js';

 const logger = pino({ name: 'data' });

-export const pool = new pg.Pool({ connectionString: config.DATABASE_URL });
+const pool = new pg.Pool({ connectionString: config.DATABASE_URL });

 /**
 * RLS tenant isolation wrapper.
 * Sets `app.tenant_id` for the duration of the callback, then resets.
- * Prevents connection pool tenant context leakage (BMad must-have).
 */
 export async function withTenant<T>(tenantId: string, fn: (client: pg.PoolClient) => Promise<T>): Promise<T> {
  const client = await pool.connect();
@@ -27,3 +26,15 @@ export async function withTenant<T>(tenantId: string, fn: (client: pg.PoolClient
    client.release();
  }
 }
+
+/** System-level queries that intentionally bypass RLS (auth, migrations, health) */
+export async function systemQuery<T extends pg.QueryResultRow = any>(
+  text: string, params?: any[]
+): Promise<pg.QueryResult<T>> {
+  return pool.query(text, params);
+}
+
+/** For auth middleware that needs direct pool access for API key lookups */
+export function getPoolForAuth(): pg.Pool {
+  return pool;
+}
--- a/products/03-alert-intelligence/src/data/redis.ts
+++ b/products/03-alert-intelligence/src/data/redis.ts
@@ -0,0 +1,9 @@
+import Redis from 'ioredis';
+import { config } from '../config/index.js';
+
+export const redis = new Redis(config.REDIS_URL, {
+  maxRetriesPerRequest: 3,
+  retryStrategy(times) {
+    return Math.min(times * 200, 3000);
+  },
+});
--- a/products/03-alert-intelligence/src/index.ts
+++ b/products/03-alert-intelligence/src/index.ts
@@ -3,12 +3,13 @@ import cors from '@fastify/cors';
 import helmet from '@fastify/helmet';
 import pino from 'pino';
 import { config } from './config/index.js';
-import { pool } from './data/db.js';
-import { registerAuth, registerAuthRoutes } from './auth/middleware.js';
+import { getPoolForAuth } from './data/db.js';
+import { authHook, decorateAuth, registerAuthRoutes, registerProtectedAuthRoutes } from './auth/middleware.js';
 import { registerWebhookRoutes } from './api/webhooks.js';
 import { registerWebhookSecretRoutes } from './api/webhook_secrets.js';
 import { registerIncidentRoutes } from './api/incidents.js';
 import { registerNotificationRoutes } from './api/notifications.js';
+import { startWebhookProcessor } from './workers/webhook-processor.js';

 const logger = pino({ name: 'dd0c-alert', level: config.LOG_LEVEL });

@@ -17,24 +18,31 @@ const app = Fastify({ logger: true });
 await app.register(cors, { origin: config.CORS_ORIGIN });
 await app.register(helmet);

-registerAuth(app, config.JWT_SECRET, pool);
+const pool = getPoolForAuth();
+decorateAuth(app);

-app.get('/health', async () => ({ status: 'ok', service: 'dd0c-alert' } /* v:c4ec43c */));
+// Public routes (no auth)
+app.get('/health', async () => ({ status: 'ok', service: 'dd0c-alert' }));
 app.get('/version', async () => ({ version: process.env.BUILD_SHA || 'dev', built: process.env.BUILD_TIME || 'unknown' }));
-
-registerAuthRoutes(app, config.JWT_SECRET, pool);
 registerWebhookRoutes(app);
-registerWebhookSecretRoutes(app);
-registerIncidentRoutes(app);
-registerNotificationRoutes(app);
+
+// Auth routes (public - login/signup)
+registerAuthRoutes(app, config.JWT_SECRET, pool);
+
+// Protected routes (auth required)
+app.register(async function protectedRoutes(protectedApp) {
+  protectedApp.addHook('onRequest', authHook(config.JWT_SECRET, pool));
+  registerProtectedAuthRoutes(protectedApp, config.JWT_SECRET, pool);
+  registerIncidentRoutes(protectedApp);
+  registerNotificationRoutes(protectedApp);
+  registerWebhookSecretRoutes(protectedApp);
+});

 try {
  await app.listen({ port: config.PORT, host: '0.0.0.0' });
  logger.info({ port: config.PORT }, 'dd0c/alert started');
+  startWebhookProcessor().catch((err) => logger.error(err, 'Webhook processor crashed'));
 } catch (err) {
  logger.fatal(err, 'Failed to start');
  process.exit(1);
 }
-// Build: 2026-03-01T06:43:58Z
-// Build: Sun Mar  1 06:47:59 UTC 2026
-// CI fix: 06:56
--- a/products/03-alert-intelligence/src/workers/webhook-processor.ts
+++ b/products/03-alert-intelligence/src/workers/webhook-processor.ts
@@ -0,0 +1,149 @@
+import pino from 'pino';
+import { redis } from '../data/redis.js';
+import { withTenant } from '../data/db.js';
+import {
+  normalizeDatadog,
+  normalizePagerduty,
+  normalizeOpsgenie,
+  type CanonicalAlert,
+} from '../ingestion/webhook.js';
+
+const logger = pino({ name: 'webhook-processor' });
+
+const INCOMING_QUEUE = 'dd0c:webhooks:incoming';
+const DEAD_LETTER_QUEUE = 'dd0c:webhooks:dead-letter';
+const MAX_RETRIES = 3;
+
+interface QueuedWebhook {
+  provider: string;
+  tenantId: string;
+  payload: any;
+  receivedAt: number;
+  retries?: number;
+}
+
+function normalizeGrafana(payload: any): CanonicalAlert {
+  const alert = payload.alerts?.[0] ?? payload;
+  return {
+    sourceProvider: 'grafana' as any,
+    sourceId: alert.fingerprint ?? crypto.randomUUID(),
+    fingerprint: alert.fingerprint ?? '',
+    title: alert.labels?.alertname ?? payload.title ?? 'Grafana Alert',
+    severity: mapGrafanaSeverity(alert.labels?.severity),
+    status: alert.status === 'resolved' ? 'resolved' : 'firing',
+    service: alert.labels?.service,
+    environment: alert.labels?.env,
+    tags: alert.labels ?? {},
+    rawPayload: payload,
+    timestamp: alert.startsAt ? new Date(alert.startsAt).getTime() : Date.now(),
+  };
+}
+
+function mapGrafanaSeverity(s: string | undefined): CanonicalAlert['severity'] {
+  switch (s) {
+    case 'critical': return 'critical';
+    case 'warning': return 'high';
+    case 'info': return 'info';
+    default: return 'medium';
+  }
+}
+
+function normalizeByProvider(provider: string, payload: any): CanonicalAlert {
+  switch (provider) {
+    case 'datadog': return normalizeDatadog(payload);
+    case 'pagerduty': return normalizePagerduty(payload);
+    case 'opsgenie': return normalizeOpsgenie(payload);
+    case 'grafana': return normalizeGrafana(payload);
+    default: throw new Error(`Unknown provider: ${provider}`);
+  }
+}
+
+async function processWebhook(item: QueuedWebhook): Promise<void> {
+  const alert = normalizeByProvider(item.provider, item.payload);
+
+  await withTenant(item.tenantId, async (client) => {
+    const alertResult = await client.query(
+      `INSERT INTO alerts (tenant_id, source_provider, source_id, fingerprint, title, severity, status, service, environment, tags, raw_payload)
+       VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
+       RETURNING id`,
+      [item.tenantId, alert.sourceProvider, alert.sourceId, alert.fingerprint, alert.title, alert.severity, alert.status, alert.service, alert.environment, JSON.stringify(alert.tags), JSON.stringify(alert.rawPayload)],
+    );
+    const alertId = alertResult.rows[0].id;
+
+    const existing = await client.query(
+      `SELECT id, alert_count FROM incidents
+       WHERE fingerprint = $1 AND status IN ('open', 'acknowledged')
+       ORDER BY created_at DESC LIMIT 1`,
+      [alert.fingerprint],
+    );
+
+    if (existing.rows[0]) {
+      await client.query('UPDATE alerts SET incident_id = $1 WHERE id = $2', [existing.rows[0].id, alertId]);
+      await client.query(
+        `UPDATE incidents SET alert_count = alert_count + 1, last_alert_at = now() WHERE id = $1`,
+        [existing.rows[0].id],
+      );
+    } else if (alert.status === 'firing') {
+      const incident = await client.query(
+        `INSERT INTO incidents (tenant_id, incident_key, fingerprint, service, title, severity, alert_count, first_alert_at, last_alert_at)
+         VALUES ($1, $2, $3, $4, $5, $6, 1, now(), now())
+         RETURNING id`,
+        [item.tenantId, `inc_${crypto.randomUUID().slice(0, 8)}`, alert.fingerprint, alert.service ?? 'unknown', alert.title, alert.severity],
+      );
+      await client.query('UPDATE alerts SET incident_id = $1 WHERE id = $2', [incident.rows[0].id, alertId]);
+    }
+
+    if (alert.status === 'resolved') {
+      await client.query(
+        `UPDATE incidents SET status = 'resolved', resolved_at = now()
+         WHERE fingerprint = $1 AND status IN ('open', 'acknowledged')`,
+        [alert.fingerprint],
+      );
+    }
+  });
+}
+
+let running = false;
+
+export async function startWebhookProcessor(): Promise<void> {
+  running = true;
+  logger.info('Webhook processor started');
+
+  while (running) {
+    try {
+      const result = await redis.brpop(INCOMING_QUEUE, 5);
+      if (!result) continue;
+
+      const [, raw] = result;
+      let item: QueuedWebhook;
+      try {
+        item = JSON.parse(raw);
+      } catch {
+        logger.error({ raw }, 'Failed to parse queued webhook');
+        await redis.lpush(DEAD_LETTER_QUEUE, raw);
+        continue;
+      }
+
+      try {
+        await processWebhook(item);
+        logger.debug({ provider: item.provider, tenantId: item.tenantId }, 'Webhook processed');
+      } catch (err) {
+        const retries = (item.retries ?? 0) + 1;
+        if (retries >= MAX_RETRIES) {
+          logger.error({ err, item }, 'Webhook processing failed, moving to dead-letter queue');
+          await redis.lpush(DEAD_LETTER_QUEUE, JSON.stringify({ ...item, retries, error: String(err) }));
+        } else {
+          logger.warn({ err, retries }, 'Webhook processing failed, retrying');
+          await redis.lpush(INCOMING_QUEUE, JSON.stringify({ ...item, retries }));
+        }
+      }
+    } catch (err) {
+      logger.error({ err }, 'Webhook processor loop error');
+      await new Promise((r) => setTimeout(r, 1000));
+    }
+  }
+}
+
+export function stopWebhookProcessor(): void {
+  running = false;
+}