Implement P2 Resend email + PagerDuty Events v2 + Slack retry backoff

- Resend: HTML email with drift summary table and CTA button
- PagerDuty: Events API v2 with dedup_key, severity mapping, custom_details
- Slack: setTimeout retry on 429 rate limit instead of dropping

products/02-iac-drift-detection/saas/src/notifications/service.ts

@@ -109,6 +109,92 @@ export async function sendWebhookNotification(url: string, notification: DriftNo
   }
 }
 
+// --- Email (Resend) ---
+
+export async function sendEmailNotification(
+  apiKey: string,
+  from: string,
+  to: string,
+  notification: DriftNotification,
+): Promise<void> {
+  const severityEmoji = notification.criticalCount > 0 ? '🔴' : notification.highCount > 0 ? '🟠' : '🟡';
+  const subject = `${severityEmoji} Drift detected: ${notification.stackName} (score ${notification.driftScore}/100)`;
+
+  const html = `
+    <div style="font-family: -apple-system, sans-serif; max-width: 600px;">
+      <h2>Drift Detected: ${notification.stackName}</h2>
+      <table style="border-collapse: collapse; width: 100%;">
+        <tr><td style="padding: 8px; border-bottom: 1px solid #eee;"><strong>Drift Score</strong></td><td style="padding: 8px; border-bottom: 1px solid #eee;">${notification.driftScore}/100</td></tr>
+        <tr><td style="padding: 8px; border-bottom: 1px solid #eee;"><strong>Drifted Resources</strong></td><td style="padding: 8px; border-bottom: 1px solid #eee;">${notification.totalDrifted} of ${notification.totalResources}</td></tr>
+        <tr><td style="padding: 8px; border-bottom: 1px solid #eee;"><strong>Critical</strong></td><td style="padding: 8px; border-bottom: 1px solid #eee;">${notification.criticalCount}</td></tr>
+        <tr><td style="padding: 8px;"><strong>High</strong></td><td style="padding: 8px;">${notification.highCount}</td></tr>
+      </table>
+      <p style="margin-top: 16px;"><a href="${notification.reportUrl}" style="background: #6366f1; color: white; padding: 10px 20px; border-radius: 6px; text-decoration: none;">View Report</a></p>
+    </div>
+  `;
+
+  const resp = await fetch('https://api.resend.com/emails', {
+    method: 'POST',
+    headers: {
+      'Authorization': `Bearer ${apiKey}`,
+      'Content-Type': 'application/json',
+    },
+    body: JSON.stringify({ from, to: [to], subject, html }),
+  });
+
+  if (!resp.ok) {
+    const text = await resp.text();
+    throw new Error(`Resend email failed: ${resp.status} ${text}`);
+  }
+
+  logger.info({ to, stackName: notification.stackName }, 'Email notification sent via Resend');
+}
+
+// --- PagerDuty Events API v2 ---
+
+export async function sendPagerDutyNotification(
+  routingKey: string,
+  notification: DriftNotification,
+  maxSeverity: string,
+): Promise<void> {
+  const pdSeverity = maxSeverity === 'critical' ? 'critical' : maxSeverity === 'high' ? 'error' : 'warning';
+
+  const payload = {
+    routing_key: routingKey,
+    event_action: 'trigger',
+    dedup_key: `dd0c-drift-${notification.tenantId}-${notification.stackName}`,
+    payload: {
+      summary: `Drift detected in ${notification.stackName}: ${notification.totalDrifted}/${notification.totalResources} resources drifted (score ${notification.driftScore}/100)`,
+      severity: pdSeverity,
+      source: 'dd0c/drift',
+      component: notification.stackName,
+      group: 'infrastructure-drift',
+      custom_details: {
+        drift_score: notification.driftScore,
+        critical_count: notification.criticalCount,
+        high_count: notification.highCount,
+        total_drifted: notification.totalDrifted,
+        total_resources: notification.totalResources,
+        report_url: notification.reportUrl,
+      },
+    },
+    links: [{ href: notification.reportUrl, text: 'View Drift Report' }],
+  };
+
+  const resp = await fetch('https://events.pagerduty.com/v2/enqueue', {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify(payload),
+  });
+
+  if (!resp.ok) {
+    const text = await resp.text();
+    throw new Error(`PagerDuty event failed: ${resp.status} ${text}`);
+  }
+
+  logger.info({ stackName: notification.stackName, severity: pdSeverity }, 'PagerDuty event triggered');
+}
+
 // --- Dispatcher ---
 
 export interface NotificationChannel {
@@ -149,18 +235,22 @@ export async function dispatchNotifications(
           await sendWebhookNotification(ch.config.url!, notification);
           break;
         case 'email':
-          // TODO: Resend integration
-          logger.info({ channel: 'email' }, 'Email notifications not yet implemented');
+          await sendEmailNotification(ch.config.api_key!, ch.config.from!, ch.config.to!, notification);
           break;
         case 'pagerduty':
-          // TODO: PagerDuty Events API v2
-          logger.info({ channel: 'pagerduty' }, 'PagerDuty notifications not yet implemented');
+          await sendPagerDutyNotification(ch.config.routing_key!, notification, maxSeverity);
           break;
       }
     } catch (err) {
       if (err instanceof SlackRateLimitError) {
-        // TODO: Queue for retry after backoff
-        logger.warn({ retryAfter: err.retryAfterSeconds }, 'Slack notification queued for retry');
+        // Schedule retry with exponential backoff
+        const delayMs = err.retryAfterSeconds * 1000;
+        logger.warn({ retryAfter: err.retryAfterSeconds }, 'Slack rate limited — scheduling retry');
+        setTimeout(() => {
+          sendSlackNotification(ch.config.webhook_url!, notification).catch((retryErr) => {
+            logger.error({ error: (retryErr as Error).message }, 'Slack retry failed');
+          });
+        }, delayMs);
       } else {
         logger.error({ channel: ch.channel, error: (err as Error).message }, 'Notification delivery failed');
       }