feat(portal): add CloudFormation/APIGateway scanners, analytics endpoints, search caching

- CloudFormation scanner: discovers stacks and maps resources to services
- API Gateway scanner: discovers REST/HTTP APIs and routes
- Analytics API: ownership coverage, health scorecards, tech debt indicators
- Redis prefix cache for Cmd+K search (60s TTL)
- 005_analytics.sql migration for aggregation helpers

products/04-lightweight-idp/migrations/005_analytics.sql

@@ -0,0 +1,66 @@
+-- dd0c/portal analytics helpers + scanner constraint updates
+
+-- Update scan_history scanner check to include new scanner types
+ALTER TABLE scan_history DROP CONSTRAINT IF EXISTS scan_history_scanner_check;
+ALTER TABLE scan_history ADD CONSTRAINT scan_history_scanner_check
+  CHECK (scanner IN ('aws', 'github', 'cloudformation', 'apigateway'));
+
+-- Update staged_updates source check to include new sources
+ALTER TABLE staged_updates DROP CONSTRAINT IF EXISTS staged_updates_source_check;
+ALTER TABLE staged_updates ADD CONSTRAINT staged_updates_source_check
+  CHECK (source IN ('aws', 'github', 'cloudformation', 'apigateway', 'manual'));
+
+-- Materialized view: ownership coverage summary
+CREATE MATERIALIZED VIEW IF NOT EXISTS mv_ownership_coverage AS
+SELECT
+  tenant_id,
+  COUNT(*)::int AS total_services,
+  COUNT(*) FILTER (WHERE owner != 'unknown')::int AS owned_services,
+  COUNT(*) FILTER (WHERE owner = 'unknown')::int AS unowned_services,
+  ROUND(
+    (COUNT(*) FILTER (WHERE owner != 'unknown')::numeric / NULLIF(COUNT(*), 0)) * 100, 1
+  ) AS coverage_pct
+FROM services
+WHERE lifecycle = 'active'
+GROUP BY tenant_id;
+
+CREATE UNIQUE INDEX IF NOT EXISTS idx_mv_ownership_tenant ON mv_ownership_coverage(tenant_id);
+
+-- Materialized view: health scorecards by tier
+CREATE MATERIALIZED VIEW IF NOT EXISTS mv_health_by_tier AS
+SELECT
+  tenant_id,
+  tier,
+  COUNT(*)::int AS count,
+  COUNT(*) FILTER (WHERE last_discovered_at < NOW() - INTERVAL '7 days')::int AS stale_count
+FROM services
+WHERE lifecycle = 'active'
+GROUP BY tenant_id, tier;
+
+CREATE UNIQUE INDEX IF NOT EXISTS idx_mv_health_tier ON mv_health_by_tier(tenant_id, tier);
+
+-- Materialized view: tech debt indicators
+CREATE MATERIALIZED VIEW IF NOT EXISTS mv_tech_debt AS
+SELECT
+  tenant_id,
+  COUNT(*)::int AS total_active,
+  COUNT(*) FILTER (WHERE description IS NULL OR description = '')::int AS missing_description,
+  COUNT(*) FILTER (WHERE owner = 'unknown')::int AS missing_owner,
+  COUNT(*) FILTER (WHERE owner_source = 'heuristic' AND owner != 'unknown')::int AS heuristic_ownership,
+  COUNT(*) FILTER (WHERE links = '{}' OR links IS NULL)::int AS missing_links,
+  COUNT(*) FILTER (WHERE last_discovered_at IS NULL)::int AS never_discovered
+FROM services
+WHERE lifecycle = 'active'
+GROUP BY tenant_id;
+
+CREATE UNIQUE INDEX IF NOT EXISTS idx_mv_tech_debt_tenant ON mv_tech_debt(tenant_id);
+
+-- Helper function to refresh all analytics materialized views
+CREATE OR REPLACE FUNCTION refresh_analytics_views()
+RETURNS void AS $$
+BEGIN
+  REFRESH MATERIALIZED VIEW CONCURRENTLY mv_ownership_coverage;
+  REFRESH MATERIALIZED VIEW CONCURRENTLY mv_health_by_tier;
+  REFRESH MATERIALIZED VIEW CONCURRENTLY mv_tech_debt;
+END;
+$$ LANGUAGE plpgsql;

products/04-lightweight-idp/src/api/analytics.ts

@@ -0,0 +1,175 @@
+import type { FastifyInstance } from 'fastify';
+import pino from 'pino';
+import { withTenant } from '../data/db.js';
+
+const logger = pino({ name: 'api-analytics' });
+
+export function registerAnalyticsRoutes(app: FastifyInstance) {
+  // Ownership coverage stats
+  app.get('/api/v1/analytics/ownership', async (req, reply) => {
+    const tenantId = (req as any).tenantId;
+
+    const result = await withTenant(tenantId, async (client) => {
+      const total = await client.query(
+        `SELECT COUNT(*)::int AS total FROM services WHERE lifecycle = 'active'`,
+      );
+      const withOwner = await client.query(
+        `SELECT COUNT(*)::int AS count FROM services WHERE lifecycle = 'active' AND owner != 'unknown'`,
+      );
+      const byTeam = await client.query(
+        `SELECT owner, owner_source, COUNT(*)::int AS service_count,
+                array_agg(name ORDER BY name) AS services
+         FROM services
+         WHERE lifecycle = 'active' AND owner != 'unknown'
+         GROUP BY owner, owner_source
+         ORDER BY service_count DESC`,
+      );
+      const bySource = await client.query(
+        `SELECT owner_source, COUNT(*)::int AS count
+         FROM services WHERE lifecycle = 'active'
+         GROUP BY owner_source
+         ORDER BY count DESC`,
+      );
+
+      const totalCount = total.rows[0]?.total ?? 0;
+      const ownedCount = withOwner.rows[0]?.count ?? 0;
+
+      return {
+        total_services: totalCount,
+        owned_services: ownedCount,
+        unowned_services: totalCount - ownedCount,
+        coverage_pct: totalCount > 0 ? Math.round((ownedCount / totalCount) * 100 * 10) / 10 : 0,
+        by_team: byTeam.rows,
+        by_source: bySource.rows,
+      };
+    });
+
+    return result;
+  });
+
+  // Health scorecards
+  app.get('/api/v1/analytics/health', async (req, reply) => {
+    const tenantId = (req as any).tenantId;
+
+    const result = await withTenant(tenantId, async (client) => {
+      const byTier = await client.query(
+        `SELECT tier, COUNT(*)::int AS count
+         FROM services WHERE lifecycle = 'active'
+         GROUP BY tier
+         ORDER BY CASE tier
+           WHEN 'critical' THEN 1 WHEN 'high' THEN 2
+           WHEN 'medium' THEN 3 WHEN 'low' THEN 4
+         END`,
+      );
+      const byLifecycle = await client.query(
+        `SELECT lifecycle, COUNT(*)::int AS count
+         FROM services
+         GROUP BY lifecycle
+         ORDER BY count DESC`,
+      );
+      const byType = await client.query(
+        `SELECT type, COUNT(*)::int AS count
+         FROM services WHERE lifecycle = 'active'
+         GROUP BY type
+         ORDER BY count DESC`,
+      );
+      const stale = await client.query(
+        `SELECT COUNT(*)::int AS count
+         FROM services
+         WHERE lifecycle = 'active'
+           AND last_discovered_at < NOW() - INTERVAL '7 days'`,
+      );
+      const recentScans = await client.query(
+        `SELECT scanner, status, discovered, started_at, completed_at
+         FROM scan_history
+         ORDER BY started_at DESC
+         LIMIT 10`,
+      );
+
+      return {
+        by_tier: byTier.rows,
+        by_lifecycle: byLifecycle.rows,
+        by_type: byType.rows,
+        stale_services: stale.rows[0]?.count ?? 0,
+        recent_scans: recentScans.rows,
+      };
+    });
+
+    return result;
+  });
+
+  // Tech debt indicators
+  app.get('/api/v1/analytics/tech-debt', async (req, reply) => {
+    const tenantId = (req as any).tenantId;
+
+    const result = await withTenant(tenantId, async (client) => {
+      const noDescription = await client.query(
+        `SELECT COUNT(*)::int AS count
+         FROM services
+         WHERE lifecycle = 'active'
+           AND (description IS NULL OR description = '')`,
+      );
+      const noOwner = await client.query(
+        `SELECT COUNT(*)::int AS count
+         FROM services
+         WHERE lifecycle = 'active' AND owner = 'unknown'`,
+      );
+      const heuristicOnly = await client.query(
+        `SELECT COUNT(*)::int AS count
+         FROM services
+         WHERE lifecycle = 'active' AND owner_source = 'heuristic' AND owner != 'unknown'`,
+      );
+      const deprecated = await client.query(
+        `SELECT name, owner, updated_at
+         FROM services
+         WHERE lifecycle = 'deprecated'
+         ORDER BY updated_at DESC`,
+      );
+      const noLinks = await client.query(
+        `SELECT COUNT(*)::int AS count
+         FROM services
+         WHERE lifecycle = 'active'
+           AND (links = '{}' OR links IS NULL)`,
+      );
+      const neverDiscovered = await client.query(
+        `SELECT COUNT(*)::int AS count
+         FROM services
+         WHERE lifecycle = 'active' AND last_discovered_at IS NULL`,
+      );
+      const totalActive = await client.query(
+        `SELECT COUNT(*)::int AS total FROM services WHERE lifecycle = 'active'`,
+      );
+
+      const total = totalActive.rows[0]?.total ?? 0;
+
+      return {
+        total_active_services: total,
+        indicators: {
+          missing_description: {
+            count: noDescription.rows[0]?.count ?? 0,
+            pct: total > 0 ? Math.round(((noDescription.rows[0]?.count ?? 0) / total) * 100 * 10) / 10 : 0,
+          },
+          missing_owner: {
+            count: noOwner.rows[0]?.count ?? 0,
+            pct: total > 0 ? Math.round(((noOwner.rows[0]?.count ?? 0) / total) * 100 * 10) / 10 : 0,
+          },
+          heuristic_ownership: {
+            count: heuristicOnly.rows[0]?.count ?? 0,
+            pct: total > 0 ? Math.round(((heuristicOnly.rows[0]?.count ?? 0) / total) * 100 * 10) / 10 : 0,
+          },
+          missing_links: {
+            count: noLinks.rows[0]?.count ?? 0,
+            pct: total > 0 ? Math.round(((noLinks.rows[0]?.count ?? 0) / total) * 100 * 10) / 10 : 0,
+          },
+          never_discovered: {
+            count: neverDiscovered.rows[0]?.count ?? 0,
+            pct: total > 0 ? Math.round(((neverDiscovered.rows[0]?.count ?? 0) / total) * 100 * 10) / 10 : 0,
+          },
+        },
+        deprecated_services: deprecated.rows,
+      };
+    });
+
+    return result;
+  });
+}

products/04-lightweight-idp/src/api/discovery.ts

@@ -5,6 +5,8 @@ import { withTenant, getPoolForAuth } from '../data/db.js';
 import { config } from '../config/index.js';
 import { AwsDiscoveryScanner } from '../discovery/aws-scanner.js';
 import { GitHubDiscoveryScanner } from '../discovery/github-scanner.js';
+import { CloudFormationDiscoveryScanner } from '../discovery/cloudformation-scanner.js';
+import { ApiGatewayDiscoveryScanner } from '../discovery/apigateway-scanner.js';
 import { CatalogService } from '../catalog/service.js';
 import { ScheduledDiscovery } from '../discovery/scheduler.js';
 
@@ -19,7 +21,6 @@ export function registerDiscoveryRoutes(app: FastifyInstance) {
   app.post('/api/v1/discovery/aws', async (req, reply) => {
     const tenantId = (req as any).tenantId;
 
-    // Get tenant's AWS config
     const tenantConfig = await withTenant(tenantId, async (client) => {
       return client.query('SELECT * FROM tenants WHERE id = $1', [tenantId]);
     });
@@ -38,6 +39,24 @@ export function registerDiscoveryRoutes(app: FastifyInstance) {
     return reply.status(202).send({ status: result.status, discovered: result.discovered });
   });
 
+  // Trigger CloudFormation discovery scan
+  app.post('/api/v1/discovery/cloudformation', async (req, reply) => {
+    const tenantId = (req as any).tenantId;
+
+    const result = await scheduler.runScan(tenantId, 'cloudformation');
+    logger.info({ tenantId, result: result.status }, 'CloudFormation discovery scan completed');
+    return reply.status(202).send({ status: result.status, discovered: result.discovered });
+  });
+
+  // Trigger API Gateway discovery scan
+  app.post('/api/v1/discovery/apigateway', async (req, reply) => {
+    const tenantId = (req as any).tenantId;
+
+    const result = await scheduler.runScan(tenantId, 'apigateway');
+    logger.info({ tenantId, result: result.status }, 'API Gateway discovery scan completed');
+    return reply.status(202).send({ status: result.status, discovered: result.discovered });
+  });
+
   // Get scan history
   app.get('/api/v1/discovery/history', async (req, reply) => {
     const tenantId = (req as any).tenantId;

products/04-lightweight-idp/src/api/search.ts

@@ -1,12 +1,16 @@
 import type { FastifyInstance } from 'fastify';
 import { z } from 'zod';
 import { MeiliSearch } from 'meilisearch';
+import { Redis } from 'ioredis';
 import pino from 'pino';
 import { config } from '../config/index.js';
 
 const logger = pino({ name: 'api-search' });
 
 let meili: MeiliSearch | null = null;
+let redis: Redis | null = null;
+
+const SEARCH_CACHE_TTL = 60; // seconds
 
 function getMeili(): MeiliSearch {
   if (!meili) {
@@ -15,6 +19,35 @@ function getMeili(): MeiliSearch {
   return meili;
 }
 
+function getRedis(): Redis {
+  if (!redis) {
+    redis = new Redis(config.REDIS_URL);
+  }
+  return redis;
+}
+
+function cacheKey(tenantId: string, prefix: string): string {
+  return `search:${tenantId}:${prefix}`;
+}
+
+/** Invalidate all search cache entries for a tenant */
+export async function invalidateSearchCache(tenantId: string): Promise<void> {
+  try {
+    const r = getRedis();
+    const pattern = `search:${tenantId}:*`;
+    let cursor = '0';
+    do {
+      const [nextCursor, keys] = await r.scan(cursor, 'MATCH', pattern, 'COUNT', 100);
+      cursor = nextCursor;
+      if (keys.length > 0) {
+        await r.del(...keys);
+      }
+    } while (cursor !== '0');
+  } catch (err) {
+    logger.warn({ tenantId, error: (err as Error).message }, 'Failed to invalidate search cache');
+  }
+}
+
 const searchQuerySchema = z.object({
   q: z.string().min(1).max(500),
   limit: z.coerce.number().min(1).max(50).default(20),
@@ -23,11 +56,23 @@ const searchQuerySchema = z.object({
 });
 
 export function registerSearchRoutes(app: FastifyInstance) {
-  // Full-text search across services
+  // Full-text search across services (Cmd+K quick search with Redis prefix cache)
   app.get('/api/v1/search', async (req, reply) => {
     const query = searchQuerySchema.parse(req.query);
     const tenantId = (req as any).tenantId;
 
+    // Check Redis prefix cache
+    const key = cacheKey(tenantId, query.q);
+    try {
+      const cached = await getRedis().get(key);
+      if (cached) {
+        const parsed = JSON.parse(cached);
+        return { ...parsed, cached: true };
+      }
+    } catch (err) {
+      logger.warn({ error: (err as Error).message }, 'Redis cache read failed — proceeding without cache');
+    }
+
     try {
       const index = getMeili().index(`services_${tenantId}`);
       const results = await index.search(query.q, {
@@ -37,12 +82,21 @@ export function registerSearchRoutes(app: FastifyInstance) {
         attributesToHighlight: ['name', 'description', 'owner'],
       });
 
-      return {
+      const response = {
         hits: results.hits,
         total: results.estimatedTotalHits,
         query: query.q,
         processingTimeMs: results.processingTimeMs,
       };
+
+      // Cache the result
+      try {
+        await getRedis().set(key, JSON.stringify(response), 'EX', SEARCH_CACHE_TTL);
+      } catch (err) {
+        logger.warn({ error: (err as Error).message }, 'Redis cache write failed');
+      }
+
+      return response;
     } catch (err) {
       logger.warn({ error: (err as Error).message }, 'Meilisearch unavailable — falling back to PG');
 
@@ -55,7 +109,16 @@ export function registerSearchRoutes(app: FastifyInstance) {
         );
       });
 
-      return { hits: result.rows, total: result.rowCount, query: query.q, fallback: true };
+      const response = { hits: result.rows, total: result.rowCount, query: query.q, fallback: true };
+
+      // Cache fallback results too
+      try {
+        await getRedis().set(key, JSON.stringify(response), 'EX', SEARCH_CACHE_TTL);
+      } catch (cacheErr) {
+        logger.warn({ error: (cacheErr as Error).message }, 'Redis cache write failed');
+      }
+
+      return response;
     }
   });
 
@@ -71,6 +134,9 @@ export function registerSearchRoutes(app: FastifyInstance) {
     const index = getMeili().index(`services_${tenantId}`);
     await index.addDocuments(services.rows.map(s => ({ ...s, id: s.id })));
 
+    // Invalidate search cache after reindex
+    await invalidateSearchCache(tenantId);
+
     logger.info({ tenantId, count: services.rowCount }, 'Reindex triggered');
     return { status: 'reindexing', documents: services.rowCount };
   });

products/04-lightweight-idp/src/api/services.ts

@@ -1,6 +1,7 @@
 import type { FastifyInstance } from 'fastify';
 import { z } from 'zod';
 import { withTenant } from '../data/db.js';
+import { invalidateSearchCache } from './search.js';
 
 const listQuerySchema = z.object({
   page: z.coerce.number().min(1).default(1),
@@ -75,6 +76,7 @@ export function registerServiceRoutes(app: FastifyInstance) {
       );
     });
 
+    await invalidateSearchCache(tenantId);
     return reply.status(201).send({ service: result.rows[0] });
   });
 
@@ -95,6 +97,7 @@ export function registerServiceRoutes(app: FastifyInstance) {
     });
 
     if (!result.rows[0]) return reply.status(404).send({ error: 'Not found' });
+    await invalidateSearchCache(tenantId);
     return { service: result.rows[0] };
   });
 
@@ -116,6 +119,7 @@ export function registerServiceRoutes(app: FastifyInstance) {
       );
     });
 
+    await invalidateSearchCache(tenantId);
     return reply.status(201).send({ service: result.rows[0] });
   });
 
@@ -128,6 +132,7 @@ export function registerServiceRoutes(app: FastifyInstance) {
       await client.query('DELETE FROM services WHERE id = $1', [id]);
     });
 
+    await invalidateSearchCache(tenantId);
     return { status: 'deleted' };
   });
 

products/04-lightweight-idp/src/discovery/apigateway-scanner.ts

@@ -0,0 +1,203 @@
+import pino from 'pino';
+import {
+  APIGatewayClient,
+  GetRestApisCommand,
+  GetResourcesCommand,
+} from '@aws-sdk/client-api-gateway';
+import {
+  ApiGatewayV2Client,
+  GetApisCommand,
+  GetRoutesCommand,
+} from '@aws-sdk/client-apigatewayv2';
+import type { DiscoveredService, ScanResult } from './aws-scanner.js';
+
+const logger = pino({ name: 'discovery-apigateway' });
+
+export class ApiGatewayDiscoveryScanner {
+  private apigwClient: APIGatewayClient;
+  private apigwV2Client: ApiGatewayV2Client;
+  private region: string;
+
+  constructor(region: string, credentials?: any) {
+    const config = { region, ...(credentials ? { credentials } : {}) };
+    this.apigwClient = new APIGatewayClient(config);
+    this.apigwV2Client = new ApiGatewayV2Client(config);
+    this.region = region;
+  }
+
+  async scan(account: string): Promise<ScanResult> {
+    const services: DiscoveredService[] = [];
+    const errors: string[] = [];
+
+    try {
+      const restApis = await this.scanRestApis(account);
+      services.push(...restApis);
+    } catch (err) {
+      errors.push(`REST API scan failed: ${(err as Error).message}`);
+      logger.warn({ region: this.region, error: (err as Error).message }, 'REST API scan failed');
+    }
+
+    try {
+      const httpApis = await this.scanHttpApis(account);
+      services.push(...httpApis);
+    } catch (err) {
+      errors.push(`HTTP API scan failed: ${(err as Error).message}`);
+      logger.warn({ region: this.region, error: (err as Error).message }, 'HTTP API scan failed');
+    }
+
+    const status = errors.length === 0
+      ? 'success'
+      : services.length > 0
+        ? 'partial_failure'
+        : 'failed';
+
+    return { status, discovered: services.length, errors, services };
+  }
+
+  private async scanRestApis(account: string): Promise<DiscoveredService[]> {
+    const results: DiscoveredService[] = [];
+    let position: string | undefined;
+
+    do {
+      const response = await this.apigwClient.send(new GetRestApisCommand({
+        position,
+        limit: 500,
+      }));
+
+      for (const api of response.items ?? []) {
+        const tags = (api.tags ?? {}) as Record<string, string>;
+        const owner = tags['owner'] ?? tags['team'] ?? tags['Owner'] ?? tags['Team'];
+
+        let routes: string[] = [];
+        try {
+          routes = await this.getRestApiRoutes(api.id!);
+        } catch (err) {
+          logger.warn({ apiId: api.id, error: (err as Error).message }, 'Failed to list REST API resources');
+        }
+
+        const arn = `arn:aws:apigateway:${this.region}::/restapis/${api.id}`;
+
+        results.push({
+          name: api.name!,
+          type: 'apigateway-rest',
+          arn,
+          region: this.region,
+          account,
+          tags,
+          owner,
+          ownerSource: owner ? 'aws-tag' : undefined,
+          metadata: {
+            apiId: api.id,
+            description: api.description,
+            createdDate: api.createdDate,
+            endpointConfiguration: api.endpointConfiguration?.types,
+            routeCount: routes.length,
+            routes: routes.slice(0, 100),
+          },
+          discoveredAt: new Date(),
+        });
+      }
+
+      position = response.position;
+    } while (position);
+
+    return results;
+  }
+
+  private async getRestApiRoutes(restApiId: string): Promise<string[]> {
+    const routes: string[] = [];
+    let position: string | undefined;
+
+    do {
+      const response = await this.apigwClient.send(new GetResourcesCommand({
+        restApiId,
+        position,
+        limit: 500,
+      }));
+
+      for (const resource of response.items ?? []) {
+        if (resource.resourceMethods) {
+          for (const method of Object.keys(resource.resourceMethods)) {
+            routes.push(`${method} ${resource.path}`);
+          }
+        }
+      }
+
+      position = response.position;
+    } while (position);
+
+    return routes;
+  }
+
+  private async scanHttpApis(account: string): Promise<DiscoveredService[]> {
+    const results: DiscoveredService[] = [];
+    let nextToken: string | undefined;
+
+    do {
+      const response = await this.apigwV2Client.send(new GetApisCommand({
+        NextToken: nextToken,
+        MaxResults: '100',
+      }));
+
+      for (const api of response.Items ?? []) {
+        const tags = (api.Tags ?? {}) as Record<string, string>;
+        const owner = tags['owner'] ?? tags['team'] ?? tags['Owner'] ?? tags['Team'];
+
+        let routes: string[] = [];
+        try {
+          routes = await this.getHttpApiRoutes(api.ApiId!);
+        } catch (err) {
+          logger.warn({ apiId: api.ApiId, error: (err as Error).message }, 'Failed to list HTTP API routes');
+        }
+
+        const arn = `arn:aws:apigateway:${this.region}::/apis/${api.ApiId}`;
+
+        results.push({
+          name: api.Name!,
+          type: `apigateway-${(api.ProtocolType ?? 'HTTP').toLowerCase()}`,
+          arn,
+          region: this.region,
+          account,
+          tags,
+          owner,
+          ownerSource: owner ? 'aws-tag' : undefined,
+          metadata: {
+            apiId: api.ApiId,
+            description: api.Description,
+            protocolType: api.ProtocolType,
+            apiEndpoint: api.ApiEndpoint,
+            createdDate: api.CreatedDate,
+            routeCount: routes.length,
+            routes: routes.slice(0, 100),
+          },
+          discoveredAt: new Date(),
+        });
+      }
+
+      nextToken = response.NextToken;
+    } while (nextToken);
+
+    return results;
+  }
+
+  private async getHttpApiRoutes(apiId: string): Promise<string[]> {
+    const routes: string[] = [];
+    let nextToken: string | undefined;
+
+    do {
+      const response = await this.apigwV2Client.send(new GetRoutesCommand({
+        ApiId: apiId,
+        NextToken: nextToken,
+        MaxResults: '100',
+      }));
+
+      for (const route of response.Items ?? []) {
+        routes.push(route.RouteKey ?? 'unknown');
+      }
+
+      nextToken = response.NextToken;
+    } while (nextToken);
+
+    return routes;
+  }
+}

products/04-lightweight-idp/src/discovery/cloudformation-scanner.ts

@@ -0,0 +1,132 @@
+import pino from 'pino';
+import {
+  CloudFormationClient,
+  ListStacksCommand,
+  DescribeStacksCommand,
+  ListStackResourcesCommand,
+  StackStatus,
+} from '@aws-sdk/client-cloudformation';
+import type { DiscoveredService, ScanResult } from './aws-scanner.js';
+
+const logger = pino({ name: 'discovery-cloudformation' });
+
+const ACTIVE_STATUSES: string[] = [
+  StackStatus.CREATE_COMPLETE,
+  StackStatus.UPDATE_COMPLETE,
+  StackStatus.UPDATE_ROLLBACK_COMPLETE,
+];
+
+export class CloudFormationDiscoveryScanner {
+  private cfnClient: CloudFormationClient;
+  private region: string;
+
+  constructor(region: string, credentials?: any) {
+    const config = { region, ...(credentials ? { credentials } : {}) };
+    this.cfnClient = new CloudFormationClient(config);
+    this.region = region;
+  }
+
+  async scan(account: string): Promise<ScanResult> {
+    const services: DiscoveredService[] = [];
+    const errors: string[] = [];
+
+    try {
+      const stacks = await this.scanStacks(account);
+      services.push(...stacks);
+    } catch (err) {
+      errors.push(`CloudFormation scan failed: ${(err as Error).message}`);
+      logger.warn({ region: this.region, error: (err as Error).message }, 'CloudFormation scan failed');
+    }
+
+    const status = errors.length === 0
+      ? 'success'
+      : services.length > 0
+        ? 'partial_failure'
+        : 'failed';
+
+    return { status, discovered: services.length, errors, services };
+  }
+
+  private async scanStacks(account: string): Promise<DiscoveredService[]> {
+    const results: DiscoveredService[] = [];
+    let nextToken: string | undefined;
+
+    do {
+      const response = await this.cfnClient.send(new ListStacksCommand({
+        StackStatusFilter: ACTIVE_STATUSES as StackStatus[],
+        NextToken: nextToken,
+      }));
+
+      for (const summary of response.StackSummaries ?? []) {
+        try {
+          const detail = await this.describeStack(summary.StackName!);
+          if (!detail) continue;
+
+          const tags = Object.fromEntries(
+            (detail.Tags ?? []).map(t => [t.Key!, t.Value!]),
+          );
+          const owner = tags['owner'] ?? tags['team'] ?? tags['Owner'] ?? tags['Team'];
+
+          const resources = await this.listResources(summary.StackName!);
+
+          results.push({
+            name: summary.StackName!,
+            type: 'cloudformation-stack',
+            arn: summary.StackId!,
+            region: this.region,
+            account,
+            tags,
+            owner,
+            ownerSource: owner ? 'aws-tag' : undefined,
+            metadata: {
+              status: summary.StackStatus,
+              creationTime: summary.CreationTime,
+              lastUpdatedTime: summary.LastUpdatedTime,
+              templateDescription: summary.TemplateDescription,
+              resourceCount: resources.length,
+              resourceTypes: [...new Set(resources.map(r => r.type))],
+              resources: resources.slice(0, 50),
+            },
+            discoveredAt: new Date(),
+          });
+        } catch (err) {
+          logger.warn({ stack: summary.StackName, error: (err as Error).message }, 'Failed to describe stack');
+        }
+      }
+
+      nextToken = response.NextToken;
+    } while (nextToken);
+
+    return results;
+  }
+
+  private async describeStack(stackName: string) {
+    const response = await this.cfnClient.send(new DescribeStacksCommand({ StackName: stackName }));
+    return response.Stacks?.[0] ?? null;
+  }
+
+  private async listResources(stackName: string): Promise<Array<{ logicalId: string; type: string; physicalId: string; status: string }>> {
+    const resources: Array<{ logicalId: string; type: string; physicalId: string; status: string }> = [];
+    let nextToken: string | undefined;
+
+    do {
+      const response = await this.cfnClient.send(new ListStackResourcesCommand({
+        StackName: stackName,
+        NextToken: nextToken,
+      }));
+
+      for (const r of response.StackResourceSummaries ?? []) {
+        resources.push({
+          logicalId: r.LogicalResourceId!,
+          type: r.ResourceType!,
+          physicalId: r.PhysicalResourceId ?? '',
+          status: r.ResourceStatus!,
+        });
+      }
+
+      nextToken = response.NextToken;
+    } while (nextToken);
+
+    return resources;
+  }
+}

products/04-lightweight-idp/src/discovery/scheduler.ts

@@ -3,11 +3,15 @@ import { Redis } from 'ioredis';
 import { Pool } from 'pg';
 import { AwsDiscoveryScanner } from './aws-scanner.js';
 import { GitHubDiscoveryScanner } from './github-scanner.js';
+import { CloudFormationDiscoveryScanner } from './cloudformation-scanner.js';
+import { ApiGatewayDiscoveryScanner } from './apigateway-scanner.js';
 import { CatalogService } from '../catalog/service.js';
 import { withTenant } from '../data/db.js';
 
 const logger = pino({ name: 'scheduled-discovery' });
 
+export type ScannerType = 'aws' | 'github' | 'cloudformation' | 'apigateway';
+
 export class ScheduledDiscovery {
   private redis: Redis;
   private pool: Pool;
@@ -30,7 +34,7 @@ export class ScheduledDiscovery {
     await this.redis.del(key);
   }
 
-  async runScan(tenantId: string, scanner: 'aws' | 'github'): Promise<{ status: string; discovered: number }> {
+  async runScan(tenantId: string, scanner: ScannerType): Promise<{ status: string; discovered: number }> {
     const locked = await this.acquireLock(tenantId, scanner);
     if (!locked) {
       logger.info({ tenantId, scanner }, 'Scan already in progress — skipping');
@@ -50,28 +54,38 @@ export class ScheduledDiscovery {
       const catalog = new CatalogService(this.pool);
 
       if (scanner === 'aws') {
-        // Get tenant's AWS config (region, credentials would come from tenant settings)
         const awsScanner = new AwsDiscoveryScanner('us-east-1');
         const result = await awsScanner.scan(tenantId);
         const isPartial = result.status === 'partial_failure';
-
         const merged = await catalog.mergeAwsDiscovery(tenantId, result.services, isPartial);
-
         await this.recordScanResult(tenantId, scanner, result.status, result.discovered, result.errors);
         return { status: result.status, discovered: merged };
-      } else {
+      } else if (scanner === 'github') {
-        // GitHub scan — would need org name + token from tenant settings
         const { Octokit } = await import('@octokit/rest');
-        const octokit = new Octokit(); // Would use tenant's GitHub token
+        const octokit = new Octokit();
         const ghScanner = new GitHubDiscoveryScanner(octokit);
-        const result = await ghScanner.scan(tenantId); // Would use tenant's org name
+        const result = await ghScanner.scan(tenantId);
         const isPartial = result.status === 'partial_failure';
-
         const merged = await catalog.mergeGitHubDiscovery(tenantId, result.repos, isPartial);
-
+        await this.recordScanResult(tenantId, scanner, result.status, result.discovered, result.errors);
+        return { status: result.status, discovered: merged };
+      } else if (scanner === 'cloudformation') {
+        const cfnScanner = new CloudFormationDiscoveryScanner('us-east-1');
+        const result = await cfnScanner.scan(tenantId);
+        const isPartial = result.status === 'partial_failure';
+        const merged = await catalog.mergeAwsDiscovery(tenantId, result.services, isPartial);
+        await this.recordScanResult(tenantId, scanner, result.status, result.discovered, result.errors);
+        return { status: result.status, discovered: merged };
+      } else if (scanner === 'apigateway') {
+        const apigwScanner = new ApiGatewayDiscoveryScanner('us-east-1');
+        const result = await apigwScanner.scan(tenantId);
+        const isPartial = result.status === 'partial_failure';
+        const merged = await catalog.mergeAwsDiscovery(tenantId, result.services, isPartial);
         await this.recordScanResult(tenantId, scanner, result.status, result.discovered, result.errors);
         return { status: result.status, discovered: merged };
       }
+
+      return { status: 'failed', discovered: 0 };
     } catch (err) {
       logger.error({ tenantId, scanner, error: (err as Error).message }, 'Scheduled scan failed');
       await this.recordScanResult(tenantId, scanner, 'failed', 0, [(err as Error).message]);

products/04-lightweight-idp/src/index.ts

@@ -9,6 +9,7 @@ import { authHook, decorateAuth, registerAuthRoutes, registerProtectedAuthRoutes
 import { registerServiceRoutes } from './api/services.js';
 import { registerDiscoveryRoutes } from './api/discovery.js';
 import { registerSearchRoutes } from './api/search.js';
+import { registerAnalyticsRoutes } from './api/analytics.js';
 
 const logger = pino({ name: 'dd0c-portal', level: config.LOG_LEVEL });
 
@@ -35,6 +36,7 @@ app.register(async function protectedRoutes(protectedApp) {
   registerServiceRoutes(protectedApp);
   registerDiscoveryRoutes(protectedApp);
   registerSearchRoutes(protectedApp);
+  registerAnalyticsRoutes(protectedApp);
 });
 
 try {