From cfe269a0313ae9c376b4970d9ce09835d4e16ad6 Mon Sep 17 00:00:00 2001 From: Max Date: Tue, 3 Mar 2026 06:36:24 +0000 Subject: [PATCH] feat(portal): add CloudFormation/APIGateway scanners, analytics endpoints, search caching - CloudFormation scanner: discovers stacks and maps resources to services - API Gateway scanner: discovers REST/HTTP APIs and routes - Analytics API: ownership coverage, health scorecards, tech debt indicators - Redis prefix cache for Cmd+K search (60s TTL) - 005_analytics.sql migration for aggregation helpers --- .../migrations/005_analytics.sql | 66 ++++++ .../04-lightweight-idp/src/api/analytics.ts | 175 +++++++++++++++ .../04-lightweight-idp/src/api/discovery.ts | 21 +- products/04-lightweight-idp/src/api/search.ts | 72 ++++++- .../04-lightweight-idp/src/api/services.ts | 5 + .../src/discovery/apigateway-scanner.ts | 203 ++++++++++++++++++ .../src/discovery/cloudformation-scanner.ts | 132 ++++++++++++ .../src/discovery/scheduler.ts | 34 ++- products/04-lightweight-idp/src/index.ts | 2 + 9 files changed, 696 insertions(+), 14 deletions(-) create mode 100644 products/04-lightweight-idp/migrations/005_analytics.sql create mode 100644 products/04-lightweight-idp/src/api/analytics.ts create mode 100644 products/04-lightweight-idp/src/discovery/apigateway-scanner.ts create mode 100644 products/04-lightweight-idp/src/discovery/cloudformation-scanner.ts diff --git a/products/04-lightweight-idp/migrations/005_analytics.sql b/products/04-lightweight-idp/migrations/005_analytics.sql new file mode 100644 index 0000000..e1ddad2 --- /dev/null +++ b/products/04-lightweight-idp/migrations/005_analytics.sql @@ -0,0 +1,66 @@ +-- dd0c/portal analytics helpers + scanner constraint updates + +-- Update scan_history scanner check to include new scanner types +ALTER TABLE scan_history DROP CONSTRAINT IF EXISTS scan_history_scanner_check; +ALTER TABLE scan_history ADD CONSTRAINT scan_history_scanner_check + CHECK (scanner IN ('aws', 'github', 'cloudformation', 'apigateway')); + +-- Update staged_updates source check to include new sources +ALTER TABLE staged_updates DROP CONSTRAINT IF EXISTS staged_updates_source_check; +ALTER TABLE staged_updates ADD CONSTRAINT staged_updates_source_check + CHECK (source IN ('aws', 'github', 'cloudformation', 'apigateway', 'manual')); + +-- Materialized view: ownership coverage summary +CREATE MATERIALIZED VIEW IF NOT EXISTS mv_ownership_coverage AS +SELECT + tenant_id, + COUNT(*)::int AS total_services, + COUNT(*) FILTER (WHERE owner != 'unknown')::int AS owned_services, + COUNT(*) FILTER (WHERE owner = 'unknown')::int AS unowned_services, + ROUND( + (COUNT(*) FILTER (WHERE owner != 'unknown')::numeric / NULLIF(COUNT(*), 0)) * 100, 1 + ) AS coverage_pct +FROM services +WHERE lifecycle = 'active' +GROUP BY tenant_id; + +CREATE UNIQUE INDEX IF NOT EXISTS idx_mv_ownership_tenant ON mv_ownership_coverage(tenant_id); + +-- Materialized view: health scorecards by tier +CREATE MATERIALIZED VIEW IF NOT EXISTS mv_health_by_tier AS +SELECT + tenant_id, + tier, + COUNT(*)::int AS count, + COUNT(*) FILTER (WHERE last_discovered_at < NOW() - INTERVAL '7 days')::int AS stale_count +FROM services +WHERE lifecycle = 'active' +GROUP BY tenant_id, tier; + +CREATE UNIQUE INDEX IF NOT EXISTS idx_mv_health_tier ON mv_health_by_tier(tenant_id, tier); + +-- Materialized view: tech debt indicators +CREATE MATERIALIZED VIEW IF NOT EXISTS mv_tech_debt AS +SELECT + tenant_id, + COUNT(*)::int AS total_active, + COUNT(*) FILTER (WHERE description IS NULL OR description = '')::int AS missing_description, + COUNT(*) FILTER (WHERE owner = 'unknown')::int AS missing_owner, + COUNT(*) FILTER (WHERE owner_source = 'heuristic' AND owner != 'unknown')::int AS heuristic_ownership, + COUNT(*) FILTER (WHERE links = '{}' OR links IS NULL)::int AS missing_links, + COUNT(*) FILTER (WHERE last_discovered_at IS NULL)::int AS never_discovered +FROM services +WHERE lifecycle = 'active' +GROUP BY tenant_id; + +CREATE UNIQUE INDEX IF NOT EXISTS idx_mv_tech_debt_tenant ON mv_tech_debt(tenant_id); + +-- Helper function to refresh all analytics materialized views +CREATE OR REPLACE FUNCTION refresh_analytics_views() +RETURNS void AS $$ +BEGIN + REFRESH MATERIALIZED VIEW CONCURRENTLY mv_ownership_coverage; + REFRESH MATERIALIZED VIEW CONCURRENTLY mv_health_by_tier; + REFRESH MATERIALIZED VIEW CONCURRENTLY mv_tech_debt; +END; +$$ LANGUAGE plpgsql; diff --git a/products/04-lightweight-idp/src/api/analytics.ts b/products/04-lightweight-idp/src/api/analytics.ts new file mode 100644 index 0000000..7f70a19 --- /dev/null +++ b/products/04-lightweight-idp/src/api/analytics.ts @@ -0,0 +1,175 @@ +import type { FastifyInstance } from 'fastify'; +import pino from 'pino'; +import { withTenant } from '../data/db.js'; + +const logger = pino({ name: 'api-analytics' }); + +export function registerAnalyticsRoutes(app: FastifyInstance) { + // Ownership coverage stats + app.get('/api/v1/analytics/ownership', async (req, reply) => { + const tenantId = (req as any).tenantId; + + const result = await withTenant(tenantId, async (client) => { + const total = await client.query( + `SELECT COUNT(*)::int AS total FROM services WHERE lifecycle = 'active'`, + ); + const withOwner = await client.query( + `SELECT COUNT(*)::int AS count FROM services WHERE lifecycle = 'active' AND owner != 'unknown'`, + ); + const byTeam = await client.query( + `SELECT owner, owner_source, COUNT(*)::int AS service_count, + array_agg(name ORDER BY name) AS services + FROM services + WHERE lifecycle = 'active' AND owner != 'unknown' + GROUP BY owner, owner_source + ORDER BY service_count DESC`, + ); + const bySource = await client.query( + `SELECT owner_source, COUNT(*)::int AS count + FROM services WHERE lifecycle = 'active' + GROUP BY owner_source + ORDER BY count DESC`, + ); + + const totalCount = total.rows[0]?.total ?? 0; + const ownedCount = withOwner.rows[0]?.count ?? 0; + + return { + total_services: totalCount, + owned_services: ownedCount, + unowned_services: totalCount - ownedCount, + coverage_pct: totalCount > 0 ? Math.round((ownedCount / totalCount) * 100 * 10) / 10 : 0, + by_team: byTeam.rows, + by_source: bySource.rows, + }; + }); + + return result; + }); + + // Health scorecards + app.get('/api/v1/analytics/health', async (req, reply) => { + const tenantId = (req as any).tenantId; + + const result = await withTenant(tenantId, async (client) => { + const byTier = await client.query( + `SELECT tier, COUNT(*)::int AS count + FROM services WHERE lifecycle = 'active' + GROUP BY tier + ORDER BY CASE tier + WHEN 'critical' THEN 1 WHEN 'high' THEN 2 + WHEN 'medium' THEN 3 WHEN 'low' THEN 4 + END`, + ); + const byLifecycle = await client.query( + `SELECT lifecycle, COUNT(*)::int AS count + FROM services + GROUP BY lifecycle + ORDER BY count DESC`, + ); + const byType = await client.query( + `SELECT type, COUNT(*)::int AS count + FROM services WHERE lifecycle = 'active' + GROUP BY type + ORDER BY count DESC`, + ); + const stale = await client.query( + `SELECT COUNT(*)::int AS count + FROM services + WHERE lifecycle = 'active' + AND last_discovered_at < NOW() - INTERVAL '7 days'`, + ); + const recentScans = await client.query( + `SELECT scanner, status, discovered, started_at, completed_at + FROM scan_history + ORDER BY started_at DESC + LIMIT 10`, + ); + + return { + by_tier: byTier.rows, + by_lifecycle: byLifecycle.rows, + by_type: byType.rows, + stale_services: stale.rows[0]?.count ?? 0, + recent_scans: recentScans.rows, + }; + }); + + return result; + }); + + // Tech debt indicators + app.get('/api/v1/analytics/tech-debt', async (req, reply) => { + const tenantId = (req as any).tenantId; + + const result = await withTenant(tenantId, async (client) => { + const noDescription = await client.query( + `SELECT COUNT(*)::int AS count + FROM services + WHERE lifecycle = 'active' + AND (description IS NULL OR description = '')`, + ); + const noOwner = await client.query( + `SELECT COUNT(*)::int AS count + FROM services + WHERE lifecycle = 'active' AND owner = 'unknown'`, + ); + const heuristicOnly = await client.query( + `SELECT COUNT(*)::int AS count + FROM services + WHERE lifecycle = 'active' AND owner_source = 'heuristic' AND owner != 'unknown'`, + ); + const deprecated = await client.query( + `SELECT name, owner, updated_at + FROM services + WHERE lifecycle = 'deprecated' + ORDER BY updated_at DESC`, + ); + const noLinks = await client.query( + `SELECT COUNT(*)::int AS count + FROM services + WHERE lifecycle = 'active' + AND (links = '{}' OR links IS NULL)`, + ); + const neverDiscovered = await client.query( + `SELECT COUNT(*)::int AS count + FROM services + WHERE lifecycle = 'active' AND last_discovered_at IS NULL`, + ); + const totalActive = await client.query( + `SELECT COUNT(*)::int AS total FROM services WHERE lifecycle = 'active'`, + ); + + const total = totalActive.rows[0]?.total ?? 0; + + return { + total_active_services: total, + indicators: { + missing_description: { + count: noDescription.rows[0]?.count ?? 0, + pct: total > 0 ? Math.round(((noDescription.rows[0]?.count ?? 0) / total) * 100 * 10) / 10 : 0, + }, + missing_owner: { + count: noOwner.rows[0]?.count ?? 0, + pct: total > 0 ? Math.round(((noOwner.rows[0]?.count ?? 0) / total) * 100 * 10) / 10 : 0, + }, + heuristic_ownership: { + count: heuristicOnly.rows[0]?.count ?? 0, + pct: total > 0 ? Math.round(((heuristicOnly.rows[0]?.count ?? 0) / total) * 100 * 10) / 10 : 0, + }, + missing_links: { + count: noLinks.rows[0]?.count ?? 0, + pct: total > 0 ? Math.round(((noLinks.rows[0]?.count ?? 0) / total) * 100 * 10) / 10 : 0, + }, + never_discovered: { + count: neverDiscovered.rows[0]?.count ?? 0, + pct: total > 0 ? Math.round(((neverDiscovered.rows[0]?.count ?? 0) / total) * 100 * 10) / 10 : 0, + }, + }, + deprecated_services: deprecated.rows, + }; + }); + + return result; + }); +} diff --git a/products/04-lightweight-idp/src/api/discovery.ts b/products/04-lightweight-idp/src/api/discovery.ts index 3ab7a0f..4e90425 100644 --- a/products/04-lightweight-idp/src/api/discovery.ts +++ b/products/04-lightweight-idp/src/api/discovery.ts @@ -5,6 +5,8 @@ import { withTenant, getPoolForAuth } from '../data/db.js'; import { config } from '../config/index.js'; import { AwsDiscoveryScanner } from '../discovery/aws-scanner.js'; import { GitHubDiscoveryScanner } from '../discovery/github-scanner.js'; +import { CloudFormationDiscoveryScanner } from '../discovery/cloudformation-scanner.js'; +import { ApiGatewayDiscoveryScanner } from '../discovery/apigateway-scanner.js'; import { CatalogService } from '../catalog/service.js'; import { ScheduledDiscovery } from '../discovery/scheduler.js'; @@ -19,7 +21,6 @@ export function registerDiscoveryRoutes(app: FastifyInstance) { app.post('/api/v1/discovery/aws', async (req, reply) => { const tenantId = (req as any).tenantId; - // Get tenant's AWS config const tenantConfig = await withTenant(tenantId, async (client) => { return client.query('SELECT * FROM tenants WHERE id = $1', [tenantId]); }); @@ -38,6 +39,24 @@ export function registerDiscoveryRoutes(app: FastifyInstance) { return reply.status(202).send({ status: result.status, discovered: result.discovered }); }); + // Trigger CloudFormation discovery scan + app.post('/api/v1/discovery/cloudformation', async (req, reply) => { + const tenantId = (req as any).tenantId; + + const result = await scheduler.runScan(tenantId, 'cloudformation'); + logger.info({ tenantId, result: result.status }, 'CloudFormation discovery scan completed'); + return reply.status(202).send({ status: result.status, discovered: result.discovered }); + }); + + // Trigger API Gateway discovery scan + app.post('/api/v1/discovery/apigateway', async (req, reply) => { + const tenantId = (req as any).tenantId; + + const result = await scheduler.runScan(tenantId, 'apigateway'); + logger.info({ tenantId, result: result.status }, 'API Gateway discovery scan completed'); + return reply.status(202).send({ status: result.status, discovered: result.discovered }); + }); + // Get scan history app.get('/api/v1/discovery/history', async (req, reply) => { const tenantId = (req as any).tenantId; diff --git a/products/04-lightweight-idp/src/api/search.ts b/products/04-lightweight-idp/src/api/search.ts index fcfa4c1..3cfa232 100644 --- a/products/04-lightweight-idp/src/api/search.ts +++ b/products/04-lightweight-idp/src/api/search.ts @@ -1,12 +1,16 @@ import type { FastifyInstance } from 'fastify'; import { z } from 'zod'; import { MeiliSearch } from 'meilisearch'; +import { Redis } from 'ioredis'; import pino from 'pino'; import { config } from '../config/index.js'; const logger = pino({ name: 'api-search' }); let meili: MeiliSearch | null = null; +let redis: Redis | null = null; + +const SEARCH_CACHE_TTL = 60; // seconds function getMeili(): MeiliSearch { if (!meili) { @@ -15,6 +19,35 @@ function getMeili(): MeiliSearch { return meili; } +function getRedis(): Redis { + if (!redis) { + redis = new Redis(config.REDIS_URL); + } + return redis; +} + +function cacheKey(tenantId: string, prefix: string): string { + return `search:${tenantId}:${prefix}`; +} + +/** Invalidate all search cache entries for a tenant */ +export async function invalidateSearchCache(tenantId: string): Promise { + try { + const r = getRedis(); + const pattern = `search:${tenantId}:*`; + let cursor = '0'; + do { + const [nextCursor, keys] = await r.scan(cursor, 'MATCH', pattern, 'COUNT', 100); + cursor = nextCursor; + if (keys.length > 0) { + await r.del(...keys); + } + } while (cursor !== '0'); + } catch (err) { + logger.warn({ tenantId, error: (err as Error).message }, 'Failed to invalidate search cache'); + } +} + const searchQuerySchema = z.object({ q: z.string().min(1).max(500), limit: z.coerce.number().min(1).max(50).default(20), @@ -23,11 +56,23 @@ const searchQuerySchema = z.object({ }); export function registerSearchRoutes(app: FastifyInstance) { - // Full-text search across services + // Full-text search across services (Cmd+K quick search with Redis prefix cache) app.get('/api/v1/search', async (req, reply) => { const query = searchQuerySchema.parse(req.query); const tenantId = (req as any).tenantId; + // Check Redis prefix cache + const key = cacheKey(tenantId, query.q); + try { + const cached = await getRedis().get(key); + if (cached) { + const parsed = JSON.parse(cached); + return { ...parsed, cached: true }; + } + } catch (err) { + logger.warn({ error: (err as Error).message }, 'Redis cache read failed — proceeding without cache'); + } + try { const index = getMeili().index(`services_${tenantId}`); const results = await index.search(query.q, { @@ -37,12 +82,21 @@ export function registerSearchRoutes(app: FastifyInstance) { attributesToHighlight: ['name', 'description', 'owner'], }); - return { + const response = { hits: results.hits, total: results.estimatedTotalHits, query: query.q, processingTimeMs: results.processingTimeMs, }; + + // Cache the result + try { + await getRedis().set(key, JSON.stringify(response), 'EX', SEARCH_CACHE_TTL); + } catch (err) { + logger.warn({ error: (err as Error).message }, 'Redis cache write failed'); + } + + return response; } catch (err) { logger.warn({ error: (err as Error).message }, 'Meilisearch unavailable — falling back to PG'); @@ -55,7 +109,16 @@ export function registerSearchRoutes(app: FastifyInstance) { ); }); - return { hits: result.rows, total: result.rowCount, query: query.q, fallback: true }; + const response = { hits: result.rows, total: result.rowCount, query: query.q, fallback: true }; + + // Cache fallback results too + try { + await getRedis().set(key, JSON.stringify(response), 'EX', SEARCH_CACHE_TTL); + } catch (cacheErr) { + logger.warn({ error: (cacheErr as Error).message }, 'Redis cache write failed'); + } + + return response; } }); @@ -71,6 +134,9 @@ export function registerSearchRoutes(app: FastifyInstance) { const index = getMeili().index(`services_${tenantId}`); await index.addDocuments(services.rows.map(s => ({ ...s, id: s.id }))); + // Invalidate search cache after reindex + await invalidateSearchCache(tenantId); + logger.info({ tenantId, count: services.rowCount }, 'Reindex triggered'); return { status: 'reindexing', documents: services.rowCount }; }); diff --git a/products/04-lightweight-idp/src/api/services.ts b/products/04-lightweight-idp/src/api/services.ts index 8adf1d1..00947de 100644 --- a/products/04-lightweight-idp/src/api/services.ts +++ b/products/04-lightweight-idp/src/api/services.ts @@ -1,6 +1,7 @@ import type { FastifyInstance } from 'fastify'; import { z } from 'zod'; import { withTenant } from '../data/db.js'; +import { invalidateSearchCache } from './search.js'; const listQuerySchema = z.object({ page: z.coerce.number().min(1).default(1), @@ -75,6 +76,7 @@ export function registerServiceRoutes(app: FastifyInstance) { ); }); + await invalidateSearchCache(tenantId); return reply.status(201).send({ service: result.rows[0] }); }); @@ -95,6 +97,7 @@ export function registerServiceRoutes(app: FastifyInstance) { }); if (!result.rows[0]) return reply.status(404).send({ error: 'Not found' }); + await invalidateSearchCache(tenantId); return { service: result.rows[0] }; }); @@ -116,6 +119,7 @@ export function registerServiceRoutes(app: FastifyInstance) { ); }); + await invalidateSearchCache(tenantId); return reply.status(201).send({ service: result.rows[0] }); }); @@ -128,6 +132,7 @@ export function registerServiceRoutes(app: FastifyInstance) { await client.query('DELETE FROM services WHERE id = $1', [id]); }); + await invalidateSearchCache(tenantId); return { status: 'deleted' }; }); diff --git a/products/04-lightweight-idp/src/discovery/apigateway-scanner.ts b/products/04-lightweight-idp/src/discovery/apigateway-scanner.ts new file mode 100644 index 0000000..3164dbb --- /dev/null +++ b/products/04-lightweight-idp/src/discovery/apigateway-scanner.ts @@ -0,0 +1,203 @@ +import pino from 'pino'; +import { + APIGatewayClient, + GetRestApisCommand, + GetResourcesCommand, +} from '@aws-sdk/client-api-gateway'; +import { + ApiGatewayV2Client, + GetApisCommand, + GetRoutesCommand, +} from '@aws-sdk/client-apigatewayv2'; +import type { DiscoveredService, ScanResult } from './aws-scanner.js'; + +const logger = pino({ name: 'discovery-apigateway' }); + +export class ApiGatewayDiscoveryScanner { + private apigwClient: APIGatewayClient; + private apigwV2Client: ApiGatewayV2Client; + private region: string; + + constructor(region: string, credentials?: any) { + const config = { region, ...(credentials ? { credentials } : {}) }; + this.apigwClient = new APIGatewayClient(config); + this.apigwV2Client = new ApiGatewayV2Client(config); + this.region = region; + } + + async scan(account: string): Promise { + const services: DiscoveredService[] = []; + const errors: string[] = []; + + try { + const restApis = await this.scanRestApis(account); + services.push(...restApis); + } catch (err) { + errors.push(`REST API scan failed: ${(err as Error).message}`); + logger.warn({ region: this.region, error: (err as Error).message }, 'REST API scan failed'); + } + + try { + const httpApis = await this.scanHttpApis(account); + services.push(...httpApis); + } catch (err) { + errors.push(`HTTP API scan failed: ${(err as Error).message}`); + logger.warn({ region: this.region, error: (err as Error).message }, 'HTTP API scan failed'); + } + + const status = errors.length === 0 + ? 'success' + : services.length > 0 + ? 'partial_failure' + : 'failed'; + + return { status, discovered: services.length, errors, services }; + } + + private async scanRestApis(account: string): Promise { + const results: DiscoveredService[] = []; + let position: string | undefined; + + do { + const response = await this.apigwClient.send(new GetRestApisCommand({ + position, + limit: 500, + })); + + for (const api of response.items ?? []) { + const tags = (api.tags ?? {}) as Record; + const owner = tags['owner'] ?? tags['team'] ?? tags['Owner'] ?? tags['Team']; + + let routes: string[] = []; + try { + routes = await this.getRestApiRoutes(api.id!); + } catch (err) { + logger.warn({ apiId: api.id, error: (err as Error).message }, 'Failed to list REST API resources'); + } + + const arn = `arn:aws:apigateway:${this.region}::/restapis/${api.id}`; + + results.push({ + name: api.name!, + type: 'apigateway-rest', + arn, + region: this.region, + account, + tags, + owner, + ownerSource: owner ? 'aws-tag' : undefined, + metadata: { + apiId: api.id, + description: api.description, + createdDate: api.createdDate, + endpointConfiguration: api.endpointConfiguration?.types, + routeCount: routes.length, + routes: routes.slice(0, 100), + }, + discoveredAt: new Date(), + }); + } + + position = response.position; + } while (position); + + return results; + } + + private async getRestApiRoutes(restApiId: string): Promise { + const routes: string[] = []; + let position: string | undefined; + + do { + const response = await this.apigwClient.send(new GetResourcesCommand({ + restApiId, + position, + limit: 500, + })); + + for (const resource of response.items ?? []) { + if (resource.resourceMethods) { + for (const method of Object.keys(resource.resourceMethods)) { + routes.push(`${method} ${resource.path}`); + } + } + } + + position = response.position; + } while (position); + + return routes; + } + + private async scanHttpApis(account: string): Promise { + const results: DiscoveredService[] = []; + let nextToken: string | undefined; + + do { + const response = await this.apigwV2Client.send(new GetApisCommand({ + NextToken: nextToken, + MaxResults: '100', + })); + + for (const api of response.Items ?? []) { + const tags = (api.Tags ?? {}) as Record; + const owner = tags['owner'] ?? tags['team'] ?? tags['Owner'] ?? tags['Team']; + + let routes: string[] = []; + try { + routes = await this.getHttpApiRoutes(api.ApiId!); + } catch (err) { + logger.warn({ apiId: api.ApiId, error: (err as Error).message }, 'Failed to list HTTP API routes'); + } + + const arn = `arn:aws:apigateway:${this.region}::/apis/${api.ApiId}`; + + results.push({ + name: api.Name!, + type: `apigateway-${(api.ProtocolType ?? 'HTTP').toLowerCase()}`, + arn, + region: this.region, + account, + tags, + owner, + ownerSource: owner ? 'aws-tag' : undefined, + metadata: { + apiId: api.ApiId, + description: api.Description, + protocolType: api.ProtocolType, + apiEndpoint: api.ApiEndpoint, + createdDate: api.CreatedDate, + routeCount: routes.length, + routes: routes.slice(0, 100), + }, + discoveredAt: new Date(), + }); + } + + nextToken = response.NextToken; + } while (nextToken); + + return results; + } + + private async getHttpApiRoutes(apiId: string): Promise { + const routes: string[] = []; + let nextToken: string | undefined; + + do { + const response = await this.apigwV2Client.send(new GetRoutesCommand({ + ApiId: apiId, + NextToken: nextToken, + MaxResults: '100', + })); + + for (const route of response.Items ?? []) { + routes.push(route.RouteKey ?? 'unknown'); + } + + nextToken = response.NextToken; + } while (nextToken); + + return routes; + } +} diff --git a/products/04-lightweight-idp/src/discovery/cloudformation-scanner.ts b/products/04-lightweight-idp/src/discovery/cloudformation-scanner.ts new file mode 100644 index 0000000..dcb9b9f --- /dev/null +++ b/products/04-lightweight-idp/src/discovery/cloudformation-scanner.ts @@ -0,0 +1,132 @@ +import pino from 'pino'; +import { + CloudFormationClient, + ListStacksCommand, + DescribeStacksCommand, + ListStackResourcesCommand, + StackStatus, +} from '@aws-sdk/client-cloudformation'; +import type { DiscoveredService, ScanResult } from './aws-scanner.js'; + +const logger = pino({ name: 'discovery-cloudformation' }); + +const ACTIVE_STATUSES: string[] = [ + StackStatus.CREATE_COMPLETE, + StackStatus.UPDATE_COMPLETE, + StackStatus.UPDATE_ROLLBACK_COMPLETE, +]; + +export class CloudFormationDiscoveryScanner { + private cfnClient: CloudFormationClient; + private region: string; + + constructor(region: string, credentials?: any) { + const config = { region, ...(credentials ? { credentials } : {}) }; + this.cfnClient = new CloudFormationClient(config); + this.region = region; + } + + async scan(account: string): Promise { + const services: DiscoveredService[] = []; + const errors: string[] = []; + + try { + const stacks = await this.scanStacks(account); + services.push(...stacks); + } catch (err) { + errors.push(`CloudFormation scan failed: ${(err as Error).message}`); + logger.warn({ region: this.region, error: (err as Error).message }, 'CloudFormation scan failed'); + } + + const status = errors.length === 0 + ? 'success' + : services.length > 0 + ? 'partial_failure' + : 'failed'; + + return { status, discovered: services.length, errors, services }; + } + + private async scanStacks(account: string): Promise { + const results: DiscoveredService[] = []; + let nextToken: string | undefined; + + do { + const response = await this.cfnClient.send(new ListStacksCommand({ + StackStatusFilter: ACTIVE_STATUSES as StackStatus[], + NextToken: nextToken, + })); + + for (const summary of response.StackSummaries ?? []) { + try { + const detail = await this.describeStack(summary.StackName!); + if (!detail) continue; + + const tags = Object.fromEntries( + (detail.Tags ?? []).map(t => [t.Key!, t.Value!]), + ); + const owner = tags['owner'] ?? tags['team'] ?? tags['Owner'] ?? tags['Team']; + + const resources = await this.listResources(summary.StackName!); + + results.push({ + name: summary.StackName!, + type: 'cloudformation-stack', + arn: summary.StackId!, + region: this.region, + account, + tags, + owner, + ownerSource: owner ? 'aws-tag' : undefined, + metadata: { + status: summary.StackStatus, + creationTime: summary.CreationTime, + lastUpdatedTime: summary.LastUpdatedTime, + templateDescription: summary.TemplateDescription, + resourceCount: resources.length, + resourceTypes: [...new Set(resources.map(r => r.type))], + resources: resources.slice(0, 50), + }, + discoveredAt: new Date(), + }); + } catch (err) { + logger.warn({ stack: summary.StackName, error: (err as Error).message }, 'Failed to describe stack'); + } + } + + nextToken = response.NextToken; + } while (nextToken); + + return results; + } + + private async describeStack(stackName: string) { + const response = await this.cfnClient.send(new DescribeStacksCommand({ StackName: stackName })); + return response.Stacks?.[0] ?? null; + } + + private async listResources(stackName: string): Promise> { + const resources: Array<{ logicalId: string; type: string; physicalId: string; status: string }> = []; + let nextToken: string | undefined; + + do { + const response = await this.cfnClient.send(new ListStackResourcesCommand({ + StackName: stackName, + NextToken: nextToken, + })); + + for (const r of response.StackResourceSummaries ?? []) { + resources.push({ + logicalId: r.LogicalResourceId!, + type: r.ResourceType!, + physicalId: r.PhysicalResourceId ?? '', + status: r.ResourceStatus!, + }); + } + + nextToken = response.NextToken; + } while (nextToken); + + return resources; + } +} diff --git a/products/04-lightweight-idp/src/discovery/scheduler.ts b/products/04-lightweight-idp/src/discovery/scheduler.ts index 57fe11a..f3b8623 100644 --- a/products/04-lightweight-idp/src/discovery/scheduler.ts +++ b/products/04-lightweight-idp/src/discovery/scheduler.ts @@ -3,11 +3,15 @@ import { Redis } from 'ioredis'; import { Pool } from 'pg'; import { AwsDiscoveryScanner } from './aws-scanner.js'; import { GitHubDiscoveryScanner } from './github-scanner.js'; +import { CloudFormationDiscoveryScanner } from './cloudformation-scanner.js'; +import { ApiGatewayDiscoveryScanner } from './apigateway-scanner.js'; import { CatalogService } from '../catalog/service.js'; import { withTenant } from '../data/db.js'; const logger = pino({ name: 'scheduled-discovery' }); +export type ScannerType = 'aws' | 'github' | 'cloudformation' | 'apigateway'; + export class ScheduledDiscovery { private redis: Redis; private pool: Pool; @@ -30,7 +34,7 @@ export class ScheduledDiscovery { await this.redis.del(key); } - async runScan(tenantId: string, scanner: 'aws' | 'github'): Promise<{ status: string; discovered: number }> { + async runScan(tenantId: string, scanner: ScannerType): Promise<{ status: string; discovered: number }> { const locked = await this.acquireLock(tenantId, scanner); if (!locked) { logger.info({ tenantId, scanner }, 'Scan already in progress — skipping'); @@ -50,28 +54,38 @@ export class ScheduledDiscovery { const catalog = new CatalogService(this.pool); if (scanner === 'aws') { - // Get tenant's AWS config (region, credentials would come from tenant settings) const awsScanner = new AwsDiscoveryScanner('us-east-1'); const result = await awsScanner.scan(tenantId); const isPartial = result.status === 'partial_failure'; - const merged = await catalog.mergeAwsDiscovery(tenantId, result.services, isPartial); - await this.recordScanResult(tenantId, scanner, result.status, result.discovered, result.errors); return { status: result.status, discovered: merged }; - } else { - // GitHub scan — would need org name + token from tenant settings + } else if (scanner === 'github') { const { Octokit } = await import('@octokit/rest'); - const octokit = new Octokit(); // Would use tenant's GitHub token + const octokit = new Octokit(); const ghScanner = new GitHubDiscoveryScanner(octokit); - const result = await ghScanner.scan(tenantId); // Would use tenant's org name + const result = await ghScanner.scan(tenantId); const isPartial = result.status === 'partial_failure'; - const merged = await catalog.mergeGitHubDiscovery(tenantId, result.repos, isPartial); - + await this.recordScanResult(tenantId, scanner, result.status, result.discovered, result.errors); + return { status: result.status, discovered: merged }; + } else if (scanner === 'cloudformation') { + const cfnScanner = new CloudFormationDiscoveryScanner('us-east-1'); + const result = await cfnScanner.scan(tenantId); + const isPartial = result.status === 'partial_failure'; + const merged = await catalog.mergeAwsDiscovery(tenantId, result.services, isPartial); + await this.recordScanResult(tenantId, scanner, result.status, result.discovered, result.errors); + return { status: result.status, discovered: merged }; + } else if (scanner === 'apigateway') { + const apigwScanner = new ApiGatewayDiscoveryScanner('us-east-1'); + const result = await apigwScanner.scan(tenantId); + const isPartial = result.status === 'partial_failure'; + const merged = await catalog.mergeAwsDiscovery(tenantId, result.services, isPartial); await this.recordScanResult(tenantId, scanner, result.status, result.discovered, result.errors); return { status: result.status, discovered: merged }; } + + return { status: 'failed', discovered: 0 }; } catch (err) { logger.error({ tenantId, scanner, error: (err as Error).message }, 'Scheduled scan failed'); await this.recordScanResult(tenantId, scanner, 'failed', 0, [(err as Error).message]); diff --git a/products/04-lightweight-idp/src/index.ts b/products/04-lightweight-idp/src/index.ts index 8bdf371..65c6b56 100644 --- a/products/04-lightweight-idp/src/index.ts +++ b/products/04-lightweight-idp/src/index.ts @@ -9,6 +9,7 @@ import { authHook, decorateAuth, registerAuthRoutes, registerProtectedAuthRoutes import { registerServiceRoutes } from './api/services.js'; import { registerDiscoveryRoutes } from './api/discovery.js'; import { registerSearchRoutes } from './api/search.js'; +import { registerAnalyticsRoutes } from './api/analytics.js'; const logger = pino({ name: 'dd0c-portal', level: config.LOG_LEVEL }); @@ -35,6 +36,7 @@ app.register(async function protectedRoutes(protectedApp) { registerServiceRoutes(protectedApp); registerDiscoveryRoutes(protectedApp); registerSearchRoutes(protectedApp); + registerAnalyticsRoutes(protectedApp); }); try {