const fs = require('fs'); const path = require('path'); const GraphStore = require('./graph.js'); const { buildSubsystems } = require('./subsystem.js'); const { extractAllContracts, buildContractXref, extractHelmContracts } = require('./contracts.js'); const { buildFlowIndex, traceFlow, detectEntryPoints } = require('./flow.js'); const { generateDependencyDiagram, generateFlowDiagram, generateContractDiagram } = require('./diagrams.js'); const { discoverCharts, chartsToGraph, generateHelmDiagram } = require('./extract-helm.js'); const { queryImpact, formatImpactMarkdown } = require('./impact.js'); const { extractAllPatterns } = require('./extract-patterns.js'); const { buildAgentKB } = require('./agent-kb.js'); const { extractDeep } = require('./extract-deep.js'); const { profileRepo, ARCHETYPES } = require('./repo-profiler.js'); const { extractDynamic } = require('./extract-dynamic.js'); /** * Phase 7D: Hierarchical Doc Generator * Orchestrates 7A, 7B, 7C, and 7E to generate a Divio-structured documentation site. */ async function generateDocs(graph, srcRoot, outDir, opts = {}) { const entryPoints = opts.entryPoints || []; const autoDetect = opts.autoDetectEntryPoints !== false; // default true const useProse = opts.prose === true; const confluenceDir = opts.confluenceDir || null; // Optional LLM module for prose enrichment let proseMod = null; let confluenceCtx = {}; if (useProse) { try { proseMod = require('./prose.js'); console.log('Prose generation enabled (LLM pass active)'); if (confluenceDir) { confluenceCtx = proseMod.loadConfluenceContext(confluenceDir); console.log(`Confluence context loaded: ${Object.keys(confluenceCtx).length} docs`); } } catch (err) { console.warn('Prose generation requested but prose.js not available. Skipping LLM pass.'); } } // 4. Discover Helm Charts (Phase 8) - Do this early to feed main graph const helmIgnore = new Set([ 'node_modules', '.git', 'venv', '__pycache__', '.terraform', '_bmad', '_bmad-output', '.codex', '.claude', '.cursor', '.gemini', '.kiro', '.agents' ]); const helmCharts = discoverCharts(srcRoot, helmIgnore); const helmGraph = chartsToGraph(helmCharts, srcRoot); console.log(`Helm: ${helmCharts.length} charts, ${helmGraph.entities.length} entities, ${helmGraph.relationships.length} relationships`); // 4b. Extract architectural patterns from code artifacts let patterns = { layers: [], appsets: [], regions: { aws: [], gcp: [], azure: [] }, cidrs: [], naming: [], techStack: { containerImages: [] }, syncWaves: [] }; let deepData = { addons: [], scriptParams: [], tfConfigs: [], helmValues: [], stateServices: [] }; let archetypeStr = ARCHETYPES ? ARCHETYPES.UNKNOWN : 'Unknown'; if (opts.legacyMode) { patterns = extractAllPatterns(srcRoot); deepData = extractDeep(srcRoot); if (!archetypeStr || archetypeStr === 'Unknown') archetypeStr = 'Infrastructure'; } else { const profile = profileRepo(srcRoot, graph); archetypeStr = profile.archetype; console.log(`Detected Repo Archetype: ${archetypeStr} (confidence: ${profile.confidence})`); const dynamicData = extractDynamic(graph, archetypeStr, srcRoot); deepData = { addons: [], scriptParams: [], tfConfigs: [], helmValues: [], stateServices: dynamicData.stateServices || [], configs: dynamicData.configs || [], deploymentPatterns: dynamicData.deploymentPatterns || [], networkTopology: dynamicData.networkTopology || [] }; } // Merge Helm into main graph so Subsystem Aggregator sees it for (const e of helmGraph.entities) { const fakePath = e.dir ? path.join(srcRoot, e.dir, 'Chart.yaml') : path.join(srcRoot, 'Chart.yaml'); graph.nodes.set(e.id, { ...e, type: e.type || 'Module', _file: fakePath }); if (!graph.fileIndex.has(fakePath)) graph.fileIndex.set(fakePath, new Set()); graph.fileIndex.get(fakePath).add(e.id); } for (const r of helmGraph.relationships) { graph.edges.push(r); } // 1. Build Subsystems (7A) const subs = buildSubsystems(graph, { srcDir: opts.srcDir || '/src/', minTraffic: opts.minTraffic || 3, crossCuttingThreshold: opts.crossCuttingThreshold || 0.6 }); // 2. Extract Contracts (7B) — TypeScript + Helm const contractsResult = extractAllContracts(subs, srcRoot); const helmContracts = extractHelmContracts(helmCharts); // Merge Helm contracts into main result contractsResult.contracts.push(...helmContracts.contracts); for (const [sub, contracts] of Object.entries(helmContracts.bySubsystem)) { if (!contractsResult.bySubsystem[sub]) contractsResult.bySubsystem[sub] = []; contractsResult.bySubsystem[sub].push(...contracts); } const xref = buildContractXref(contractsResult.contracts, graph, (p) => p.replace(/^\/?src\//, '')); // 3. Trace Flows (7C) — auto-detect entry points if none provided const flowIndex = buildFlowIndex(graph, subs); let allEntryPoints = [...entryPoints]; if (autoDetect) { const detected = detectEntryPoints(graph); console.log(`Auto-detected ${detected.length} entry points`); for (const ep of detected) { if (!allEntryPoints.includes(ep.id)) allEntryPoints.push(ep.id); } } const flowResults = allEntryPoints.map(ep => traceFlow(ep, flowIndex)); const validFlows = flowResults.filter(f => !f.error && f.flow.length > 1); console.log(`Flow traces: ${validFlows.length} valid out of ${flowResults.length} attempted`); // 3b. Change Impact Analysis — pick high-value nodes const impactTargets = []; // Find shared secrets/configmaps for (const [id, node] of graph.nodes) { if (node.kind === 'terraform-module' || node.kind === 'terraform-resource') { impactTargets.push(id); } } // Also pick Helm charts with many interactions for (const c of helmCharts) { if (c.interactions.length > 3) { const chartId = `helm:${c.dir}:${c.chart.name}`; if (graph.nodes.has(chartId)) impactTargets.push(chartId); } } const impactResults = impactTargets.slice(0, 20).map(t => queryImpact(graph, t, 5)) .filter(r => r.impactedCount > 0); console.log(`Impact analysis: ${impactResults.length} nodes with downstream dependents`); // Initialize output directory structure (Divio) const dirs = [ 'reference/subsystems', 'reference/contracts', 'reference/modules', 'reference/helm', 'reference/helm/charts', 'explanation', 'tutorials', 'how-to', 'diagrams' ]; for (const d of dirs) { fs.mkdirSync(path.join(outDir, d), { recursive: true }); } // Generate Reference: System Architecture const sysArchPath = path.join(outDir, 'reference/system-architecture.md'); const depDiag = generateDependencyDiagram(subs); const depDiagPath = 'diagrams/system-deps.mmd'; fs.writeFileSync(path.join(outDir, depDiagPath), depDiag); let archProse = ''; if (proseMod) { console.log('Generating architecture overview...'); archProse = await proseMod.describeArchitecture(subs.subsystems, subs.crossCutting, {}, { confluenceCtx }, { deps: subs.dependencyMatrix, confluenceCtx }); archProse = `\n${archProse.trim()}\n\n`; } const sysArchContent = `# System Architecture ${archProse} ## Summary Statistics - **Subsystems:** ${subs.subsystems.length} - **Helm Charts:** ${helmCharts.length} - **Total Contracts:** ${contractsResult.contracts.length} - **Cross-Cutting Concerns:** ${subs.crossCutting.join(', ') || 'none'} ## Platform Architecture Patterns ### Layered Architecture The system is organized into the following logical layers (top to bottom): ${patterns.layers.map(l => `- **${l.layer}** (${l.repos.join(', ')})`).join('\n')} ### Deployment Topology (Hub & Spoke) ArgoCD ApplicationSets define the following ownership model: **Hub (Infrastructure/Control Plane):** ${patterns.appsets.filter(a => a.location === 'hub').map(a => `- \`${a.name}\` manages \`${a.repoName}\``).join('\n')} **Spoke (Applications/Runtime):** ${patterns.appsets.filter(a => a.location === 'spoke').map(a => `- \`${a.name}\` manages \`${a.repoName}\``).join('\n')} ### Cloud Regions Supported - **AWS:** ${patterns.regions.aws.join(', ')} - **GCP:** ${patterns.regions.gcp.join(', ')} - **Azure:** ${patterns.regions.azure.join(', ')} ### Network CIDR Allocations | CIDR Block | Context | File | |---|---|---| ${patterns.cidrs.slice(0, 15).map(c => `| \`${c.cidr}\` | ${c.refs[0].context} | \`${c.refs[0].file}\` |`).join('\n')} ### Naming Conventions The following resource naming patterns are enforced: ${patterns.naming.slice(0, 15).map(n => `- \`${n.pattern}\` (via \`${n.file}\`)`).join('\n')} ### Tech Stack & Dependencies **Core Images:** ${patterns.techStack.containerImages.slice(0, 20).map(i => `- \`${i}\``).join('\n')} ## Configuration & Operational Defaults **State Management Services:** ${deepData.stateServices.map(s => s.name).filter((v, i, a) => a.indexOf(v) === i).join(', ')} **Cluster Addons (EKS/GKE):** ${deepData.addons.map(a => `- ${a.name}: ${a.version}`).join('\n')} **Key Script Parameters (Timeouts/Retries):** ${deepData.scriptParams.filter(p => p.name.includes('TIMEOUT') || p.name.includes('WAIT')).map(p => `- ${p.name} = ${p.value} (${p.file})`).join('\n')} **Infrastructure Configs:** ${deepData.tfConfigs.filter(c => c.key.includes('backup')).map(c => `- ${c.key}: ${c.value} (${c.file})`).join('\n')} **Account/Template Values:** ${deepData.helmValues.map(v => `- ${v.key}: ${v.value} (${v.file})`).join('\n')} ## Subsystems | Subsystem | Kind | Files | Modules | Functions | |---|---|---|---|---| ${subs.subsystems.map(s => `| ${s.name} | ${s.kind} | ${s.files.length} | ${s.entities.modules} | ${s.entities.functions} |`).join('\n')} ## Cross-Cutting Concerns ${subs.crossCutting.map(c => `- **${c}**`).join('\n') || '*None detected*'} ## Cross-Subsystem Dependencies | From | To | Calls | Imports | |---|---|---|---| ${Object.entries(subs.dependencyMatrix).filter(([, v]) => (v.calls + v.imports) > 0).sort((a, b) => (b[1].calls + b[1].imports) - (a[1].calls + a[1].imports)).map(([k, v]) => { const [from, to] = k.split('→'); return `| ${from} | ${to} | ${v.calls} | ${v.imports} |`; }).join('\n')} ## Top Helm Charts by Resource Count | Chart | Path | Resources | |---|---|---| ${[...helmCharts].sort((a, b) => b.templates.resources.length - a.templates.resources.length).slice(0, 10).map(c => `| ${c.chart.name} | \`${c.dir}\` | ${c.templates.resources.length} |`).join('\n')} ## Kubernetes Resource Types (across all charts) | Kind | Count | |---|---| ${(() => { const kinds = {}; for (const c of helmCharts) for (const r of c.templates.resources) kinds[r.kind] = (kinds[r.kind] || 0) + 1; return Object.entries(kinds).sort((a, b) => b[1] - a[1]).slice(0, 15).map(([k, v]) => `| ${k} | ${v} |`).join('\n'); })()} ## Dependency Map \`\`\`mermaid ${depDiag} \`\`\` `; fs.writeFileSync(sysArchPath, sysArchContent); // Generate Reference: Subsystem Docs for (const sub of subs.subsystems) { const subDocPath = path.join(outDir, `reference/subsystems/${sub.name}.md`); const subContracts = contractsResult.bySubsystem[sub.name] || []; let contractSection = ''; if (subContracts.length > 0) { const contractDiag = generateContractDiagram(subContracts); fs.writeFileSync(path.join(outDir, `diagrams/${sub.name}-contracts.mmd`), contractDiag); contractSection = `\n## Contracts\n\`\`\`mermaid\n${contractDiag}\n\`\`\`\n`; } let subProse = ''; if (proseMod) { console.log(`Generating prose for subsystem: ${sub.name}...`); subProse = await proseMod.describeSubsystem(sub, subs.dependencyMatrix, { confluenceCtx }); subProse = `\n${subProse.trim()}\n\n`; } const subContent = `# Subsystem: ${sub.name} ${subProse} **Kind:** ${sub.kind} **Files:** ${sub.files.length} ## Public Exports ${sub.publicExports.length > 0 ? sub.publicExports.map(e => `- \`${e}\``).join('\n') : '*None*'} ${contractSection} ## Files ${sub.files.map(f => `- \`${f}\``).join('\n')} `; fs.writeFileSync(subDocPath, subContent); } // Generate Reference: Contracts const contractDocPath = path.join(outDir, 'reference/contracts/index.md'); const allContractsDiag = generateContractDiagram(contractsResult.contracts); fs.writeFileSync(path.join(outDir, 'diagrams/all-contracts.mmd'), allContractsDiag); let contractProseList = ''; if (proseMod && contractsResult.contracts.length > 0) { console.log(`Generating prose for ${contractsResult.contracts.length} contracts...`); // Batch processing to avoid overloading the API const batchSize = 10; const contractDocs = []; for (let i = 0; i < contractsResult.contracts.length; i += batchSize) { const batch = contractsResult.contracts.slice(i, i + batchSize); const docs = await Promise.all(batch.map(c => proseMod.describeContract(c, xref, {}))); contractDocs.push(...docs); } contractProseList = contractsResult.contracts.map((c, i) => `### ${c.name}\n${contractDocs[i].trim()}\n`).join('\n'); } fs.writeFileSync(contractDocPath, `# System Contracts\n\n\`\`\`mermaid\n${allContractsDiag}\n\`\`\`\n\n${contractProseList}`); // Generate Reference: Helm Charts const helmIndexPath = path.join(outDir, 'reference/helm/index.md'); let helmIndexContent = '# Helm Charts\n\n'; helmIndexContent += '## Helm Sync Waves (Bootstrapping Order)\n\n| Wave | Count | Resources |\n|---|---|---|\n'; helmIndexContent += patterns.syncWaves.map(w => `| ${w.wave} | ${w.resources.length} | ${w.resources.slice(0, 5).map(r => r.kind + ':' + r.name).join(', ')}${w.resources.length > 5 ? '...' : ''} |`).join('\n') + '\n\n'; helmIndexContent += patterns.syncWaves.map(w => `| ${w.wave} | ${w.resources.length} | ${w.resources.slice(0, 5).map(r => r.kind + ':' + r.name).join(', ')}${w.resources.length > 5 ? '...' : ''} |`).join('\n') + '\n\n'; // Name-to-file lookup for agent navigation helmIndexContent += '## Quick Lookup (by chart name)\n\n'; const nameGroups = {}; for (const c of helmCharts) { const safeName = c.dir.replace(/[^a-zA-Z0-9]/g, '-').replace(/-+/g, '-').replace(/^-|-$/g, ''); if (!nameGroups[c.chart.name]) nameGroups[c.chart.name] = []; nameGroups[c.chart.name].push({ dir: c.dir, safeName }); } for (const [name, entries] of Object.entries(nameGroups).sort((a, b) => a[0].localeCompare(b[0]))) { if (entries.length === 1) { helmIndexContent += `- **${name}** → [${entries[0].dir}](charts/${entries[0].safeName}.md)\n`; } else { helmIndexContent += `- **${name}**:\n`; for (const e of entries) { helmIndexContent += ` - [${e.dir}](charts/${e.safeName}.md)\n`; } } } helmIndexContent += '\n## All Charts\n\n| Chart | Path | Version | AppVersion | Resources | Dependencies | Values Keys | Interactions |\n|---|---|---|---|---|---|---|---|\n'; // Use dir-based filenames to avoid collisions between same-named charts for (const c of helmCharts) { const safeName = c.dir.replace(/[^a-zA-Z0-9]/g, '-').replace(/-+/g, '-').replace(/^-|-$/g, ''); const chartDocPath = path.join(outDir, `reference/helm/charts/${safeName}.md`); helmIndexContent += `| [${c.chart.name}](charts/${safeName}.md) | \`${c.dir}\` | ${c.chart.version} | ${c.chart.appVersion || 'N/A'} | ${c.templates.resources.length} | ${c.chart.dependencies.map(d => d.name).join(', ') || 'none'} | ${c.values.keys.length} | ${c.interactions.length} |\n`; let chartContent = `# Chart: ${c.chart.name}\n\n`; chartContent += `**Version:** ${c.chart.version} \n`; chartContent += `**App Version:** ${c.chart.appVersion || 'N/A'} \n`; chartContent += `**Path:** \`${c.dir}\`\n\n`; if (c.chart.description) { chartContent += `${c.chart.description}\n\n`; } if (c.chart.dependencies.length > 0) { chartContent += `## Dependencies\n`; for (const d of c.chart.dependencies) { chartContent += `- **${d.name}** (${d.version})${d.condition ? ` *if ${d.condition}*` : ''}\n`; } chartContent += '\n'; } if (c.interactions.length > 0) { chartContent += `## Interactions (Contracts)\n`; for (const i of c.interactions) { chartContent += `- **${i.type}**: \`${i.target}\` (via \`${i.file}\`)\n`; } chartContent += '\n'; } if (c.templates.resources.length > 0) { chartContent += `## Resources Generated\n`; for (const r of c.templates.resources) { chartContent += `- **${r.kind}**: \`${r.name}\` (${r.file})\n`; } chartContent += '\n'; } if (c.values.keys.length > 0) { chartContent += `## Configuration Surface (values.yaml)\n`; chartContent += `| Key | Type | Default |\n|---|---|---|\n`; for (const k of c.values.keys) { let defStr = k.defaultValue !== undefined ? String(k.defaultValue).replace(/\\n/g, ' ') : (k.hasDefault ? 'yes' : 'no'); if (defStr.includes('|')) defStr = defStr.replace(/\\|/g, '\\\\|'); chartContent += `| \`${k.name}\` | ${k.type} | ${defStr} |\n`; } chartContent += '\n'; } fs.writeFileSync(chartDocPath, chartContent); } // Generate Helm interaction diagram const helmDiag = generateHelmDiagram(helmCharts); fs.writeFileSync(path.join(outDir, 'diagrams/helm-interactions.mmd'), helmDiag); // Shared secrets/configmaps cross-reference const configUsers = {}; for (const c of helmCharts) { for (const i of c.interactions) { if (i.type === 'config-ref') { if (!configUsers[i.target]) configUsers[i.target] = []; configUsers[i.target].push(c.chart.name); } } } // Port map: which charts expose which ports const portMap = {}; for (const c of helmCharts) { for (const i of c.interactions) { if (i.type === 'port' && i.target !== '0') { if (!portMap[i.target]) portMap[i.target] = []; if (!portMap[i.target].includes(c.chart.name)) portMap[i.target].push(c.chart.name); } } } helmIndexContent += `\n## Interaction Diagram\n\`\`\`mermaid\n${helmDiag}\n\`\`\`\n`; // Shared config/secrets table const sharedConfigs = Object.entries(configUsers).filter(([, users]) => users.length > 1); if (sharedConfigs.length > 0) { helmIndexContent += `\n## Shared Secrets & ConfigMaps\n| Secret/ConfigMap | Used By |\n|---|---|\n`; for (const [name, users] of sharedConfigs) { helmIndexContent += `| \`${name}\` | ${users.join(', ')} |\n`; } } // Port allocation table const sharedPorts = Object.entries(portMap).filter(([, users]) => users.length > 1).sort((a, b) => Number(a[0]) - Number(b[0])); if (sharedPorts.length > 0) { helmIndexContent += `\n## Port Allocation (shared)\n| Port | Charts |\n|---|---|\n`; for (const [port, users] of sharedPorts) { helmIndexContent += `| ${port} | ${users.join(', ')} |\n`; } } // K8s service references const svcRefs = []; for (const c of helmCharts) { for (const i of c.interactions) { if (i.type === 'k8s-service') { svcRefs.push({ from: c.chart.name, to: i.target }); } } } if (svcRefs.length > 0) { helmIndexContent += `\n## Service-to-Service References\n| From Chart | Calls Service |\n|---|---|\n`; for (const ref of svcRefs) { helmIndexContent += `| ${ref.from} | \`${ref.to}\` |\n`; } } fs.writeFileSync(helmIndexPath, helmIndexContent); // Generate Explanation: Data Flows const flowsPath = path.join(outDir, 'explanation/data-flows.md'); let flowsContent = '# Data Flows\n\n'; for (let i = 0; i < flowResults.length; i++) { const fr = flowResults[i]; if (fr.error) { flowsContent += `## Flow: ${fr.entryPoint}\n**Error:** ${fr.error}\n\n`; continue; } const flowDiag = generateFlowDiagram(fr); const diagName = `flow-${i}.mmd`; fs.writeFileSync(path.join(outDir, `diagrams/${diagName}`), flowDiag); let flowProse = ''; if (proseMod) { console.log(`Generating prose for flow: ${fr.entryPoint}...`); flowProse = await proseMod.describeFlow(fr, {}); flowProse = `${flowProse.trim()}\n\n`; } flowsContent += `## Flow: ${fr.entryPoint}\n`; flowsContent += flowProse; flowsContent += `**Subsystem Sequence:** ${fr.subsystemSequence.join(' → ')}\n\n`; flowsContent += `\`\`\`mermaid\n${flowDiag}\n\`\`\`\n\n`; } fs.writeFileSync(flowsPath, flowsContent); // Generate Explanation: Change Impact Analysis if (impactResults.length > 0) { const impactPath = path.join(outDir, 'explanation/change-impact.md'); let impactContent = '# Change Impact Analysis\n\n'; impactContent += 'This section documents the blast radius of modifying key infrastructure components.\n\n'; for (const result of impactResults) { impactContent += formatImpactMarkdown(result); impactContent += '\n\n---\n\n'; } // Summary table impactContent += '## Impact Summary\n\n'; impactContent += '| Component | Kind | Downstream Count | Max Depth |\n|---|---|---|---|\n'; for (const r of impactResults.sort((a, b) => b.impactedCount - a.impactedCount)) { const node = r.targetNode || {}; impactContent += `| \`${r.target}\` | ${node.kind || 'unknown'} | ${r.impactedCount} | ${Math.max(...r.impacted.map(i => i.depth), 0)} |\n`; } fs.writeFileSync(impactPath, impactContent); } // Generate Agent Knowledge Base (JSON) const agentKB = buildAgentKB(graph, srcRoot, helmCharts, subs, contractsResult, patterns, impactResults); fs.writeFileSync(path.join(outDir, 'agent-kb.json'), JSON.stringify(agentKB, null, 2)); console.log(`Agent KB: ${agentKB.reference.subsystems.length} subsystems, ${agentKB.reference.helm.charts.length} charts`); if (proseMod) { await proseMod.synthesizeReferencePages(agentKB, deepData, outDir, archetypeStr, { confluenceCtx, model: process.env.LLM_MODEL || 'claude-haiku-4.5' }); } return { subsystems: subs.subsystems.length, contracts: contractsResult.contracts.length, flows: flowResults.length, outDir }; } if (require.main === module) { const snapshotPath = process.argv[2]; const srcRoot = process.argv[3]; const outDir = process.argv[4]; const useProse = process.argv.includes('--prose'); const confluenceArg = process.argv.find(a => a.startsWith('--confluence=')); const confluenceDir = confluenceArg ? confluenceArg.split('=')[1] : null; const entryPoints = process.argv.slice(5).filter(a => a !== '--prose' && !a.startsWith('--confluence=')); if (!snapshotPath || !srcRoot || !outDir) { console.error('Usage: node sysdoc.js [--prose] [--confluence=] [entryPoint1] ...'); process.exit(1); } const graph = GraphStore.loadSnapshot(snapshotPath); // Using an IIFE to support top-level await (async () => { try { const result = await generateDocs(graph, srcRoot, outDir, { srcDir: srcRoot.endsWith('/') ? srcRoot : srcRoot + '/', entryPoints, prose: useProse, legacyMode: process.argv.includes("--legacy"), confluenceDir }); console.log(`Generated docs in ${result.outDir}`); console.log(`- ${result.subsystems} subsystems`); console.log(`- ${result.contracts} contracts`); console.log(`- ${result.flows} flows`); } catch (err) { console.error('Error generating docs:', err); process.exit(1); } })(); } module.exports = { generateDocs };