dev-intel-v2/sysdoc.js

const fs = require('fs');
const path = require('path');
const GraphStore = require('./graph.js');
const { buildSubsystems } = require('./subsystem.js');
const { extractAllContracts, buildContractXref, extractHelmContracts } = require('./contracts.js');
const { buildFlowIndex, traceFlow } = require('./flow.js');
const { generateDependencyDiagram, generateFlowDiagram, generateContractDiagram } = require('./diagrams.js');
const { discoverCharts, chartsToGraph, generateHelmDiagram } = require('./extract-helm.js');

/**
 * Phase 7D: Hierarchical Doc Generator
 * Orchestrates 7A, 7B, 7C, and 7E to generate a Divio-structured documentation site.
 */

async function generateDocs(graph, srcRoot, outDir, opts = {}) {
  const entryPoints = opts.entryPoints || [];
  const useProse = opts.prose === true;
  
  // Optional LLM module for prose enrichment
  let proseMod = null;
  if (useProse) {
    try {
      proseMod = require('./prose.js');
      console.log('Prose generation enabled (LLM pass active)');
    } catch (err) {
      console.warn('Prose generation requested but prose.js not available. Skipping LLM pass.');
    }
  }

  // 4. Discover Helm Charts (Phase 8) - Do this early to feed main graph
  const helmIgnore = new Set([
    'node_modules', '.git', 'venv', '__pycache__', '.terraform',
    '_bmad', '_bmad-output', '.codex', '.claude', '.cursor', '.gemini', '.kiro', '.agents'
  ]);
  const helmCharts = discoverCharts(srcRoot, helmIgnore);
  const helmGraph = chartsToGraph(helmCharts, srcRoot);
  console.log(`Helm: ${helmCharts.length} charts, ${helmGraph.entities.length} entities, ${helmGraph.relationships.length} relationships`);

  // Merge Helm into main graph so Subsystem Aggregator sees it
  for (const e of helmGraph.entities) {
    const fakePath = e.dir ? path.join(srcRoot, e.dir, 'Chart.yaml') : path.join(srcRoot, 'Chart.yaml');
    graph.nodes.set(e.id, { ...e, type: e.type || 'Module', _file: fakePath });
    if (!graph.fileIndex.has(fakePath)) graph.fileIndex.set(fakePath, new Set());
    graph.fileIndex.get(fakePath).add(e.id);
  }
  for (const r of helmGraph.relationships) {
    graph.edges.push(r);
  }
  
  // 1. Build Subsystems (7A)
  const subs = buildSubsystems(graph, {
    srcDir: opts.srcDir || '/src/',
    minTraffic: opts.minTraffic || 3,
    crossCuttingThreshold: opts.crossCuttingThreshold || 0.6
  });

  // 2. Extract Contracts (7B) — TypeScript + Helm
  const contractsResult = extractAllContracts(subs, srcRoot);
  const helmContracts = extractHelmContracts(helmCharts);
  
  // Merge Helm contracts into main result
  contractsResult.contracts.push(...helmContracts.contracts);
  for (const [sub, contracts] of Object.entries(helmContracts.bySubsystem)) {
    if (!contractsResult.bySubsystem[sub]) contractsResult.bySubsystem[sub] = [];
    contractsResult.bySubsystem[sub].push(...contracts);
  }
  
  const xref = buildContractXref(contractsResult.contracts, graph, (p) => p.replace(/^\/?src\//, ''));
  
  // 3. Trace Flows (7C)
  const flowIndex = buildFlowIndex(graph, subs);
  const flowResults = entryPoints.map(ep => traceFlow(ep, flowIndex));

  // Initialize output directory structure (Divio)
  const dirs = [
    'reference/subsystems',
    'reference/contracts',
    'reference/modules',
    'reference/helm',
    'reference/helm/charts',
    'explanation',
    'tutorials',
    'how-to',
    'diagrams'
  ];
  
  for (const d of dirs) {
    fs.mkdirSync(path.join(outDir, d), { recursive: true });
  }

  // Generate Reference: System Architecture
  const sysArchPath = path.join(outDir, 'reference/system-architecture.md');
  const depDiag = generateDependencyDiagram(subs);
  const depDiagPath = 'diagrams/system-deps.mmd';
  fs.writeFileSync(path.join(outDir, depDiagPath), depDiag);
  
  let archProse = '';
  if (proseMod) {
    console.log('Generating architecture overview...');
    archProse = await proseMod.describeArchitecture(subs.subsystems, subs.crossCutting, {}, {});
    archProse = `\n${archProse.trim()}\n\n`;
  }
  
  const sysArchContent = `# System Architecture
${archProse}
## Summary Statistics
- **Subsystems:** ${subs.subsystems.length}
- **Helm Charts:** ${helmCharts.length}
- **Total Contracts:** ${contractsResult.contracts.length}
- **Cross-Cutting Concerns:** ${subs.crossCutting.join(', ') || 'none'}

## Subsystems

| Subsystem | Kind | Files | Modules | Functions |
|---|---|---|---|---|
${subs.subsystems.map(s => `| ${s.name} | ${s.kind} | ${s.files.length} | ${s.entities.modules} | ${s.entities.functions} |`).join('\n')}

## Cross-Cutting Concerns
${subs.crossCutting.map(c => `- **${c}**`).join('\n') || '*None detected*'}

## Cross-Subsystem Dependencies

| From | To | Calls | Imports |
|---|---|---|---|
${Object.entries(subs.dependencyMatrix).filter(([, v]) => (v.calls + v.imports) > 0).sort((a, b) => (b[1].calls + b[1].imports) - (a[1].calls + a[1].imports)).map(([k, v]) => { const [from, to] = k.split('→'); return `| ${from} | ${to} | ${v.calls} | ${v.imports} |`; }).join('\n')}

## Top Helm Charts by Resource Count

| Chart | Path | Resources |
|---|---|---|
${[...helmCharts].sort((a, b) => b.templates.resources.length - a.templates.resources.length).slice(0, 10).map(c => `| ${c.chart.name} | \`${c.dir}\` | ${c.templates.resources.length} |`).join('\n')}

## Kubernetes Resource Types (across all charts)

| Kind | Count |
|---|---|
${(() => { const kinds = {}; for (const c of helmCharts) for (const r of c.templates.resources) kinds[r.kind] = (kinds[r.kind] || 0) + 1; return Object.entries(kinds).sort((a, b) => b[1] - a[1]).slice(0, 15).map(([k, v]) => `| ${k} | ${v} |`).join('\n'); })()}

## Dependency Map
\`\`\`mermaid
${depDiag}
\`\`\`
`;
  fs.writeFileSync(sysArchPath, sysArchContent);

  // Generate Reference: Subsystem Docs
  for (const sub of subs.subsystems) {
    const subDocPath = path.join(outDir, `reference/subsystems/${sub.name}.md`);
    const subContracts = contractsResult.bySubsystem[sub.name] || [];
    
    let contractSection = '';
    if (subContracts.length > 0) {
      const contractDiag = generateContractDiagram(subContracts);
      fs.writeFileSync(path.join(outDir, `diagrams/${sub.name}-contracts.mmd`), contractDiag);
      contractSection = `\n## Contracts\n\`\`\`mermaid\n${contractDiag}\n\`\`\`\n`;
    }

    let subProse = '';
    if (proseMod) {
      console.log(`Generating prose for subsystem: ${sub.name}...`);
      subProse = await proseMod.describeSubsystem(sub, subs.dependencyMatrix, {});
      subProse = `\n${subProse.trim()}\n\n`;
    }

    const subContent = `# Subsystem: ${sub.name}
${subProse}
**Kind:** ${sub.kind}
**Files:** ${sub.files.length}

## Public Exports
${sub.publicExports.length > 0 ? sub.publicExports.map(e => `- \`${e}\``).join('\n') : '*None*'}
${contractSection}
## Files
${sub.files.map(f => `- \`${f}\``).join('\n')}
`;
    fs.writeFileSync(subDocPath, subContent);
  }

  // Generate Reference: Contracts
  const contractDocPath = path.join(outDir, 'reference/contracts/index.md');
  const allContractsDiag = generateContractDiagram(contractsResult.contracts);
  fs.writeFileSync(path.join(outDir, 'diagrams/all-contracts.mmd'), allContractsDiag);
  
  let contractProseList = '';
  if (proseMod && contractsResult.contracts.length > 0) {
    console.log(`Generating prose for ${contractsResult.contracts.length} contracts...`);
    // Batch processing to avoid overloading the API
    const batchSize = 10;
    const contractDocs = [];
    for (let i = 0; i < contractsResult.contracts.length; i += batchSize) {
      const batch = contractsResult.contracts.slice(i, i + batchSize);
      const docs = await Promise.all(batch.map(c => proseMod.describeContract(c, xref, {})));
      contractDocs.push(...docs);
    }
    contractProseList = contractsResult.contracts.map((c, i) => `### ${c.name}\n${contractDocs[i].trim()}\n`).join('\n');
  }

  fs.writeFileSync(contractDocPath, `# System Contracts\n\n\`\`\`mermaid\n${allContractsDiag}\n\`\`\`\n\n${contractProseList}`);

  // Generate Reference: Helm Charts
  const helmIndexPath = path.join(outDir, 'reference/helm/index.md');
  let helmIndexContent = '# Helm Charts\n\n';
  
  // Name-to-file lookup for agent navigation
  helmIndexContent += '## Quick Lookup (by chart name)\n\n';
  const nameGroups = {};
  for (const c of helmCharts) {
    const safeName = c.dir.replace(/[^a-zA-Z0-9]/g, '-').replace(/-+/g, '-').replace(/^-|-$/g, '');
    if (!nameGroups[c.chart.name]) nameGroups[c.chart.name] = [];
    nameGroups[c.chart.name].push({ dir: c.dir, safeName });
  }
  for (const [name, entries] of Object.entries(nameGroups).sort((a, b) => a[0].localeCompare(b[0]))) {
    if (entries.length === 1) {
      helmIndexContent += `- **${name}** → [${entries[0].dir}](charts/${entries[0].safeName}.md)\n`;
    } else {
      helmIndexContent += `- **${name}**:\n`;
      for (const e of entries) {
        helmIndexContent += `  - [${e.dir}](charts/${e.safeName}.md)\n`;
      }
    }
  }
  
  helmIndexContent += '\n## All Charts\n\n| Chart | Path | Version | AppVersion | Resources | Dependencies | Values Keys | Interactions |\n|---|---|---|---|---|---|---|---|\n';
  
  // Use dir-based filenames to avoid collisions between same-named charts
  for (const c of helmCharts) {
    const safeName = c.dir.replace(/[^a-zA-Z0-9]/g, '-').replace(/-+/g, '-').replace(/^-|-$/g, '');
    const chartDocPath = path.join(outDir, `reference/helm/charts/${safeName}.md`);
    
    helmIndexContent += `| [${c.chart.name}](charts/${safeName}.md) | \`${c.dir}\` | ${c.chart.version} | ${c.chart.appVersion || 'N/A'} | ${c.templates.resources.length} | ${c.chart.dependencies.map(d => d.name).join(', ') || 'none'} | ${c.values.keys.length} | ${c.interactions.length} |\n`;
    
    let chartContent = `# Chart: ${c.chart.name}\n\n`;
    chartContent += `**Version:** ${c.chart.version}  \n`;
    chartContent += `**App Version:** ${c.chart.appVersion || 'N/A'}  \n`;
    chartContent += `**Path:** \`${c.dir}\`\n\n`;
    
    if (c.chart.description) {
      chartContent += `${c.chart.description}\n\n`;
    }
    
    if (c.chart.dependencies.length > 0) {
      chartContent += `## Dependencies\n`;
      for (const d of c.chart.dependencies) {
        chartContent += `- **${d.name}** (${d.version})${d.condition ? ` *if ${d.condition}*` : ''}\n`;
      }
      chartContent += '\n';
    }
    
    if (c.interactions.length > 0) {
      chartContent += `## Interactions (Contracts)\n`;
      for (const i of c.interactions) {
        chartContent += `- **${i.type}**: \`${i.target}\` (via \`${i.file}\`)\n`;
      }
      chartContent += '\n';
    }
    
    if (c.templates.resources.length > 0) {
      chartContent += `## Resources Generated\n`;
      for (const r of c.templates.resources) {
        chartContent += `- **${r.kind}**: \`${r.name}\` (${r.file})\n`;
      }
      chartContent += '\n';
    }
    
    if (c.values.keys.length > 0) {
      chartContent += `## Configuration Surface (values.yaml)\n`;
      chartContent += `| Key | Type | Default |\n|---|---|---|\n`;
      for (const k of c.values.keys) {
        let defStr = k.defaultValue !== undefined ? String(k.defaultValue).replace(/\\n/g, ' ') : (k.hasDefault ? 'yes' : 'no');
        if (defStr.includes('|')) defStr = defStr.replace(/\\|/g, '\\\\|');
        chartContent += `| \`${k.name}\` | ${k.type} | ${defStr} |\n`;
      }
      chartContent += '\n';
    }
    
    fs.writeFileSync(chartDocPath, chartContent);
  }
  
  // Generate Helm interaction diagram
  const helmDiag = generateHelmDiagram(helmCharts);
  fs.writeFileSync(path.join(outDir, 'diagrams/helm-interactions.mmd'), helmDiag);
  
  // Shared secrets/configmaps cross-reference
  const configUsers = {};
  for (const c of helmCharts) {
    for (const i of c.interactions) {
      if (i.type === 'config-ref') {
        if (!configUsers[i.target]) configUsers[i.target] = [];
        configUsers[i.target].push(c.chart.name);
      }
    }
  }
  
  // Port map: which charts expose which ports
  const portMap = {};
  for (const c of helmCharts) {
    for (const i of c.interactions) {
      if (i.type === 'port' && i.target !== '0') {
        if (!portMap[i.target]) portMap[i.target] = [];
        if (!portMap[i.target].includes(c.chart.name)) portMap[i.target].push(c.chart.name);
      }
    }
  }
  
  helmIndexContent += `\n## Interaction Diagram\n\`\`\`mermaid\n${helmDiag}\n\`\`\`\n`;
  
  // Shared config/secrets table
  const sharedConfigs = Object.entries(configUsers).filter(([, users]) => users.length > 1);
  if (sharedConfigs.length > 0) {
    helmIndexContent += `\n## Shared Secrets & ConfigMaps\n| Secret/ConfigMap | Used By |\n|---|---|\n`;
    for (const [name, users] of sharedConfigs) {
      helmIndexContent += `| \`${name}\` | ${users.join(', ')} |\n`;
    }
  }
  
  // Port allocation table
  const sharedPorts = Object.entries(portMap).filter(([, users]) => users.length > 1).sort((a, b) => Number(a[0]) - Number(b[0]));
  if (sharedPorts.length > 0) {
    helmIndexContent += `\n## Port Allocation (shared)\n| Port | Charts |\n|---|---|\n`;
    for (const [port, users] of sharedPorts) {
      helmIndexContent += `| ${port} | ${users.join(', ')} |\n`;
    }
  }
  
  // K8s service references
  const svcRefs = [];
  for (const c of helmCharts) {
    for (const i of c.interactions) {
      if (i.type === 'k8s-service') {
        svcRefs.push({ from: c.chart.name, to: i.target });
      }
    }
  }
  if (svcRefs.length > 0) {
    helmIndexContent += `\n## Service-to-Service References\n| From Chart | Calls Service |\n|---|---|\n`;
    for (const ref of svcRefs) {
      helmIndexContent += `| ${ref.from} | \`${ref.to}\` |\n`;
    }
  }
  
  fs.writeFileSync(helmIndexPath, helmIndexContent);

  // Generate Explanation: Data Flows
  const flowsPath = path.join(outDir, 'explanation/data-flows.md');
  let flowsContent = '# Data Flows\n\n';
  
  for (let i = 0; i < flowResults.length; i++) {
    const fr = flowResults[i];
    if (fr.error) {
      flowsContent += `## Flow: ${fr.entryPoint}\n**Error:** ${fr.error}\n\n`;
      continue;
    }
    const flowDiag = generateFlowDiagram(fr);
    const diagName = `flow-${i}.mmd`;
    fs.writeFileSync(path.join(outDir, `diagrams/${diagName}`), flowDiag);
    
    let flowProse = '';
    if (proseMod) {
      console.log(`Generating prose for flow: ${fr.entryPoint}...`);
      flowProse = await proseMod.describeFlow(fr, {});
      flowProse = `${flowProse.trim()}\n\n`;
    }
    
    flowsContent += `## Flow: ${fr.entryPoint}\n`;
    flowsContent += flowProse;
    flowsContent += `**Subsystem Sequence:** ${fr.subsystemSequence.join(' → ')}\n\n`;
    flowsContent += `\`\`\`mermaid\n${flowDiag}\n\`\`\`\n\n`;
  }
  fs.writeFileSync(flowsPath, flowsContent);

  return {
    subsystems: subs.subsystems.length,
    contracts: contractsResult.contracts.length,
    flows: flowResults.length,
    outDir
  };
}

if (require.main === module) {
  const snapshotPath = process.argv[2];
  const srcRoot = process.argv[3];
  const outDir = process.argv[4];
  const useProse = process.argv.includes('--prose');
  const entryPoints = process.argv.slice(5).filter(a => a !== '--prose');

  if (!snapshotPath || !srcRoot || !outDir) {
    console.error('Usage: node sysdoc.js <snapshot.json> <srcRoot> <outDir> [--prose] [entryPoint1] ...');
    process.exit(1);
  }

  const graph = GraphStore.loadSnapshot(snapshotPath);
  // Using an IIFE to support top-level await
  (async () => {
    try {
      const result = await generateDocs(graph, srcRoot, outDir, { 
        srcDir: srcRoot.endsWith('/') ? srcRoot : srcRoot + '/',
        entryPoints, 
        prose: useProse 
      });
      console.log(`Generated docs in ${result.outDir}`);
      console.log(`- ${result.subsystems} subsystems`);
      console.log(`- ${result.contracts} contracts`);
      console.log(`- ${result.flows} flows`);
    } catch (err) {
      console.error('Error generating docs:', err);
      process.exit(1);
    }
  })();
}

module.exports = { generateDocs };