dev-intel-v2/pipeline-v3.js

/**
 * Dev Intel V3 — Pipeline Glue Layer
 * 
 * Orchestrates OSS tools (terraform-docs, helm-docs) + custom analysis
 * (graph builder, subsystem aggregator, cross-chart interactions, LLM prose).
 * 
 * Data Contract:
 *   terraform-docs → markdown per module dir → merged into docs/reference/terraform/
 *   helm-docs     → README.md per chart dir  → merged into docs/reference/helm/charts/
 *   graph.js      → snapshot.json (nodes + edges + fileIndex)
 *   subsystem.js  → subsystems[] with dependency matrix
 *   prose.js      → LLM-enriched descriptions injected into final markdown
 *   impact.js     → change impact analysis from graph traversal
 */

const fs = require('fs');
const path = require('path');
const { execSync } = require('child_process');

const GraphStore = require('./graph.js');
const { buildSubsystems } = require('./subsystem.js');
const { discoverCharts, chartsToGraph } = require('./extract-helm.js');
const { detectAnomalies } = require('./prose.js');
const { queryImpact, formatImpactMarkdown } = require('./impact.js');
const { detectEntryPoints } = require('./flow.js');

const PATH_PREFIX = '/home/node/.local/bin:' + process.env.PATH;

const IGNORE_DIRS = new Set([
  'node_modules', '.git', 'dist', 'build', '__pycache__', '.next',
  '.turbo', 'coverage', '.nyc_output', 'vendor', 'venv',
  '.codex', '.claude', '.cursor', '.gemini', '.kiro', '.agents',
  '_bmad', '_bmad-output', '.terraform', 'skills', '.ci', 'docs', '.workdir',
]);

/**
 * Step 1: Run terraform-docs on all directories containing .tf files.
 * Returns: Map<dirPath, markdownString>
 */
function runTerraformDocs(repoRoot) {
  const results = new Map();
  const tfDirs = new Set();

  // Find all dirs with .tf files
  function walk(dir) {
    const base = path.basename(dir);
    if (IGNORE_DIRS.has(base)) return;
    let hasTf = false;
    try {
      const entries = fs.readdirSync(dir, { withFileTypes: true });
      for (const e of entries) {
        if (e.isDirectory()) walk(path.join(dir, e.name));
        else if (e.name.endsWith('.tf')) hasTf = true;
      }
    } catch { return; }
    if (hasTf) tfDirs.add(dir);
  }
  walk(repoRoot);

  console.log(`terraform-docs: scanning ${tfDirs.size} directories...`);
  let success = 0, fail = 0;

  for (const dir of tfDirs) {
    try {
      const md = execSync(`terraform-docs markdown table "${dir}" 2>/dev/null`, {
        env: { ...process.env, PATH: PATH_PREFIX },
        timeout: 15000,
        maxBuffer: 1024 * 1024,
      }).toString();
      if (md.trim().length > 10) {
        results.set(path.relative(repoRoot, dir), md);
        success++;
      }
    } catch {
      fail++;
    }
  }
  console.log(`terraform-docs: ${success} generated, ${fail} skipped`);
  return results;
}

/**
 * Step 2: Run helm-docs on all chart directories.
 * Returns: Map<chartDir, markdownString>
 */
function runHelmDocs(repoRoot) {
  const results = new Map();
  const chartDirs = [];

  function walk(dir) {
    const base = path.basename(dir);
    if (IGNORE_DIRS.has(base)) return;
    try {
      const entries = fs.readdirSync(dir, { withFileTypes: true });
      const hasChart = entries.some(e => e.name === 'Chart.yaml');
      if (hasChart) chartDirs.push(dir);
      for (const e of entries) {
        if (e.isDirectory()) walk(path.join(dir, e.name));
      }
    } catch { return; }
  }
  walk(repoRoot);

  console.log(`helm-docs: scanning ${chartDirs.length} charts...`);
  let success = 0, fail = 0;

  for (const dir of chartDirs) {
    try {
      execSync(`helm-docs --chart-search-root "${dir}" --dry-run 2>/dev/null`, {
        env: { ...process.env, PATH: PATH_PREFIX },
        timeout: 15000,
        maxBuffer: 1024 * 1024,
      });
      // helm-docs writes README.md in-place with --dry-run it outputs to stdout
      const md = execSync(`helm-docs --chart-search-root "${dir}" --dry-run 2>/dev/null`, {
        env: { ...process.env, PATH: PATH_PREFIX },
        timeout: 15000,
        maxBuffer: 1024 * 1024,
      }).toString();
      if (md.trim().length > 10) {
        results.set(path.relative(repoRoot, dir), md);
        success++;
      }
    } catch {
      fail++;
    }
  }
  console.log(`helm-docs: ${success} generated, ${fail} skipped`);
  return results;
}

/**
 * Step 3: Run custom graph extraction + subsystem analysis.
 * Uses the existing pipeline.js batch extraction.
 */
function runGraphExtraction(repoRoot, snapshotPath) {
  console.log('Running graph extraction pipeline...');
  execSync(`node pipeline.js batch "${repoRoot}" --output ./output 2>&1`, {
    cwd: __dirname,
    timeout: 300000,
    maxBuffer: 10 * 1024 * 1024,
    stdio: ['pipe', 'pipe', 'pipe'],
  });
  return GraphStore.loadSnapshot(snapshotPath);
}

/**
 * Step 4: Assemble final docs directory.
 */
function assembleDocs(outDir, { tfDocs, helmDocs, graph, subs, helmCharts, impactResults, entryPoints }) {
  // Create directory structure
  const dirs = [
    'reference/terraform', 'reference/helm/charts', 'reference/subsystems',
    'explanation', 'diagrams'
  ];
  for (const d of dirs) {
    fs.mkdirSync(path.join(outDir, d), { recursive: true });
  }

  // Write terraform docs
  for (const [dir, md] of tfDocs) {
    const safeName = dir.replace(/\//g, '-');
    fs.writeFileSync(path.join(outDir, `reference/terraform/${safeName}.md`), `# Terraform: ${dir}\n\n${md}`);
  }

  // Write helm docs
  for (const [dir, md] of helmDocs) {
    const safeName = dir.replace(/\//g, '-');
    fs.writeFileSync(path.join(outDir, `reference/helm/charts/${safeName}.md`), md);
  }

  // Write helm index with inline dependency data
  let helmIndex = '# Helm Charts\n\n## Quick Lookup\n\n';
  helmIndex += '| Chart | Path | Dependencies | Values Keys |\n|---|---|---|---|\n';
  for (const c of helmCharts) {
    const deps = (c.chart.dependencies || []).map(d => d.name).join(', ') || 'none';
    const valKeys = Object.keys(c.values || {}).slice(0, 5).join(', ');
    const safeName = c.dir.replace(/\//g, '-');
    helmIndex += `| ${c.chart.name} | [${c.dir}](charts/${safeName}.md) | ${deps} | ${valKeys} |\n`;
  }
  fs.writeFileSync(path.join(outDir, 'reference/helm/index.md'), helmIndex);

  // Write terraform index
  let tfIndex = '# Terraform Modules\n\n';
  tfIndex += `| Directory | Generated |\n|---|---|\n`;
  for (const [dir] of tfDocs) {
    const safeName = dir.replace(/\//g, '-');
    tfIndex += `| ${dir} | [docs](terraform/${safeName}.md) |\n`;
  }
  fs.writeFileSync(path.join(outDir, 'reference/terraform/index.md'), tfIndex);

  // Write subsystem docs
  for (const sub of subs.subsystems) {
    const anomalies = detectAnomalies(sub, subs.dependencyMatrix || {});
    let content = `# Subsystem: ${sub.name}\n\n`;
    content += `**Kind:** ${sub.kind}\n**Files:** ${sub.files.length}\n`;
    content += `**Functions:** ${sub.entities.functions}, **Classes:** ${sub.entities.classes}\n\n`;

    if (anomalies.length > 0) {
      content += `## Structural Notes\n${anomalies.map(a => `- ${a}`).join('\n')}\n\n`;
    }

    const outDeps = Object.entries(subs.dependencyMatrix || {})
      .filter(([k]) => k.startsWith(sub.name + '→'))
      .map(([k, v]) => `- → ${k.split('→')[1]}: ${v.calls} calls, ${v.imports} imports`);
    const inDeps = Object.entries(subs.dependencyMatrix || {})
      .filter(([k]) => k.endsWith('→' + sub.name))
      .map(([k, v]) => `- ← ${k.split('→')[0]}: ${v.calls} calls, ${v.imports} imports`);

    if (outDeps.length > 0 || inDeps.length > 0) {
      content += `## Dependencies\n`;
      if (outDeps.length) content += `### Depends On\n${outDeps.join('\n')}\n\n`;
      if (inDeps.length) content += `### Depended On By\n${inDeps.join('\n')}\n\n`;
    }

    if (sub.publicExports.length > 0) {
      content += `## Public Exports\n${sub.publicExports.map(e => `- \`${e}\``).join('\n')}\n`;
    }

    fs.writeFileSync(path.join(outDir, `reference/subsystems/${sub.name}.md`), content);
  }

  // Write architecture overview
  let arch = '# System Architecture\n\n';
  arch += `## Summary\n- **Subsystems:** ${subs.subsystems.length}\n`;
  arch += `- **Helm Charts:** ${helmCharts.length}\n`;
  arch += `- **Terraform Modules:** ${tfDocs.size}\n`;
  arch += `- **Cross-Cutting:** ${subs.crossCutting.join(', ') || 'none'}\n\n`;

  arch += '## Subsystems\n\n| Subsystem | Kind | Files | Functions |\n|---|---|---|---|\n';
  for (const s of subs.subsystems) {
    arch += `| ${s.name} | ${s.kind} | ${s.files.length} | ${s.entities.functions} |\n`;
  }
  fs.writeFileSync(path.join(outDir, 'reference/system-architecture.md'), arch);

  // Write change impact analysis
  if (impactResults.length > 0) {
    let impactContent = '# Change Impact Analysis\n\n';
    for (const r of impactResults.filter(r => r.impactedCount > 0).slice(0, 15)) {
      impactContent += formatImpactMarkdown(r) + '\n\n---\n\n';
    }
    fs.writeFileSync(path.join(outDir, 'explanation/change-impact.md'), impactContent);
  }

  // Write entry points
  if (entryPoints.length > 0) {
    let epContent = '# Entry Points\n\n| Entry | Kind | Reason |\n|---|---|---|\n';
    for (const ep of entryPoints) {
      epContent += `| \`${ep.id}\` | ${ep.kind} | ${ep.reason} |\n`;
    }
    fs.writeFileSync(path.join(outDir, 'explanation/entry-points.md'), epContent);
  }

  // Count output
  let fileCount = 0;
  function countFiles(dir) {
    for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
      if (e.isDirectory()) countFiles(path.join(dir, e.name));
      else fileCount++;
    }
  }
  countFiles(outDir);
  return fileCount;
}

// Main
async function main() {
  const repoRoot = process.argv[2] || '/home/node/.openclaw/workspace/agents/max/foxtrot/';
  const outDir = process.argv[3] || './foxtrot-docs-v3';
  const snapshotPath = './output/snapshot.json';

  console.log('=== Dev Intel V3 Pipeline ===');
  console.log(`Repo: ${repoRoot}`);
  console.log(`Output: ${outDir}`);

  // Clean output
  if (fs.existsSync(outDir)) {
    fs.rmSync(outDir, { recursive: true });
  }

  // Step 1: terraform-docs
  const tfDocs = runTerraformDocs(repoRoot);

  // Step 2: helm-docs
  const helmDocs = runHelmDocs(repoRoot);

  // Step 3: Graph extraction (reuse existing pipeline)
  const graph = runGraphExtraction(repoRoot, snapshotPath);

  // Step 4: Helm chart discovery + merge into graph
  const helmIgnore = new Set([...IGNORE_DIRS]);
  const helmCharts = discoverCharts(repoRoot, helmIgnore);
  chartsToGraph(helmCharts, graph, repoRoot);

  // Step 5: Subsystem analysis
  const subs = buildSubsystems(graph, { srcDir: repoRoot });

  // Step 6: Entry points
  const entryPoints = detectEntryPoints(graph);
  console.log(`Entry points: ${entryPoints.length} detected`);

  // Step 7: Impact analysis
  const impactTargets = [];
  for (const [id, node] of graph.nodes) {
    if (node.kind === 'terraform-module' || node.kind === 'terraform-resource') {
      impactTargets.push(id);
    }
  }
  for (const c of helmCharts) {
    if (c.interactions && c.interactions.length > 3) {
      const chartId = `helm:${c.dir}:${c.chart.name}`;
      if (graph.nodes.has(chartId)) impactTargets.push(chartId);
    }
  }
  const impactResults = impactTargets.slice(0, 20).map(t => queryImpact(graph, t, 5));
  console.log(`Impact analysis: ${impactResults.filter(r => r.impactedCount > 0).length} nodes with dependents`);

  // Step 8: Assemble
  const fileCount = assembleDocs(outDir, {
    tfDocs, helmDocs, graph, subs, helmCharts, impactResults, entryPoints
  });

  console.log(`\n=== V3 Pipeline Complete ===`);
  console.log(`Output: ${fileCount} files in ${outDir}`);
  console.log(`Terraform modules: ${tfDocs.size}`);
  console.log(`Helm charts: ${helmDocs.size}`);
  console.log(`Subsystems: ${subs.subsystems.length}`);
}

main().catch(err => {
  console.error('Pipeline failed:', err.message);
  process.exit(1);
});