feat: confluence benchmark, pattern extractor, agent KB, UX spec

- extract-patterns.js: mines layered arch, ArgoCD appsets, cloud regions,
  CIDR allocations, naming conventions, sync waves, tech stack from code
- agent-kb.js: token-efficient JSON rendering of same doc tree
- eval-confluence-ref-questions.json: 32 reference-only benchmark questions
- wiggum-v2.sh: Ralph Wiggum loop targeting confluence baseline (77.8%)
- docs/human-ux-spec.md: BMad UX designer spec for human doc structure
- Eval results: V2 at 28.7% vs confluence 77.8% baseline
- Hub/spoke ownership now correctly extracted (95% on that question)
- Naming conventions, regions, CIDRs surfaced in system-architecture.md
This commit is contained in:
Jarvis Prime
2026-03-10 14:20:35 +00:00
parent 049609a358
commit 0265ec7a60
844 changed files with 2129910 additions and 30 deletions

87
wiggum-fix.js Normal file
View File

@@ -0,0 +1,87 @@
const fs = require('fs');
const { callLLM } = require('./prose.js');
const EVAL_REPORT = process.argv[2];
const FILE_TO_FIX = process.argv[3] || './sysdoc.js';
if (!EVAL_REPORT || !fs.existsSync(EVAL_REPORT)) {
console.error('Usage: node wiggum-fix.js <eval-report.json> [file-to-fix.js]');
process.exit(1);
}
const report = JSON.parse(fs.readFileSync(EVAL_REPORT, 'utf8'));
// Find failures (score < 80)
const failures = report.results.filter(r => r.composite < 80).slice(0, 3); // Top 3
if (failures.length === 0) {
console.log('No major failures to fix.');
process.exit(0);
}
const sourceCode = fs.readFileSync(FILE_TO_FIX, 'utf8');
const prompt = `You are a Senior Engineer fixing a documentation generator script.
The script (${FILE_TO_FIX}) generates markdown documentation from a codebase graph.
We ran an AI agent against the generated docs using a set of ground-truth questions. The agent failed to answer the following questions because the required information was missing, incomplete, or not navigable in the generated docs.
FAILURES TO FIX:
${failures.map(f => `Question ID: ${f.id}
Category: ${f.category}
Question: ${f.question}
Ground Truth Answer: ${f.groundTruth}
Agent Answer: ${f.llmAnswer}
Score: ${f.composite}% (Accuracy: ${f.score.accuracy}, Completeness: ${f.score.completeness}, Precision: ${f.score.precision}, Navigation: ${f.score.navigation})
Notes: ${f.score.notes || ''}
Agent Files Checked: ${f.filesRead.join(', ')}
---`).join('\n')}
DIAGNOSIS TASK:
1. Identify WHY the agent failed. Is the data missing from the docs entirely? Is it buried in the wrong file? Is the index missing a link? Is the layout confusing?
2. Propose a concrete code change to ${FILE_TO_FIX} to add or fix this documentation. For example: Adding a new table to the index file, enriching the output of a specific markdown file, adding a new section for cross-cutting details, etc.
You MUST provide your fix as a fully rewritten function or a large continuous block of code that we can copy-paste to replace the old section. Do NOT provide diffs or snippets.
Your response must be a JSON object with this exact structure:
{
"diagnosis": "Short explanation of the failure root causes.",
"fixDescription": "What you are changing in the code.",
"targetFunctionOrSection": "The name of the function or section you are replacing (e.g. generateIndexDoc, buildChartDoc).",
"oldCodeSnippet": "A snippet of the existing code you are replacing to help us find it.",
"newCode": "The fully rewritten code block to replace it with."
}`;
async function run() {
console.log('Brainstorming fix for ' + failures.length + ' failures using claude-sonnet-4.6...');
try {
const raw = await callLLM(prompt, { model: 'claude-sonnet-4.6', json: true });
// Attempt to apply the fix
const fix = JSON.parse(raw);
console.log('Diagnosis:', fix.diagnosis);
console.log('Fix:', fix.fixDescription);
// Apply string replace
if (fix.oldCodeSnippet && fix.newCode) {
// Clean up the snippet slightly to increase match chance
const searchStr = fix.oldCodeSnippet.trim();
const code = fs.readFileSync(FILE_TO_FIX, 'utf8');
if (code.includes(searchStr)) {
const newCode = code.replace(searchStr, fix.newCode);
fs.writeFileSync(FILE_TO_FIX, newCode);
console.log('✅ Fix applied to ' + FILE_TO_FIX);
process.exit(0);
} else {
console.error('❌ Could not find exact match for oldCodeSnippet in ' + FILE_TO_FIX);
console.log('Expected snippet:\n', searchStr.substring(0, 100) + '...');
process.exit(1);
}
}
} catch (err) {
console.error('Failed to generate or apply fix:', err.message);
process.exit(1);
}
}
run();