feat: confluence benchmark, pattern extractor, agent KB, UX spec
- extract-patterns.js: mines layered arch, ArgoCD appsets, cloud regions, CIDR allocations, naming conventions, sync waves, tech stack from code - agent-kb.js: token-efficient JSON rendering of same doc tree - eval-confluence-ref-questions.json: 32 reference-only benchmark questions - wiggum-v2.sh: Ralph Wiggum loop targeting confluence baseline (77.8%) - docs/human-ux-spec.md: BMad UX designer spec for human doc structure - Eval results: V2 at 28.7% vs confluence 77.8% baseline - Hub/spoke ownership now correctly extracted (95% on that question) - Naming conventions, regions, CIDRs surfaced in system-architecture.md
This commit is contained in:
87
wiggum-fix.js
Normal file
87
wiggum-fix.js
Normal file
@@ -0,0 +1,87 @@
|
||||
const fs = require('fs');
|
||||
const { callLLM } = require('./prose.js');
|
||||
|
||||
const EVAL_REPORT = process.argv[2];
|
||||
const FILE_TO_FIX = process.argv[3] || './sysdoc.js';
|
||||
|
||||
if (!EVAL_REPORT || !fs.existsSync(EVAL_REPORT)) {
|
||||
console.error('Usage: node wiggum-fix.js <eval-report.json> [file-to-fix.js]');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const report = JSON.parse(fs.readFileSync(EVAL_REPORT, 'utf8'));
|
||||
|
||||
// Find failures (score < 80)
|
||||
const failures = report.results.filter(r => r.composite < 80).slice(0, 3); // Top 3
|
||||
if (failures.length === 0) {
|
||||
console.log('No major failures to fix.');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const sourceCode = fs.readFileSync(FILE_TO_FIX, 'utf8');
|
||||
|
||||
const prompt = `You are a Senior Engineer fixing a documentation generator script.
|
||||
|
||||
The script (${FILE_TO_FIX}) generates markdown documentation from a codebase graph.
|
||||
We ran an AI agent against the generated docs using a set of ground-truth questions. The agent failed to answer the following questions because the required information was missing, incomplete, or not navigable in the generated docs.
|
||||
|
||||
FAILURES TO FIX:
|
||||
${failures.map(f => `Question ID: ${f.id}
|
||||
Category: ${f.category}
|
||||
Question: ${f.question}
|
||||
Ground Truth Answer: ${f.groundTruth}
|
||||
Agent Answer: ${f.llmAnswer}
|
||||
Score: ${f.composite}% (Accuracy: ${f.score.accuracy}, Completeness: ${f.score.completeness}, Precision: ${f.score.precision}, Navigation: ${f.score.navigation})
|
||||
Notes: ${f.score.notes || ''}
|
||||
Agent Files Checked: ${f.filesRead.join(', ')}
|
||||
---`).join('\n')}
|
||||
|
||||
DIAGNOSIS TASK:
|
||||
1. Identify WHY the agent failed. Is the data missing from the docs entirely? Is it buried in the wrong file? Is the index missing a link? Is the layout confusing?
|
||||
2. Propose a concrete code change to ${FILE_TO_FIX} to add or fix this documentation. For example: Adding a new table to the index file, enriching the output of a specific markdown file, adding a new section for cross-cutting details, etc.
|
||||
|
||||
You MUST provide your fix as a fully rewritten function or a large continuous block of code that we can copy-paste to replace the old section. Do NOT provide diffs or snippets.
|
||||
|
||||
Your response must be a JSON object with this exact structure:
|
||||
{
|
||||
"diagnosis": "Short explanation of the failure root causes.",
|
||||
"fixDescription": "What you are changing in the code.",
|
||||
"targetFunctionOrSection": "The name of the function or section you are replacing (e.g. generateIndexDoc, buildChartDoc).",
|
||||
"oldCodeSnippet": "A snippet of the existing code you are replacing to help us find it.",
|
||||
"newCode": "The fully rewritten code block to replace it with."
|
||||
}`;
|
||||
|
||||
async function run() {
|
||||
console.log('Brainstorming fix for ' + failures.length + ' failures using claude-sonnet-4.6...');
|
||||
try {
|
||||
const raw = await callLLM(prompt, { model: 'claude-sonnet-4.6', json: true });
|
||||
|
||||
// Attempt to apply the fix
|
||||
const fix = JSON.parse(raw);
|
||||
console.log('Diagnosis:', fix.diagnosis);
|
||||
console.log('Fix:', fix.fixDescription);
|
||||
|
||||
// Apply string replace
|
||||
if (fix.oldCodeSnippet && fix.newCode) {
|
||||
// Clean up the snippet slightly to increase match chance
|
||||
const searchStr = fix.oldCodeSnippet.trim();
|
||||
const code = fs.readFileSync(FILE_TO_FIX, 'utf8');
|
||||
|
||||
if (code.includes(searchStr)) {
|
||||
const newCode = code.replace(searchStr, fix.newCode);
|
||||
fs.writeFileSync(FILE_TO_FIX, newCode);
|
||||
console.log('✅ Fix applied to ' + FILE_TO_FIX);
|
||||
process.exit(0);
|
||||
} else {
|
||||
console.error('❌ Could not find exact match for oldCodeSnippet in ' + FILE_TO_FIX);
|
||||
console.log('Expected snippet:\n', searchStr.substring(0, 100) + '...');
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Failed to generate or apply fix:', err.message);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
run();
|
||||
Reference in New Issue
Block a user