b8403be96c
- NEW: repo-profiler.js — deterministic archetype detection (Infra, Frontend, Backend, etc.) - NEW: extract-dynamic.js — generic extractor replacing hardcoded Foxtrot patterns - NEW: eval-generator.js — dynamic ground-truth question generation from any repo graph - NEW: specs/bmad-agnostic-refactor-spec.md — full BMad spec with acceptance criteria - REFACTORED: prose.js — two-pass LLM synthesis with rich context (shared secrets, ports, service refs) - REFACTORED: sysdoc.js — wired repo-profiler + extract-dynamic, --legacy escape hatch - REFACTORED: wiggum-v2.sh — uses eval-generator before benchmarks - FIXED: graph.js — _edgeSet rebuilt on loadSnapshot() (edge dedup was broken) - FIXED: graph.js — recursive sortKeys() for deep equality in diffing - FIXED: prose.js — robust JSON array extraction from LLM output - FIXED: ratchet.js — syntax validation (node --check) before saving LLM mutations - FIXED: extract-dynamic.js — centralized state services regex, added console.warn for silent failures - TESTS: test-eval-generator, test-repo-profiler, test-synthesis-quality + mock fixtures Eval: 81.5% on Foxtrot (fully repo-agnostic, no hardcoded reference pages) BMad reviews: Architect B+, Dev Lead B-, TEA B-
73 lines
2.9 KiB
JavaScript
73 lines
2.9 KiB
JavaScript
const assert = require('node:assert');
|
|
const fs = require('fs');
|
|
const path = require('path');
|
|
const os = require('os');
|
|
const { generateQuestions } = require('../eval-generator');
|
|
|
|
async function runTests() {
|
|
console.log('Running test-eval-generator.js...');
|
|
|
|
// Create a temp directory with mock snapshot and minimal repo structure
|
|
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'eval-gen-test-'));
|
|
const mockRepoDir = path.join(tmpDir, 'repo');
|
|
fs.mkdirSync(mockRepoDir, { recursive: true });
|
|
|
|
// Write the mock graph as a snapshot file
|
|
const mockGraph = require('./fixtures/mock-express-graph.json');
|
|
const snapshotPath = path.join(tmpDir, 'snapshot.json');
|
|
fs.writeFileSync(snapshotPath, JSON.stringify(mockGraph));
|
|
|
|
// Create a minimal package.json so repo-profiler can detect it
|
|
fs.writeFileSync(path.join(mockRepoDir, 'package.json'), JSON.stringify({
|
|
name: 'test-api',
|
|
dependencies: { express: '^4.18.0' }
|
|
}));
|
|
|
|
try {
|
|
// 1. Test that generator runs without crashing on a small repo
|
|
const questions = generateQuestions(snapshotPath, mockRepoDir);
|
|
|
|
// 2. Test that questions are returned as an array
|
|
assert.ok(Array.isArray(questions), 'Should return an array');
|
|
console.log(` Generated ${questions.length} questions from mock repo`);
|
|
|
|
// 3. Test that each question has required fields
|
|
for (const q of questions) {
|
|
assert.ok(q.question, 'Missing question field');
|
|
assert.ok(q.expected_answer || q.answer, 'Missing expected_answer/answer field');
|
|
assert.ok(q.category, 'Missing category field');
|
|
assert.ok(q.source_entity || q.source, 'Missing source_entity/source field');
|
|
}
|
|
|
|
// 4. Test no Foxtrot-specific strings
|
|
for (const q of questions) {
|
|
const str = JSON.stringify(q).toLowerCase();
|
|
assert.ok(!str.includes('foxtrot'), `Should not contain "foxtrot": ${q.question}`);
|
|
assert.ok(!str.includes('vpc_cidr'), `Should not contain "vpc_cidr": ${q.question}`);
|
|
assert.ok(!str.includes('mdm-app'), `Should not contain "mdm-app": ${q.question}`);
|
|
}
|
|
|
|
// 5. Test that a real repo (Foxtrot) generates >= 20 questions
|
|
const foxtrotSnapshot = path.resolve(__dirname, '../snapshots/foxtrot-clean.json');
|
|
const foxtrotRepo = '/home/node/.openclaw/workspace/agents/max/foxtrot/';
|
|
if (fs.existsSync(foxtrotSnapshot) && fs.existsSync(foxtrotRepo)) {
|
|
const foxtrotQuestions = generateQuestions(foxtrotSnapshot, foxtrotRepo);
|
|
assert.ok(foxtrotQuestions.length >= 20,
|
|
`Foxtrot should generate >= 20 questions, got ${foxtrotQuestions.length}`);
|
|
console.log(` Foxtrot: ${foxtrotQuestions.length} questions generated`);
|
|
} else {
|
|
console.log(' Skipping Foxtrot integration test (snapshot/repo not found)');
|
|
}
|
|
|
|
console.log('✅ eval-generator tests passed!');
|
|
} finally {
|
|
// Cleanup
|
|
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
runTests().catch(err => {
|
|
console.error(err);
|
|
process.exit(1);
|
|
});
|