73 lines
2.9 KiB
JavaScript
73 lines
2.9 KiB
JavaScript
|
const assert = require('node:assert');
|
||
|
|
const fs = require('fs');
|
||
|
|
const path = require('path');
|
||
|
|
const os = require('os');
|
||
|
|
const { generateQuestions } = require('../eval-generator');
|
||
|
|
|
||
|
|
async function runTests() {
|
||
|
|
console.log('Running test-eval-generator.js...');
|
||
|
|
|
||
|
|
// Create a temp directory with mock snapshot and minimal repo structure
|
||
|
|
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'eval-gen-test-'));
|
||
|
|
const mockRepoDir = path.join(tmpDir, 'repo');
|
||
|
|
fs.mkdirSync(mockRepoDir, { recursive: true });
|
||
|
|
|
||
|
|
// Write the mock graph as a snapshot file
|
||
|
|
const mockGraph = require('./fixtures/mock-express-graph.json');
|
||
|
|
const snapshotPath = path.join(tmpDir, 'snapshot.json');
|
||
|
|
fs.writeFileSync(snapshotPath, JSON.stringify(mockGraph));
|
||
|
|
|
||
|
|
// Create a minimal package.json so repo-profiler can detect it
|
||
|
|
fs.writeFileSync(path.join(mockRepoDir, 'package.json'), JSON.stringify({
|
||
|
|
name: 'test-api',
|
||
|
|
dependencies: { express: '^4.18.0' }
|
||
|
|
}));
|
||
|
|
|
||
|
|
try {
|
||
|
|
// 1. Test that generator runs without crashing on a small repo
|
||
|
|
const questions = generateQuestions(snapshotPath, mockRepoDir);
|
||
|
|
|
||
|
|
// 2. Test that questions are returned as an array
|
||
|
|
assert.ok(Array.isArray(questions), 'Should return an array');
|
||
|
|
console.log(` Generated ${questions.length} questions from mock repo`);
|
||
|
|
|
||
|
|
// 3. Test that each question has required fields
|
||
|
|
for (const q of questions) {
|
||
|
|
assert.ok(q.question, 'Missing question field');
|
||
|
|
assert.ok(q.expected_answer || q.answer, 'Missing expected_answer/answer field');
|
||
|
|
assert.ok(q.category, 'Missing category field');
|
||
|
|
assert.ok(q.source_entity || q.source, 'Missing source_entity/source field');
|
||
|
|
}
|
||
|
|
|
||
|
|
// 4. Test no Foxtrot-specific strings
|
||
|
|
for (const q of questions) {
|
||
|
|
const str = JSON.stringify(q).toLowerCase();
|
||
|
|
assert.ok(!str.includes('foxtrot'), `Should not contain "foxtrot": ${q.question}`);
|
||
|
|
assert.ok(!str.includes('vpc_cidr'), `Should not contain "vpc_cidr": ${q.question}`);
|
||
|
|
assert.ok(!str.includes('mdm-app'), `Should not contain "mdm-app": ${q.question}`);
|
||
|
|
}
|
||
|
|
|
||
|
|
// 5. Test that a real repo (Foxtrot) generates >= 20 questions
|
||
|
|
const foxtrotSnapshot = path.resolve(__dirname, '../snapshots/foxtrot-clean.json');
|
||
|
|
const foxtrotRepo = '/home/node/.openclaw/workspace/agents/max/foxtrot/';
|
||
|
|
if (fs.existsSync(foxtrotSnapshot) && fs.existsSync(foxtrotRepo)) {
|
||
|
|
const foxtrotQuestions = generateQuestions(foxtrotSnapshot, foxtrotRepo);
|
||
|
|
assert.ok(foxtrotQuestions.length >= 20,
|
||
|
|
`Foxtrot should generate >= 20 questions, got ${foxtrotQuestions.length}`);
|
||
|
|
console.log(` Foxtrot: ${foxtrotQuestions.length} questions generated`);
|
||
|
|
} else {
|
||
|
|
console.log(' Skipping Foxtrot integration test (snapshot/repo not found)');
|
||
|
|
}
|
||
|
|
|
||
|
|
console.log('✅ eval-generator tests passed!');
|
||
|
|
} finally {
|
||
|
|
// Cleanup
|
||
|
|
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
runTests().catch(err => {
|
||
|
|
console.error(err);
|
||
|
|
process.exit(1);
|
||
|
|
});
|