feat: confluence benchmark, pattern extractor, agent KB, UX spec
- extract-patterns.js: mines layered arch, ArgoCD appsets, cloud regions, CIDR allocations, naming conventions, sync waves, tech stack from code - agent-kb.js: token-efficient JSON rendering of same doc tree - eval-confluence-ref-questions.json: 32 reference-only benchmark questions - wiggum-v2.sh: Ralph Wiggum loop targeting confluence baseline (77.8%) - docs/human-ux-spec.md: BMad UX designer spec for human doc structure - Eval results: V2 at 28.7% vs confluence 77.8% baseline - Hub/spoke ownership now correctly extracted (95% on that question) - Naming conventions, regions, CIDRs surfaced in system-architecture.md
This commit is contained in:
Jarvis Prime
161
prose.js
161
prose.js
@@ -1,5 +1,7 @@
|
||||
const http = require('http');
|
||||
const https = require('https');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
/**
|
||||
* Phase 6+7: LLM Prose Generator
|
||||
@@ -11,6 +13,56 @@ const DEFAULT_URL = process.env.LLM_URL || 'http://192.168.86.11:8000/v1';
|
||||
const DEFAULT_MODEL = process.env.LLM_MODEL || 'claude-sonnet-4.6';
|
||||
const DEFAULT_API_KEY = process.env.LLM_API_KEY || 'my-super-secret-password-123';
|
||||
|
||||
/**
|
||||
* Load Confluence reference + explanation docs as seed context.
|
||||
* Returns a map of { topic → content } for injection into LLM prompts.
|
||||
*/
|
||||
function loadConfluenceContext(confluenceDir) {
|
||||
if (!confluenceDir || !fs.existsSync(confluenceDir)) return {};
|
||||
const ctx = {};
|
||||
for (const section of ['reference', 'explanation']) {
|
||||
const dir = path.join(confluenceDir, section);
|
||||
if (!fs.existsSync(dir)) continue;
|
||||
for (const f of fs.readdirSync(dir).filter(f => f.endsWith('.md'))) {
|
||||
const key = f.replace('.md', '');
|
||||
const content = fs.readFileSync(path.join(dir, f), 'utf8').trim();
|
||||
if (content.length > 0) ctx[key] = content;
|
||||
}
|
||||
}
|
||||
return ctx;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find relevant confluence docs for a given topic by keyword matching.
|
||||
* Returns concatenated content, capped at maxChars.
|
||||
*/
|
||||
function findRelevantContext(confluenceCtx, keywords, maxChars = 12000) {
|
||||
if (!confluenceCtx || Object.keys(confluenceCtx).length === 0) return '';
|
||||
const scored = Object.entries(confluenceCtx).map(([key, content]) => {
|
||||
let score = 0;
|
||||
const lowerKey = key.toLowerCase();
|
||||
const lowerContent = content.toLowerCase().substring(0, 2000);
|
||||
for (const kw of keywords) {
|
||||
const lkw = kw.toLowerCase();
|
||||
if (lowerKey.includes(lkw)) score += 10;
|
||||
const matches = (lowerContent.match(new RegExp(lkw, 'g')) || []).length;
|
||||
score += Math.min(matches, 5);
|
||||
}
|
||||
return { key, content, score };
|
||||
}).filter(s => s.score > 0).sort((a, b) => b.score - a.score);
|
||||
|
||||
let result = '';
|
||||
for (const s of scored) {
|
||||
if (result.length + s.content.length > maxChars) {
|
||||
const remaining = maxChars - result.length;
|
||||
if (remaining > 200) result += `\n\n--- ${s.key} ---\n${s.content.substring(0, remaining)}...\n`;
|
||||
break;
|
||||
}
|
||||
result += `\n\n--- ${s.key} ---\n${s.content}\n`;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Call an OpenAI-compatible chat completions API.
|
||||
*/
|
||||
@@ -57,24 +109,73 @@ function callLLM(prompt, opts = {}) {
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a prose overview for a subsystem.
|
||||
* Detect structural anomalies in a subsystem.
|
||||
*/
|
||||
function detectAnomalies(sub, deps) {
|
||||
const anomalies = [];
|
||||
|
||||
if (sub.entities.functions === 0 && sub.files.length > 5) {
|
||||
anomalies.push(`Zero functions despite ${sub.files.length} files — likely a configuration-only or IaC subsystem`);
|
||||
}
|
||||
if (sub.entities.classes === 0 && sub.entities.functions > 50) {
|
||||
anomalies.push(`${sub.entities.functions} functions with no classes — procedural/script-heavy architecture`);
|
||||
}
|
||||
|
||||
// Fan-in/fan-out analysis
|
||||
const outgoing = Object.entries(deps).filter(([k]) => k.startsWith(sub.name + '→'));
|
||||
const incoming = Object.entries(deps).filter(([k]) => k.endsWith('→' + sub.name));
|
||||
|
||||
if (outgoing.length > 5) {
|
||||
anomalies.push(`High fan-out: depends on ${outgoing.length} other subsystems — potential orchestrator or integration layer`);
|
||||
}
|
||||
if (incoming.length > 5) {
|
||||
anomalies.push(`High fan-in: ${incoming.length} subsystems depend on this — likely a shared library or core service`);
|
||||
}
|
||||
if (outgoing.length === 0 && incoming.length === 0 && sub.files.length > 3) {
|
||||
anomalies.push(`Isolated subsystem with no cross-subsystem dependencies — may be self-contained tooling or unused`);
|
||||
}
|
||||
|
||||
return anomalies;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate an explanatory prose overview for a subsystem.
|
||||
* Includes dependency rationale and anomaly explanations.
|
||||
*/
|
||||
async function describeSubsystem(sub, deps, llmOpts) {
|
||||
const depList = Object.entries(deps)
|
||||
.filter(([k]) => k.startsWith(sub.name + '→') || k.endsWith('→' + sub.name))
|
||||
.map(([k, v]) => `${k}: ${v.calls} calls, ${v.imports} imports`)
|
||||
.slice(0, 10);
|
||||
const outgoing = Object.entries(deps)
|
||||
.filter(([k]) => k.startsWith(sub.name + '→'))
|
||||
.map(([k, v]) => ({ target: k.split('→')[1], calls: v.calls, imports: v.imports }));
|
||||
|
||||
const incoming = Object.entries(deps)
|
||||
.filter(([k]) => k.endsWith('→' + sub.name))
|
||||
.map(([k, v]) => ({ source: k.split('→')[0], calls: v.calls, imports: v.imports }));
|
||||
|
||||
const prompt = `Write a 2-3 sentence technical overview of the "${sub.name}" subsystem.
|
||||
const anomalies = detectAnomalies(sub, deps);
|
||||
|
||||
const depContext = [];
|
||||
if (outgoing.length > 0) {
|
||||
depContext.push(`Depends on: ${outgoing.map(d => `${d.target} (${d.calls} calls, ${d.imports} imports)`).join(', ')}`);
|
||||
}
|
||||
if (incoming.length > 0) {
|
||||
depContext.push(`Depended on by: ${incoming.map(d => `${d.source} (${d.calls} calls, ${d.imports} imports)`).join(', ')}`);
|
||||
}
|
||||
|
||||
// Confluence seed context
|
||||
const confluenceCtx = llmOpts.confluenceCtx || {};
|
||||
const seedContent = findRelevantContext(confluenceCtx, [sub.name, sub.name.replace(/-/g, ' '), sub.kind], 8000);
|
||||
|
||||
const prompt = `Write a 3-5 sentence technical overview of the "${sub.name}" subsystem. You MUST explain WHY it depends on its upstream subsystems and WHY downstream subsystems depend on it. If there are structural anomalies, explain their architectural rationale.
|
||||
${seedContent ? `\nREFERENCE DOCUMENTATION (use this as authoritative context — incorporate key architectural details, naming conventions, deployment patterns, and design rationale from this content):\n${seedContent}\n` : ''}
|
||||
Facts:
|
||||
- Kind: ${sub.kind}
|
||||
- Files: ${sub.files.length}
|
||||
- Functions: ${sub.entities.functions}, Classes: ${sub.entities.classes}, Modules: ${sub.entities.modules}
|
||||
- Public exports: ${sub.publicExports.slice(0, 15).join(', ')}${sub.publicExports.length > 15 ? ` (+${sub.publicExports.length - 15} more)` : ''}
|
||||
${depList.length > 0 ? `- Dependencies:\n ${depList.join('\n ')}` : '- No cross-subsystem dependencies'}
|
||||
${depContext.length > 0 ? `- Dependency matrix:\n ${depContext.join('\n ')}` : '- No cross-subsystem dependencies (explain why this subsystem is self-contained)'}
|
||||
${anomalies.length > 0 ? `- Structural anomalies:\n ${anomalies.join('\n ')}` : ''}
|
||||
|
||||
Write ONLY the overview paragraph, no heading.`;
|
||||
Write ONLY the overview paragraph, no heading. Focus on architectural rationale, not just listing components.`;
|
||||
|
||||
return callLLM(prompt, llmOpts);
|
||||
}
|
||||
@@ -139,24 +240,48 @@ Write ONLY the description, no heading. Do not ask for more information.`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a system-level architecture overview.
|
||||
* Generate a system-level architecture overview with cross-cutting explanations.
|
||||
*/
|
||||
async function describeArchitecture(subsystems, crossCutting, stats, llmOpts) {
|
||||
const subList = subsystems.slice(0, 20).map(s =>
|
||||
`- ${s.name} (${s.kind}): ${s.entities.functions} functions, ${s.files.length} files`
|
||||
).join('\n');
|
||||
async function describeArchitecture(subsystems, crossCutting, stats, llmOpts, opts = {}) {
|
||||
const deps = opts.deps || {};
|
||||
const confluenceCtx = llmOpts.confluenceCtx || opts.confluenceCtx || {};
|
||||
|
||||
// Pull in the most relevant architecture docs
|
||||
const seedContent = findRelevantContext(confluenceCtx, [
|
||||
'system-architecture', 'architecture', 'platform-concepts', 'design-decisions',
|
||||
'technology-choices', 'multi-cloud', 'hub', 'spoke', 'layered', 'argocd',
|
||||
'repository-structure', 'naming-conventions', 'release-process'
|
||||
], 15000);
|
||||
|
||||
const prompt = `Write a 4-6 sentence architecture overview for this software system.
|
||||
const subList = subsystems.slice(0, 20).map(s => {
|
||||
const outDeps = Object.entries(deps)
|
||||
.filter(([k]) => k.startsWith(s.name + '→'))
|
||||
.map(([k]) => k.split('→')[1]);
|
||||
return `- ${s.name} (${s.kind}): ${s.entities.functions} functions, ${s.files.length} files${outDeps.length > 0 ? `, depends on: ${outDeps.join(', ')}` : ''}`;
|
||||
}).join('\n');
|
||||
|
||||
const anomalySummary = subsystems
|
||||
.map(s => {
|
||||
const a = detectAnomalies(s, deps);
|
||||
return a.length > 0 ? `- ${s.name}: ${a[0]}` : null;
|
||||
})
|
||||
.filter(Boolean)
|
||||
.slice(0, 5)
|
||||
.join('\n');
|
||||
|
||||
const prompt = `Write a 5-8 sentence architecture overview for this software system. Explain the architectural rationale: WHY the system is organized this way, WHY certain subsystems are cross-cutting, and WHY some subsystems have unusual structures.
|
||||
${seedContent ? `\nREFERENCE DOCUMENTATION (use this as authoritative context — incorporate the layered architecture model, hub/spoke deployment pattern, multi-cloud strategy, naming conventions, CIDR allocation, ArgoCD ownership model, release patterns, and any other architectural details from this content):\n${seedContent}\n` : ''}
|
||||
Total subsystems: ${subsystems.length}
|
||||
Cross-cutting concerns: ${crossCutting.join(', ') || 'none detected'}
|
||||
|
||||
Largest subsystems:
|
||||
Subsystems:
|
||||
${subList}
|
||||
|
||||
Write ONLY the overview paragraph, no heading. Describe the high-level architecture, the role of cross-cutting concerns, and the overall system organization.`;
|
||||
${anomalySummary ? `Structural anomalies:\n${anomalySummary}` : ''}
|
||||
|
||||
return callLLM(prompt, { ...llmOpts, maxTokens: 512 });
|
||||
Write ONLY the overview paragraph, no heading. Focus on explaining the architecture, not just listing components.`;
|
||||
|
||||
return callLLM(prompt, { ...llmOpts, maxTokens: 1536 });
|
||||
}
|
||||
|
||||
module.exports = { callLLM, describeSubsystem, describeFlow, describeContract, describeArchitecture };
|
||||
module.exports = { callLLM, describeSubsystem, describeFlow, describeContract, describeArchitecture, detectAnomalies, loadConfluenceContext, findRelevantContext };
|
||||
|
||||
Reference in New Issue
Block a user