b8403be96c
- NEW: repo-profiler.js — deterministic archetype detection (Infra, Frontend, Backend, etc.) - NEW: extract-dynamic.js — generic extractor replacing hardcoded Foxtrot patterns - NEW: eval-generator.js — dynamic ground-truth question generation from any repo graph - NEW: specs/bmad-agnostic-refactor-spec.md — full BMad spec with acceptance criteria - REFACTORED: prose.js — two-pass LLM synthesis with rich context (shared secrets, ports, service refs) - REFACTORED: sysdoc.js — wired repo-profiler + extract-dynamic, --legacy escape hatch - REFACTORED: wiggum-v2.sh — uses eval-generator before benchmarks - FIXED: graph.js — _edgeSet rebuilt on loadSnapshot() (edge dedup was broken) - FIXED: graph.js — recursive sortKeys() for deep equality in diffing - FIXED: prose.js — robust JSON array extraction from LLM output - FIXED: ratchet.js — syntax validation (node --check) before saving LLM mutations - FIXED: extract-dynamic.js — centralized state services regex, added console.warn for silent failures - TESTS: test-eval-generator, test-repo-profiler, test-synthesis-quality + mock fixtures Eval: 81.5% on Foxtrot (fully repo-agnostic, no hardcoded reference pages) BMad reviews: Architect B+, Dev Lead B-, TEA B-
401 lines
19 KiB
JSON
401 lines
19 KiB
JSON
{
|
|
"generated": "2026-03-11T07:22:28.865Z",
|
|
"count": 27,
|
|
"questions": [
|
|
{
|
|
"id": "q-001",
|
|
"category": "structural",
|
|
"difficulty": "easy",
|
|
"audience": [
|
|
"human",
|
|
"machine"
|
|
],
|
|
"question": "How many Helm charts are defined across this repository?",
|
|
"expected_answer": "76",
|
|
"answer": "76",
|
|
"answerType": "exact",
|
|
"source": "Chart.yaml discovery",
|
|
"source_entity": "all charts"
|
|
},
|
|
{
|
|
"id": "q-002",
|
|
"category": "structural",
|
|
"difficulty": "easy",
|
|
"audience": [
|
|
"human",
|
|
"machine"
|
|
],
|
|
"question": "How many subsystems does this codebase contain?",
|
|
"expected_answer": "12",
|
|
"answer": "12",
|
|
"answerType": "exact",
|
|
"source": "subsystem aggregation",
|
|
"source_entity": "all subsystems"
|
|
},
|
|
{
|
|
"id": "q-003",
|
|
"category": "structural",
|
|
"difficulty": "medium",
|
|
"audience": [
|
|
"human",
|
|
"machine"
|
|
],
|
|
"question": "Which 5 Helm charts produce the most Kubernetes resources? List them with their resource counts.",
|
|
"expected_answer": "cluster (app-common/apps-provider/addons/charts/common/cluster): 55\ncluster (app-common/charts/cluster): 32\njenkins (app-common/charts/jenkins): 29\nkube-state-metrics (app-common/charts/otel-collector/charts/kube-state-metrics): 27\nfoxtrot-eks (compute-common/templates/foxtrot-eks): 24",
|
|
"answer": "cluster (app-common/apps-provider/addons/charts/common/cluster): 55\ncluster (app-common/charts/cluster): 32\njenkins (app-common/charts/jenkins): 29\nkube-state-metrics (app-common/charts/otel-collector/charts/kube-state-metrics): 27\nfoxtrot-eks (compute-common/templates/foxtrot-eks): 24",
|
|
"answerType": "ranked-list",
|
|
"source": "template scanning",
|
|
"source_entity": "cluster, cluster, jenkins, kube-state-metrics, foxtrot-eks"
|
|
},
|
|
{
|
|
"id": "q-004",
|
|
"category": "structural",
|
|
"difficulty": "easy",
|
|
"audience": [
|
|
"human"
|
|
],
|
|
"question": "Which subsystem contains the most files, and how many?",
|
|
"expected_answer": "runtime: 16309 files",
|
|
"answer": "runtime: 16309 files",
|
|
"answerType": "exact",
|
|
"source": "subsystem aggregation",
|
|
"source_entity": "runtime"
|
|
},
|
|
{
|
|
"id": "q-005",
|
|
"category": "dependency",
|
|
"difficulty": "medium",
|
|
"audience": [
|
|
"human",
|
|
"machine"
|
|
],
|
|
"question": "What are the dependencies of the \"external-dns\" chart (at app-common/apps-provider/addons/charts/common/external-dns)?",
|
|
"expected_answer": "external-dns (1.17.0) [condition: external-dns.enabled]",
|
|
"answer": "external-dns (1.17.0) [condition: external-dns.enabled]",
|
|
"answerType": "list",
|
|
"source": "app-common/apps-provider/addons/charts/common/external-dns/Chart.yaml",
|
|
"source_entity": "external-dns"
|
|
},
|
|
{
|
|
"id": "q-006",
|
|
"category": "dependency",
|
|
"difficulty": "medium",
|
|
"audience": [
|
|
"human",
|
|
"machine"
|
|
],
|
|
"question": "What are the dependencies of the \"ingress-nginx\" chart (at app-common/apps-provider/addons/charts/common/ingress-nginx)?",
|
|
"expected_answer": "ingress-nginx (4.12.0) [condition: ingress-nginx.enabled]",
|
|
"answer": "ingress-nginx (4.12.0) [condition: ingress-nginx.enabled]",
|
|
"answerType": "list",
|
|
"source": "app-common/apps-provider/addons/charts/common/ingress-nginx/Chart.yaml",
|
|
"source_entity": "ingress-nginx"
|
|
},
|
|
{
|
|
"id": "q-007",
|
|
"category": "dependency",
|
|
"difficulty": "medium",
|
|
"audience": [
|
|
"human",
|
|
"machine"
|
|
],
|
|
"question": "What are the dependencies of the \"kube-state-metrics\" chart (at app-common/apps-provider/addons/charts/common/kube-state-metrics)?",
|
|
"expected_answer": "kube-state-metrics (5.26.0) [condition: kube-state-metrics.enabled]",
|
|
"answer": "kube-state-metrics (5.26.0) [condition: kube-state-metrics.enabled]",
|
|
"answerType": "list",
|
|
"source": "app-common/apps-provider/addons/charts/common/kube-state-metrics/Chart.yaml",
|
|
"source_entity": "kube-state-metrics"
|
|
},
|
|
{
|
|
"id": "q-008",
|
|
"category": "dependency",
|
|
"difficulty": "medium",
|
|
"audience": [
|
|
"human",
|
|
"machine"
|
|
],
|
|
"question": "What are the dependencies of the \"open-telemetry\" chart (at app-common/apps-provider/addons/charts/common/otel-collector)?",
|
|
"expected_answer": "opentelemetry-operator (0.26.4) [condition: opentelemetry-operator.enabled]\nk8s-metrics (0.1.0) [condition: k8s-metrics.enabled]\nagent (0.1.1) [condition: agent.enabled]\ncommon (0.1.0) [condition: common.enabled]\ngw-metrics (0.1.0) [condition: gw-metrics.enabled]\ngw-traces (0.1.0) [condition: gw-traces.enabled]\nprom-scraper (0.1.0) [condition: prom-scraper.enabled]",
|
|
"answer": "opentelemetry-operator (0.26.4) [condition: opentelemetry-operator.enabled]\nk8s-metrics (0.1.0) [condition: k8s-metrics.enabled]\nagent (0.1.1) [condition: agent.enabled]\ncommon (0.1.0) [condition: common.enabled]\ngw-metrics (0.1.0) [condition: gw-metrics.enabled]\ngw-traces (0.1.0) [condition: gw-traces.enabled]\nprom-scraper (0.1.0) [condition: prom-scraper.enabled]",
|
|
"answerType": "list",
|
|
"source": "app-common/apps-provider/addons/charts/common/otel-collector/Chart.yaml",
|
|
"source_entity": "open-telemetry"
|
|
},
|
|
{
|
|
"id": "q-009",
|
|
"category": "dependency",
|
|
"difficulty": "medium",
|
|
"audience": [
|
|
"human",
|
|
"machine"
|
|
],
|
|
"question": "What are the dependencies of the \"reltio-prometheus\" chart (at app-common/apps-provider/addons/charts/common/prometheus)?",
|
|
"expected_answer": "kube-prometheus-stack (69.4.1) [condition: kube-prometheus-stack.enabled]\nprometheus-adapter (4.11.0) [condition: prometheus-adapter.enabled]",
|
|
"answer": "kube-prometheus-stack (69.4.1) [condition: kube-prometheus-stack.enabled]\nprometheus-adapter (4.11.0) [condition: prometheus-adapter.enabled]",
|
|
"answerType": "list",
|
|
"source": "app-common/apps-provider/addons/charts/common/prometheus/Chart.yaml",
|
|
"source_entity": "reltio-prometheus"
|
|
},
|
|
{
|
|
"id": "q-010",
|
|
"category": "contract",
|
|
"difficulty": "medium",
|
|
"audience": [
|
|
"human",
|
|
"machine"
|
|
],
|
|
"question": "Which secrets or ConfigMaps are shared across multiple Helm charts? List each with the charts that use it.",
|
|
"expected_answer": "logdna-agent-key: cluster\nlogdna-agent-tags: cluster\nvault-secret: ai-app, cleanse-app, pms-app, rdm-app, shared-app\nvault-secret-rdm-tasks: ai-app, cleanse-app, pms-app, rdm-app, shared-app",
|
|
"answer": "logdna-agent-key: cluster\nlogdna-agent-tags: cluster\nvault-secret: ai-app, cleanse-app, pms-app, rdm-app, shared-app\nvault-secret-rdm-tasks: ai-app, cleanse-app, pms-app, rdm-app, shared-app",
|
|
"answerType": "list",
|
|
"source": "template interaction scanning",
|
|
"source_entity": "logdna-agent-key, logdna-agent-tags, vault-secret, vault-secret-rdm-tasks"
|
|
},
|
|
{
|
|
"id": "q-011",
|
|
"category": "configuration",
|
|
"difficulty": "easy",
|
|
"audience": [
|
|
"machine"
|
|
],
|
|
"question": "How many top-level configuration keys does the twistlock-defender chart (at app-common/apps-provider/addons/charts/common/twistlock) expose in its values.yaml?",
|
|
"expected_answer": "60",
|
|
"answer": "60",
|
|
"answerType": "exact",
|
|
"source": "app-common/apps-provider/addons/charts/common/twistlock/values.yaml",
|
|
"source_entity": "twistlock-defender"
|
|
},
|
|
{
|
|
"id": "q-012",
|
|
"category": "configuration",
|
|
"difficulty": "easy",
|
|
"audience": [
|
|
"machine"
|
|
],
|
|
"question": "How many top-level configuration keys does the twistlock chart (at app-common/charts/twistlock) expose in its values.yaml?",
|
|
"expected_answer": "60",
|
|
"answer": "60",
|
|
"answerType": "exact",
|
|
"source": "app-common/charts/twistlock/values.yaml",
|
|
"source_entity": "twistlock"
|
|
},
|
|
{
|
|
"id": "q-013",
|
|
"category": "configuration",
|
|
"difficulty": "easy",
|
|
"audience": [
|
|
"machine"
|
|
],
|
|
"question": "How many top-level configuration keys does the kube-state-metrics chart (at app-common/charts/otel-collector/charts/kube-state-metrics) expose in its values.yaml?",
|
|
"expected_answer": "54",
|
|
"answer": "54",
|
|
"answerType": "exact",
|
|
"source": "app-common/charts/otel-collector/charts/kube-state-metrics/values.yaml",
|
|
"source_entity": "kube-state-metrics"
|
|
},
|
|
{
|
|
"id": "q-014",
|
|
"category": "configuration",
|
|
"difficulty": "easy",
|
|
"audience": [
|
|
"machine"
|
|
],
|
|
"question": "How many top-level configuration keys does the milvus chart (at app-common/charts/milvus) expose in its values.yaml?",
|
|
"expected_answer": "50",
|
|
"answer": "50",
|
|
"answerType": "exact",
|
|
"source": "app-common/charts/milvus/values.yaml",
|
|
"source_entity": "milvus"
|
|
},
|
|
{
|
|
"id": "q-015",
|
|
"category": "configuration",
|
|
"difficulty": "easy",
|
|
"audience": [
|
|
"machine"
|
|
],
|
|
"question": "How many top-level configuration keys does the cluster chart (at app-common/charts/cluster) expose in its values.yaml?",
|
|
"expected_answer": "41",
|
|
"answer": "41",
|
|
"answerType": "exact",
|
|
"source": "app-common/charts/cluster/values.yaml",
|
|
"source_entity": "cluster"
|
|
},
|
|
{
|
|
"id": "q-016",
|
|
"category": "interaction",
|
|
"difficulty": "medium",
|
|
"audience": [
|
|
"human",
|
|
"machine"
|
|
],
|
|
"question": "Which Helm charts reference Kubernetes services from other components? List each chart and the service it calls.",
|
|
"expected_answer": "hazelcast (app-common/apps-provider/addons/charts/state-apps/hz) → kubernetes\nhazelcast (app-common/charts/hazelcast) → kubernetes\ngw-metrics (app-common/charts/otel-collector/charts/gw-metrics) → argo-rollouts-metrics\nredis (app-common/charts/redis) → headless",
|
|
"answer": "hazelcast (app-common/apps-provider/addons/charts/state-apps/hz) → kubernetes\nhazelcast (app-common/charts/hazelcast) → kubernetes\ngw-metrics (app-common/charts/otel-collector/charts/gw-metrics) → argo-rollouts-metrics\nredis (app-common/charts/redis) → headless",
|
|
"answerType": "list",
|
|
"source": "template interaction scanning",
|
|
"source_entity": "hazelcast, hazelcast, gw-metrics, redis"
|
|
},
|
|
{
|
|
"id": "q-017",
|
|
"category": "interaction",
|
|
"difficulty": "hard",
|
|
"audience": [
|
|
"human",
|
|
"machine"
|
|
],
|
|
"question": "Which network ports are used by multiple Helm charts? List each port and the charts that expose it.",
|
|
"expected_answer": "Port 443: cluster, twistlock-defender, eck-operator, twistlock\nPort 8889: gw-metrics, gw-traces, kube-state-metrics\nPort 9200: eck-operator, elasticsearch\nPort 13133: agent, gw-metrics, gw-traces, k8s-metrics, prom-scraper, elasticsearch",
|
|
"answer": "Port 443: cluster, twistlock-defender, eck-operator, twistlock\nPort 8889: gw-metrics, gw-traces, kube-state-metrics\nPort 9200: eck-operator, elasticsearch\nPort 13133: agent, gw-metrics, gw-traces, k8s-metrics, prom-scraper, elasticsearch",
|
|
"answerType": "list",
|
|
"source": "template port scanning",
|
|
"source_entity": "shared ports"
|
|
},
|
|
{
|
|
"id": "q-018",
|
|
"category": "resource",
|
|
"difficulty": "medium",
|
|
"audience": [
|
|
"human",
|
|
"machine"
|
|
],
|
|
"question": "What are the most common Kubernetes resource types generated across all Helm charts?",
|
|
"expected_answer": "ClusterRole: 51\nServiceAccount: 49\nSecret: 43\nExternalSecret: 38\nService: 36\nRole: 30\nClusterRoleBinding: 29\nDeployment: 29\nConfigMap: 25\nRoleBinding: 25",
|
|
"answer": "ClusterRole: 51\nServiceAccount: 49\nSecret: 43\nExternalSecret: 38\nService: 36\nRole: 30\nClusterRoleBinding: 29\nDeployment: 29\nConfigMap: 25\nRoleBinding: 25",
|
|
"answerType": "ranked-list",
|
|
"source": "template resource scanning",
|
|
"source_entity": "all charts"
|
|
},
|
|
{
|
|
"id": "q-019",
|
|
"category": "resource",
|
|
"difficulty": "medium",
|
|
"audience": [
|
|
"human",
|
|
"machine"
|
|
],
|
|
"question": "What Kubernetes resource types does the cluster chart generate? List each type and count.",
|
|
"expected_answer": "Secret: 6\nExternalSecret: 5\nServiceAccount: 5\nClusterRole: 5\nClusterRoleBinding: 5\nNamespace: 4\nStorageClass: 4\nDeployment: 3\nClusterSecretStore: 3\nConfigMap: 2\nRole: 2\nRoleBinding: 2\nDaemonSet: 2\nService: 2\nClusterIssuer: 1\nAzureIdentity: 1\nAzureIdentityBinding: 1\nAPIService: 1\nClusterExternalSecret: 1",
|
|
"answer": "Secret: 6\nExternalSecret: 5\nServiceAccount: 5\nClusterRole: 5\nClusterRoleBinding: 5\nNamespace: 4\nStorageClass: 4\nDeployment: 3\nClusterSecretStore: 3\nConfigMap: 2\nRole: 2\nRoleBinding: 2\nDaemonSet: 2\nService: 2\nClusterIssuer: 1\nAzureIdentity: 1\nAzureIdentityBinding: 1\nAPIService: 1\nClusterExternalSecret: 1",
|
|
"answerType": "list",
|
|
"source": "app-common/apps-provider/addons/charts/common/cluster/templates/",
|
|
"source_entity": "cluster"
|
|
},
|
|
{
|
|
"id": "q-020",
|
|
"category": "resource",
|
|
"difficulty": "medium",
|
|
"audience": [
|
|
"human",
|
|
"machine"
|
|
],
|
|
"question": "What Kubernetes resource types does the cluster chart generate? List each type and count.",
|
|
"expected_answer": "ExternalSecret: 4\nStorageClass: 4\nSecret: 3\nClusterSecretStore: 3\nNamespace: 2\nConfigMap: 2\nServiceAccount: 2\nClusterRole: 2\nClusterRoleBinding: 2\nDaemonSet: 2\nClusterIssuer: 1\nRole: 1\nRoleBinding: 1\nService: 1\nDeployment: 1\nClusterExternalSecret: 1",
|
|
"answer": "ExternalSecret: 4\nStorageClass: 4\nSecret: 3\nClusterSecretStore: 3\nNamespace: 2\nConfigMap: 2\nServiceAccount: 2\nClusterRole: 2\nClusterRoleBinding: 2\nDaemonSet: 2\nClusterIssuer: 1\nRole: 1\nRoleBinding: 1\nService: 1\nDeployment: 1\nClusterExternalSecret: 1",
|
|
"answerType": "list",
|
|
"source": "app-common/charts/cluster/templates/",
|
|
"source_entity": "cluster"
|
|
},
|
|
{
|
|
"id": "q-021",
|
|
"category": "resource",
|
|
"difficulty": "medium",
|
|
"audience": [
|
|
"human",
|
|
"machine"
|
|
],
|
|
"question": "What Kubernetes resource types does the jenkins chart generate? List each type and count.",
|
|
"expected_answer": "ConfigMap: 5\nService: 3\nSecret: 3\nIngress: 2\nServiceAccount: 2\nPersistentVolumeClaim: 1\nSecurityGroupPolicy: 1\nPrometheusRule: 1\nBackendConfig: 1\nNetworkPolicy: 1\nPodDisruptionBudget: 1\nPodMonitoring: 1\nRoute: 1\nServiceMonitor: 1\nStatefulSet: 1\nRole: 1\nRoleBinding: 1\nSecretClaim: 1\nPod: 1",
|
|
"answer": "ConfigMap: 5\nService: 3\nSecret: 3\nIngress: 2\nServiceAccount: 2\nPersistentVolumeClaim: 1\nSecurityGroupPolicy: 1\nPrometheusRule: 1\nBackendConfig: 1\nNetworkPolicy: 1\nPodDisruptionBudget: 1\nPodMonitoring: 1\nRoute: 1\nServiceMonitor: 1\nStatefulSet: 1\nRole: 1\nRoleBinding: 1\nSecretClaim: 1\nPod: 1",
|
|
"answerType": "list",
|
|
"source": "app-common/charts/jenkins/templates/",
|
|
"source_entity": "jenkins"
|
|
},
|
|
{
|
|
"id": "q-022",
|
|
"category": "cross-subsystem",
|
|
"difficulty": "easy",
|
|
"audience": [
|
|
"human"
|
|
],
|
|
"question": "Which subsystems are identified as cross-cutting concerns?",
|
|
"expected_answer": "root",
|
|
"answer": "root",
|
|
"answerType": "list",
|
|
"source": "subsystem aggregation",
|
|
"source_entity": "root"
|
|
},
|
|
{
|
|
"id": "q-023",
|
|
"category": "cross-subsystem",
|
|
"difficulty": "hard",
|
|
"audience": [
|
|
"human",
|
|
"machine"
|
|
],
|
|
"question": "Which subsystems depend on root, and how heavily (by call+import count)?",
|
|
"expected_answer": "app-tools: 98\nipam-tools: 78\ncompute-tools: 70\nruntime: 42\ncompute-common: 30",
|
|
"answer": "app-tools: 98\nipam-tools: 78\ncompute-tools: 70\nruntime: 42\ncompute-common: 30",
|
|
"answerType": "list",
|
|
"source": "dependency matrix",
|
|
"source_entity": "root"
|
|
},
|
|
{
|
|
"id": "q-024",
|
|
"category": "architectural",
|
|
"difficulty": "hard",
|
|
"audience": [
|
|
"human"
|
|
],
|
|
"question": "The following subsystems have 0 detected functions and 0 modules: account-common, network-common, network-core, root. Why might this be the case, and what do they actually contain?",
|
|
"expected_answer": "These subsystems primarily contain Helm charts with Go-templated YAML, Terraform HCL, and/or Crossplane compositions. The code analysis pipeline detects functions/modules from Python, Go, TypeScript, and shell scripts — but Helm templates use Go template syntax which does not produce traditional function/module entities. Their content is captured through the Helm chart extraction phase instead.",
|
|
"answer": "These subsystems primarily contain Helm charts, Terraform, or Crossplane compositions rather than traditional code.",
|
|
"answerType": "explanation",
|
|
"source": "architectural analysis",
|
|
"source_entity": "account-common, network-common, network-core, root"
|
|
},
|
|
{
|
|
"id": "q-025",
|
|
"category": "architectural",
|
|
"difficulty": "easy",
|
|
"audience": [
|
|
"human",
|
|
"machine"
|
|
],
|
|
"question": "What is the current version and appVersion of the cluster Helm chart?",
|
|
"expected_answer": "version: 1.0.0, appVersion: 1.0",
|
|
"answer": "version: 1.0.0, appVersion: 1.0",
|
|
"answerType": "exact",
|
|
"source": "app-common/apps-provider/addons/charts/common/cluster/Chart.yaml",
|
|
"source_entity": "cluster"
|
|
},
|
|
{
|
|
"id": "q-026",
|
|
"category": "scenario",
|
|
"difficulty": "hard",
|
|
"audience": [
|
|
"human"
|
|
],
|
|
"question": "If you need to rotate the \"logdna-agent-key\" shared secret, which Helm charts would be affected and need redeployment?",
|
|
"expected_answer": "cluster",
|
|
"answer": "cluster",
|
|
"answerType": "list",
|
|
"source": "template interaction scanning",
|
|
"source_entity": "logdna-agent-key"
|
|
},
|
|
{
|
|
"id": "q-027",
|
|
"category": "scenario",
|
|
"difficulty": "hard",
|
|
"audience": [
|
|
"human"
|
|
],
|
|
"question": "A new engineer needs to deploy the cluster application. What charts, configuration values, and external dependencies should they understand first?",
|
|
"expected_answer": "Chart: cluster (app-common/apps-provider/addons/charts/common/cluster), Version: 1.0.0\nKey values: cloud, arch, clusterName, namespaceHacks, deploySentinelOne, deployPrisma, prismaWSAddress, deployMetricsServer, metricsServerRegistry, metricsServerVersion (+30 more)\nResources generated: 55 K8s resources\nInteractions: config-ref: logdna-agent-key, config-ref: logdna-agent-tags, port: 4443, port: 443, port: 6443",
|
|
"answer": "Chart: cluster (app-common/apps-provider/addons/charts/common/cluster), Version: 1.0.0",
|
|
"answerType": "explanation",
|
|
"source": "app-common/apps-provider/addons/charts/common/cluster",
|
|
"source_entity": "cluster"
|
|
}
|
|
]
|
|
} |