max/dev-intel-v2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Phase 9b: structural documentation improvements\n\n- sysdoc.js: Added Summary Statistics, Top Charts, and K8s Resource Types to architecture doc\n- Addresses ratchet failures where system-wide rollups were missing from generated prose\n- Eval v2 shows minor improvement, though RAG context window still limits wide scatter-gather queries

Browse Source
This commit is contained in:
Jarvis Prime
2026-03-09 23:40:07 +00:00
parent b99341e8bc
commit 0cc4abcb0f
5 changed files with 1452 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

419
eval-questions-iter1.json Normal file
View File

@@ -0,0 +1,419 @@
{
"generated": "2026-03-09T23:20:55.589Z",
"count": 33,
"questions": [
{
"id": "struct-001",
"category": "structural",
"difficulty": "easy",
"audience": [
"human",
"machine"
],
"question": "How many Helm charts are defined across the Foxtrot monorepo?",
"answer": "76",
"answerType": "exact",
"source": "Chart.yaml discovery"
},
{
"id": "struct-002",
"category": "structural",
"difficulty": "medium",
"audience": [
"human",
"machine"
],
"question": "Which 5 Helm charts produce the most Kubernetes resources? List them with their resource counts.",
"answer": "cluster (app-common/apps-provider/addons/charts/common/cluster): 55\ncluster (app-common/charts/cluster): 32\njenkins (app-common/charts/jenkins): 29\nkube-state-metrics (app-common/charts/otel-collector/charts/kube-state-metrics): 27\nfoxtrot-eks (compute-common/templates/foxtrot-eks): 24",
"answerType": "ranked-list",
"source": "template scanning"
},
{
"id": "struct-003",
"category": "structural",
"difficulty": "easy",
"audience": [
"human",
"machine"
],
"question": "How many subsystems does the Foxtrot codebase contain?",
"answer": "11",
"answerType": "exact",
"source": "subsystem aggregation"
},
{
"id": "struct-004",
"category": "structural",
"difficulty": "easy",
"audience": [
"human"
],
"question": "Which subsystem contains the most files, and how many?",
"answer": "runtime: 16309 files",
"answerType": "exact",
"source": "subsystem aggregation"
},
{
"id": "dep-external-dns-001",
"category": "dependency",
"difficulty": "medium",
"audience": [
"human",
"machine"
],
"question": "What are the dependencies of the \"external-dns\" chart (at app-common/apps-provider/addons/charts/common/external-dns)?",
"answer": "external-dns (1.17.0) [condition: external-dns.enabled]",
"answerType": "list",
"source": "app-common/apps-provider/addons/charts/common/external-dns/Chart.yaml"
},
{
"id": "dep-ingress-nginx-001",
"category": "dependency",
"difficulty": "medium",
"audience": [
"human",
"machine"
],
"question": "What are the dependencies of the \"ingress-nginx\" chart (at app-common/apps-provider/addons/charts/common/ingress-nginx)?",
"answer": "ingress-nginx (4.12.0) [condition: ingress-nginx.enabled]",
"answerType": "list",
"source": "app-common/apps-provider/addons/charts/common/ingress-nginx/Chart.yaml"
},
{
"id": "dep-kube-state-metrics-001",
"category": "dependency",
"difficulty": "medium",
"audience": [
"human",
"machine"
],
"question": "What are the dependencies of the \"kube-state-metrics\" chart (at app-common/apps-provider/addons/charts/common/kube-state-metrics)?",
"answer": "kube-state-metrics (5.26.0) [condition: kube-state-metrics.enabled]",
"answerType": "list",
"source": "app-common/apps-provider/addons/charts/common/kube-state-metrics/Chart.yaml"
},
{
"id": "dep-open-telemetry-001",
"category": "dependency",
"difficulty": "medium",
"audience": [
"human",
"machine"
],
"question": "What are the dependencies of the \"open-telemetry\" chart (at app-common/apps-provider/addons/charts/common/otel-collector)?",
"answer": "opentelemetry-operator (0.26.4) [condition: opentelemetry-operator.enabled]\nk8s-metrics (0.1.0) [condition: k8s-metrics.enabled]\nagent (0.1.1) [condition: agent.enabled]\ncommon (0.1.0) [condition: common.enabled]\ngw-metrics (0.1.0) [condition: gw-metrics.enabled]\ngw-traces (0.1.0) [condition: gw-traces.enabled]\nprom-scraper (0.1.0) [condition: prom-scraper.enabled]",
"answerType": "list",
"source": "app-common/apps-provider/addons/charts/common/otel-collector/Chart.yaml"
},
{
"id": "dep-reltio-prometheus-001",
"category": "dependency",
"difficulty": "medium",
"audience": [
"human",
"machine"
],
"question": "What are the dependencies of the \"reltio-prometheus\" chart (at app-common/apps-provider/addons/charts/common/prometheus)?",
"answer": "kube-prometheus-stack (69.4.1) [condition: kube-prometheus-stack.enabled]\nprometheus-adapter (4.11.0) [condition: prometheus-adapter.enabled]",
"answerType": "list",
"source": "app-common/apps-provider/addons/charts/common/prometheus/Chart.yaml"
},
{
"id": "contract-001",
"category": "contract",
"difficulty": "medium",
"audience": [
"human",
"machine"
],
"question": "Which secrets or ConfigMaps are shared across multiple Helm charts? List each with the charts that use it.",
"answer": "logdna-agent-key: cluster\nlogdna-agent-tags: cluster\nvault-secret: ai-app, cleanse-app, pms-app, rdm-app, shared-app\nvault-secret-rdm-tasks: ai-app, cleanse-app, pms-app, rdm-app, shared-app",
"answerType": "list",
"source": "template interaction scanning"
},
{
"id": "config-mdm-app-domain",
"category": "configuration",
"difficulty": "easy",
"audience": [
"human"
],
"question": "What is the default value of \"domain\" in the mdm-app chart?",
"answer": "cloud.reltio.com",
"answerType": "exact",
"source": "app-common/charts/mdm-app/values.yaml"
},
{
"id": "config-mdm-app-count",
"category": "configuration",
"difficulty": "easy",
"audience": [
"machine"
],
"question": "How many top-level configuration keys does the mdm-app chart (at app-common/charts/mdm-app) expose in its values.yaml?",
"answer": "38",
"answerType": "exact",
"source": "app-common/charts/mdm-app/values.yaml"
},
{
"id": "config-ai-app-domain",
"category": "configuration",
"difficulty": "easy",
"audience": [
"human"
],
"question": "What is the default value of \"domain\" in the ai-app chart?",
"answer": "cloud.reltio.com",
"answerType": "exact",
"source": "app-common/charts/ai-app/values.yaml"
},
{
"id": "config-ai-app-count",
"category": "configuration",
"difficulty": "easy",
"audience": [
"machine"
],
"question": "How many top-level configuration keys does the ai-app chart (at app-common/charts/ai-app) expose in its values.yaml?",
"answer": "34",
"answerType": "exact",
"source": "app-common/charts/ai-app/values.yaml"
},
{
"id": "config-elasticsearch-domain",
"category": "configuration",
"difficulty": "easy",
"audience": [
"human"
],
"question": "What is the default value of \"domain\" in the elasticsearch chart?",
"answer": "etalon.reltio.com",
"answerType": "exact",
"source": "app-common/apps-provider/addons/charts/state-apps/es/values.yaml"
},
{
"id": "config-elasticsearch-count",
"category": "configuration",
"difficulty": "easy",
"audience": [
"machine"
],
"question": "How many top-level configuration keys does the elasticsearch chart (at app-common/apps-provider/addons/charts/state-apps/es) expose in its values.yaml?",
"answer": "32",
"answerType": "exact",
"source": "app-common/apps-provider/addons/charts/state-apps/es/values.yaml"
},
{
"id": "config-hazelcast-domain",
"category": "configuration",
"difficulty": "easy",
"audience": [
"human"
],
"question": "What is the default value of \"domain\" in the hazelcast chart?",
"answer": "etalon.reltio.com",
"answerType": "exact",
"source": "app-common/apps-provider/addons/charts/state-apps/hz/values.yaml"
},
{
"id": "config-hazelcast-count",
"category": "configuration",
"difficulty": "easy",
"audience": [
"machine"
],
"question": "How many top-level configuration keys does the hazelcast chart (at app-common/apps-provider/addons/charts/state-apps/hz) expose in its values.yaml?",
"answer": "32",
"answerType": "exact",
"source": "app-common/apps-provider/addons/charts/state-apps/hz/values.yaml"
},
{
"id": "config-redis-domain",
"category": "configuration",
"difficulty": "easy",
"audience": [
"human"
],
"question": "What is the default value of \"domain\" in the redis chart?",
"answer": "internal.reltio.com",
"answerType": "exact",
"source": "app-common/charts/redis/values.yaml"
},
{
"id": "config-redis-count",
"category": "configuration",
"difficulty": "easy",
"audience": [
"machine"
],
"question": "How many top-level configuration keys does the redis chart (at app-common/charts/redis) expose in its values.yaml?",
"answer": "23",
"answerType": "exact",
"source": "app-common/charts/redis/values.yaml"
},
{
"id": "interaction-001",
"category": "interaction",
"difficulty": "medium",
"audience": [
"human",
"machine"
],
"question": "Which Helm charts reference Kubernetes services from other components? List each chart and the service it calls.",
"answer": "hazelcast (app-common/apps-provider/addons/charts/state-apps/hz) → kubernetes\nhazelcast (app-common/charts/hazelcast) → kubernetes\ngw-metrics (app-common/charts/otel-collector/charts/gw-metrics) → argo-rollouts-metrics\nredis (app-common/charts/redis) → headless",
"answerType": "list",
"source": "template interaction scanning"
},
{
"id": "interaction-002",
"category": "interaction",
"difficulty": "hard",
"audience": [
"human",
"machine"
],
"question": "Which network ports are used by multiple Helm charts? List each port and the charts that expose it.",
"answer": "Port 443: cluster, twistlock-defender, eck-operator, twistlock\nPort 8889: gw-metrics, gw-traces, kube-state-metrics\nPort 9200: eck-operator, elasticsearch\nPort 13133: agent, gw-metrics, gw-traces, k8s-metrics, prom-scraper, elasticsearch",
"answerType": "list",
"source": "template port scanning"
},
{
"id": "resource-001",
"category": "resource",
"difficulty": "medium",
"audience": [
"human",
"machine"
],
"question": "What are the 10 most common Kubernetes resource types generated across all Foxtrot Helm charts?",
"answer": "ClusterRole: 51\nServiceAccount: 49\nSecret: 43\nExternalSecret: 38\nService: 36\nRole: 30\nClusterRoleBinding: 29\nDeployment: 29\nConfigMap: 25\nRoleBinding: 25",
"answerType": "ranked-list",
"source": "template resource scanning"
},
{
"id": "resource-mdm-app-001",
"category": "resource",
"difficulty": "medium",
"audience": [
"human",
"machine"
],
"question": "What Kubernetes resource types does the mdm-app chart generate? List each type and count.",
"answer": "Secret: 5\nJob: 4\nDeployment: 2\nPersistentVolumeClaim: 2\nService: 2\nExternalSecret: 2\nHorizontalPodAutoscaler: 1\nRollout: 1\nPersistentVolume: 1\nPodDisruptionBudget: 1\nServiceAccount: 1",
"answerType": "list",
"source": "app-common/charts/mdm-app/templates/"
},
{
"id": "resource-cassandra-001",
"category": "resource",
"difficulty": "medium",
"audience": [
"human",
"machine"
],
"question": "What Kubernetes resource types does the cassandra chart generate? List each type and count.",
"answer": "Service: 2\nStatefulSet: 1\nExternalSecret: 1\nClusterSecretStore: 1\nConfigMap: 1\nSecret: 1\nServiceAccount: 1\nRole: 1\nRoleBinding: 1\nClusterRole: 1\nClusterRoleBinding: 1\nCronJob: 1\nJob: 1",
"answerType": "list",
"source": "app-common/apps-provider/addons/charts/state-cluster/cassandra/templates/"
},
{
"id": "resource-jenkins-001",
"category": "resource",
"difficulty": "medium",
"audience": [
"human",
"machine"
],
"question": "What Kubernetes resource types does the jenkins chart generate? List each type and count.",
"answer": "ConfigMap: 5\nService: 3\nSecret: 3\nIngress: 2\nServiceAccount: 2\nPersistentVolumeClaim: 1\nSecurityGroupPolicy: 1\nPrometheusRule: 1\nBackendConfig: 1\nNetworkPolicy: 1\nPodDisruptionBudget: 1\nPodMonitoring: 1\nRoute: 1\nServiceMonitor: 1\nStatefulSet: 1\nRole: 1\nRoleBinding: 1\nSecretClaim: 1\nPod: 1",
"answerType": "list",
"source": "app-common/charts/jenkins/templates/"
},
{
"id": "cross-sub-001",
"category": "cross-subsystem",
"difficulty": "hard",
"audience": [
"human",
"machine"
],
"question": "Which subsystems depend on app-tools, and how heavily (by call+import count)?",
"answer": "ipam-tools: 6\ncompute-tools: 4\napp-common: 1",
"answerType": "list",
"source": "dependency matrix"
},
{
"id": "cross-sub-002",
"category": "cross-subsystem",
"difficulty": "easy",
"audience": [
"human"
],
"question": "Which subsystems are identified as cross-cutting concerns?",
"answer": "app-tools",
"answerType": "list",
"source": "subsystem aggregation"
},
{
"id": "arch-001",
"category": "architectural",
"difficulty": "hard",
"audience": [
"human"
],
"question": "The following subsystems have 0 detected functions and 0 modules: account-common, network-common, network-core. Why might this be the case, and what do they actually contain?",
"answer": "These subsystems primarily contain Helm charts with Go-templated YAML, Terraform HCL, and Crossplane compositions. The code analysis pipeline detects functions/modules from Python, Go, TypeScript, and shell scripts — but Helm templates use Go template syntax ({{ }}) which doesn't produce traditional function/module entities. Their content is captured through the Helm chart extraction phase instead.",
"answerType": "explanation",
"source": "architectural analysis"
},
{
"id": "arch-002",
"category": "architectural",
"difficulty": "easy",
"audience": [
"human",
"machine"
],
"question": "What is the current version and appVersion of the mdm-app Helm chart?",
"answer": "version: 0.7.4, appVersion: 2025.12-12-develop",
"answerType": "exact",
"source": "app-common/charts/mdm-app/Chart.yaml"
},
{
"id": "scenario-001",
"category": "scenario",
"difficulty": "hard",
"audience": [
"human"
],
"question": "If you need to rotate the \"vault-secret\" shared secret, which Helm charts would be affected and need redeployment?",
"answer": "ai-app, cleanse-app, pms-app, rdm-app, shared-app",
"answerType": "list",
"source": "template interaction scanning"
},
{
"id": "scenario-002",
"category": "scenario",
"difficulty": "hard",
"audience": [
"human"
],
"question": "A new engineer needs to deploy the MDM application stack. What charts, configuration values, and external dependencies should they understand first?",
"answer": "Chart: mdm-app (app-common/charts/mdm-app), Version: 0.7.4\nKey values: clusterName, environmentType, arch, cloudProvider, region, vaultEnv, imagePullPolicy, clusterDomain, terminationGracePeriodSeconds, etalonsubDomain (+28 more)\nResources generated: 22 K8s resources\nInteractions: none detected",
"answerType": "explanation",
"source": "app-common/charts/mdm-app"
},
{
"id": "scenario-003",
"category": "scenario",
"difficulty": "medium",
"audience": [
"human"
],
"question": "What network ports does the Cassandra chart expose, and what are they typically used for?",
"answer": "Ports: 7000, 7001, 7199, 9042, 9160\n7000: Cassandra inter-node cluster communication\n7001: Cassandra SSL inter-node\n7199: JMX monitoring\n9042: CQL native transport (client connections)\n9160: Thrift client API (legacy)",
"answerType": "explanation",
"source": "app-common/apps-provider/addons/charts/state-cluster/cassandra/templates/"
}
]
}