779 lines
47 KiB
JSON
779 lines
47 KiB
JSON
|
{
|
||
|
|
"evalType": "human",
|
||
|
|
"timestamp": "2026-03-09T23:55:39.520Z",
|
||
|
|
"docsDir": "./foxtrot-docs",
|
||
|
|
"totalQuestions": 28,
|
||
|
|
"overallScore": "63.9",
|
||
|
|
"avgClarity": "3.29",
|
||
|
|
"avgActionability": "2.82",
|
||
|
|
"avgCompleteness": "3.21",
|
||
|
|
"avgFindability": "3.46",
|
||
|
|
"notFoundCount": 5,
|
||
|
|
"notFoundRate": "17.9%",
|
||
|
|
"byCategory": {
|
||
|
|
"structural": {
|
||
|
|
"avg": "63.8",
|
||
|
|
"count": 4
|
||
|
|
},
|
||
|
|
"dependency": {
|
||
|
|
"avg": "20.0",
|
||
|
|
"count": 5
|
||
|
|
},
|
||
|
|
"contract": {
|
||
|
|
"avg": "85.0",
|
||
|
|
"count": 1
|
||
|
|
},
|
||
|
|
"configuration": {
|
||
|
|
"avg": "67.0",
|
||
|
|
"count": 5
|
||
|
|
},
|
||
|
|
"interaction": {
|
||
|
|
"avg": "95.0",
|
||
|
|
"count": 2
|
||
|
|
},
|
||
|
|
"resource": {
|
||
|
|
"avg": "91.3",
|
||
|
|
"count": 4
|
||
|
|
},
|
||
|
|
"cross-subsystem": {
|
||
|
|
"avg": "65.0",
|
||
|
|
"count": 2
|
||
|
|
},
|
||
|
|
"architectural": {
|
||
|
|
"avg": "87.5",
|
||
|
|
"count": 2
|
||
|
|
},
|
||
|
|
"scenario": {
|
||
|
|
"avg": "51.7",
|
||
|
|
"count": 3
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"byDifficulty": {
|
||
|
|
"easy": {
|
||
|
|
"avg": "64.0",
|
||
|
|
"count": 10
|
||
|
|
},
|
||
|
|
"medium": {
|
||
|
|
"avg": "58.1",
|
||
|
|
"count": 13
|
||
|
|
},
|
||
|
|
"hard": {
|
||
|
|
"avg": "79.0",
|
||
|
|
"count": 5
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"results": [
|
||
|
|
{
|
||
|
|
"id": "struct-001",
|
||
|
|
"category": "structural",
|
||
|
|
"difficulty": "easy",
|
||
|
|
"question": "How many Helm charts are defined across the Foxtrot monorepo?",
|
||
|
|
"groundTruth": "76",
|
||
|
|
"humanAnswer": "76",
|
||
|
|
"docsUsed": [
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/helm/index.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"clarity": 5,
|
||
|
|
"actionability": 2,
|
||
|
|
"completeness": 3,
|
||
|
|
"findability": 5,
|
||
|
|
"notes": "Reader extracted correct answer from well-organized reference docs. Helm index and system architecture are intuitive locations. However, knowing the chart count alone provides minimal actionability—engineers need chart names, purposes, and dependencies to act. Docs appear to contain the metric but likely lack operational context."
|
||
|
|
},
|
||
|
|
"composite": 75,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "struct-002",
|
||
|
|
"category": "structural",
|
||
|
|
"difficulty": "medium",
|
||
|
|
"question": "Which 5 Helm charts produce the most Kubernetes resources? List them with their resource counts.",
|
||
|
|
"groundTruth": "cluster (app-common/apps-provider/addons/charts/common/cluster): 55\ncluster (app-common/charts/cluster): 32\njenkins (app-common/charts/jenkins): 29\nkube-state-metrics (app-common/charts/otel-collector/charts/kube-state-metrics): 27\nfoxtrot-eks (compute-common/templates/foxtrot-eks): 24",
|
||
|
|
"humanAnswer": "1. cluster (`app-common/apps-provider/addons/charts/common/cluster`) - 55 resources\n2. cluster (`app-common/charts/cluster`) - 32 resources\n3. jenkins - 29 resources\n4. kube-state-metrics - 27 resources\n5. foxtrot-eks - 24 resources",
|
||
|
|
"docsUsed": [
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/helm/index.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"clarity": 4,
|
||
|
|
"actionability": 4,
|
||
|
|
"completeness": 5,
|
||
|
|
"findability": 4,
|
||
|
|
"notes": "Reader extracted all 5 charts with exact resource counts correctly. Documentation was clear enough to find complete information, though it required consulting two reference files rather than a single authoritative source. Data is actionable for capacity planning but lacks context on implications or recommended actions."
|
||
|
|
},
|
||
|
|
"composite": 85,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "struct-003",
|
||
|
|
"category": "structural",
|
||
|
|
"difficulty": "easy",
|
||
|
|
"question": "How many subsystems does the Foxtrot codebase contain?",
|
||
|
|
"groundTruth": "11",
|
||
|
|
"humanAnswer": "12",
|
||
|
|
"docsUsed": [
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/subsystems/account-common.md",
|
||
|
|
"reference/subsystems/app-tools.md",
|
||
|
|
"reference/subsystems/compute-common.md",
|
||
|
|
"reference/subsystems/compute-tools.md",
|
||
|
|
"reference/subsystems/control-core.md",
|
||
|
|
"reference/subsystems/ipam-core.md",
|
||
|
|
"reference/subsystems/ipam-tools.md",
|
||
|
|
"reference/subsystems/network-common.md",
|
||
|
|
"reference/subsystems/network-core.md",
|
||
|
|
"reference/subsystems/root.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"clarity": 2,
|
||
|
|
"actionability": 2,
|
||
|
|
"completeness": 2,
|
||
|
|
"findability": 4,
|
||
|
|
"notes": "Reader found the right documentation in logical locations but arrived at incorrect count (12 vs 11). Suggests system-architecture.md lacks explicit total count or subsystem definitions are ambiguous. Reader consulted 11 subsystem files yet answered 12, indicating either miscounting or unclear delineation of what constitutes a subsystem. Documentation structure is discoverable, but the core metric is not clearly stated."
|
||
|
|
},
|
||
|
|
"composite": 50,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "struct-004",
|
||
|
|
"category": "structural",
|
||
|
|
"difficulty": "easy",
|
||
|
|
"question": "Which subsystem contains the most files, and how many?",
|
||
|
|
"groundTruth": "runtime: 16309 files",
|
||
|
|
"humanAnswer": "The **runtime** subsystem contains the most files with **16,309** files.",
|
||
|
|
"docsUsed": [
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/subsystems/account-common.md",
|
||
|
|
"reference/subsystems/app-tools.md",
|
||
|
|
"reference/subsystems/compute-common.md",
|
||
|
|
"reference/subsystems/compute-tools.md",
|
||
|
|
"reference/subsystems/control-core.md",
|
||
|
|
"reference/subsystems/ipam-core.md",
|
||
|
|
"reference/subsystems/ipam-tools.md",
|
||
|
|
"reference/subsystems/network-common.md",
|
||
|
|
"reference/subsystems/network-core.md",
|
||
|
|
"reference/subsystems/root.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"clarity": 3,
|
||
|
|
"actionability": 2,
|
||
|
|
"completeness": 2,
|
||
|
|
"findability": 2,
|
||
|
|
"notes": "Reader found correct answer, but runtime.md is absent from consulted docs despite being the largest subsystem. Information likely buried in system-architecture.md rather than in dedicated runtime subsystem documentation. Documentation structure is incomplete—runtime should have its own reference file like other subsystems (account-common, compute-core, network-core, etc.). Metric is informational only, not actionable for engineers."
|
||
|
|
},
|
||
|
|
"composite": 45,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "dep-external-dns-001",
|
||
|
|
"category": "dependency",
|
||
|
|
"difficulty": "medium",
|
||
|
|
"question": "What are the dependencies of the \"external-dns\" chart (at app-common/apps-provider/addons/charts/common/external-dns)?",
|
||
|
|
"groundTruth": "external-dns (1.17.0) [condition: external-dns.enabled]",
|
||
|
|
"humanAnswer": "Looking at the Helm Charts index table in `reference/helm/index.md`, the external-dns chart at `app-common/apps-provider/addons/charts/common/external-dns` shows:\n\n| Chart | Path | Version | Resources | Dependencies | Interactions |\n|---|---|---|---|---|---|\n| external-dns | `app-common/apps-provider/addons/charts/common/external-dns` | 1.17.0 | 0 | **1** | 0 |\n\nThe table indicates it has **1 dependency**, but the specific name of that dependency is not listed in the provided documentation.\n\n**Answer: NOT_FOUND** (the count is documented as 1, but the specific dependency name is not provided in these docs)",
|
||
|
|
"docsUsed": [
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/helm/index.md",
|
||
|
|
"reference/subsystems/account-common.md",
|
||
|
|
"reference/subsystems/app-tools.md",
|
||
|
|
"reference/subsystems/compute-common.md",
|
||
|
|
"reference/subsystems/compute-tools.md",
|
||
|
|
"reference/subsystems/ipam-core.md",
|
||
|
|
"reference/subsystems/ipam-tools.md",
|
||
|
|
"reference/subsystems/network-common.md",
|
||
|
|
"reference/subsystems/network-core.md",
|
||
|
|
"reference/subsystems/root.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"clarity": 0,
|
||
|
|
"actionability": 0,
|
||
|
|
"completeness": 0,
|
||
|
|
"findability": 0,
|
||
|
|
"notes": "Documentation shows dependency count (1) but omits the actual dependency name and version. The table structure indicates dependencies exist without providing the essential details needed to understand or act on them. An engineer cannot determine what external-dns depends on from the provided documentation."
|
||
|
|
},
|
||
|
|
"composite": 0,
|
||
|
|
"notFound": true
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "dep-ingress-nginx-001",
|
||
|
|
"category": "dependency",
|
||
|
|
"difficulty": "medium",
|
||
|
|
"question": "What are the dependencies of the \"ingress-nginx\" chart (at app-common/apps-provider/addons/charts/common/ingress-nginx)?",
|
||
|
|
"groundTruth": "ingress-nginx (4.12.0) [condition: ingress-nginx.enabled]",
|
||
|
|
"humanAnswer": "NOT_FOUND\n\nThe documentation shows that the ingress-nginx chart at `app-common/apps-provider/addons/charts/common/ingress-nginx` has 1 dependency (from the Helm Charts index table), but the specific dependency is not listed in the provided documentation.",
|
||
|
|
"docsUsed": [
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/helm/index.md",
|
||
|
|
"reference/subsystems/account-common.md",
|
||
|
|
"reference/subsystems/app-tools.md",
|
||
|
|
"reference/subsystems/compute-common.md",
|
||
|
|
"reference/subsystems/compute-tools.md",
|
||
|
|
"reference/subsystems/ipam-core.md",
|
||
|
|
"reference/subsystems/ipam-tools.md",
|
||
|
|
"reference/subsystems/network-common.md",
|
||
|
|
"reference/subsystems/network-core.md",
|
||
|
|
"reference/subsystems/root.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"clarity": 0,
|
||
|
|
"actionability": 0,
|
||
|
|
"completeness": 0,
|
||
|
|
"findability": 0,
|
||
|
|
"notes": "Reader could not locate the ingress-nginx dependency information despite it being referenced in the Helm Charts index. The documentation indicates a dependency exists but fails to provide the actual dependency details (ingress-nginx 4.12.0 with condition flag). Critical information gap between index reference and actual documentation content."
|
||
|
|
},
|
||
|
|
"composite": 0,
|
||
|
|
"notFound": true
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "dep-kube-state-metrics-001",
|
||
|
|
"category": "dependency",
|
||
|
|
"difficulty": "medium",
|
||
|
|
"question": "What are the dependencies of the \"kube-state-metrics\" chart (at app-common/apps-provider/addons/charts/common/kube-state-metrics)?",
|
||
|
|
"groundTruth": "kube-state-metrics (5.26.0) [condition: kube-state-metrics.enabled]",
|
||
|
|
"humanAnswer": "NOT_FOUND\n\nThe documentation shows that the kube-state-metrics chart at `app-common/apps-provider/addons/charts/common/kube-state-metrics` has 1 dependency (from the Helm Charts index table), but the specific dependency is not listed in the provided documentation. The detailed chart documentation file that would contain this information is not included in the reference materials.",
|
||
|
|
"docsUsed": [
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/helm/index.md",
|
||
|
|
"reference/subsystems/account-common.md",
|
||
|
|
"reference/subsystems/app-tools.md",
|
||
|
|
"reference/subsystems/compute-common.md",
|
||
|
|
"reference/subsystems/compute-tools.md",
|
||
|
|
"reference/subsystems/ipam-core.md",
|
||
|
|
"reference/subsystems/ipam-tools.md",
|
||
|
|
"reference/subsystems/network-common.md",
|
||
|
|
"reference/subsystems/network-core.md",
|
||
|
|
"reference/subsystems/root.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"clarity": 0,
|
||
|
|
"actionability": 0,
|
||
|
|
"completeness": 0,
|
||
|
|
"findability": 0,
|
||
|
|
"notes": "Reader could not locate the dependency information. While the Helm index indicates kube-state-metrics has 1 dependency, the actual chart documentation containing the dependency details (kube-state-metrics 5.26.0 with condition flag) was not included in the consulted reference materials. The specific chart documentation file would need to be referenced directly."
|
||
|
|
},
|
||
|
|
"composite": 0,
|
||
|
|
"notFound": true
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "dep-open-telemetry-001",
|
||
|
|
"category": "dependency",
|
||
|
|
"difficulty": "medium",
|
||
|
|
"question": "What are the dependencies of the \"open-telemetry\" chart (at app-common/apps-provider/addons/charts/common/otel-collector)?",
|
||
|
|
"groundTruth": "opentelemetry-operator (0.26.4) [condition: opentelemetry-operator.enabled]\nk8s-metrics (0.1.0) [condition: k8s-metrics.enabled]\nagent (0.1.1) [condition: agent.enabled]\ncommon (0.1.0) [condition: common.enabled]\ngw-metrics (0.1.0) [condition: gw-metrics.enabled]\ngw-traces (0.1.0) [condition: gw-traces.enabled]\nprom-scraper (0.1.0) [condition: prom-scraper.enabled]",
|
||
|
|
"humanAnswer": "\n\nThe \"open-telemetry\" chart at `app-common/apps-provider/addons/charts/common/otel-collector` (version 1.0.1) has the following dependencies:\n\n- opentelemetry-operator (0.26.4)\n- k8s-metrics (0.1.0)\n- agent (0.1.1)\n- common (0.1.0)\n- gw-metrics (0.1.0)\n- gw-traces (0.1.0)\n- prom-scraper (0.1.0)\n\nAll dependencies are conditional and enabled via feature flags in the values configuration.",
|
||
|
|
"docsUsed": [
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/helm/index.md",
|
||
|
|
"reference/helm/charts/app-common-apps-provider-addons-charts-common-otel-collector-charts-agent.md",
|
||
|
|
"reference/helm/charts/app-common-apps-provider-addons-charts-common-otel-collector-charts-common.md",
|
||
|
|
"reference/helm/charts/app-common-apps-provider-addons-charts-common-otel-collector-charts-gw-metrics.md",
|
||
|
|
"reference/helm/charts/app-common-apps-provider-addons-charts-common-otel-collector-charts-gw-traces.md",
|
||
|
|
"reference/helm/charts/app-common-apps-provider-addons-charts-common-otel-collector-charts-k8s-metrics.md",
|
||
|
|
"reference/helm/charts/app-common-apps-provider-addons-charts-common-otel-collector-charts-prom-scraper.md",
|
||
|
|
"reference/helm/charts/app-common-apps-provider-addons-charts-common-otel-collector.md",
|
||
|
|
"reference/helm/charts/app-common-charts-otel-collector-charts-agent.md",
|
||
|
|
"reference/helm/charts/app-common-charts-otel-collector-charts-common.md",
|
||
|
|
"reference/helm/charts/app-common-charts-otel-collector-charts-gw-metrics.md",
|
||
|
|
"reference/helm/charts/app-common-charts-otel-collector-charts-gw-traces.md",
|
||
|
|
"reference/helm/charts/app-common-charts-otel-collector-charts-k8s-metrics.md",
|
||
|
|
"reference/helm/charts/app-common-charts-otel-collector-charts-kube-state-metrics.md",
|
||
|
|
"reference/helm/charts/app-common-charts-otel-collector-charts-opentelemetry-operator.md",
|
||
|
|
"reference/helm/charts/app-common-charts-otel-collector-charts-prom-scraper.md",
|
||
|
|
"reference/helm/charts/app-common-charts-otel-collector.md",
|
||
|
|
"reference/subsystems/account-common.md",
|
||
|
|
"reference/subsystems/app-tools.md",
|
||
|
|
"reference/subsystems/compute-tools.md",
|
||
|
|
"reference/subsystems/ipam-core.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"clarity": 5,
|
||
|
|
"actionability": 5,
|
||
|
|
"completeness": 5,
|
||
|
|
"findability": 5,
|
||
|
|
"notes": "Reader extracted all 7 dependencies with exact versions and correctly identified the conditional nature via feature flags. Documentation structure with dedicated chart files for each dependency and main chart reference enabled straightforward discovery. Information is immediately actionable for deployment configuration."
|
||
|
|
},
|
||
|
|
"composite": 100,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "dep-reltio-prometheus-001",
|
||
|
|
"category": "dependency",
|
||
|
|
"difficulty": "medium",
|
||
|
|
"question": "What are the dependencies of the \"reltio-prometheus\" chart (at app-common/apps-provider/addons/charts/common/prometheus)?",
|
||
|
|
"groundTruth": "kube-prometheus-stack (69.4.1) [condition: kube-prometheus-stack.enabled]\nprometheus-adapter (4.11.0) [condition: prometheus-adapter.enabled]",
|
||
|
|
"humanAnswer": "Looking at the Helm Charts index in `reference/helm/index.md`, the reltio-prometheus chart entry shows:\n\n| Chart | Path | Version | Resources | Dependencies | Interactions |\n|---|---|---|---|---|---|\n| [reltio-prometheus](charts/app-common-apps-provider-addons-charts-common-prometheus.md) | `app-common/apps-provider/addons/charts/common/prometheus` | 1.0.0 | 0 | 2 | 0 |\n\nThe table indicates the chart has **2 dependencies**, but the specific names of those dependencies are not listed in the provided documentation. The detailed dependency information would be in the referenced file `charts/app-common-apps-provider-addons-charts-common-prometheus.md`, which is not included in these docs.\n\n**Answer: NOT_FOUND** (specific dependency names not documented in provided materials)",
|
||
|
|
"docsUsed": [
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/helm/index.md",
|
||
|
|
"reference/subsystems/account-common.md",
|
||
|
|
"reference/subsystems/app-tools.md",
|
||
|
|
"reference/subsystems/compute-common.md",
|
||
|
|
"reference/subsystems/compute-tools.md",
|
||
|
|
"reference/subsystems/ipam-core.md",
|
||
|
|
"reference/subsystems/ipam-tools.md",
|
||
|
|
"reference/subsystems/network-common.md",
|
||
|
|
"reference/subsystems/network-core.md",
|
||
|
|
"reference/subsystems/root.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"clarity": 0,
|
||
|
|
"actionability": 0,
|
||
|
|
"completeness": 0,
|
||
|
|
"findability": 0,
|
||
|
|
"notes": "Reader answered NOT_FOUND. Documentation shows dependency count (2) but omits the actual dependency names, versions, and conditions. The referenced detail file was not provided in the documentation set, making the information inaccessible and unusable."
|
||
|
|
},
|
||
|
|
"composite": 0,
|
||
|
|
"notFound": true
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "contract-001",
|
||
|
|
"category": "contract",
|
||
|
|
"difficulty": "medium",
|
||
|
|
"question": "Which secrets or ConfigMaps are shared across multiple Helm charts? List each with the charts that use it.",
|
||
|
|
"groundTruth": "logdna-agent-key: cluster\nlogdna-agent-tags: cluster\nvault-secret: ai-app, cleanse-app, pms-app, rdm-app, shared-app\nvault-secret-rdm-tasks: ai-app, cleanse-app, pms-app, rdm-app, shared-app",
|
||
|
|
"humanAnswer": "\n\nBased on the documentation, here are the shared secrets and ConfigMaps across multiple Helm charts:\n\n| Secret/ConfigMap | Used By |\n|---|---|\n| `logdna-agent-key` | cluster, cluster |\n| `logdna-agent-tags` | cluster, cluster |\n| `vault-secret` | ai-app, cleanse-app, pms-app, rdm-app, shared-app |\n| `vault-secret-rdm-tasks` | ai-app, cleanse-app, pms-app, rdm-app, shared-app |\n\nThe `vault-secret` and `vault-secret-rdm-tasks` ConfigMaps are the most widely shared, used across five application charts. The logdna secrets are used by the cluster chart (appearing twice, likely indicating multiple uses within that chart).",
|
||
|
|
"docsUsed": [
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/helm/index.md",
|
||
|
|
"reference/contracts/index.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"clarity": 4,
|
||
|
|
"actionability": 4,
|
||
|
|
"completeness": 5,
|
||
|
|
"findability": 4,
|
||
|
|
"notes": "Reader correctly identified all four secrets and their chart associations. Minor issue: misinterpreted logdna secrets as appearing twice within cluster chart when ground truth shows single usage. Docs lack explicit distinction between 'used by' vs 'used multiple times within' a chart, creating ambiguity. All required information present and accessible from appropriate reference files."
|
||
|
|
},
|
||
|
|
"composite": 85,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "config-mdm-app-domain",
|
||
|
|
"category": "configuration",
|
||
|
|
"difficulty": "easy",
|
||
|
|
"question": "What is the default value of \"domain\" in the mdm-app chart?",
|
||
|
|
"groundTruth": "cloud.reltio.com",
|
||
|
|
"humanAnswer": "`cloud.reltio.com`",
|
||
|
|
"docsUsed": [
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/helm/index.md",
|
||
|
|
"reference/helm/charts/app-common-charts-mdm-app.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"clarity": 5,
|
||
|
|
"actionability": 5,
|
||
|
|
"completeness": 5,
|
||
|
|
"findability": 5,
|
||
|
|
"notes": "Reader extracted exact answer from appropriate Helm chart documentation. Default value was clearly documented and readily discoverable in the mdm-app chart reference file."
|
||
|
|
},
|
||
|
|
"composite": 100,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "config-ai-app-domain",
|
||
|
|
"category": "configuration",
|
||
|
|
"difficulty": "easy",
|
||
|
|
"question": "What is the default value of \"domain\" in the ai-app chart?",
|
||
|
|
"groundTruth": "cloud.reltio.com",
|
||
|
|
"humanAnswer": "`cloud.reltio.com`",
|
||
|
|
"docsUsed": [
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/helm/index.md",
|
||
|
|
"reference/helm/charts/app-common-charts-ai-app.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"clarity": 5,
|
||
|
|
"actionability": 5,
|
||
|
|
"completeness": 5,
|
||
|
|
"findability": 5,
|
||
|
|
"notes": "Reader successfully located the correct default value. The ai-app chart reference file is the logical location for Helm chart defaults, and the information was presented clearly enough for accurate extraction."
|
||
|
|
},
|
||
|
|
"composite": 100,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "config-elasticsearch-domain",
|
||
|
|
"category": "configuration",
|
||
|
|
"difficulty": "easy",
|
||
|
|
"question": "What is the default value of \"domain\" in the elasticsearch chart?",
|
||
|
|
"groundTruth": "etalon.reltio.com",
|
||
|
|
"humanAnswer": "NOT_FOUND\n\nThe elasticsearch chart documentation lists `domain` as a configuration key with type `unknown` and indicates a default exists (marked as \"yes\"), but the actual default value is not specified in the provided documentation.",
|
||
|
|
"docsUsed": [
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/helm/index.md",
|
||
|
|
"reference/helm/charts/app-common-apps-provider-addons-charts-state-cluster-elasticsearch.md",
|
||
|
|
"reference/helm/charts/app-common-charts-elasticsearch.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"clarity": 0,
|
||
|
|
"actionability": 0,
|
||
|
|
"completeness": 0,
|
||
|
|
"findability": 0,
|
||
|
|
"notes": "Documentation indicates a default value exists for 'domain' but fails to specify the actual value (etalon.reltio.com). Reader reached the correct chart documentation but found incomplete information, making the answer unfindable despite the right location being consulted."
|
||
|
|
},
|
||
|
|
"composite": 0,
|
||
|
|
"notFound": true
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "config-hazelcast-domain",
|
||
|
|
"category": "configuration",
|
||
|
|
"difficulty": "easy",
|
||
|
|
"question": "What is the default value of \"domain\" in the hazelcast chart?",
|
||
|
|
"groundTruth": "etalon.reltio.com",
|
||
|
|
"humanAnswer": "`internal.reltio.com`",
|
||
|
|
"docsUsed": [
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/helm/index.md",
|
||
|
|
"reference/helm/charts/app-common-apps-provider-addons-charts-state-cluster-hazelcast.md",
|
||
|
|
"reference/helm/charts/app-common-charts-hazelcast.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"clarity": 2,
|
||
|
|
"actionability": 1,
|
||
|
|
"completeness": 1,
|
||
|
|
"findability": 3,
|
||
|
|
"notes": "Reader located documentation in appropriate files but extracted incorrect default value (internal.reltio.com vs. etalon.reltio.com). Suggests docs either presented wrong default clearly, contained conflicting values, or lacked explicit default designation. Information was findable but inaccurate, making it actionable in the wrong direction—worse than missing entirely."
|
||
|
|
},
|
||
|
|
"composite": 35,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "config-redis-domain",
|
||
|
|
"category": "configuration",
|
||
|
|
"difficulty": "easy",
|
||
|
|
"question": "What is the default value of \"domain\" in the redis chart?",
|
||
|
|
"groundTruth": "internal.reltio.com",
|
||
|
|
"humanAnswer": "`internal.reltio.com`",
|
||
|
|
"docsUsed": [
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/helm/index.md",
|
||
|
|
"reference/helm/charts/app-common-charts-redis.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"clarity": 5,
|
||
|
|
"actionability": 5,
|
||
|
|
"completeness": 5,
|
||
|
|
"findability": 5,
|
||
|
|
"notes": "Reader located correct answer in appropriate chart reference documentation. The redis chart file is the logical location for this configuration value. Information was sufficiently clear to extract the exact default value without ambiguity."
|
||
|
|
},
|
||
|
|
"composite": 100,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "interaction-001",
|
||
|
|
"category": "interaction",
|
||
|
|
"difficulty": "medium",
|
||
|
|
"question": "Which Helm charts reference Kubernetes services from other components? List each chart and the service it calls.",
|
||
|
|
"groundTruth": "hazelcast (app-common/apps-provider/addons/charts/state-apps/hz) → kubernetes\nhazelcast (app-common/charts/hazelcast) → kubernetes\ngw-metrics (app-common/charts/otel-collector/charts/gw-metrics) → argo-rollouts-metrics\nredis (app-common/charts/redis) → headless",
|
||
|
|
"humanAnswer": "Based on the \"Service-to-Service References\" section in the documentation:\n\n- **hazelcast** → calls `kubernetes` service\n- **hazelcast** → calls `kubernetes` service (appears to be a second instance)\n- **gw-metrics** → calls `argo-rollouts-metrics` service\n- **redis** → calls `headless` service",
|
||
|
|
"docsUsed": [
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/helm/index.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"clarity": 4,
|
||
|
|
"actionability": 4,
|
||
|
|
"completeness": 5,
|
||
|
|
"findability": 5,
|
||
|
|
"notes": "Reader extracted all four chart-service references correctly from a dedicated 'Service-to-Service References' section. Information is specific and actionable, though context about downstream usage (network policies, service discovery) would enhance actionability. Placement in reference/helm and system-architecture docs aligns with natural search patterns."
|
||
|
|
},
|
||
|
|
"composite": 90,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "interaction-002",
|
||
|
|
"category": "interaction",
|
||
|
|
"difficulty": "hard",
|
||
|
|
"question": "Which network ports are used by multiple Helm charts? List each port and the charts that expose it.",
|
||
|
|
"groundTruth": "Port 443: cluster, twistlock-defender, eck-operator, twistlock\nPort 8889: gw-metrics, gw-traces, kube-state-metrics\nPort 9200: eck-operator, elasticsearch\nPort 13133: agent, gw-metrics, gw-traces, k8s-metrics, prom-scraper, elasticsearch",
|
||
|
|
"humanAnswer": "**Port 443**\n- cluster\n- twistlock-defender\n- eck-operator\n- twistlock\n\n**Port 8889**\n- gw-metrics\n- gw-traces\n- kube-state-metrics\n\n**Port 9200**\n- eck-operator\n- elasticsearch\n\n**Port 13133**\n- agent\n- gw-metrics\n- gw-traces\n- k8s-metrics\n- prom-scraper\n- elasticsearch",
|
||
|
|
"docsUsed": [
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/helm/index.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"clarity": 5,
|
||
|
|
"actionability": 5,
|
||
|
|
"completeness": 5,
|
||
|
|
"findability": 5,
|
||
|
|
"notes": "Reader extracted exact answer matching ground truth. Documentation presented port-to-chart mappings clearly with proper formatting. All four ports and their associated charts were found and correctly listed. Information was located in logical reference sections (system-architecture, helm index) where engineers would naturally search for network configuration details."
|
||
|
|
},
|
||
|
|
"composite": 100,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "resource-001",
|
||
|
|
"category": "resource",
|
||
|
|
"difficulty": "medium",
|
||
|
|
"question": "What are the 10 most common Kubernetes resource types generated across all Foxtrot Helm charts?",
|
||
|
|
"groundTruth": "ClusterRole: 51\nServiceAccount: 49\nSecret: 43\nExternalSecret: 38\nService: 36\nRole: 30\nClusterRoleBinding: 29\nDeployment: 29\nConfigMap: 25\nRoleBinding: 25",
|
||
|
|
"humanAnswer": "Based on the \"Kubernetes Resource Types (across all charts)\" table in the system architecture documentation, the 10 most common Kubernetes resource types are:\n\n1. ClusterRole — 51\n2. ServiceAccount — 49\n3. Secret — 43\n4. ExternalSecret — 38\n5. Service — 36\n6. Role — 30\n7. ClusterRoleBinding — 29\n8. Deployment — 29\n9. ConfigMap — 25\n10. RoleBinding — 25",
|
||
|
|
"docsUsed": [
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/helm/index.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"clarity": 5,
|
||
|
|
"actionability": 3,
|
||
|
|
"completeness": 5,
|
||
|
|
"findability": 4,
|
||
|
|
"notes": "Reader extracted exact ground truth from a dedicated table. Information is presented clearly with counts. Docs are in expected locations (system-architecture, helm reference). Actionability limited because data is informational rather than prescriptive—useful for understanding chart composition but lacks guidance on what to do with these metrics."
|
||
|
|
},
|
||
|
|
"composite": 85,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "resource-mdm-app-001",
|
||
|
|
"category": "resource",
|
||
|
|
"difficulty": "medium",
|
||
|
|
"question": "What Kubernetes resource types does the mdm-app chart generate? List each type and count.",
|
||
|
|
"groundTruth": "Secret: 5\nJob: 4\nDeployment: 2\nPersistentVolumeClaim: 2\nService: 2\nExternalSecret: 2\nHorizontalPodAutoscaler: 1\nRollout: 1\nPersistentVolume: 1\nPodDisruptionBudget: 1\nServiceAccount: 1",
|
||
|
|
"humanAnswer": "**Kubernetes Resource Types Generated by mdm-app Chart:**\n\n- HorizontalPodAutoscaler: 1\n- Deployment: 2\n- Rollout: 1\n- Job: 4\n- PersistentVolume: 1\n- PersistentVolumeClaim: 2\n- Service: 2\n- PodDisruptionBudget: 1\n- Secret: 5\n- ExternalSecret: 2\n- ServiceAccount: 1\n\n**Total: 22 resources**",
|
||
|
|
"docsUsed": [
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/helm/index.md",
|
||
|
|
"reference/helm/charts/app-common-charts-mdm-app.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"clarity": 5,
|
||
|
|
"actionability": 5,
|
||
|
|
"completeness": 5,
|
||
|
|
"findability": 5,
|
||
|
|
"notes": "Reader extracted all 11 resource types with correct counts. Information presented in clear, scannable format. Documentation path (app-common-charts-mdm-app.md) is exactly where an engineer would search for chart-specific resource details. Complete accuracy indicates docs contain all necessary information in accessible form."
|
||
|
|
},
|
||
|
|
"composite": 100,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "resource-cassandra-001",
|
||
|
|
"category": "resource",
|
||
|
|
"difficulty": "medium",
|
||
|
|
"question": "What Kubernetes resource types does the cassandra chart generate? List each type and count.",
|
||
|
|
"groundTruth": "Service: 2\nStatefulSet: 1\nExternalSecret: 1\nClusterSecretStore: 1\nConfigMap: 1\nSecret: 1\nServiceAccount: 1\nRole: 1\nRoleBinding: 1\nClusterRole: 1\nClusterRoleBinding: 1\nCronJob: 1\nJob: 1",
|
||
|
|
"humanAnswer": "**Kubernetes Resource Types Generated by cassandra Chart:**\n\n- Service: 2\n- StatefulSet: 1\n- ExternalSecret: 1\n- ClusterSecretStore: 1\n- ConfigMap: 1\n- Secret: 1\n- ServiceAccount: 1\n- Role: 1\n- RoleBinding: 1\n- ClusterRole: 1\n- ClusterRoleBinding: 1\n- CronJob: 1\n- Job: 1\n\n**Total: 14 resources across 13 resource types**",
|
||
|
|
"docsUsed": [
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/helm/index.md",
|
||
|
|
"reference/helm/charts/app-common-apps-provider-addons-charts-state-cluster-cassandra.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"clarity": 5,
|
||
|
|
"actionability": 4,
|
||
|
|
"completeness": 5,
|
||
|
|
"findability": 5,
|
||
|
|
"notes": "Reader extracted all 13 resource types with exact counts matching ground truth. Formatting with bullets and bold headers makes data immediately scannable. Documentation path is intuitive for cassandra chart specifics. Actionability is high for verification/planning but limited to informational rather than prescriptive guidance."
|
||
|
|
},
|
||
|
|
"composite": 95,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "resource-jenkins-001",
|
||
|
|
"category": "resource",
|
||
|
|
"difficulty": "medium",
|
||
|
|
"question": "What Kubernetes resource types does the jenkins chart generate? List each type and count.",
|
||
|
|
"groundTruth": "ConfigMap: 5\nService: 3\nSecret: 3\nIngress: 2\nServiceAccount: 2\nPersistentVolumeClaim: 1\nSecurityGroupPolicy: 1\nPrometheusRule: 1\nBackendConfig: 1\nNetworkPolicy: 1\nPodDisruptionBudget: 1\nPodMonitoring: 1\nRoute: 1\nServiceMonitor: 1\nStatefulSet: 1\nRole: 1\nRoleBinding: 1\nSecretClaim: 1\nPod: 1",
|
||
|
|
"humanAnswer": "Based on the Jenkins chart documentation, here are the Kubernetes resource types generated:\n\n- **ConfigMap**: 4\n- **Service**: 3\n- **Secret**: 3\n- **Ingress**: 2\n- **ServiceAccount**: 2\n- **PersistentVolumeClaim**: 1\n- **SecurityGroupPolicy**: 1\n- **PrometheusRule**: 1\n- **BackendConfig**: 1\n- **NetworkPolicy**: 1\n- **PodDisruptionBudget**: 1\n- **PodMonitoring**: 1\n- **Route**: 1\n- **ServiceMonitor**: 1\n- **StatefulSet**: 1\n- **Role**: 1\n- **RoleBinding**: 1\n- **SecretClaim**: 1\n- **Pod**: 1",
|
||
|
|
"docsUsed": [
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/helm/index.md",
|
||
|
|
"reference/helm/charts/app-common-charts-jenkins.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"clarity": 4,
|
||
|
|
"actionability": 4,
|
||
|
|
"completeness": 4,
|
||
|
|
"findability": 5,
|
||
|
|
"notes": "Reader successfully identified all 19 resource types and got 18/19 counts correct. ConfigMap count error (4 vs 5) indicates minor clarity issue—documentation may not have made this count explicit enough. Docs were in the expected location and contained all necessary information. Error is small enough that it doesn't significantly impact actionability, but precision matters for resource validation."
|
||
|
|
},
|
||
|
|
"composite": 85,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "cross-sub-001",
|
||
|
|
"category": "cross-subsystem",
|
||
|
|
"difficulty": "hard",
|
||
|
|
"question": "Which subsystems depend on app-tools, and how heavily (by call+import count)?",
|
||
|
|
"groundTruth": "ipam-tools: 6\ncompute-tools: 4\napp-common: 1",
|
||
|
|
"humanAnswer": "Based on the Cross-Subsystem Dependencies table in the system-architecture.md documentation:\n\n**Subsystems that depend on app-tools:**\n\n1. **ipam-tools** — 6 calls + 0 imports = **6 total**\n2. **compute-tools** — 4 calls + 0 imports = **4 total**\n3. **app-common** — 1 call + 0 imports = **1 total**\n\nipam-tools is the heaviest consumer of app-tools, followed by compute-tools. All dependencies are call-based rather than import-based.",
|
||
|
|
"docsUsed": [
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/helm/index.md",
|
||
|
|
"reference/subsystems/account-common.md",
|
||
|
|
"reference/subsystems/app-tools.md",
|
||
|
|
"reference/subsystems/compute-common.md",
|
||
|
|
"reference/subsystems/compute-tools.md",
|
||
|
|
"reference/subsystems/ipam-core.md",
|
||
|
|
"reference/subsystems/ipam-tools.md",
|
||
|
|
"reference/subsystems/network-common.md",
|
||
|
|
"reference/subsystems/network-core.md",
|
||
|
|
"reference/subsystems/root.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"clarity": 5,
|
||
|
|
"actionability": 4,
|
||
|
|
"completeness": 5,
|
||
|
|
"findability": 5,
|
||
|
|
"notes": "Reader extracted exact dependency counts matching ground truth with clear structural presentation. Cross-Subsystem Dependencies table in system-architecture.md is the logical location for this information. Actionability is 4 rather than 5 because the response lacks context about *why* these dependencies exist or their operational impact, which would be needed for architectural decisions."
|
||
|
|
},
|
||
|
|
"composite": 95,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "cross-sub-002",
|
||
|
|
"category": "cross-subsystem",
|
||
|
|
"difficulty": "easy",
|
||
|
|
"question": "Which subsystems are identified as cross-cutting concerns?",
|
||
|
|
"groundTruth": "app-tools",
|
||
|
|
"humanAnswer": "Based on the documentation, specifically the system architecture overview and the subsystems table:\n\n**root**\n\nThis is the only subsystem identified as a cross-cutting concern. All other subsystems (account-common, app-common, app-tools, compute-common, compute-tools, control-core, ipam-core, ipam-tools, network-common, network-core, and runtime) are classified as domain subsystems.",
|
||
|
|
"docsUsed": [
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/subsystems/account-common.md",
|
||
|
|
"reference/subsystems/app-tools.md",
|
||
|
|
"reference/subsystems/compute-common.md",
|
||
|
|
"reference/subsystems/compute-tools.md",
|
||
|
|
"reference/subsystems/control-core.md",
|
||
|
|
"reference/subsystems/ipam-core.md",
|
||
|
|
"reference/subsystems/ipam-tools.md",
|
||
|
|
"reference/subsystems/network-common.md",
|
||
|
|
"reference/subsystems/network-core.md",
|
||
|
|
"reference/subsystems/root.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"clarity": 1,
|
||
|
|
"actionability": 1,
|
||
|
|
"completeness": 2,
|
||
|
|
"findability": 3,
|
||
|
|
"notes": "Reader consulted correct files but extracted wrong answer (root instead of app-tools). Documentation failed to clearly communicate that app-tools is a cross-cutting concern. Despite accessing reference/subsystems/app-tools.md, the classification was either ambiguous, contradictory, or buried. This creates actionability risk—engineers would implement incorrect architecture based on this misclassification."
|
||
|
|
},
|
||
|
|
"composite": 35,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "arch-001",
|
||
|
|
"category": "architectural",
|
||
|
|
"difficulty": "hard",
|
||
|
|
"question": "The following subsystems have 0 detected functions and 0 modules: account-common, network-common, network-core. Why might this be the case, and what do they actually contain?",
|
||
|
|
"groundTruth": "These subsystems primarily contain Helm charts with Go-templated YAML, Terraform HCL, and Crossplane compositions. The code analysis pipeline detects functions/modules from Python, Go, TypeScript, and shell scripts — but Helm templates use Go template syntax ({{ }}) which doesn't produce traditional function/module entities. Their content is captured through the Helm chart extraction phase instead.",
|
||
|
|
"humanAnswer": "These subsystems contain configuration, data structures, and schemas rather than executable code, which explains the 0 functions and 0 modules.\n\n**account-common** holds Helm chart definitions and values files—foundational data structures and constants for account operations without executable logic. Its eight files are YAML-based chart configurations.\n\n**network-common** provides Helm chart templates and values for network infrastructure across cloud providers (AWS VPC, Azure VNet, GCP VPC). It establishes shared constants, types, and schemas through declarative configuration files rather than code.\n\n**network-core** contains Terraform modules and configuration files that define networking abstractions and domain models. Its 87 files include `.tf` files, YAML manifests, and lock files—all configuration-driven infrastructure definitions rather than behavioral code.\n\nThe documentation explicitly describes these as \"configuration-centric\" or \"data-driven\" architectures. They serve as reference layers and stable foundations that other subsystems depend on for shared definitions, but they don't expose functions or modules because they're composed of declarative infrastructure-as-code and configuration files, not executable functions.",
|
||
|
|
"docsUsed": [
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/subsystems/account-common.md",
|
||
|
|
"reference/subsystems/app-tools.md",
|
||
|
|
"reference/subsystems/compute-common.md",
|
||
|
|
"reference/subsystems/compute-tools.md",
|
||
|
|
"reference/subsystems/control-core.md",
|
||
|
|
"reference/subsystems/ipam-core.md",
|
||
|
|
"reference/subsystems/ipam-tools.md",
|
||
|
|
"reference/subsystems/network-common.md",
|
||
|
|
"reference/subsystems/network-core.md",
|
||
|
|
"reference/subsystems/root.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"clarity": 4,
|
||
|
|
"actionability": 3,
|
||
|
|
"completeness": 3,
|
||
|
|
"findability": 5,
|
||
|
|
"notes": "Reader correctly inferred the answer by understanding these are configuration-centric subsystems, but documentation doesn't explicitly state that the analysis pipeline only detects functions/modules from Python/Go/TypeScript/shell—not Helm templates or Terraform. Findability is excellent (subsystem docs in expected locations), but completeness gaps exist: Crossplane compositions mentioned in ground truth aren't reflected in reader's answer, and the pipeline's language limitations aren't documented as the root cause. Actionability is moderate—engineers understand what's there but lack explicit guidance on implications for code analysis or integration patterns."
|
||
|
|
},
|
||
|
|
"composite": 75,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "arch-002",
|
||
|
|
"category": "architectural",
|
||
|
|
"difficulty": "easy",
|
||
|
|
"question": "What is the current version and appVersion of the mdm-app Helm chart?",
|
||
|
|
"groundTruth": "version: 0.7.4, appVersion: 2025.12-12-develop",
|
||
|
|
"humanAnswer": "**Version:** 0.7.4 \n**App Version:** 2025.12-12-develop",
|
||
|
|
"docsUsed": [
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/helm/index.md",
|
||
|
|
"reference/helm/charts/app-common-charts-mdm-app.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"clarity": 5,
|
||
|
|
"actionability": 5,
|
||
|
|
"completeness": 5,
|
||
|
|
"findability": 5,
|
||
|
|
"notes": "Reader extracted exact version values with clear formatting. Information is immediately usable for deployment. Helm chart versions are logically located in the dedicated chart reference file. No gaps in documentation coverage."
|
||
|
|
},
|
||
|
|
"composite": 100,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "scenario-001",
|
||
|
|
"category": "scenario",
|
||
|
|
"difficulty": "hard",
|
||
|
|
"question": "If you need to rotate the \"vault-secret\" shared secret, which Helm charts would be affected and need redeployment?",
|
||
|
|
"groundTruth": "ai-app, cleanse-app, pms-app, rdm-app, shared-app",
|
||
|
|
"humanAnswer": "Based on the \"Shared Secrets & ConfigMaps\" table in the Helm Charts documentation, the following charts use the \"vault-secret\" shared secret and would require redeployment upon rotation:\n\n- ai-app\n- cleanse-app\n- pms-app\n- rdm-app\n- shared-app",
|
||
|
|
"docsUsed": [
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/helm/index.md",
|
||
|
|
"reference/contracts/index.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"clarity": 4,
|
||
|
|
"actionability": 3,
|
||
|
|
"completeness": 5,
|
||
|
|
"findability": 4,
|
||
|
|
"notes": "Reader correctly identified all five affected charts using a dedicated 'Shared Secrets & ConfigMaps' table in the Helm documentation. Information was clear and complete for identifying scope. Actionability limited because docs appear to specify WHICH charts are affected but lack rotation procedures or redeployment steps. Findability strong—reference/helm/index.md is the natural location for this information, though reader needed to consult multiple documents to confirm."
|
||
|
|
},
|
||
|
|
"composite": 80,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "scenario-002",
|
||
|
|
"category": "scenario",
|
||
|
|
"difficulty": "hard",
|
||
|
|
"question": "A new engineer needs to deploy the MDM application stack. What charts, configuration values, and external dependencies should they understand first?",
|
||
|
|
"groundTruth": "Chart: mdm-app (app-common/charts/mdm-app), Version: 0.7.4\nKey values: clusterName, environmentType, arch, cloudProvider, region, vaultEnv, imagePullPolicy, clusterDomain, terminationGracePeriodSeconds, etalonsubDomain (+28 more)\nResources generated: 22 K8s resources\nInteractions: none detected",
|
||
|
|
"humanAnswer": "To deploy the MDM application stack, a new engineer should understand:\n\n**Primary Chart**\n- `mdm-app` (v0.7.4) located at `app-common/charts/mdm-app` with 22 Kubernetes resources. The chart declares zero dependencies, but this is misleading operationally.\n\n**Configuration Values**\nThe chart requires Helm values, though the specific schema isn't detailed in these docs. However, the app-tools subsystem exposes functions that reveal required configuration patterns:\n- Cross-account MDM companion ARN patching (`patch_crossaccount_mdm_companions`)\n- DynamoDB shared DevOps API configuration (`patch_mdm_systemdynamodb_shared_devops_api`)\n- Shared DevOps API ARN mappings (`patch_shared_devops_api_mdm_arns`)\n\n**External Dependencies**\n- **Vault secrets**: MDM consumes `vault-secret` and `vault-secret-rdm-tasks` ConfigMaps/Secrets, shared with ai-app, cleanse-app, pms-app, rdm-app, and shared-app\n- **RDM companion relationship**: MDM operates alongside rdm-app (v0.5.12) as a companion application; the app-tools subsystem includes `get_rdm_mdm_companions` logic indicating they're provisioned together\n- **Root subsystem**: Like all app-common charts, mdm-app depends on root cross-cutting concerns for foundational types and configuration\n\n**Deployment Context**\nMDM deploys within the app-common subsystem (935 files, 28 functions), which provides infrastructure provisioning and lifecycle management. The deployment likely requires running app-tools provisioning scripts to establish vault secrets and cross-account IAM configurations before Helm chart deployment.",
|
||
|
|
"docsUsed": [
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/helm/index.md",
|
||
|
|
"reference/subsystems/account-common.md",
|
||
|
|
"reference/subsystems/app-tools.md",
|
||
|
|
"reference/subsystems/compute-common.md",
|
||
|
|
"reference/subsystems/compute-tools.md",
|
||
|
|
"reference/subsystems/ipam-core.md",
|
||
|
|
"reference/subsystems/ipam-tools.md",
|
||
|
|
"reference/subsystems/network-common.md",
|
||
|
|
"reference/subsystems/network-core.md",
|
||
|
|
"reference/subsystems/root.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"clarity": 3,
|
||
|
|
"actionability": 2,
|
||
|
|
"completeness": 2,
|
||
|
|
"findability": 2,
|
||
|
|
"notes": "Reader provided well-structured context but diverged from ground truth. Critical gap: the 30+ specific configuration values (clusterName, environmentType, arch, etc.) are not listed or referenced. Instead, reader inferred deployment patterns from app-tools functions, which is valuable but doesn't address the documented values schema. Information scattered across 11 files suggests poor consolidation in helm documentation. New engineer would lack concrete values to configure before deployment."
|
||
|
|
},
|
||
|
|
"composite": 45,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "scenario-003",
|
||
|
|
"category": "scenario",
|
||
|
|
"difficulty": "medium",
|
||
|
|
"question": "What network ports does the Cassandra chart expose, and what are they typically used for?",
|
||
|
|
"groundTruth": "Ports: 7000, 7001, 7199, 9042, 9160\n7000: Cassandra inter-node cluster communication\n7001: Cassandra SSL inter-node\n7199: JMX monitoring\n9042: CQL native transport (client connections)\n9160: Thrift client API (legacy)",
|
||
|
|
"humanAnswer": "The Cassandra chart exposes five network ports via the StatefulSet template:\n\n- Port `7000`\n- Port `7001`\n- Port `7199`\n- Port `9042`\n- Port `9160`\n\nHowever, the documentation doesn't specify what each port is typically used for. The chart definition only lists the port numbers and their source template file (`templates/cassandra-statefulset.yaml`), but doesn't include descriptions of their purposes or protocols.",
|
||
|
|
"docsUsed": [
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/helm/index.md",
|
||
|
|
"reference/helm/charts/app-common-apps-provider-addons-charts-state-cluster-cassandra.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"clarity": 2,
|
||
|
|
"actionability": 1,
|
||
|
|
"completeness": 1,
|
||
|
|
"findability": 2,
|
||
|
|
"notes": "Documentation lists port numbers but omits their purposes entirely. Reader correctly identified all 5 ports but explicitly noted the absence of functional descriptions. Without knowing what each port does (inter-node communication, JMX, CQL, Thrift), engineers cannot configure networking, firewall rules, or understand cluster architecture. Critical operational context is missing from all consulted docs."
|
||
|
|
},
|
||
|
|
"composite": 30,
|
||
|
|
"notFound": false
|
||
|
|
}
|
||
|
|
]
|
||
|
|
}
|