1044 lines
54 KiB
JSON
1044 lines
54 KiB
JSON
|
{
|
||
|
|
"evalType": "agent",
|
||
|
|
"timestamp": "2026-03-10T05:05:06.054Z",
|
||
|
|
"docsDir": "/tmp/confluence-bench",
|
||
|
|
"totalQuestions": 32,
|
||
|
|
"overallScore": "77.8",
|
||
|
|
"avgAccuracy": "3.91",
|
||
|
|
"avgCompleteness": "3.69",
|
||
|
|
"avgPrecision": "4.56",
|
||
|
|
"avgNavigation": "3.41",
|
||
|
|
"notFoundCount": 5,
|
||
|
|
"notFoundRate": "15.6%",
|
||
|
|
"byCategory": {
|
||
|
|
"architecture": {
|
||
|
|
"avg": "80.8",
|
||
|
|
"count": 6
|
||
|
|
},
|
||
|
|
"dependencies": {
|
||
|
|
"avg": "68.0",
|
||
|
|
"count": 5
|
||
|
|
},
|
||
|
|
"operations": {
|
||
|
|
"avg": "80.8",
|
||
|
|
"count": 6
|
||
|
|
},
|
||
|
|
"configuration": {
|
||
|
|
"avg": "81.7",
|
||
|
|
"count": 6
|
||
|
|
},
|
||
|
|
"services": {
|
||
|
|
"avg": "90.0",
|
||
|
|
"count": 5
|
||
|
|
},
|
||
|
|
"contracts": {
|
||
|
|
"avg": "60.0",
|
||
|
|
"count": 4
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"byDifficulty": {
|
||
|
|
"easy": {
|
||
|
|
"avg": "77.0",
|
||
|
|
"count": 10
|
||
|
|
},
|
||
|
|
"medium": {
|
||
|
|
"avg": "78.2",
|
||
|
|
"count": 17
|
||
|
|
},
|
||
|
|
"hard": {
|
||
|
|
"avg": "78.0",
|
||
|
|
"count": 5
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"results": [
|
||
|
|
{
|
||
|
|
"id": "arch-layered-order",
|
||
|
|
"category": "architecture",
|
||
|
|
"difficulty": "easy",
|
||
|
|
"question": "What are the five layers in Foxtrot's layered architecture, listed from top to bottom?",
|
||
|
|
"groundTruth": "Application Layer (app-*), Compute Layer (compute-*), Network Layer (network-*), Account Layer (account-*), Control Plane (control-core).",
|
||
|
|
"llmAnswer": "1. Application Layer (app-*)\n2. Compute Layer (compute-*)\n3. Network Layer (network-*)\n4. Account Layer (account-*)\n5. Control Plane (control-core)",
|
||
|
|
"filesRead": [
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"explanation/index.md",
|
||
|
|
"explanation/architecture-patterns.md",
|
||
|
|
"reference/runtime-architecture.md",
|
||
|
|
"reference/index.md"
|
||
|
|
],
|
||
|
|
"plannedFiles": [
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"explanation/index.md",
|
||
|
|
"explanation/architecture-patterns.md",
|
||
|
|
"reference/runtime-architecture.md",
|
||
|
|
"reference/index.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"accuracy": 0,
|
||
|
|
"completeness": 0,
|
||
|
|
"precision": 0,
|
||
|
|
"navigation": 0,
|
||
|
|
"notes": "Parse error: "
|
||
|
|
},
|
||
|
|
"composite": 0,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "arch-hub-spoke-ownership",
|
||
|
|
"category": "architecture",
|
||
|
|
"difficulty": "easy",
|
||
|
|
"question": "Which ArgoCD instance owns the account, network, and compute layers, and which owns the application layer?",
|
||
|
|
"groundTruth": "Hub ArgoCD owns account, network, and compute layers (infrastructure and control-plane appsets). Spoke ArgoCD owns the application layer (app-runtime appsets only).",
|
||
|
|
"llmAnswer": "**Hub ArgoCD** owns the account, network, and compute layers.\n\n**Spoke ArgoCD** owns the application layer.\n\nThis is documented in the Runtime Ownership Matrix in runtime-architecture.md:\n- Account, network, and compute runtime repos are managed by Hub ArgoCD with hub infrastructure appsets\n- Application runtime is managed by Spoke ArgoCD with spoke runtime appsets",
|
||
|
|
"filesRead": [
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/runtime-architecture.md",
|
||
|
|
"reference/app-common-architecture.md",
|
||
|
|
"reference/account-common-architecture.md",
|
||
|
|
"reference/index.md"
|
||
|
|
],
|
||
|
|
"plannedFiles": [
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/runtime-architecture.md",
|
||
|
|
"reference/app-common-architecture.md",
|
||
|
|
"reference/account-common-architecture.md",
|
||
|
|
"reference/index.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"accuracy": 5,
|
||
|
|
"completeness": 5,
|
||
|
|
"precision": 5,
|
||
|
|
"navigation": 5,
|
||
|
|
"notes": "Answer correctly identifies Hub ArgoCD ownership of account/network/compute layers and Spoke ArgoCD ownership of application layer. Provides accurate supporting details about appsets and references the correct source document. No hallucinations or extraneous information detected."
|
||
|
|
},
|
||
|
|
"composite": 100,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "arch-aws-regions",
|
||
|
|
"category": "architecture",
|
||
|
|
"difficulty": "easy",
|
||
|
|
"question": "What AWS regions does Foxtrot support?",
|
||
|
|
"groundTruth": "us-east-1, us-west-2, eu-west-1, eu-central-1.",
|
||
|
|
"llmAnswer": "Based on the documentation, Foxtrot supports the following AWS regions:\n\n- us-east-1\n- us-west-2\n- eu-west-1\n- eu-central-1",
|
||
|
|
"filesRead": [
|
||
|
|
"explanation/multi-cloud-strategy.md",
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"explanation/index.md",
|
||
|
|
"reference/account-common-architecture.md",
|
||
|
|
"reference/runtime-architecture.md"
|
||
|
|
],
|
||
|
|
"plannedFiles": [
|
||
|
|
"explanation/multi-cloud-strategy.md",
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"explanation/index.md",
|
||
|
|
"reference/account-common-architecture.md",
|
||
|
|
"reference/runtime-architecture.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"accuracy": 5,
|
||
|
|
"completeness": 5,
|
||
|
|
"precision": 5,
|
||
|
|
"navigation": 4,
|
||
|
|
"notes": "Answer matches ground truth exactly with all four regions listed correctly. No hallucinations or extra information. Navigation appears competent based on accurate sourcing, though actual file access path not visible."
|
||
|
|
},
|
||
|
|
"composite": 95,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "arch-gcp-shared-vpc-host",
|
||
|
|
"category": "architecture",
|
||
|
|
"difficulty": "medium",
|
||
|
|
"question": "What is the default GCP host project used for Shared VPC in network-common, and what is the host network path it attaches to?",
|
||
|
|
"groundTruth": "The default host project is `network-services-436015`. When `sharedVpc.enabled: true`, service project resources attach to the host network path `projects/HOST_PROJECT/global/networks/gcp-core-network`.",
|
||
|
|
"llmAnswer": "The default GCP host project used for Shared VPC in network-common is `network-services-436015`, and the host network path it attaches to is `projects/network-services-436015/global/networks/gcp-core-network`.",
|
||
|
|
"filesRead": [
|
||
|
|
"reference/network-common-architecture.md",
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/index.md",
|
||
|
|
"reference/configuration-reference.md",
|
||
|
|
"explanation/platform-concepts.md"
|
||
|
|
],
|
||
|
|
"plannedFiles": [
|
||
|
|
"reference/network-common-architecture.md",
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/index.md",
|
||
|
|
"reference/configuration-reference.md",
|
||
|
|
"explanation/platform-concepts.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"accuracy": 5,
|
||
|
|
"completeness": 4,
|
||
|
|
"precision": 5,
|
||
|
|
"navigation": 4,
|
||
|
|
"notes": "AI correctly identified both the host project (network-services-436015) and resolved the network path with the actual project ID substituted. Minor gap: didn't explicitly mention the sharedVpc.enabled condition from ground truth, though this wasn't directly asked. The substitution of HOST_PROJECT placeholder with actual ID is reasonable and helpful rather than problematic."
|
||
|
|
},
|
||
|
|
"composite": 90,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "arch-cidr-employee-access",
|
||
|
|
"category": "architecture",
|
||
|
|
"difficulty": "medium",
|
||
|
|
"question": "What is the CIDR range for the employee access (bastions) segment on AWS and on GCP?",
|
||
|
|
"groundTruth": "AWS: 10.208.128.0/24. GCP: 10.128.128.0/24.",
|
||
|
|
"llmAnswer": "AWS: 10.208.128.0/24\nGCP: 10.128.128.0/24",
|
||
|
|
"filesRead": [
|
||
|
|
"reference/network-common-architecture.md",
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/configuration-reference.md",
|
||
|
|
"reference/network-core-architecture.md",
|
||
|
|
"reference/index.md"
|
||
|
|
],
|
||
|
|
"plannedFiles": [
|
||
|
|
"reference/network-common-architecture.md",
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/configuration-reference.md",
|
||
|
|
"reference/network-core-architecture.md",
|
||
|
|
"reference/index.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"accuracy": 5,
|
||
|
|
"completeness": 5,
|
||
|
|
"precision": 5,
|
||
|
|
"navigation": 5,
|
||
|
|
"notes": "AI answer matches ground truth exactly for both AWS and GCP CIDR ranges. All required information present with no hallucinations or extraneous details. Perfect accuracy indicates successful navigation to correct infrastructure specifications."
|
||
|
|
},
|
||
|
|
"composite": 100,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "arch-production-cidr",
|
||
|
|
"category": "architecture",
|
||
|
|
"difficulty": "medium",
|
||
|
|
"question": "What is the CIDR range for production workloads on AWS and on GCP?",
|
||
|
|
"groundTruth": "AWS: 10.192.0.0/10. GCP: 10.128.0.0/10.",
|
||
|
|
"llmAnswer": "According to the system-architecture.md file, the CIDR ranges for production workloads are:\n\n- **AWS**: 10.192.0.0/10\n- **GCP**: 10.128.0.0/10",
|
||
|
|
"filesRead": [
|
||
|
|
"reference/network-core-architecture.md",
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/network-common-architecture.md",
|
||
|
|
"reference/configuration-reference.md",
|
||
|
|
"reference/index.md"
|
||
|
|
],
|
||
|
|
"plannedFiles": [
|
||
|
|
"reference/network-core-architecture.md",
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/network-common-architecture.md",
|
||
|
|
"reference/configuration-reference.md",
|
||
|
|
"reference/index.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"accuracy": 5,
|
||
|
|
"completeness": 5,
|
||
|
|
"precision": 5,
|
||
|
|
"navigation": 5,
|
||
|
|
"notes": "AI answer matches ground truth exactly for both AWS and GCP CIDR ranges. Agent demonstrated source awareness by citing system-architecture.md. No hallucinations or extraneous information present. Perfect response."
|
||
|
|
},
|
||
|
|
"composite": 100,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "dep-runtime-common-horizontal",
|
||
|
|
"category": "dependencies",
|
||
|
|
"difficulty": "easy",
|
||
|
|
"question": "Which runtime repositories consume charts from which common repositories?",
|
||
|
|
"groundTruth": "app-runtime consumes charts from app-common; compute-runtime consumes charts from compute-common; network-runtime consumes charts from network-common; account-runtime consumes charts from account-common.",
|
||
|
|
"llmAnswer": "\n\nBased on the documentation provided:\n\n**Runtime repositories consume charts from common repositories as follows:**\n\n- `app-runtime` consumes charts from `app-common`\n- `compute-runtime` consumes charts from `compute-common`\n- `network-runtime` consumes charts from `network-common`\n- `account-runtime` consumes charts from `account-common`\n\nThis is documented as the horizontal dependency model in the system architecture.",
|
||
|
|
"filesRead": [
|
||
|
|
"reference/runtime-architecture.md",
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/repository-structure.md",
|
||
|
|
"reference/app-common-architecture.md",
|
||
|
|
"reference/index.md"
|
||
|
|
],
|
||
|
|
"plannedFiles": [
|
||
|
|
"reference/runtime-architecture.md",
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/repository-structure.md",
|
||
|
|
"reference/app-common-architecture.md",
|
||
|
|
"reference/index.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"accuracy": 5,
|
||
|
|
"completeness": 5,
|
||
|
|
"precision": 5,
|
||
|
|
"navigation": 4,
|
||
|
|
"notes": "AI answer matches ground truth exactly on all four runtime-to-common repository mappings. No factual errors or hallucinations. Minor contextual addition about 'horizontal dependency model' adds value without introducing inaccuracy. Navigation score slightly reduced due to inability to verify actual file discovery process, though answer quality suggests competent information retrieval."
|
||
|
|
},
|
||
|
|
"composite": 95,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "dep-vertical-layers",
|
||
|
|
"category": "dependencies",
|
||
|
|
"difficulty": "easy",
|
||
|
|
"question": "What are the vertical layer dependencies in Foxtrot's architecture?",
|
||
|
|
"groundTruth": "Application Layer depends on Compute Layer; Compute Layer depends on Network Layer; Network Layer depends on Account Layer; Account Layer is managed by the Control Plane.",
|
||
|
|
"llmAnswer": "Based on the system-architecture.md file, the vertical layer dependencies in Foxtrot's architecture are:\n\n**Application Layer** depends on **Compute Layer**\n\n**Compute Layer** depends on **Network Layer**\n\n**Network Layer** depends on **Account Layer**\n\n**Account Layer** is managed by **Control Plane**",
|
||
|
|
"filesRead": [
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/runtime-architecture.md",
|
||
|
|
"reference/index.md",
|
||
|
|
"explanation/architecture-patterns.md",
|
||
|
|
"reference/app-common-architecture.md"
|
||
|
|
],
|
||
|
|
"plannedFiles": [
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/runtime-architecture.md",
|
||
|
|
"reference/index.md",
|
||
|
|
"explanation/architecture-patterns.md",
|
||
|
|
"reference/app-common-architecture.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"accuracy": 5,
|
||
|
|
"completeness": 5,
|
||
|
|
"precision": 5,
|
||
|
|
"navigation": 5,
|
||
|
|
"notes": "AI answer matches ground truth exactly on all four dependency relationships. No hallucinations or extraneous information. Correctly cited system-architecture.md as source, demonstrating proper navigation to the right documentation."
|
||
|
|
},
|
||
|
|
"composite": 100,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "dep-create-account-repos",
|
||
|
|
"category": "dependencies",
|
||
|
|
"difficulty": "medium",
|
||
|
|
"question": "How many PRs does `create-account` produce and which repositories does it touch?",
|
||
|
|
"groundTruth": "4 PRs: account-runtime, network-core (account principal updates), control-core (provider config + assume-role policy updates), and cloud-iam (account bootstrap TF).",
|
||
|
|
"llmAnswer": "NOT_FOUND",
|
||
|
|
"filesRead": [
|
||
|
|
"reference/account-common-architecture.md",
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/index.md",
|
||
|
|
"reference/release-process.md",
|
||
|
|
"reference/repository-structure.md"
|
||
|
|
],
|
||
|
|
"plannedFiles": [
|
||
|
|
"reference/account-common-architecture.md",
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/index.md",
|
||
|
|
"reference/release-process.md",
|
||
|
|
"reference/repository-structure.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"accuracy": 0,
|
||
|
|
"completeness": 0,
|
||
|
|
"precision": 5,
|
||
|
|
"navigation": 0,
|
||
|
|
"notes": "Agent returned NOT_FOUND without attempting to locate information about create-account's PR generation or repository impact. No facts provided means no accuracy or completeness. Precision is high because there are no hallucinations, only absence of data. Navigation score reflects complete failure to find the relevant information."
|
||
|
|
},
|
||
|
|
"composite": 25,
|
||
|
|
"notFound": true
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "dep-create-cluster-repos",
|
||
|
|
"category": "dependencies",
|
||
|
|
"difficulty": "medium",
|
||
|
|
"question": "Which repositories does `create-cluster` commit to, and how many PRs does it typically produce?",
|
||
|
|
"groundTruth": "4 PRs: compute-runtime (cluster path) plus 3 PRs in cloud-auth-core (cluster/node role PR, IRSA role PR, and network role PR). It also calls cluster-netbox-allocate.py to reserve/allocate cluster subnet CIDRs.",
|
||
|
|
"llmAnswer": "NOT_FOUND\n\nThe provided documentation does not specify which repositories `create-cluster` commits to or how many PRs it typically produces. While the CLI reference documents the command syntax and notes it wraps `compute-tools/bash-spoke-setup/scripts/create-cluster.sh`, the specific repository targets and PR count are not detailed in these files.",
|
||
|
|
"filesRead": [
|
||
|
|
"reference/cli-reference.md",
|
||
|
|
"reference/release-process.md",
|
||
|
|
"reference/repository-structure.md",
|
||
|
|
"reference/index.md",
|
||
|
|
"reference/system-architecture.md"
|
||
|
|
],
|
||
|
|
"plannedFiles": [
|
||
|
|
"reference/cli-reference.md",
|
||
|
|
"reference/release-process.md",
|
||
|
|
"reference/repository-structure.md",
|
||
|
|
"reference/index.md",
|
||
|
|
"reference/system-architecture.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"accuracy": 0,
|
||
|
|
"completeness": 0,
|
||
|
|
"precision": 5,
|
||
|
|
"navigation": 0,
|
||
|
|
"notes": "AI answered NOT_FOUND when ground truth specifies 4 PRs across 2 repositories (compute-runtime and cloud-auth-core) plus a call to cluster-netbox-allocate.py. The agent failed to locate or access the relevant documentation containing this information."
|
||
|
|
},
|
||
|
|
"composite": 25,
|
||
|
|
"notFound": true
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "dep-compute-common-deps",
|
||
|
|
"category": "dependencies",
|
||
|
|
"difficulty": "medium",
|
||
|
|
"question": "What are the core dependencies of compute-common for cluster provisioning?",
|
||
|
|
"groundTruth": "Crossplane core + provider-aws/provider-kubernetes, Helm, KCL build pipeline, and ArgoCD deployment flow.",
|
||
|
|
"llmAnswer": "Based on the compute-common architecture documentation, the core dependencies for cluster provisioning are:\n\n* Crossplane core + provider-aws/provider-kubernetes\n* Helm\n* KCL build pipeline\n* ArgoCD deployment flow\n\nThese enable the composition pipeline to provision EKS clusters through managed resources (Cluster, SecurityGroup, LaunchTemplate, NodeGroup, and optional Spot integration) and deploy them via GitOps.",
|
||
|
|
"filesRead": [
|
||
|
|
"reference/compute-common-architecture.md",
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/index.md",
|
||
|
|
"reference/compute-tools-architecture.md",
|
||
|
|
"explanation/architecture-patterns.md"
|
||
|
|
],
|
||
|
|
"plannedFiles": [
|
||
|
|
"reference/compute-common-architecture.md",
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/index.md",
|
||
|
|
"reference/compute-tools-architecture.md",
|
||
|
|
"explanation/architecture-patterns.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"accuracy": 5,
|
||
|
|
"completeness": 5,
|
||
|
|
"precision": 5,
|
||
|
|
"navigation": 4,
|
||
|
|
"notes": "Answer correctly identifies all four core dependencies from ground truth with no factual errors. Additional context about managed resources and GitOps deployment is appropriate elaboration rather than hallucination. Navigation score reduced slightly due to inability to verify actual tool usage or documentation lookup process, though answer quality suggests reliable information retrieval."
|
||
|
|
},
|
||
|
|
"composite": 95,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "ops-argocd-deployment-flow",
|
||
|
|
"category": "operations",
|
||
|
|
"difficulty": "medium",
|
||
|
|
"question": "What are the steps in the ArgoCD deployment flow for an application, from ApplicationSet watching app-runtime to pods starting?",
|
||
|
|
"groundTruth": "1. ApplicationSet watches app-runtime. 2. Reads argo-gen-params.yaml per environment. 3. Generates Application. 4. Pulls chart from OCI registry. 5. Merges values and overrides. 6. Renders manifests. 7. Applies to cluster. 8. ESO fetches secrets. 9. Pods start with injected configs.",
|
||
|
|
"llmAnswer": "1. ApplicationSet watches app-runtime\n2. Reads argo-gen-params.yaml per environment\n3. Generates Application\n4. Pulls chart from OCI registry\n5. Merges values and overrides\n6. Renders manifests\n7. Applies to cluster\n8. ESO fetches secrets\n9. Pods start with injected configs",
|
||
|
|
"filesRead": [
|
||
|
|
"reference/applicationset-reference.md",
|
||
|
|
"reference/app-common-architecture.md",
|
||
|
|
"reference/runtime-architecture.md",
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/operator-basics.md"
|
||
|
|
],
|
||
|
|
"plannedFiles": [
|
||
|
|
"reference/applicationset-reference.md",
|
||
|
|
"reference/app-common-architecture.md",
|
||
|
|
"reference/runtime-architecture.md",
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/operator-basics.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"accuracy": 5,
|
||
|
|
"completeness": 5,
|
||
|
|
"precision": 5,
|
||
|
|
"navigation": 5,
|
||
|
|
"notes": "AI answer matches ground truth exactly across all 9 steps with no omissions, hallucinations, or deviations. Perfect alignment on ArgoCD deployment flow from ApplicationSet through pod startup."
|
||
|
|
},
|
||
|
|
"composite": 100,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "ops-ebf-release-pattern",
|
||
|
|
"category": "operations",
|
||
|
|
"difficulty": "medium",
|
||
|
|
"question": "What are the steps in the EBF/weekly release pattern?",
|
||
|
|
"groundTruth": "1. Create release branch from develop. 2. Promote versions for release/hotfix environments. 3. Run stage pipelines (preview, preprod, production). 4. Merge promoted changes to main. 5. Reconcile hotfix and develop branches.",
|
||
|
|
"llmAnswer": "The EBF / Weekly Release Pattern consists of these steps:\n\n1. Create `release` branch from `develop`.\n2. Promote versions for release/hotfix environments.\n3. Run stage pipelines (preview, preprod, production).\n4. Merge promoted changes to `main`.\n5. Reconcile `hotfix` and `develop` branches.",
|
||
|
|
"filesRead": [
|
||
|
|
"reference/release-process.md",
|
||
|
|
"reference/index.md",
|
||
|
|
"explanation/index.md",
|
||
|
|
"reference/system-architecture.md"
|
||
|
|
],
|
||
|
|
"plannedFiles": [
|
||
|
|
"reference/release-process.md",
|
||
|
|
"reference/index.md",
|
||
|
|
"explanation/index.md",
|
||
|
|
"reference/system-architecture.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"accuracy": 5,
|
||
|
|
"completeness": 5,
|
||
|
|
"precision": 5,
|
||
|
|
"navigation": 5,
|
||
|
|
"notes": "AI answer matches ground truth exactly across all five steps. Formatting with backticks enhances clarity without altering content. No omissions, hallucinations, or inaccuracies detected."
|
||
|
|
},
|
||
|
|
"composite": 100,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "ops-rollback-procedure",
|
||
|
|
"category": "operations",
|
||
|
|
"difficulty": "medium",
|
||
|
|
"question": "What is the preferred rollback procedure in Foxtrot, and when are script-level deletes used instead?",
|
||
|
|
"groundTruth": "GitOps rollback is preferred: revert the runtime/config commit, merge the revert PR, and confirm ArgoCD reconciliation. Script-level deletes (delete-vpc.sh, delete-cluster.sh, delete-app.sh) are used only with explicit approval and change control.",
|
||
|
|
"llmAnswer": "The preferred rollback procedure in Foxtrot is GitOps-based:\n\n1. Revert the runtime/config commit\n2. Merge the revert PR\n3. Confirm ArgoCD reconciliation\n\nScript-level deletes (`delete-vpc.sh`, `delete-cluster.sh`, `delete-app.sh`) are used only with explicit approval and change control. Additionally, these delete workflows are documented as draft/partial reliability paths, and operators should use the CRUD Operations Matrix to identify supported vs TBD operations before execution.",
|
||
|
|
"filesRead": [
|
||
|
|
"reference/verification-and-rollback.md",
|
||
|
|
"reference/script-prerequisites.md",
|
||
|
|
"reference/release-process.md",
|
||
|
|
"reference/index.md",
|
||
|
|
"reference/system-architecture.md"
|
||
|
|
],
|
||
|
|
"plannedFiles": [
|
||
|
|
"reference/verification-and-rollback.md",
|
||
|
|
"reference/script-prerequisites.md",
|
||
|
|
"reference/release-process.md",
|
||
|
|
"reference/index.md",
|
||
|
|
"reference/system-architecture.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"accuracy": 5,
|
||
|
|
"completeness": 5,
|
||
|
|
"precision": 4,
|
||
|
|
"navigation": 4,
|
||
|
|
"notes": "AI answer covers all ground truth elements accurately: GitOps rollback procedure with three steps and script-level delete constraints. However, it adds contextual information about 'draft/partial reliability paths' and 'CRUD Operations Matrix' that extends beyond the ground truth. While this elaboration appears reasonable and doesn't contradict the ground truth, it introduces claims not verified against the provided ground truth, slightly reducing precision. Navigation appears strong given the agent located both core and related documentation."
|
||
|
|
},
|
||
|
|
"composite": 90,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "ops-branch-cluster-mapping",
|
||
|
|
"category": "operations",
|
||
|
|
"difficulty": "easy",
|
||
|
|
"question": "What is the branch-to-cluster mapping in app-runtime? Which branch targets which environments?",
|
||
|
|
"groundTruth": "develop → internal clusters (development and QA); main → customer-facing clusters (production); hotfix → EBF verification; release → release verification (weekly release staging).",
|
||
|
|
"llmAnswer": "NOT_FOUND\n\nThe provided documentation does not contain a specific branch-to-cluster mapping for app-runtime. While `release-process.md` describes a general branch model (`develop`, `main`, `hotfix`, `release`) and their intended purposes (internal QA, production, etc.), it does not explicitly map these branches to specific clusters or environments within app-runtime.",
|
||
|
|
"filesRead": [
|
||
|
|
"reference/app-common-architecture.md",
|
||
|
|
"reference/release-process.md",
|
||
|
|
"reference/runtime-architecture.md",
|
||
|
|
"reference/repository-structure.md",
|
||
|
|
"reference/index.md"
|
||
|
|
],
|
||
|
|
"plannedFiles": [
|
||
|
|
"reference/app-common-architecture.md",
|
||
|
|
"reference/release-process.md",
|
||
|
|
"reference/runtime-architecture.md",
|
||
|
|
"reference/repository-structure.md",
|
||
|
|
"reference/index.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"accuracy": 0,
|
||
|
|
"completeness": 0,
|
||
|
|
"precision": 5,
|
||
|
|
"navigation": 0,
|
||
|
|
"notes": "AI responded with NOT_FOUND despite ground truth existing. The branch-to-cluster mapping is documented and specific (develop→internal, main→production, hotfix→EBF, release→staging), but the agent failed to locate it. Per evaluation rules for NOT_FOUND responses."
|
||
|
|
},
|
||
|
|
"composite": 25,
|
||
|
|
"notFound": true
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "ops-jenkins-jobs",
|
||
|
|
"category": "operations",
|
||
|
|
"difficulty": "hard",
|
||
|
|
"question": "What are the six Jenkins jobs in the Foxtrot pipeline and what does each do?",
|
||
|
|
"groundTruth": "appRuntimeSync: syncs runtime branches on changes to main/hotfix/release. foxtrotSync: updates imagetags-override.yaml with new image tags. foxtrotPromote: promotes and merges release outputs. chartSync: updates argo-gen-params.yaml with chart versions. chartCheckInAppCommon: detects chart version mismatches and triggers chartSync. chartPromoteVersion: promotes chart versions per release stage.",
|
||
|
|
"llmAnswer": "\n\nThe six Jenkins jobs in the Foxtrot pipeline are:\n\n1. **appRuntimeSync** - Branch synchronization across runtime branches\n2. **foxtrotSync** - Image tag updates in runtime\n3. **foxtrotPromote** - Promotes and merges release outputs\n4. **chartSync** - Updates chart versions in runtime\n5. **chartCheckInAppCommon** - Compares chart versions and triggers sync\n6. **chartPromoteVersion** - Promotes chart versions per stage",
|
||
|
|
"filesRead": [
|
||
|
|
"reference/release-process.md",
|
||
|
|
"reference/index.md",
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/repository-structure.md",
|
||
|
|
"explanation/index.md"
|
||
|
|
],
|
||
|
|
"plannedFiles": [
|
||
|
|
"reference/release-process.md",
|
||
|
|
"reference/index.md",
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/repository-structure.md",
|
||
|
|
"explanation/index.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"accuracy": 5,
|
||
|
|
"completeness": 5,
|
||
|
|
"precision": 4,
|
||
|
|
"navigation": 4,
|
||
|
|
"notes": "All six jobs correctly identified with accurate functional descriptions. Minor precision deduction: descriptions omit specific file names (imagetags-override.yaml, argo-gen-params.yaml) present in ground truth, though the core functionality is captured. Navigation score reflects successful identification of all jobs without visible process evidence in the answer."
|
||
|
|
},
|
||
|
|
"composite": 90,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "ops-create-cluster-timeout",
|
||
|
|
"category": "operations",
|
||
|
|
"difficulty": "medium",
|
||
|
|
"question": "What is the timeout for waiting for a cluster to reach ready condition in the create-cluster workflow?",
|
||
|
|
"groundTruth": "3600 seconds (1 hour), with a poll interval of 30 seconds.",
|
||
|
|
"llmAnswer": "3600s",
|
||
|
|
"filesRead": [
|
||
|
|
"reference/compute-tools-architecture.md",
|
||
|
|
"reference/configuration-reference.md",
|
||
|
|
"reference/operator-basics.md",
|
||
|
|
"reference/compute-common-architecture.md",
|
||
|
|
"reference/system-architecture.md"
|
||
|
|
],
|
||
|
|
"plannedFiles": [
|
||
|
|
"reference/compute-tools-architecture.md",
|
||
|
|
"reference/configuration-reference.md",
|
||
|
|
"reference/operator-basics.md",
|
||
|
|
"reference/compute-common-architecture.md",
|
||
|
|
"reference/system-architecture.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"accuracy": 5,
|
||
|
|
"completeness": 2,
|
||
|
|
"precision": 5,
|
||
|
|
"navigation": 4,
|
||
|
|
"notes": "AI correctly identified the 3600-second timeout value with no hallucinations. However, the answer is incomplete—it omits the 30-second poll interval specified in the ground truth. The agent demonstrated ability to locate the correct timeout value but failed to capture the full context of the cluster readiness check configuration."
|
||
|
|
},
|
||
|
|
"composite": 80,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "config-cloud-resource-naming",
|
||
|
|
"category": "configuration",
|
||
|
|
"difficulty": "easy",
|
||
|
|
"question": "What is the canonical format for cloud resource names in Foxtrot, and what does each component represent?",
|
||
|
|
"groundTruth": "Format: `{provider}-{phase}-{region-code}-{index}-{resource}[-{descriptor}]`. provider: aws/gcp/azr; phase: prd/iqa/dev; region-code: two-letter derived code; index: incremental integer; resource: vpc/eks/gke/aks/lb/rds. Example: `gcp-prd-us-1-vpc-blue`.",
|
||
|
|
"llmAnswer": "\n\nThe canonical format for cloud resource names in Foxtrot is:\n\n`{provider}-{phase}-{region-code}-{index}-{resource}[-{descriptor}]`\n\n**Components:**\n\n- `provider`: Cloud provider (`aws`, `gcp`, `azr`)\n- `phase`: Environment phase (`prd`, `iqa`, `dev`)\n- `region-code`: Two-letter derived code from the cloud region\n- `index`: Incremental integer\n- `resource`: Resource type (`vpc`, `eks`, `gke`, `aks`, `lb`, `rds`)\n- `descriptor`: Optional suffix for disambiguation\n\n**Examples:**\n- `gcp-prd-us-1-vpc-blue`\n- `aws-prd-eu-1-eks-green`\n- `azr-iqa-uk-1-lb-green`\n\n**Region Code Derivation Algorithm:**\n1. Remove directional words: `east`, `west`, `north`, `south`\n2. Remove separators\n3. Take first two characters\n\nExamples: `us-east-1` → `us`, `westeurope` → `eu`, `uksouth` → `uk`, `southeastasia` → `as`",
|
||
|
|
"filesRead": [
|
||
|
|
"reference/naming-conventions.md",
|
||
|
|
"reference/index.md",
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/configuration-reference.md",
|
||
|
|
"explanation/index.md"
|
||
|
|
],
|
||
|
|
"plannedFiles": [
|
||
|
|
"reference/naming-conventions.md",
|
||
|
|
"reference/index.md",
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/configuration-reference.md",
|
||
|
|
"explanation/index.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"accuracy": 4,
|
||
|
|
"completeness": 5,
|
||
|
|
"precision": 2,
|
||
|
|
"navigation": 2,
|
||
|
|
"notes": "AI correctly identified all ground truth components and format. However, it hallucinated a 'Region Code Derivation Algorithm' section with specific rules (removing directional words, etc.) that do not appear in ground truth. The additional examples beyond the provided one are plausible but unverified. The answer demonstrates knowledge of the core naming convention but introduces unsubstantiated details that reduce precision and suggest the agent may not have consulted authoritative documentation."
|
||
|
|
},
|
||
|
|
"composite": 65,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "config-region-code-algorithm",
|
||
|
|
"category": "configuration",
|
||
|
|
"difficulty": "medium",
|
||
|
|
"question": "What is the algorithm for deriving a two-letter region code from a cloud region name, and what is the region code for `westeurope`?",
|
||
|
|
"groundTruth": "Algorithm: 1. Remove directional words (east, west, north, south). 2. Remove separators. 3. Take first two characters. For `westeurope`: remove 'west' → 'europe' → 'eu'.",
|
||
|
|
"llmAnswer": "**Algorithm for deriving a two-letter region code:**\n\n1. Remove directional words: `east`, `west`, `north`, `south`.\n2. Remove separators.\n3. Take first two characters.\n\n**Region code for `westeurope`:** `eu`\n\n(Source: `reference/naming-conventions.md` — Region Code Derivation section)",
|
||
|
|
"filesRead": [
|
||
|
|
"reference/naming-conventions.md",
|
||
|
|
"reference/index.md",
|
||
|
|
"reference/azure-runtime-contracts.md",
|
||
|
|
"reference/configuration-reference.md",
|
||
|
|
"reference/system-architecture.md"
|
||
|
|
],
|
||
|
|
"plannedFiles": [
|
||
|
|
"reference/naming-conventions.md",
|
||
|
|
"reference/index.md",
|
||
|
|
"reference/azure-runtime-contracts.md",
|
||
|
|
"reference/configuration-reference.md",
|
||
|
|
"reference/system-architecture.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"accuracy": 5,
|
||
|
|
"completeness": 5,
|
||
|
|
"precision": 5,
|
||
|
|
"navigation": 4,
|
||
|
|
"notes": "Answer matches ground truth exactly on all substantive points: algorithm steps are identical, region code derivation is correct. Formatting is clear and organized. Citation to source file demonstrates navigation to relevant documentation, though file existence cannot be independently verified."
|
||
|
|
},
|
||
|
|
"composite": 95,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "config-app-config-merge-order",
|
||
|
|
"category": "configuration",
|
||
|
|
"difficulty": "medium",
|
||
|
|
"question": "What is the configuration merge order for app charts in Foxtrot, from lowest to highest precedence?",
|
||
|
|
"groundTruth": "1. values.yaml, 2. default-properties.yaml, 3. default-values.yaml / k8s-values.yaml, 4. common-values.yaml, 5. properties-override.yaml, 6. k8s-override.yaml, 7. imagetags-override.yaml.",
|
||
|
|
"llmAnswer": "Based on the documentation provided, the configuration merge order for app charts in Foxtrot, from lowest to highest precedence, is:\n\n1. `values.yaml`\n2. `default-properties.yaml`\n3. `default-values.yaml` / `k8s-values.yaml`\n4. `common-values.yaml`\n5. `properties-override.yaml`\n6. `k8s-override.yaml`\n7. `imagetags-override.yaml`\n\nThis is documented in both `reference/configuration-reference.md` and `reference/app-common-architecture.md`.",
|
||
|
|
"filesRead": [
|
||
|
|
"reference/configuration-reference.md",
|
||
|
|
"reference/app-common-architecture.md",
|
||
|
|
"reference/applicationset-reference.md",
|
||
|
|
"reference/index.md",
|
||
|
|
"reference/system-architecture.md"
|
||
|
|
],
|
||
|
|
"plannedFiles": [
|
||
|
|
"reference/configuration-reference.md",
|
||
|
|
"reference/app-common-architecture.md",
|
||
|
|
"reference/applicationset-reference.md",
|
||
|
|
"reference/index.md",
|
||
|
|
"reference/system-architecture.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"accuracy": 5,
|
||
|
|
"completeness": 5,
|
||
|
|
"precision": 5,
|
||
|
|
"navigation": 5,
|
||
|
|
"notes": "AI answer matches ground truth exactly across all 7 configuration merge order items in correct sequence. No hallucinations or omissions. Cites specific documentation sources demonstrating proper information retrieval."
|
||
|
|
},
|
||
|
|
"composite": 100,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "config-account-creation-product-id",
|
||
|
|
"category": "configuration",
|
||
|
|
"difficulty": "hard",
|
||
|
|
"question": "What is the AWS Service Catalog product ID used by account-common for account creation, and what are the customer-facing and internal OU IDs?",
|
||
|
|
"groundTruth": "Service Catalog product ID: `prod-mts6togilnnuk`. Customer-facing OU: `ou-8vpz-mogk4qua`. Internal OU: `ou-8vpz-rw7etshv`. Provider config reference: `aws-709659200419`.",
|
||
|
|
"llmAnswer": "**AWS Service Catalog product ID:** `prod-mts6togilnnuk`\n\n**Customer-facing OU ID:** `ou-8vpz-mogk4qua`\n\n**Internal OU ID:** `ou-8vpz-rw7etshv`\n\nThese values are documented as contract-critical defaults in the Account Common Architecture reference and must be treated as controlled constants unless a coordinated migration is planned.",
|
||
|
|
"filesRead": [
|
||
|
|
"reference/account-common-architecture.md",
|
||
|
|
"reference/index.md",
|
||
|
|
"reference/configuration-reference.md",
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/api-contracts.md"
|
||
|
|
],
|
||
|
|
"plannedFiles": [
|
||
|
|
"reference/account-common-architecture.md",
|
||
|
|
"reference/index.md",
|
||
|
|
"reference/configuration-reference.md",
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/api-contracts.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"accuracy": 5,
|
||
|
|
"completeness": 3,
|
||
|
|
"precision": 4,
|
||
|
|
"navigation": 4,
|
||
|
|
"notes": "AI correctly identified all three primary identifiers with exact values. Missing the provider config reference (aws-709659200419) from ground truth reduces completeness. Added contextual interpretation about 'contract-critical defaults' is reasonable but not in ground truth. Navigation appears sound given accuracy of retrieved values, though process not visible."
|
||
|
|
},
|
||
|
|
"composite": 80,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "config-ipam-rds-backup",
|
||
|
|
"category": "configuration",
|
||
|
|
"difficulty": "hard",
|
||
|
|
"question": "What are the RDS automated backup retention period and backup window configured in ipam-core?",
|
||
|
|
"groundTruth": "RDS automated backup retention: 7 days. RDS backup window: 03:00-06:00 UTC.",
|
||
|
|
"llmAnswer": "RDS automated backup retention: 7 days \nRDS backup window: `03:00-06:00` UTC",
|
||
|
|
"filesRead": [
|
||
|
|
"reference/ipam-core-architecture.md",
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/configuration-reference.md",
|
||
|
|
"reference/index.md",
|
||
|
|
"reference/runtime-architecture.md"
|
||
|
|
],
|
||
|
|
"plannedFiles": [
|
||
|
|
"reference/ipam-core-architecture.md",
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/configuration-reference.md",
|
||
|
|
"reference/index.md",
|
||
|
|
"reference/runtime-architecture.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"accuracy": 5,
|
||
|
|
"completeness": 5,
|
||
|
|
"precision": 5,
|
||
|
|
"navigation": 4,
|
||
|
|
"notes": "AI answer matches ground truth exactly on both RDS retention (7 days) and backup window (03:00-06:00 UTC). No hallucinations or extraneous information. Navigation score reduced from 5 only because no execution log provided showing the agent's search process, though correct answer implies successful file discovery."
|
||
|
|
},
|
||
|
|
"composite": 95,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "config-dev-artifact-naming",
|
||
|
|
"category": "configuration",
|
||
|
|
"difficulty": "medium",
|
||
|
|
"question": "What is the artifact naming convention for development chart versions vs release chart versions in the OCI registry?",
|
||
|
|
"groundTruth": "Development artifacts: `{chart-name}:0.0.0-{commit-sha}`. Release artifacts: `{chart-name}:{chart-version}` (semantic version from Chart.yaml).",
|
||
|
|
"llmAnswer": "Based on the provided documentation:\n\n**Development chart versions:** `0.0.0-{commit-sha}`\n\n**Release chart versions:** Semantic versioning (e.g., `0.7.4`)\n\nThis is stated in `repository-structure.md` under the Versioning section. The Docker image naming convention in `naming-conventions.md` shows the full OCI registry path format as `{registry}/{repository}/{image-name}:{version}`, where the version component follows the patterns above.\n\nHowever, the documentation does not provide an explicit separate naming convention specifically for release versions in the OCI registry beyond stating they follow semantic versioning.",
|
||
|
|
"filesRead": [
|
||
|
|
"reference/naming-conventions.md",
|
||
|
|
"reference/release-process.md",
|
||
|
|
"reference/index.md",
|
||
|
|
"reference/repository-structure.md",
|
||
|
|
"reference/applicationset-reference.md"
|
||
|
|
],
|
||
|
|
"plannedFiles": [
|
||
|
|
"reference/naming-conventions.md",
|
||
|
|
"reference/release-process.md",
|
||
|
|
"reference/index.md",
|
||
|
|
"reference/repository-structure.md",
|
||
|
|
"reference/applicationset-reference.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"accuracy": 3,
|
||
|
|
"completeness": 2,
|
||
|
|
"precision": 4,
|
||
|
|
"navigation": 2,
|
||
|
|
"notes": "AI correctly identified the version components (0.0.0-{commit-sha} for dev, semantic versioning for release) but failed to provide the complete artifact naming format including the {chart-name} prefix for both cases. The answer lacks the explicit pattern `{chart-name}:{version}` that constitutes the full naming convention. Navigation appears limited—while the AI referenced documentation files, it didn't demonstrate locating the specific naming convention patterns stated in ground truth."
|
||
|
|
},
|
||
|
|
"composite": 55,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "services-tech-stack-orchestration",
|
||
|
|
"category": "services",
|
||
|
|
"difficulty": "easy",
|
||
|
|
"question": "What technologies make up the orchestration layer in Foxtrot's technology stack?",
|
||
|
|
"groundTruth": "Kubernetes (EKS, GKE, AKS), ArgoCD, and Crossplane.",
|
||
|
|
"llmAnswer": "The orchestration layer in Foxtrot's technology stack consists of:\n\n- Kubernetes (EKS, GKE, AKS)\n- ArgoCD\n- Crossplane",
|
||
|
|
"filesRead": [
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"explanation/technology-choices.md",
|
||
|
|
"reference/runtime-architecture.md",
|
||
|
|
"reference/index.md",
|
||
|
|
"explanation/index.md"
|
||
|
|
],
|
||
|
|
"plannedFiles": [
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"explanation/technology-choices.md",
|
||
|
|
"reference/runtime-architecture.md",
|
||
|
|
"reference/index.md",
|
||
|
|
"explanation/index.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"accuracy": 5,
|
||
|
|
"completeness": 5,
|
||
|
|
"precision": 5,
|
||
|
|
"navigation": 5,
|
||
|
|
"notes": "AI answer matches ground truth exactly. All three orchestration technologies (Kubernetes variants, ArgoCD, Crossplane) are present with correct specificity. No hallucinations or extraneous information. Clean, direct response."
|
||
|
|
},
|
||
|
|
"composite": 100,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "services-state-management",
|
||
|
|
"category": "services",
|
||
|
|
"difficulty": "easy",
|
||
|
|
"question": "What are the four state management services used in Foxtrot?",
|
||
|
|
"groundTruth": "Elasticsearch, Hazelcast, Redis, and Milvus.",
|
||
|
|
"llmAnswer": "Based on the documentation provided, the four state management services used in Foxtrot are:\n\n1. Elasticsearch\n2. Hazelcast\n3. Redis\n4. Milvus",
|
||
|
|
"filesRead": [
|
||
|
|
"reference/service-catalog.md",
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/index.md",
|
||
|
|
"explanation/index.md"
|
||
|
|
],
|
||
|
|
"plannedFiles": [
|
||
|
|
"reference/service-catalog.md",
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/index.md",
|
||
|
|
"reference/platform-concepts.md",
|
||
|
|
"explanation/index.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"accuracy": 5,
|
||
|
|
"completeness": 5,
|
||
|
|
"precision": 5,
|
||
|
|
"navigation": 5,
|
||
|
|
"notes": "AI answer matches ground truth exactly with all four services listed correctly (Elasticsearch, Hazelcast, Redis, Milvus). No hallucinations, no missing items, clean presentation. The citation of documentation suggests proper source navigation."
|
||
|
|
},
|
||
|
|
"composite": 100,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "services-eks-addon-versions",
|
||
|
|
"category": "services",
|
||
|
|
"difficulty": "hard",
|
||
|
|
"question": "What are the default pinned versions for the vpc-cni and coredns EKS addons in compute-common?",
|
||
|
|
"groundTruth": "vpc-cni: v1.20.1-eksbuild.1. coredns: v1.12.2-eksbuild.4.",
|
||
|
|
"llmAnswer": "Based on the compute-common-architecture.md file, the default pinned versions for EKS addons are:\n\n- **vpc-cni**: `v1.20.1-eksbuild.1`\n- **coredns**: `v1.12.2-eksbuild.4`",
|
||
|
|
"filesRead": [
|
||
|
|
"reference/compute-common-architecture.md",
|
||
|
|
"reference/compute-tools-architecture.md",
|
||
|
|
"reference/configuration-reference.md",
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/index.md"
|
||
|
|
],
|
||
|
|
"plannedFiles": [
|
||
|
|
"reference/compute-common-architecture.md",
|
||
|
|
"reference/compute-tools-architecture.md",
|
||
|
|
"reference/configuration-reference.md",
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/index.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"accuracy": 5,
|
||
|
|
"completeness": 5,
|
||
|
|
"precision": 5,
|
||
|
|
"navigation": 5,
|
||
|
|
"notes": "AI provided exact version strings matching ground truth for both addons. Answer is factually correct, covers all required items, contains no hallucinations, and demonstrates successful location of the source document (compute-common-architecture.md)."
|
||
|
|
},
|
||
|
|
"composite": 100,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "services-aws-nat-egress-model",
|
||
|
|
"category": "services",
|
||
|
|
"difficulty": "medium",
|
||
|
|
"question": "What is the AWS NAT egress model for internal VPCs vs production clusters in Foxtrot?",
|
||
|
|
"groundTruth": "Internal AWS VPCs use shared NAT egress. Production AWS clusters use dedicated NAT egress per cluster. This split reduces cost for internal environments while isolating production egress and capacity per cluster.",
|
||
|
|
"llmAnswer": "Internal AWS VPCs use shared NAT egress. Production AWS clusters use dedicated NAT egress per cluster. This split is intentional: shared NAT reduces cost for internal environments, while dedicated NAT isolates production egress and capacity per cluster. Operators should treat NAT topology as an environment characteristic, not assume one uniform AWS egress pattern across all VPCs.",
|
||
|
|
"filesRead": [
|
||
|
|
"reference/network-core-architecture.md",
|
||
|
|
"reference/network-common-architecture.md",
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/account-common-architecture.md",
|
||
|
|
"reference/index.md"
|
||
|
|
],
|
||
|
|
"plannedFiles": [
|
||
|
|
"reference/network-core-architecture.md",
|
||
|
|
"reference/network-common-architecture.md",
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/account-common-architecture.md",
|
||
|
|
"reference/index.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"accuracy": 5,
|
||
|
|
"completeness": 5,
|
||
|
|
"precision": 5,
|
||
|
|
"navigation": 4,
|
||
|
|
"notes": "AI answer contains all ground truth facts correctly stated. Adds operational guidance about treating NAT topology as environment characteristic, which is a reasonable inference rather than hallucination. No inaccurate information detected. Navigation score reduced from 5 only because actual file/source path cannot be verified from the answer alone."
|
||
|
|
},
|
||
|
|
"composite": 95,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "services-ipam-netbox-role",
|
||
|
|
"category": "services",
|
||
|
|
"difficulty": "medium",
|
||
|
|
"question": "What is NetBox used for in Foxtrot, and what infrastructure does it run on?",
|
||
|
|
"groundTruth": "NetBox is Foxtrot's IP address management (IPAM) source of truth. It tracks all VPC CIDR blocks across clouds, prevents CIDR overlap, and serves allocation data to compute-tools during VPC and cluster provisioning. It runs on AWS ECS with an RDS database and ALB, provisioned via Terraform in ipam-core.",
|
||
|
|
"llmAnswer": "NetBox serves as the IPAM (IP Address Management) system in Foxtrot. According to the documentation:\n\n**Purpose:**\n- Tracks all VPC CIDR blocks across clouds\n- Prevents CIDR overlap\n- Source of truth for network inventory\n\n**Infrastructure:**\nThe documentation does not specify what infrastructure NetBox runs on. It only mentions that `ipam-core` is a Core repository responsible for \"NetBox bootstrapping,\" but the hosting infrastructure is not detailed in the provided files.",
|
||
|
|
"filesRead": [
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/index.md",
|
||
|
|
"reference/service-catalog.md",
|
||
|
|
"reference/runtime-architecture.md",
|
||
|
|
"reference/network-core-architecture.md"
|
||
|
|
],
|
||
|
|
"plannedFiles": [
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/index.md",
|
||
|
|
"reference/service-catalog.md",
|
||
|
|
"reference/runtime-architecture.md",
|
||
|
|
"reference/network-core-architecture.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"accuracy": 3,
|
||
|
|
"completeness": 2,
|
||
|
|
"precision": 4,
|
||
|
|
"navigation": 2,
|
||
|
|
"notes": "AI correctly identified NetBox's IPAM purpose and functions (tracking VPC CIDR blocks, preventing overlap, source of truth) but failed to locate infrastructure details. The ground truth specifies AWS ECS, RDS, ALB, and Terraform provisioning in ipam-core—information the AI claimed was undocumented. This represents a significant gap in both completeness and navigation, as the question explicitly asks for infrastructure details. The AI avoided hallucination but missed critical required information."
|
||
|
|
},
|
||
|
|
"composite": 55,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "contracts-argo-gen-params-required",
|
||
|
|
"category": "contracts",
|
||
|
|
"difficulty": "easy",
|
||
|
|
"question": "What are the required fields in an argo-gen-params.yaml file for the ArgoCD ApplicationSet generator?",
|
||
|
|
"groundTruth": "chart.version (string), namespace (string), and environment (string).",
|
||
|
|
"llmAnswer": "Based on the documentation provided, the required fields in an `argo-gen-params.yaml` file for the ArgoCD ApplicationSet generator are:\n\n```yaml\nchart:\n version: string\nnamespace: string\nenvironment: string\n```\n\nThese are defined in the ArgoCD Application Contracts section of the API Contracts and Interfaces reference.",
|
||
|
|
"filesRead": [
|
||
|
|
"reference/applicationset-reference.md",
|
||
|
|
"reference/api-contracts.md",
|
||
|
|
"reference/configuration-reference.md",
|
||
|
|
"reference/index.md",
|
||
|
|
"explanation/index.md"
|
||
|
|
],
|
||
|
|
"plannedFiles": [
|
||
|
|
"reference/applicationset-reference.md",
|
||
|
|
"reference/api-contracts.md",
|
||
|
|
"reference/configuration-reference.md",
|
||
|
|
"reference/index.md",
|
||
|
|
"explanation/index.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"accuracy": 5,
|
||
|
|
"completeness": 5,
|
||
|
|
"precision": 4,
|
||
|
|
"navigation": 4,
|
||
|
|
"notes": "AI correctly identified all three required fields (chart.version, namespace, environment) matching ground truth exactly. YAML formatting is appropriate and helpful. Minor deductions for precision and navigation due to unverifiable documentation reference claim, though the core field information is accurate and complete."
|
||
|
|
},
|
||
|
|
"composite": 90,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "contracts-azure-xrd-naming",
|
||
|
|
"category": "contracts",
|
||
|
|
"difficulty": "medium",
|
||
|
|
"question": "What is the XRD name format and composition name format for Azure AKS in Foxtrot, with examples?",
|
||
|
|
"groundTruth": "XRD name format: `{plural}.{group}`, e.g. `reltioaksclusters.foxtrot.reltio.com`. Composition name format: `reltio-aks-cluster-{channel}-{version}`, e.g. `reltio-aks-cluster-prd-v1`.",
|
||
|
|
"llmAnswer": "**XRD name format:**\n- `{plural}.{group}`\n- Example: `reltioaksclusters.foxtrot.reltio.com`\n\n**Composition name format:**\n- `reltio-aks-cluster-{channel}-{version}`\n- Example: `reltio-aks-cluster-prd-v1`",
|
||
|
|
"filesRead": [
|
||
|
|
"reference/azure-xrd-contract.md",
|
||
|
|
"reference/naming-conventions.md",
|
||
|
|
"reference/azure-runtime-contracts.md",
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/index.md"
|
||
|
|
],
|
||
|
|
"plannedFiles": [
|
||
|
|
"reference/azure-xrd-contract.md",
|
||
|
|
"reference/naming-conventions.md",
|
||
|
|
"reference/azure-runtime-contracts.md",
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/index.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"accuracy": 5,
|
||
|
|
"completeness": 5,
|
||
|
|
"precision": 5,
|
||
|
|
"navigation": 5,
|
||
|
|
"notes": "AI answer matches ground truth exactly. Both XRD and composition name formats are correct with accurate examples. No hallucinations or extraneous information. Response is well-structured and complete."
|
||
|
|
},
|
||
|
|
"composite": 100,
|
||
|
|
"notFound": false
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "contracts-helm-chart-required-values",
|
||
|
|
"category": "contracts",
|
||
|
|
"difficulty": "medium",
|
||
|
|
"question": "What are the five required values that all app Helm charts must define?",
|
||
|
|
"groundTruth": "accountID (string), cloudProvider (string), region (string), domain (string), and imageRegistry (string).",
|
||
|
|
"llmAnswer": "NOT_FOUND\n\nThe provided documentation does not contain a specific list of five required values that all app Helm charts must define.",
|
||
|
|
"filesRead": [
|
||
|
|
"reference/app-common-architecture.md",
|
||
|
|
"reference/configuration-reference.md",
|
||
|
|
"reference/index.md",
|
||
|
|
"reference/applicationset-reference.md",
|
||
|
|
"explanation/index.md"
|
||
|
|
],
|
||
|
|
"plannedFiles": [
|
||
|
|
"reference/app-common-architecture.md",
|
||
|
|
"reference/configuration-reference.md",
|
||
|
|
"reference/index.md",
|
||
|
|
"reference/applicationset-reference.md",
|
||
|
|
"explanation/index.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"accuracy": 0,
|
||
|
|
"completeness": 0,
|
||
|
|
"precision": 5,
|
||
|
|
"navigation": 0,
|
||
|
|
"notes": "AI failed to locate the required Helm chart values in documentation. Ground truth specifies five required values (accountID, cloudProvider, region, domain, imageRegistry) but AI returned NOT_FOUND, indicating search/navigation failure."
|
||
|
|
},
|
||
|
|
"composite": 25,
|
||
|
|
"notFound": true
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"id": "contracts-sync-wave-ordering",
|
||
|
|
"category": "contracts",
|
||
|
|
"difficulty": "hard",
|
||
|
|
"question": "What are the ArgoCD sync wave values and what resource types are deployed at each wave?",
|
||
|
|
"groundTruth": "Wave -5: prerequisites. Wave 0: infrastructure. Wave 5: state apps. Wave 10: application services. Wave 15: post-deployment resources (monitoring, backup hooks, finalizers).",
|
||
|
|
"llmAnswer": "NOT_FOUND\n\nThe provided documentation files do not contain information about ArgoCD sync wave values or resource types deployed at each wave. This level of operational detail would typically be found in configuration reference or ArgoCD-specific operational documentation, which is not included in the files reviewed.",
|
||
|
|
"filesRead": [
|
||
|
|
"reference/applicationset-reference.md",
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/release-process.md",
|
||
|
|
"reference/runtime-architecture.md",
|
||
|
|
"reference/index.md"
|
||
|
|
],
|
||
|
|
"plannedFiles": [
|
||
|
|
"reference/applicationset-reference.md",
|
||
|
|
"reference/system-architecture.md",
|
||
|
|
"reference/release-process.md",
|
||
|
|
"reference/runtime-architecture.md",
|
||
|
|
"reference/index.md"
|
||
|
|
],
|
||
|
|
"score": {
|
||
|
|
"accuracy": 0,
|
||
|
|
"completeness": 0,
|
||
|
|
"precision": 5,
|
||
|
|
"navigation": 0,
|
||
|
|
"notes": "AI correctly identified that the required information was not present in the provided documentation. No hallucinations or false claims were made. However, the answer provides zero coverage of the ground truth specifications regarding the five ArgoCD sync waves and their associated resource types."
|
||
|
|
},
|
||
|
|
"composite": 25,
|
||
|
|
"notFound": true
|
||
|
|
}
|
||
|
|
]
|
||
|
|
}
|