feat: confluence benchmark, pattern extractor, agent KB, UX spec
- extract-patterns.js: mines layered arch, ArgoCD appsets, cloud regions, CIDR allocations, naming conventions, sync waves, tech stack from code - agent-kb.js: token-efficient JSON rendering of same doc tree - eval-confluence-ref-questions.json: 32 reference-only benchmark questions - wiggum-v2.sh: Ralph Wiggum loop targeting confluence baseline (77.8%) - docs/human-ux-spec.md: BMad UX designer spec for human doc structure - Eval results: V2 at 28.7% vs confluence 77.8% baseline - Hub/spoke ownership now correctly extracted (95% on that question) - Naming conventions, regions, CIDRs surfaced in system-architecture.md
This commit is contained in:
Jarvis Prime
263
eval-confluence-questions.json
Normal file
263
eval-confluence-questions.json
Normal file
@@ -0,0 +1,263 @@
|
||||
{
|
||||
"generated": "2026-03-10T04:53:00Z",
|
||||
"benchmark": "confluence-gold-standard",
|
||||
"count": 32,
|
||||
"questions": [
|
||||
{
|
||||
"id": "arch-layered-order",
|
||||
"category": "architecture",
|
||||
"question": "What are the five layers in Foxtrot's layered architecture, listed from top to bottom?",
|
||||
"answer": "Application Layer (app-*), Compute Layer (compute-*), Network Layer (network-*), Account Layer (account-*), Control Plane (control-core).",
|
||||
"source": "reference/system-architecture.md",
|
||||
"difficulty": "easy"
|
||||
},
|
||||
{
|
||||
"id": "arch-hub-spoke-ownership",
|
||||
"category": "architecture",
|
||||
"question": "Which ArgoCD instance owns the account, network, and compute layers, and which owns the application layer?",
|
||||
"answer": "Hub ArgoCD owns account, network, and compute layers (infrastructure and control-plane appsets). Spoke ArgoCD owns the application layer (app-runtime appsets only).",
|
||||
"source": "reference/system-architecture.md, reference/runtime-architecture.md, explanation/platform-concepts.md",
|
||||
"difficulty": "easy"
|
||||
},
|
||||
{
|
||||
"id": "arch-aws-regions",
|
||||
"category": "architecture",
|
||||
"question": "What AWS regions does Foxtrot support?",
|
||||
"answer": "us-east-1, us-west-2, eu-west-1, eu-central-1.",
|
||||
"source": "reference/system-architecture.md",
|
||||
"difficulty": "easy"
|
||||
},
|
||||
{
|
||||
"id": "arch-gcp-shared-vpc-host",
|
||||
"category": "architecture",
|
||||
"question": "What is the default GCP host project used for Shared VPC in network-common, and what is the host network path it attaches to?",
|
||||
"answer": "The default host project is `network-services-436015`. When `sharedVpc.enabled: true`, service project resources attach to the host network path `projects/HOST_PROJECT/global/networks/gcp-core-network`.",
|
||||
"source": "reference/network-common-architecture.md",
|
||||
"difficulty": "medium"
|
||||
},
|
||||
{
|
||||
"id": "arch-cidr-employee-access",
|
||||
"category": "architecture",
|
||||
"question": "What is the CIDR range for the employee access (bastions) segment on AWS and on GCP?",
|
||||
"answer": "AWS: 10.208.128.0/24. GCP: 10.128.128.0/24.",
|
||||
"source": "reference/system-architecture.md",
|
||||
"difficulty": "medium"
|
||||
},
|
||||
{
|
||||
"id": "arch-production-cidr",
|
||||
"category": "architecture",
|
||||
"question": "What is the CIDR range for production workloads on AWS and on GCP?",
|
||||
"answer": "AWS: 10.192.0.0/10. GCP: 10.128.0.0/10.",
|
||||
"source": "reference/system-architecture.md",
|
||||
"difficulty": "medium"
|
||||
},
|
||||
{
|
||||
"id": "dep-runtime-common-horizontal",
|
||||
"category": "dependencies",
|
||||
"question": "Which runtime repositories consume charts from which common repositories?",
|
||||
"answer": "app-runtime consumes charts from app-common; compute-runtime consumes charts from compute-common; network-runtime consumes charts from network-common; account-runtime consumes charts from account-common.",
|
||||
"source": "reference/system-architecture.md, reference/repository-structure.md",
|
||||
"difficulty": "easy"
|
||||
},
|
||||
{
|
||||
"id": "dep-vertical-layers",
|
||||
"category": "dependencies",
|
||||
"question": "What are the vertical layer dependencies in Foxtrot's architecture?",
|
||||
"answer": "Application Layer depends on Compute Layer; Compute Layer depends on Network Layer; Network Layer depends on Account Layer; Account Layer is managed by the Control Plane.",
|
||||
"source": "reference/system-architecture.md",
|
||||
"difficulty": "easy"
|
||||
},
|
||||
{
|
||||
"id": "dep-create-account-repos",
|
||||
"category": "dependencies",
|
||||
"question": "How many PRs does `create-account` produce and which repositories does it touch?",
|
||||
"answer": "4 PRs: account-runtime, network-core (account principal updates), control-core (provider config + assume-role policy updates), and cloud-iam (account bootstrap TF).",
|
||||
"source": "reference/compute-tools-architecture.md",
|
||||
"difficulty": "medium"
|
||||
},
|
||||
{
|
||||
"id": "dep-create-cluster-repos",
|
||||
"category": "dependencies",
|
||||
"question": "Which repositories does `create-cluster` commit to, and how many PRs does it typically produce?",
|
||||
"answer": "4 PRs: compute-runtime (cluster path) plus 3 PRs in cloud-auth-core (cluster/node role PR, IRSA role PR, and network role PR). It also calls cluster-netbox-allocate.py to reserve/allocate cluster subnet CIDRs.",
|
||||
"source": "reference/compute-tools-architecture.md",
|
||||
"difficulty": "medium"
|
||||
},
|
||||
{
|
||||
"id": "dep-compute-common-deps",
|
||||
"category": "dependencies",
|
||||
"question": "What are the core dependencies of compute-common for cluster provisioning?",
|
||||
"answer": "Crossplane core + provider-aws/provider-kubernetes, Helm, KCL build pipeline, and ArgoCD deployment flow.",
|
||||
"source": "reference/compute-common-architecture.md",
|
||||
"difficulty": "medium"
|
||||
},
|
||||
{
|
||||
"id": "ops-argocd-deployment-flow",
|
||||
"category": "operations",
|
||||
"question": "What are the steps in the ArgoCD deployment flow for an application, from ApplicationSet watching app-runtime to pods starting?",
|
||||
"answer": "1. ApplicationSet watches app-runtime. 2. Reads argo-gen-params.yaml per environment. 3. Generates Application. 4. Pulls chart from OCI registry. 5. Merges values and overrides. 6. Renders manifests. 7. Applies to cluster. 8. ESO fetches secrets. 9. Pods start with injected configs.",
|
||||
"source": "reference/system-architecture.md",
|
||||
"difficulty": "medium"
|
||||
},
|
||||
{
|
||||
"id": "ops-ebf-release-pattern",
|
||||
"category": "operations",
|
||||
"question": "What are the steps in the EBF/weekly release pattern?",
|
||||
"answer": "1. Create release branch from develop. 2. Promote versions for release/hotfix environments. 3. Run stage pipelines (preview, preprod, production). 4. Merge promoted changes to main. 5. Reconcile hotfix and develop branches.",
|
||||
"source": "reference/release-process.md",
|
||||
"difficulty": "medium"
|
||||
},
|
||||
{
|
||||
"id": "ops-rollback-procedure",
|
||||
"category": "operations",
|
||||
"question": "What is the preferred rollback procedure in Foxtrot, and when are script-level deletes used instead?",
|
||||
"answer": "GitOps rollback is preferred: revert the runtime/config commit, merge the revert PR, and confirm ArgoCD reconciliation. Script-level deletes (delete-vpc.sh, delete-cluster.sh, delete-app.sh) are used only with explicit approval and change control.",
|
||||
"source": "reference/verification-and-rollback.md",
|
||||
"difficulty": "medium"
|
||||
},
|
||||
{
|
||||
"id": "ops-branch-cluster-mapping",
|
||||
"category": "operations",
|
||||
"question": "What is the branch-to-cluster mapping in app-runtime? Which branch targets which environments?",
|
||||
"answer": "develop \u2192 internal clusters (development and QA); main \u2192 customer-facing clusters (production); hotfix \u2192 EBF verification; release \u2192 release verification (weekly release staging).",
|
||||
"source": "reference/system-architecture.md, reference/release-process.md",
|
||||
"difficulty": "easy"
|
||||
},
|
||||
{
|
||||
"id": "ops-jenkins-jobs",
|
||||
"category": "operations",
|
||||
"question": "What are the six Jenkins jobs in the Foxtrot pipeline and what does each do?",
|
||||
"answer": "appRuntimeSync: syncs runtime branches on changes to main/hotfix/release. foxtrotSync: updates imagetags-override.yaml with new image tags. foxtrotPromote: promotes and merges release outputs. chartSync: updates argo-gen-params.yaml with chart versions. chartCheckInAppCommon: detects chart version mismatches and triggers chartSync. chartPromoteVersion: promotes chart versions per release stage.",
|
||||
"source": "reference/api-contracts.md, reference/release-process.md",
|
||||
"difficulty": "hard"
|
||||
},
|
||||
{
|
||||
"id": "ops-create-cluster-timeout",
|
||||
"category": "operations",
|
||||
"question": "What is the timeout for waiting for a cluster to reach ready condition in the create-cluster workflow?",
|
||||
"answer": "3600 seconds (1 hour), with a poll interval of 30 seconds.",
|
||||
"source": "reference/compute-tools-architecture.md",
|
||||
"difficulty": "medium"
|
||||
},
|
||||
{
|
||||
"id": "config-cloud-resource-naming",
|
||||
"category": "configuration",
|
||||
"question": "What is the canonical format for cloud resource names in Foxtrot, and what does each component represent?",
|
||||
"answer": "Format: `{provider}-{phase}-{region-code}-{index}-{resource}[-{descriptor}]`. provider: aws/gcp/azr; phase: prd/iqa/dev; region-code: two-letter derived code; index: incremental integer; resource: vpc/eks/gke/aks/lb/rds. Example: `gcp-prd-us-1-vpc-blue`.",
|
||||
"source": "reference/naming-conventions.md",
|
||||
"difficulty": "easy"
|
||||
},
|
||||
{
|
||||
"id": "config-region-code-algorithm",
|
||||
"category": "configuration",
|
||||
"question": "What is the algorithm for deriving a two-letter region code from a cloud region name, and what is the region code for `westeurope`?",
|
||||
"answer": "Algorithm: 1. Remove directional words (east, west, north, south). 2. Remove separators. 3. Take first two characters. For `westeurope`: remove 'west' \u2192 'europe' \u2192 'eu'.",
|
||||
"source": "reference/naming-conventions.md",
|
||||
"difficulty": "medium"
|
||||
},
|
||||
{
|
||||
"id": "config-app-config-merge-order",
|
||||
"category": "configuration",
|
||||
"question": "What is the configuration merge order for app charts in Foxtrot, from lowest to highest precedence?",
|
||||
"answer": "1. values.yaml, 2. default-properties.yaml, 3. default-values.yaml / k8s-values.yaml, 4. common-values.yaml, 5. properties-override.yaml, 6. k8s-override.yaml, 7. imagetags-override.yaml.",
|
||||
"source": "reference/configuration-reference.md, reference/repository-structure.md, reference/app-common-architecture.md",
|
||||
"difficulty": "medium"
|
||||
},
|
||||
{
|
||||
"id": "config-account-creation-product-id",
|
||||
"category": "configuration",
|
||||
"question": "What is the AWS Service Catalog product ID used by account-common for account creation, and what are the customer-facing and internal OU IDs?",
|
||||
"answer": "Service Catalog product ID: `prod-mts6togilnnuk`. Customer-facing OU: `ou-8vpz-mogk4qua`. Internal OU: `ou-8vpz-rw7etshv`. Provider config reference: `aws-709659200419`.",
|
||||
"source": "reference/account-common-architecture.md",
|
||||
"difficulty": "hard"
|
||||
},
|
||||
{
|
||||
"id": "config-ipam-rds-backup",
|
||||
"category": "configuration",
|
||||
"question": "What are the RDS automated backup retention period and backup window configured in ipam-core?",
|
||||
"answer": "RDS automated backup retention: 7 days. RDS backup window: 03:00-06:00 UTC.",
|
||||
"source": "reference/ipam-core-architecture.md",
|
||||
"difficulty": "hard"
|
||||
},
|
||||
{
|
||||
"id": "config-dev-artifact-naming",
|
||||
"category": "configuration",
|
||||
"question": "What is the artifact naming convention for development chart versions vs release chart versions in the OCI registry?",
|
||||
"answer": "Development artifacts: `{chart-name}:0.0.0-{commit-sha}`. Release artifacts: `{chart-name}:{chart-version}` (semantic version from Chart.yaml).",
|
||||
"source": "reference/api-contracts.md, reference/network-common-architecture.md",
|
||||
"difficulty": "medium"
|
||||
},
|
||||
{
|
||||
"id": "services-tech-stack-orchestration",
|
||||
"category": "services",
|
||||
"question": "What technologies make up the orchestration layer in Foxtrot's technology stack?",
|
||||
"answer": "Kubernetes (EKS, GKE, AKS), ArgoCD, and Crossplane.",
|
||||
"source": "reference/system-architecture.md",
|
||||
"difficulty": "easy"
|
||||
},
|
||||
{
|
||||
"id": "services-state-management",
|
||||
"category": "services",
|
||||
"question": "What are the four state management services used in Foxtrot?",
|
||||
"answer": "Elasticsearch, Hazelcast, Redis, and Milvus.",
|
||||
"source": "reference/system-architecture.md",
|
||||
"difficulty": "easy"
|
||||
},
|
||||
{
|
||||
"id": "services-eks-addon-versions",
|
||||
"category": "services",
|
||||
"question": "What are the default pinned versions for the vpc-cni and coredns EKS addons in compute-common?",
|
||||
"answer": "vpc-cni: v1.20.1-eksbuild.1. coredns: v1.12.2-eksbuild.4.",
|
||||
"source": "reference/compute-common-architecture.md",
|
||||
"difficulty": "hard"
|
||||
},
|
||||
{
|
||||
"id": "services-aws-nat-egress-model",
|
||||
"category": "services",
|
||||
"question": "What is the AWS NAT egress model for internal VPCs vs production clusters in Foxtrot?",
|
||||
"answer": "Internal AWS VPCs use shared NAT egress. Production AWS clusters use dedicated NAT egress per cluster. This split reduces cost for internal environments while isolating production egress and capacity per cluster.",
|
||||
"source": "reference/network-common-architecture.md, reference/operator-basics.md",
|
||||
"difficulty": "medium"
|
||||
},
|
||||
{
|
||||
"id": "services-ipam-netbox-role",
|
||||
"category": "services",
|
||||
"question": "What is NetBox used for in Foxtrot, and what infrastructure does it run on?",
|
||||
"answer": "NetBox is Foxtrot's IP address management (IPAM) source of truth. It tracks all VPC CIDR blocks across clouds, prevents CIDR overlap, and serves allocation data to compute-tools during VPC and cluster provisioning. It runs on AWS ECS with an RDS database and ALB, provisioned via Terraform in ipam-core.",
|
||||
"source": "reference/ipam-core-architecture.md, reference/system-architecture.md",
|
||||
"difficulty": "medium"
|
||||
},
|
||||
{
|
||||
"id": "contracts-argo-gen-params-required",
|
||||
"category": "contracts",
|
||||
"question": "What are the required fields in an argo-gen-params.yaml file for the ArgoCD ApplicationSet generator?",
|
||||
"answer": "chart.version (string), namespace (string), and environment (string).",
|
||||
"source": "reference/api-contracts.md, reference/azure-runtime-contracts.md",
|
||||
"difficulty": "easy"
|
||||
},
|
||||
{
|
||||
"id": "contracts-azure-xrd-naming",
|
||||
"category": "contracts",
|
||||
"question": "What is the XRD name format and composition name format for Azure AKS in Foxtrot, with examples?",
|
||||
"answer": "XRD name format: `{plural}.{group}`, e.g. `reltioaksclusters.foxtrot.reltio.com`. Composition name format: `reltio-aks-cluster-{channel}-{version}`, e.g. `reltio-aks-cluster-prd-v1`.",
|
||||
"source": "reference/azure-xrd-contract.md",
|
||||
"difficulty": "medium"
|
||||
},
|
||||
{
|
||||
"id": "contracts-helm-chart-required-values",
|
||||
"category": "contracts",
|
||||
"question": "What are the five required values that all app Helm charts must define?",
|
||||
"answer": "accountID (string), cloudProvider (string), region (string), domain (string), and imageRegistry (string).",
|
||||
"source": "reference/api-contracts.md",
|
||||
"difficulty": "medium"
|
||||
},
|
||||
{
|
||||
"id": "contracts-sync-wave-ordering",
|
||||
"category": "contracts",
|
||||
"question": "What are the ArgoCD sync wave values and what resource types are deployed at each wave?",
|
||||
"answer": "Wave -5: prerequisites. Wave 0: infrastructure. Wave 5: state apps. Wave 10: application services. Wave 15: post-deployment resources (monitoring, backup hooks, finalizers).",
|
||||
"source": "reference/api-contracts.md",
|
||||
"difficulty": "hard"
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user