dev-intel-v2/eval-questions.json

{
  "generated": "2026-03-10T00:27:35.845Z",
  "count": 33,
  "questions": [
    {
      "id": "struct-001",
      "category": "structural",
      "difficulty": "easy",
      "audience": [
        "human",
        "machine"
      ],
      "question": "How many Helm charts are defined across the Foxtrot monorepo?",
      "answer": "76",
      "answerType": "exact",
      "source": "Chart.yaml discovery"
    },
    {
      "id": "struct-002",
      "category": "structural",
      "difficulty": "medium",
      "audience": [
        "human",
        "machine"
      ],
      "question": "Which 5 Helm charts produce the most Kubernetes resources? List them with their resource counts.",
      "answer": "cluster (app-common/apps-provider/addons/charts/common/cluster): 55\ncluster (app-common/charts/cluster): 32\njenkins (app-common/charts/jenkins): 29\nkube-state-metrics (app-common/charts/otel-collector/charts/kube-state-metrics): 27\nfoxtrot-eks (compute-common/templates/foxtrot-eks): 24",
      "answerType": "ranked-list",
      "source": "template scanning"
    },
    {
      "id": "struct-003",
      "category": "structural",
      "difficulty": "easy",
      "audience": [
        "human",
        "machine"
      ],
      "question": "How many subsystems does the Foxtrot codebase contain?",
      "answer": "12",
      "answerType": "exact",
      "source": "subsystem aggregation"
    },
    {
      "id": "struct-004",
      "category": "structural",
      "difficulty": "easy",
      "audience": [
        "human"
      ],
      "question": "Which subsystem contains the most files, and how many?",
      "answer": "runtime: 16309 files",
      "answerType": "exact",
      "source": "subsystem aggregation"
    },
    {
      "id": "dep-external-dns-001",
      "category": "dependency",
      "difficulty": "medium",
      "audience": [
        "human",
        "machine"
      ],
      "question": "What are the dependencies of the \"external-dns\" chart (at app-common/apps-provider/addons/charts/common/external-dns)?",
      "answer": "external-dns (1.17.0) [condition: external-dns.enabled]",
      "answerType": "list",
      "source": "app-common/apps-provider/addons/charts/common/external-dns/Chart.yaml"
    },
    {
      "id": "dep-ingress-nginx-001",
      "category": "dependency",
      "difficulty": "medium",
      "audience": [
        "human",
        "machine"
      ],
      "question": "What are the dependencies of the \"ingress-nginx\" chart (at app-common/apps-provider/addons/charts/common/ingress-nginx)?",
      "answer": "ingress-nginx (4.12.0) [condition: ingress-nginx.enabled]",
      "answerType": "list",
      "source": "app-common/apps-provider/addons/charts/common/ingress-nginx/Chart.yaml"
    },
    {
      "id": "dep-kube-state-metrics-001",
      "category": "dependency",
      "difficulty": "medium",
      "audience": [
        "human",
        "machine"
      ],
      "question": "What are the dependencies of the \"kube-state-metrics\" chart (at app-common/apps-provider/addons/charts/common/kube-state-metrics)?",
      "answer": "kube-state-metrics (5.26.0) [condition: kube-state-metrics.enabled]",
      "answerType": "list",
      "source": "app-common/apps-provider/addons/charts/common/kube-state-metrics/Chart.yaml"
    },
    {
      "id": "dep-open-telemetry-001",
      "category": "dependency",
      "difficulty": "medium",
      "audience": [
        "human",
        "machine"
      ],
      "question": "What are the dependencies of the \"open-telemetry\" chart (at app-common/apps-provider/addons/charts/common/otel-collector)?",
      "answer": "opentelemetry-operator (0.26.4) [condition: opentelemetry-operator.enabled]\nk8s-metrics (0.1.0) [condition: k8s-metrics.enabled]\nagent (0.1.1) [condition: agent.enabled]\ncommon (0.1.0) [condition: common.enabled]\ngw-metrics (0.1.0) [condition: gw-metrics.enabled]\ngw-traces (0.1.0) [condition: gw-traces.enabled]\nprom-scraper (0.1.0) [condition: prom-scraper.enabled]",
      "answerType": "list",
      "source": "app-common/apps-provider/addons/charts/common/otel-collector/Chart.yaml"
    },
    {
      "id": "dep-reltio-prometheus-001",
      "category": "dependency",
      "difficulty": "medium",
      "audience": [
        "human",
        "machine"
      ],
      "question": "What are the dependencies of the \"reltio-prometheus\" chart (at app-common/apps-provider/addons/charts/common/prometheus)?",
      "answer": "kube-prometheus-stack (69.4.1) [condition: kube-prometheus-stack.enabled]\nprometheus-adapter (4.11.0) [condition: prometheus-adapter.enabled]",
      "answerType": "list",
      "source": "app-common/apps-provider/addons/charts/common/prometheus/Chart.yaml"
    },
    {
      "id": "contract-001",
      "category": "contract",
      "difficulty": "medium",
      "audience": [
        "human",
        "machine"
      ],
      "question": "Which secrets or ConfigMaps are shared across multiple Helm charts? List each with the charts that use it.",
      "answer": "logdna-agent-key: cluster\nlogdna-agent-tags: cluster\nvault-secret: ai-app, cleanse-app, pms-app, rdm-app, shared-app\nvault-secret-rdm-tasks: ai-app, cleanse-app, pms-app, rdm-app, shared-app",
      "answerType": "list",
      "source": "template interaction scanning"
    },
    {
      "id": "config-mdm-app-domain",
      "category": "configuration",
      "difficulty": "easy",
      "audience": [
        "human"
      ],
      "question": "What is the default value of \"domain\" in the mdm-app chart?",
      "answer": "cloud.reltio.com",
      "answerType": "exact",
      "source": "app-common/charts/mdm-app/values.yaml"
    },
    {
      "id": "config-mdm-app-count",
      "category": "configuration",
      "difficulty": "easy",
      "audience": [
        "machine"
      ],
      "question": "How many top-level configuration keys does the mdm-app chart (at app-common/charts/mdm-app) expose in its values.yaml?",
      "answer": "38",
      "answerType": "exact",
      "source": "app-common/charts/mdm-app/values.yaml"
    },
    {
      "id": "config-ai-app-domain",
      "category": "configuration",
      "difficulty": "easy",
      "audience": [
        "human"
      ],
      "question": "What is the default value of \"domain\" in the ai-app chart?",
      "answer": "cloud.reltio.com",
      "answerType": "exact",
      "source": "app-common/charts/ai-app/values.yaml"
    },
    {
      "id": "config-ai-app-count",
      "category": "configuration",
      "difficulty": "easy",
      "audience": [
        "machine"
      ],
      "question": "How many top-level configuration keys does the ai-app chart (at app-common/charts/ai-app) expose in its values.yaml?",
      "answer": "34",
      "answerType": "exact",
      "source": "app-common/charts/ai-app/values.yaml"
    },
    {
      "id": "config-elasticsearch-domain",
      "category": "configuration",
      "difficulty": "easy",
      "audience": [
        "human"
      ],
      "question": "What is the default value of \"domain\" in the elasticsearch chart?",
      "answer": "etalon.reltio.com",
      "answerType": "exact",
      "source": "app-common/apps-provider/addons/charts/state-apps/es/values.yaml"
    },
    {
      "id": "config-elasticsearch-count",
      "category": "configuration",
      "difficulty": "easy",
      "audience": [
        "machine"
      ],
      "question": "How many top-level configuration keys does the elasticsearch chart (at app-common/apps-provider/addons/charts/state-apps/es) expose in its values.yaml?",
      "answer": "32",
      "answerType": "exact",
      "source": "app-common/apps-provider/addons/charts/state-apps/es/values.yaml"
    },
    {
      "id": "config-hazelcast-domain",
      "category": "configuration",
      "difficulty": "easy",
      "audience": [
        "human"
      ],
      "question": "What is the default value of \"domain\" in the hazelcast chart?",
      "answer": "etalon.reltio.com",
      "answerType": "exact",
      "source": "app-common/apps-provider/addons/charts/state-apps/hz/values.yaml"
    },
    {
      "id": "config-hazelcast-count",
      "category": "configuration",
      "difficulty": "easy",
      "audience": [
        "machine"
      ],
      "question": "How many top-level configuration keys does the hazelcast chart (at app-common/apps-provider/addons/charts/state-apps/hz) expose in its values.yaml?",
      "answer": "32",
      "answerType": "exact",
      "source": "app-common/apps-provider/addons/charts/state-apps/hz/values.yaml"
    },
    {
      "id": "config-redis-domain",
      "category": "configuration",
      "difficulty": "easy",
      "audience": [
        "human"
      ],
      "question": "What is the default value of \"domain\" in the redis chart?",
      "answer": "internal.reltio.com",
      "answerType": "exact",
      "source": "app-common/charts/redis/values.yaml"
    },
    {
      "id": "config-redis-count",
      "category": "configuration",
      "difficulty": "easy",
      "audience": [
        "machine"
      ],
      "question": "How many top-level configuration keys does the redis chart (at app-common/charts/redis) expose in its values.yaml?",
      "answer": "23",
      "answerType": "exact",
      "source": "app-common/charts/redis/values.yaml"
    },
    {
      "id": "interaction-001",
      "category": "interaction",
      "difficulty": "medium",
      "audience": [
        "human",
        "machine"
      ],
      "question": "Which Helm charts reference Kubernetes services from other components? List each chart and the service it calls.",
      "answer": "hazelcast (app-common/apps-provider/addons/charts/state-apps/hz) → kubernetes\nhazelcast (app-common/charts/hazelcast) → kubernetes\ngw-metrics (app-common/charts/otel-collector/charts/gw-metrics) → argo-rollouts-metrics\nredis (app-common/charts/redis) → headless",
      "answerType": "list",
      "source": "template interaction scanning"
    },
    {
      "id": "interaction-002",
      "category": "interaction",
      "difficulty": "hard",
      "audience": [
        "human",
        "machine"
      ],
      "question": "Which network ports are used by multiple Helm charts? List each port and the charts that expose it.",
      "answer": "Port 443: cluster, twistlock-defender, eck-operator, twistlock\nPort 8889: gw-metrics, gw-traces, kube-state-metrics\nPort 9200: eck-operator, elasticsearch\nPort 13133: agent, gw-metrics, gw-traces, k8s-metrics, prom-scraper, elasticsearch",
      "answerType": "list",
      "source": "template port scanning"
    },
    {
      "id": "resource-001",
      "category": "resource",
      "difficulty": "medium",
      "audience": [
        "human",
        "machine"
      ],
      "question": "What are the 10 most common Kubernetes resource types generated across all Foxtrot Helm charts?",
      "answer": "ClusterRole: 51\nServiceAccount: 49\nSecret: 43\nExternalSecret: 38\nService: 36\nRole: 30\nClusterRoleBinding: 29\nDeployment: 29\nConfigMap: 25\nRoleBinding: 25",
      "answerType": "ranked-list",
      "source": "template resource scanning"
    },
    {
      "id": "resource-mdm-app-001",
      "category": "resource",
      "difficulty": "medium",
      "audience": [
        "human",
        "machine"
      ],
      "question": "What Kubernetes resource types does the mdm-app chart generate? List each type and count.",
      "answer": "Secret: 5\nJob: 4\nDeployment: 2\nPersistentVolumeClaim: 2\nService: 2\nExternalSecret: 2\nHorizontalPodAutoscaler: 1\nRollout: 1\nPersistentVolume: 1\nPodDisruptionBudget: 1\nServiceAccount: 1",
      "answerType": "list",
      "source": "app-common/charts/mdm-app/templates/"
    },
    {
      "id": "resource-cassandra-001",
      "category": "resource",
      "difficulty": "medium",
      "audience": [
        "human",
        "machine"
      ],
      "question": "What Kubernetes resource types does the cassandra chart generate? List each type and count.",
      "answer": "Service: 2\nStatefulSet: 1\nExternalSecret: 1\nClusterSecretStore: 1\nConfigMap: 1\nSecret: 1\nServiceAccount: 1\nRole: 1\nRoleBinding: 1\nClusterRole: 1\nClusterRoleBinding: 1\nCronJob: 1\nJob: 1",
      "answerType": "list",
      "source": "app-common/apps-provider/addons/charts/state-cluster/cassandra/templates/"
    },
    {
      "id": "resource-jenkins-001",
      "category": "resource",
      "difficulty": "medium",
      "audience": [
        "human",
        "machine"
      ],
      "question": "What Kubernetes resource types does the jenkins chart generate? List each type and count.",
      "answer": "ConfigMap: 5\nService: 3\nSecret: 3\nIngress: 2\nServiceAccount: 2\nPersistentVolumeClaim: 1\nSecurityGroupPolicy: 1\nPrometheusRule: 1\nBackendConfig: 1\nNetworkPolicy: 1\nPodDisruptionBudget: 1\nPodMonitoring: 1\nRoute: 1\nServiceMonitor: 1\nStatefulSet: 1\nRole: 1\nRoleBinding: 1\nSecretClaim: 1\nPod: 1",
      "answerType": "list",
      "source": "app-common/charts/jenkins/templates/"
    },
    {
      "id": "cross-sub-001",
      "category": "cross-subsystem",
      "difficulty": "hard",
      "audience": [
        "human",
        "machine"
      ],
      "question": "Which subsystems depend on app-tools, and how heavily (by call+import count)?",
      "answer": "ipam-tools: 6\ncompute-tools: 4\napp-common: 1",
      "answerType": "list",
      "source": "dependency matrix"
    },
    {
      "id": "cross-sub-002",
      "category": "cross-subsystem",
      "difficulty": "easy",
      "audience": [
        "human"
      ],
      "question": "Which subsystems are identified as cross-cutting concerns?",
      "answer": "root",
      "answerType": "list",
      "source": "subsystem aggregation"
    },
    {
      "id": "arch-001",
      "category": "architectural",
      "difficulty": "hard",
      "audience": [
        "human"
      ],
      "question": "The following subsystems have 0 detected functions and 0 modules: account-common, network-common, network-core, root. Why might this be the case, and what do they actually contain?",
      "answer": "These subsystems primarily contain Helm charts with Go-templated YAML, Terraform HCL, and Crossplane compositions. The code analysis pipeline detects functions/modules from Python, Go, TypeScript, and shell scripts — but Helm templates use Go template syntax ({{ }}) which doesn't produce traditional function/module entities. Their content is captured through the Helm chart extraction phase instead.",
      "answerType": "explanation",
      "source": "architectural analysis"
    },
    {
      "id": "arch-002",
      "category": "architectural",
      "difficulty": "easy",
      "audience": [
        "human",
        "machine"
      ],
      "question": "What is the current version and appVersion of the mdm-app Helm chart?",
      "answer": "version: 0.7.4, appVersion: 2025.12-12-develop",
      "answerType": "exact",
      "source": "app-common/charts/mdm-app/Chart.yaml"
    },
    {
      "id": "scenario-001",
      "category": "scenario",
      "difficulty": "hard",
      "audience": [
        "human"
      ],
      "question": "If you need to rotate the \"vault-secret\" shared secret, which Helm charts would be affected and need redeployment?",
      "answer": "ai-app, cleanse-app, pms-app, rdm-app, shared-app",
      "answerType": "list",
      "source": "template interaction scanning"
    },
    {
      "id": "scenario-002",
      "category": "scenario",
      "difficulty": "hard",
      "audience": [
        "human"
      ],
      "question": "A new engineer needs to deploy the MDM application stack. What charts, configuration values, and external dependencies should they understand first?",
      "answer": "Chart: mdm-app (app-common/charts/mdm-app), Version: 0.7.4\nKey values: clusterName, environmentType, arch, cloudProvider, region, vaultEnv, imagePullPolicy, clusterDomain, terminationGracePeriodSeconds, etalonsubDomain (+28 more)\nResources generated: 22 K8s resources\nInteractions: none detected",
      "answerType": "explanation",
      "source": "app-common/charts/mdm-app"
    },
    {
      "id": "scenario-003",
      "category": "scenario",
      "difficulty": "medium",
      "audience": [
        "human"
      ],
      "question": "What network ports does the Cassandra chart expose, and what are they typically used for?",
      "answer": "Ports: 7000, 7001, 7199, 9042, 9160\n7000: Cassandra inter-node cluster communication\n7001: Cassandra SSL inter-node\n7199: JMX monitoring\n9042: CQL native transport (client connections)\n9160: Thrift client API (legacy)",
      "answerType": "explanation",
      "source": "app-common/apps-provider/addons/charts/state-cluster/cassandra/templates/"
    }
  ]
}