#!/bin/bash # Integration Test Script for dd0c Stack # Goes beyond smoke tests: exercises actual product flows (CRUD, webhooks, etc.) set -euo pipefail RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' NC='\033[0m' BASE_URL="${BASE_URL:-192.168.86.11}" PROTOCOL="${PROTOCOL:-http}" TS=$(date +%s) TOTAL=0 PASSED=0 check() { local name=$1 status=$2 ((TOTAL++)) || true if [ "$status" = "pass" ]; then echo -e " ${GREEN}✓${NC} $name" >&2 ((PASSED++)) || true else echo -e " ${RED}✗${NC} $name" >&2 fi } url() { echo "${PROTOCOL}://${BASE_URL}:$1$2"; } auth() { local port=$1 local resp resp=$(curl -s -X POST "$(url "$port" /api/v1/auth/signup)" \ -H "Content-Type: application/json" \ -d "{\"email\":\"integ-${TS}-${port}@dd0c.dev\",\"password\":\"integtest123!\",\"tenant_name\":\"Integ ${TS} ${port}\"}") echo "$resp" | grep -o '"token":"[^"]*' | cut -d'"' -f4 } api() { local method=$1 port=$2 path=$3 token=$4 shift 4 curl -s -X "$method" "$(url "$port" "$path")" \ -H "Authorization: Bearer $token" \ -H "Content-Type: application/json" \ "$@" 2>/dev/null } echo -e "${YELLOW}dd0c Integration Tests — $(date -u '+%Y-%m-%d %H:%M UTC')${NC}" echo -e "Target: ${PROTOCOL}://${BASE_URL}\n" # ─── P3 Alert Intelligence (:3003) ─── echo -e "${YELLOW}P3 Alert Intelligence (:3003)${NC}" ALERT_TOKEN=$(auth 3003) # Summary endpoint resp=$(api GET 3003 /api/v1/summary "$ALERT_TOKEN") echo "$resp" | grep -q "open_total" && check "Summary endpoint" "pass" || check "Summary endpoint" "fail" # Notification config resp=$(api PUT 3003 /api/v1/notifications/slack "$ALERT_TOKEN" \ -d '{"enabled":true,"config":{"webhook_url":"https://hooks.slack.com/test"},"min_severity":"high"}') code=$(curl -s -o /dev/null -w "%{http_code}" -X PUT "$(url 3003 /api/v1/notifications/slack)" \ -H "Authorization: Bearer $ALERT_TOKEN" -H "Content-Type: application/json" \ -d '{"enabled":true,"config":{"webhook_url":"https://hooks.slack.com/test"},"min_severity":"high"}') [ "$code" = "200" ] || [ "$code" = "201" ] && check "Set notification config" "pass" || check "Set notification config (HTTP $code)" "fail" # List notifications resp=$(api GET 3003 /api/v1/notifications "$ALERT_TOKEN") echo "$resp" | grep -q "slack" && check "List notifications" "pass" || check "List notifications" "fail" # ─── P4 Portal / Service Catalog (:3004) ─── echo -e "\n${YELLOW}P4 Portal / Service Catalog (:3004)${NC}" PORTAL_TOKEN=$(auth 3004) # Create a service resp=$(api PUT 3004 /api/v1/services "$PORTAL_TOKEN" \ -d '{"name":"payment-api","description":"Handles payments","owner":"platform-team","tier":"critical","language":"typescript","repo_url":"https://github.com/dd0c/payment-api","tags":{"env":"prod"}}') echo "$resp" | grep -q "payment-api" && check "Create service" "pass" || check "Create service" "fail" # List services resp=$(api GET 3004 /api/v1/services "$PORTAL_TOKEN") echo "$resp" | grep -q "payment-api" && check "List services (has created)" "pass" || check "List services" "fail" # Ownership report code=$(curl -s -o /dev/null -w "%{http_code}" "$(url 3004 /api/v1/ownership)" \ -H "Authorization: Bearer $PORTAL_TOKEN") [ "$code" = "200" ] && check "Ownership report" "pass" || check "Ownership report (HTTP $code)" "fail" # Search code=$(curl -s -o /dev/null -w "%{http_code}" "$(url 3004 '/api/v1/search?q=payment')" \ -H "Authorization: Bearer $PORTAL_TOKEN") [ "$code" = "200" ] && check "Search services" "pass" || check "Search services (HTTP $code)" "fail" # ─── P5 Cost Anomaly (:3007) ─── echo -e "\n${YELLOW}P5 Cost Anomaly (:3007)${NC}" COST_TOKEN=$(auth 3007) # Ingest cost data resp=$(api POST 3007 /api/v1/ingest "$COST_TOKEN" \ -d '{"events":[{"account_id":"123456789012","resource_type":"ec2","region":"us-east-1","hourly_cost":42.50,"currency":"USD","timestamp":"2026-03-01T00:00:00Z"}]}') code=$(curl -s -o /dev/null -w "%{http_code}" -X POST "$(url 3007 /api/v1/ingest)" \ -H "Authorization: Bearer $COST_TOKEN" -H "Content-Type: application/json" \ -d '{"events":[{"account_id":"123456789012","resource_type":"ec2","region":"us-east-1","hourly_cost":42.50,"currency":"USD","timestamp":"2026-03-01T00:00:00Z"}]}') [ "$code" = "200" ] || [ "$code" = "201" ] && check "Ingest cost data" "pass" || check "Ingest cost data (HTTP $code)" "fail" # Dashboard code=$(curl -s -o /dev/null -w "%{http_code}" "$(url 3007 /api/v1/dashboard)" \ -H "Authorization: Bearer $COST_TOKEN") [ "$code" = "200" ] && check "Dashboard" "pass" || check "Dashboard (HTTP $code)" "fail" # Governance code=$(curl -s -o /dev/null -w "%{http_code}" "$(url 3007 /api/v1/governance)" \ -H "Authorization: Bearer $COST_TOKEN") [ "$code" = "200" ] && check "Governance rules" "pass" || check "Governance rules (HTTP $code)" "fail" # ─── P6 Runbook Automation (:3006) ─── echo -e "\n${YELLOW}P6 Runbook Automation (:3006)${NC}" RUN_TOKEN=$(auth 3006) # Create a runbook resp=$(api POST 3006 /api/v1/runbooks "$RUN_TOKEN" \ -d '{"name":"restart-nginx","description":"Restart nginx on web servers","yaml_content":"steps:\n - name: restart\n command: systemctl restart nginx\n requires_approval: true"}') echo "$resp" | grep -q "restart-nginx\|id" && check "Create runbook" "pass" || check "Create runbook" "fail" # List runbooks resp=$(api GET 3006 /api/v1/runbooks "$RUN_TOKEN") echo "$resp" | grep -q "restart-nginx" && check "List runbooks (has created)" "pass" || check "List runbooks" "fail" # List approvals code=$(curl -s -o /dev/null -w "%{http_code}" "$(url 3006 /api/v1/approvals)" \ -H "Authorization: Bearer $RUN_TOKEN") [ "$code" = "200" ] && check "List approvals" "pass" || check "List approvals (HTTP $code)" "fail" # ─── P2 Drift Detection (:3002) ─── echo -e "\n${YELLOW}P2 Drift Detection (:3002)${NC}" DRIFT_TOKEN=$(auth 3002) # Dashboard code=$(curl -s -o /dev/null -w "%{http_code}" "$(url 3002 /api/v1/dashboard)" \ -H "Authorization: Bearer $DRIFT_TOKEN") [ "$code" = "200" ] && check "Dashboard" "pass" || check "Dashboard (HTTP $code)" "fail" # List stacks (empty) resp=$(api GET 3002 /api/v1/stacks "$DRIFT_TOKEN") echo "$resp" | grep -q "\[\]" && check "List stacks (empty)" "pass" || check "List stacks" "fail" # ─── P3 Webhook Secrets Management ─── echo -e "\n${YELLOW}P3 Webhook Secrets Management (:3003)${NC}" resp=$(api PUT 3003 /api/v1/webhooks/secrets "$ALERT_TOKEN" \ -d '{"provider":"datadog","secret":"whsec_test_datadog_secret_123"}') echo "$resp" | grep -q "success" && check "Register Datadog webhook secret" "pass" || check "Register Datadog webhook secret" "fail" resp=$(api GET 3003 /api/v1/webhooks/secrets "$ALERT_TOKEN") echo "$resp" | grep -q "datadog" && check "List webhook secrets" "pass" || check "List webhook secrets" "fail" code=$(curl -s -o /dev/null -w "%{http_code}" -X DELETE "$(url 3003 /api/v1/webhooks/secrets/datadog)" \ -H "Authorization: Bearer $ALERT_TOKEN") [ "$code" = "200" ] && check "Delete webhook secret" "pass" || check "Delete webhook secret (HTTP $code)" "fail" # ─── P2 Drift Report Submission ─── echo -e "\n${YELLOW}P2 Drift Report Submission (:3002)${NC}" resp=$(api POST 3002 /api/v1/reports "$DRIFT_TOKEN" \ -d '{"stack_name":"vpc-prod","stack_fingerprint":"abc123","state_serial":42,"total_resources":15,"drift_score":12.5,"raw_report":{"drifted_resources":["aws_security_group.main"]}}') echo "$resp" | grep -q "vpc-prod" && check "Submit drift report" "pass" || check "Submit drift report" "fail" resp=$(api GET 3002 /api/v1/stacks "$DRIFT_TOKEN") echo "$resp" | grep -q "vpc-prod" && check "List stacks (has submitted)" "pass" || check "List stacks (has submitted)" "fail" resp=$(api GET 3002 "/api/v1/stacks/vpc-prod/history" "$DRIFT_TOKEN") echo "$resp" | grep -q "drift_score" && check "Stack drift history" "pass" || check "Stack drift history" "fail" code=$(curl -s -o /dev/null -w "%{http_code}" -X DELETE "$(url 3002 /api/v1/stacks/vpc-prod)" \ -H "Authorization: Bearer $DRIFT_TOKEN") [ "$code" = "200" ] && check "Delete stack" "pass" || check "Delete stack (HTTP $code)" "fail" # ─── P6 Runbook Execution ─── echo -e "\n${YELLOW}P6 Runbook Execution (:3006)${NC}" # Get the runbook ID we created earlier RUNBOOK_ID=$(api GET 3006 /api/v1/runbooks "$RUN_TOKEN" | grep -o '"id":"[^"]*' | head -1 | cut -d'"' -f4) if [ -n "$RUNBOOK_ID" ]; then resp=$(api POST 3006 "/api/v1/runbooks/$RUNBOOK_ID/execute" "$RUN_TOKEN" \ -d '{"dry_run":true,"variables":{"target":"web-01"}}') echo "$resp" | grep -q "pending\|execution_id" && check "Execute runbook (dry run)" "pass" || check "Execute runbook (dry run)" "fail" EXEC_ID=$(echo "$resp" | grep -o '"execution_id":"[^"]*' | cut -d'"' -f4) if [ -n "$EXEC_ID" ]; then resp=$(api GET 3006 "/api/v1/executions/$EXEC_ID" "$RUN_TOKEN") echo "$resp" | grep -q "pending\|dry_run" && check "Get execution status" "pass" || check "Get execution status" "fail" else check "Get execution status (no exec ID)" "fail" fi resp=$(api GET 3006 "/api/v1/runbooks/$RUNBOOK_ID/executions" "$RUN_TOKEN") echo "$resp" | grep -q "executions\|$EXEC_ID" && check "List runbook executions" "pass" || check "List runbook executions" "fail" else check "Execute runbook (no runbook found)" "fail" check "Get execution status (skipped)" "fail" check "List runbook executions (skipped)" "fail" fi # ─── Cross-service: API Key generation ─── echo -e "\n${YELLOW}Cross-service: API Key Auth${NC}" resp=$(api POST 3003 /api/v1/auth/api-keys "$ALERT_TOKEN" -d "{}") API_KEY=$(echo "$resp" | grep -o '"api_key":"[^"]*' | cut -d'"' -f4) if [ -n "$API_KEY" ]; then check "Generate API key" "pass" # Use API key to hit an endpoint code=$(curl -s -o /dev/null -w "%{http_code}" "$(url 3003 /api/v1/incidents)" \ -H "X-API-Key: $API_KEY") [ "$code" = "200" ] && check "Auth via API key" "pass" || check "Auth via API key (HTTP $code)" "fail" else check "Generate API key" "fail" check "Auth via API key (skipped)" "fail" fi # Summary echo "" echo -e "${YELLOW}═══════════════════════════${NC}" if [ "$PASSED" -eq "$TOTAL" ]; then echo -e "${GREEN}ALL PASSED: $PASSED/$TOTAL${NC}" exit 0 else echo -e "${RED}$PASSED/$TOTAL passed ($((TOTAL - PASSED)) failed)${NC}" exit 1 fi