{
  "schema": "epok.setup-manifest/1",
  "id": "aws-ecs-otel-traces",
  "version": "1.0.0",
  "title": "AWS ECS on Fargate traces -> Epok with an OpenTelemetry Collector sidecar",
  "summary": "Registers a new task definition revision of an existing ECS service that adds an OpenTelemetry Collector sidecar (OTLP in on localhost:4318, OTLP/HTTP out to Epok with x-api-key from an SSM SecureString) and points the app container's OTel SDK at it, then rolls the service. Uninstall rolls back to the previous revision.",
  "provider": "aws",
  "signal": "traces",
  "source": "otlp_traces_ecs",
  "tool": "aws",
  "estimated_minutes": 8,
  "last_verified": null,
  "prerequisites": {
    "cli": [
      {
        "name": "aws",
        "min_version": "2.15.0",
        "check": "aws --version",
        "version_regex": "aws-cli/([0-9]+\\.[0-9]+\\.[0-9]+)"
      },
      {
        "name": "python3",
        "min_version": "3.8.0",
        "check": "python3 --version",
        "version_regex": "Python ([0-9]+\\.[0-9]+\\.[0-9]+)"
      },
      {
        "name": "curl",
        "min_version": "7.68.0",
        "check": "curl --version",
        "version_regex": "^curl ([0-9]+\\.[0-9]+\\.[0-9]+)"
      }
    ],
    "permissions": [
      "ecs:DescribeServices",
      "ecs:DescribeTaskDefinition",
      "ecs:RegisterTaskDefinition",
      "ecs:UpdateService",
      "ecs:DeregisterTaskDefinition",
      "ecs:TagResource",
      "ssm:PutParameter",
      "ssm:GetParameter",
      "ssm:AddTagsToResource",
      "ssm:DeleteParameter",
      "(s3 store) s3:PutObject, s3:GetObject, s3:DeleteObject, s3:GetBucketPublicAccessBlock",
      "iam:PutRolePolicy",
      "iam:DeleteRolePolicy",
      "iam:PassRole"
    ],
    "permissions_note": "iam:PassRole on the task's execution and task roles (needed to register a revision). The app must already use an OpenTelemetry SDK that reads the standard OTEL_* variables (examples/otel-traces/service shows the minimum); this setup does not change application code.",
    "check_commands": [
      "aws sts get-caller-identity --query Account --output text >/dev/null",
      "aws ecs describe-services --region \"$AWS_REGION\" --cluster \"$ECS_CLUSTER\" --services \"$ECS_SERVICE\" --query 'services[?status==`ACTIVE`].serviceName' --output text | grep -q ."
    ]
  },
  "inputs": [
    {
      "name": "AWS_REGION",
      "required": true,
      "actor": "agent",
      "pattern": "^[a-z]{2}(-[a-z]+)+-[0-9]$",
      "description": "Region of the cluster."
    },
    {
      "name": "ECS_CLUSTER",
      "required": true,
      "actor": "agent",
      "pattern": "^[A-Za-z0-9_-]{1,255}$",
      "description": "ECS cluster name.",
      "how_to_get": "aws ecs list-clusters"
    },
    {
      "name": "ECS_SERVICE",
      "required": true,
      "actor": "agent",
      "pattern": "^[A-Za-z0-9_-]{1,255}$",
      "description": "ECS service (Fargate or awsvpc network mode) to instrument.",
      "how_to_get": "aws ecs list-services --cluster $ECS_CLUSTER (confirm with the human)"
    },
    {
      "name": "APP_CONTAINER",
      "required": true,
      "actor": "agent",
      "pattern": "^[A-Za-z0-9_-]{1,255}$",
      "description": "Name of the application container in the task definition (it gets the OTEL_* variables)."
    },
    {
      "name": "OTEL_SERVICE_NAME",
      "required": true,
      "actor": "agent",
      "pattern": "^[A-Za-z0-9._-]{1,128}$",
      "description": "service.name for the spans (how the service appears in Epok).",
      "how_to_get": "Ask the human, or use the workload name."
    },
    {
      "name": "EPOK_KEY_PARAMETER",
      "default": "/epok/ingest-api-key",
      "actor": "agent",
      "pattern": "^/[A-Za-z0-9_./-]{1,200}$",
      "description": "SSM Parameter Store name for the ingest key (SecureString, default aws/ssm KMS key; free standard tier). The sidecar reads it through the task execution role. An existing parameter is reused, not overwritten."
    },
    {
      "name": "EPOK_KEY_STORE",
      "default": "ssm",
      "actor": "agent",
      "pattern": "^(ssm|s3)$",
      "description": "Where the sidecar's key lives. ssm (default, recommended): SSM Parameter Store SecureString EPOK_KEY_PARAMETER, injected with secrets/valueFrom. s3: an ECS environment file (EPOK_API_KEY=<key>) at EPOK_KEY_S3_URI in a private bucket with SSE-S3, injected with environmentFiles; use it only where SSM and Secrets Manager are not available to you."
    },
    {
      "name": "EPOK_KEY_S3_URI",
      "required": false,
      "default": "",
      "actor": "agent",
      "pattern": "^(s3://[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]/[A-Za-z0-9_./-]{1,200}[.]env)?$",
      "description": "Only for EPOK_KEY_STORE=s3: s3://<existing private bucket>/<path>.env. The bucket must block all public access.",
      "how_to_get": "Ask the human which private bucket holds service configuration."
    },
    {
      "name": "EPOK_COLLECTOR_IMAGE",
      "default": "ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector:0.163.2",
      "actor": "agent",
      "pattern": "^[a-z0-9.-]+(/[A-Za-z0-9._-]+)+:[A-Za-z0-9._-]+$",
      "description": "Collector image for the sidecar (core distribution: otlp receiver + otlphttp exporter; multi-arch). Mirror it into ECR if tasks cannot reach ghcr.io."
    },
    {
      "name": "EPOK_RESOURCE_TAGS",
      "default": "managed-by=epok",
      "actor": "agent",
      "pattern": "^[A-Za-z0-9_.:/=+@-]+=[A-Za-z0-9_.:/=+@ -]*(,[A-Za-z0-9_.:/=+@-]+=[A-Za-z0-9_.:/=+@ -]*){0,9}$",
      "description": "Comma-separated key=value tags for the SSM parameter."
    },
    {
      "name": "APP_URL",
      "required": false,
      "default": "",
      "actor": "agent",
      "pattern": "^(https?://[A-Za-z0-9.:-]+)?$",
      "description": "Base URL that reaches the service (load balancer DNS name or a task address), no trailing slash. Needed only by verify.",
      "how_to_get": "aws elbv2 describe-load-balancers --query 'LoadBalancers[].DNSName' (match the service's target group), or ask the human."
    },
    {
      "name": "EPOK_VERIFY_PATH",
      "default": "/",
      "actor": "agent",
      "pattern": "^/[A-Za-z0-9_./?=&-]*$",
      "description": "Path verify requests. Pick one your instrumented HTTP server handles (the example app traces /work)."
    },
    {
      "name": "EPOK_API_KEY",
      "required": true,
      "secret": true,
      "scope": "ingest",
      "actor": "human",
      "pattern": "^[A-Za-z0-9_.-]{8,200}$",
      "description": "Epok ingest-scope API key, dedicated to this workload.",
      "how_to_get": "A human creates it: Epok -> Settings -> API Keys -> New key (scope: ingest, name: traces-<workload>). Agents must ask for it, never create or guess it, and never write it to a file or a commit."
    },
    {
      "name": "EPOK_READ_API_KEY",
      "required": true,
      "secret": true,
      "scope": "read",
      "actor": "human",
      "pattern": "^[A-Za-z0-9_.-]{8,200}$",
      "description": "Epok API key with read scope, used only by verify to look the canary trace up. May be the same key as EPOK_API_KEY if that key also has read scope.",
      "how_to_get": "A human creates it: Epok -> Settings -> API Keys (scope: read). Agents must ask for it."
    },
    {
      "name": "EPOK_INGEST_HOST",
      "default": "https://ingest.epok.cloud",
      "actor": "agent",
      "pattern": "^(https://[A-Za-z0-9.-]+|http://(127\\.0\\.0\\.1|localhost))(:[0-9]{1,5})?$",
      "description": "Epok OTLP/HTTP ingest base URL, no trailing slash. The exporter appends /v1/traces. Change only if Epok support tells you to."
    },
    {
      "name": "EPOK_API_HOST",
      "default": "https://app.epok.cloud",
      "actor": "agent",
      "pattern": "^(https://[A-Za-z0-9.-]+|http://(127\\.0\\.0\\.1|localhost))(:[0-9]{1,5})?$",
      "description": "Epok API base URL used by verify, no trailing slash."
    },
    {
      "name": "EPOK_TENANT_ID",
      "required": true,
      "actor": "agent",
      "pattern": "^[0-9]{1,10}$",
      "description": "Your Epok tenant (account) id, used in the verify API path.",
      "how_to_get": "Epok -> Settings -> Account (the numeric account id), or ask the human."
    },
    {
      "name": "NONCE",
      "generate": "nonce",
      "actor": "agent",
      "pattern": "^[0-9a-f]{24}$",
      "description": "Canary token: verify sends one request with W3C traceparent trace id 00000000<NONCE> and looks that trace up in Epok. Generated by the runner if not set (24 lowercase hex)."
    }
  ],
  "cost_notes": "No new compute: the sidecar runs inside the existing tasks (memoryReservation 64 MiB, typically <0.05 vCPU; the task's cpu/memory must have that headroom). One SSM standard SecureString parameter (free). Epok bills the decoded OTLP bytes; the collector sends gzip and Epok meters after decompression. About 1-2 KB per 3-span trace.",
  "notes": [
    "Fargate and other awsvpc tasks share localhost between containers, so the app exports to http://localhost:4318 and only the sidecar holds the key.",
    "The sidecar is essential=false: if it crashes the app keeps serving, spans are lost (and the collector logs say why).",
    "The new revision is tagged epok-previous=<old revision ARN>; uninstall uses that tag to roll back.",
    "AWS Distro for OpenTelemetry (public.ecr.aws/aws-observability/aws-otel-collector) works the same way if your platform team standardises on it; set EPOK_COLLECTOR_IMAGE and keep the config."
  ],
  "steps": [
    {
      "id": "inspect-service",
      "actor": "agent",
      "destructive": false,
      "idempotent": true,
      "description": "Check the service is ACTIVE, uses awsvpc, contains APP_CONTAINER and has an execution role. Prints EPOK_TASK_FAMILY, EPOK_EXECUTION_ROLE_NAME and EPOK_ACCOUNT_ID.",
      "run": "python3 -c '\nimport json, os, subprocess, sys\ne = os.environ\ndef aws(*a):\n    return json.loads(subprocess.check_output([\"aws\", *a, \"--region\", e[\"AWS_REGION\"], \"--output\", \"json\"]))\ns = aws(\"ecs\", \"describe-services\", \"--cluster\", e[\"ECS_CLUSTER\"], \"--services\", e[\"ECS_SERVICE\"])[\"services\"]\nif not s or s[0][\"status\"] != \"ACTIVE\":\n    sys.exit(\"service \" + e[\"ECS_SERVICE\"] + \" not found or not ACTIVE in cluster \" + e[\"ECS_CLUSTER\"])\ntd = aws(\"ecs\", \"describe-task-definition\", \"--task-definition\", s[0][\"taskDefinition\"])[\"taskDefinition\"]\nif td.get(\"networkMode\") != \"awsvpc\":\n    sys.exit(\"network mode \" + str(td.get(\"networkMode\")) + \": the sidecar needs awsvpc (Fargate) so localhost is shared\")\nnames = [c[\"name\"] for c in td[\"containerDefinitions\"]]\nif e[\"APP_CONTAINER\"] not in names:\n    sys.exit(\"container \" + e[\"APP_CONTAINER\"] + \" not in task definition; containers: \" + \", \".join(names))\nrole = td.get(\"executionRoleArn\")\nif not role:\n    sys.exit(\"task definition has no executionRoleArn; ECS needs one to read the key parameter\")\nprint(\"task definition \" + td[\"taskDefinitionArn\"] + \", containers \" + \", \".join(names))\nprint(\"EPOK_TASK_FAMILY=\" + td[\"family\"])\nprint(\"EPOK_EXECUTION_ROLE_NAME=\" + role.split(\"/\")[-1])\nprint(\"EPOK_ACCOUNT_ID=\" + role.split(\":\")[4])'",
      "exports": [
        "EPOK_TASK_FAMILY",
        "EPOK_EXECUTION_ROLE_NAME",
        "EPOK_ACCOUNT_ID"
      ]
    },
    {
      "id": "store-key",
      "actor": "agent",
      "destructive": false,
      "idempotent_check": "B=\"${EPOK_KEY_S3_URI#s3://}\"; BUCKET=\"${B%%/*}\"; OBJ=\"${B#*/}\"\ncase \"$EPOK_KEY_STORE\" in\n  ssm) aws ssm get-parameter --name \"$EPOK_KEY_PARAMETER\" --region \"$AWS_REGION\" --query Parameter.Name --output text >/dev/null 2>&1 ;;\n  s3) aws s3api head-object --bucket \"$BUCKET\" --key \"$OBJ\" --region \"$AWS_REGION\" >/dev/null 2>&1 ;;\nesac",
      "on_exists": "skip",
      "description": "Store the ingest key (ssm: SecureString via JSON on stdin, never on the command line; s3: an env file streamed from stdin with SSE-S3, refused unless the bucket blocks public access). Skipped if the parameter/object exists.",
      "run": "# AWS CLI v2 (2.17.0 to 2.40.3 tested) rejects --cli-input-json read from a pipe (\"Invalid JSON received\"). The key goes in through a plain parameter file on stdin instead; the other arguments are not secret.\nB=\"${EPOK_KEY_S3_URI#s3://}\"; BUCKET=\"${B%%/*}\"; OBJ=\"${B#*/}\"\ncase \"$EPOK_KEY_STORE\" in\n  ssm)\n    python3 -c '\nimport json, os, subprocess, sys\ne = os.environ\ntags = [dict(zip((\"Key\", \"Value\"), p.split(\"=\", 1))) for p in e[\"EPOK_RESOURCE_TAGS\"].split(\",\")]\nsys.exit(subprocess.run([\"aws\", \"ssm\", \"put-parameter\", \"--region\", e[\"AWS_REGION\"], \"--name\", e[\"EPOK_KEY_PARAMETER\"],\n                         \"--type\", \"SecureString\", \"--tags\", json.dumps(tags), \"--value\", \"file:///dev/stdin\",\n                         \"--query\", \"Version\", \"--output\", \"text\"], input=e[\"EPOK_API_KEY\"], text=True).returncode)\n'\n    echo \"stored the key in SSM SecureString $EPOK_KEY_PARAMETER (default aws/ssm KMS key)\" ;;\n  s3)\n    test -n \"$OBJ\" -a \"$OBJ\" != \"$B\" || { echo \"set EPOK_KEY_S3_URI=s3://<bucket>/<path>.env\" >&2; exit 1; }\n    aws s3api get-public-access-block --bucket \"$BUCKET\" --query 'PublicAccessBlockConfiguration.[BlockPublicAcls,BlockPublicPolicy,IgnorePublicAcls,RestrictPublicBuckets]' --output text | grep -qv False \\\n      || { echo \"bucket $BUCKET does not block public access; stop and ask\" >&2; exit 1; }\n    python3 -c 'import os; print(\"EPOK_API_KEY=\" + os.environ[\"EPOK_API_KEY\"])' | aws s3 cp - \"$EPOK_KEY_S3_URI\" --sse AES256 --only-show-errors\n    echo \"stored the key as an ECS environment file at $EPOK_KEY_S3_URI (SSE-S3)\" ;;\n  *) echo \"EPOK_KEY_STORE must be ssm or s3\" >&2; exit 1 ;;\nesac"
    },
    {
      "id": "grant-execution-role",
      "actor": "agent",
      "destructive": false,
      "idempotent": true,
      "description": "Allow the task execution role to read exactly that parameter or object (inline policy epok-read-ingest-key).",
      "run": "B=\"${EPOK_KEY_S3_URI#s3://}\"; BUCKET=\"${B%%/*}\"; OBJ=\"${B#*/}\"\ncase \"$EPOK_KEY_STORE\" in\n  ssm) DOC=\"{\\\"Version\\\":\\\"2012-10-17\\\",\\\"Statement\\\":[{\\\"Effect\\\":\\\"Allow\\\",\\\"Action\\\":\\\"ssm:GetParameters\\\",\\\"Resource\\\":\\\"arn:aws:ssm:$AWS_REGION:$EPOK_ACCOUNT_ID:parameter$EPOK_KEY_PARAMETER\\\"}]}\"\n       WHAT=\"$EPOK_KEY_PARAMETER\" ;;\n  s3) DOC=\"{\\\"Version\\\":\\\"2012-10-17\\\",\\\"Statement\\\":[{\\\"Effect\\\":\\\"Allow\\\",\\\"Action\\\":\\\"s3:GetObject\\\",\\\"Resource\\\":\\\"arn:aws:s3:::$BUCKET/$OBJ\\\"},{\\\"Effect\\\":\\\"Allow\\\",\\\"Action\\\":\\\"s3:GetBucketLocation\\\",\\\"Resource\\\":\\\"arn:aws:s3:::$BUCKET\\\"}]}\"\n      WHAT=\"$EPOK_KEY_S3_URI\" ;;\nesac\naws iam put-role-policy --role-name \"$EPOK_EXECUTION_ROLE_NAME\" --policy-name epok-read-ingest-key --policy-document \"$DOC\"\necho \"execution role $EPOK_EXECUTION_ROLE_NAME may read $WHAT\" "
    },
    {
      "id": "register-revision",
      "actor": "agent",
      "destructive": false,
      "idempotent": true,
      "description": "Register a new revision: the current one plus the epok-otel-collector sidecar, plus OTEL_SERVICE_NAME / OTEL_TRACES_EXPORTER / OTEL_EXPORTER_OTLP_PROTOCOL / OTEL_EXPORTER_OTLP_ENDPOINT on APP_CONTAINER. Re-running replaces (does not duplicate) the sidecar. Prints EPOK_TASK_DEF_ARN.",
      "run": "python3 -c '\nimport json, os, subprocess, sys\ne = os.environ\ndef aws(*a):\n    out = subprocess.run([\"aws\", *a, \"--region\", e[\"AWS_REGION\"], \"--output\", \"json\"],\n                         capture_output=True, text=True)\n    if out.returncode != 0:\n        sys.exit(out.stderr)\n    return json.loads(out.stdout)\nsvc = aws(\"ecs\", \"describe-services\", \"--cluster\", e[\"ECS_CLUSTER\"], \"--services\", e[\"ECS_SERVICE\"])[\"services\"][0]\ncur = aws(\"ecs\", \"describe-task-definition\", \"--task-definition\", svc[\"taskDefinition\"], \"--include\", \"TAGS\")\ntd, tags = cur[\"taskDefinition\"], cur.get(\"tags\") or []\nprevious = next((t[\"value\"] for t in tags if t[\"key\"] == \"epok-previous\"), td[\"taskDefinitionArn\"])\nfor k in (\"taskDefinitionArn\", \"revision\", \"status\", \"requiresAttributes\", \"compatibilities\",\n          \"registeredAt\", \"registeredBy\", \"deregisteredAt\"):\n    td.pop(k, None)\nSIDECAR = \"epok-otel-collector\"\nours = {\"OTEL_SERVICE_NAME\", \"OTEL_TRACES_EXPORTER\", \"OTEL_EXPORTER_OTLP_PROTOCOL\", \"OTEL_EXPORTER_OTLP_ENDPOINT\"}\ncfg = \"receivers:\\n  otlp:\\n    protocols:\\n      http:\\n        endpoint: localhost:4318\\nexporters:\\n  otlphttp/epok:\\n    endpoint: \\x24{env:EPOK_INGEST_HOST}\\n    headers:\\n      x-api-key: \\x24{env:EPOK_API_KEY}\\nservice:\\n  pipelines:\\n    traces:\\n      receivers: [otlp]\\n      exporters: [otlphttp/epok]\\n\"\nparam = \"arn:aws:ssm:\" + e[\"AWS_REGION\"] + \":\" + e[\"EPOK_ACCOUNT_ID\"] + \":parameter\" + e[\"EPOK_KEY_PARAMETER\"]\ncontainers = [c for c in td[\"containerDefinitions\"] if c[\"name\"] != SIDECAR]\napp_log = None\nfor c in containers:\n    if c[\"name\"] == e[\"APP_CONTAINER\"]:\n        c[\"environment\"] = [v for v in c.get(\"environment\", []) if v[\"name\"] not in ours] + [\n            {\"name\": \"OTEL_SERVICE_NAME\", \"value\": e[\"OTEL_SERVICE_NAME\"]},\n            {\"name\": \"OTEL_TRACES_EXPORTER\", \"value\": \"otlp\"},\n            {\"name\": \"OTEL_EXPORTER_OTLP_PROTOCOL\", \"value\": \"http/protobuf\"},\n            {\"name\": \"OTEL_EXPORTER_OTLP_ENDPOINT\", \"value\": \"http://localhost:4318\"}]\n        deps = [d for d in c.get(\"dependsOn\", []) if d[\"containerName\"] != SIDECAR]\n        c[\"dependsOn\"] = deps + [{\"containerName\": SIDECAR, \"condition\": \"START\"}]\n        app_log = c.get(\"logConfiguration\")\nsidecar = {\"name\": SIDECAR, \"image\": e[\"EPOK_COLLECTOR_IMAGE\"], \"essential\": False,\n           \"command\": [\"--config=env:EPOK_OTEL_COLLECTOR_CONFIG\"], \"memoryReservation\": 64,\n           \"environment\": [{\"name\": \"EPOK_OTEL_COLLECTOR_CONFIG\", \"value\": cfg},\n                           {\"name\": \"EPOK_INGEST_HOST\", \"value\": e[\"EPOK_INGEST_HOST\"]}],\n           }\nif e[\"EPOK_KEY_STORE\"] == \"s3\":\n    sidecar[\"environmentFiles\"] = [{\"type\": \"s3\", \"value\": \"arn:aws:s3:::\" + e[\"EPOK_KEY_S3_URI\"][len(\"s3://\"):]}]\nelse:\n    sidecar[\"secrets\"] = [{\"name\": \"EPOK_API_KEY\", \"valueFrom\": param}]\nif app_log and app_log.get(\"logDriver\") == \"awslogs\":\n    opts = dict(app_log.get(\"options\", {}))\n    opts[\"awslogs-stream-prefix\"] = SIDECAR\n    sidecar[\"logConfiguration\"] = {\"logDriver\": \"awslogs\", \"options\": opts}\ntd[\"containerDefinitions\"] = containers + [sidecar]\ntags = [t for t in tags if t[\"key\"] != \"epok-previous\"] + [{\"key\": \"epok-previous\", \"value\": previous}]\nif tags:\n    td[\"tags\"] = tags\nnew = aws(\"ecs\", \"register-task-definition\", \"--cli-input-json\", json.dumps(td))  # no secret inside: only references\nprint(\"previous revision \" + previous)\nprint(\"EPOK_TASK_DEF_ARN=\" + new[\"taskDefinition\"][\"taskDefinitionArn\"])'",
      "exports": [
        "EPOK_TASK_DEF_ARN"
      ]
    },
    {
      "id": "deploy-revision",
      "actor": "agent",
      "destructive": false,
      "idempotent": true,
      "description": "Roll the service to the new revision and wait until it is stable (normal ECS rolling deployment).",
      "run": "aws ecs update-service --region \"$AWS_REGION\" --cluster \"$ECS_CLUSTER\" --service \"$ECS_SERVICE\" \\\n  --task-definition \"$EPOK_TASK_DEF_ARN\" --query service.taskDefinition --output text\naws ecs wait services-stable --region \"$AWS_REGION\" --cluster \"$ECS_CLUSTER\" --services \"$ECS_SERVICE\"\necho \"service $ECS_SERVICE runs $EPOK_TASK_DEF_ARN\" ",
      "timeout_seconds": 1500
    }
  ],
  "verify": {
    "emit": "if [ -z \"${APP_URL:-}\" ]; then echo \"set APP_URL to the service URL (load balancer or task address)\" >&2; exit 1; fi\ncurl -fsS -o /dev/null -w \"HTTP %{http_code}\\n\" -H \"traceparent: 00-00000000$NONCE-00f067aa0ba902b7-01\" \"${APP_URL}${EPOK_VERIFY_PATH}\"",
    "call": "printf 'header = \"X-API-Key: %s\"\\n' \"$EPOK_READ_API_KEY\" | curl -K - -fsS \\\n  \"$EPOK_API_HOST/api/v1/tenants/$EPOK_TENANT_ID/traces/00000000$NONCE\" | python3 -c '\nimport json, sys\nd = json.load(sys.stdin)\nspans = [e for es in d.get(\"services\", {}).values() for e in es\n         if e.get(\"fields\", {}).get(\"record_type\") == \"span\"]\nids = {e.get(\"span_id\") for e in spans}\nservices = sorted({e.get(\"service\") for e in spans})\nout = {\n    \"trace_id\": d.get(\"trace_id\"),\n    \"spans\": len(spans),\n    \"services\": services,\n    \"roots\": sum(1 for e in spans if e.get(\"parent_span_id\") not in ids),\n    \"children\": sum(1 for e in spans if e.get(\"parent_span_id\") in ids),\n    \"error_spans\": sum(1 for e in spans if e.get(\"fields\", {}).get(\"span_status\") == \"error\"),\n}\nout[\"ok\"] = True if spans and services == [sys.argv[1]] else None\nprint(json.dumps(out))' \"$OTEL_SERVICE_NAME\"",
    "success": {
      "field": "ok",
      "is": "non_null"
    },
    "timeout_seconds": 180,
    "poll_seconds": 10,
    "reemit_seconds": 60
  },
  "failure_modes": [
    {
      "id": "permission-denied",
      "step": [
        "inspect-service",
        "store-key",
        "grant-execution-role",
        "register-revision",
        "deploy-revision"
      ],
      "symptom": "AccessDenied / AccessDeniedException / not authorized to perform iam:PassRole.",
      "check": "aws sts get-caller-identity --query Arn --output text",
      "fix": "The caller lacks a permission in prerequisites.permissions. Stop and ask a human. Never grant IAM to yourself.",
      "stop_and_ask": true
    },
    {
      "id": "not-awsvpc",
      "step": "inspect-service",
      "symptom": "inspect-service exits with 'the sidecar needs awsvpc'.",
      "check": "aws ecs describe-task-definition --task-definition \"$EPOK_TASK_FAMILY\" --region \"$AWS_REGION\" --query taskDefinition.networkMode",
      "fix": "bridge/host-mode EC2 tasks need a daemon collector per instance instead of a sidecar. Stop and ask.",
      "stop_and_ask": true
    },
    {
      "id": "deployment-not-stable",
      "step": "deploy-revision",
      "symptom": "services-stable times out, or tasks stop with CannotPullContainerError / ResourceInitializationError.",
      "check": "aws ecs describe-services --region \"$AWS_REGION\" --cluster \"$ECS_CLUSTER\" --services \"$ECS_SERVICE\" --query 'services[0].events[:5].message'",
      "fix": "ResourceInitializationError ... environment file / s3: the execution role cannot read the S3 env file (re-run grant-execution-role; an SSE-KMS bucket also needs kms:Decrypt). CannotPullContainerError: tasks have no route to ghcr.io (private subnet without NAT): mirror the collector image to ECR and set EPOK_COLLECTOR_IMAGE. ResourceInitializationError ... ssm: the execution role cannot read the parameter (re-run grant-execution-role; a customer-managed KMS key also needs kms:Decrypt). Out of memory: raise the task memory. If unsure, run the uninstall to roll back and stop and ask.",
      "stop_and_ask": true
    },
    {
      "id": "collector-not-exporting",
      "step": "verify",
      "symptom": "Requests succeed but the trace never arrives.",
      "check": "G=\"$(aws ecs describe-task-definition --task-definition \"$EPOK_TASK_DEF_ARN\" --region \"$AWS_REGION\" --query 'taskDefinition.containerDefinitions[?name==`epok-otel-collector`].logConfiguration.options.\"awslogs-group\" | [0]' --output text)\"; aws logs tail \"$G\" --region \"$AWS_REGION\" --since 10m --log-stream-name-prefix epok-otel-collector | tail -30",
      "fix": "401/403 = key (see epok-auth-rejected). No collector log stream = the app container has no awslogs config to copy; check the task's stopped reason. 'connection refused' in the app = the app's SDK is not reading OTEL_EXPORTER_OTLP_ENDPOINT (the app is not instrumented: stop and ask).",
      "stop_and_ask": false
    },
    {
      "id": "epok-auth-rejected",
      "step": "verify",
      "symptom": "Spans never arrive; exporter or collector logs show HTTP 401 or 403 from the Epok ingest host.",
      "check": "Ask the human which key they gave you and its scope (do not print it).",
      "fix": "The ingest key is wrong, revoked, or not ingest scope. Stop and ask the human for a new ingest-scope key. Do not retry in a loop.",
      "stop_and_ask": true
    },
    {
      "id": "verify-read-auth",
      "step": "verify",
      "symptom": "The verify call fails with HTTP 401 or 403 on every poll.",
      "check": "Does EPOK_READ_API_KEY have read scope, and is EPOK_TENANT_ID its tenant?",
      "fix": "403 = the key lacks read scope or belongs to another tenant. Stop and ask the human.",
      "stop_and_ask": true
    },
    {
      "id": "trace-not-found",
      "step": "verify",
      "symptom": "The verify call keeps failing with HTTP 404 (Trace not found) until the timeout.",
      "check": "Re-read the emit output: did the request/invocation succeed? Then check the failure modes for this runtime.",
      "fix": "The canary trace did not reach Epok, or your code does not continue an incoming W3C traceparent (then spans arrive under another trace id: look for OTEL_SERVICE_NAME in Epok -> Traces). Report; do not change sampling or keys to make it pass.",
      "stop_and_ask": false
    },
    {
      "id": "wrong-service-name",
      "step": "verify",
      "symptom": "Verify finds the trace but ok is null and services is not [OTEL_SERVICE_NAME].",
      "check": "Look at services in the verify JSON.",
      "fix": "Something else sets service.name (OTEL_RESOURCE_ATTRIBUTES or code). Report it to the human; do not edit application code without asking.",
      "stop_and_ask": true
    }
  ],
  "uninstall": [
    {
      "id": "restore-previous-revision",
      "actor": "agent",
      "destructive": true,
      "description": "If the running revision carries the epok-previous tag, roll the service back to that revision, wait until stable, and deregister the Epok revision. Otherwise do nothing.",
      "run": "python3 -c '\nimport json, os, subprocess, sys\ne = os.environ\ndef aws(*a):\n    out = subprocess.run([\"aws\", *a, \"--region\", e[\"AWS_REGION\"], \"--output\", \"json\"], capture_output=True, text=True)\n    if out.returncode != 0:\n        if \"NotFound\" in out.stderr or \"ServiceNotActive\" in out.stderr:\n            return None\n        sys.exit(out.stderr)\n    return json.loads(out.stdout) if out.stdout.strip() else {}\ns = aws(\"ecs\", \"describe-services\", \"--cluster\", e[\"ECS_CLUSTER\"], \"--services\", e[\"ECS_SERVICE\"])\ns = [x for x in (s or {}).get(\"services\", []) if x[\"status\"] == \"ACTIVE\"]\nif not s:\n    print(\"service not found or inactive; nothing to restore\")\n    sys.exit(0)\ncur = aws(\"ecs\", \"describe-task-definition\", \"--task-definition\", s[0][\"taskDefinition\"], \"--include\", \"TAGS\")\nprev = next((t[\"value\"] for t in cur.get(\"tags\") or [] if t[\"key\"] == \"epok-previous\"), None)\nif not prev:\n    print(\"current revision \" + s[0][\"taskDefinition\"] + \" was not created by this setup; nothing to restore\")\n    sys.exit(0)\naws(\"ecs\", \"update-service\", \"--cluster\", e[\"ECS_CLUSTER\"], \"--service\", e[\"ECS_SERVICE\"], \"--task-definition\", prev)\nsubprocess.run([\"aws\", \"ecs\", \"wait\", \"services-stable\", \"--region\", e[\"AWS_REGION\"], \"--cluster\", e[\"ECS_CLUSTER\"],\n                \"--services\", e[\"ECS_SERVICE\"]], check=True)\naws(\"ecs\", \"deregister-task-definition\", \"--task-definition\", s[0][\"taskDefinition\"])\nprint(\"service back on \" + prev + \"; deregistered \" + s[0][\"taskDefinition\"])'",
      "timeout_seconds": 1500
    },
    {
      "id": "remove-role-policy",
      "actor": "agent",
      "destructive": true,
      "description": "Delete the inline policy epok-read-ingest-key from the execution role (already gone counts as done).",
      "run": "ROLE=\"$(python3 -c '\nimport json, os, subprocess\ne = os.environ\nr = subprocess.run([\"aws\", \"ecs\", \"describe-services\", \"--cluster\", e[\"ECS_CLUSTER\"], \"--services\", e[\"ECS_SERVICE\"],\n                    \"--region\", e[\"AWS_REGION\"], \"--output\", \"json\"], capture_output=True, text=True)\ns = [x for x in json.loads(r.stdout or \"{}\").get(\"services\", []) if x[\"status\"] == \"ACTIVE\"] if r.returncode == 0 else []\nif s:\n    td = json.loads(subprocess.check_output([\"aws\", \"ecs\", \"describe-task-definition\", \"--task-definition\",\n                    s[0][\"taskDefinition\"], \"--region\", e[\"AWS_REGION\"], \"--output\", \"json\"]))[\"taskDefinition\"]\n    print((td.get(\"executionRoleArn\") or \"\").split(\"/\")[-1])\n')\"\nROLE=\"${ROLE:-${EPOK_EXECUTION_ROLE_NAME:-}}\"\nif [ -z \"$ROLE\" ]; then echo \"execution role unknown (service gone); set EPOK_EXECUTION_ROLE_NAME to clean up\"; exit 0; fi\nif OUT=\"$(aws iam delete-role-policy --role-name \"$ROLE\" --policy-name epok-read-ingest-key 2>&1)\"; then\n  echo \"removed policy epok-read-ingest-key from $ROLE\"\nelif grep -q NoSuchEntity <<< \"$OUT\"; then echo \"policy already gone\"; else echo \"$OUT\" >&2; exit 1; fi"
    },
    {
      "id": "delete-key-parameter",
      "actor": "agent",
      "destructive": true,
      "description": "Delete the SSM parameter or S3 env file holding the key (already gone counts as done). Skip this step if another service still uses it.",
      "run": "B=\"${EPOK_KEY_S3_URI#s3://}\"; BUCKET=\"${B%%/*}\"; OBJ=\"${B#*/}\"\ncase \"$EPOK_KEY_STORE\" in\n  ssm)\n    if OUT=\"$(aws ssm delete-parameter --region \"$AWS_REGION\" --name \"$EPOK_KEY_PARAMETER\" 2>&1)\"; then\n      echo \"deleted $EPOK_KEY_PARAMETER\"\n    elif grep -q ParameterNotFound <<< \"$OUT\"; then echo \"parameter already gone\"; else echo \"$OUT\" >&2; exit 1; fi ;;\n  s3)\n    aws s3 rm \"$EPOK_KEY_S3_URI\" --only-show-errors && echo \"deleted $EPOK_KEY_S3_URI (or it was already gone)\" ;;\nesac"
    }
  ]
}
