{
  "schema": "epok.setup-manifest/1",
  "id": "aws-lambda-otel-traces",
  "version": "1.0.0",
  "title": "AWS Lambda (Python) traces -> Epok with the OpenTelemetry Lambda layers",
  "summary": "Adds the public OpenTelemetry Python layer (auto-instrumentation, AWS_LAMBDA_EXEC_WRAPPER) and the OpenTelemetry Collector layer to an existing Python Lambda function. The in-function collector forwards OTLP/HTTP to Epok with the x-api-key header. No code change, no redeploy of the function package.",
  "provider": "aws",
  "signal": "traces",
  "source": "otlp_traces_lambda",
  "tool": "aws",
  "estimated_minutes": 3,
  "last_verified": null,
  "prerequisites": {
    "cli": [
      {
        "name": "aws",
        "min_version": "2.15.0",
        "check": "aws --version",
        "version_regex": "aws-cli/([0-9]+\\.[0-9]+\\.[0-9]+)"
      },
      {
        "name": "python3",
        "min_version": "3.8.0",
        "check": "python3 --version",
        "version_regex": "Python ([0-9]+\\.[0-9]+\\.[0-9]+)"
      },
      {
        "name": "curl",
        "min_version": "7.68.0",
        "check": "curl --version",
        "version_regex": "^curl ([0-9]+\\.[0-9]+\\.[0-9]+)"
      }
    ],
    "permissions": [
      "lambda:GetFunctionConfiguration",
      "lambda:UpdateFunctionConfiguration",
      "lambda:GetLayerVersion",
      "lambda:InvokeFunction",
      "secretsmanager:CreateSecret",
      "secretsmanager:DescribeSecret",
      "secretsmanager:TagResource",
      "secretsmanager:DeleteSecret",
      "iam:PutRolePolicy",
      "iam:DeleteRolePolicy"
    ],
    "permissions_note": "lambda:InvokeFunction is only for the verify canary. The layers are public (account 184161586896). The secretsmanager:* and iam:*RolePolicy permissions are for the default key store (an inline policy on the function's own role, scoped to one secret ARN); EPOK_KEY_STORE=env needs none of them.",
    "check_commands": [
      "aws sts get-caller-identity --query Account --output text >/dev/null",
      "aws lambda get-function-configuration --function-name \"$FUNCTION_NAME\" --region \"$AWS_REGION\" --query Runtime --output text | grep -q '^python3\\.'",
      "aws lambda get-layer-version-by-arn --region \"$AWS_REGION\" --arn \"arn:aws:lambda:$AWS_REGION:184161586896:layer:opentelemetry-python-$EPOK_OTEL_PYTHON_LAYER_VERSION\" --query LayerVersionArn --output text >/dev/null"
    ]
  },
  "inputs": [
    {
      "name": "AWS_REGION",
      "required": true,
      "actor": "agent",
      "pattern": "^[a-z]{2}(-[a-z]+)+-[0-9]$",
      "description": "Region of the function.",
      "how_to_get": "aws configure get region, or the function ARN."
    },
    {
      "name": "FUNCTION_NAME",
      "required": true,
      "actor": "agent",
      "pattern": "^[A-Za-z0-9_-]{1,64}$",
      "description": "Name of the existing Python (python3.x, zip package) Lambda function to instrument.",
      "how_to_get": "aws lambda list-functions --query 'Functions[?starts_with(Runtime, `python3`)].FunctionName' (confirm with the human)."
    },
    {
      "name": "OTEL_SERVICE_NAME",
      "required": true,
      "actor": "agent",
      "pattern": "^[A-Za-z0-9._-]{1,128}$",
      "description": "service.name for the spans (how the service appears in Epok).",
      "how_to_get": "Ask the human, or use the workload name."
    },
    {
      "name": "EPOK_KEY_STORE",
      "default": "secretsmanager",
      "actor": "agent",
      "pattern": "^(secretsmanager|env)$",
      "description": "Where the collector reads the key. secretsmanager (default, recommended): a Secrets Manager secret read by the collector's secretsmanager config provider with the function role. env (NOT RECOMMENDED): the key as the plain function environment variable EPOK_API_KEY, readable by anyone with lambda:GetFunctionConfiguration; use only when Secrets Manager is unavailable and the human agrees."
    },
    {
      "name": "EPOK_KEY_SECRET_NAME",
      "default": "epok/ingest-api-key",
      "actor": "agent",
      "pattern": "^[A-Za-z0-9/_+=.@-]{1,200}$",
      "description": "Secrets Manager secret name for the key (EPOK_KEY_STORE=secretsmanager). An existing secret is reused, not overwritten. About $0.40/month per secret plus $0.05 per 10,000 API calls (one read per cold start)."
    },
    {
      "name": "EPOK_RESOURCE_TAGS",
      "default": "managed-by=epok",
      "actor": "agent",
      "pattern": "^[A-Za-z0-9_.:/=+@-]+=[A-Za-z0-9_.:/=+@ -]*(,[A-Za-z0-9_.:/=+@-]+=[A-Za-z0-9_.:/=+@ -]*){0,9}$",
      "description": "Comma-separated key=value tags for the secret."
    },
    {
      "name": "EPOK_OTEL_PYTHON_LAYER_VERSION",
      "default": "0_22_0:1",
      "actor": "agent",
      "pattern": "^[0-9]+_[0-9]+_[0-9]+:[0-9]+$",
      "description": "Version suffix of the public layer arn:aws:lambda:<region>:184161586896:layer:opentelemetry-python-<v> (layer-python/0.22.0 = opentelemetry-python 1.45.1). Supports arm64 and x86_64."
    },
    {
      "name": "EPOK_OTEL_COLLECTOR_LAYER_VERSION",
      "default": "0_24_0:1",
      "actor": "agent",
      "pattern": "^[0-9]+_[0-9]+_[0-9]+:[0-9]+$",
      "description": "Version suffix of the public layer arn:aws:lambda:<region>:184161586896:layer:opentelemetry-collector-<amd64|arm64>-<v> (layer-collector/0.24.0 = collector v0.163.0). The architecture is read from the function."
    },
    {
      "name": "EPOK_API_KEY",
      "required": true,
      "secret": true,
      "scope": "ingest",
      "actor": "human",
      "pattern": "^[A-Za-z0-9_.-]{8,200}$",
      "description": "Epok ingest-scope API key, dedicated to this workload.",
      "how_to_get": "A human creates it: Epok -> Settings -> API Keys -> New key (scope: ingest, name: traces-<workload>). Agents must ask for it, never create or guess it, and never write it to a file or a commit."
    },
    {
      "name": "EPOK_READ_API_KEY",
      "required": true,
      "secret": true,
      "scope": "read",
      "actor": "human",
      "pattern": "^[A-Za-z0-9_.-]{8,200}$",
      "description": "Epok API key with read scope, used only by verify to look the canary trace up. May be the same key as EPOK_API_KEY if that key also has read scope.",
      "how_to_get": "A human creates it: Epok -> Settings -> API Keys (scope: read). Agents must ask for it."
    },
    {
      "name": "EPOK_INGEST_HOST",
      "default": "https://ingest.epok.cloud",
      "actor": "agent",
      "pattern": "^(https://[A-Za-z0-9.-]+|http://(127\\.0\\.0\\.1|localhost))(:[0-9]{1,5})?$",
      "description": "Epok OTLP/HTTP ingest base URL, no trailing slash. The exporter appends /v1/traces. Change only if Epok support tells you to."
    },
    {
      "name": "EPOK_API_HOST",
      "default": "https://app.epok.cloud",
      "actor": "agent",
      "pattern": "^(https://[A-Za-z0-9.-]+|http://(127\\.0\\.0\\.1|localhost))(:[0-9]{1,5})?$",
      "description": "Epok API base URL used by verify, no trailing slash."
    },
    {
      "name": "EPOK_TENANT_ID",
      "required": true,
      "actor": "agent",
      "pattern": "^[0-9]{1,10}$",
      "description": "Your Epok tenant (account) id, used in the verify API path.",
      "how_to_get": "Epok -> Settings -> Account (the numeric account id), or ask the human."
    },
    {
      "name": "NONCE",
      "generate": "nonce",
      "actor": "agent",
      "pattern": "^[0-9a-f]{24}$",
      "description": "Canary token: verify sends one request with W3C traceparent trace id 00000000<NONCE> and looks that trace up in Epok. Generated by the runner if not set (24 lowercase hex)."
    }
  ],
  "cost_notes": "One Secrets Manager secret (about $0.40/month, plus $0.05 per 10,000 reads; the collector reads it once per cold start). The collector extension shares the function's memory and adds roughly 100-300 ms to cold starts and a few ms per invocation (billed duration). Epok bills the decoded OTLP bytes: about 1-2 KB per 3-span trace from these layers, so 1 million invocations/day with 3 spans each is about 1.5 GB/day.",
  "notes": [
    "Python functions only (zip package). For Node.js use the opentelemetry-nodejs layer with the same collector layer and collector config; the steps are otherwise identical.",
    "The collector config is passed in the EPOK_OTEL_COLLECTOR_CONFIG environment variable (OPENTELEMETRY_COLLECTOR_CONFIG_URI=env:...), so the function package does not change.",
    "Default key store: Secrets Manager. The collector layer's secretsmanager config provider resolves ${secretsmanager:<secret ARN>} at collector start (one GetSecretValue per cold start) with the function role, which gets GetSecretValue on that one ARN. The key never appears in the function configuration. A customer-managed KMS key on the secret also needs kms:Decrypt for the role.",
    "EPOK_KEY_STORE=env is NOT RECOMMENDED: it writes the key into the function environment (EPOK_API_KEY), encrypted at rest but readable by anyone with lambda:GetFunctionConfiguration and visible in the console.",
    "The root span is created by the layer (one SERVER span per invocation) and continues a W3C traceparent found in event.headers (API Gateway, Function URLs, ALB).",
    "Verify invokes the function once with the event {\"headers\":{\"traceparent\":...}}. If invoking the function has side effects, ask the human before verify."
  ],
  "steps": [
    {
      "id": "inspect-function",
      "actor": "agent",
      "destructive": false,
      "idempotent": true,
      "description": "Read the function configuration: it must be a python3.x zip function. Prints EPOK_COLLECTOR_ARCH (amd64 or arm64) for the collector layer.",
      "run": "aws lambda get-function-configuration --function-name \"$FUNCTION_NAME\" --region \"$AWS_REGION\" --output json | python3 -c '\nimport json, sys\nc = json.load(sys.stdin)\nrt = c.get(\"Runtime\") or \"\"\nif not rt.startswith(\"python3.\"):\n    sys.exit(\"runtime \" + (rt or str(c.get(\"PackageType\"))) + \" is not a Python zip function; this manifest instruments Python only\")\narch = (c.get(\"Architectures\") or [\"x86_64\"])[0]\nprint(\"runtime \" + rt + \", architecture \" + arch + \", layers \" + str(len(c.get(\"Layers\") or [])))\nprint(\"EPOK_COLLECTOR_ARCH=\" + (\"arm64\" if arch == \"arm64\" else \"amd64\"))\nprint(\"EPOK_FUNCTION_ROLE_NAME=\" + c[\"Role\"].split(\"/\")[-1])\nprint(\"EPOK_ACCOUNT_ID=\" + c[\"FunctionArn\"].split(\":\")[4])'",
      "exports": [
        "EPOK_COLLECTOR_ARCH",
        "EPOK_FUNCTION_ROLE_NAME",
        "EPOK_ACCOUNT_ID"
      ]
    },
    {
      "id": "store-key",
      "actor": "agent",
      "destructive": false,
      "idempotent_check": "case \"$EPOK_KEY_STORE\" in\n  env) exit 0 ;;   # nothing to store: the key goes into the function environment (not recommended)\n  secretsmanager) aws secretsmanager describe-secret --secret-id \"$EPOK_KEY_SECRET_NAME\" --region \"$AWS_REGION\" --query ARN --output text >/dev/null 2>&1 ;;\nesac",
      "on_exists": "skip",
      "description": "EPOK_KEY_STORE=secretsmanager (default): create the Secrets Manager secret EPOK_KEY_SECRET_NAME holding the key (value read from JSON on stdin, never on the command line). Skipped if the secret exists, and always skipped with EPOK_KEY_STORE=env.",
      "run": "# AWS CLI v2 (2.17.0 to 2.40.3 tested) rejects --cli-input-json read from a pipe (\"Invalid JSON received\"). The key goes in through a plain parameter file on stdin instead; the other arguments are not secret.\npython3 -c '\nimport json, os, subprocess, sys\ne = os.environ\ntags = [dict(zip((\"Key\", \"Value\"), p.split(\"=\", 1))) for p in e[\"EPOK_RESOURCE_TAGS\"].split(\",\")]\nsys.exit(subprocess.run([\"aws\", \"secretsmanager\", \"create-secret\", \"--region\", e[\"AWS_REGION\"],\n                         \"--name\", e[\"EPOK_KEY_SECRET_NAME\"], \"--description\", \"Epok ingest API key for OpenTelemetry traces\",\n                         \"--tags\", json.dumps(tags), \"--secret-string\", \"file:///dev/stdin\",\n                         \"--query\", \"ARN\", \"--output\", \"text\"], input=e[\"EPOK_API_KEY\"], text=True).returncode)\n'"
    },
    {
      "id": "grant-function-role",
      "actor": "agent",
      "destructive": false,
      "idempotent": true,
      "description": "Allow the function's execution role secretsmanager:GetSecretValue on exactly that secret ARN (inline policy epok-read-ingest-key). No-op with EPOK_KEY_STORE=env.",
      "run": "case \"$EPOK_KEY_STORE\" in\n  env) echo \"EPOK_KEY_STORE=env: no IAM change (not recommended: the key sits in the function configuration)\"; exit 0 ;;\nesac\nARN=\"$(aws secretsmanager describe-secret --secret-id \"$EPOK_KEY_SECRET_NAME\" --region \"$AWS_REGION\" --query ARN --output text)\"\naws iam put-role-policy --role-name \"$EPOK_FUNCTION_ROLE_NAME\" --policy-name epok-read-ingest-key \\\n  --policy-document \"{\\\"Version\\\":\\\"2012-10-17\\\",\\\"Statement\\\":[{\\\"Effect\\\":\\\"Allow\\\",\\\"Action\\\":\\\"secretsmanager:GetSecretValue\\\",\\\"Resource\\\":\\\"$ARN\\\"}]}\"\necho \"function role $EPOK_FUNCTION_ROLE_NAME may read exactly $ARN\" "
    },
    {
      "id": "configure-function",
      "actor": "agent",
      "destructive": false,
      "idempotent": true,
      "description": "Add the two OpenTelemetry layers (replacing older versions of them, keeping every other layer) and merge the OTEL_* / Epok variables into the existing environment (every other variable is kept). The collector reads the key with ${secretsmanager:<secret ARN>} (default) or, with EPOK_KEY_STORE=env, from the EPOK_API_KEY variable. Waits until the update is applied.",
      "run": "# AWS CLI v2 (2.17.0 to 2.40.3 tested) rejects --cli-input-json read from a pipe (\"Invalid JSON received\"). The key goes in through a plain parameter file on stdin instead; the other arguments are not secret.\npython3 -c '\nimport json, os, subprocess, sys\ne = os.environ\nc = json.loads(subprocess.check_output([\"aws\", \"lambda\", \"get-function-configuration\", \"--function-name\",\n                                        e[\"FUNCTION_NAME\"], \"--region\", e[\"AWS_REGION\"], \"--output\", \"json\"]))\notel = \":184161586896:layer:opentelemetry-\"\npy = \"arn:aws:lambda:\" + e[\"AWS_REGION\"] + otel[:-1] + \"-python-\" + e[\"EPOK_OTEL_PYTHON_LAYER_VERSION\"]\ncol = (\"arn:aws:lambda:\" + e[\"AWS_REGION\"] + otel + \"collector-\" + e[\"EPOK_COLLECTOR_ARCH\"] + \"-\"\n       + e[\"EPOK_OTEL_COLLECTOR_LAYER_VERSION\"])\nlayers = [l[\"Arn\"] for l in c.get(\"Layers\") or [] if otel not in l[\"Arn\"]] + [py, col]\nif len(layers) > 5:\n    sys.exit(\"the function would have more than 5 layers; stop and ask\")\nenv = (c.get(\"Environment\") or {}).get(\"Variables\") or {}\nwrapper = env.get(\"AWS_LAMBDA_EXEC_WRAPPER\")\nif wrapper not in (None, \"/opt/otel-instrument\"):\n    sys.exit(\"AWS_LAMBDA_EXEC_WRAPPER is already \" + wrapper + \"; stop and ask\")\ncfg = \"receivers:\\n  otlp:\\n    protocols:\\n      http:\\n        endpoint: localhost:4318\\nexporters:\\n  otlphttp/epok:\\n    endpoint: \\x24{env:EPOK_INGEST_HOST}\\n    headers:\\n      x-api-key: \\x24{env:EPOK_API_KEY}\\nservice:\\n  pipelines:\\n    traces:\\n      receivers: [otlp]\\n      exporters: [otlphttp/epok]\\n\"\nfor k in (\"EPOK_API_KEY\", \"EPOK_KEY_SECRET_ARN\"):\n    env.pop(k, None)\nif e[\"EPOK_KEY_STORE\"] == \"secretsmanager\":\n    arn = subprocess.check_output([\"aws\", \"secretsmanager\", \"describe-secret\", \"--secret-id\", e[\"EPOK_KEY_SECRET_NAME\"],\n                                   \"--region\", e[\"AWS_REGION\"], \"--query\", \"ARN\", \"--output\", \"text\"], text=True).strip()\n    cfg = cfg.replace(\"\\x24{env:EPOK_API_KEY}\", \"\\x24{secretsmanager:\" + arn + \"}\")   # \\x24 = \"$\" (collector placeholder, not shell)\nelse:\n    env[\"EPOK_API_KEY\"] = e[\"EPOK_API_KEY\"]   # not recommended: readable with lambda:GetFunctionConfiguration\nenv.update({\n    \"AWS_LAMBDA_EXEC_WRAPPER\": \"/opt/otel-instrument\",\n    \"OTEL_SERVICE_NAME\": e[\"OTEL_SERVICE_NAME\"],\n    \"OTEL_TRACES_EXPORTER\": \"otlp\",\n    \"OTEL_METRICS_EXPORTER\": \"none\",\n    \"OTEL_LOGS_EXPORTER\": \"none\",\n    \"OTEL_EXPORTER_OTLP_PROTOCOL\": \"http/protobuf\",\n    \"OTEL_EXPORTER_OTLP_ENDPOINT\": \"http://localhost:4318\",\n    \"OPENTELEMETRY_COLLECTOR_CONFIG_URI\": \"env:EPOK_OTEL_COLLECTOR_CONFIG\",\n    \"EPOK_OTEL_COLLECTOR_CONFIG\": cfg,\n    \"EPOK_INGEST_HOST\": e[\"EPOK_INGEST_HOST\"],\n})\n# The environment may hold the key (EPOK_KEY_STORE=env): stdin, never argv.\nr = subprocess.run([\"aws\", \"lambda\", \"update-function-configuration\", \"--region\", e[\"AWS_REGION\"],\n                    \"--function-name\", e[\"FUNCTION_NAME\"], \"--layers\", *layers, \"--environment\", \"file:///dev/stdin\",\n                    \"--query\", \"join(`,`, [FunctionName, LastUpdateStatus])\", \"--output\", \"text\"],\n                   input=json.dumps({\"Variables\": env}), text=True)\nif r.returncode:\n    sys.exit(r.returncode)\n'\naws lambda wait function-updated-v2 --function-name \"$FUNCTION_NAME\" --region \"$AWS_REGION\"\necho \"instrumented $FUNCTION_NAME\" ",
      "timeout_seconds": 300
    }
  ],
  "verify": {
    "emit": "OUT=\"$(mktemp)\"\naws lambda invoke --function-name \"$FUNCTION_NAME\" --region \"$AWS_REGION\" --cli-binary-format raw-in-base64-out \\\n  --payload '{\"headers\":{\"traceparent\":\"00-00000000'\"$NONCE\"'-00f067aa0ba902b7-01\"}}' \\\n  --query StatusCode --output text \"$OUT\"\nrm -f \"$OUT\"",
    "call": "printf 'header = \"X-API-Key: %s\"\\n' \"$EPOK_READ_API_KEY\" | curl -K - -fsS \\\n  \"$EPOK_API_HOST/api/v1/tenants/$EPOK_TENANT_ID/traces/00000000$NONCE\" | python3 -c '\nimport json, sys\nd = json.load(sys.stdin)\nspans = [e for es in d.get(\"services\", {}).values() for e in es\n         if e.get(\"fields\", {}).get(\"record_type\") == \"span\"]\nids = {e.get(\"span_id\") for e in spans}\nservices = sorted({e.get(\"service\") for e in spans})\nout = {\n    \"trace_id\": d.get(\"trace_id\"),\n    \"spans\": len(spans),\n    \"services\": services,\n    \"roots\": sum(1 for e in spans if e.get(\"parent_span_id\") not in ids),\n    \"children\": sum(1 for e in spans if e.get(\"parent_span_id\") in ids),\n    \"error_spans\": sum(1 for e in spans if e.get(\"fields\", {}).get(\"span_status\") == \"error\"),\n}\nout[\"ok\"] = True if spans and services == [sys.argv[1]] else None\nprint(json.dumps(out))' \"$OTEL_SERVICE_NAME\"",
    "success": {
      "field": "ok",
      "is": "non_null"
    },
    "timeout_seconds": 180,
    "poll_seconds": 10,
    "reemit_seconds": 60
  },
  "failure_modes": [
    {
      "id": "permission-denied",
      "step": [
        "inspect-function",
        "configure-function"
      ],
      "symptom": "AccessDeniedException on get-function-configuration or update-function-configuration.",
      "check": "aws sts get-caller-identity --query Arn --output text",
      "fix": "The caller lacks a permission in prerequisites.permissions. Stop and ask a human to grant it. Never grant IAM to yourself.",
      "stop_and_ask": true
    },
    {
      "id": "secret-access-denied",
      "step": [
        "store-key",
        "grant-function-role",
        "verify"
      ],
      "symptom": "AccessDeniedException on create-secret/describe-secret, or (verify) collector logs show 'secretsmanager' / 'GetSecretValue' AccessDenied and no trace arrives.",
      "check": "aws logs tail \"/aws/lambda/$FUNCTION_NAME\" --region \"$AWS_REGION\" --since 10m | grep -iE 'secretsmanager|GetSecretValue|AccessDenied' | tail -10",
      "fix": "Caller side: you lack a secretsmanager:* permission in prerequisites.permissions (stop and ask; never grant IAM to yourself). Function side: re-run grant-function-role; IAM can take up to a minute to propagate (verify re-invokes every 60 s); a customer-managed KMS key also needs kms:Decrypt. Do not fall back to EPOK_KEY_STORE=env without the human's agreement.",
      "stop_and_ask": true
    },
    {
      "id": "not-python",
      "step": "inspect-function",
      "symptom": "inspect-function exits with 'is not a Python zip function'.",
      "check": "aws lambda get-function-configuration --function-name \"$FUNCTION_NAME\" --region \"$AWS_REGION\" --query '[Runtime,PackageType]'",
      "fix": "Container-image or non-Python functions need the SDK in the image (see the Epok ECS/Cloud Run guides) or the matching language layer. Stop and ask which path the human wants.",
      "stop_and_ask": true
    },
    {
      "id": "layer-not-found",
      "step": "configure-function",
      "symptom": "update-function-configuration fails with 'Layer version ... does not exist' or AccessDenied on the layer.",
      "check": "aws lambda get-layer-version-by-arn --region \"$AWS_REGION\" --arn \"arn:aws:lambda:$AWS_REGION:184161586896:layer:opentelemetry-collector-$EPOK_COLLECTOR_ARCH-$EPOK_OTEL_COLLECTOR_LAYER_VERSION\"",
      "fix": "The layer version is not published in this region. Pick a published version from https://github.com/open-telemetry/opentelemetry-lambda/releases and re-run with EPOK_OTEL_*_LAYER_VERSION set.",
      "stop_and_ask": false
    },
    {
      "id": "wrapper-conflict",
      "step": "configure-function",
      "symptom": "configure-function exits with 'AWS_LAMBDA_EXEC_WRAPPER is already ...' or 'more than 5 layers'.",
      "check": "aws lambda get-function-configuration --function-name \"$FUNCTION_NAME\" --region \"$AWS_REGION\" --query '[Layers[].Arn, Environment.Variables.AWS_LAMBDA_EXEC_WRAPPER]'",
      "fix": "Another wrapper or APM layer (Datadog, New Relic, ADOT) is installed. Two tracers on one function conflict. Stop and ask the human which one to keep.",
      "stop_and_ask": true
    },
    {
      "id": "collector-not-exporting",
      "step": "verify",
      "symptom": "The invocation succeeds but the trace never arrives.",
      "check": "aws logs tail \"/aws/lambda/$FUNCTION_NAME\" --region \"$AWS_REGION\" --since 10m | grep -iE 'otel|collector|export|401|403' | tail -20",
      "fix": "Look for collector errors: 401/403 = key (see epok-auth-rejected); 'connection refused' on 4318 = the collector layer is missing or failed to start (check the layer ARN architecture matches the function); DNS/timeout = the function runs in a VPC without internet egress (add a NAT gateway or VPC endpoint route; stop and ask).",
      "stop_and_ask": false
    },
    {
      "id": "epok-auth-rejected",
      "step": "verify",
      "symptom": "Spans never arrive; exporter or collector logs show HTTP 401 or 403 from the Epok ingest host.",
      "check": "Ask the human which key they gave you and its scope (do not print it).",
      "fix": "The ingest key is wrong, revoked, or not ingest scope. Stop and ask the human for a new ingest-scope key. Do not retry in a loop.",
      "stop_and_ask": true
    },
    {
      "id": "verify-read-auth",
      "step": "verify",
      "symptom": "The verify call fails with HTTP 401 or 403 on every poll.",
      "check": "Does EPOK_READ_API_KEY have read scope, and is EPOK_TENANT_ID its tenant?",
      "fix": "403 = the key lacks read scope or belongs to another tenant. Stop and ask the human.",
      "stop_and_ask": true
    },
    {
      "id": "trace-not-found",
      "step": "verify",
      "symptom": "The verify call keeps failing with HTTP 404 (Trace not found) until the timeout.",
      "check": "Re-read the emit output: did the request/invocation succeed? Then check the failure modes for this runtime.",
      "fix": "The canary trace did not reach Epok, or your code does not continue an incoming W3C traceparent (then spans arrive under another trace id: look for OTEL_SERVICE_NAME in Epok -> Traces). Report; do not change sampling or keys to make it pass.",
      "stop_and_ask": false
    },
    {
      "id": "wrong-service-name",
      "step": "verify",
      "symptom": "Verify finds the trace but ok is null and services is not [OTEL_SERVICE_NAME].",
      "check": "Look at services in the verify JSON.",
      "fix": "Something else sets service.name (OTEL_RESOURCE_ATTRIBUTES or code). Report it to the human; do not edit application code without asking.",
      "stop_and_ask": true
    }
  ],
  "uninstall": [
    {
      "id": "remove-instrumentation",
      "actor": "agent",
      "destructive": true,
      "description": "Remove the two OpenTelemetry layers and exactly the variables configure-function added (other layers and variables are kept). A missing function counts as done.",
      "run": "CFG_JSON=\"$(python3 -c '\nimport json, os, subprocess\ne = os.environ\nr = subprocess.run([\"aws\", \"lambda\", \"get-function-configuration\", \"--function-name\", e[\"FUNCTION_NAME\"],\n                    \"--region\", e[\"AWS_REGION\"], \"--output\", \"json\"], capture_output=True, text=True)\nif r.returncode == 0:\n    c = json.loads(r.stdout)\n    otel = \":184161586896:layer:opentelemetry-\"\n    ours = {\"AWS_LAMBDA_EXEC_WRAPPER\", \"OTEL_SERVICE_NAME\", \"OTEL_TRACES_EXPORTER\", \"OTEL_METRICS_EXPORTER\",\n            \"OTEL_LOGS_EXPORTER\", \"OTEL_EXPORTER_OTLP_PROTOCOL\", \"OTEL_EXPORTER_OTLP_ENDPOINT\",\n            \"OPENTELEMETRY_COLLECTOR_CONFIG_URI\", \"EPOK_OTEL_COLLECTOR_CONFIG\", \"EPOK_INGEST_HOST\", \"EPOK_API_KEY\",\n            \"EPOK_KEY_SECRET_ARN\"}\n    env = {k: v for k, v in ((c.get(\"Environment\") or {}).get(\"Variables\") or {}).items() if k not in ours}\n    layers = [l[\"Arn\"] for l in c.get(\"Layers\") or [] if otel not in l[\"Arn\"]]\n    print(json.dumps({\"FunctionName\": e[\"FUNCTION_NAME\"], \"Layers\": layers, \"Environment\": {\"Variables\": env}}))\n')\"\nif [ -z \"$CFG_JSON\" ]; then echo \"function $FUNCTION_NAME not found; nothing to do\"; exit 0; fi\naws lambda update-function-configuration --region \"$AWS_REGION\" \\\n  --cli-input-json \"$CFG_JSON\" --query 'join(`,`, [FunctionName, LastUpdateStatus])' --output text\naws lambda wait function-updated-v2 --function-name \"$FUNCTION_NAME\" --region \"$AWS_REGION\"\necho \"removed Epok tracing layers and variables from $FUNCTION_NAME\" ",
      "timeout_seconds": 300
    },
    {
      "id": "remove-role-policy",
      "actor": "agent",
      "destructive": true,
      "skip_if_failed": [
        "remove-instrumentation"
      ],
      "description": "Delete the inline policy epok-read-ingest-key from the function role (already gone counts as done). Not run while the function may still read the secret.",
      "run": "ROLE=\"$(aws lambda get-function-configuration --function-name \"$FUNCTION_NAME\" --region \"$AWS_REGION\" --query Role --output text 2>/dev/null | sed 's#.*/##')\" || true\nROLE=\"${ROLE:-${EPOK_FUNCTION_ROLE_NAME:-}}\"\nif [ -z \"$ROLE\" ]; then echo \"function role unknown (function gone); set EPOK_FUNCTION_ROLE_NAME to clean up\"; exit 0; fi\nif OUT=\"$(aws iam delete-role-policy --role-name \"$ROLE\" --policy-name epok-read-ingest-key 2>&1)\"; then\n  echo \"removed policy epok-read-ingest-key from $ROLE\"\nelif grep -q NoSuchEntity <<< \"$OUT\"; then echo \"policy already gone\"; else echo \"$OUT\" >&2; exit 1; fi"
    },
    {
      "id": "delete-key-secret",
      "actor": "agent",
      "destructive": true,
      "skip_if_failed": [
        "remove-instrumentation"
      ],
      "description": "Delete the Secrets Manager secret (no recovery window: the key itself lives in Epok and can be re-issued). Skipped with EPOK_KEY_STORE=env; already gone counts as done. Skip this step if another function still uses the same secret.",
      "run": "if [ \"$EPOK_KEY_STORE\" = env ]; then echo \"EPOK_KEY_STORE=env: no secret to delete\"; exit 0; fi\nif OUT=\"$(aws secretsmanager delete-secret --region \"$AWS_REGION\" --secret-id \"$EPOK_KEY_SECRET_NAME\" --force-delete-without-recovery 2>&1)\"; then\n  echo \"deleted secret $EPOK_KEY_SECRET_NAME\"\nelif grep -q ResourceNotFoundException <<< \"$OUT\"; then echo \"secret already gone\"; else echo \"$OUT\" >&2; exit 1; fi"
    }
  ]
}
