Implement reversible PII de-identification round trip (T1–T7)

Build the AWS side of the HNCB demo end to end (region ap-southeast-1): - T1 /tokenize + T2 /restore: Lambdas behind a public API Gateway (shared-secret auth), Presidio detection, random per-request tokens, DynamoDB vault; overlap resolution so a ROC ID stays TW_ROC_ID. - T3: Presidio made private (SG-locked to the tokenize Lambda in-VPC; DynamoDB gateway endpoint); only /tokenize + /restore are public. - T4: RAG Lambda registered as an MCP tool on an AgentCore Gateway (AWS_IAM/SigV4); agentcore_setup.sh + a SigV4 MCP invoke test. - T5: Strands agent deployed to AgentCore Runtime; SigV4 gateway auth, apac inference profile, pinned deps. - T6: advisor UI on S3+CloudFront with a Fusion-less demo orchestrator (/demo) chaining tokenize -> runtime -> restore. - T7: README runbook + trace check; teardown deletes gateway/runtime/memory/ECR. Verified live: the cloud AgentCore/Bedrock trace shows only tokens, never the real name. Secrets stay in gitignored local.auto.tfvars. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-01 17:10:58 +10:00
parent 6b2a051be3
commit 78d67a2469
21 changed files with 1439 additions and 107 deletions
--- a/scripts/agentcore_setup.sh
+++ b/scripts/agentcore_setup.sh
@@ -1,41 +1,117 @@
 #!/usr/bin/env bash
 # AgentCore setup (steps 3-4-7). Terraform's AgentCore coverage lags, so this
-# stage uses the AgentCore CLI / SDK. Commands below reflect the current CLI
-# shape -- CHECK against `agentcore --help`, the surface moves fast.
+# stage uses the AWS CLI's `bedrock-agentcore-control` API. Verified against the
+# CLI on 2026-07-01; the surface moves fast, so re-check with
+# `aws bedrock-agentcore-control help` if a call rejects.
+#
+# T4 (implemented below): create an MCP Gateway and register the RAG Lambda as a
+# tool. Inbound auth is AWS_IAM (SigV4) -- no Cognito needed. The Gateway assumes
+# the IAM role from Terraform (`terraform output -raw gateway_role_arn`) to invoke
+# the Lambda.
 set -euo pipefail
+cd "$(dirname "$0")"

 : "${REGION:?set REGION}"
-: "${RAG_LAMBDA_ARN:?set RAG_LAMBDA_ARN (from: terraform output -raw rag_lambda_arn)}"
+: "${RAG_LAMBDA_ARN:?set RAG_LAMBDA_ARN (from: terraform -chdir=terraform output -raw rag_lambda_arn)}"
+: "${GATEWAY_ROLE_ARN:?set GATEWAY_ROLE_ARN (from: terraform -chdir=terraform output -raw gateway_role_arn)}"
 : "${BEDROCK_MODEL_ID:=anthropic.claude-3-5-sonnet-20241022-v2:0}"
+GW_NAME="${GW_NAME:-hncb-rag-gateway}"
+TAG='conan hncb demo'
+export AWS_PAGER=""

-echo "==> 1. Install the AgentCore CLI (recommended tooling)"
-echo "    npm install -g @aws/agentcore    # or: pip install bedrock-agentcore-starter-toolkit"
+echo "==> 1. Create the MCP Gateway (AWS_IAM inbound auth)"
+GW_JSON="$(aws bedrock-agentcore-control create-gateway \
+  --region "$REGION" \
+  --name "$GW_NAME" \
+  --role-arn "$GATEWAY_ROLE_ARN" \
+  --protocol-type MCP \
+  --authorizer-type AWS_IAM \
+  --tags "Owner=$TAG" 2>/dev/null \
+  || aws bedrock-agentcore-control list-gateways --region "$REGION" \
+       --query "items[?name=='$GW_NAME']|[0]" --output json)"
+GW_ID="$(echo "$GW_JSON" | python3 -c 'import sys,json;print(json.load(sys.stdin)["gatewayId"])')"
+echo "    gatewayId=$GW_ID"

-echo "==> 2. Create a Gateway that exposes the RAG Lambda as an MCP tool"
-echo "    AgentCore Gateway turns a Lambda into an MCP tool with no code changes."
+echo "==> 2. Wait for the Gateway to be READY"
+for i in $(seq 1 20); do
+  ST="$(aws bedrock-agentcore-control get-gateway --region "$REGION" --gateway-identifier "$GW_ID" --query status --output text)"
+  echo "    [$i] status=$ST"
+  [ "$ST" = "READY" ] && break
+  sleep 6
+done
+GW_URL="$(aws bedrock-agentcore-control get-gateway --region "$REGION" --gateway-identifier "$GW_ID" --query gatewayUrl --output text)"
+
+echo "==> 3. Register the RAG Lambda as an MCP tool target"
+# Build the target config: mcp.lambda with the tool schema from agent/tool_schema.json
+# (wrapped in the inlinePayload list). The Gateway's inputSchema is a RESTRICTED
+# JSON-Schema subset (only type/properties/required/items/description), so we strip
+# anything else (e.g. the "default" on period_days) recursively.
+python3 - "$RAG_LAMBDA_ARN" > /tmp/hncb_target.json <<'PY'
+import json, sys, os
+ALLOWED = {"type", "properties", "required", "items", "description"}
+def clean(s):
+    if not isinstance(s, dict):
+        return s
+    out = {k: v for k, v in s.items() if k in ALLOWED}
+    if "properties" in out:
+        out["properties"] = {k: clean(v) for k, v in out["properties"].items()}
+    if "items" in out:
+        out["items"] = clean(out["items"])
+    return out
+lambda_arn = sys.argv[1]
+schema = json.load(open(os.path.join("..", "agent", "tool_schema.json")))
+schema["inputSchema"] = clean(schema["inputSchema"])
+if "outputSchema" in schema:
+    schema["outputSchema"] = clean(schema["outputSchema"])
+print(json.dumps({
+    "mcp": {"lambda": {"lambdaArn": lambda_arn, "toolSchema": {"inlinePayload": [schema]}}}
+}))
+PY
+if aws bedrock-agentcore-control list-gateway-targets --region "$REGION" \
+     --gateway-identifier "$GW_ID" --query 'items[?name==`rag`]' --output text | grep -q rag; then
+  echo "    target 'rag' already exists -- reusing"
+else
+  aws bedrock-agentcore-control create-gateway-target \
+    --region "$REGION" \
+    --gateway-identifier "$GW_ID" \
+    --name "rag" \
+    --target-configuration file:///tmp/hncb_target.json \
+    --credential-provider-configurations '[{"credentialProviderType":"GATEWAY_IAM_ROLE"}]' \
+    --query '{target:targetId,status:status}' --output json
+fi
+
+echo ""
+echo "Gateway ready. MCP URL:"
+echo "    $GW_URL"
+echo "Export it for the agent + the invoke test:"
+echo "    export AGENTCORE_GATEWAY_URL=$GW_URL"
+echo ""
+echo "Verify T4 (manual invoke returns the evidence package):"
+echo "    REGION=$REGION AGENTCORE_GATEWAY_URL=$GW_URL python3 scripts/gateway_invoke_test.py CUST_000123"
+
+########################################
+# T5: deploy the agent to AgentCore Runtime (uses the starter toolkit CLI).
+# In ap-southeast-1 Claude 3.5 Sonnet v2 is INFERENCE_PROFILE-only -> use the
+# apac.* profile id, not the raw model id.
+########################################
+APAC_MODEL="apac.anthropic.claude-3-5-sonnet-20241022-v2:0"
+echo ""
+echo "==> (T5) Deploy the agent to AgentCore Runtime:"
 cat <<EOF
-    agentcore gateway create \\
-      --name hncb-rag-gateway \\
-      --region ${REGION} \\
-      --target-type lambda \\
-      --target-arn ${RAG_LAMBDA_ARN} \\
-      --tool-name get_customer_activity_summary \\
-      --tool-description "Return a de-identified activity summary for a customer TOKEN" \\
-      --tool-schema file://../agent/tool_schema.json
-    # Note the printed Gateway MCP URL -> export AGENTCORE_GATEWAY_URL=...
+    cd ../agent   # requirements.txt is version-pinned
+    pip install bedrock-agentcore-starter-toolkit          # provides 'agentcore'
+    agentcore configure -e agent.py -n hncb_advisor_agent -r ${REGION} \\
+      -rf requirements.txt --non-interactive
+    agentcore launch --env AGENTCORE_GATEWAY_URL=${GW_URL} --env BEDROCK_MODEL_ID=${APAC_MODEL}
+    # -> note the printed Runtime ARN (Fusion / the UI will invoke this).
+
+    # REQUIRED post-launch: the auto-created execution role can't call the Gateway
+    # yet. Grant it (role name is printed by launch as AmazonBedrockAgentCoreSDKRuntime-*):
+    ROLE=\$(aws iam list-roles --query "Roles[?starts_with(RoleName,'AmazonBedrockAgentCoreSDKRuntime-${REGION}')].RoleName | [0]" --output text)
+    GW_ARN=\$(aws bedrock-agentcore-control get-gateway --region ${REGION} --gateway-identifier ${GW_ID} --query gatewayArn --output text)
+    aws iam put-role-policy --role-name "\$ROLE" --policy-name hncb-gateway-invoke \\
+      --policy-document "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"bedrock-agentcore:InvokeGateway\"],\"Resource\":[\"\$GW_ARN\",\"\$GW_ARN/*\"]}]}"
+
+    # Verify: agentcore invoke '{"prompt":"talking points for CUST_000123"}'
 EOF
-
-echo "==> 3. Deploy the agent to AgentCore Runtime"
-cat <<EOF
-    cd ../agent
-    agentcore configure --entrypoint agent.py --name hncb-advisor-agent --region ${REGION}
-    AGENTCORE_GATEWAY_URL=<from step 2> BEDROCK_MODEL_ID=${BEDROCK_MODEL_ID} \\
-      agentcore launch
-    # Note the printed Runtime ARN -> Fusion will invoke this.
-EOF
-
-echo "==> 4. (recommended) Lock the Runtime to accept calls only from the Gateway"
-echo "    Use a resource policy (SigV4) or allowedWorkloadConfiguration (JWT)."
-echo "    See AgentCore 'Runtime targets' docs."
-
-echo "Done printing steps. Fill AGENTCORE_GATEWAY_URL + Runtime ARN into fusion/POLICY_SETUP.md."
+echo "Fill AGENTCORE_GATEWAY_URL + Runtime ARN into fusion/POLICY_SETUP.md."
--- a/scripts/deploy.sh
+++ b/scripts/deploy.sh
@@ -16,7 +16,8 @@ terraform -chdir=terraform apply -target=aws_lambda_function.rag -auto-approve

 echo "==> Step 2: build + push the Presidio detector image"
 aws ecr describe-repositories --repository-names hncb-presidio --region "$REGION" >/dev/null 2>&1 \
-  || aws ecr create-repository --repository-name hncb-presidio --region "$REGION" >/dev/null
+  || aws ecr create-repository --repository-name hncb-presidio --region "$REGION" \
+       --tags Key=Owner,Value="conan hncb demo" >/dev/null
 aws ecr get-login-password --region "$REGION" | docker login --username AWS --password-stdin "$ECR"
 docker build -t "$ECR/hncb-presidio:latest" ./presidio
 docker push "$ECR/hncb-presidio:latest"
--- a/scripts/gateway_invoke_test.py
+++ b/scripts/gateway_invoke_test.py
@@ -0,0 +1,104 @@
+"""
+Manual T4 verification: call the RAG tool THROUGH the AgentCore Gateway (MCP) and
+print the de-identified evidence package. This is the "manual agentcore invoke"
+the task's acceptance asks for -- it exercises the real Gateway -> Lambda path,
+not the Lambda directly.
+
+Auth is AWS_IAM: every MCP request is SigV4-signed for the `bedrock-agentcore`
+service with your local creds. MCP uses streamable HTTP (JSON-RPC, responses may
+come back as SSE), so we do initialize -> tools/list -> tools/call.
+
+Usage:
+    REGION=ap-southeast-1 AGENTCORE_GATEWAY_URL=<url> \
+        python3 scripts/gateway_invoke_test.py CUST_000123
+
+Requires botocore (pip install boto3). Depends on nothing else.
+"""
+import json
+import os
+import sys
+import urllib.request
+
+from botocore.auth import SigV4Auth
+from botocore.awsrequest import AWSRequest
+from botocore.session import Session
+
+REGION = os.environ["REGION"]
+URL = os.environ["AGENTCORE_GATEWAY_URL"]
+TOKEN = sys.argv[1] if len(sys.argv) > 1 else "CUST_000123"
+
+_creds = Session().get_credentials()
+_session_id = None  # set from the initialize response
+
+
+def _rpc(method, params=None, notify=False):
+    """One SigV4-signed MCP JSON-RPC call. Returns the parsed result (or None)."""
+    global _session_id
+    body = {"jsonrpc": "2.0", "method": method}
+    if not notify:
+        body["id"] = 1
+    if params is not None:
+        body["params"] = params
+    data = json.dumps(body).encode()
+
+    headers = {"Content-Type": "application/json", "Accept": "application/json, text/event-stream"}
+    if _session_id:
+        headers["Mcp-Session-Id"] = _session_id
+
+    req = AWSRequest(method="POST", url=URL, data=data, headers=headers)
+    SigV4Auth(_creds, "bedrock-agentcore", REGION).add_auth(req)
+
+    try:
+        resp = urllib.request.urlopen(urllib.request.Request(URL, data=data, headers=dict(req.headers)), timeout=40)
+    except urllib.error.HTTPError as e:
+        print(f"HTTP {e.code} on {method}: {e.read().decode()[:400]}", file=sys.stderr)
+        raise
+
+    sid = resp.headers.get("Mcp-Session-Id")
+    if sid:
+        _session_id = sid
+    raw = resp.read().decode()
+    if notify:
+        return None
+
+    # Response is either JSON or SSE (event-stream). Extract the JSON-RPC object.
+    payload = raw
+    if "text/event-stream" in (resp.headers.get("Content-Type") or "") or raw.lstrip().startswith("event:"):
+        for line in raw.splitlines():
+            if line.startswith("data:"):
+                payload = line[5:].strip()
+    obj = json.loads(payload)
+    if "error" in obj:
+        raise RuntimeError(f"MCP error on {method}: {obj['error']}")
+    return obj.get("result")
+
+
+def main():
+    _rpc("initialize", {
+        "protocolVersion": "2025-06-18",
+        "capabilities": {},
+        "clientInfo": {"name": "hncb-t4-test", "version": "1.0"},
+    })
+    _rpc("notifications/initialized", notify=True)
+
+    tools = _rpc("tools/list").get("tools", [])
+    names = [t["name"] for t in tools]
+    print("tools exposed by gateway:", names)
+    # Gateway namespaces tool names (e.g. rag___get_customer_activity_summary).
+    tool = next((n for n in names if n.endswith("get_customer_activity_summary")), None)
+    if not tool:
+        sys.exit("get_customer_activity_summary not found on the gateway")
+
+    result = _rpc("tools/call", {"name": tool, "arguments": {"customer_token": TOKEN}})
+    print(f"\n=== evidence package for {TOKEN} (via Gateway MCP) ===")
+    # MCP tool results come back as content blocks; print text blocks.
+    for block in result.get("content", []):
+        if block.get("type") == "text":
+            try:
+                print(json.dumps(json.loads(block["text"]), indent=2, ensure_ascii=False))
+            except json.JSONDecodeError:
+                print(block["text"])
+
+
+if __name__ == "__main__":
+    main()
--- a/scripts/teardown.sh
+++ b/scripts/teardown.sh
@@ -6,14 +6,47 @@ cd "$(dirname "$0")/.."

 REGION="${REGION:-us-east-1}"

-echo "==> Delete AgentCore resources (adjust names to what agentcore_setup created)"
-echo "    agentcore runtime delete --name hncb-advisor-agent --region ${REGION} || true"
-echo "    agentcore gateway delete --name hncb-rag-gateway   --region ${REGION} || true"
+echo "==> Delete AgentCore Gateway + targets (not in Terraform state)"
+export AWS_PAGER=""
+GW_ID="$(aws bedrock-agentcore-control list-gateways --region "$REGION" \
+  --query "items[?name=='hncb-rag-gateway'].gatewayId | [0]" --output text 2>/dev/null || echo None)"
+if [ "$GW_ID" != "None" ] && [ -n "$GW_ID" ]; then
+  for TID in $(aws bedrock-agentcore-control list-gateway-targets --region "$REGION" \
+      --gateway-identifier "$GW_ID" --query 'items[].targetId' --output text 2>/dev/null); do
+    aws bedrock-agentcore-control delete-gateway-target --region "$REGION" \
+      --gateway-identifier "$GW_ID" --target-id "$TID" || true
+  done
+  aws bedrock-agentcore-control delete-gateway --region "$REGION" --gateway-identifier "$GW_ID" || true
+  echo "    deleted gateway $GW_ID"
+fi
+echo "==> Delete AgentCore Runtime + Memory (T5; not in Terraform state)"
+for RT in $(aws bedrock-agentcore-control list-agent-runtimes --region "$REGION" \
+    --query "agentRuntimes[?starts_with(agentRuntimeName,'hncb')].agentRuntimeId" --output text 2>/dev/null); do
+  aws bedrock-agentcore-control delete-agent-runtime --region "$REGION" --agent-runtime-id "$RT" || true
+  echo "    deleted runtime $RT"
+done
+for MEM in $(aws bedrock-agentcore-control list-memories --region "$REGION" \
+    --query "memories[?starts_with(id,'hncb')].id" --output text 2>/dev/null); do
+  aws bedrock-agentcore-control delete-memory --region "$REGION" --memory-id "$MEM" || true
+  echo "    deleted memory $MEM"
+done
+echo "    (agentcore launch also auto-created an ECR repo, a CodeBuild project, and an"
+echo "     AmazonBedrockAgentCoreSDKRuntime-* execution role -- delete by hand if desired.)"

 echo "==> Terraform destroy (DynamoDB, Lambda, ECS, IAM)"
 terraform -chdir=terraform destroy -auto-approve

-echo "==> (optional) delete the Presidio ECR repo"
-echo "    aws ecr delete-repository --repository-name hncb-presidio --force --region ${REGION} || true"
+echo "==> Delete ECR repos (Presidio + the agentcore-built agent image; not in TF state)"
+for REPO in hncb-presidio bedrock-agentcore-hncb_advisor_agent; do
+  aws ecr delete-repository --repository-name "$REPO" --force --region "$REGION" 2>/dev/null \
+    && echo "    deleted ECR $REPO" || true
+done
+echo "    (also free/leftover: the AmazonBedrockAgentCoreSDKRuntime-* role and a CodeBuild"
+echo "     project agentcore created -- no cost idle; delete by hand if you want it spotless.)"

-echo "Teardown complete (mind any S3/CloudFront you created for the UI)."
+echo "==> (manual) default VPC created by hand for the T1 live deploy (ap-southeast-1)"
+echo "    Not in Terraform state; tagged maintenance=manual-cleanup-required. Free while"
+echo "    empty, so usually leave it. To remove: aws ec2 delete-vpc --vpc-id vpc-0e97f4fdb643c3e26"
+
+echo "Teardown complete. (UI S3+CloudFront are Terraform-managed now, so the destroy"
+echo " above removes them; the S3 bucket has force_destroy=true.)"