demo: add scripts/prove.sh (the money shot)
Runs a de-id round trip then searches the live AgentCore/Bedrock trace to show the token appears and the real name has 0 hits — proof the cloud only ever saw a token. `--last` proves against recent trace without a new round trip (snappy for running right after the UI demo). Discovers the runtime log group automatically. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
74
scripts/prove.sh
Executable file
74
scripts/prove.sh
Executable file
@@ -0,0 +1,74 @@
|
||||
#!/usr/bin/env bash
|
||||
# Demo "money shot": prove the cloud (Bedrock/AgentCore) only ever saw a TOKEN,
|
||||
# never the real name.
|
||||
#
|
||||
# scripts/prove.sh run a fresh round trip, then prove it
|
||||
# scripts/prove.sh --last skip the round trip; prove against the last few
|
||||
# minutes of trace (snappy: run right after the UI demo)
|
||||
#
|
||||
# Env overrides: REGION, ENDPOINT (round-trip URL), NAME (the PII to hunt for).
|
||||
set -uo pipefail
|
||||
|
||||
REGION="${REGION:-ap-southeast-1}"
|
||||
ENDPOINT="${ENDPOINT:-https://hncb-deid.apim-apac-demo.com/tokenize}"
|
||||
NAME="${NAME:-王小明}"
|
||||
QUERY="${QUERY:-請幫我整理${NAME}最近三個月的理財往來,並給我下次拜訪話術。}"
|
||||
export AWS_PAGER=""
|
||||
|
||||
# Discover the AgentCore runtime trace log group (no hard-coded runtime id).
|
||||
LG=$(aws logs describe-log-groups --region "$REGION" \
|
||||
--log-group-name-prefix /aws/bedrock-agentcore/runtimes/ \
|
||||
--query "logGroups[?contains(logGroupName,'hncb') && ends_with(logGroupName,'-DEFAULT')].logGroupName | [0]" \
|
||||
--output text 2>/dev/null)
|
||||
if [ -z "$LG" ] || [ "$LG" = "None" ]; then
|
||||
echo "✗ couldn't find the AgentCore runtime log group (is the runtime deployed?)"; exit 1
|
||||
fi
|
||||
|
||||
# Count trace events (since START ms) whose message contains $1.
|
||||
count() {
|
||||
aws logs filter-log-events --region "$REGION" --log-group-name "$LG" \
|
||||
--start-time "$1" --filter-pattern "$2" --no-paginate \
|
||||
--query 'events[].eventId' --output text 2>/dev/null | tr '\t' '\n' | grep -c . || true
|
||||
}
|
||||
|
||||
if [ "${1:-}" = "--last" ]; then
|
||||
START=$(( ($(date +%s) - 300) * 1000 ))
|
||||
TOKEN=$(aws logs filter-log-events --region "$REGION" --log-group-name "$LG" \
|
||||
--start-time "$START" --filter-pattern 'CUST_' --no-paginate \
|
||||
--query 'events[*].message' --output text 2>/dev/null | grep -oE 'CUST_[0-9]+' | head -1)
|
||||
else
|
||||
echo "▶ Advisor asks about ${NAME}"
|
||||
echo " \"${QUERY}\""
|
||||
echo
|
||||
echo "▶ Running the de-identification round trip through the gateway …"
|
||||
START=$(( $(date +%s) * 1000 - 3000 ))
|
||||
RESP=$(curl -s -m 60 -X POST "$ENDPOINT" -H 'content-type: application/json' \
|
||||
-d "{\"query\":\"${QUERY}\"}")
|
||||
TOKEN=$(printf '%s' "$RESP" | grep -oE 'CUST_[0-9]+' | head -1)
|
||||
if [ -z "$TOKEN" ]; then echo " ✗ no token minted (detector miss?) — re-run"; exit 1; fi
|
||||
echo " ✓ ${NAME} → ${TOKEN} (this token is all that left for the cloud)"
|
||||
echo " ✓ advisor got talking points; identity restored on-prem"
|
||||
fi
|
||||
[ -z "$TOKEN" ] && { echo "✗ no token seen in the recent trace — run a query first"; exit 1; }
|
||||
|
||||
echo
|
||||
echo "▶ Proof — searching the cloud (Bedrock/AgentCore) trace:"
|
||||
|
||||
# Wait for the trace to flush (token to show up), then hunt for the name.
|
||||
THITS=0
|
||||
for _ in $(seq 1 15); do
|
||||
THITS=$(count "$START" "\"$TOKEN\"")
|
||||
[ "${THITS:-0}" -gt 0 ] && break
|
||||
sleep 4
|
||||
done
|
||||
NHITS=$(count "$START" "\"$NAME\"")
|
||||
|
||||
printf " token %-14s → %s hits ← the model reasoned on this\n" "$TOKEN" "$THITS"
|
||||
printf " name %-14s → %s hits ← the model NEVER saw it\n" "$NAME" "$NHITS"
|
||||
echo
|
||||
if [ "${THITS:-0}" -gt 0 ] && [ "${NHITS:-0}" -eq 0 ]; then
|
||||
echo " ✅ The cloud only ever handled a token. PII never left the on-prem zone."
|
||||
else
|
||||
echo " ⚠ trace still settling (token=$THITS, name=$NHITS). Give it a few seconds and re-run:"
|
||||
echo " scripts/prove.sh --last"
|
||||
fi
|
||||
Reference in New Issue
Block a user