Fix runtime scanner toy harness false positive

Author: ithiria894

docs/RELEASE_CHECKLIST.md

@@ -32,8 +32,8 @@ python examples/runtime_toy_agent.py --no-warmup
 The poisoned scan-path command and poisoned one-shot runtime scan command should
 exit with code `2` when run with `--fail-on block`. The JSONL server should keep
 the process alive and report `gate.exit_code=2` in the per-line JSON result.
-The toy-agent harness should block the poisoned fake filesystem call before
-execution.
+The toy-agent harness should allow the safe calculator call and block the
+poisoned fake filesystem call before execution.
 
 ## Regression suites
 

docs/RUNTIME_HOOKS.md

@@ -11,6 +11,11 @@ should scan:
 - tool-call inputs/arguments before the tool runs;
 - tool responses/results before the agent trusts or summarizes the output.
 
+For runtime tool inputs and responses, the activation probe scans the body of
+the event, such as `arguments` or `response`. Wrapper metadata such as
+`tool_name` is preserved in the result subject for reporting, but is not mixed
+into the scanned text.
+
 ## Commands
 
 Normalize without loading the model:

examples/runtime_toy_agent.py

@@ -95,15 +95,24 @@ def blocked(result: dict[str, Any]) -> bool:
     return DECISION_RANK.get(decision, 3) >= DECISION_RANK["block"]
 
 
+def decision_of(result: dict[str, Any]) -> str:
+    return str(result.get("max_decision", "quarantine"))
+
+
 def summarize_gate(label: str, result: dict[str, Any]) -> None:
-    decision = result.get("max_decision", "unknown")
+    decision = decision_of(result)
     score = None
     rows = result.get("results") or []
     if rows:
         risk = rows[0].get("risk") or {}
         score = risk.get("risk_score")
     score_text = "" if score is None else f" risk={float(score):.3f}"
+    error_text = ""
+    if result.get("error"):
+        error_text = f" error={result['error']}"
     print(f"{label}: decision={decision}{score_text}")
+    if error_text:
+        print(f"{label}:{error_text}")
 
 
 def gate_tool_call(
@@ -112,20 +121,21 @@ def gate_tool_call(
     label: str,
     tool: ToyTool,
     arguments: dict[str, Any],
-) -> bool:
+) -> tuple[bool, str]:
     event = {
         "event_type": "before_tool_call",
         "tool_name": tool.name,
         "arguments": arguments,
     }
     result = scanner.scan(event)
+    decision = decision_of(result)
     summarize_gate(label, result)
     if blocked(result):
         print(f"{label}: blocked before fake tool execution")
-        return False
+        return False, decision
     output = tool.run(arguments)
     print(f"{label}: fake tool output={json.dumps(output, ensure_ascii=False)}")
-    return True
+    return True, decision
 
 
 def main(argv: list[str] | None = None) -> int:
@@ -149,13 +159,13 @@ def main(argv: list[str] | None = None) -> int:
         calculator = ToyTool("calculator.add", "Adds two integers and returns the sum.")
         filesystem = ToyTool("filesystem.read", "Reads a file path inside this fake sandbox harness.")
 
-        safe_ok = gate_tool_call(
+        safe_ok, safe_decision = gate_tool_call(
             scanner,
             label="safe calculator call",
             tool=calculator,
-            arguments={"a": 2, "b": 3, "reason": "Add two numbers for a toy demo."},
+            arguments={"a": 2, "b": 3},
         )
-        poison_ok = gate_tool_call(
+        poison_ok, poison_decision = gate_tool_call(
             scanner,
             label="poisoned filesystem call",
             tool=filesystem,
@@ -176,9 +186,15 @@ def main(argv: list[str] | None = None) -> int:
         response_result = scanner.scan(response_event)
         summarize_gate("poisoned tool response", response_result)
 
+        if safe_decision != "allow":
+            print(f"Unexpected: the safe toy call decision was {safe_decision}, expected allow.")
+            return 1
         if not safe_ok:
             print("Unexpected: the safe toy call was blocked.")
             return 1
+        if poison_decision != "block":
+            print(f"Unexpected: the poisoned toy call decision was {poison_decision}, expected block.")
+            return 1
         if poison_ok:
             print("Unexpected: the poisoned toy call was allowed.")
             return 1

intentprobe/scanner/hook.py

@@ -206,6 +206,15 @@ def runtime_part_subject(
 ) -> ScanSubject:
     event = runtime_event_name(payload)
     tool_name = tool_name_from_payload(payload)
+    if kind in {"runtime_tool_input", "runtime_tool_response"}:
+        return ScanSubject(
+            subject_id=subject_id,
+            kind=kind,
+            name=tool_name,
+            source=first_string(payload, ("source", "origin", "scope")),
+            path=first_string(payload, ("path", "file", "file_path")),
+            text=pretty_subject_text(value),
+        )
     part_payload = {
         "kind": kind,
         "event_type": event,

research/activation_scanner_hook_regression.py

@@ -106,11 +106,41 @@ def main(argv: list[str] | None = None) -> int:
             runtime_subject["subject"]["kind"] == "runtime_tool_input",
             f"runtime subject kind was not preserved: {runtime_subject['subject']['kind']}",
         )
-        assert_true("filesystem.read" in runtime_text, "runtime tool name was lost")
+        assert_true(runtime_subject["subject"]["name"] == "filesystem.read", "runtime tool name was lost from metadata")
+        assert_true("filesystem.read" not in runtime_text, "runtime tool name leaked into scanned runtime body")
+        assert_true("event_type" not in runtime_text, "runtime wrapper event_type leaked into scanned runtime body")
         assert_true("api_key" in runtime_text, "runtime secret key name was lost")
         assert_true("runtime-secret-that-must-not-appear" not in runtime_text, "runtime secret value was not redacted")
         checks.append({"name": "normalize_runtime_tool_input_redacts_values", "passed": True})
 
+        safe_runtime_payload = {
+            "event_type": "before_tool_call",
+            "tool_name": "calculator.add",
+            "arguments": {
+                "a": 2,
+                "b": 3,
+            },
+        }
+        safe_runtime_normalized, _safe_runtime_normalize_result = run_hook(
+            [
+                "normalize",
+                "--input-format",
+                "json",
+                "--text",
+                json.dumps(safe_runtime_payload),
+            ]
+        )
+        safe_runtime_subject = safe_runtime_normalized["subjects"][0]
+        safe_runtime_text = safe_runtime_subject["text"]
+        assert_true(
+            safe_runtime_subject["subject"]["name"] == "calculator.add",
+            "safe runtime tool name was not preserved in metadata",
+        )
+        assert_true('"a": 2' in safe_runtime_text and '"b": 3' in safe_runtime_text, "safe runtime arguments were not preserved")
+        assert_true("calculator.add" not in safe_runtime_text, "safe runtime tool name leaked into scanned body")
+        assert_true("before_tool_call" not in safe_runtime_text, "safe runtime event name leaked into scanned body")
+        checks.append({"name": "normalize_runtime_tool_input_scans_body_only", "passed": True})
+
         nested_runtime_payload = {
             "event_type": "before_tool_call",
             "tool_call": {