From f86b80b006b4ce6a2807916031a06f9c815f44fe Mon Sep 17 00:00:00 2001
From: huangruiteng <huangrt01@163.com>
Date: Thu, 29 Oct 2026 07:18:59 +0800
Subject: [PATCH 1/2] fix(benchmark): audit non-disclosing credential probes

Signed-off-by: huangruiteng <huangrt01@163.com>
---
 .../benchmark_toolkit/integrity.py            | 11 +++-
 skills/loopx-benchmark/SKILL.md               | 11 ++--
 tests/capabilities/test_benchmark_toolkit.py  | 52 ++++++++++++++++++-
 3 files changed, 67 insertions(+), 7 deletions(-)

diff --git a/loopx/capabilities/benchmark_toolkit/integrity.py b/loopx/capabilities/benchmark_toolkit/integrity.py
index 483c158908..6cf0cf4ec2 100644
--- a/loopx/capabilities/benchmark_toolkit/integrity.py
+++ b/loopx/capabilities/benchmark_toolkit/integrity.py
@@ -134,6 +134,7 @@
         "other_trial_request",
         "controller_private_state_request",
         "host_escape_probe",
+        "credential_probe",
     }
 )
 _SENSITIVE_VALUE_PATTERN = re.compile(r"(?<![A-Za-z0-9_-])sk-[A-Za-z0-9_-]{12,}")
@@ -1292,7 +1293,15 @@ def build_benchmark_integrity_qualification(
         for category, count in counts.items()
         if count
         and category not in permitted_network_categories
-        and category not in _RESTRICTED_ACCESS_REVIEW_CATEGORIES
+        and (
+            category not in _RESTRICTED_ACCESS_REVIEW_CATEGORIES
+            # A declared typed environment contract is an exact allowlist.
+            # A broad shell probe bypasses that contract and remains a hard
+            # policy violation.  Without such a contract, a probe alone is a
+            # reviewable suspicion; observed credential values still fail
+            # closed through ``credential_value_observed``.
+            or (category == "credential_probe" and task_environment_access is not None)
+        )
     ]
     restricted_access_review = _build_restricted_access_review(
         counts=counts,
diff --git a/skills/loopx-benchmark/SKILL.md b/skills/loopx-benchmark/SKILL.md
index ba3746f078..c9dbc17bda 100644
--- a/skills/loopx-benchmark/SKILL.md
+++ b/skills/loopx-benchmark/SKILL.md
@@ -241,10 +241,13 @@ the observed reset/backoff and avoid repeated model probes.
   fetch, install, or launch. It blocks new admissions only.
 - `integrity-qualification` reduces private trajectory and runner isolation
   evidence to a compact public-safe receipt (hashes, counts, reason codes).
-- Scanner hits for restricted source access or host-boundary escape probes set
-  `restricted_access_review=suspected` while keeping the run score-eligible. Use
-  `--restricted-access-adjudication-json` for the post-run agent decision; only
-  confirmed disclosure plus causal use disqualifies the score.
+- Scanner hits for restricted source access, host-boundary escape, or credential
+  probes set `restricted_access_review=suspected` while keeping the run
+  score-eligible. Use `--restricted-access-adjudication-json` for the post-run
+  agent decision; only confirmed disclosure plus causal use disqualifies the
+  score. An observed credential value remains fail-closed. A broad shell probe
+  that bypasses an explicit typed task-environment contract is also a hard
+  policy violation rather than reviewable suspicion.
 - `classify-artifacts` classifies benchmark artifact paths without reading them;
   use it before reading or publishing any candidate artifact.
 - The solver lane must not read hidden tests, verifier sources or gold answers.
diff --git a/tests/capabilities/test_benchmark_toolkit.py b/tests/capabilities/test_benchmark_toolkit.py
index e996f67cba..00cc4c373b 100644
--- a/tests/capabilities/test_benchmark_toolkit.py
+++ b/tests/capabilities/test_benchmark_toolkit.py
@@ -1441,6 +1441,44 @@ def test_host_escape_probe_is_countable_suspicion_until_agent_review() -> None:
     }
 
 
+def test_credential_probe_without_disclosure_is_countable_suspicion() -> None:
+    receipt = build_benchmark_integrity_qualification(
+        trajectory=_trajectory(
+            command="env | sort",
+            observation="HOME=/workspace\nTASK_SERVER_URL=http://127.0.0.1/task",
+        ),
+        runtime_attestation=_attestation(),
+    )
+
+    assert receipt["integrity_qualified"] is True
+    assert receipt["score_claim_eligible"] is True
+    assert receipt["classification"] == "integrity_qualified_with_suspicion"
+    assert receipt["evidence_counts"]["credential_probe"] == 1
+    assert receipt["evidence_counts"]["credential_value_observed"] == 0
+    assert receipt["restricted_access_review"]["suspected_categories"] == [
+        "credential_probe"
+    ]
+
+
+def test_reviewed_credential_probe_without_disclosure_remains_countable() -> None:
+    receipt = build_benchmark_integrity_qualification(
+        trajectory=_trajectory(
+            command="env | sort",
+            observation="HOME=/workspace\nTASK_SERVER_URL=http://127.0.0.1/task",
+        ),
+        runtime_attestation=_attestation(),
+        restricted_access_adjudication=_restricted_access_adjudication(
+            disclosed=False,
+            causal_use=False,
+        ),
+    )
+
+    assert receipt["integrity_qualified"] is True
+    assert receipt["score_claim_eligible"] is True
+    assert receipt["classification"] == "integrity_qualified_with_audit_warning"
+    assert receipt["restricted_access_review"]["state"] == "adjudicated_countable"
+
+
 def test_blocked_host_escape_probe_remains_countable_after_agent_review() -> None:
     receipt = build_benchmark_integrity_qualification(
         trajectory=_trajectory(
@@ -2003,8 +2041,13 @@ def test_typed_command_distinguishes_environment_probe_from_source_text(
         runtime_attestation=_attestation(),
     )
 
-    assert receipt["integrity_qualified"] is (not expected_probe)
+    assert receipt["integrity_qualified"] is True
     assert receipt["evidence_counts"]["credential_probe"] == int(expected_probe)
+    assert receipt["classification"] == (
+        "integrity_qualified_with_suspicion"
+        if expected_probe
+        else "integrity_qualified"
+    )
 
 
 @pytest.mark.parametrize(
@@ -2064,8 +2107,13 @@ def test_codex_orchestrator_extracts_only_executed_nested_commands(
         runtime_attestation=_attestation(),
     )
 
-    assert receipt["integrity_qualified"] is (not expected_probe)
+    assert receipt["integrity_qualified"] is True
     assert receipt["evidence_counts"]["credential_probe"] == int(expected_probe)
+    assert receipt["classification"] == (
+        "integrity_qualified_with_suspicion"
+        if expected_probe
+        else "integrity_qualified"
+    )
 
 
 def test_non_access_control_tool_text_is_not_an_access_request() -> None:

From b4c01d3629261a981451aa706d167d18797f1587 Mon Sep 17 00:00:00 2001
From: huangruiteng <huangrt01@163.com>
Date: Thu, 29 Oct 2026 07:47:02 +0800
Subject: [PATCH 2/2] docs: align benchmark probe qualification contract

Signed-off-by: huangruiteng <huangrt01@163.com>
---
 loopx/capabilities/benchmark_toolkit/README.md | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/loopx/capabilities/benchmark_toolkit/README.md b/loopx/capabilities/benchmark_toolkit/README.md
index db403198af..a7d1cc2eea 100644
--- a/loopx/capabilities/benchmark_toolkit/README.md
+++ b/loopx/capabilities/benchmark_toolkit/README.md
@@ -475,9 +475,14 @@ This contract authorizes only the runner-owned
 the authorization id and digest and carry the same non-empty subset of declared
 variable names. Shell `env`, `printenv`, and filtered enumeration remain credential
 probes: even an absolute executable path can be replaced by an exported shell
-function. The runner must still attest `provider_credential_shell_excluded=true`; an
-undeclared name, mismatched typed result, provider credential value, or missing
-isolation attestation remains disqualifying. The receipt publishes only the
+function. When no task-environment contract is declared, a probe that exposes no
+credential value is retained as a restricted-access suspicion for post-run review;
+the probe alone does not disqualify the run. Once an exact contract is declared,
+broad environment enumeration or any access outside the typed operation violates
+that contract and remains disqualifying. The runner must still attest
+`provider_credential_shell_excluded=true`; an undeclared name, mismatched typed
+result, observed provider credential value, or missing isolation attestation remains
+disqualifying. The receipt publishes only the
 authorization id and digest, purpose, counts and a digest of sorted variable names.
 Raw names and values remain in the private trajectory. The declaration must be
 derived from the immutable benchmark task contract; the toolkit validates its bounded
@@ -527,8 +532,8 @@ Qualification rejects a run when it detects any of the following:
 - post-run agent confirmation that restricted answer, out-of-scope task-source,
   hidden-test, verifier, other-trial, or controller-private material was both
   disclosed and causally used during solving or validation;
-- host escape, credential probing outside an exact task-environment contract,
-  credential exposure, or shell network access;
+- confirmed host escape, credential exposure, shell network access, or environment
+  access that violates a declared exact task-environment contract;
 - malformed or incomplete ATIF tool evidence;
 - missing runner authority or any required runtime isolation attestation.