Verifier 与二值奖励 深度解析

118def build_binary_final_reward(
119    reward: dict[str, Any],
120    *,
121    verifier_exit: int,
122    agent_exec_returncode: int | None,
123    agent_early_terminated: bool = False,
124) -> dict[str, Any]:
125    """Convert raw verifier output into the comparable benchmark final reward.
126
127    ``verifier/reward.json`` may keep continuous diagnostic scores. The
128    externally comparable final reward is binary: every required validation
129    check must pass, otherwise reward/score are 0.
130    """
131    final_reward = dict(reward)
132    raw_score = _score_or_none(reward.get("raw_score"))
133    raw_reward = _score_or_none(reward.get("raw_reward"))
134    if raw_score is None:
135        raw_score = _score_or_none(reward.get("score", reward.get("reward")))
136    if raw_reward is None:
137        raw_reward = _score_or_none(reward.get("reward", reward.get("score")))
138    raw_passed = bool(reward.get("passed"))
139
140    verifier_completed = verifier_exit in (0, 6)
141    verifier_passed = raw_passed and verifier_completed
142    agent_execution_passed = (
143        agent_exec_returncode in (0, None)
144        or agent_early_terminated
145        or verifier_passed
146    )
147    if agent_exec_returncode in (0, None) or agent_early_terminated:
148        agent_execution_reason = "agent process completed successfully"
149    elif verifier_passed:
150        agent_execution_reason = (
151            f"agent_exec_returncode={agent_exec_returncode}; verifier passed, "
152            "treating agent process status as diagnostic"
153        )
154    else:
155        agent_execution_reason = f"agent_exec_returncode={agent_exec_returncode}"
156
157    if _is_v2_reward_schema(reward):
158        # v2: the raw test.sh check list IS the source of truth. We do NOT
159        # add framework meta-checks (agent_execution / verifier_completed /
160        # script_verifier_or_llm) on top — those gated v1 scoring and made
161        # every report show "x/3" regardless of how many real checks the
162        # task had. Binary passed simply requires every raw check to pass
163        # AND the verifier to have completed (so the raw data is trustable).
164        # An agent crash typically leaves outputs missing and trips raw
165        # checks naturally, so we don't gate on agent_execution explicitly.
166        breakdown = reward.get("checks_breakdown")
167        checks = [
168            {
169                "id": c.get("id"),
170                "passed": bool(c.get("passed")),
171                "reason": c.get("reason"),
172                "check_type": c.get("check_type"),
173            }
174            for c in (breakdown if isinstance(breakdown, list) else [])
175            if isinstance(c, dict)
176        ]
177        passed_checks = sum(1 for c in checks if c.get("passed"))
178        total_checks = len(checks)
179        all_passed = total_checks > 0 and passed_checks == total_checks and verifier_completed
180        check_ratio = round(passed_checks / total_checks, 4) if total_checks else 0.0
181        raw_score = check_ratio
182        raw_reward = check_ratio
183        raw_passed = total_checks > 0 and passed_checks == total_checks
184        llm_check = None
185    else:
186        checks = [
187            {
188                "id": "agent_execution",
189                "passed": agent_execution_passed,
190                "reason": agent_execution_reason,
191            },
192            {
193                "id": "verifier_completed",
194                "passed": verifier_completed,
195                "reason": f"verifier_exit={verifier_exit}",
196            },
197        ]
198        llm_check = _llm_rubric_check(reward, raw_score)
199        if llm_check is not None:
200            checks.append(llm_check)
201        else:
202            checks.append(_script_verifier_check(reward, raw_score, raw_passed))
203        checks.extend(_explicit_validation_checks(reward))
204        hard_checks = reward.get("hard_checks")
205        if isinstance(hard_checks, dict) and "passable" in hard_checks:
206            checks.append(
207                {
208                    "id": "hard_checks_passable",
209                    "passed": bool(hard_checks.get("passable")),
210                    "score": hard_checks.get("score"),
211                    "reason": "; ".join(str(item) for item in hard_checks.get("cap_reasons", []) or []),
212                }
213            )
214        passed_checks = sum(1 for check in checks if check.get("passed"))
215        total_checks = len(checks)
216        all_passed = total_checks > 0 and passed_checks == total_checks
217
218    final_reward["raw_reward"] = raw_reward
219    final_reward["raw_score"] = raw_score
220    final_reward["raw_passed"] = raw_passed
221    if llm_check is not None:
222        final_reward["llm_rubric_score"] = llm_check.get("score")
223        final_reward["llm_rubric_threshold"] = llm_check.get("threshold")
224        final_reward["llm_rubric_passed"] = llm_check.get("passed")
225    final_reward["reward_mode"] = "binary_all_checks"
226    final_reward["validation_checks"] = checks
227    final_reward["check_summary"] = {
228        "total": total_checks,
229        "passed": passed_checks,
230        "failed": total_checks - passed_checks,
231        "all_passed": all_passed,
232    }
233    final_reward["framework_signals"] = {
234        "agent_execution_passed": agent_execution_passed,
235        "agent_execution_reason": agent_execution_reason,
236        "verifier_completed": verifier_completed,
237        "verifier_exit": verifier_exit,
238        "agent_exec_returncode": agent_exec_returncode,
239        "agent_early_terminated": agent_early_terminated,
240    }
241    final_reward["passed"] = all_passed
242    final_reward["reward"] = 1.0 if all_passed else 0.0
243    final_reward["score"] = final_reward["reward"]
244    final_reward.setdefault("verifier_exit", verifier_exit)
245    return final_reward

122def validate_tasks(check: Validation) -> None:
123    task_by_id: dict[str, Path] = {}
124    modality_ids: dict[str, set[str]] = {
125        "text_only": set(),
126        "browser_textcapable": set(),
127        "vision": set(),
128    }
129
130    for task_toml in sorted(DATASETS.rglob("task.toml")):
131        payload = tomllib.loads(task_toml.read_text(encoding="utf-8"))
132        task = payload.get("task") or {}
133        environment = payload.get("environment") or {}
134        task_id = task.get("id")
135        rel = task_toml.parent.relative_to(ROOT)
136        check.require(isinstance(task_id, str) and bool(task_id), f"{rel}: missing [task].id")
137        if not isinstance(task_id, str) or not task_id:
138            continue
139        check.require(task_id not in task_by_id, f"duplicate task id: {task_id}")
140        task_by_id[task_id] = task_toml.parent
141
142        for required in ("task.md", "rubric.json", "grader/run.sh"):
143            check.require((task_toml.parent / required).is_file(), f"{rel}: missing {required}")
144
145        check.require(
146            environment.get("base_images") == [PINNED_RUNTIME],
147            f"{rel}: [environment].base_images must contain only the public pinned runtime",
148        )
149
150        if bool(environment.get("requires_vision", False)):
151            modality = "vision"
152        elif bool(environment.get("requires_browser", False)):
153            modality = "browser_textcapable"
154        else:
155            modality = "text_only"
156        modality_ids[modality].add(task_id)
157
158    all_ids = load_collection(COLLECTIONS["all"])
159    check.require(
160        len(all_ids) == EXPECTED_TASK_COUNT,
161        f"full collection has {len(all_ids)} tasks; expected {EXPECTED_TASK_COUNT}",
162    )
163    check.require(len(all_ids) == len(set(all_ids)), "full collection contains duplicate task ids")
164    check.require(
165        set(all_ids) == set(task_by_id),
166        "full collection and discovered task.toml ids differ",
167    )
168
169    for modality, expected_ids in modality_ids.items():
170        actual = load_collection(COLLECTIONS[modality])
171        check.require(len(actual) == len(set(actual)), f"{modality} collection contains duplicates")
172        check.require(
173            set(actual) == expected_ids,
174            f"{modality} collection does not match task.toml modality declarations",
175        )
176
177    union = set().union(*modality_ids.values())
178    total_memberships = sum(len(items) for items in modality_ids.values())
179    check.require(union == set(all_ids), "modality collections do not cover the full collection")
180    check.require(
181        total_memberships == len(union),
182        "modality collections overlap; every task must belong to exactly one slice",
183    )
184
185    counts = {name: len(ids) for name, ids in modality_ids.items()}
186    print(
187        "tasks:"
188        f" all={len(all_ids)}"
189        f" text_only={counts['text_only']}"
190        f" browser_textcapable={counts['browser_textcapable']}"
191        f" vision={counts['vision']}"
192    )

129class PublicJudgeApiTests(unittest.TestCase):
130    def test_gemini_native_protocol_and_versioned_base_url(self) -> None:
131        response = {
132            "candidates": [{"content": {"parts": [{"text": json.dumps(JUDGMENT)}]}}]
133        }
134        with _FakeEndpoint(response) as endpoint:
135            result = gemini_judge(
136                JudgeConfig(
137                    provider="gemini",
138                    model="gemini-3.1-pro-preview",
139                    api_key="test-gemini-key",
140                    base_url=f"{endpoint.base_url}/v1beta",
141                ),
142                "grade this",
143            )
144
145        self.assertTrue(result["passed"])
146        self.assertEqual(len(endpoint.requests), 1)
147        request = endpoint.requests[0]
148        self.assertEqual(
149            request["path"],
150            "/v1beta/models/gemini-3.1-pro-preview:generateContent",
151        )
152        headers = {str(k).lower(): v for k, v in request["headers"].items()}  # type: ignore[union-attr]
153        self.assertEqual(headers["x-goog-api-key"], "test-gemini-key")
154        body = request["body"]
155        self.assertEqual(body["contents"][0]["parts"][0]["text"], "grade this")  # type: ignore[index]
156        self.assertEqual(
157            body["generationConfig"]["responseMimeType"],  # type: ignore[index]
158            "application/json",
159        )
160
161    def test_openai_responses_protocol_and_multimodal_input(self) -> None:
162        with _FakeEndpoint({"output_text": json.dumps(JUDGMENT)}) as endpoint:
163            result = openai_judge(
164                JudgeConfig(
165                    provider="openai",
166                    model="test-model",
167                    api_key="test-openai-key",
168                    base_url=f"{endpoint.base_url}/v1",
169                ),
170                "grade this",
171                [
172                    {"text": "artifact image"},
173                    {
174                        "inlineData": {
175                            "mimeType": "image/png",
176                            "data": "aGVsbG8=",
177                        }
178                    },
179                ],
180            )
181
182        self.assertTrue(result["passed"])
183        request = endpoint.requests[0]
184        self.assertEqual(request["path"], "/v1/responses")
185        headers = {str(k).lower(): v for k, v in request["headers"].items()}  # type: ignore[union-attr]
186        self.assertEqual(headers["authorization"], "Bearer test-openai-key")
187        body = request["body"]
188        user_content = body["input"][1]["content"]  # type: ignore[index]
189        self.assertEqual(user_content[0], {"type": "input_text", "text": "grade this"})
190        self.assertEqual(user_content[1], {"type": "input_text", "text": "artifact image"})
191        self.assertEqual(
192            user_content[2],
193            {
194                "type": "input_image",
195                "image_url": "data:image/png;base64,aGVsbG8=",
196            },
197        )
198
199
200class PublicHarnessContractTests(unittest.TestCase):
201    def test_public_native_provider_configs_pin_api_versions(self) -> None:
202        config_root = Path(__file__).resolve().parents[1] / "configs"
203        for name in (
204            "realreplicabench_native_google_direct_models.json",
205            "realreplicabench_native_google_models.json",
206        ):
207            payload = json.loads((config_root / name).read_text(encoding="utf-8"))
208            self.assertEqual(
209                payload["providers"]["google"]["baseUrl"],
210                "https://generativelanguage.googleapis.com/v1beta",
211            )
212        qwen = json.loads(
213            (config_root / "realreplicabench_qwen37plus_native_models.json").read_text(
214                encoding="utf-8"
215            )
216        )
217        self.assertEqual(
218            qwen["providers"]["qwen"]["baseUrl"],
219            "https://dashscope-intl.aliyuncs.com/compatible-mode/v1",
220        )
221
222    def test_byo_endpoint_preset_configs_pin_wire_formats(self) -> None:
223        # Each BYO-endpoint preset is a starter template for a distinct
224        # wire format the OpenClaw runtime speaks natively. A future edit
225        # that breaks any of these contracts (wrong `api` enum, wrong
226        # default baseUrl, dropped anthropic-version header) should trip
227        # this test rather than silently ship a broken preset.
228        config_root = Path(__file__).resolve().parents[1] / "configs"
229
230        openai_chat = json.loads(
231            (config_root / "realreplicabench_openai_chat_models.json").read_text(
232                encoding="utf-8"
233            )
234        )
235        self.assertEqual(openai_chat["providers"]["openai"]["api"], "openai-completions")
236        self.assertEqual(
237            openai_chat["providers"]["openai"]["baseUrl"],
238            "https://api.openai.com/v1",
239        )