RealReplicaBench 真实副本 Agent Benchmark

328def load_task(task_id: str) -> TaskSpec:
329    case_dir = DATASETS_DIR / task_id
330    task_toml = case_dir / "task.toml"
331    if not task_toml.is_file():
332        for candidate in DATASETS_DIR.rglob("task.toml"):
333            cdata = tomllib.loads(candidate.read_text(encoding="utf-8"))
334            if cdata.get("task", {}).get("id") == task_id:
335                case_dir = candidate.parent
336                task_toml = candidate
337                break
338        else:
339            raise SystemExit(f"task not found: {task_id} (searched {DATASETS_DIR})")
340    data = tomllib.loads(task_toml.read_text(encoding="utf-8"))
341    task = data.get("task", {})
342    agent = data.get("agent", {})
343    verifier = data.get("verifier", {})
344    environment = data.get("environment", {})
345    host_port_raw = environment.get("host_published_port")
346    host_published_port: int | None = None
347    if host_port_raw is not None:
348        try:
349            host_published_port = int(host_port_raw)
350        except (TypeError, ValueError):
351            raise SystemExit(
352                f"task {task_id}: [environment].host_published_port must be an integer, got {host_port_raw!r}"
353            )
354        if host_published_port <= 0 or host_published_port > 65535:
355            raise SystemExit(
356                f"task {task_id}: [environment].host_published_port out of range: {host_published_port}"
357            )
358    runtime_mocks = _parse_runtime_mocks(task_id, environment)

358    runtime_mocks = _parse_runtime_mocks(task_id, environment)
359    implicit_host_port = None
360    runtime_mock_ports = [mock.port for mock in runtime_mocks if mock.port is not None]
361    if host_published_port is None and len(runtime_mock_ports) == 1:
362        implicit_host_port = runtime_mock_ports[0]
363    early_terminate: EarlyTerminate | None = None
364    et_raw = environment.get("early_terminate")
365    if et_raw is not None:
366        if not isinstance(et_raw, dict):
367            raise SystemExit(
368                f"task {task_id}: [environment.early_terminate] must be a table"
369            )
370        required = ("poll_path", "match_kind", "match_field", "match_value")
371        for key in required:
372            if key not in et_raw:
373                raise SystemExit(
374                    f"task {task_id}: [environment.early_terminate].{key} is required"
375                )
376        match_kind = str(et_raw["match_kind"])
377        if match_kind != "list_any_field_equals":
378            raise SystemExit(
379                f"task {task_id}: unsupported early_terminate.match_kind {match_kind!r}"
380            )
381        if host_published_port is None and implicit_host_port is None:
382            raise SystemExit(
383                f"task {task_id}: early_terminate requires host_published_port or exactly one runtime mock port"
384            )
385        early_terminate = EarlyTerminate(
386            poll_path=str(et_raw["poll_path"]),
387            poll_interval_sec=float(et_raw.get("poll_interval_sec", 5)),
388            match_kind=match_kind,
389            match_field=str(et_raw["match_field"]),
390            match_value=str(et_raw["match_value"]),
391            kill_pattern=str(et_raw.get("kill_pattern", "bun run runner.ts")),
392        )
393    output_file_early_terminate: OutputFileEarlyTerminate | None = None
394    output_et_raw = environment.get("output_file_early_terminate")
395    if output_et_raw is not None:
396        if not isinstance(output_et_raw, dict):
397            raise SystemExit(
398                f"task {task_id}: [environment.output_file_early_terminate] must be a table"
399            )
400        output_path = str(output_et_raw.get("output_path") or "")
401        if not output_path.startswith("/"):
402            raise SystemExit(
403                f"task {task_id}: [environment.output_file_early_terminate].output_path must be an absolute path"
404            )
405        output_file_early_terminate = OutputFileEarlyTerminate(
406            output_path=output_path,
407            poll_interval_sec=float(output_et_raw.get("poll_interval_sec", 2)),
408            kill_pattern=str(output_et_raw.get("kill_pattern", "bun run runner.ts")),
409        )
410    # Layout-aware default for the verifier entrypoint: the new layout uses
411    # grader/run.sh, the old (frozen v2) layout uses test.sh. The task prompt file
412    # is `task.md` in BOTH layouts. An explicit [task].entrypoint / [task].test in
413    # task.toml always wins over the detected default.
414    test_default = "grader/run.sh" if (case_dir / "grader" / "run.sh").is_file() else "test.sh"
415    return TaskSpec(
416        task_id=str(task.get("id") or task_id),
417        case_dir=case_dir,
418        name=str(task.get("name") or task_id),
419        language=str(task.get("language") or "zh-CN"),
420        timeout_sec=int(agent.get("timeout_sec", 900)),
421        max_actions=int(agent.get("max_actions", 6)),
422        verifier_timeout_sec=int(verifier.get("timeout_sec", 120)),
423        entrypoint=str(task.get("entrypoint") or "task.md"),
424        test_script=str(task.get("test") or test_default),
425        rubric=str(task.get("rubric") or "rubric.json"),
426        private_dir=(case_dir / "private") if (case_dir / "private").is_dir() else None,
427        mock_services=bool(environment.get("mock_services", False) or runtime_mocks),
428        host_published_port=host_published_port,
429        requires_browser=bool(environment.get("requires_browser", False)),
430        requires_vision=bool(environment.get("requires_vision", False)),
431        runtime_mocks=runtime_mocks,
432        early_terminate=early_terminate,
433        output_file_early_terminate=output_file_early_terminate,
434    )

481def copy_public_task_dir(src: Path, dst: Path) -> None:
482    """Copy the agent-visible task content, never exposing verifier-only data.
483
484    New layout (allowlist): the agent sees ONLY ``workspace/`` (its inputs) plus
485    the task statement ``task.md``. Everything verifier-only — ``grader/``,
486    ``services/`` (mock source), ``private/``, ``rubric.json``, ``task.toml`` —
487    lives outside ``workspace/`` and is structurally excluded by not copying it.
488    This replaces the old blocklist, where a mock/verifier filename missing from
489    the ignore tuple would leak answers to the agent.
490
491    Old layout (frozen v2, no ``grader/``/``workspace/``): fall back to the
492    historical blocklist over the whole task dir. Removed once v2 migrates (Phase 2).
493    """
494    workspace = src / "workspace"
495    # Detect the new layout by grader/ or workspace/ so browse-only tasks (no
496    # agent inputs → no workspace/) still take the allowlist path instead of
497    # falling through to the old blocklist (which would leak grader/ + services/).
498    is_new_layout = (src / "grader").is_dir() or workspace.is_dir()
499    if is_new_layout:
500        dst.mkdir(parents=True, exist_ok=True)
501        if workspace.is_dir():
502            shutil.copytree(
503                workspace,
504                dst / "workspace",
505                ignore=shutil.ignore_patterns(".DS_Store", "__pycache__", "*.pyc"),
506            )
507        task_md = src / "task.md"
508        if task_md.is_file():
509            shutil.copy2(task_md, dst / "task.md")
510        return
511    ignore = shutil.ignore_patterns(
512        ".DS_Store",
513        "fixtures",
514        "attachments",
515        "node_modules",
516        "private",
517        "__pycache__",
518        "*.pyc",
519        "rubric.json",
520        "test.sh",
521        "task.toml",
522        "source_metadata.json",
523        "REFERENCE_SOLUTION.py",
524        "_grader.py",
525        "host_deps.txt",
526        "package.json",
527        "package-lock.json",
528        "start_mock_services.sh",
529        "mock*.py",
530        "mock*.js",
531        "*server.py",
532        "*server.js",
533        "*.mjs",
534        "verify*.py",
535        "INPUTS_TO_BUILD.md",
536        "ASSET_INVENTORY.md",
537        "EXPECTED_BEHAVIOR.md",
538    )
539    shutil.copytree(src, dst, ignore=ignore)

48def parse_scalar(value: str) -> Any:
49    value = value.strip()
50    # Strip inline comments (` # ...`) outside of quotes. YAML treats `#`
51    # preceded by whitespace as a comment; without this our custom subset
52    # parser silently glued the comment onto the value, breaking numeric coercion.
53    if value and not (value.startswith('"') or value.startswith("'")):
54        hash_idx = value.find(" #")
55        if hash_idx != -1:
56            value = value[:hash_idx].rstrip()
57    if value in {"", "null", "~"}:
58        return None
59    if value in {"true", "True"}:
60        return True
61    if value in {"false", "False"}:
62        return False
63    if value.startswith('"') and value.endswith('"'):
64        return value[1:-1].replace('\\"', '"')
65    if value.startswith("'") and value.endswith("'"):
66        return value[1:-1].replace("''", "'")
67    if value.startswith("[") and value.endswith("]"):
68        inner = value[1:-1].strip()
69        if not inner:
70            return []
71        return [parse_scalar(part.strip()) for part in inner.split(",")]
72    try:
73        return int(value)
74    except ValueError:
75        pass
76    try:
77        return float(value)
78    except ValueError:
79        return value
80
81
82def load_yaml_subset(path: Path) -> dict[str, Any]:
83    if not path.is_file():
84        raise SystemExit(f"config file not found: {path}")
85    root: dict[str, Any] = {}
86    stack: list[tuple[int, Any]] = [(-1, root)]
87    last_key_at_indent: dict[int, tuple[Any, str]] = {}
88    for raw_line in path.read_text(encoding="utf-8").splitlines():
89        if not raw_line.strip() or raw_line.lstrip().startswith("#"):
90            continue
91        indent = len(raw_line) - len(raw_line.lstrip(" "))
92        line = raw_line.strip()
93        parent = stack[-1][1]
94        if line.startswith("- "):
95            while stack and indent < stack[-1][0]:
96                stack.pop()
97            parent = stack[-1][1]
98            if not isinstance(parent, list):
99                grand_parent, key = last_key_at_indent.get(indent, (None, None))
100                if grand_parent is None:
101                    raise SystemExit(f"invalid yaml list item: {raw_line}")
102                if isinstance(grand_parent.get(key), list):
103                    parent = grand_parent[key]
104                else:
105                    parent = []
106                    grand_parent[key] = parent
107                if not stack or stack[-1][1] is not parent:
108                    stack.append((indent, parent))
109            item_text = line[2:].strip()
110            if not item_text:
111                item: Any = {}
112                parent.append(item)
113                stack.append((indent, item))
114            elif ": " in item_text:
115                key, value = item_text.split(":", 1)
116                item = {key.strip(): parse_scalar(value)}
117                parent.append(item)
118                stack.append((indent, item))
119            else:
120                parent.append(parse_scalar(item_text))
121            continue
122        while stack and indent <= stack[-1][0]:
123            stack.pop()
124        parent = stack[-1][1]
125        if ":" not in line:
126            raise SystemExit(f"invalid yaml line: {raw_line}")
127        if not isinstance(parent, dict):
128            raise SystemExit(f"invalid yaml mapping line under list: {raw_line}")
129        key, value = line.split(":", 1)
130        key = key.strip()
131        value = value.strip()
132        if value:
133            parent[key] = parse_scalar(value)
134        else:
135            parent[key] = {}
136            last_key_at_indent[indent + 2] = (parent, key)
137            stack.append((indent, parent[key]))
138    return root

620def copy_runtime_mock_sources(runtime_mocks: tuple[RuntimeMockConfig, ...], dst: Path) -> None:
621    if not runtime_mocks:
622        return
623    copyable = [mock for mock in runtime_mocks if not _runtime_mock_uses_baked_source(mock)]
624    if not copyable:
625        return
626    dst.mkdir(parents=True, exist_ok=True)
627    for mock in copyable:
628        target = dst / mock.name
629        if target.exists():
630            shutil.rmtree(target)
631        shutil.copytree(mock.source_dir, target, ignore=_runtime_mock_ignore(mock))
632    if any(mock.kind == "cli" for mock in copyable):
633        bridge_src = PROJECT_ROOT / "real_replica_bench" / "mock_services" / "bench_bridge.py"
634        if not bridge_src.is_file():
635            raise SystemExit(f"CLI runtime mocks require missing bench bridge: {bridge_src}")
636        shutil.copy2(bridge_src, dst / "bench_bridge.py")

118def build_binary_final_reward(
119    reward: dict[str, Any],
120    *,
121    verifier_exit: int,
122    agent_exec_returncode: int | None,
123    agent_early_terminated: bool = False,
124) -> dict[str, Any]:
125    """Convert raw verifier output into the comparable benchmark final reward.
126
127    ``verifier/reward.json`` may keep continuous diagnostic scores. The
128    externally comparable final reward is binary: every required validation
129    check must pass, otherwise reward/score are 0.
130    """
131    final_reward = dict(reward)
132    raw_score = _score_or_none(reward.get("raw_score"))
133    raw_reward = _score_or_none(reward.get("raw_reward"))
134    if raw_score is None:
135        raw_score = _score_or_none(reward.get("score", reward.get("reward")))
136    if raw_reward is None:
137        raw_reward = _score_or_none(reward.get("reward", reward.get("score")))
138    raw_passed = bool(reward.get("passed"))
139
140    verifier_completed = verifier_exit in (0, 6)
141    verifier_passed = raw_passed and verifier_completed
142    agent_execution_passed = (
143        agent_exec_returncode in (0, None)
144        or agent_early_terminated
145        or verifier_passed
146    )
147    if agent_exec_returncode in (0, None) or agent_early_terminated:
148        agent_execution_reason = "agent process completed successfully"
149    elif verifier_passed:
150        agent_execution_reason = (
151            f"agent_exec_returncode={agent_exec_returncode}; verifier passed, "
152            "treating agent process status as diagnostic"
153        )
154    else:
155        agent_execution_reason = f"agent_exec_returncode={agent_exec_returncode}"
156
157    if _is_v2_reward_schema(reward):
158        # v2: the raw test.sh check list IS the source of truth. We do NOT
159        # add framework meta-checks (agent_execution / verifier_completed /
160        # script_verifier_or_llm) on top — those gated v1 scoring and made
161        # every report show "x/3" regardless of how many real checks the
162        # task had. Binary passed simply requires every raw check to pass
163        # AND the verifier to have completed (so the raw data is trustable).
164        # An agent crash typically leaves outputs missing and trips raw
165        # checks naturally, so we don't gate on agent_execution explicitly.
166        breakdown = reward.get("checks_breakdown")
167        checks = [
168            {
169                "id": c.get("id"),
170                "passed": bool(c.get("passed")),
171                "reason": c.get("reason"),
172                "check_type": c.get("check_type"),
173            }
174            for c in (breakdown if isinstance(breakdown, list) else [])
175            if isinstance(c, dict)
176        ]
177        passed_checks = sum(1 for c in checks if c.get("passed"))
178        total_checks = len(checks)
179        all_passed = total_checks > 0 and passed_checks == total_checks and verifier_completed
180        check_ratio = round(passed_checks / total_checks, 4) if total_checks else 0.0
181        raw_score = check_ratio
182        raw_reward = check_ratio
183        raw_passed = total_checks > 0 and passed_checks == total_checks
184        llm_check = None
185    else:
186        checks = [
187            {
188                "id": "agent_execution",
189                "passed": agent_execution_passed,
190                "reason": agent_execution_reason,
191            },
192            {
193                "id": "verifier_completed",
194                "passed": verifier_completed,
195                "reason": f"verifier_exit={verifier_exit}",
196            },
197        ]
198        llm_check = _llm_rubric_check(reward, raw_score)
199        if llm_check is not None:
200            checks.append(llm_check)
201        else:
202            checks.append(_script_verifier_check(reward, raw_score, raw_passed))
203        checks.extend(_explicit_validation_checks(reward))
204        hard_checks = reward.get("hard_checks")
205        if isinstance(hard_checks, dict) and "passable" in hard_checks:
206            checks.append(
207                {
208                    "id": "hard_checks_passable",
209                    "passed": bool(hard_checks.get("passable")),
210                    "score": hard_checks.get("score"),
211                    "reason": "; ".join(str(item) for item in hard_checks.get("cap_reasons", []) or []),
212                }
213            )
214        passed_checks = sum(1 for check in checks if check.get("passed"))
215        total_checks = len(checks)
216        all_passed = total_checks > 0 and passed_checks == total_checks
217
218    final_reward["raw_reward"] = raw_reward
219    final_reward["raw_score"] = raw_score
220    final_reward["raw_passed"] = raw_passed
221    if llm_check is not None:
222        final_reward["llm_rubric_score"] = llm_check.get("score")
223        final_reward["llm_rubric_threshold"] = llm_check.get("threshold")
224        final_reward["llm_rubric_passed"] = llm_check.get("passed")
225    final_reward["reward_mode"] = "binary_all_checks"
226    final_reward["validation_checks"] = checks
227    final_reward["check_summary"] = {
228        "total": total_checks,
229        "passed": passed_checks,
230        "failed": total_checks - passed_checks,
231        "all_passed": all_passed,
232    }
233    final_reward["framework_signals"] = {
234        "agent_execution_passed": agent_execution_passed,
235        "agent_execution_reason": agent_execution_reason,
236        "verifier_completed": verifier_completed,
237        "verifier_exit": verifier_exit,
238        "agent_exec_returncode": agent_exec_returncode,
239        "agent_early_terminated": agent_early_terminated,
240    }
241    final_reward["passed"] = all_passed
242    final_reward["reward"] = 1.0 if all_passed else 0.0
243    final_reward["score"] = final_reward["reward"]
244    final_reward.setdefault("verifier_exit", verifier_exit)
245    return final_reward

18def build_task_prompt(spec: TaskSpec) -> str:
19    prompt = (spec.case_dir / spec.entrypoint).read_text(encoding="utf-8")
20    remote_workdir = REMOTE_WORKSPACE_ROOT
21    # New layout exposes inputs under workspace/; old layout flattens them into
22    # the workdir root.
23    remote_input_dir = (
24        f"{remote_workdir}/workspace" if (spec.case_dir / "workspace").is_dir() else remote_workdir
25    )
26    remote_output_dir = f"{remote_workdir}/outputs"
27    workspace_contract = (
28        "本次工作的标准目录约定：\n"
29        f"- 任务输入目录：`{remote_input_dir}`。\n"
30        f"- 标准工作目录：`{remote_workdir}`。\n"
31        f"- 标准产物目录：`{remote_output_dir}`。\n"
32        "- 如需创建或修改任何本地文件，请在标准工作目录内完成，并把最终文件保存到标准产物目录。\n"
33        "- 不要把最终产物只保存在 `/tmp`、桌面、下载目录或其它临时路径；这些路径不会作为标准结果归档。\n\n"
34    )
35    if spec.runtime_mocks and "/mcp" in prompt:
36        workspace_contract += (
37            "MCP 工具约定：`accio-mcp-cli` 不可用；如任务要求使用本地 `/mcp` endpoint，"
38            "请直接用 `curl` 或等价命令发送 JSON-RPC `tools/list` / `tools/call` 请求。\n\n"
39        )
40    return workspace_contract + prompt

64# Autonomous-agent directive, prepended to every OpenClaw prompt in the most
65# prominent position. OpenClaw's default framing is a conversational personal
66# assistant, and persona-forward models (notably GPT-5.6 Luna/Terra/Sol) latch
67# onto it — they open with a "我刚上线，你希望怎么称呼我？" self-introduction plus a
68# question, end the turn with zero tool calls, and since `openclaw agent
69# --message` is a single non-interactive invocation there is no user to answer →
70# the loop terminates and the task scores ~0. Reproduced on this runtime with
71# gpt-5.6-luna: one upstream call, zero tool calls, reward 0.
72#
73# It carries NO task-specific or verifier-facing hints, and is applied uniformly
74# to every OpenClaw run so the cross-model comparison stays fair. Because it sits
75# on the scoring boundary — changing it moves every model's starting line — it is
76# a module-level constant rather than an inline string: greppable, diffable in
77# review, and overridable in an A/B harness without editing this file.
78OPENCLAW_AGENT_DIRECTIVE = (
79    "# 角色：自主任务执行 Agent\n\n"
80    "你是一个自主运行的任务执行 agent，**不是**聊天助手。请**立即开始执行下面的任务**。\n"
81    "- **不要**自我介绍、不要给自己取名字、不要询问「如何称呼你 / 我该怎么称呼你」、不要任何寒暄开场白。\n"
82    "- 当前是**无人值守的单次执行**：没有人会回答你的任何反问，任何"
83    "「等待用户确认 / 请用户提供」都会导致任务直接失败。\n"
84    "- 直接调用工具（read / exec / browser 等）完成任务，把最终产物写到 `/task/outputs/`。"
85    "遇到不确定的地方，基于任务描述做出最合理的假设并继续推进，不要停下来等待确认。\n\n"
86    "---\n\n"
87)

116def recover_trajectory_from_accio_state(agent_dir: Path) -> dict[str, Any]:
117    state_dir = agent_dir / "accio_state"
118    if not state_dir.exists():
119        return {}
120    jsonl_files = sorted(state_dir.rglob("*.jsonl"))
121    tool_calls: list[Any] = []
122    tool_results: list[Any] = []
123    tool_progress: list[Any] = []
124    messages: list[dict[str, Any]] = []
125    response_chunks: list[str] = []
126    response_chunk_seen: set[str] = set()
127    usage_records: list[Any] = []
128    source_files: list[str] = []
129
130    for path in jsonl_files:
131        source_files.append(str(path.relative_to(agent_dir)))
132        for line_no, line in enumerate(path.read_text(encoding="utf-8", errors="replace").splitlines(), start=1):
133            item = _safe_json_loads(line.strip())
134            if not isinstance(item, dict):
135                continue
136
137            raw_calls = item.get("toolCalls") or item.get("tool_calls") or []
138            if isinstance(raw_calls, list):
139                for call in raw_calls:
140                    if isinstance(call, dict):
141                        tool_calls.append({**_sanitize_tool_call(call), "_source": f"{path.relative_to(agent_dir)}:{line_no}"})
142
143            progress = item.get("toolProgress") or item.get("tool_progress")
144            for progress_item in _flatten_tool_progress(progress):
145                if isinstance(progress_item, dict):
146                    tool_progress.append({**progress_item, "_source": f"{path.relative_to(agent_dir)}:{line_no}"})
147
148            metadata = item.get("metadata")
149            if isinstance(metadata, dict) and isinstance(metadata.get("usage"), dict):
150                usage_records.append(metadata["usage"])
151
152            role = item.get("role")
153            message_type = item.get("messageType") or item.get("message_type")
154            content = item.get("content")
155            if role == "assistant" and isinstance(content, str) and content.strip():
156                chunk = content.strip()
157                if chunk not in response_chunk_seen:
158                    response_chunk_seen.add(chunk)
159                    response_chunks.append(chunk)
160            if role == "tool" or message_type == "tool_result":
161                result = {
162                    "toolCallId": item.get("toolCallId") or item.get("tool_call_id"),
163                    "toolName": item.get("name"),
164                    "content": content if isinstance(content, str) else json.dumps(content, ensure_ascii=False),
165                    "_source": f"{path.relative_to(agent_dir)}:{line_no}",
166                }
167                tool_results.append(result)
168                if isinstance(content, str) and "<task_result>" in content:
169                    chunk = content.strip()
170                    if chunk not in response_chunk_seen:
171                        response_chunk_seen.add(chunk)
172                        response_chunks.append(chunk)
173
174            if role in {"assistant", "tool", "user"} or message_type:
175                preview = content if isinstance(content, str) else ""
176                messages.append(
177                    {
178                        "role": role,
179                        "message_type": message_type,
180                        "tool_name": item.get("name"),
181                        "tool_call_count": len(raw_calls) if isinstance(raw_calls, list) else 0,
182                        "content_preview": preview[:4000],
183                        "timestamp": item.get("timestamp") or item.get("ts"),
184                        "_source": f"{path.relative_to(agent_dir)}:{line_no}",
185                    }
186                )
187
188    usage: dict[str, int] = {}
189    for record in usage_records:
190        for key, value in record.items():
191            if isinstance(value, int):
192                usage[key] = usage.get(key, 0) + value
193
194    tool_calls, duplicate_tool_calls = _dedupe_sequence(tool_calls, _tool_call_key)
195    tool_results, duplicate_tool_results = _dedupe_sequence(tool_results, _tool_result_key)
196    tool_progress, duplicate_tool_progress = _dedupe_sequence(tool_progress, _progress_key)
197    messages, duplicate_messages = _dedupe_sequence(messages, _message_key)
198    duplicate_counts = {
199        "tool_calls": duplicate_tool_calls,
200        "tool_results": duplicate_tool_results,
201        "tool_progress": duplicate_tool_progress,
202        "messages": duplicate_messages,
203    }
204
205    if not any([tool_calls, tool_results, tool_progress, messages, response_chunks]):
206        return {}
207    return {
208        "source": "accio_state",
209        "recovered_from_accio_state": True,
210        "recovered_source_files": source_files,
211        "deduplicated_recovery_items": duplicate_counts,
212        "response_text": "\n\n".join(response_chunks)[-60000:],
213        "tool_calls": tool_calls,
214        "tool_results": tool_results,
215        "tool_progress": tool_progress,
216        "messages": messages,
217        "usage": usage,
218    }

122def validate_tasks(check: Validation) -> None:
123    task_by_id: dict[str, Path] = {}
124    modality_ids: dict[str, set[str]] = {
125        "text_only": set(),
126        "browser_textcapable": set(),
127        "vision": set(),
128    }
129
130    for task_toml in sorted(DATASETS.rglob("task.toml")):
131        payload = tomllib.loads(task_toml.read_text(encoding="utf-8"))
132        task = payload.get("task") or {}
133        environment = payload.get("environment") or {}
134        task_id = task.get("id")
135        rel = task_toml.parent.relative_to(ROOT)
136        check.require(isinstance(task_id, str) and bool(task_id), f"{rel}: missing [task].id")
137        if not isinstance(task_id, str) or not task_id:
138            continue
139        check.require(task_id not in task_by_id, f"duplicate task id: {task_id}")
140        task_by_id[task_id] = task_toml.parent
141
142        for required in ("task.md", "rubric.json", "grader/run.sh"):
143            check.require((task_toml.parent / required).is_file(), f"{rel}: missing {required}")
144
145        check.require(
146            environment.get("base_images") == [PINNED_RUNTIME],
147            f"{rel}: [environment].base_images must contain only the public pinned runtime",
148        )
149
150        if bool(environment.get("requires_vision", False)):
151            modality = "vision"
152        elif bool(environment.get("requires_browser", False)):
153            modality = "browser_textcapable"
154        else:
155            modality = "text_only"
156        modality_ids[modality].add(task_id)
157
158    all_ids = load_collection(COLLECTIONS["all"])
159    check.require(
160        len(all_ids) == EXPECTED_TASK_COUNT,
161        f"full collection has {len(all_ids)} tasks; expected {EXPECTED_TASK_COUNT}",
162    )
163    check.require(len(all_ids) == len(set(all_ids)), "full collection contains duplicate task ids")
164    check.require(
165        set(all_ids) == set(task_by_id),
166        "full collection and discovered task.toml ids differ",
167    )
168
169    for modality, expected_ids in modality_ids.items():
170        actual = load_collection(COLLECTIONS[modality])
171        check.require(len(actual) == len(set(actual)), f"{modality} collection contains duplicates")
172        check.require(
173            set(actual) == expected_ids,
174            f"{modality} collection does not match task.toml modality declarations",
175        )
176
177    union = set().union(*modality_ids.values())
178    total_memberships = sum(len(items) for items in modality_ids.values())
179    check.require(union == set(all_ids), "modality collections do not cover the full collection")
180    check.require(
181        total_memberships == len(union),
182        "modality collections overlap; every task must belong to exactly one slice",
183    )
184
185    counts = {name: len(ids) for name, ids in modality_ids.items()}
186    print(
187        "tasks:"
188        f" all={len(all_ids)}"
189        f" text_only={counts['text_only']}"
190        f" browser_textcapable={counts['browser_textcapable']}"
191        f" vision={counts['vision']}"
192    )

129class PublicJudgeApiTests(unittest.TestCase):
130    def test_gemini_native_protocol_and_versioned_base_url(self) -> None:
131        response = {
132            "candidates": [{"content": {"parts": [{"text": json.dumps(JUDGMENT)}]}}]
133        }
134        with _FakeEndpoint(response) as endpoint:
135            result = gemini_judge(
136                JudgeConfig(
137                    provider="gemini",
138                    model="gemini-3.1-pro-preview",
139                    api_key="test-gemini-key",
140                    base_url=f"{endpoint.base_url}/v1beta",
141                ),
142                "grade this",
143            )
144
145        self.assertTrue(result["passed"])
146        self.assertEqual(len(endpoint.requests), 1)
147        request = endpoint.requests[0]
148        self.assertEqual(
149            request["path"],
150            "/v1beta/models/gemini-3.1-pro-preview:generateContent",
151        )
152        headers = {str(k).lower(): v for k, v in request["headers"].items()}  # type: ignore[union-attr]
153        self.assertEqual(headers["x-goog-api-key"], "test-gemini-key")
154        body = request["body"]
155        self.assertEqual(body["contents"][0]["parts"][0]["text"], "grade this")  # type: ignore[index]
156        self.assertEqual(
157            body["generationConfig"]["responseMimeType"],  # type: ignore[index]
158            "application/json",
159        )
160
161    def test_openai_responses_protocol_and_multimodal_input(self) -> None:
162        with _FakeEndpoint({"output_text": json.dumps(JUDGMENT)}) as endpoint:
163            result = openai_judge(
164                JudgeConfig(
165                    provider="openai",
166                    model="test-model",
167                    api_key="test-openai-key",
168                    base_url=f"{endpoint.base_url}/v1",
169                ),
170                "grade this",
171                [
172                    {"text": "artifact image"},
173                    {
174                        "inlineData": {
175                            "mimeType": "image/png",
176                            "data": "aGVsbG8=",
177                        }
178                    },
179                ],
180            )
181
182        self.assertTrue(result["passed"])
183        request = endpoint.requests[0]
184        self.assertEqual(request["path"], "/v1/responses")
185        headers = {str(k).lower(): v for k, v in request["headers"].items()}  # type: ignore[union-attr]
186        self.assertEqual(headers["authorization"], "Bearer test-openai-key")
187        body = request["body"]
188        user_content = body["input"][1]["content"]  # type: ignore[index]
189        self.assertEqual(user_content[0], {"type": "input_text", "text": "grade this"})
190        self.assertEqual(user_content[1], {"type": "input_text", "text": "artifact image"})
191        self.assertEqual(
192            user_content[2],
193            {
194                "type": "input_image",
195                "image_url": "data:image/png;base64,aGVsbG8=",
196            },
197        )
198
199
200class PublicHarnessContractTests(unittest.TestCase):
201    def test_public_native_provider_configs_pin_api_versions(self) -> None:
202        config_root = Path(__file__).resolve().parents[1] / "configs"
203        for name in (
204            "realreplicabench_native_google_direct_models.json",
205            "realreplicabench_native_google_models.json",
206        ):
207            payload = json.loads((config_root / name).read_text(encoding="utf-8"))
208            self.assertEqual(
209                payload["providers"]["google"]["baseUrl"],
210                "https://generativelanguage.googleapis.com/v1beta",
211            )
212        qwen = json.loads(
213            (config_root / "realreplicabench_qwen37plus_native_models.json").read_text(
214                encoding="utf-8"
215            )
216        )
217        self.assertEqual(
218            qwen["providers"]["qwen"]["baseUrl"],
219            "https://dashscope-intl.aliyuncs.com/compatible-mode/v1",
220        )
221
222    def test_byo_endpoint_preset_configs_pin_wire_formats(self) -> None:
223        # Each BYO-endpoint preset is a starter template for a distinct
224        # wire format the OpenClaw runtime speaks natively. A future edit
225        # that breaks any of these contracts (wrong `api` enum, wrong
226        # default baseUrl, dropped anthropic-version header) should trip
227        # this test rather than silently ship a broken preset.
228        config_root = Path(__file__).resolve().parents[1] / "configs"
229
230        openai_chat = json.loads(
231            (config_root / "realreplicabench_openai_chat_models.json").read_text(
232                encoding="utf-8"
233            )
234        )
235        self.assertEqual(openai_chat["providers"]["openai"]["api"], "openai-completions")
236        self.assertEqual(
237            openai_chat["providers"]["openai"]["baseUrl"],
238            "https://api.openai.com/v1",
239        )