Files
DeepInfer/eval/cvbench/bench.py
T
shunyadandClaude Opus 5.5 addd6e6c3e fix(eval/cvbench): do not count missing sources as 0% in the overall score
summarize_extra averaged ADE20K/COCO/Omni3D with .get(source, 0.0), so a run
that scored no rows of a source (e.g. --limit 200, which keeps only ADE20K
rows because records are taken in file order) reported an overall CV-Bench
score of at most a quarter of the real accuracy.

The overall/2D/3D scores are now reported only when every source they
average has scored rows; otherwise summary.json lists
cvbench_missing_sources and the scorer warns. Per-source and per-task
accuracies are reported either way.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-18 22:22:15 -08:00

168 lines
6.3 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations
import json
import os
import re
import sys
from dataclasses import dataclass
from pathlib import Path
from typing import Any
EVAL_DIR = Path(__file__).resolve().parent.parent # eval/
if str(EVAL_DIR) not in sys.path:
sys.path.insert(0, str(EVAL_DIR))
from common.io import require_file # noqa: E402
# CV-Bench as lmms-eval runs it: the image, then the dataset's prompt with this suffix, 0-shot, no thinking.
PROMPT_SUFFIX = "\nAnswer with the option's letter from the given choices directly."
THINKING_MODE = "chat"
# The first option letter in the response, with or without parentheses.
_LETTER_RE = re.compile(r"\(?\b([A-F])\b\)?")
_TRAILING_SPECIAL_RE = re.compile(r"(?:\s*(?:<|[^|<>]*|>|<\|[^|<>]*\|>|</s>))+\s*$")
DATA_HINT = (
"CV-Bench download: https://huggingface.co/datasets/nyu-visionx/CV-Bench (test_2d.parquet, test_3d.parquet); "
"convert it with eval/cvbench/prepare.py"
)
@dataclass(frozen=True)
class Problem:
task_id: str
prompt: str
answer: str # the option letter, A-F
source: str # ADE20K / COCO / Omni3D
task: str # Count / Relation / Depth / Distance
image_path: str
image_size: tuple[int, int]
# --------------------------------------------------------------------------- #
# Data
# --------------------------------------------------------------------------- #
def load(data_file: str | os.PathLike) -> list[Problem]:
path = Path(data_file)
require_file(path, DATA_HINT)
problems: list[Problem] = []
with open(path) as f:
for line_no, line in enumerate(f):
line = line.strip()
if not line:
continue
row = json.loads(line)
answer = str(row["answer"]).strip("() ")
assert re.fullmatch(r"[A-F]", answer), f"{path}:{line_no + 1}: answer {row['answer']!r} is not an option letter"
image = Path(row["image"])
if not image.is_absolute():
image = path.parent / image
problems.append(
Problem(
task_id=str(row["task_id"]),
prompt=str(row["prompt"]),
answer=answer,
source=str(row["source"]),
task=str(row["task"]),
image_path=str(image),
image_size=(int(row["image_size"][0]), int(row["image_size"][1])),
)
)
assert problems, f"no problem parsed from {path}"
assert len({p.task_id for p in problems}) == len(problems), f"duplicate task_id in {path}"
return problems
def build_messages(problem: Problem) -> list[dict[str, Any]]:
content = [
{"type": "image_url", "image_url": {"url": problem.image_path}},
{"type": "text", "text": problem.prompt + PROMPT_SUFFIX},
]
return [{"role": "user", "content": content}]
# --------------------------------------------------------------------------- #
# Answer extraction and scoring
# --------------------------------------------------------------------------- #
def final_answer_text(raw_text: str) -> str:
return _TRAILING_SPECIAL_RE.sub("", raw_text.rsplit("</think>", 1)[-1]).strip()
def extract_letter(text: str) -> str | None:
match = _LETTER_RE.search(text)
return match.group(1) if match else None
# --------------------------------------------------------------------------- #
# Entry points used by run_eval.py / score.py
# --------------------------------------------------------------------------- #
def build_records(problems: list[Problem], *, limit: int | None = None) -> list[dict[str, Any]]:
selected = problems if limit is None else problems[: max(limit, 1)]
return [
{
"task_id": problem.task_id,
"question": problem.prompt,
"answer": problem.answer,
"source": problem.source,
"task": problem.task,
"image_paths": [problem.image_path],
"image_sizes": [list(problem.image_size)],
"messages": build_messages(problem),
}
for problem in selected
]
def score_rollout(rollout: dict[str, Any], record: dict[str, Any]) -> dict[str, Any]:
pred = extract_letter(final_answer_text(rollout.get("raw_text", "")))
row = dict(rollout)
row["pred"] = pred
row["gold"] = record["answer"]
row["extract_source"] = "letter" if pred is not None else "none"
row["correct"] = pred == record["answer"]
return row
_SOURCES_2D = ("ADE20K", "COCO")
_SOURCES_3D = ("Omni3D",)
def summarize_extra(results: list[dict[str, Any]], by_id: dict[str, dict[str, Any]]) -> dict[str, Any]:
"""CV-Bench's overall score: the mean of the 2D accuracy (itself the mean of ADE20K and COCO) and the 3D accuracy.
The overall, 2D and 3D scores are only reported when every source they average has scored rows; a partial run
(e.g. --limit, which keeps the first records in file order) reports the missing sources instead of counting
them as 0%.
"""
def accuracy(rows: list[dict[str, Any]]) -> float:
return sum(1 for row in rows if row["correct"]) / len(rows) if rows else 0.0
by_source: dict[str, list[dict[str, Any]]] = {}
by_task: dict[str, list[dict[str, Any]]] = {}
for row in results:
record = by_id[row["task_id"]]
by_source.setdefault(record["source"], []).append(row)
by_task.setdefault(record["task"], []).append(row)
source_accuracy = {source: accuracy(rows) for source, rows in sorted(by_source.items())}
summary: dict[str, Any] = {
"by_source": source_accuracy,
"by_task": {task: accuracy(rows) for task, rows in sorted(by_task.items())},
}
missing = [source for source in _SOURCES_2D + _SOURCES_3D if source not in source_accuracy]
if missing:
print(f"[cvbench] no scored rows from {missing}; CV-Bench overall/2D/3D are not reported", file=sys.stderr)
summary["cvbench_missing_sources"] = missing
return summary
accuracy_2d = sum(source_accuracy[source] for source in _SOURCES_2D) / len(_SOURCES_2D)
accuracy_3d = sum(source_accuracy[source] for source in _SOURCES_3D) / len(_SOURCES_3D)
summary.update(cvbench_overall=(accuracy_2d + accuracy_3d) / 2, cvbench_2d=accuracy_2d, cvbench_3d=accuracy_3d)
return summary