mirror of
https://github.com/deepseek-ai/DeepInfer.git
synced 2026-11-20 04:22:27 +00:00
summarize_extra averaged ADE20K/COCO/Omni3D with .get(source, 0.0), so a run that scored no rows of a source (e.g. --limit 200, which keeps only ADE20K rows because records are taken in file order) reported an overall CV-Bench score of at most a quarter of the real accuracy. The overall/2D/3D scores are now reported only when every source they average has scored rows; otherwise summary.json lists cvbench_missing_sources and the scorer warns. Per-source and per-task accuracies are reported either way. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
168 lines
6.3 KiB
Python
168 lines
6.3 KiB
Python
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||
# SPDX-License-Identifier: Apache-2.0
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import os
|
||
import re
|
||
import sys
|
||
from dataclasses import dataclass
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
EVAL_DIR = Path(__file__).resolve().parent.parent # eval/
|
||
if str(EVAL_DIR) not in sys.path:
|
||
sys.path.insert(0, str(EVAL_DIR))
|
||
|
||
from common.io import require_file # noqa: E402
|
||
|
||
# CV-Bench as lmms-eval runs it: the image, then the dataset's prompt with this suffix, 0-shot, no thinking.
|
||
PROMPT_SUFFIX = "\nAnswer with the option's letter from the given choices directly."
|
||
THINKING_MODE = "chat"
|
||
# The first option letter in the response, with or without parentheses.
|
||
_LETTER_RE = re.compile(r"\(?\b([A-F])\b\)?")
|
||
_TRAILING_SPECIAL_RE = re.compile(r"(?:\s*(?:<|[^|<>]*|>|<\|[^|<>]*\|>|</s>))+\s*$")
|
||
DATA_HINT = (
|
||
"CV-Bench download: https://huggingface.co/datasets/nyu-visionx/CV-Bench (test_2d.parquet, test_3d.parquet); "
|
||
"convert it with eval/cvbench/prepare.py"
|
||
)
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Problem:
|
||
task_id: str
|
||
prompt: str
|
||
answer: str # the option letter, A-F
|
||
source: str # ADE20K / COCO / Omni3D
|
||
task: str # Count / Relation / Depth / Distance
|
||
image_path: str
|
||
image_size: tuple[int, int]
|
||
|
||
|
||
# --------------------------------------------------------------------------- #
|
||
# Data
|
||
# --------------------------------------------------------------------------- #
|
||
|
||
|
||
def load(data_file: str | os.PathLike) -> list[Problem]:
|
||
path = Path(data_file)
|
||
require_file(path, DATA_HINT)
|
||
problems: list[Problem] = []
|
||
with open(path) as f:
|
||
for line_no, line in enumerate(f):
|
||
line = line.strip()
|
||
if not line:
|
||
continue
|
||
row = json.loads(line)
|
||
answer = str(row["answer"]).strip("() ")
|
||
assert re.fullmatch(r"[A-F]", answer), f"{path}:{line_no + 1}: answer {row['answer']!r} is not an option letter"
|
||
image = Path(row["image"])
|
||
if not image.is_absolute():
|
||
image = path.parent / image
|
||
problems.append(
|
||
Problem(
|
||
task_id=str(row["task_id"]),
|
||
prompt=str(row["prompt"]),
|
||
answer=answer,
|
||
source=str(row["source"]),
|
||
task=str(row["task"]),
|
||
image_path=str(image),
|
||
image_size=(int(row["image_size"][0]), int(row["image_size"][1])),
|
||
)
|
||
)
|
||
assert problems, f"no problem parsed from {path}"
|
||
assert len({p.task_id for p in problems}) == len(problems), f"duplicate task_id in {path}"
|
||
return problems
|
||
|
||
|
||
def build_messages(problem: Problem) -> list[dict[str, Any]]:
|
||
content = [
|
||
{"type": "image_url", "image_url": {"url": problem.image_path}},
|
||
{"type": "text", "text": problem.prompt + PROMPT_SUFFIX},
|
||
]
|
||
return [{"role": "user", "content": content}]
|
||
|
||
|
||
# --------------------------------------------------------------------------- #
|
||
# Answer extraction and scoring
|
||
# --------------------------------------------------------------------------- #
|
||
|
||
|
||
def final_answer_text(raw_text: str) -> str:
|
||
return _TRAILING_SPECIAL_RE.sub("", raw_text.rsplit("</think>", 1)[-1]).strip()
|
||
|
||
|
||
def extract_letter(text: str) -> str | None:
|
||
match = _LETTER_RE.search(text)
|
||
return match.group(1) if match else None
|
||
|
||
|
||
# --------------------------------------------------------------------------- #
|
||
# Entry points used by run_eval.py / score.py
|
||
# --------------------------------------------------------------------------- #
|
||
|
||
|
||
def build_records(problems: list[Problem], *, limit: int | None = None) -> list[dict[str, Any]]:
|
||
selected = problems if limit is None else problems[: max(limit, 1)]
|
||
return [
|
||
{
|
||
"task_id": problem.task_id,
|
||
"question": problem.prompt,
|
||
"answer": problem.answer,
|
||
"source": problem.source,
|
||
"task": problem.task,
|
||
"image_paths": [problem.image_path],
|
||
"image_sizes": [list(problem.image_size)],
|
||
"messages": build_messages(problem),
|
||
}
|
||
for problem in selected
|
||
]
|
||
|
||
|
||
def score_rollout(rollout: dict[str, Any], record: dict[str, Any]) -> dict[str, Any]:
|
||
pred = extract_letter(final_answer_text(rollout.get("raw_text", "")))
|
||
row = dict(rollout)
|
||
row["pred"] = pred
|
||
row["gold"] = record["answer"]
|
||
row["extract_source"] = "letter" if pred is not None else "none"
|
||
row["correct"] = pred == record["answer"]
|
||
return row
|
||
|
||
|
||
_SOURCES_2D = ("ADE20K", "COCO")
|
||
_SOURCES_3D = ("Omni3D",)
|
||
|
||
|
||
def summarize_extra(results: list[dict[str, Any]], by_id: dict[str, dict[str, Any]]) -> dict[str, Any]:
|
||
"""CV-Bench's overall score: the mean of the 2D accuracy (itself the mean of ADE20K and COCO) and the 3D accuracy.
|
||
|
||
The overall, 2D and 3D scores are only reported when every source they average has scored rows; a partial run
|
||
(e.g. --limit, which keeps the first records in file order) reports the missing sources instead of counting
|
||
them as 0%.
|
||
"""
|
||
|
||
def accuracy(rows: list[dict[str, Any]]) -> float:
|
||
return sum(1 for row in rows if row["correct"]) / len(rows) if rows else 0.0
|
||
|
||
by_source: dict[str, list[dict[str, Any]]] = {}
|
||
by_task: dict[str, list[dict[str, Any]]] = {}
|
||
for row in results:
|
||
record = by_id[row["task_id"]]
|
||
by_source.setdefault(record["source"], []).append(row)
|
||
by_task.setdefault(record["task"], []).append(row)
|
||
source_accuracy = {source: accuracy(rows) for source, rows in sorted(by_source.items())}
|
||
summary: dict[str, Any] = {
|
||
"by_source": source_accuracy,
|
||
"by_task": {task: accuracy(rows) for task, rows in sorted(by_task.items())},
|
||
}
|
||
missing = [source for source in _SOURCES_2D + _SOURCES_3D if source not in source_accuracy]
|
||
if missing:
|
||
print(f"[cvbench] no scored rows from {missing}; CV-Bench overall/2D/3D are not reported", file=sys.stderr)
|
||
summary["cvbench_missing_sources"] = missing
|
||
return summary
|
||
accuracy_2d = sum(source_accuracy[source] for source in _SOURCES_2D) / len(_SOURCES_2D)
|
||
accuracy_3d = sum(source_accuracy[source] for source in _SOURCES_3D) / len(_SOURCES_3D)
|
||
summary.update(cvbench_overall=(accuracy_2d + accuracy_3d) / 2, cvbench_2d=accuracy_2d, cvbench_3d=accuracy_3d)
|
||
return summary
|