mirror of
https://github.com/deepseek-ai/DeepInfer.git
synced 2026-10-26 16:35:05 +00:00
39 lines
1.2 KiB
Python
39 lines
1.2 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
from collections.abc import Iterable
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
def read_jsonl(path: str | os.PathLike) -> list[dict[str, Any]]:
|
|
rows: list[dict[str, Any]] = []
|
|
with open(path) as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if line:
|
|
rows.append(json.loads(line))
|
|
return rows
|
|
|
|
|
|
def write_jsonl(path: str | os.PathLike, rows: Iterable[dict[str, Any]]) -> int:
|
|
os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True)
|
|
count = 0
|
|
with open(path, "w") as f:
|
|
for row in rows:
|
|
f.write(json.dumps(row, ensure_ascii=False) + "\n")
|
|
count += 1
|
|
return count
|
|
|
|
|
|
def require_file(path: Path, hint: str) -> Path:
|
|
"""Raise an actionable error when the data file is missing (this directory ships no datasets)."""
|
|
if not path.is_file():
|
|
raise FileNotFoundError(
|
|
f"data file not found: {path}\n"
|
|
"The evaluation scripts do not download datasets. Obtain it and place it at this path, "
|
|
f"or set DATA_FILE / --data-file.\n{hint}"
|
|
)
|
|
return path
|