from __future__ import annotations
import json
from pathlib import Path
from typing import Any, Iterable, Iterator
BESPOKE_TOKEN_REPLACEMENTS = {
"<|begin_of_thought|>": "",
"<|end_of_thought|>": "",
"<|begin_of_solution|>": "",
"<|end_of_solution|>": "",
}
def clean_text(value: Any) -> str:
if value is None:
return ""
return str(value).strip()
def normalize_reasoning_tokens(value: Any) -> str:
text = clean_text(value)
for old, new in BESPOKE_TOKEN_REPLACEMENTS.items():
text = text.replace(old, new)
return text.strip()
def make_user_assistant_record(
user: Any,
assistant: Any,
*,
source: str | None = None,
include_source: bool = False,
) -> dict[str, str] | None:
user_text = clean_text(user)
assistant_text = normalize_reasoning_tokens(assistant)
if not user_text or not assistant_text:
return None
record = {
"user": user_text,
"assistant": assistant_text,
}
if include_source and source:
record["source"] = source
return record
def read_jsonl(path: str | Path) -> Iterator[tuple[int, dict[str, Any]]]:
jsonl_path = Path(path)
with jsonl_path.open("r", encoding="utf-8") as f:
for row_index, line in enumerate(f):
line = line.strip()
if not line:
continue
yield row_index, json.loads(line)
def write_jsonl(records: Iterable[dict[str, Any]], path: str | Path) -> None:
out_path = Path(path)
out_path.parent.mkdir(parents=True, exist_ok=True)
with out_path.open("w", encoding="utf-8") as f:
for record in records:
f.write(json.dumps(record, ensure_ascii=False))
f.write("\n")
def resolve_paths(paths: str | Path | Iterable[str | Path]) -> list[Path]:
if isinstance(paths, (str, Path)):
return [Path(paths)]
return [Path(path) for path in paths]
def render_records(records: Iterable[dict[str, Any]]) -> str:
return json.dumps(list(records), ensure_ascii=False, indent=2)