razor5050's picture
Add tokenizer, inference code, model card, and 20-query report
ca2f8ca verified
raw
history blame contribute delete
652 Bytes
import json, csv, time
from pathlib import Path
class MetricsLogger:
def __init__(self, path):
self.path=Path(path); self.path.parent.mkdir(parents=True,exist_ok=True)
def log(self, **kw):
kw.setdefault('time', time.time())
with self.path.open('a') as f: f.write(json.dumps(kw)+'\n')
def jsonl_to_csv(jsonl_path, csv_path):
rows=[json.loads(l) for l in Path(jsonl_path).read_text().splitlines() if l.strip()]
if not rows: return
keys=sorted(set().union(*(r.keys() for r in rows)))
with open(csv_path,'w',newline='') as f:
w=csv.DictWriter(f,fieldnames=keys); w.writeheader(); w.writerows(rows)