|
|
| import json |
| import pandas as pd |
| from transformers import TrainerCallback |
|
|
|
|
| class ContinuousLogCallback(TrainerCallback): |
| def __init__(self, json_path, csv_path): |
| self.json_path = json_path |
| self.csv_path = csv_path |
| self.logs = [] |
|
|
| def save_logs(self): |
| with open(self.json_path, "w", encoding="utf-8") as f: |
| json.dump(self.logs, f, ensure_ascii=False, indent=2) |
|
|
| pd.DataFrame(self.logs).to_csv(self.csv_path, index=False) |
|
|
| def on_log(self, args, state, control, logs=None, **kwargs): |
| if logs is None: |
| return |
|
|
| item = { |
| "step": int(state.global_step), |
| "epoch": float(state.epoch) if state.epoch is not None else None, |
| "event": "log" |
| } |
|
|
| for key, value in logs.items(): |
| if isinstance(value, (int, float)): |
| item[key] = float(value) |
|
|
| self.logs.append(item) |
| self.save_logs() |
|
|
| def on_evaluate(self, args, state, control, metrics=None, **kwargs): |
| if metrics is None: |
| return |
|
|
| item = { |
| "step": int(state.global_step), |
| "epoch": float(state.epoch) if state.epoch is not None else None, |
| "event": "evaluate" |
| } |
|
|
| for key, value in metrics.items(): |
| if isinstance(value, (int, float)): |
| item[key] = float(value) |
|
|
| self.logs.append(item) |
| self.save_logs() |
|
|
