Omarrran's picture
Add Transformers-compatible ks_byte_lm SpaceByte release
45121cf verified
Raw
History Blame Contribute Delete
1.25 kB
"""Logging setup shared by every entrypoint.
Console handler is UTF-8 (so Kashmiri samples print on Windows cp1252 shells)
and a per-run file handler mirrors everything to <run_dir>/train.log.
"""
from __future__ import annotations
import logging
import os
import sys
_CONFIGURED = False
def get_logger(name: str = "ksbyte", run_dir: str | None = None,
level: int = logging.INFO) -> logging.Logger:
global _CONFIGURED
logger = logging.getLogger(name)
logger.setLevel(level)
if not _CONFIGURED:
fmt = logging.Formatter(
"%(asctime)s | %(levelname)-7s | %(message)s", datefmt="%H:%M:%S"
)
# Console: force UTF-8 so Arabic-script samples never crash the stream.
stream = sys.stdout
try:
stream.reconfigure(encoding="utf-8") # py3.7+
except Exception:
pass
ch = logging.StreamHandler(stream)
ch.setFormatter(fmt)
logger.addHandler(ch)
if run_dir:
os.makedirs(run_dir, exist_ok=True)
fh = logging.FileHandler(os.path.join(run_dir, "train.log"), encoding="utf-8")
fh.setFormatter(fmt)
logger.addHandler(fh)
_CONFIGURED = True
return logger