nathanael-fijalkow commited on
Commit
f97097f
·
1 Parent(s): 39a0b64

reverse order for tokenizers load

Browse files
Files changed (1) hide show
  1. src/evaluate.py +4 -3
src/evaluate.py CHANGED
@@ -767,11 +767,12 @@ def load_model_from_hub(model_id: str, device: str = "auto"):
767
  from src.model import ChessConfig, ChessForCausalLM
768
  from src.tokenizer import ChessTokenizer
769
 
770
- # Try loading with custom tokenizer first, fall back to AutoTokenizer
 
771
  try:
772
- tokenizer = ChessTokenizer.from_pretrained(model_id)
773
- except Exception:
774
  tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
 
 
775
 
776
  model = AutoModelForCausalLM.from_pretrained(
777
  model_id,
 
767
  from src.model import ChessConfig, ChessForCausalLM
768
  from src.tokenizer import ChessTokenizer
769
 
770
+ # Try AutoTokenizer with trust_remote_code first to load custom tokenizer.py from Hub
771
+ # Fall back to local ChessTokenizer if the model doesn't have a custom tokenizer
772
  try:
 
 
773
  tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
774
+ except Exception:
775
+ tokenizer = ChessTokenizer.from_pretrained(model_id)
776
 
777
  model = AutoModelForCausalLM.from_pretrained(
778
  model_id,