Spaces:
Running
Running
Commit
·
f97097f
1
Parent(s):
39a0b64
reverse order for tokenizers load
Browse files- src/evaluate.py +4 -3
src/evaluate.py
CHANGED
|
@@ -767,11 +767,12 @@ def load_model_from_hub(model_id: str, device: str = "auto"):
|
|
| 767 |
from src.model import ChessConfig, ChessForCausalLM
|
| 768 |
from src.tokenizer import ChessTokenizer
|
| 769 |
|
| 770 |
-
# Try
|
|
|
|
| 771 |
try:
|
| 772 |
-
tokenizer = ChessTokenizer.from_pretrained(model_id)
|
| 773 |
-
except Exception:
|
| 774 |
tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
|
|
|
|
|
|
|
| 775 |
|
| 776 |
model = AutoModelForCausalLM.from_pretrained(
|
| 777 |
model_id,
|
|
|
|
| 767 |
from src.model import ChessConfig, ChessForCausalLM
|
| 768 |
from src.tokenizer import ChessTokenizer
|
| 769 |
|
| 770 |
+
# Try AutoTokenizer with trust_remote_code first to load custom tokenizer.py from Hub
|
| 771 |
+
# Fall back to local ChessTokenizer if the model doesn't have a custom tokenizer
|
| 772 |
try:
|
|
|
|
|
|
|
| 773 |
tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
|
| 774 |
+
except Exception:
|
| 775 |
+
tokenizer = ChessTokenizer.from_pretrained(model_id)
|
| 776 |
|
| 777 |
model = AutoModelForCausalLM.from_pretrained(
|
| 778 |
model_id,
|