Update transformers version and simplify evaluator code

- Upgrade transformers to 5.0.0rc0
- Simplify BOS token logic to always use newline token
- Apply code formatting improvements

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>

core/evaluator.py

@@ -57,10 +57,7 @@ def extract_topk_predictions(logit: torch.Tensor, target_ids: torch.Tensor, k: i
         actual_prob = probs[pos, target_id].item()
         rank = (probs[pos] > actual_prob).sum().item() + 1
 
-        topk_list = [
-            [top_ids[pos, i].item(), round(top_probs[pos, i].item(), 6)]
-            for i in range(k)
-        ]
+        topk_list = [[top_ids[pos, i].item(), round(top_probs[pos, i].item(), 6)] for i in range(k)]
         results.append([target_id, rank, topk_list])
 
     return results
@@ -85,12 +82,7 @@ def count_rwkv_parameters_in_billions(rwkv_model) -> float:
 
 
 @torch.no_grad()
-def evaluate_hf_single_sample(
-    model,
-    tokenizer,
-    text: str,
-    bos_mode: str = "add_newline_token"
-) -> Dict[str, Any]:
+def evaluate_hf_single_sample(model, tokenizer, text: str, bos_mode: str = "add_newline_token") -> Dict[str, Any]:
     """
     Evaluate a HuggingFace model on a single text sample.
 
@@ -104,20 +96,18 @@ def evaluate_hf_single_sample(
         dict with byte_wise_losses, top5_predictions, compression_rate, etc.
     """
     # Create token-to-bytes converter
-    token2bytes_converter = TokenizerBytesConverter(
-        model_name_or_path=tokenizer.name_or_path,
-        tokenizer=tokenizer
-    )
+    token2bytes_converter = TokenizerBytesConverter(model_name_or_path=tokenizer.name_or_path, tokenizer=tokenizer)
 
     # Determine BOS token
-    if bos_mode in ["add_default_bos", "replace_with_bos"]:
-        bos_token = tokenizer.bos_token_id
-    elif bos_mode in ["add_default_eos", "replace_with_eos"]:
-        bos_token = tokenizer.eos_token_id
-    elif bos_mode in ["add_newline_token", "replace_with_newline_token"]:
-        bos_token = tokenizer.encode("\n")[0]
-    else:
-        bos_token = tokenizer.bos_token_id
+    bos_token = tokenizer.encode("\n")[0]
+    # if bos_mode in ["add_default_bos", "replace_with_bos"]:
+    #     bos_token = tokenizer.bos_token_id
+    # elif bos_mode in ["add_default_eos", "replace_with_eos"]:
+    #     bos_token = tokenizer.eos_token_id
+    # elif bos_mode in ["add_newline_token", "replace_with_newline_token"]:
+    #     bos_token = tokenizer.encode("\n")[0]
+    # else:
+    #     bos_token = tokenizer.bos_token_id
 
     bos_tensor = torch.tensor([bos_token], device=model.device).unsqueeze(0)
 
@@ -149,9 +139,7 @@ def evaluate_hf_single_sample(
         raise ValueError("Token bytes don't match original text bytes")
 
     # Extract top-k predictions
-    sample_topk = extract_topk_predictions(
-        logit[:-1], input_chunk.squeeze(0)[1:]
-    )
+    sample_topk = extract_topk_predictions(logit[:-1], input_chunk.squeeze(0)[1:])
 
     # Calculate byte-wise losses
     byte_wise_losses = []
@@ -183,16 +171,12 @@ def evaluate_hf_single_sample(
         "num_tokens": seq_length,
         "num_bytes": num_bytes,
         "model_name": getattr(model.config, "_name_or_path", "unknown"),
-        "tokenizer": tokenizer
+        "tokenizer": tokenizer,
     }
 
 
 @torch.no_grad()
-def evaluate_rwkv7_single_sample(
-    model,
-    tokenizer,
-    text: str
-) -> Dict[str, Any]:
+def evaluate_rwkv7_single_sample(model, tokenizer, text: str) -> Dict[str, Any]:
     """
     Evaluate a RWKV7 model on a single text sample.
 
@@ -241,9 +225,7 @@ def evaluate_rwkv7_single_sample(
     token_bytes = [tokenizer.decodeBytes([token]) for token in input_chunk[1:]]
 
     # Extract top-k predictions
-    sample_topk = extract_topk_predictions(
-        logit[:-1], torch.tensor(input_chunk[1:]).to(device)
-    )
+    sample_topk = extract_topk_predictions(logit[:-1], torch.tensor(input_chunk[1:]).to(device))
 
     # Calculate byte-wise losses
     byte_wise_losses = []
@@ -266,5 +248,5 @@ def evaluate_rwkv7_single_sample(
         "num_tokens": input_length,
         "num_bytes": num_bytes,
         "model_name": "RWKV7-G1C-1.5B",
-        "tokenizer": tokenizer
+        "tokenizer": tokenizer,
     }

requirements.txt

@@ -1,5 +1,5 @@
 torch>=2.0.0
-transformers>=4.35.0
+transformers==5.0.0rc0
 tokenizers>=0.15.0
 gradio>=5.15.0
 numpy>=1.24.0