optimize parameter calculation

generate_overview_json.py

@@ -1,12 +1,17 @@
-from pathlib import Path
 import json
+from pathlib import Path
 
+from huggingface_hub import get_safetensors_metadata
 from tqdm import tqdm
 from transformers import AutoModelForCausalLM
 
 
 def get_num_parameters(model_name: str) -> int:
-    return AutoModelForCausalLM.from_pretrained(model_name).num_parameters()
+    try:
+        metadata = get_safetensors_metadata(model_name)
+        return sum(metadata.parameter_count.values())
+    except Exception:
+        return AutoModelForCausalLM.from_pretrained(model_name).num_parameters()
 
 
 def main():

generate_overview_requirements.txt

@@ -1 +1,2 @@
+huggingface_hub
 transformers