Upload config_generator.py with huggingface_hub
Browse files- config_generator.py +5 -2
config_generator.py
CHANGED
|
@@ -66,8 +66,11 @@ def generate_flags(
|
|
| 66 |
target_size_mib: float = None,
|
| 67 |
) -> dict:
|
| 68 |
is_qat = model.get("features", {}).get("is_qat", False)
|
| 69 |
-
|
| 70 |
-
|
|
|
|
|
|
|
|
|
|
| 71 |
|
| 72 |
max_layer = model.get("features", {}).get("n_layers", 31)
|
| 73 |
|
|
|
|
| 66 |
target_size_mib: float = None,
|
| 67 |
) -> dict:
|
| 68 |
is_qat = model.get("features", {}).get("is_qat", False)
|
| 69 |
+
# Normally Q5_K pins (see EMBD_DEPLOY_TIER); with --free-pins the
|
| 70 |
+
# classifier may assign output/token_embd other tiers — honor them here
|
| 71 |
+
# since the binary applies these two flags unconditionally.
|
| 72 |
+
output_type = assignments.get("output.weight", "Q5_K")
|
| 73 |
+
token_embd_type = assignments.get("token_embd.weight", "Q5_K")
|
| 74 |
|
| 75 |
max_layer = model.get("features", {}).get("n_layers", 31)
|
| 76 |
|