abedgemma commited on
Commit
730ebbb
ยท
verified ยท
1 Parent(s): d7e263e

Update model_manager.py

Browse files
Files changed (1) hide show
  1. model_manager.py +33 -49
model_manager.py CHANGED
@@ -51,86 +51,70 @@ class ModelDef:
51
  CATALOG: list[ModelDef] = [
52
 
53
  # โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•
54
- # TIER 1 โ€” ุงู„ุฎูŠุงุฑ ุงู„ุงูุชุฑุงุถูŠ: ุฌูˆุฏุฉ ุนุงู„ูŠุฉ + ุงุณุชู‚ุฑุงุฑ ู…ูุซุจูŽุช
 
 
55
  # โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•
56
  ModelDef(
57
  id = "qwen3-8b",
58
- name = "Qwen3 8B โ€” Thinking (bartowski Q4_K_M)",
59
- repo = "bartowski/Qwen3-8B-Instruct-GGUF",
60
- glob_primary = "*Q4_K_M*",
61
- glob_fallback= "*Q4_K_S*",
62
- size_gb = 5.2,
63
  ctx_size = 16384,
64
  quality = 88,
65
- has_vision = True,
66
  has_thinking = True,
67
- mmproj_repo = "bartowski/Qwen3-8B-Instruct-GGUF",
68
- mmproj_glob = "*mmproj*",
69
  extra_flags = "--jinja --reasoning-format deepseek",
70
  temp=0.6, top_k=20, top_p=0.95, min_p=0.0,
71
- notes="ุงู„ุฎูŠุงุฑ ุงู„ุงูุชุฑุงุถูŠ. Qwen3 (ู…ุด 3.5) ูŠุชุฌู†ุจ KV cache reuse bug. "
72
- "bartowski Q4_K_M: KLD=0.0087 โ€” ุฃูุถู„ ุฏู‚ุฉ ููŠ ู‡ุฐุง ุงู„ุญุฌู….",
73
  ),
74
 
75
  # โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•
76
- # TIER 2 โ€” ุฃุณุฑุน ู„ู„ู…ู‡ุงู… ุงู„ุจุณูŠุทุฉ
 
 
77
  # โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•
78
  ModelDef(
79
  id = "qwen3-4b",
80
- name = "Qwen3 4B โ€” Fast (bartowski Q4_K_M)",
81
- repo = "bartowski/Qwen3-4B-Instruct-GGUF",
82
- glob_primary = "*Q4_K_M*",
83
- glob_fallback= "*Q4_K_S*",
84
- size_gb = 2.9,
85
  ctx_size = 16384,
86
  quality = 76,
87
- has_vision = True,
88
  has_thinking = True,
89
- mmproj_repo = "bartowski/Qwen3-4B-Instruct-GGUF",
90
- mmproj_glob = "*mmproj*",
91
  extra_flags = "--jinja --reasoning-format deepseek",
92
  temp=0.6, top_k=20, top_p=0.95, min_p=0.0,
93
- notes="ุฃุณุฑุน 2x ู…ู† 8B. ู„ู„ู…ู‡ุงู… ุงู„ุจุณูŠุทุฉ ูˆุงู„ู€ classification ูˆุงู„ู€ routing.",
94
  ),
95
 
96
  # โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•
97
- # TIER 3 โ€” ุฌูˆุฏุฉ ุฃุนู„ู‰ ู„ู…ุง RAM ูŠุณู…ุญ
98
  # โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•
99
  ModelDef(
100
  id = "qwen3-14b",
101
- name = "Qwen3 14B โ€” Premium (bartowski Q4_K_M)",
102
- repo = "bartowski/Qwen3-14B-Instruct-GGUF",
103
- glob_primary = "*Q4_K_M*",
104
- glob_fallback= "*Q4_K_S*",
105
  size_gb = 9.5,
106
- ctx_size = 12288, # ุฃุตุบุฑ context ู„ุฃู† ุงู„ุฃูˆุฒุงู† ุฃูƒุจุฑ
107
  quality = 94,
108
- has_vision = True,
109
- has_thinking = True,
110
- mmproj_repo = "bartowski/Qwen3-14B-Instruct-GGUF",
111
- mmproj_glob = "*mmproj*",
112
- extra_flags = "--jinja --reasoning-format deepseek",
113
- temp=0.6, top_k=20, top_p=0.95, min_p=0.0,
114
- notes="Premium tier. ูŠุญุชุงุฌ 16GB ุตุงููŠ (HF CPU Pro). ู„ูˆ RAM ู…ุชุงุญ ุงุฎุชุงุฑู‡.",
115
- ),
116
-
117
- # โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•
118
- # TIER 4 โ€” ุงู„ู…ุณุชู‚ุจู„ (ุชุฑุงู‚ุจู‡ู… model_manager ุฃุณุจูˆุนูŠุงู‹)
119
- # โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•
120
- ModelDef(
121
- id = "qwen3-32b-q2",
122
- name = "Qwen3 32B โ€” IQ2_XXS (Future โ€” 24GB+)",
123
- repo = "bartowski/Qwen3-32B-Instruct-GGUF",
124
- glob_primary = "*IQ2_XXS*",
125
- glob_fallback= "*Q2_K*",
126
- size_gb = 12.0,
127
- ctx_size = 8192,
128
- quality = 98,
129
  has_vision = False,
130
  has_thinking = True,
 
 
131
  extra_flags = "--jinja --reasoning-format deepseek",
132
  temp=0.6, top_k=20, top_p=0.95, min_p=0.0,
133
- notes="ู„ู„ู…ุณุชู‚ุจู„ ู„ูˆ ุชุฑู‚ูŠุช ู„ู€ CPU Pro. ูŠุญุชุงุฌ >14GB RAM ุตุงููŠ.",
134
  ),
135
  ]
136
 
 
51
  CATALOG: list[ModelDef] = [
52
 
53
  # โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•
54
+ # TIER 1 โ€” DEFAULT: Qwen3-8B Q4_K_M
55
+ # Verified repo: unsloth/Qwen3-8B-GGUF (5.03 GB, ~8-12 tok/s on CPU)
56
+ # Exact filename confirmed on HF Hub: Qwen3-8B-Q4_K_M.gguf
57
  # โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•
58
  ModelDef(
59
  id = "qwen3-8b",
60
+ name = "Qwen3 8B Q4_K_M (unsloth)",
61
+ repo = "unsloth/Qwen3-8B-GGUF",
62
+ glob_primary = "Qwen3-8B-Q4_K_M.gguf",
63
+ glob_fallback= "Qwen3-8B-Q4_K_S.gguf",
64
+ size_gb = 5.1,
65
  ctx_size = 16384,
66
  quality = 88,
67
+ has_vision = False,
68
  has_thinking = True,
69
+ mmproj_repo = "",
70
+ mmproj_glob = "",
71
  extra_flags = "--jinja --reasoning-format deepseek",
72
  temp=0.6, top_k=20, top_p=0.95, min_p=0.0,
73
+ notes="Default. Verified filename on HF Hub. 5.03GB. ~8-12 tok/s on CPU.",
 
74
  ),
75
 
76
  # โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•
77
+ # TIER 2 โ€” FAST: Qwen3-4B Q4_K_M (fits in 4GB, ~20-30 tok/s)
78
+ # Verified repo: unsloth/Qwen3-4B-GGUF (2.5 GB)
79
+ # Exact filename confirmed: Qwen3-4B-Q4_K_M.gguf
80
  # โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•
81
  ModelDef(
82
  id = "qwen3-4b",
83
+ name = "Qwen3 4B Q4_K_M (unsloth)",
84
+ repo = "unsloth/Qwen3-4B-GGUF",
85
+ glob_primary = "Qwen3-4B-Q4_K_M.gguf",
86
+ glob_fallback= "Qwen3-4B-Q4_K_S.gguf",
87
+ size_gb = 2.5,
88
  ctx_size = 16384,
89
  quality = 76,
90
+ has_vision = False,
91
  has_thinking = True,
92
+ mmproj_repo = "",
93
+ mmproj_glob = "",
94
  extra_flags = "--jinja --reasoning-format deepseek",
95
  temp=0.6, top_k=20, top_p=0.95, min_p=0.0,
96
+ notes="Fast fallback. 2.5GB. ~20-30 tok/s on CPU. Good for simple tasks.",
97
  ),
98
 
99
  # โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•
100
+ # TIER 3 โ€” PREMIUM: Qwen3-14B (needs 10GB+ RAM)
101
  # โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•
102
  ModelDef(
103
  id = "qwen3-14b",
104
+ name = "Qwen3 14B Q4_K_M (unsloth)",
105
+ repo = "unsloth/Qwen3-14B-GGUF",
106
+ glob_primary = "Qwen3-14B-Q4_K_M.gguf",
107
+ glob_fallback= "*Q4_K_M*",
108
  size_gb = 9.5,
109
+ ctx_size = 12288,
110
  quality = 94,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
111
  has_vision = False,
112
  has_thinking = True,
113
+ mmproj_repo = "",
114
+ mmproj_glob = "",
115
  extra_flags = "--jinja --reasoning-format deepseek",
116
  temp=0.6, top_k=20, top_p=0.95, min_p=0.0,
117
+ notes="Premium. Needs 16GB+ RAM. ~4-6 tok/s on CPU.",
118
  ),
119
  ]
120