sidmaz666 commited on
Commit
2afe5c2
·
verified ·
1 Parent(s): e3aa692

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +53 -19
app.py CHANGED
@@ -79,12 +79,14 @@ ort_session = None
79
  model_load_error = None
80
  MODEL_LOCK = asyncio.Lock()
81
 
82
- # Cached input/output names
83
  kv_cache_input_names = []
84
  kv_cache_output_names = []
85
  model_input_names = []
86
  model_output_names = []
87
- has_num_logits_input = False
 
 
88
 
89
  # ---------- Helper Functions ----------
90
  def _verify_api_key(request: Request) -> None:
@@ -135,7 +137,8 @@ def _create_ort_session(model_path: str) -> ort.InferenceSession:
135
 
136
  async def _ensure_loaded():
137
  global tokenizer, ort_session, model_load_error
138
- global kv_cache_input_names, kv_cache_output_names, model_input_names, model_output_names, has_num_logits_input
 
139
 
140
  async with MODEL_LOCK:
141
  if tokenizer is not None and ort_session is not None:
@@ -148,16 +151,32 @@ async def _ensure_loaded():
148
  onnx_path = os.path.join(local_dir, "onnx", ONNX_MODEL_FILE)
149
  ort_session = _create_ort_session(onnx_path)
150
 
151
- model_input_names = [inp.name for inp in ort_session.get_inputs()]
152
- model_output_names = [out.name for out in ort_session.get_outputs()]
 
 
 
153
 
154
  kv_cache_input_names = [name for name in model_input_names if name.startswith("past_key_values")]
155
  kv_cache_output_names = [name for name in model_output_names if name.startswith("present")]
156
- has_num_logits_input = "num_logits_to_keep" in model_input_names
157
 
158
- logger.info(f"Model loaded successfully: {MODEL_ID} ({MODEL_QUANTIZATION})")
 
 
 
 
 
 
 
 
 
 
 
 
 
159
  logger.info(f"KV cache inputs: {len(kv_cache_input_names)}, outputs: {len(kv_cache_output_names)}")
160
- logger.info(f"Has num_logits_to_keep input: {has_num_logits_input}")
 
161
  except Exception as e:
162
  model_load_error = str(e)
163
  logger.exception("Model loading failed")
@@ -212,11 +231,10 @@ def _sample_token(logits: np.ndarray, temperature: float, top_p: float) -> int:
212
  return int(np.random.choice(len(probs), p=probs))
213
 
214
  def _init_past_key_values(batch_size: int = 1) -> Dict[str, np.ndarray]:
215
- """Create zero-filled past_key_values tensors based on the session's input specs."""
216
  past = {}
217
  for inp in ort_session.get_inputs():
218
  if inp.name.startswith("past_key_values"):
219
- # Convert symbolic dimensions to actual values
220
  shape = []
221
  for dim in inp.shape:
222
  if isinstance(dim, str):
@@ -225,12 +243,12 @@ def _init_past_key_values(batch_size: int = 1) -> Dict[str, np.ndarray]:
225
  elif "sequence" in dim.lower():
226
  shape.append(0) # start with zero sequence length
227
  else:
228
- shape.append(1) # fallback
229
  elif isinstance(dim, int) and dim > 0:
230
  shape.append(dim)
231
  else:
232
- shape.append(0) # for any negative or zero
233
- past[inp.name] = np.zeros(shape, dtype=np.float32)
234
  return past
235
 
236
  def _update_past_key_values(outputs: List[np.ndarray]) -> Dict[str, np.ndarray]:
@@ -244,17 +262,33 @@ def _update_past_key_values(outputs: List[np.ndarray]) -> Dict[str, np.ndarray]:
244
  return new_past
245
 
246
  def _prepare_inputs(input_ids: np.ndarray, attention_mask: np.ndarray, past_kv: Dict[str, np.ndarray]) -> Dict[str, np.ndarray]:
247
- """Build the input feed dictionary with all required inputs."""
248
  feed = {
249
- "input_ids": input_ids,
250
- "attention_mask": attention_mask,
251
  }
252
  # Add KV cache tensors
253
  for name in kv_cache_input_names:
254
- feed[name] = past_kv.get(name, np.zeros((1, 0), dtype=np.float32))
 
255
  # Add num_logits_to_keep if required
256
- if has_num_logits_input:
257
- feed["num_logits_to_keep"] = np.array([1], dtype=np.int64)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
258
  return feed
259
 
260
  def _generate_full_sync(
 
79
  model_load_error = None
80
  MODEL_LOCK = asyncio.Lock()
81
 
82
+ # Model metadata
83
  kv_cache_input_names = []
84
  kv_cache_output_names = []
85
  model_input_names = []
86
  model_output_names = []
87
+ num_logits_input_info = None # (name, expected_shape, dtype)
88
+ position_ids_input_info = None
89
+ past_kv_dtype = np.float32
90
 
91
  # ---------- Helper Functions ----------
92
  def _verify_api_key(request: Request) -> None:
 
137
 
138
  async def _ensure_loaded():
139
  global tokenizer, ort_session, model_load_error
140
+ global kv_cache_input_names, kv_cache_output_names, model_input_names, model_output_names
141
+ global num_logits_input_info, position_ids_input_info, past_kv_dtype
142
 
143
  async with MODEL_LOCK:
144
  if tokenizer is not None and ort_session is not None:
 
151
  onnx_path = os.path.join(local_dir, "onnx", ONNX_MODEL_FILE)
152
  ort_session = _create_ort_session(onnx_path)
153
 
154
+ # Cache input/output metadata
155
+ inputs = ort_session.get_inputs()
156
+ outputs = ort_session.get_outputs()
157
+ model_input_names = [inp.name for inp in inputs]
158
+ model_output_names = [out.name for out in outputs]
159
 
160
  kv_cache_input_names = [name for name in model_input_names if name.startswith("past_key_values")]
161
  kv_cache_output_names = [name for name in model_output_names if name.startswith("present")]
 
162
 
163
+ # Determine dtype for past_key_values (use first available)
164
+ for inp in inputs:
165
+ if inp.name.startswith("past_key_values"):
166
+ past_kv_dtype = np.float16 if inp.type == "tensor(float16)" else np.float32
167
+ break
168
+
169
+ # Check for special inputs
170
+ for inp in inputs:
171
+ if inp.name == "num_logits_to_keep":
172
+ num_logits_input_info = (inp.name, inp.shape, past_kv_dtype)
173
+ elif inp.name == "position_ids":
174
+ position_ids_input_info = (inp.name, inp.shape, np.int64)
175
+
176
+ logger.info(f"Model loaded: {MODEL_ID} ({MODEL_QUANTIZATION})")
177
  logger.info(f"KV cache inputs: {len(kv_cache_input_names)}, outputs: {len(kv_cache_output_names)}")
178
+ logger.info(f"num_logits_to_keep: {num_logits_input_info is not None}")
179
+ logger.info(f"position_ids: {position_ids_input_info is not None}")
180
  except Exception as e:
181
  model_load_error = str(e)
182
  logger.exception("Model loading failed")
 
231
  return int(np.random.choice(len(probs), p=probs))
232
 
233
  def _init_past_key_values(batch_size: int = 1) -> Dict[str, np.ndarray]:
234
+ """Create zero-filled past_key_values tensors using the correct dtype and shapes."""
235
  past = {}
236
  for inp in ort_session.get_inputs():
237
  if inp.name.startswith("past_key_values"):
 
238
  shape = []
239
  for dim in inp.shape:
240
  if isinstance(dim, str):
 
243
  elif "sequence" in dim.lower():
244
  shape.append(0) # start with zero sequence length
245
  else:
246
+ shape.append(1)
247
  elif isinstance(dim, int) and dim > 0:
248
  shape.append(dim)
249
  else:
250
+ shape.append(0)
251
+ past[inp.name] = np.zeros(shape, dtype=past_kv_dtype)
252
  return past
253
 
254
  def _update_past_key_values(outputs: List[np.ndarray]) -> Dict[str, np.ndarray]:
 
262
  return new_past
263
 
264
  def _prepare_inputs(input_ids: np.ndarray, attention_mask: np.ndarray, past_kv: Dict[str, np.ndarray]) -> Dict[str, np.ndarray]:
265
+ """Build the input feed dictionary with all required inputs and correct shapes/dtypes."""
266
  feed = {
267
+ "input_ids": input_ids.astype(np.int64),
268
+ "attention_mask": attention_mask.astype(np.int64),
269
  }
270
  # Add KV cache tensors
271
  for name in kv_cache_input_names:
272
+ feed[name] = past_kv.get(name, np.zeros((1, 0), dtype=past_kv_dtype))
273
+
274
  # Add num_logits_to_keep if required
275
+ if num_logits_input_info is not None:
276
+ name, shape, dtype = num_logits_input_info
277
+ # The value should be 1 (only keep logits for the last token)
278
+ # Build tensor according to expected shape
279
+ if shape == [] or shape == [1] or shape == (1,):
280
+ # Scalar or 1D of length 1
281
+ feed[name] = np.array(1, dtype=np.int64)
282
+ else:
283
+ # Unexpected shape; fallback to scalar
284
+ feed[name] = np.array(1, dtype=np.int64)
285
+
286
+ # Add position_ids if required
287
+ if position_ids_input_info is not None:
288
+ # For prefill, position_ids are [0, 1, ..., seq_len-1]
289
+ seq_len = input_ids.shape[1]
290
+ feed["position_ids"] = np.arange(seq_len, dtype=np.int64).reshape(1, -1)
291
+
292
  return feed
293
 
294
  def _generate_full_sync(