Update app.py
Browse files
app.py
CHANGED
|
@@ -79,12 +79,14 @@ ort_session = None
|
|
| 79 |
model_load_error = None
|
| 80 |
MODEL_LOCK = asyncio.Lock()
|
| 81 |
|
| 82 |
-
#
|
| 83 |
kv_cache_input_names = []
|
| 84 |
kv_cache_output_names = []
|
| 85 |
model_input_names = []
|
| 86 |
model_output_names = []
|
| 87 |
-
|
|
|
|
|
|
|
| 88 |
|
| 89 |
# ---------- Helper Functions ----------
|
| 90 |
def _verify_api_key(request: Request) -> None:
|
|
@@ -135,7 +137,8 @@ def _create_ort_session(model_path: str) -> ort.InferenceSession:
|
|
| 135 |
|
| 136 |
async def _ensure_loaded():
|
| 137 |
global tokenizer, ort_session, model_load_error
|
| 138 |
-
global kv_cache_input_names, kv_cache_output_names, model_input_names, model_output_names
|
|
|
|
| 139 |
|
| 140 |
async with MODEL_LOCK:
|
| 141 |
if tokenizer is not None and ort_session is not None:
|
|
@@ -148,16 +151,32 @@ async def _ensure_loaded():
|
|
| 148 |
onnx_path = os.path.join(local_dir, "onnx", ONNX_MODEL_FILE)
|
| 149 |
ort_session = _create_ort_session(onnx_path)
|
| 150 |
|
| 151 |
-
|
| 152 |
-
|
|
|
|
|
|
|
|
|
|
| 153 |
|
| 154 |
kv_cache_input_names = [name for name in model_input_names if name.startswith("past_key_values")]
|
| 155 |
kv_cache_output_names = [name for name in model_output_names if name.startswith("present")]
|
| 156 |
-
has_num_logits_input = "num_logits_to_keep" in model_input_names
|
| 157 |
|
| 158 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 159 |
logger.info(f"KV cache inputs: {len(kv_cache_input_names)}, outputs: {len(kv_cache_output_names)}")
|
| 160 |
-
logger.info(f"
|
|
|
|
| 161 |
except Exception as e:
|
| 162 |
model_load_error = str(e)
|
| 163 |
logger.exception("Model loading failed")
|
|
@@ -212,11 +231,10 @@ def _sample_token(logits: np.ndarray, temperature: float, top_p: float) -> int:
|
|
| 212 |
return int(np.random.choice(len(probs), p=probs))
|
| 213 |
|
| 214 |
def _init_past_key_values(batch_size: int = 1) -> Dict[str, np.ndarray]:
|
| 215 |
-
"""Create zero-filled past_key_values tensors
|
| 216 |
past = {}
|
| 217 |
for inp in ort_session.get_inputs():
|
| 218 |
if inp.name.startswith("past_key_values"):
|
| 219 |
-
# Convert symbolic dimensions to actual values
|
| 220 |
shape = []
|
| 221 |
for dim in inp.shape:
|
| 222 |
if isinstance(dim, str):
|
|
@@ -225,12 +243,12 @@ def _init_past_key_values(batch_size: int = 1) -> Dict[str, np.ndarray]:
|
|
| 225 |
elif "sequence" in dim.lower():
|
| 226 |
shape.append(0) # start with zero sequence length
|
| 227 |
else:
|
| 228 |
-
shape.append(1)
|
| 229 |
elif isinstance(dim, int) and dim > 0:
|
| 230 |
shape.append(dim)
|
| 231 |
else:
|
| 232 |
-
shape.append(0)
|
| 233 |
-
past[inp.name] = np.zeros(shape, dtype=
|
| 234 |
return past
|
| 235 |
|
| 236 |
def _update_past_key_values(outputs: List[np.ndarray]) -> Dict[str, np.ndarray]:
|
|
@@ -244,17 +262,33 @@ def _update_past_key_values(outputs: List[np.ndarray]) -> Dict[str, np.ndarray]:
|
|
| 244 |
return new_past
|
| 245 |
|
| 246 |
def _prepare_inputs(input_ids: np.ndarray, attention_mask: np.ndarray, past_kv: Dict[str, np.ndarray]) -> Dict[str, np.ndarray]:
|
| 247 |
-
"""Build the input feed dictionary with all required inputs."""
|
| 248 |
feed = {
|
| 249 |
-
"input_ids": input_ids,
|
| 250 |
-
"attention_mask": attention_mask,
|
| 251 |
}
|
| 252 |
# Add KV cache tensors
|
| 253 |
for name in kv_cache_input_names:
|
| 254 |
-
feed[name] = past_kv.get(name, np.zeros((1, 0), dtype=
|
|
|
|
| 255 |
# Add num_logits_to_keep if required
|
| 256 |
-
if
|
| 257 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 258 |
return feed
|
| 259 |
|
| 260 |
def _generate_full_sync(
|
|
|
|
| 79 |
model_load_error = None
|
| 80 |
MODEL_LOCK = asyncio.Lock()
|
| 81 |
|
| 82 |
+
# Model metadata
|
| 83 |
kv_cache_input_names = []
|
| 84 |
kv_cache_output_names = []
|
| 85 |
model_input_names = []
|
| 86 |
model_output_names = []
|
| 87 |
+
num_logits_input_info = None # (name, expected_shape, dtype)
|
| 88 |
+
position_ids_input_info = None
|
| 89 |
+
past_kv_dtype = np.float32
|
| 90 |
|
| 91 |
# ---------- Helper Functions ----------
|
| 92 |
def _verify_api_key(request: Request) -> None:
|
|
|
|
| 137 |
|
| 138 |
async def _ensure_loaded():
|
| 139 |
global tokenizer, ort_session, model_load_error
|
| 140 |
+
global kv_cache_input_names, kv_cache_output_names, model_input_names, model_output_names
|
| 141 |
+
global num_logits_input_info, position_ids_input_info, past_kv_dtype
|
| 142 |
|
| 143 |
async with MODEL_LOCK:
|
| 144 |
if tokenizer is not None and ort_session is not None:
|
|
|
|
| 151 |
onnx_path = os.path.join(local_dir, "onnx", ONNX_MODEL_FILE)
|
| 152 |
ort_session = _create_ort_session(onnx_path)
|
| 153 |
|
| 154 |
+
# Cache input/output metadata
|
| 155 |
+
inputs = ort_session.get_inputs()
|
| 156 |
+
outputs = ort_session.get_outputs()
|
| 157 |
+
model_input_names = [inp.name for inp in inputs]
|
| 158 |
+
model_output_names = [out.name for out in outputs]
|
| 159 |
|
| 160 |
kv_cache_input_names = [name for name in model_input_names if name.startswith("past_key_values")]
|
| 161 |
kv_cache_output_names = [name for name in model_output_names if name.startswith("present")]
|
|
|
|
| 162 |
|
| 163 |
+
# Determine dtype for past_key_values (use first available)
|
| 164 |
+
for inp in inputs:
|
| 165 |
+
if inp.name.startswith("past_key_values"):
|
| 166 |
+
past_kv_dtype = np.float16 if inp.type == "tensor(float16)" else np.float32
|
| 167 |
+
break
|
| 168 |
+
|
| 169 |
+
# Check for special inputs
|
| 170 |
+
for inp in inputs:
|
| 171 |
+
if inp.name == "num_logits_to_keep":
|
| 172 |
+
num_logits_input_info = (inp.name, inp.shape, past_kv_dtype)
|
| 173 |
+
elif inp.name == "position_ids":
|
| 174 |
+
position_ids_input_info = (inp.name, inp.shape, np.int64)
|
| 175 |
+
|
| 176 |
+
logger.info(f"Model loaded: {MODEL_ID} ({MODEL_QUANTIZATION})")
|
| 177 |
logger.info(f"KV cache inputs: {len(kv_cache_input_names)}, outputs: {len(kv_cache_output_names)}")
|
| 178 |
+
logger.info(f"num_logits_to_keep: {num_logits_input_info is not None}")
|
| 179 |
+
logger.info(f"position_ids: {position_ids_input_info is not None}")
|
| 180 |
except Exception as e:
|
| 181 |
model_load_error = str(e)
|
| 182 |
logger.exception("Model loading failed")
|
|
|
|
| 231 |
return int(np.random.choice(len(probs), p=probs))
|
| 232 |
|
| 233 |
def _init_past_key_values(batch_size: int = 1) -> Dict[str, np.ndarray]:
|
| 234 |
+
"""Create zero-filled past_key_values tensors using the correct dtype and shapes."""
|
| 235 |
past = {}
|
| 236 |
for inp in ort_session.get_inputs():
|
| 237 |
if inp.name.startswith("past_key_values"):
|
|
|
|
| 238 |
shape = []
|
| 239 |
for dim in inp.shape:
|
| 240 |
if isinstance(dim, str):
|
|
|
|
| 243 |
elif "sequence" in dim.lower():
|
| 244 |
shape.append(0) # start with zero sequence length
|
| 245 |
else:
|
| 246 |
+
shape.append(1)
|
| 247 |
elif isinstance(dim, int) and dim > 0:
|
| 248 |
shape.append(dim)
|
| 249 |
else:
|
| 250 |
+
shape.append(0)
|
| 251 |
+
past[inp.name] = np.zeros(shape, dtype=past_kv_dtype)
|
| 252 |
return past
|
| 253 |
|
| 254 |
def _update_past_key_values(outputs: List[np.ndarray]) -> Dict[str, np.ndarray]:
|
|
|
|
| 262 |
return new_past
|
| 263 |
|
| 264 |
def _prepare_inputs(input_ids: np.ndarray, attention_mask: np.ndarray, past_kv: Dict[str, np.ndarray]) -> Dict[str, np.ndarray]:
|
| 265 |
+
"""Build the input feed dictionary with all required inputs and correct shapes/dtypes."""
|
| 266 |
feed = {
|
| 267 |
+
"input_ids": input_ids.astype(np.int64),
|
| 268 |
+
"attention_mask": attention_mask.astype(np.int64),
|
| 269 |
}
|
| 270 |
# Add KV cache tensors
|
| 271 |
for name in kv_cache_input_names:
|
| 272 |
+
feed[name] = past_kv.get(name, np.zeros((1, 0), dtype=past_kv_dtype))
|
| 273 |
+
|
| 274 |
# Add num_logits_to_keep if required
|
| 275 |
+
if num_logits_input_info is not None:
|
| 276 |
+
name, shape, dtype = num_logits_input_info
|
| 277 |
+
# The value should be 1 (only keep logits for the last token)
|
| 278 |
+
# Build tensor according to expected shape
|
| 279 |
+
if shape == [] or shape == [1] or shape == (1,):
|
| 280 |
+
# Scalar or 1D of length 1
|
| 281 |
+
feed[name] = np.array(1, dtype=np.int64)
|
| 282 |
+
else:
|
| 283 |
+
# Unexpected shape; fallback to scalar
|
| 284 |
+
feed[name] = np.array(1, dtype=np.int64)
|
| 285 |
+
|
| 286 |
+
# Add position_ids if required
|
| 287 |
+
if position_ids_input_info is not None:
|
| 288 |
+
# For prefill, position_ids are [0, 1, ..., seq_len-1]
|
| 289 |
+
seq_len = input_ids.shape[1]
|
| 290 |
+
feed["position_ids"] = np.arange(seq_len, dtype=np.int64).reshape(1, -1)
|
| 291 |
+
|
| 292 |
return feed
|
| 293 |
|
| 294 |
def _generate_full_sync(
|