Bound stage generation budgets with Codex
Browse files
README.md
CHANGED
|
@@ -31,6 +31,8 @@ The backend always executes seven isolated modules sequentially:
|
|
| 31 |
|
| 32 |
Every module attempts its own small-model call. If one call fails, only that stage uses a deterministic fallback and the pipeline continues. Runtime Details shows the source used by every stage.
|
| 33 |
|
|
|
|
|
|
|
| 34 |
## Topologies
|
| 35 |
|
| 36 |
- Single Prompt
|
|
|
|
| 31 |
|
| 32 |
Every module attempts its own small-model call. If one call fails, only that stage uses a deterministic fallback and the pipeline continues. Runtime Details shows the source used by every stage.
|
| 33 |
|
| 34 |
+
Each module also has a bounded token budget appropriate to its contract. `CONTEXTFORGE_MAX_NEW_TOKENS` is the global ceiling, while stage budgets keep the seven-call CPU path practical.
|
| 35 |
+
|
| 36 |
## Topologies
|
| 37 |
|
| 38 |
- Single Prompt
|
app.py
CHANGED
|
@@ -46,6 +46,15 @@ STAGE_NAMES = [
|
|
| 46 |
"qa_repair",
|
| 47 |
"final_assembly",
|
| 48 |
]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 49 |
|
| 50 |
|
| 51 |
def parse_bool_env(name: str, default: bool = False) -> bool:
|
|
@@ -237,7 +246,7 @@ def generate_json(stage: str, instruction: str, payload: dict[str, Any]) -> tupl
|
|
| 237 |
with torch.no_grad():
|
| 238 |
output_ids = model.generate(
|
| 239 |
**inputs,
|
| 240 |
-
max_new_tokens=MAX_NEW_TOKENS,
|
| 241 |
do_sample=False,
|
| 242 |
repetition_penalty=1.05,
|
| 243 |
pad_token_id=tokenizer.eos_token_id,
|
|
|
|
| 46 |
"qa_repair",
|
| 47 |
"final_assembly",
|
| 48 |
]
|
| 49 |
+
STAGE_TOKEN_BUDGETS = {
|
| 50 |
+
"intake_analysis": 180,
|
| 51 |
+
"topology_decision": 140,
|
| 52 |
+
"vital_structure": 180,
|
| 53 |
+
"reasoning_architecture": 240,
|
| 54 |
+
"prompt_pack_generation": 520,
|
| 55 |
+
"qa_repair": 260,
|
| 56 |
+
"final_assembly": 260,
|
| 57 |
+
}
|
| 58 |
|
| 59 |
|
| 60 |
def parse_bool_env(name: str, default: bool = False) -> bool:
|
|
|
|
| 246 |
with torch.no_grad():
|
| 247 |
output_ids = model.generate(
|
| 248 |
**inputs,
|
| 249 |
+
max_new_tokens=min(MAX_NEW_TOKENS, STAGE_TOKEN_BUDGETS.get(stage, MAX_NEW_TOKENS)),
|
| 250 |
do_sample=False,
|
| 251 |
repetition_penalty=1.05,
|
| 252 |
pad_token_id=tokenizer.eos_token_id,
|