RthItalia commited on
Commit
75b79d0
·
verified ·
1 Parent(s): cad26d4

Bound stage generation budgets with Codex

Browse files
Files changed (2) hide show
  1. README.md +2 -0
  2. app.py +10 -1
README.md CHANGED
@@ -31,6 +31,8 @@ The backend always executes seven isolated modules sequentially:
31
 
32
  Every module attempts its own small-model call. If one call fails, only that stage uses a deterministic fallback and the pipeline continues. Runtime Details shows the source used by every stage.
33
 
 
 
34
  ## Topologies
35
 
36
  - Single Prompt
 
31
 
32
  Every module attempts its own small-model call. If one call fails, only that stage uses a deterministic fallback and the pipeline continues. Runtime Details shows the source used by every stage.
33
 
34
+ Each module also has a bounded token budget appropriate to its contract. `CONTEXTFORGE_MAX_NEW_TOKENS` is the global ceiling, while stage budgets keep the seven-call CPU path practical.
35
+
36
  ## Topologies
37
 
38
  - Single Prompt
app.py CHANGED
@@ -46,6 +46,15 @@ STAGE_NAMES = [
46
  "qa_repair",
47
  "final_assembly",
48
  ]
 
 
 
 
 
 
 
 
 
49
 
50
 
51
  def parse_bool_env(name: str, default: bool = False) -> bool:
@@ -237,7 +246,7 @@ def generate_json(stage: str, instruction: str, payload: dict[str, Any]) -> tupl
237
  with torch.no_grad():
238
  output_ids = model.generate(
239
  **inputs,
240
- max_new_tokens=MAX_NEW_TOKENS,
241
  do_sample=False,
242
  repetition_penalty=1.05,
243
  pad_token_id=tokenizer.eos_token_id,
 
46
  "qa_repair",
47
  "final_assembly",
48
  ]
49
+ STAGE_TOKEN_BUDGETS = {
50
+ "intake_analysis": 180,
51
+ "topology_decision": 140,
52
+ "vital_structure": 180,
53
+ "reasoning_architecture": 240,
54
+ "prompt_pack_generation": 520,
55
+ "qa_repair": 260,
56
+ "final_assembly": 260,
57
+ }
58
 
59
 
60
  def parse_bool_env(name: str, default: bool = False) -> bool:
 
246
  with torch.no_grad():
247
  output_ids = model.generate(
248
  **inputs,
249
+ max_new_tokens=min(MAX_NEW_TOKENS, STAGE_TOKEN_BUDGETS.get(stage, MAX_NEW_TOKENS)),
250
  do_sample=False,
251
  repetition_penalty=1.05,
252
  pad_token_id=tokenizer.eos_token_id,