Update app.js

app.js

@@ -1735,7 +1735,10 @@ SCOPE axl_meta [level: 0]
 END SCOPE
 `;
 
-function buildPrompt(n, recentDomains, maxVars=20) {
+function buildPrompt(n, recentDomains, 
+                     maxVars=8
+                    // maxVars=20
+                    ) {
     const recentStr = recentDomains.slice(-10).join(", ") || "none yet";
     return `${AXL_SPEC}
 
@@ -1881,83 +1884,80 @@ app.get("/domains", (_, res) => res.json(tracker.stats()));
 app.post("/generate", async (req, res) => {
     const n = parseInt(req.query.n ?? req.body.n ?? "50");
     const rps = Math.min(parseFloat(req.query.rps ?? req.body.rps ?? DEFAULT_RPS.toString()), MAX_RPS);
-    const syntheticFraction = parseFloat(req.query.synthetic_fraction ?? req.body.synthetic_fraction ?? "0.4");
+    const synthFraction = parseFloat(req.query.synthetic_fraction ?? req.body.synthetic_fraction ?? "0.4");
     const maxVars = parseInt(req.query.max_vars ?? req.body.max_vars ?? "20");
 
-    log(`Generation request: n=${n} rps=${rps} synthetic=${(syntheticFraction*100).toFixed(0)}%`, "HEAD");
+    log(`Generation request: n=${n} rps=${rps} synthetic=${(synthFraction * 100).toFixed(0)}%`, "HEAD");
 
-    const generated =[];
-    let errors = 0;
+    let batchErrors = 0;
+    let newSamplesCount = 0;
 
-    // ── SYNTHETIC BATCH ──
+    // ── 1. SYNTHETIC BATCH (Immediate Save) ──
     const nSynthetic = Math.floor(n * syntheticFraction);
     if (nSynthetic > 0) {
-        const gens    = Object.values(SYNTHETIC_GENERATORS);
-        const perGen  = Math.max(1, Math.ceil(nSynthetic / gens.length));
-        
-        let tempSynth =[];
-        for (const genFn of gens) {
-            tempSynth.push(...genFn(perGen));
-        }
-        
-        // Shuffle slightly for mix, then slice to EXACTLY nSynthetic
-        tempSynth.sort(() => 0.5 - Math.random());
+        const gens = Object.values(SYNTHETIC_GENERATORS);
+        const perGen = Math.max(1, Math.ceil(nSynthetic / gens.length));
+        let tempSynth = [];
+        for (const genFn of gens) { tempSynth.push(...genFn(perGen)); }
         tempSynth = tempSynth.slice(0, nSynthetic);
 
         for (const raw of tempSynth) {
-            if (!validateSample(raw)) { errors++; continue; }
             const norm = normalizeSample(raw);
-            if (!norm) { errors++; continue; }
-            generated.push(norm);
-            tracker.record(norm.domain);
+            if (norm && validateSample(norm)) {
+                dataset.push(norm);
+                tracker.record(norm.domain);
+                newSamplesCount++;
+            } else { batchErrors++; }
         }
-        log(`Synthetic: ${generated.length} valid`, "OK");
+        saveDataset(); // Commit synthetics to disk immediately
+        log(`Synthetic batch committed to disk: ${newSamplesCount} samples`, "OK");
     }
 
-    // ── AI BATCH ──
-    const nAI         = n - generated.length;
+    // ── 2. AI BATCH (Incremental Save per Call) ──
+    const nAI = n - nSynthetic;
     const callsNeeded = Math.ceil(nAI / SAMPLES_PER_CALL);
     const minInterval = 1000 / rps;
-    log(`AI generation: ${nAI} samples via ${callsNeeded} Bedrock calls`);
 
-    for (let ci = 0; ci < callsNeeded; ci++) {
-        const t0 = Date.now();
-        try {
-            const batchN   = Math.min(SAMPLES_PER_CALL, nAI - generated.filter(s=>s.metadata?.source==="ai").length + 2);
-            const prompt   = buildPrompt(batchN, tracker.recent, maxVars);
-            const text     = await callBedrock(prompt);
-            const parsed   = parseAIResponse(text);
-            let valid = 0;
-            for (const raw of parsed) {
-                if (!validateSample(raw)) { errors++; continue; }
-                const norm = normalizeSample(raw);
-                if (!norm) { errors++; continue; }
-                norm.metadata = norm.metadata || {};
-                norm.metadata.source = "ai";
-                generated.push(norm);
-                tracker.record(norm.domain);
-                valid++;
+    // Send initial response so the connection doesn't timeout, 
+    // but the background loop keeps writing to disk.
+    res.json({ message: "Generation started. Use /dataset/download to track progress.", total_requested: n });
+
+    (async () => {
+        for (let ci = 0; ci < callsNeeded; ci++) {
+            const t0 = Date.now();
+            try {
+                const prompt = buildPrompt(SAMPLES_PER_CALL, tracker.recent, maxVars);
+                const text = await callBedrock(prompt);
+                const parsed = parseAIResponse(text);
+                
+                let callValid = 0;
+                for (const raw of parsed) {
+                    const norm = normalizeSample(raw);
+                    if (norm && validateSample(norm)) {
+                        norm.metadata = norm.metadata || {};
+                        norm.metadata.source = "ai";
+                        dataset.push(norm);
+                        tracker.record(norm.domain);
+                        callValid++;
+                        newSamplesCount++;
+                    } else { batchErrors++; }
+                }
+
+                // THE FIX: Save to disk after every successful Bedrock call
+                if (callValid > 0) {
+                    saveDataset();
+                    log(`Incremental Save: Call ${ci + 1}/${callsNeeded} committed (${callValid} samples). Total dataset: ${dataset.length}`, "OK");
+                }
+            } catch (e) {
+                log(`Bedrock call ${ci + 1} failed: ${e.message}`, "ERROR");
+                batchErrors++;
             }
-            log(`Call ${ci+1}/${callsNeeded}: ${valid}/${parsed.length} valid`, "OK");
-        } catch(e) {
-            log(`Bedrock call ${ci+1} failed: ${e.message}`, "ERROR");
-            errors++;
+            
+            const elapsed = Date.now() - t0;
+            if (elapsed < minInterval) await new Promise(r => setTimeout(r, minInterval - elapsed));
         }
-        // rate limit
-        const elapsed = Date.now() - t0;
-        if (elapsed < minInterval) await new Promise(r=>setTimeout(r, minInterval - elapsed));
-    }
-
-    dataset.push(...generated);
-    saveDataset();
-
-    log(`Done: ${generated.length} new | total=${dataset.length} | errors=${errors}`, "OK");
-    res.json({
-        generated: generated.length,
-        errors,
-        total_dataset: dataset.length,
-        samples: generated,
-    });
+        log(`Batch Process Finished. New: ${newSamplesCount} | Errors: ${batchErrors}`, "HEAD");
+    })();
 });
 
 app.get("/dataset/download", (req, res) => {