app.js

/**
 * axl_dataset_server.js
 * ══════════════════════════════════════════════════════════════════════════════
 * AXL Dataset Generation Server — Full Rewrite
 *
 * What changed:
 *   1. ALL synthetic generators now produce multi-scope AXL programs
 *      matching the depth the AI produces (scope_depth 2-3, EMERGES,
 *      CONTRADICT, CONSERVES, STEP/CANDIDATE where applicable)
 *   2. POST-PROCESSING NORMALIZER: every sample's c_values are mapped
 *      to[0,1] relative to declared bounds before saving. No more
 *      raw voltages, payoffs, or angles in c_values.
 *   3. VALIDATION: catches string c_values, missing VARS prefix,
 *      mask length mismatch, non-finite b_raw — all logged and rejected.
 *   4. FOCUS TRACKER: guides AI generation away from overrepresented domains.
 *   5. DOWNLOAD endpoint returns dated filename.
 *   6. RPS throttle on AI calls, SAMPLES_PER_CALL configurable.
 *   7. AWS BEDROCK updated to ConverseCommand with NodeHttpHandler.
 *   8. [FIXED] Synthetic array strictly sliced to exact requested fraction.
 *   9.[FIXED] SAMPLES_PER_CALL lowered to 4 to prevent AWS token truncation.
 *
 * Endpoints:
 *   GET  /status
 *   POST /generate?n=50&rps=3&synthetic_fraction=0.4
 *   GET  /dataset/download
 *   POST /dataset/clear
 *   GET  /dataset/sample?n=5
 *   GET  /domains
 *
 * Usage:
 *   node axl_dataset_server.js
 *   PORT=8000 AWS_REGION=us-east-1 node axl_dataset_server.js
 * ══════════════════════════════════════════════════════════════════════════════
 */

import express      from "express";
import cors         from "cors";
import { randomUUID } from "crypto";
import fs           from "fs";
import path         from "path";
import {
    BedrockRuntimeClient,
    ConverseCommand
} from "@aws-sdk/client-bedrock-runtime";
import { NodeHttpHandler } from "@smithy/node-http-handler";

// ── CONFIG ────────────────────────────────────────────────────────────────────
const PORT             = parseInt(process.env.PORT          || "7860");
const AWS_REGION       = process.env.AWS_REGION             || "us-east-1";
const BEDROCK_MODEL_ID = "arn:aws:bedrock:us-east-1:106774395747:inference-profile/global.anthropic.claude-haiku-4-5-20251001-v1:0";
  //"arn:aws:bedrock:us-east-1:106774395747:inference-profile/global.anthropic.claude-sonnet-4-6";
       
const DATASET_PATH     = process.env.DATASET_PATH           || "axl_dataset.json";
const DEFAULT_RPS      = parseFloat(process.env.DEFAULT_RPS || "3");
const MAX_RPS          = parseFloat(process.env.MAX_RPS     || "8");
const SAMPLES_PER_CALL = parseInt(process.env.SAMPLES_PER_CALL || "4"); // Reduced to prevent token cutoff
const SCALE_K          = 30.0;
const LATENT_D_MAX     = 8; 

// ── LOGGING ───────────────────────────────────────────────────────────────────
const ts  = () => new Date().toISOString().slice(11, 23);
const log = (msg, level = "INFO") => {
    const icons = { INFO:"  ", WARN:"⚠ ", ERROR:"✖ ", OK:"✔ ", HEAD:"━━" };
    console.log(`[${ts()}] ${icons[level]||"  "} ${msg}`);
};

// ── MATH HELPERS ──────────────────────────────────────────────────────────────
const randint  = (lo, hi) => Math.floor(Math.random() * (hi - lo + 1)) + lo;
const rnd      = (lo, hi) => Math.random() * (hi - lo) + lo;
const fmt      = (v, d=6)  => Number(v.toFixed(d));
const safeNorm = (b) => {
    if (!isFinite(b)) return 0.0;
    return Math.sign(b) * Math.log1p(Math.abs(b)) / SCALE_K;
};
const randchoice = (arr) => arr[Math.floor(Math.random() * arr.length)];

// ── NORMALIZER ────────────────────────────────────────────────────────────────
function normalizeSample(s) {
    const parsed = parseVarsBounds(s.vars_text, s.n_vars);
    
    if (!Array.isArray(s.c_values) || s.c_values.length < s.n_vars) return null;
    
    const raw = s.c_values.slice(0, s.n_vars);

    const normalized = raw.map((v, i) => {
        if (typeof v !== "number" || !isFinite(v)) return null;
        const [lo, hi] = parsed[i] || [0, 1];
        if (hi <= lo) return Math.min(1, Math.max(0, v));
        return Math.min(1, Math.max(0, (v - lo) / (hi - lo)));
    });

    if (normalized.includes(null)) return null;

    const mask =[...Array(s.n_vars).fill(1), ...Array(Math.max(0, LATENT_D_MAX - s.n_vars)).fill(0)];

    return {
        ...s,
        c_values:  normalized.map(v => fmt(v)),
        mask:      mask,
        b_norm:    fmt(safeNorm(s.b_raw), 8),
        vars_text: ensureVarsPrefix(s.vars_text),
    };
}

function parseVarsBounds(varsText, nVars) {
    const bounds = [];
    const re = /IN\s*[\[({]?\s*(-?[\d.e+\-]+)[\s,]+(-?[\d.e+\-]+)\s*[\])}]?/gi;
    let m;
    while ((m = re.exec(varsText)) !== null) {
        bounds.push([parseFloat(m[1]), parseFloat(m[2])]);
    }
    while (bounds.length < nVars) bounds.push([0, 1]);
    return bounds;
}

function ensureVarsPrefix(vt) {
    if (!vt) return "VARS c0 IN [0 1]";
    const t = vt.trim();
    return t.startsWith("VARS") ? t : "VARS " + t;
}

// ── VALIDATION ────────────────────────────────────────────────────────────────
function validateSample(s) {
    const required =["domain","sense","n_vars","vars_text","expr_text",
                      "sense_text","b_raw","c_values","mask"];
    for (const f of required) {
        if (s[f] === undefined || s[f] === null) {
            log(`Reject [${s.id||"?"}]: missing field ${f}`, "WARN");
            return false;
        }
    }
    if (!Array.isArray(s.c_values))              { log(`Reject: c_values not array`,      "WARN"); return false; }
    if (s.c_values.length < s.n_vars)            { log(`Reject: c_values len mismatch`,   "WARN"); return false; }
    if (s.c_values.some(v => typeof v !== "number" || !isFinite(v))) {
        log(`Reject: non-numeric c_values in ${s.domain}`, "WARN"); return false;
    }
    if (!isFinite(s.b_raw))                      { log(`Reject: non-finite b_raw`,         "WARN"); return false; }
    if (!s.vars_text.trimStart().startsWith("VARS")) {
        log(`Auto-fix vars_text prefix for ${s.domain}`, "WARN");
    }
    if (!s.id) s.id = randomUUID();
    return true;
}

// ═════════════════════════════════════════════════════════════════════════════
// DEEP SYNTHETIC GENERATORS
// ═════════════════════════════════════════════════════════════════════════════

// ── PHYSICS ───────────────────────────────────────────────────────────────────
function synthPhysicsSamples(n) {
    const samples = [];
    const templates =[
        {
            name: "Kinetic Energy",
            fn: (c) => 0.5 * (c[0]) * (c[1])**2,
            bounds: [[0.1,100],[0.1,50]],
            units: "Joules",
            formula: "KE = 0.5 * m * v^2",
            conserves: "energy ACROSS elastic_collision",
        },
        {
            name: "Gravitational PE",
            fn: (c) => c[0] * 9.81 * c[1],
            bounds: [[0.1,100],[0.1,100]],
            units: "Joules",
            formula: "PE = m * g * h",
            conserves: "energy ACROSS conservative_field",
        },
        {
            name: "Ohm Power",
            fn: (c) => (c[0]**2) / Math.max(c[1], 0.001),
            bounds: [[0.1,240],[0.1,1000]],
            units: "Watts",
            formula: "P = V^2 / R",
            conserves: "charge ACROSS resistive_element",
        },
        {
            name: "Ideal Gas Temperature",
            fn: (c) => (c[0]*c[1]) / (c[2]*8.314),
            bounds: [[0.1,10],[0.001,0.1],[0.1,5]],
            units: "Kelvin",
            formula: "T = PV / nR",
            conserves: "particle_count ACROSS isothermal",
        },
        {
            name: "Snell Refraction",
            fn: (c) => (c[0]*2+1) * Math.sin(c[1]*1.5) / Math.max((c[2]*2+1), 0.001),
            bounds: [[0,1],[0,1],[0,1]],
            units: "dimensionless",
            formula: "n1*sin(t1)/n2",
            conserves: "wavevector_component ACROSS interface",
        },
        {
            name: "Centripetal Acceleration",
            fn: (c) => (c[0]**2) / Math.max(c[1], 0.001),
            bounds: [[0.1,50],[0.1,10]],
            units: "m/s²",
            formula: "a = v^2 / r",
            conserves: "angular_momentum ACROSS circular_motion",
        },
    ];

    for (let i = 0; i < n; i++) {
        const t = randchoice(templates);
        const cRaw = t.bounds.map(([lo,hi]) => rnd(lo, hi));
        try {
            const bRaw = t.fn(cRaw);
            if (!isFinite(bRaw)) continue;
            const nVars = t.bounds.length;
            const varsBounds = t.bounds.map(([lo,hi],j) => `c${j} IN [${lo} ${hi}]`).join(" ");

            const axl = [
                `SCOPE physics_primitives [level: 0]`,
                `  ASSUME measurement EXISTS`,
                `  ASSUME ${t.conserves}`,
                `END SCOPE`,
                ``,
                `SCOPE ${t.name.toLowerCase().replace(/ /g,"_")}[level: 1]`,
                `  ASSUME physics_primitives SCOPE HOLDS`,
                ``,
                `  ENTITY physical_system`,
                ...t.bounds.map(([lo,hi],j) =>
                    `    FIELD c${j}: REAL BOUND [${lo}, ${hi}]`),
                `    FIELD result: REAL`,
                `    DERIVES result FROM {${cRaw.map((_,j)=>`c${j}`).join(", ")}} BY ${t.name.toLowerCase().replace(/ /g,"_")}_formula`,
                `  END ENTITY`,
                ``,
                `  TRANSFORM ${t.name.toLowerCase().replace(/ /g,"_")}_formula`,
                `    INPUT  {${t.bounds.map((_,j)=>`c${j}: REAL BOUND [${t.bounds[j][0]}, ${t.bounds[j][1]}]`).join(", ")}}`,
                `    OUTPUT {result: REAL}`,
                `    PRODUCES result EQUALS ${t.formula}`,
                `    # Computed: ${t.fn(cRaw).toFixed(4)} ${t.units}`,
                `  END TRANSFORM`,
                ``,
                `  OBSERVE result EQUALS ${bRaw.toFixed(6)}`,
                `  UNKNOWN ${cRaw.map((_,j)=>`c${j}: REAL BOUND[${t.bounds[j][0]}, ${t.bounds[j][1]}]`).join("\n  UNKNOWN ")}`,
                ``,
                `  COLLAPSE ${t.name.toLowerCase().replace(/ /g,"_")}`,
                `    GIVEN {result EQUALS ${bRaw.toFixed(4)}}`,
                `    FIND  {${cRaw.map((_,j)=>`c${j}`).join(", ")}}`,
                `    CONFIDENCE 0.97`,
                `  END COLLAPSE`,
                `END SCOPE`,
            ].join("\n");

            samples.push({
                id: randomUUID(), domain: "physics", subdomain: t.name,
                sense: "EQUAL", n_vars: nVars,
                vars_text: `VARS ${varsBounds}`,
                expr_text: `EXPR ${t.formula}`,
                sense_text: "SENSE EQUAL",
                b_raw: fmt(bRaw), b_norm: fmt(safeNorm(bRaw), 8),
                c_values: cRaw.map(v => fmt(v)),
                mask: Array(nVars).fill(1),
                candidates:[], step_count: 0, axl_source: axl,
                metadata: {
                    description: `${t.name} constraint inversion: ${t.formula} = ${bRaw.toFixed(4)}`,
                    units: t.units, difficulty: "medium",
                    scope_depth: 2, has_emergence: false, has_contradiction: false
                }
            });
        } catch(e) {}
    }
    return samples;
}

// ── FINANCE ───────────────────────────────────────────────────────────────────
function synthFinanceSamples(n) {
    const samples =[];
    for (let i = 0; i < n; i++) {
        const kind = randchoice(["compound","loan","portfolio_variance","sharpe"]);
        try {
            let bRaw, cRaw, bounds, formula, desc, subdomain;

            if (kind === "compound") {
                const P = rnd(1000, 100000), r = rnd(0.01, 0.25), t = rnd(1, 30);
                bRaw = P * Math.exp(r * t);
                cRaw = [P, r, t];
                bounds = [[1000,100000],[0.01,0.25],[1,30]];
                formula = "P * exp(r * t)";
                subdomain = "compound_interest";
                desc = `Compound interest P=${P.toFixed(0)} r=${r.toFixed(3)} t=${t.toFixed(1)}y → ${bRaw.toFixed(2)}`;
            } else if (kind === "loan") {
                const P = rnd(50000,500000), r = rnd(0.002,0.02), np = randint(60,360);
                bRaw = P*r*(1+r)**np / ((1+r)**np - 1 + 1e-9);
                cRaw =[P, r, np];
                bounds = [[50000,500000],[0.002,0.02],[60,360]];
                formula = "P*r*(1+r)^n / ((1+r)^n - 1)";
                subdomain = "loan_payment";
                desc = `Monthly payment: ${bRaw.toFixed(2)}`;
            } else if (kind === "portfolio_variance") {
                const w =[rnd(0,1), rnd(0,1), rnd(0,1)];
                const s = w.reduce((a,b)=>a+b,0); w[0]/=s; w[1]/=s; w[2]/=s;
                const sig =[rnd(0.05,0.3), rnd(0.02,0.15), rnd(0.01,0.08)];
                const rho01 = rnd(-0.3,0.5), rho02 = rnd(-0.2,0.4), rho12 = rnd(-0.1,0.3);
                bRaw = w[0]**2*sig[0]**2 + w[1]**2*sig[1]**2 + w[2]**2*sig[2]**2
                     + 2*w[0]*w[1]*rho01*sig[0]*sig[1]
                     + 2*w[0]*w[2]*rho02*sig[0]*sig[2]
                     + 2*w[1]*w[2]*rho12*sig[1]*sig[2];
                cRaw = [w[0], w[1], w[2]];
                bounds = [[0,1],[0,1],[0,1]];
                formula = "portfolio variance with covariance matrix";
                subdomain = "portfolio_variance";
                desc = `Minimum variance portfolio: variance=${bRaw.toFixed(5)}`;
            } else {
                const ret =[rnd(0.05,0.25), rnd(0.02,0.12), rnd(0.01,0.05)];
                const risk =[rnd(0.1,0.4), rnd(0.05,0.2), rnd(0.01,0.05)];
                const w =[rnd(0.2,0.6), rnd(0.1,0.4), rnd(0.05,0.3)];
                const s = w.reduce((a,b)=>a+b,0); w[0]/=s; w[1]/=s; w[2]/=s;
                const portRet = w.reduce((a,v,j)=>a+v*ret[j],0);
                const portRisk = Math.sqrt(w.reduce((a,v,j)=>a+v**2*risk[j]**2,0));
                bRaw = portRet / Math.max(portRisk, 0.001);
                cRaw = w;
                bounds = [[0,1],[0,1],[0,1]];
                formula = "Sharpe = return / risk";
                subdomain = "sharpe_ratio";
                desc = `Sharpe ratio: ${bRaw.toFixed(3)}`;
            }

            if (!isFinite(bRaw)) continue;
            const nVars = cRaw.length;
            const varsBounds = bounds.map(([lo,hi],j)=>`c${j} IN [${lo} ${hi}]`).join(" ");

            const axl = [
                `SCOPE financial_primitives[level: 0]`,
                `  ASSUME capital EXISTS`,
                `  ASSUME time EXISTS`,
                `  CONSERVES total_capital APPROXIMATELY ACROSS ${subdomain}`,
                `END SCOPE`,
                ``,
                `SCOPE ${subdomain} [level: 1]`,
                `  ASSUME financial_primitives SCOPE HOLDS`,
                ``,
                `  ENTITY financial_system`,
                ...bounds.map(([lo,hi],j) => `    FIELD c${j}: REAL BOUND[${lo}, ${hi}]`),
                `    FIELD output: REAL`,
                `    DERIVES output FROM {${cRaw.map((_,j)=>`c${j}`).join(", ")}} BY ${subdomain}_formula`,
                `    TENDS output ${ kind==="portfolio_variance"?"MIN":"MAX" }`,
                `  END ENTITY`,
                ``,
                `  TRANSFORM ${subdomain}_formula`,
                `    INPUT  {${bounds.map(([lo,hi],j)=>`c${j}: REAL BOUND[${lo},${hi}]`).join(", ")}}`,
                `    OUTPUT {result: REAL}`,
                `    PRODUCES result EQUALS ${formula}`,
                `  END TRANSFORM`,
                ``,
                `  OBSERVE output EQUALS ${bRaw.toFixed(6)}`,
                `  UNKNOWN ${cRaw.map((_,j)=>`c${j}: REAL BOUND [${bounds[j][0]}, ${bounds[j][1]}]`).join("\n  UNKNOWN ")}`,
                ``,
                `  COLLAPSE ${subdomain}`,
                `    GIVEN {output EQUALS ${bRaw.toFixed(4)}}`,
                `    FIND  {${cRaw.map((_,j)=>`c${j}`).join(", ")}}`,
                `    CONFIDENCE 0.95`,
                `  END COLLAPSE`,
                `END SCOPE`,
            ].join("\n");

            samples.push({
                id: randomUUID(), domain: "finance", subdomain,
                sense: kind==="portfolio_variance" ? "MINIMIZE" : "EQUAL",
                n_vars: nVars,
                vars_text: `VARS ${varsBounds}`,
                expr_text: `EXPR FINANCE_${subdomain.toUpperCase()}`,
                sense_text: kind==="portfolio_variance" ? "SENSE MINIMIZE" : "SENSE EQUAL",
                b_raw: fmt(bRaw), b_norm: fmt(safeNorm(bRaw), 8),
                c_values: cRaw.map(v => fmt(v)),
                mask: Array(nVars).fill(1),
                candidates:[], step_count: 0, axl_source: axl,
                metadata: {
                    description: desc, units: "currency",
                    difficulty: "medium", scope_depth: 2,
                    has_emergence: false, has_contradiction: false
                }
            });
        } catch(e) {}
    }
    return samples;
}

// ── CRYPTOGRAPHY ─────────────────────────────────────────────────────────────
function synthXorSamples(n) {
    const samples =[];
    for (let i = 0; i < n; i++) {
        const key = randint(0,255), plain = randint(0,255);
        const cipher = key ^ plain;
        const keyBits   = Array.from({length:8},(_,b)=>(key   >>(7-b))&1);
        const plainBits = Array.from({length:8},(_,b)=>(plain >>(7-b))&1);

        const bitEntities = Array.from({length:8},(_,b)=>[
            `  ENTITY bit_${b}`,
            `    FIELD key_bit:   DISCRETE VALUES {0, 1} EQUALS ${keyBits[b]}`,
            `    FIELD plain_bit: DISCRETE VALUES {0, 1} EQUALS ${plainBits[b]}`,
            `    FIELD out_bit:   DISCRETE VALUES {0, 1}`,
            `    DERIVES out_bit FROM {key_bit, plain_bit} BY xor_gate`,
            `  END ENTITY`,
        ].join("\n")).join("\n");

        const axl =[
            `SCOPE xor_bit_level[level: 0]`,
            `  ASSUME bit EXISTS`,
            `  ASSUME bit DISCRETE VALUES {0, 1}`,
            `  TRANSFORM xor_gate`,
            `    INPUT  {a: bit, b: bit}`,
            `    OUTPUT {out: bit}`,
            `    PRODUCES 0 WHEN a EQUALS b`,
            `    PRODUCES 1 WHEN a NOT_EQUALS b`,
            `  END TRANSFORM`,
            bitEntities,
            `END SCOPE`,
            ``,
            `SCOPE xor_byte_level[level: 1]`,
            `  ASSUME xor_bit_level SCOPE HOLDS`,
            `  ENTITY byte_assembler`,
            `    FIELD value: INTEGER BOUND[0, 255]`,
            `    DERIVES value FROM {bit_7..bit_0} BY binary_weighted_sum`,
            `  END ENTITY`,
            `  CONSERVES bit_count ACROSS binary_weighted_sum`,
            `  ENTITY key_byte`,
            `    FIELD raw:        INTEGER BOUND[0, 255]`,
            `    FIELD normalised: REAL    BOUND [0, 1]`,
            `    DERIVES normalised FROM raw BY divide_255`,
            `  END ENTITY`,
            `  ENTITY plain_byte`,
            `    FIELD raw: INTEGER BOUND[0, 255] EQUALS ${plain}`,
            `  END ENTITY`,
            `  ENTITY cipher_byte`,
            `    FIELD raw: INTEGER BOUND[0, 255]`,
            `    DERIVES raw FROM {key_byte.raw, plain_byte.raw} BY xor_byte_transform`,
            `  END ENTITY`,
            `  TRANSFORM xor_byte_transform`,
            `    INPUT  {key: INTEGER BOUND [0,255], plain: INTEGER BOUND[0,255]}`,
            `    OUTPUT {cipher: INTEGER BOUND [0,255]}`,
            `    CONSERVES nothing`,
            `    CHANGES cipher`,
            `    PRODUCES cipher = key XOR plain`,
            `  END TRANSFORM`,
            `  OBSERVE cipher_byte.raw EQUALS ${cipher}`,
            `  UNKNOWN key_byte.normalised: REAL BOUND [0, 1]`,
            `  COLLAPSE xor_byte_level`,
            `    GIVEN cipher_byte.raw`,
            `    FIND  key_byte.normalised`,
            `    CONFIDENCE 0.99`,
            `  END COLLAPSE`,
            `END SCOPE`,
        ].join("\n");

        samples.push({
            id: randomUUID(), domain: "cryptography", subdomain: "xor_byte",
            sense: "EQUAL", n_vars: 1,
            vars_text: "VARS c0 IN [0 255]",
            expr_text: `EXPR XOR plain=${plain} cipher=${cipher} key=UNKNOWN`,
            sense_text: "SENSE EQUAL",
            b_raw: fmt(cipher), b_norm: fmt(safeNorm(cipher), 8),
            c_values:[key],
            mask: [1], candidates:[], step_count: 0, axl_source: axl,
            metadata: {
                description: `XOR byte: plain=${plain} key=${key} → cipher=${cipher}`,
                units: "character_code", difficulty: "medium",
                scope_depth: 2, has_emergence: false, has_contradiction: true,
                key_bits: keyBits, plain_bits: plainBits,
                out_bits: keyBits.map((kb,b)=>kb^plainBits[b])
            }
        });
    }
    return samples;
}

function synthCaesarSamples(n) {
    const samples =[];
    for (let i = 0; i < n; i++) {
        const msgLen = randint(1,4);
        const shift  = randint(0,25);
        const plains = Array.from({length:msgLen},()=>randint(0,25));
        const ciphers = plains.map(p=>(p+shift)%26);
        const bRaw   = ciphers.reduce((a,b)=>a+b,0);

        const charEntities = plains.map((p,idx)=>[
            `  ENTITY char_${idx}`,
            `    FIELD plaintext:  INTEGER DISCRETE VALUES {0..25} EQUALS ${p}`,
            `    FIELD ciphertext: INTEGER DISCRETE VALUES {0..25}`,
            `    DERIVES ciphertext FROM {plaintext, shift} BY caesar_transform`,
            `  END ENTITY`,
        ].join("\n")).join("\n");

        const axl = [
            `SCOPE alphabet_ring [level: 0]`,
            `  ASSUME integer EXISTS`,
            `  ASSUME modular_arithmetic HOLDS`,
            `  ENTITY alphabet`,
            `    FIELD size: INTEGER EQUALS 26`,
            `    BOUND any_char WITHIN [0, 25]`,
            `  END ENTITY`,
            `  TRANSFORM modular_add`,
            `    INPUT  {a: INTEGER, b: INTEGER, mod: INTEGER}`,
            `    OUTPUT {result: INTEGER}`,
            `    PRODUCES result = (a + b) % mod`,
            `    CONSERVES mod`,
            `  END TRANSFORM`,
            `END SCOPE`,
            ``,
            `SCOPE caesar_char [level: 1]`,
            `  ASSUME alphabet_ring SCOPE HOLDS`,
            `  TRANSFORM caesar_transform`,
            `    INPUT  {plaintext: INTEGER BOUND [0,25], shift: INTEGER BOUND [0,25]}`,
            `    OUTPUT {ciphertext: INTEGER BOUND [0,25]}`,
            `    PRODUCES ciphertext = modular_add(plaintext, shift, 26)`,
            `    CONSERVES alphabet_size`,
            `    CHANGES  character_identity`,
            `  END TRANSFORM`,
            `  ENTITY shift_key`,
            `    FIELD raw:        INTEGER BOUND [0, 25]`,
            `    FIELD normalised: REAL    BOUND [0, 1]`,
            `    DERIVES normalised FROM raw BY divide_25`,
            `  END ENTITY`,
            charEntities,
            `END SCOPE`,
            ``,
            `SCOPE caesar_message [level: 2]`,
            `  ASSUME caesar_char SCOPE HOLDS`,
            `  ENTITY message`,
            `    FIELD plaintext:  VECTOR OF INTEGER EQUALS[${plains.join(", ")}]`,
            `    FIELD ciphertext: VECTOR OF INTEGER`,
            `    FIELD length:     INTEGER EQUALS ${msgLen}`,
            `    DERIVES ciphertext FROM {plaintext, shift_key.raw} BY caesar_transform APPLIED_TO_EACH`,
            `  END ENTITY`,
            `  OBSERVE message.ciphertext EQUALS[${ciphers.join(", ")}]`,
            `  OBSERVE SUM(message.ciphertext) EQUALS ${bRaw}`,
            `  UNKNOWN shift_key.normalised: REAL BOUND [0, 1]`,
            `  COLLAPSE caesar_message`,
            `    GIVEN {message.plaintext, SUM(message.ciphertext)}`,
            `    FIND  shift_key.normalised`,
            `    CONFIDENCE 0.99`,
            `  END COLLAPSE`,
            `END SCOPE`,
        ].join("\n");

        samples.push({
            id: randomUUID(), domain: "cryptography", subdomain: "caesar",
            sense: "EQUAL", n_vars: 1,
            vars_text: "VARS c0 IN[0 25]",
            expr_text: `EXPR CAESAR plains=[${plains}] ciphers=[${ciphers}] shift=UNKNOWN`,
            sense_text: "SENSE EQUAL",
            b_raw: fmt(bRaw), b_norm: fmt(safeNorm(bRaw), 8),
            c_values: [shift],
            mask: [1], candidates:[], step_count: 0, axl_source: axl,
            metadata: {
                description: `Caesar: shift=${shift} plains=[${plains}] → [${ciphers}]`,
                units: "character_code_sum",
                difficulty: msgLen > 1 ? "medium" : "easy",
                scope_depth: 3, has_emergence: false, has_contradiction: true
            }
        });
    }
    return samples;
}

function synthVigenereSamples(n) {
    const samples =[];
    for (let i = 0; i < n; i++) {
        const keyLen = randint(2,5);
        const key    = Array.from({length:keyLen},()=>randint(0,25));
        const plain  = Array.from({length:keyLen},()=>randint(0,25));
        const cipher = plain.map((p,idx)=>(p+key[idx])%26);
        const bRaw   = cipher.reduce((a,b)=>a+b,0);

        const posEntities = plain.map((p,idx)=>[
            `  ENTITY position_${idx}`,
            `    FIELD plaintext:  INTEGER EQUALS ${p}`,
            `    FIELD key_char:   INTEGER BOUND [0, 25]`,
            `    FIELD ciphertext: INTEGER BOUND[0, 25]`,
            `    DERIVES ciphertext FROM {plaintext, key_char} BY modular_add_26`,
            `  END ENTITY`,
        ].join("\n")).join("\n");

        const axl = [
            `SCOPE alphabet_ring [level: 0]`,
            `  ASSUME integer EXISTS`,
            `  TRANSFORM modular_add_26`,
            `    INPUT  {a: INTEGER BOUND [0,25], b: INTEGER BOUND[0,25]}`,
            `    OUTPUT {result: INTEGER BOUND [0,25]}`,
            `    PRODUCES result = (a + b) % 26`,
            `  END TRANSFORM`,
            `END SCOPE`,
            ``,
            `SCOPE vigenere_position [level: 1]`,
            `  ASSUME alphabet_ring SCOPE HOLDS`,
            posEntities,
            `  EMERGES key_periodicity`,
            `    FROM COMPOSITION OF {position_0..position_${keyLen-1}}`,
            `    AT SCOPE vigenere_composition`,
            `    NOT PRESENT AT SCOPE alphabet_ring`,
            `  END EMERGES`,
            `END SCOPE`,
            ``,
            `SCOPE vigenere_composition [level: 2]`,
            `  ASSUME vigenere_position SCOPE HOLDS`,
            `  ENTITY key_vector`,
            `    FIELD chars:      VECTOR OF INTEGER BOUND [0, 25]`,
            `    FIELD normalised: VECTOR OF REAL    BOUND[0, 1]`,
            `    DERIVES normalised FROM chars BY divide_each_by_25`,
            `  END ENTITY`,
            `  ENTITY plaintext_vector`,
            `    FIELD chars: VECTOR OF INTEGER EQUALS [${plain.join(", ")}]`,
            `  END ENTITY`,
            `  ENTITY ciphertext_vector`,
            `    FIELD chars: VECTOR OF INTEGER`,
            `    DERIVES chars FROM {plaintext_vector.chars, key_vector.chars}`,
            `      BY modular_add_26 APPLIED_PAIRWISE`,
            `  END ENTITY`,
            `  OBSERVE ciphertext_vector.chars EQUALS[${cipher.join(", ")}]`,
            `  OBSERVE SUM(ciphertext_vector.chars) EQUALS ${bRaw}`,
            `  UNKNOWN key_vector.normalised: VECTOR OF REAL BOUND[0, 1]`,
            `  COLLAPSE vigenere_composition`,
            `    GIVEN {plaintext_vector.chars, ciphertext_vector.chars}`,
            `    FIND  key_vector.normalised`,
            `    CONFIDENCE 0.99`,
            `  END COLLAPSE`,
            `END SCOPE`,
        ].join("\n");

        const varsText = `VARS ${Array.from({length:keyLen},(_,j)=>`c${j} IN [0 25]`).join(" ")}`;
        samples.push({
            id: randomUUID(), domain: "cryptography",
            subdomain: `vigenere_${keyLen}char`,
            sense: "EQUAL", n_vars: keyLen, vars_text: varsText,
            expr_text: `EXPR VIGENERE keyLen=${keyLen} plains=[${plain}] ciphers=[${cipher}] key=UNKNOWN`,
            sense_text: "SENSE EQUAL",
            b_raw: fmt(bRaw), b_norm: fmt(safeNorm(bRaw), 8),
            c_values: key,
            mask: Array(keyLen).fill(1),
            candidates:[], step_count: 0, axl_source: axl,
            metadata: {
                description: `Vigenère ${keyLen}-char key=[${key}]`,
                units: "character_code_sum",
                difficulty: keyLen <= 2 ? "medium" : keyLen <= 4 ? "hard" : "expert",
                scope_depth: 3, has_emergence: true, has_contradiction: true
            }
        });
    }
    return samples;
}

function synthAffineSamples(n) {
    const VALID_A   =[1,3,5,7,9,11,15,17,19,21,23,25];
    const INV_A_MAP = {1:1,3:9,5:21,7:15,9:3,11:19,15:7,17:23,19:11,21:5,23:17,25:25};
    const samples   =[];
    for (let i = 0; i < n; i++) {
        const aIdx  = randint(0,11);
        const a     = VALID_A[aIdx];
        const b     = randint(0,25);
        const p     = randint(0,25);
        const cipher = (a*p+b)%26;
        const aInv  = INV_A_MAP[a];
        const verify = ((aInv*(cipher-b))%26+26)%26;
        if (verify !== p) continue;

        const axl =[
            `SCOPE modular_arithmetic [level: 0]`,
            `  ASSUME integer EXISTS`,
            `  TRANSFORM modular_multiply_add`,
            `    INPUT  {a: INTEGER, p: INTEGER, b: INTEGER, mod: INTEGER}`,
            `    OUTPUT {result: INTEGER}`,
            `    PRODUCES result = (a * p + b) % mod`,
            `  END TRANSFORM`,
            `  TRANSFORM modular_inverse`,
            `    INPUT  {a: INTEGER, mod: INTEGER}`,
            `    OUTPUT {a_inv: INTEGER}`,
            `    REQUIRES gcd(a, mod) EQUALS 1`,
            `    PRODUCES a_inv SUCH_THAT (a * a_inv) % mod EQUALS 1`,
            `  END TRANSFORM`,
            `END SCOPE`,
            ``,
            `SCOPE affine_cipher [level: 1]`,
            `  ASSUME modular_arithmetic SCOPE HOLDS`,
            `  ENTITY affine_key`,
            `    FIELD a:      INTEGER DISCRETE VALUES {1,3,5,7,9,11,15,17,19,21,23,25}`,
            `    FIELD b:      INTEGER BOUND[0, 25]`,
            `    FIELD a_idx:  INTEGER BOUND[0, 11]`,
            `    FIELD a_norm: REAL    BOUND[0, 1]`,
            `    FIELD b_norm: REAL    BOUND [0, 1]`,
            `    DERIVES a     FROM a_idx BY lookup_valid_a`,
            `    DERIVES a_norm FROM a_idx BY divide_11`,
            `    DERIVES b_norm FROM b    BY divide_25`,
            `    BOUND a COPRIME_TO 26`,
            `  END ENTITY`,
            `  ENTITY affine_transform_result`,
            `    FIELD plaintext:  INTEGER BOUND[0, 25] EQUALS ${p}`,
            `    FIELD ciphertext: INTEGER BOUND[0, 25]`,
            `    DERIVES ciphertext FROM {affine_key.a, plaintext, affine_key.b} BY modular_multiply_add`,
            `  END ENTITY`,
            `  ENTITY affine_inverse`,
            `    FIELD a_inv: INTEGER EQUALS ${aInv}`,
            `    FIELD recovery: INTEGER`,
            `    DERIVES recovery FROM {a_inv, ciphertext, b} BY modular_multiply_add`,
            `  END ENTITY`,
            `  CONTRADICT invertible WITH non_coprime_a`,
            `    AT SCOPE affine_cipher`,
            `    BOUNDARY_TYPE INFORMATION`,
            `  END CONTRADICT`,
            `  OBSERVE affine_transform_result.ciphertext EQUALS ${cipher}`,
            `  UNKNOWN affine_key.a_norm: REAL BOUND[0, 1]`,
            `  UNKNOWN affine_key.b_norm: REAL BOUND[0, 1]`,
            `  COLLAPSE affine_cipher`,
            `    GIVEN {affine_transform_result.plaintext, affine_transform_result.ciphertext}`,
            `    FIND  {affine_key.a_norm, affine_key.b_norm}`,
            `    CONFIDENCE 0.99`,
            `  END COLLAPSE`,
            `END SCOPE`,
        ].join("\n");

        samples.push({
            id: randomUUID(), domain: "cryptography", subdomain: "affine",
            sense: "EQUAL", n_vars: 2,
            vars_text: "VARS c0 IN[0 11] c1 IN [0 25]",
            expr_text: `EXPR AFFINE a=${a}(idx=${aIdx}) b=${b} plain=${p} cipher=${cipher}`,
            sense_text: "SENSE EQUAL",
            b_raw: fmt(cipher), b_norm: fmt(safeNorm(cipher), 8),
            c_values: [aIdx, b],
            mask: [1,1], candidates:[], step_count: 0, axl_source: axl,
            metadata: {
                description: `Affine a=${a} b=${b} plain=${p} → cipher=${cipher}`,
                units: "character_code", difficulty: "hard",
                scope_depth: 2, has_emergence: false, has_contradiction: true,
                a_inverse: aInv, coprimality_proof: `gcd(${a},26)=1`
            }
        });
    }
    return samples;
}

// ── OPTIMIZATION (deep AXL) ───────────────────────────────────────────────────
function synthOptimizationSamples(n) {
    const samples =[];
    for (let i = 0; i < n; i++) {
        const sense  = randchoice(["MINIMIZE","MAXIMIZE"]);
        const nVars  = randint(2,6);
        try {
            const coeffs = Array.from({length:nVars}, ()=>rnd(0.1,10.0));
            const cOpt   = sense === "MINIMIZE" ? Array(nVars).fill(0.0) : Array(nVars).fill(1.0);
            const cVals  = cOpt.map(v=>Math.min(1.0,Math.max(0.0,v+rnd(-0.05,0.05))));
            const bRaw   = cVals.reduce((acc,v,j)=>acc+v*coeffs[j],0);
            const exprStr = coeffs.map((c,j)=>`${c.toFixed(3)}*c${j}`).join(" + ");

            const axl =[
                `SCOPE constraint_primitives [level: 0]`,
                `  ASSUME variable EXISTS`,
                `  ASSUME variable BOUND [0, 1]`,
                `  ASSUME objective EXISTS`,
                `END SCOPE`,
                ``,
                `SCOPE linear_program [level: 1]`,
                `  ASSUME constraint_primitives SCOPE HOLDS`,
                ``,
                `  ENTITY decision_variables`,
                ...Array.from({length:nVars},(_,j)=>
                    `    FIELD c${j}: REAL BOUND[0, 1]`),
                `  END ENTITY`,
                ``,
                `  ENTITY objective_function`,
                `    FIELD value: REAL`,
                `    DERIVES value FROM {${Array.from({length:nVars},(_,j)=>`c${j}`).join(", ")}} BY linear_combination`,
                `    TENDS value ${sense === "MINIMIZE" ? "MIN" : "MAX"}`,
                `  END ENTITY`,
                ``,
                `  TRANSFORM linear_combination`,
                `    INPUT  {${Array.from({length:nVars},(_,j)=>`c${j}: REAL BOUND [0,1]`).join(", ")}}`,
                `    OUTPUT {value: REAL}`,
                `    PRODUCES value EQUALS ${exprStr}`,
                `  END TRANSFORM`,
                ``,
                `  OBJECTIVE ${sense.toLowerCase()}_objective`,
                `    OPTIMIZE value ${sense}`,
                `    SUBJECT TO ${Array.from({length:nVars},(_,j)=>`c${j} WITHIN [0,1]`).join("\n    SUBJECT TO ")}`,
                `  END OBJECTIVE`,
                ``,
                `  OBSERVE value EQUALS ${bRaw.toFixed(6)}`,
                `  UNKNOWN ${Array.from({length:nVars},(_,j)=>`c${j}: REAL BOUND [0, 1]`).join("\n  UNKNOWN ")}`,
                ``,
                `  COLLAPSE linear_program`,
                `    GIVEN {value EQUALS ${bRaw.toFixed(4)}}`,
                `    FIND  {${Array.from({length:nVars},(_,j)=>`c${j}`).join(", ")}}`,
                `    CONFIDENCE 0.95`,
                `  END COLLAPSE`,
                `END SCOPE`,
            ].join("\n");

            samples.push({
                id: randomUUID(), domain: "optimization",
                subdomain: `linear_${sense.toLowerCase()}_${nVars}var`,
                sense, n_vars: nVars,
                vars_text: `VARS ${Array.from({length:nVars},(_,j)=>`c${j} IN[0 1]`).join(" ")}`,
                expr_text: `EXPR ${exprStr}`,
                sense_text: `SENSE ${sense}`,
                b_raw: fmt(bRaw), b_norm: fmt(safeNorm(bRaw), 8),
                c_values: cVals.map(v=>fmt(v)),
                mask: Array(nVars).fill(1),
                candidates:[], step_count: 0, axl_source: axl,
                metadata: {
                    description: `Linear ${sense.toLowerCase()} over ${nVars} variables`,
                    units: "objective", difficulty: "easy",
                    scope_depth: 2, has_emergence: false, has_contradiction: false
                }
            });
        } catch(e) {}
    }
    return samples;
}

// ── SYNTHETIC GENERATOR MAP ───────────────────────────────────────────────────
const SYNTHETIC_GENERATORS = {
    physics:              synthPhysicsSamples,
    finance:              synthFinanceSamples,
    cryptography_xor:     synthXorSamples,
    cryptography_caesar:  synthCaesarSamples,
    cryptography_vigenere:synthVigenereSamples,
    cryptography_affine:  synthAffineSamples,
    optimization:         synthOptimizationSamples,
};

// ── FOCUS TRACKER ─────────────────────────────────────────────────────────────
const tracker = {
    recent:[], counts: {}, total: 0,
    record(domain) {
        this.recent.push(domain);
        if (this.recent.length > 100) this.recent.shift();
        this.counts[domain] = (this.counts[domain]||0) + 1;
        this.total++;
    },
    stats() {
        return { total: this.total, counts: this.counts, recent: this.recent.slice(-20) };
    }
};

// ── DATASET STATE ─────────────────────────────────────────────────────────────
let dataset =[];
function loadDataset() {
    if (fs.existsSync(DATASET_PATH)) {
        dataset = JSON.parse(fs.readFileSync(DATASET_PATH,"utf8"));
        log(`Loaded ${dataset.length} existing samples`, "OK");
    }
}
function saveDataset() {
    fs.writeFileSync(DATASET_PATH, JSON.stringify(dataset, null, 2));
    log(`Saved ${dataset.length} samples → ${DATASET_PATH}`, "OK");
}

// ── BEDROCK CLIENT ────────────────────────────────────────────────────────────

// Initialize Bedrock Client with NodeHttpHandler explicitly
const bedrockClient = new BedrockRuntimeClient({
    region: AWS_REGION,
    requestHandler: new NodeHttpHandler({
        http2Handler: undefined, 
    }) 
});

const AXL_SPEC = `
# AXL — Abstraction Exchange Language
### A Constraint-First, Bidirectionally Navigable Abstraction Language
---

## PHILOSOPHY

AXL treats existence as a consequence of constraint satisfaction.
Things exist not because they are declared, but because they are
the only things consistent with the active assumptions and bounds.
The language navigates upward (increasing abstraction, emergence)
and downward (reducing abstraction, collapse to concrete).

The system does not know biology, physics, or cryptography.
It knows only: what is bounded, what is tended toward, what is
observed, and what must follow. Domain knowledge lives in the
encoder. AXL is what survives after domain is stripped away.

---

## PART I — FULL VOCABULARY

### 1. SCOPE DECLARATION
Declares an abstraction level. Everything inside a scope inherits
its level tag. Scopes are numbered — lower is more fundamental.

SCOPE <name> [level: <int>]
  ...contents...
END SCOPE

Example levels (not hardcoded, user defined):
- level 0 = most fundamental (quantum, bits)
- level 1 = composed structures (atoms, bytes)
- level 2 = functional systems (molecules, data blocks)
- level 3 = emergent behavior (cells, algorithms)
- level N = arbitrary abstraction ceiling

---

### 2. ENTITY
A thing that exists within a scope. Has fields, states, and
existence conditions. An entity does not exist unless its
REQUIRES clause is satisfied.

ENTITY <name>
  FIELD <name>: <type>[BOUND <range>] [CONSERVES] [DISCRETE]
  STATE <name>: REQUIRES <condition>
  TENDS <field> <direction>
  DERIVES FROM <entity_list>
END ENTITY

**FIELD** — a measurable or logical property of the entity
**CONSERVES** — this field cannot change across any TRANSFORM
**DISCRETE** — field only takes integer or enumerated values
**TENDS** — soft directional pull on a field (not a hard target)

---

### 3. ASSUME
Axiom declaration. Taken as true without derivation.
The foundation everything else builds on.
Multiple ASSUMEs compound — all must hold simultaneously.

ASSUME <statement>
ASSUME <entity> EXISTS
ASSUME <field> EQUALS <value>
ASSUME <relationship> HOLDS

---

### 4. BOUND
Hard constraint on a field or relationship.
Nothing outside a BOUND can exist in this scope.

BOUND <field> WITHIN [<min>, <max>]
BOUND <field> ABOVE <value>
BOUND <field> BELOW <value>
BOUND <field> EQUALS <value>          # exact, no freedom
BOUND <entity> COUNT WITHIN [<min>, <max>]
BOUND <a> FORBIDS <b>                 # mutual exclusion

---

### 5. TEND
Soft objective. The system pulls toward this but does not
require it. Multiple TENDs create competing pressures that
produce equilibrium — emergence happens here.

TEND <field> MIN                      # pull toward minimum
TEND <field> MAX                      # pull toward maximum
TEND <field> TOWARD <value>           # pull toward specific value
TEND <system> STABLE                  # resist perturbation
TEND <field> FOLLOW <other_field>     # track another field

---

### 6. OBJECTIVE
Hard optimization target. Unlike TEND, this must be satisfied
for the scope to be considered resolved.

OBJECTIVE <name>
  OPTIMIZE <field>[MIN | MAX | EQUAL <value>]
  SUBJECT TO <constraint_list>
END OBJECTIVE

---

### 7. DERIVE
Declares that something follows necessarily from other things.
If the sources hold, the derived thing must hold.
This is the upward propagation engine.

DERIVE <entity | field | state>
  FROM <source_list>
  WHEN <condition>
  BY <transform_name>
END DERIVE

---

### 8. TRANSFORM
A process that maps one configuration to another.
Transforms are the edges between scope levels.
They must declare what they CONSERVE and what they CHANGE.

TRANSFORM <name>
  INPUT <entity_or_field_list>
  OUTPUT <entity_or_field_list>
  CONSERVES <field_list>
  CHANGES <field_list>
  REQUIRES <precondition>
  PRODUCES <postcondition>
END TRANSFORM

---

### 9. OBSERVE
What has been measured. This is the B value in the UCE.
The system will work backwards from this to find what
configuration of unknowns is consistent with it.

OBSERVE <field | entity | state> EQUALS <value>
OBSERVE <field> WITHIN [<min>, <max>]
OBSERVE <pattern> IN <output>

---

### 10. UNKNOWN
What we are solving for. The C vector in the UCE.
The system navigates the abstraction domain to find
values of UNKNOWNs that satisfy all constraints
given the OBSERVATIONs.

UNKNOWN <name>: <type> BOUND[<min>, <max>]
UNKNOWN <name>: DISCRETE WITHIN <set>

---

### 11. COLLAPSE
Instruction to reduce abstraction — find the most concrete
configuration satisfying all active constraints.
This is reverse abstraction in one word.

COLLAPSE <scope | entity | system>
  GIVEN <observation_list>
  FIND <unknown_list>
  CONFIDENCE <threshold>
END COLLAPSE

---

### 12. EXPAND
Opposite of COLLAPSE. Given a concrete configuration,
derive upward — what higher-level behaviors emerge?

EXPAND <entity | configuration>
  THROUGH <scope_list>
  OBSERVE_EMERGENT <property_list>
END EXPAND

---

### 13. CONTRADICT
Flags that two things cannot both be true simultaneously.
Not an error — a scope boundary marker.
Where contradictions appear is where the interesting
physics, biology, and logic lives.

CONTRADICT <statement_a> WITH <statement_b>
  AT SCOPE <level>
  BOUNDARY_TYPE[SCALE | SPEED | INFORMATION | ENERGY]

---

### 14. EMERGES
Declares a property that does not exist at lower scopes
but appears when lower entities compose.
Emergence is not reducible to its parts in AXL —
it must be explicitly declared.

EMERGES <property>
  FROM COMPOSITION OF <entity_list>
  AT SCOPE <level>
  NOT PRESENT AT SCOPE <lower_level>

---

### 15. COMPOSES
Declares that an entity is built from other entities.
Composition respects all lower-scope constraints.

COMPOSES <higher_entity>
  FROM <lower_entity_list>
  BOUND count: <range>
  CONSERVING <field_list>
END COMPOSES

---

### 16. STEP
Sequence counter for iterative or feedback processes.
Enables the system to learn groups across attempts.
Critical for the mining feedback loop.

STEP <n>
  STATE <snapshot>
  CANDIDATE <configuration>
  DELTA <change_from_previous>
END STEP

---

### 17. CANDIDATE
A proposed solution configuration.
Multiple candidates can coexist until COLLAPSE selects.

CANDIDATE <n>
  <field>: <value>
  SCORE: <objective_value>
  CONFIDENCE: <float 0-1>
END CANDIDATE

---

### 18. UNLESS
Conditional override. Higher scope can override lower scope
constraints under specific conditions.

UNLESS <condition>
  OVERRIDE <constraint>
  WITH <replacement>
END UNLESS

---

### 19. CONSERVES
Declares an invariant across the entire system regardless
of what transforms occur. Conservation laws in physics,
invariants in computation, conservation of mass — all CONSERVES.

CONSERVES <quantity> ACROSS <transform_list>
CONSERVES <quantity> WITHIN SCOPE <level>

---

### 20. SAMPLE
A specific observed instance used as training signal
for the diffusion engine. The raw material for learning.

SAMPLE <id>
  INPUT <configuration>
  OUTPUT <observed_value>
  STEP <n>
END SAMPLE


---

## PART II — GRAMMAR RULES

program         := scope_block+
scope_block     := SCOPE name [level] body END SCOPE
body            := statement*
statement       := assume | entity | bound | tend | objective
                 | derive | transform | observe | unknown
                 | collapse | expand | contradict | emerges
                 | composes | step | candidate | conserves
                 | sample | unless

condition       := field op value | entity EXISTS | state ACTIVE
                 | condition AND condition | condition OR condition
                 | NOT condition

range           := [min, max] | {discrete_set}
direction       := MIN | MAX | STABLE | TOWARD value | FOLLOW field
type            := REAL | INTEGER | BOOLEAN | SYMBOLIC | VECTOR
op              := EQUALS | ABOVE | BELOW | WITHIN | FORBIDS

---

## PART III — DEMONSTRATIONS

---

### DEMONSTRATION 1: THE ATOM
#### Encoding atomic structure from quantum scope upward

axl:
# ════════════════════════════════════════════════════
# ATOM SYSTEM
# Starting assumption: energy exists and is quantized
# ════════════════════════════════════════════════════

SCOPE quantum[level: 0]

  ASSUME energy EXISTS
  ASSUME energy DISCRETE                    # quantization axiom
  ASSUME charge EXISTS
  CONSERVES charge ACROSS ALL_TRANSFORMS
  CONSERVES energy ACROSS ALL_TRANSFORMS UNLESS transform EQUALS radiation

  ENTITY quark
    FIELD charge: REAL BOUND [-1, 1] DISCRETE VALUES {-2/3, -1/3, 1/3, 2/3}
    FIELD color: SYMBOLIC DISCRETE VALUES {red, green, blue, antired, antigreen, antiblue}
    FIELD mass: REAL BOUND [0.002, 4.5] # GeV
    STATE free: REQUIRES energy ABOVE confinement_threshold
    STATE confined: REQUIRES energy BELOW confinement_threshold
    TENDS state TOWARD confined                # quarks want to be bound
  END ENTITY

  ENTITY gluon
    FIELD charge: BOUND[0, 0] CONSERVES
    FIELD color: SYMBOLIC                      # carries color charge
    TENDS quark_binding MAX                    # gluons maximize binding
  END ENTITY

  BOUND quark COLOR_NEUTRAL IN any_free_composite  # color confinement

END SCOPE


SCOPE hadron [level: 1]

  ASSUME quantum SCOPE HOLDS                  # inherit all quantum rules

  COMPOSES proton
    FROM quark COUNT EQUALS 3
    CONFIGURATION {up, up, down}              # charge = 2/3+2/3-1/3 = +1
    CONSERVING charge
    BOUND total_charge EQUALS 1
    BOUND color EQUALS neutral                # must be colorless
  END COMPOSES

  COMPOSES neutron
    FROM quark COUNT EQUALS 3
    CONFIGURATION {up, down, down}            # charge = 2/3-1/3-1/3 = 0
    CONSERVING charge
    BOUND total_charge EQUALS 0
    BOUND color EQUALS neutral
  END COMPOSES

  EMERGES stability
    FROM COMPOSITION OF {proton, neutron}
    AT SCOPE hadron
    NOT PRESENT AT SCOPE quantum
    # individual quarks don't have nuclear stability — it emerges here

END SCOPE


SCOPE nucleus[level: 2]

  ASSUME hadron SCOPE HOLDS

  COMPOSES nucleus
    FROM {proton, neutron}
    BOUND proton_count WITHIN [1, 118]        # known elements
    BOUND neutron_count WITHIN [0, 177]
    TEND binding_energy MAX                   # nucleus maximizes binding
    TEND neutron_to_proton_ratio TOWARD 1.0   # stability tendency
  END COMPOSES

  ENTITY strong_force
    FIELD range: BOUND[0, 3e-15]             # femtometers — very short range
    FIELD strength: REAL
    TENDS nucleon_separation MIN              # pulls nucleons together
  END ENTITY

  ENTITY weak_force
    TRANSFORMS neutron INTO proton
    WHEN neutron_to_proton_ratio ABOVE stable_ratio
    PRODUCES {proton, electron, antineutrino}  # beta decay
    CONSERVING charge
    CONSERVING lepton_number
  END ENTITY

  EMERGES atomic_number
    FROM proton COUNT
    AT SCOPE nucleus
    # proton count alone determines element identity — pure emergence

END SCOPE


SCOPE atom[level: 3]

  ASSUME nucleus SCOPE HOLDS

  ENTITY electron
    FIELD charge: BOUND[-1, -1] CONSERVES
    FIELD mass: BOUND[9.109e-31, 9.109e-31] CONSERVES
    FIELD spin: DISCRETE VALUES {-0.5, 0.5}
    FIELD orbital: DISCRETE VALUES {s, p, d, f}
    FIELD energy_level: INTEGER BOUND [1, 7]
    TENDS energy MIN                          # electrons seek lowest energy
    BOUND same_orbital_same_spin FORBIDS      # Pauli exclusion principle
  END ENTITY

  COMPOSES atom
    FROM {nucleus, electron}
    BOUND electron_count EQUALS proton_count  # neutral atom
    TEND electron ORBITAL_FILL lowest_energy_first  # Aufbau principle
  END COMPOSES

  EMERGES chemical_behavior
    FROM valence_electron_configuration
    AT SCOPE atom
    NOT PRESENT AT SCOPE nucleus
    # chemistry is purely a scope 3 emergence — nucleus knows nothing of it

  EMERGES periodic_properties
    FROM atomic_number MODULO orbital_filling_pattern
    AT SCOPE atom

END SCOPE


# ══ COLLAPSE EXAMPLE: Given observed spectral lines, find element ══

OBSERVE spectral_emission EQUALS {656nm, 486nm, 434nm, 410nm}

UNKNOWN element: SYMBOLIC WITHIN periodic_table
UNKNOWN atomic_number: INTEGER BOUND [1, 118]
UNKNOWN electron_configuration: VECTOR

COLLAPSE atom
  GIVEN {spectral_emission}
  FIND {element, atomic_number, electron_configuration}
  CONFIDENCE 0.95
END COLLAPSE

# Answer the system should derive: Hydrogen (Z=1)


---

### DEMONSTRATION 2: SHA-256 MINING
#### Encoding the constraint and the intelligent nonce search

axl:
# ════════════════════════════════════════════════════
# SHA-256 BITCOIN MINING SYSTEM
# Objective: find nonce such that hash < target
# The system does not know what SHA-256 is.
# It knows: there is a transform, it has an output,
# the output must satisfy a bound.
# ════════════════════════════════════════════════════

SCOPE bitfield[level: 0]

  ASSUME bit EXISTS
  ASSUME bit DISCRETE VALUES {0, 1}
  ENTITY bit_array
    FIELD length: INTEGER
    FIELD value: VECTOR of bit
    BOUND value COUNT EQUALS length
  END ENTITY

END SCOPE


SCOPE sha256_internals[level: 1]

  ASSUME bitfield SCOPE HOLDS

  ENTITY message_schedule
    FIELD W: VECTOR COUNT EQUALS 64        # 64 32-bit words
    DERIVES FROM input_block
    BY expansion_transform
  END ENTITY

  TRANSFORM round_function
    INPUT {working_vars: VECTOR COUNT 8, W_i: bit_array}
    OUTPUT {working_vars_next: VECTOR COUNT 8}
    CONSERVES bit_count
    CHANGES working_vars
    # 64 applications of this = full SHA-256
    # Each application: sigma, choice, majority, mod add
    # AXL does not encode the arithmetic — that's the domain encoder's job
    # AXL encodes only: this is a deterministic transform with known structure
  END TRANSFORM

  CONTRADICT input_a WITH input_b
    AT SCOPE sha256_internals
    BOUNDARY_TYPE INFORMATION
    # Two different inputs CAN produce same output (collision)
    # But finding one is computationally infeasible
    # This contradiction is the security boundary

  TRANSFORM sha256
    INPUT {message: bit_array BOUND length WITHIN [0, 2^64]}
    OUTPUT {digest: bit_array BOUND length EQUALS 256}
    CONSERVES nothing                       # one-way: no conservation
    CHANGES everything                      # avalanche: all output bits affected
    REQUIRES input EXISTS
    PRODUCES digest PSEUDO_RANDOM_FROM input
    # PSEUDO_RANDOM_FROM is the key declaration:
    # output is deterministic but statistically independent of input structure
  END TRANSFORM

END SCOPE


SCOPE block_header[level: 2]

  ASSUME sha256_internals SCOPE HOLDS

  ENTITY block_header
    FIELD version: INTEGER BOUND[1, 4]
    FIELD prev_hash: bit_array BOUND length EQUALS 256   # GIVEN, fixed
    FIELD merkle_root: bit_array BOUND length EQUALS 256 # GIVEN, fixed
    FIELD timestamp: INTEGER                              # GIVEN, approximately fixed
    FIELD bits: INTEGER                                   # GIVEN, encodes target
    FIELD nonce: INTEGER BOUND[0, 4294967295]           # UNKNOWN — 32 bits of freedom
  END ENTITY

  DERIVE target
    FROM bits
    BY difficulty_decode_transform
    # target = a 256-bit number. Valid hash must be below it.
    # More leading zeros in target = harder problem

END SCOPE


SCOPE mining [level: 3]

  ASSUME block_header SCOPE HOLDS

  TRANSFORM double_sha256
    INPUT {block_header}
    OUTPUT {hash: bit_array BOUND length EQUALS 256}
    BY {sha256 APPLIED_TWICE}
  END TRANSFORM

  OBJECTIVE valid_block
    OPTIMIZE hash MIN                        # minimize hash value numerically
    SUBJECT TO hash BELOW target             # hard bound — must be satisfied
    SUBJECT TO nonce WITHIN[0, 4294967295]
  END OBJECTIVE

  # ══ HERE IS THE INTELLIGENCE LAYER ══
  # The nonce space has NO smooth gradient (SHA-256 destroys it)
  # But the SEARCH SPACE has learnable structure:
  # - which regions have been explored (coverage)
  # - step count gives sequence context
  # - candidate scoring gives relative ranking
  # This is what the UCE learns — not the hash, but the search

  UNKNOWN nonce: INTEGER BOUND[0, 4294967295]

  STEP 0
    STATE {nonce_space: unexplored}
    CANDIDATE 0
      nonce: SAMPLE uniform
      SCORE: UNKNOWN
      CONFIDENCE: 0.0
    END CANDIDATE
  END STEP

  STEP N
    STATE {
      explored_regions: VECTOR,              # coverage map
      best_hash_so_far: bit_array,
      distance_to_target: INTEGER,
      step_count: N
    }
    CANDIDATE 0..50                          # 50 parallel candidates
      nonce: DERIVES FROM {
        explored_regions,
        step_count,
        best_hash_so_far
      }
      # The system learns: given search history,
      # what nonce regions have NOT been tried
      # and which historically neighboring regions
      # produced closer hashes (within this block's structure)
      SCORE: distance_to_target MIN
      CONFIDENCE: LEARNS
    END CANDIDATE
  END STEP

  OBSERVE hash BELOW target                  # this is the termination condition

  COLLAPSE mining
    GIVEN {block_header_fixed_fields, target, search_history}
    FIND {nonce}
    CONFIDENCE 0.99
  END COLLAPSE

  # ══ WHAT THE SYSTEM LEARNS ══
  # Not: how to reverse SHA-256
  # But: how to cover the nonce space intelligently
  # avoiding redundant regions, clustering search
  # around previously productive neighborhoods
  # The intelligence is in the search, not the hash

END SCOPE


---

### DEMONSTRATION 3: CELL DIVISION (MITOSIS)
#### Encoding life as constraint satisfaction

axl:
# ════════════════════════════════════════════════════
# CELL DIVISION SYSTEM
# Life as constraints on information replication
# with error correction and state machines
# ════════════════════════════════════════════════════

SCOPE molecular[level: 0]

  ASSUME molecule EXISTS
  ASSUME information CAN BE ENCODED IN molecule
  ASSUME chemical_bond EXISTS
  CONSERVES atom_count ACROSS chemical_reactions
  CONSERVES charge ACROSS chemical_reactions

  ENTITY nucleotide
    FIELD base: SYMBOLIC DISCRETE VALUES {adenine, thymine, guanine, cytosine}
    FIELD sugar: SYMBOLIC EQUALS deoxyribose
    FIELD phosphate: BOOLEAN
    BOUND adenine PAIRS_WITH thymine ONLY      # base pairing rule
    BOUND guanine PAIRS_WITH cytosine ONLY
  END ENTITY

  ENTITY dna_strand
    FIELD sequence: VECTOR of nucleotide
    FIELD length: INTEGER BOUND [1, 3e9]       # up to 3 billion base pairs
    FIELD direction: SYMBOLIC DISCRETE VALUES {5_to_3, 3_to_5}
    CONSERVES sequence UNLESS transform EQUALS mutation
    TENDS base_pair_bonds STABLE               # double helix stability
  END ENTITY

  TRANSFORM base_pairing
    INPUT {strand_a: dna_strand, strand_b: dna_strand}
    OUTPUT {double_helix}
    REQUIRES strand_a.direction FORBIDS strand_b.direction  # antiparallel
    REQUIRES each_base PAIRS_WITH complement
    CONSERVES sequence_information
  END TRANSFORM

END SCOPE


SCOPE genome[level: 1]

  ASSUME molecular SCOPE HOLDS

  ENTITY chromosome
    FIELD dna: dna_strand
    FIELD histone_proteins: VECTOR
    FIELD centromere_position: INTEGER
    FIELD telomere_length: INTEGER BOUND [0, 15000]  # erosion limit
    TENDS compaction MAX WHEN cell_dividing EQUALS true
    TENDS compaction MIN WHEN cell_active EQUALS true
  END ENTITY

  ENTITY genome
    FIELD chromosomes: VECTOR of chromosome
    BOUND chromosome_count EQUALS 46          # human diploid — pairs of 23
    CONSERVES chromosome_count UNLESS transform EQUALS meiosis
    TENDS integrity MAX                       # genome resists damage
  END ENTITY

  ENTITY dna_polymerase
    FIELD error_rate: REAL BOUND[1e-9, 1e-9] # one error per billion bases
    TENDS accuracy MAX
    TRANSFORMS dna_strand INTO dna_strand_copy
    CONSERVES sequence UNLESS error_rate TRIGGERS
  END ENTITY

  EMERGES genetic_code
    FROM nucleotide_triplet_sequence
    AT SCOPE genome
    NOT PRESENT AT SCOPE molecular
    # individual nucleotides have no meaning — codons emerge at this level

END SCOPE


SCOPE cell[level: 2]

  ASSUME genome SCOPE HOLDS

  ENTITY cell
    FIELD genome: genome
    FIELD membrane: BOOLEAN EQUALS true        # boundary condition
    FIELD atp_level: REAL BOUND[0, 1]         # energy state
    FIELD size: REAL BOUND[minimum_viable, maximum_before_division]
    FIELD age: INTEGER                         # division counter
    FIELD state: SYMBOLIC DISCRETE VALUES {
      G1,                                      # growth phase 1
      S,                                       # synthesis (DNA replication)
      G2,                                      # growth phase 2
      M,                                       # mitosis (division)
      G0,                                      # quiescent (resting)
      apoptosis                                # programmed death
    }
    TENDS atp_level MAX                        # cells maximize energy
    TENDS genome_integrity MAX                 # cells protect DNA
    BOUND age BELOW hayflick_limit             # telomere erosion limit
  END ENTITY

  ENTITY checkpoint_protein
    FIELD name: SYMBOLIC DISCRETE VALUES {p53, rb, cyclin_B, cdk1}
    FIELD active: BOOLEAN
    TENDS genome_damage OBSERVE                # monitors DNA integrity
    TRANSFORMS cell.state INTO apoptosis
      WHEN genome_damage ABOVE repair_threshold
    # p53 is the key constraint enforcer — if damage is too great, die
  END ENTITY

END SCOPE


SCOPE mitosis [level: 3]

  ASSUME cell SCOPE HOLDS

  # ══ TRIGGER CONDITIONS ══
  # Cell division is itself a constraint satisfaction problem:
  # divide WHEN AND ONLY WHEN all conditions hold

  OBJECTIVE division_trigger
    OPTIMIZE cell.state EQUAL M
    SUBJECT TO {
      cell.size ABOVE division_threshold,
      cell.atp_level ABOVE 0.7,
      dna_replication EQUALS complete,
      checkpoint_all EQUALS passed,
      external_growth_signal EQUALS present,
      cell.age BELOW hayflick_limit
    }
    # ALL conditions must hold — AND logic, not OR
    # This is why cancer is a constraint violation:
    # it occurs when checkpoint constraints are BROKEN
  END OBJECTIVE

  TRANSFORM prophase
    INPUT {cell: STATE G2}
    OUTPUT {cell: chromosomes_condensed, spindle_forming}
    REQUIRES division_trigger EQUALS satisfied
    CONSERVES chromosome_count
    CONSERVES genome_content
  END TRANSFORM

  TRANSFORM metaphase
    INPUT {cell: chromosomes_condensed}
    OUTPUT {cell: chromosomes_aligned_at_plate}
    REQUIRES spindle_attached_to EQUALS all_chromosomes
    # Checkpoint: spindle assembly checkpoint
    # If any chromosome unattached — PAUSE here
    BOUND unattached_chromosomes EQUALS 0
      UNLESS OVERRIDE apoptosis
  END TRANSFORM

  TRANSFORM anaphase
    INPUT {cell: chromosomes_aligned}
    OUTPUT {sister_chromatids: separating}
    CONSERVES chromosome_count_total
    CHANGES chromosome_location
    PRODUCES two_sets OF 46_chromosomes
  END TRANSFORM

  TRANSFORM telophase_and_cytokinesis
    INPUT {cell: chromatids_separated}
    OUTPUT {daughter_cell_1, daughter_cell_2}
    CONSERVES genome_content IN each_daughter
    CONSERVES organelle_distribution APPROXIMATELY
    PRODUCES {
      daughter_cell_1: IDENTICAL_GENOME_TO parent,
      daughter_cell_2: IDENTICAL_GENOME_TO parent
    }
  END TRANSFORM

  EMERGES tissue
    FROM COMPOSITION OF {cell COUNT ABOVE 1}
    AT SCOPE mitosis
    NOT PRESENT AT SCOPE cell
    # individual cells have no tissue identity — it emerges from composition

  EMERGES cancer
    FROM checkpoint_protein.active EQUALS false
      AND division_trigger SATISFIED_WITHOUT all_conditions
    AT SCOPE mitosis
    # Cancer is not a new thing — it is constraint violation
    # The cell divides when it should not because the objective
    # function has been corrupted
    CONTRADICT cancer WITH normal_division
      AT SCOPE mitosis
      BOUNDARY_TYPE INFORMATION           # corrupted information in genome

  # ══ COLLAPSE EXAMPLE: Cancer diagnosis ══

  OBSERVE {
    cell.division_rate ABOVE normal_range,
    checkpoint_protein.p53 EQUALS inactive,
    cell.age ABOVE hayflick_limit
  }

  UNKNOWN {
    mutation_site: dna_strand LOCATION,
    checkpoint_broken: SYMBOLIC,
    division_trigger_violated: OBJECTIVE_CLAUSE
  }

  COLLAPSE mitosis
    GIVEN {abnormal_observations}
    FIND {mutation_site, checkpoint_broken, division_trigger_violated}
    CONFIDENCE 0.90
  END COLLAPSE

END SCOPE


# ════════════════════════════════════════════════════
# META: THE LANGUAGE DESCRIBING ITSELF
# AXL encoding its own structure
# ════════════════════════════════════════════════════

SCOPE axl_meta [level: 0]

  ASSUME symbol EXISTS
  ASSUME symbol CAN CARRY meaning
  ASSUME meaning IS RELATIVE TO scope

  ENTITY keyword
    FIELD token: SYMBOLIC
    FIELD semantic: SYMBOLIC
    FIELD abstraction_direction: DISCRETE VALUES {up, down, both, none}
    FIELD arity: INTEGER                      # how many arguments it takes
  END ENTITY

  ENTITY statement
    FIELD keywords: VECTOR of keyword
    FIELD scope_level: INTEGER
    DERIVES meaning FROM {keywords, scope_level, context}
    # Same token, different scope = different meaning
    # BOUND in scope 0 (physics) ≠ BOUND in scope 3 (biology)
  END ENTITY

  EMERGES program_meaning
    FROM COMPOSITION OF {statement COUNT ABOVE 1}
    AT SCOPE axl_meta
    NOT PRESENT AT SCOPE symbol
    # Individual symbols have no program meaning
    # Meaning emerges from composition and scope

  TEND expressiveness MAX
  TEND ambiguity MIN
  CONTRADICT expressiveness WITH ambiguity
    AT SCOPE axl_meta
    BOUNDARY_TYPE INFORMATION
    # All languages face this — AXL resolves it via scope tagging

END SCOPE
`;

function buildPrompt(n, recentDomains, 
                     maxVars=8
                    // maxVars=20
                    ) {
    const recentStr = recentDomains.slice(-10).join(", ") || "none yet";
    return `${AXL_SPEC}

════════════════════════════════════════════════════════════
GENERATION TASK
════════════════════════════════════════════════════════════

Generate exactly ${n} UCE training samples as a JSON array.

STRICT REQUIREMENTS:
1. Output ONLY a valid JSON array. No markdown, no backticks, no preamble.
2. Every sample MUST have all fields in the schema below.
3. b_raw must be a computed finite float matching the AXL program's logic.
4. c_values must be in RAW domain units matching the declared bounds in vars_text.
   DO NOT pre-normalize to[0,1] — the pipeline does that. Example: if
   vars_text says "VARS c0 IN [200 800]" then c_values should be like[650, 210].
5. Every vars_text MUST start with the word VARS.
6. Every sample must use a DIFFERENT domain or structural pattern.
7. Vary: sense (EQUAL/MINIMIZE/MAXIMIZE/SELECT/TEND), n_vars (1-${maxVars}),
   abstraction depth (scope_depth 2-10), domain.
8. AXL programs must have 2-3 nested SCOPE blocks minimum.
9. Include EMERGES for at least 40% of samples.
10. Include STEP/CANDIDATE blocks for iterative search problems.
11. Include CONTRADICT for problems with hard boundaries.
12. Do NOT use domains recently overrepresented: ${recentStr}
13. Be expressive, use all the information known on that problem and domain. minimum scope_depth: 3

JSON SCHEMA (output array of these):
{
  "id": "<uuid>",
  "domain": "<domain>",
  "subdomain": "<specific topic>",
  "sense": "<EQUAL|MINIMIZE|MAXIMIZE|SELECT|TEND>",
  "n_vars": <integer>,
  "vars_text": "VARS c0 IN[lo hi] c1 IN [lo hi] ...",
  "expr_text": "EXPR <description>",
  "sense_text": "SENSE <EQUAL|MINIMIZE|MAXIMIZE|SELECT|TEND>",
  "b_raw": <float>,
  "b_norm": <sign(b)*log(1+|b|)/30>,
  "c_values":[<raw domain values matching bounds>],
  "mask": [1,1,...],
  "candidates":[],
  "step_count": <integer>,
  "axl_source": "<full multi-scope AXL program as single string>",
  "metadata": {
    "description": "<one sentence>",
    "units": "<units of b_raw>",
    "difficulty": "<easy|medium|hard|expert>",
    "scope_depth": <integer>,
    "has_emergence": <boolean>,
    "has_contradiction": <boolean>
  }
}

OUTPUT THE JSON ARRAY NOW:`;
}

async function callBedrock(prompt) {
  const model = "haiku";
  
    const command = new ConverseCommand({
        modelId: BEDROCK_MODEL_ID,
        messages: [{ role: "user", content:[{ text: prompt }] }],
        inferenceConfig: { 
            maxTokens: 8000, //10000, 
            temperature: 1
        }/*,
      additionalModelRequestFields: (function() {
                    if (model.includes("haiku")) {
                        return {
                            reasoning_config: {
                                type: "enabled",
                                budget_tokens: 2048 
                            } 
                        };
                    } else if (model.includes("claude")) {
                        return {
                            // thinking: { type: "adaptive" },
                            output_config: { effort: "high" }
                        };
                    }
                    return undefined;
                })() */
    });

    const t0 = Date.now();
    const res = await bedrockClient.send(command);
    const text = res.output.message.content.find(b => b.text)?.text || "";
    log(`Bedrock call ${((Date.now()-t0)/1000).toFixed(1)}s — ${text.length} chars`);
    
    const tokenUsage = res.usage ? (res.usage.inputTokens + res.usage.outputTokens) : 0;
    console.log("Token Usage: " + tokenUsage);
    
    return text;
}

function parseAIResponse(text) {
    let t = text.trim();
    for (const fence of ["```json","```JSON","```"]) {
        if (t.startsWith(fence)) { t = t.slice(fence.length); break; }
    }
    if (t.endsWith("```")) t = t.slice(0,-3);
    t = t.trim();
    const start = t.indexOf("["), end = t.lastIndexOf("]");
    if (start === -1 || end === -1) return[];
    try {
        return JSON.parse(t.slice(start, end+1));
    } catch(e) {
        log(`JSON parse error: ${e.message} — attempting salvage`, "WARN");
        const saved =[];
        let depth = 0, buf = "", inObj = false;
        for (const ch of t.slice(start+1, end)) {
            if (ch === "{") { depth++; inObj = true; }
            if (inObj) buf += ch;
            if (ch === "}") {
                depth--;
                if (depth === 0 && buf.trim()) {
                    try { saved.push(JSON.parse(buf)); } catch(_) {}
                    buf = ""; inObj = false;
                }
            }
        }
        log(`Salvaged ${saved.length} samples`);
        return saved;
    }
}

// ── EXPRESS APP ───────────────────────────────────────────────────────────────
const app = express();
app.use(cors());
app.use(express.json());

app.get("/status", (_, res) => res.json({
    dataset_size: dataset.length,
    dataset_path: DATASET_PATH,
    bedrock_model: BEDROCK_MODEL_ID,
    focus_tracker: tracker.stats(),
    server_time: new Date().toISOString(),
}));

app.get("/domains", (_, res) => res.json(tracker.stats()));

app.post("/generate", async (req, res) => {
    // Correcting the variable names to match the logic below
    const n = parseInt(req.query.n ?? req.body.n ?? "50");
    const rps = Math.min(parseFloat(req.query.rps ?? req.body.rps ?? DEFAULT_RPS.toString()), MAX_RPS);
    const syntheticFraction = parseFloat(req.query.synthetic_fraction ?? req.body.synthetic_fraction ?? "0.4");
    const maxVars = parseInt(req.query.max_vars ?? req.body.max_vars ?? "20");

    log(`Generation request: n=${n} rps=${rps} synthetic=${(syntheticFraction * 100).toFixed(0)}%`, "HEAD");

    let batchErrors = 0;
    let newSamplesCount = 0;

    // ── 1. SYNTHETIC BATCH (Immediate Save) ──
    const nSynthetic = Math.floor(n * syntheticFraction);
    if (nSynthetic > 0) {
        const gens = Object.values(SYNTHETIC_GENERATORS);
        const perGen = Math.max(1, Math.ceil(nSynthetic / gens.length));
        let tempSynth = [];
        for (const genFn of gens) { 
            try { tempSynth.push(...genFn(perGen)); } catch(e) { log(`Gen error: ${e.message}`, "ERROR"); }
        }
        tempSynth = tempSynth.slice(0, nSynthetic);

        for (const raw of tempSynth) {
            const norm = normalizeSample(raw);
            if (norm && validateSample(norm)) {
                dataset.push(norm);
                tracker.record(norm.domain);
                newSamplesCount++;
            } else { batchErrors++; }
        }
        saveDataset(); 
        log(`Synthetic batch committed to disk: ${newSamplesCount} samples`, "OK");
    }

    // ── 2. AI BATCH (Incremental Save) ──
    const nAI = n - nSynthetic;
    const callsNeeded = Math.ceil(nAI / SAMPLES_PER_CALL);
    const minInterval = 1000 / rps;

    // Respond immediately to prevent client-side timeouts
    res.json({ 
        message: "Generation started in background.", 
        total_requested: n, 
        synthetic_count: nSynthetic, 
        ai_tasks: callsNeeded 
    });

    // Run AI loop in background
    (async () => {
        for (let ci = 0; ci < callsNeeded; ci++) {
            const t0 = Date.now();
            try {
                const prompt = buildPrompt(SAMPLES_PER_CALL, tracker.recent, maxVars);
                const text = await callBedrock(prompt);
                const parsed = parseAIResponse(text);
                
                let callValid = 0;
                for (const raw of parsed) {
                    const norm = normalizeSample(raw);
                    if (norm && validateSample(norm)) {
                        norm.metadata = norm.metadata || {};
                        norm.metadata.source = "ai";
                        dataset.push(norm);
                        tracker.record(norm.domain);
                        callValid++;
                        newSamplesCount++;
                    } else { batchErrors++; }
                }

                if (callValid > 0) {
                    saveDataset();
                    log(`Saved Call ${ci + 1}/${callsNeeded} (${callValid} samples). Total: ${dataset.length}`, "OK");
                }
            } catch (e) {
                log(`Bedrock call ${ci + 1} failed: ${e.message}`, "ERROR");
                batchErrors++;
            }
            
            const elapsed = Date.now() - t0;
            if (elapsed < minInterval) await new Promise(r => setTimeout(r, minInterval - elapsed));
        }
        log(`BACKGROUND PROCESS COMPLETE. New samples: ${newSamplesCount} | Errors: ${batchErrors}`, "HEAD");
    })();
});

app.get("/dataset/download", (req, res) => {
    if (!fs.existsSync(DATASET_PATH)) {
        return res.status(404).json({ error: "No dataset yet. Run /generate first." });
    }
    const dated = `axl_dataset_${new Date().toISOString().slice(0,19).replace(/[:T]/g,"-")}.json`;
    res.download(DATASET_PATH, dated, (err) => {
        if (err) log(`Download error: ${err.message}`, "ERROR");
        else     log(`Dataset downloaded as ${dated}`, "OK");
    });
});

app.post("/dataset/clear", (_, res) => {
    const count = dataset.length;
    dataset =[];
    if (fs.existsSync(DATASET_PATH)) fs.unlinkSync(DATASET_PATH);
    log(`Cleared ${count} samples`, "OK");
    res.json({ cleared: count });
});

app.get("/dataset/sample", (req, res) => {
    const n = Math.min(parseInt(req.query.n||"5"), dataset.length);
    const shuffled = [...dataset].sort(()=>Math.random()-0.5);
    res.json({ samples: shuffled.slice(0,n) });
});

// ── STARTUP ───────────────────────────────────────────────────────────────────
loadDataset();
app.listen(PORT, '0.0.0.0', () => {
    log(`AXL Dataset Server listening on :${PORT}`, "OK");
    log(`Model:   ${BEDROCK_MODEL_ID}`);
    log(`Dataset: ${DATASET_PATH}`);
    log(`RPS:     ${DEFAULT_RPS} default / ${MAX_RPS} max`);
});