ui/static/app.js

    const config = window.MASTERMAP_CONFIG || {};
    const cleanPath = config.cleanPath || "";
    let applyWorkbookPath = config.applyWorkbookPath || "";
    let applyBlueprintPath = config.applyBlueprintPath || "";
    const defaultOutputSheet = config.defaultOutputSheet || "Cleaned_Data";
    const sheetSelect = document.getElementById("sheetSelect");
    const applySheetSelect = document.getElementById("applySheetSelect");
    const outputSheet = document.getElementById("outputSheet");
    const models = document.getElementById("models");
    const fetchModels = document.getElementById("fetchModels");
    const runButton = document.getElementById("runButton");
    const applyButton = document.getElementById("applyButton");
    const saveReferencesButton = document.getElementById("saveReferencesButton");
    const applyWorkbookForm = document.getElementById("applyWorkbookForm");
    const applyBlueprintForm = document.getElementById("applyBlueprintForm");
    const applyWorkbookInput = document.getElementById("applyWorkbookInput");
    const applyBlueprintInput = document.getElementById("applyBlueprintInput");
    const runStatus = document.getElementById("runStatus");
    const applyStatus = document.getElementById("applyStatus");
    const referencesStatus = document.getElementById("referencesStatus");
    const applyWorkbookFile = document.getElementById("applyWorkbookFile");
    const applyBlueprintFile = document.getElementById("applyBlueprintFile");
    const cleanLogs = document.getElementById("cleanLogs");
    const applyLogs = document.getElementById("applyLogs");
    const cleanProgressPanel = document.getElementById("cleanProgressPanel");
    const cleanProgressSummary = document.getElementById("cleanProgressSummary");
    const cleanProgressList = document.getElementById("cleanProgressList");
    const cleanResult = document.getElementById("cleanResult");
    const applyResult = document.getElementById("applyResult");
    let cleanRawLogText = "";
    let cleanLiveLine = "";
    let cleanProgressOrder = [];
    let cleanProgressByColumn = {};
    let applyRawLogText = "";
    let applyLiveLine = "";
    let activeRunStream = null;
    let activeRunJobId = "";
    let stopRequested = false;

    function submitUploadForm(formId, statusId, event) {
      if (event) {
        event.preventDefault();
      }
      const form = document.getElementById(formId);
      const status = document.getElementById(statusId);
      [
        ["clean_selected_sheet", sheetSelect ? sheetSelect.value : ""],
        ["output_sheet", outputSheet ? outputSheet.value : ""],
        ["models", models ? models.value : ""],
        ["apply_selected_sheet", applySheetSelect ? applySheetSelect.value : ""]
      ].forEach(([name, value]) => {
        let input = form.querySelector(`input[name="${name}"]`);
        if (!input) {
          input = document.createElement("input");
          input.type = "hidden";
          input.name = name;
          form.appendChild(input);
        }
        input.value = value;
      });
      status.textContent = "Loading...";

      if (formId === "applyWorkbookForm" || formId === "applyBlueprintForm") {
        uploadApplyFile(form, status, formId);
        return;
      }

      form.submit();
    }

    async function uploadApplyFile(form, status, formId) {
      try {
        const res = await fetch(form.action, {
          method: "POST",
          body: new FormData(form),
          headers: { "Accept": "application/json" }
        });
        const data = await res.json();
        if (!res.ok) {
          status.textContent = data.error || "Upload failed.";
          status.classList.add("error");
          return;
        }

        status.classList.remove("error");
        status.textContent = data.message || "Loaded.";
        if (formId === "applyWorkbookForm") {
          applyWorkbookPath = data.apply_workbook_path || "";
          applyWorkbookFile.innerHTML = data.apply_workbook_filename
            ? `<div class="file-pill">${escapeHtml(data.apply_workbook_filename)}</div>`
            : "";
        }
        if (formId === "applyBlueprintForm") {
          applyBlueprintPath = data.apply_blueprint_path || "";
          applyBlueprintFile.innerHTML = data.apply_blueprint_filename
            ? `<div class="file-pill">${escapeHtml(data.apply_blueprint_filename)}</div>`
            : "";
        }
        if (data.apply_sheets) {
          setApplySheets(data.apply_sheets, data.apply_selected_sheet);
        }
        applyButton.disabled = !(applyWorkbookPath && applyBlueprintPath && applySheetSelect.value);
      } catch (error) {
        status.textContent = "Upload failed.";
        status.classList.add("error");
      }
    }

    function clearCleanOutput() {
      cleanRawLogText = "";
      cleanLiveLine = "";
      cleanProgressOrder = [];
      cleanProgressByColumn = {};
      cleanLogs.textContent = "";
      cleanProgressPanel.classList.remove("active");
      cleanProgressSummary.textContent = "Waiting...";
      cleanProgressList.innerHTML = "";
      cleanResult.classList.remove("active");
      cleanResult.innerHTML = "";
    }

    function clearApplyOutput() {
      applyRawLogText = "";
      applyLiveLine = "";
      applyLogs.textContent = "";
      applyResult.classList.remove("active");
      applyResult.innerHTML = "";
    }

    function parseProgressLine(line) {
      const cleanLine = line.replace(/\x1b\[[0-9;]*m/g, "").trim();
      const match = cleanLine.match(/^Cleaning\s+(.+?):\s+(\d+)%\|.*?\|\s+(\d+)\/(\d+)\s+\[([^\]]*)\]/);
      if (!match) return null;

      const columnName = match[1].trim();
      const percent = Number(match[2]);
      const current = match[3];
      const total = match[4];
      const bracketParts = match[5].split(",").map(part => part.trim()).filter(Boolean);
      const timingParts = bracketParts.slice(0, 2);
      const metricParts = bracketParts.slice(2);
      const timing = timingParts.join(", ").includes("?") ? "estimating..." : timingParts.join(", ");
      const metrics = metricParts.join(", ");

      return { columnName, percent, current, total, timing, metrics };
    }

    function escapeHtml(value) {
      return String(value).replace(/[&<>"']/g, char => ({
        "&": "&amp;",
        "<": "&lt;",
        ">": "&gt;",
        '"': "&quot;",
        "'": "&#39;"
      }[char]));
    }

    function downloadUrl(path, filename, endpoint) {
      const params = new URLSearchParams({ path });
      if (filename) {
        params.set("filename", filename);
      }
      return `${endpoint}?${params.toString()}`;
    }

    function renderCleanProgressRows() {
      cleanProgressPanel.classList.add("active");
      cleanProgressSummary.textContent = `${cleanProgressOrder.length} column${cleanProgressOrder.length === 1 ? "" : "s"}`;
      cleanProgressList.innerHTML = cleanProgressOrder.map(columnName => {
        const item = cleanProgressByColumn[columnName];
        const percent = Math.max(0, Math.min(100, item.percent));
        const meta = `${item.current}/${item.total}${item.timing ? " | " + item.timing : ""}${item.metrics ? " | " + item.metrics : ""}`;
        return `
          <div class="progress-row">
            <div class="progress-top">
              <strong>${escapeHtml(columnName)}</strong>
              <span>${percent}%</span>
            </div>
            <div class="progress-track">
              <div class="progress-fill" style="width:${percent}%"></div>
            </div>
            <div class="progress-meta">${escapeHtml(meta)}</div>
          </div>
        `;
      }).join("");
    }

    function renderCleanProgressLine(line) {
      const parsed = parseProgressLine(line);
      if (!parsed) return false;

      if (!cleanProgressByColumn[parsed.columnName]) {
        cleanProgressOrder.push(parsed.columnName);
      }
      cleanProgressByColumn[parsed.columnName] = parsed;
      renderCleanProgressRows();
      return true;
    }

    function appendCleanLogChunk(chunk) {
      for (const char of chunk) {
        cleanRawLogText += char === "\r" ? "\n" : char;
        if (char === "\r") {
          renderCleanProgressLine(cleanLiveLine);
          cleanLiveLine = "";
        } else if (char === "\n") {
          renderCleanProgressLine(cleanLiveLine);
          cleanLiveLine = "";
        } else {
          cleanLiveLine += char;
        }
      }
      const isPartialProgress = cleanLiveLine.startsWith("Cleaning ") && cleanLiveLine.includes("|");
      if (!isPartialProgress) {
        renderCleanProgressLine(cleanLiveLine);
      }
      cleanLogs.textContent = cleanRawLogText;
      cleanLogs.scrollTop = cleanLogs.scrollHeight;
    }

    function renderApplySummary() {
      const changed = applyRawLogText.match(/Success!\s+(\d+)\s+corrections injected/i);
      const added = applyRawLogText.match(/Memory updated:\s+(\d+)\s+new approved values added/i);
      if (!changed && !added) return;

      applyResult.classList.add("active");
      applyResult.innerHTML = `
        <strong>Blueprint applied</strong>
        <div class="status">${changed ? changed[1] : "0"} workbook row value${changed && changed[1] === "1" ? "" : "s"} updated from human overrides.</div>
        <div class="status">${added ? added[1] : "0"} new unique reference value${added && added[1] === "1" ? "" : "s"} added to manual references.</div>
        <a class="download-link" href="${downloadUrl(applyWorkbookPath, "", "/download-applied-workbook")}">Download Cleaned Workbook</a>
      `;
    }

    function appendApplyLogChunk(chunk) {
      for (const char of chunk) {
        applyRawLogText += char === "\r" ? "\n" : char;
        if (char === "\r" || char === "\n") {
          applyLiveLine = "";
        } else {
          applyLiveLine += char;
        }
      }
      applyLogs.textContent = applyRawLogText;
      applyLogs.scrollTop = applyLogs.scrollHeight;
      renderApplySummary();
    }

    function clearPersistedOutputs() {
      [
        "mastermap.cleanRawLogText",
        "mastermap.applyRawLogText",
        "mastermap.cleanResultHtml",
        "mastermap.applyResultHtml",
        "mastermap.cleanResultActive",
        "mastermap.applyResultActive"
      ].forEach(key => localStorage.removeItem(key));
    }

    function setRunButtonIdle() {
      activeRunStream = null;
      activeRunJobId = "";
      stopRequested = false;
      runButton.disabled = false;
      runButton.textContent = "Run Cleaning";
      runButton.classList.remove("danger");
    }

    async function stopActiveRun() {
      if (!activeRunJobId) return;
      stopRequested = true;
      runButton.disabled = true;
      runButton.textContent = "Stopping...";
      runStatus.textContent = "Stopping run...";
      await fetch(`/stop?job_id=${encodeURIComponent(activeRunJobId)}`, { method: "POST" });
    }

    function selectApplySheet(sheetName) {
      if (!sheetName) return;
      const existing = Array.from(applySheetSelect.options).some(option => option.value === sheetName);
      if (!existing) {
        const option = document.createElement("option");
        option.value = sheetName;
        option.textContent = sheetName;
        applySheetSelect.appendChild(option);
      }
      applySheetSelect.value = sheetName;
      applySheetSelect.disabled = false;
    }

    function setApplySheets(sheets, preferredSheet) {
      const selected = preferredSheet || applySheetSelect.value;
      applySheetSelect.innerHTML = "";
      sheets.forEach(sheetName => {
        const option = document.createElement("option");
        option.value = sheetName;
        option.textContent = sheetName;
        applySheetSelect.appendChild(option);
      });
      if (selected && sheets.includes(selected)) {
        applySheetSelect.value = selected;
      } else if (sheets.length) {
        applySheetSelect.value = sheets[0];
      }
      applySheetSelect.disabled = sheets.length === 0;
    }

    async function refreshApplySheets(preferredSheet) {
      if (!applyWorkbookPath) return selectApplySheet(preferredSheet);
      const params = new URLSearchParams({ path: applyWorkbookPath });
      const res = await fetch(`/sheets?${params.toString()}`);
      if (!res.ok) {
        selectApplySheet(preferredSheet);
        return;
      }
      const data = await res.json();
      setApplySheets(data.sheets || [], preferredSheet);
    }

    async function refreshReferenceSyncStatus(updateText = true) {
      try {
        const res = await fetch("/references/status");
        const data = await res.json();
        saveReferencesButton.disabled = !data.enabled;
        if (updateText) {
          referencesStatus.textContent = data.enabled
            ? `Ready to save manual references to ${data.space_id}.`
            : data.reason || "Reference sync is unavailable.";
        }
      } catch (error) {
        saveReferencesButton.disabled = true;
        if (updateText) {
          referencesStatus.textContent = "Reference sync status unavailable.";
        }
      }
    }

    clearPersistedOutputs();
    refreshReferenceSyncStatus();

    applyWorkbookForm.addEventListener("submit", event => {
      submitUploadForm("applyWorkbookForm", "applyWorkbookUploadStatus", event);
    });
    applyBlueprintForm.addEventListener("submit", event => {
      submitUploadForm("applyBlueprintForm", "applyBlueprintUploadStatus", event);
    });
    applyWorkbookInput.addEventListener("change", event => {
      submitUploadForm("applyWorkbookForm", "applyWorkbookUploadStatus", event);
    });
    applyBlueprintInput.addEventListener("change", event => {
      submitUploadForm("applyBlueprintForm", "applyBlueprintUploadStatus", event);
    });

    fetchModels.addEventListener("click", async () => {
      fetchModels.disabled = true;
      runStatus.textContent = "Fetching Groq models...";
      const res = await fetch("/models");
      const data = await res.json();
      fetchModels.disabled = false;
      if (!res.ok) {
        runStatus.textContent = data.error || "Could not fetch models";
        return;
      }
      models.value = data.models.join(",");
      runStatus.textContent = "Model list updated.";
    });

    saveReferencesButton.addEventListener("click", async () => {
      saveReferencesButton.disabled = true;
      referencesStatus.textContent = "Saving manual references...";
      try {
        const res = await fetch("/references/save", { method: "POST" });
        const data = await res.json();
        if (!res.ok) {
          referencesStatus.textContent = data.error || "Could not save manual references.";
          await refreshReferenceSyncStatus();
          return;
        }
        referencesStatus.textContent = data.message || "Manual references saved.";
      } catch (error) {
        referencesStatus.textContent = "Could not save manual references.";
      } finally {
        await refreshReferenceSyncStatus(false);
      }
    });

    runButton.addEventListener("click", () => {
      if (activeRunStream) {
        stopActiveRun();
        return;
      }
      if (!cleanPath || !sheetSelect.value) return;
      clearCleanOutput();
      activeRunJobId = window.crypto && window.crypto.randomUUID ? window.crypto.randomUUID() : String(Date.now());
      stopRequested = false;
      runButton.disabled = false;
      runButton.textContent = "Stop Cleaning";
      runButton.classList.add("danger");
      runStatus.textContent = "Running...";
      const params = new URLSearchParams({
        job_id: activeRunJobId,
        input: cleanPath,
        sheet: sheetSelect.value,
        output_sheet: outputSheet.value || defaultOutputSheet,
        models: models.value.trim()
      });
      const generatedBlueprintPath = `data/uploads/Blueprint_${activeRunJobId}.xlsx`;
      const stream = new EventSource(`/run?${params.toString()}`);
      activeRunStream = stream;
      stream.onmessage = event => appendCleanLogChunk(JSON.parse(event.data));
      stream.addEventListener("done", async () => {
        stream.close();
        if (!stopRequested) {
          applyBlueprintPath = generatedBlueprintPath;
          const targetSheet = outputSheet.value || defaultOutputSheet;
          await refreshApplySheets(targetSheet);
          applyButton.disabled = !(applyWorkbookPath && applyBlueprintPath && applySheetSelect.value);
          applyStatus.textContent = "Generated blueprint is ready for Apply Blueprint.";
          cleanResult.classList.add("active");
          cleanResult.innerHTML = `
            <strong>Blueprint generated</strong>
            <div class="status">Blueprint saved for this session.</div>
            <a class="download-link" href="${downloadUrl(generatedBlueprintPath, "Blueprint.xlsx", "/download-blueprint")}">Download Blueprint</a>
            <a class="download-link" href="${downloadUrl(cleanPath, "", "/download-cleaned-workbook")}">Download Cleaned Workbook</a>
          `;
          runStatus.textContent = "Finished.";
        } else {
          runStatus.textContent = "Stopped.";
        }
        setRunButtonIdle();
      });
      stream.addEventListener("failed", () => {
        stream.close();
        setRunButtonIdle();
        runStatus.textContent = "Run failed. Check logs.";
      });
      stream.addEventListener("error", () => {
        stream.close();
        setRunButtonIdle();
        runStatus.textContent = "Run stopped. Check logs.";
      });
    });

    applyButton.addEventListener("click", () => {
      if (!applyWorkbookPath || !applyBlueprintPath || !applySheetSelect.value) return;
      clearApplyOutput();
      applyButton.disabled = true;
      applyStatus.textContent = "Applying blueprint...";
      const params = new URLSearchParams({
        input: applyWorkbookPath,
        blueprint: applyBlueprintPath,
        sheet: applySheetSelect.value
      });
      const stream = new EventSource(`/apply?${params.toString()}`);
      stream.onmessage = event => appendApplyLogChunk(JSON.parse(event.data));
      stream.addEventListener("done", () => {
        stream.close();
        applyButton.disabled = false;
        applyStatus.textContent = "Finished.";
        renderApplySummary();
      });
      stream.addEventListener("failed", () => {
        stream.close();
        applyButton.disabled = false;
        applyStatus.textContent = "Apply failed. Check logs.";
      });
      stream.addEventListener("error", () => {
        stream.close();
        applyButton.disabled = false;
        applyStatus.textContent = "Apply stopped. Check logs.";
      });
    });