clean up

src/App.svelte

@@ -12,6 +12,13 @@
   let inputValue: { data: string[][]; headers: string[] } = { data: [[]], headers: [] };
   let cleanedValue: { data: string[][]; headers: string[] } = { data: [[]], headers: [] };
 
+  // Cleaning UI state
+  let cleaningInProgress = false;
+  let cleaningError: string | null = null;
+  let modelWarning: string | null = null;
+  let progressCurrent = 0;
+  let progressTotal = 0;
+
 
   function parseCSVorTSV(text: string) {
     if (!text) return { data: [[]], headers: [] };
@@ -20,7 +27,35 @@
     const sep = lines[0].includes('\t') ? '\t' : ',';
     const headers = lines[0].split(sep).map(h => h.trim());
     const data = lines.slice(1).map(line => line.split(sep).map(cell => cell.trim()));
-    return { data, headers };
+    return normalizeTable({ data, headers });
+  }
+
+  function hasNonEmptyTable(value: { data: string[][]; headers: string[] } | null | undefined): boolean {
+    if (!value) return false;
+    const headersOk = Array.isArray(value.headers) && value.headers.length > 0;
+    const rowsOk = Array.isArray(value.data) && value.data.length > 0;
+    if (!headersOk || !rowsOk) return false;
+    // At least one non-empty cell
+    return value.data.some((row) => Array.isArray(row) && row.some((cell) => String(cell ?? '').trim() !== ''));
+  }
+
+  function normalizeTable(value: { data: unknown[][]; headers: unknown[] } | null | undefined): { data: string[][]; headers: string[] } {
+    if (!value || !Array.isArray(value.headers) || value.headers.length === 0) {
+      return { data: [], headers: [] };
+    }
+    const headers: string[] = value.headers.map((h) => String(h ?? ''));
+    const width = headers.length;
+    const data: string[][] = (Array.isArray(value.data) ? value.data : []).map((row) => {
+      const safeRow = Array.isArray(row) ? row : [];
+      const padded: string[] = Array.from({ length: width }, (_, i) => String(safeRow[i] ?? ''));
+      return padded;
+    });
+    return { headers, data };
+  }
+
+  function padRowToWidth(row: unknown[] | null | undefined, width: number): string[] {
+    const safeRow = Array.isArray(row) ? row : [];
+    return Array.from({ length: width }, (_, i) => String(safeRow[i] ?? ''));
   }
 
   function updateInputValueFromRaw() {
@@ -48,37 +83,120 @@
   }
 
   function handleInputChange(e: CustomEvent) {
-    inputValue = e.detail;
+    inputValue = normalizeTable(e.detail);
     updateRawFromInputValue();
   }
 
   let generator: any = null;
   let loadingModel = false;
 
+  // Heuristic fallback cleaner if model output is unusable
+  function fallbackClean(text: string): string {
+    if (!text) return '';
+    // Remove URLs
+    let out = text.replace(/https?:\/\/\S+/g, '');
+    // Remove emojis and non-text pictographs
+    out = out.replace(/[\p{Extended_Pictographic}]/gu, '');
+    // Normalize quotes and whitespace
+    out = out
+      .replace(/[\u2018\u2019\u201A\u201B]/g, "'")
+      .replace(/[\u201C\u201D\u201E\u201F]/g, '"')
+      .normalize('NFC')
+      .replace(/\s+/g, ' ')
+      .trim();
+    return out;
+  }
+
+  async function cleanTextWithModel(text: string): Promise<string> {
+    if (!text || !text.trim()) return '';
+    if (!generator) {
+      loadingModel = true;
+      generator = await pipeline('text-generation', 'Xenova/gpt2');
+      loadingModel = false;
+    }
+
+    const prompt =
+      'You are a text cleaning assistant. Clean the following text by fixing casing, spelling, removing URLs/emojis, and normalizing spaces. Return ONLY the cleaned text and nothing else.\nInput:\n"' +
+      text.replace(/\n/g, ' ') +
+      '"\nOutput:';
+
+    try {
+      const output = await generator(prompt, {
+        max_new_tokens: 80,
+        temperature: 0.2,
+        top_p: 0.9,
+        repetition_penalty: 1.1,
+        do_sample: false
+      });
+      const generated = output?.[0]?.generated_text ?? '';
+      const continuation = generated.slice(prompt.length).trim();
+      // If model echoes input or returns empty, use fallback
+      if (!continuation || continuation.length < 1) return fallbackClean(text);
+      // Trim to first line to avoid rambling
+      const firstLine = continuation.split('\n')[0].trim();
+      // Guard against the model including the word "Output:" again
+      return firstLine.replace(/^Output:\s*/i, '').trim();
+    } catch (e) {
+      return fallbackClean(text);
+    }
+  }
+
   async function analyzeAndClean() {
+    cleaningError = null;
+    modelWarning = null;
     showSteps = false;
     cleaningSteps = [];
     cleanedValue = { data: [[]], headers: [] };
     cleanedData = '';
-    loadingModel = true;
-    try {
-      if (!generator) {
-        generator = await pipeline('text-generation', 'Xenova/gpt2');
+
+    // Ensure there is a table to clean
+    if (!inputValue.headers?.length) {
+      cleaningError = 'No headers detected. Please paste a table with headers in the first row.';
+      return;
+    }
+
+    // Prepare output schema: add a cleaned column for every original header
+    const cleanedHeaders = inputValue.headers.map((h) => `${h}_cleaned`);
+    const outHeaders = [...inputValue.headers, ...cleanedHeaders];
+
+    // Iterate rows
+    cleaningInProgress = true;
+    progressCurrent = 0;
+    const numRows = inputValue.data?.length ?? 0;
+    const numCols = inputValue.headers.length;
+    progressTotal = numRows * numCols;
+    const outRows: string[][] = [];
+
+    for (const row of inputValue.data) {
+      const baseRow = padRowToWidth(row, numCols);
+      const cleanedCells: string[] = [];
+      for (let c = 0; c < numCols; c += 1) {
+        const cell = (baseRow?.[c] ?? '').toString();
+        let cleaned = '';
+        try {
+          cleaned = await cleanTextWithModel(cell);
+        } catch (e) {
+          if (!modelWarning) modelWarning = 'Model unavailable. Used deterministic fallback cleaning.';
+          cleaned = fallbackClean(cell);
+        }
+        if (!cleaned) cleaned = fallbackClean(cell);
+        cleanedCells.push(cleaned);
+        progressCurrent += 1;
       }
-      loadingModel = false;
-      // Prepare a prompt for the model
-      const tableString = [inputValue.headers, ...inputValue.data].map(row => row.join(',')).join('\n');
-      const prompt = `Clean this table and suggest steps. Table:\n${tableString}\nDescribe the cleaning steps and show a cleaned version.`;
-      const output = await generator(prompt, { max_new_tokens: 128 });
-      let content = output?.[0]?.generated_text || '';
-      cleaningSteps = [{ step: content, accepted: true }];
-      cleanedValue = { headers: inputValue.headers, data: inputValue.data };
-      cleanedData = content;
-      showSteps = true;
-    } catch (err) {
-      alert('Failed to load or run the model. Please check your internet connection and model support.');
-      loadingModel = false;
+      const newRow = [...baseRow, ...cleanedCells];
+      outRows.push(newRow);
     }
+
+    cleanedValue = normalizeTable({ headers: outHeaders, data: outRows });
+    // Keep a CSV copy in cleanedData for export convenience
+    const sep = ',';
+    const lines = [outHeaders.join(sep), ...outRows.map(r => r.join(sep))];
+    cleanedData = lines.join('\n');
+
+    // Note for transparency
+    cleaningSteps = [{ step: `Cleaned all ${numCols} columns using a text-generation model.`, accepted: true }];
+    showSteps = true;
+    cleaningInProgress = false;
   }
 
   function toggleStep(idx: number) {
@@ -101,6 +219,7 @@
   }
 
   $: updateInputValueFromRaw();
+  // No-op reactive blocks here
 </script>
 
 <div class="df-theme">
@@ -113,17 +232,39 @@
         <input type="file" accept=".csv,.tsv,.txt" bind:this={fileInput} on:change={handleFileUpload} />
       </div>
       <div style="margin: 1rem 0; width: 100%;">
-        <Dataframe
-          bind:value={inputValue}
-          show_search="search"
-          show_row_numbers={true}
-          show_copy_button={true}
-          show_fullscreen_button={true}
-          editable={true}
-          on:change={handleInputChange}
-        />
+        
+        {#if hasNonEmptyTable(inputValue)}
+          <Dataframe
+            bind:value={inputValue}
+            show_search="search"
+            show_row_numbers={true}
+            show_copy_button={true}
+            show_fullscreen_button={true}
+            editable={true}
+            on:change={handleInputChange}
+          />
+        {/if}
       </div>
-      <button on:click={analyzeAndClean}>Analyze &amp; Clean</button>
+      <button on:click={analyzeAndClean} disabled={cleaningInProgress || loadingModel || !inputValue.headers?.length}>
+        {#if loadingModel}
+          Loading model...
+        {:else if cleaningInProgress}
+          Cleaning {progressCurrent}/{progressTotal}...
+        {:else}
+          Analyze &amp; Clean
+        {/if}
+      </button>
+
+      {#if cleaningError}
+        <div role="alert" aria-live="assertive" style="margin-top: 0.5rem; color: #b00020;">
+          {cleaningError}
+        </div>
+      {/if}
+      {#if modelWarning}
+        <div role="status" aria-live="polite" style="margin-top: 0.5rem; color: #5f5f00;">
+          {modelWarning}
+        </div>
+      {/if}
     </section>
 
     {#if showSteps}
@@ -131,39 +272,29 @@
         <h2>AI-Suggested Cleaning Steps</h2>
         <ul>
           {#each cleaningSteps as step, idx}
-            <li>
+            <div>
               <input type="checkbox" bind:checked={step.accepted} on:change={() => toggleStep(idx)} />
               {step.step}
-            </li>
+            </div>
           {/each}
         </ul>
       </section>
     {/if}
 
     <section class="dataframes-section">
-      <div class="dataframe original">
-        <h3>Original Data</h3>
-        <Dataframe
-          bind:value={inputValue}
-          show_search="search"
-          show_row_numbers={true}
-          show_copy_button={true}
-          show_fullscreen_button={true}
-          editable={true}
-          on:change={handleInputChange}
-        />
-      </div>
-      <div class="dataframe cleaned">
-        <h3>Cleaned Data (Preview)</h3>
-        <Dataframe
-          bind:value={cleanedValue}
-          show_search="search"
-          show_row_numbers={true}
-          show_copy_button={true}
-          show_fullscreen_button={true}
-          editable={false}
-        />
-      </div>
+      {#if hasNonEmptyTable(cleanedValue)}
+        <div class="dataframe cleaned">
+          <h3>Cleaned Data (Preview)</h3>
+          <Dataframe
+            bind:value={cleanedValue}
+            show_search="search"
+            show_row_numbers={true}
+            show_copy_button={true}
+            show_fullscreen_button={true}
+            editable={false}
+          />
+        </div>
+      {/if}
     </section>
 
     <button on:click={exportCleaned}>Export Cleaned Data</button>
@@ -171,14 +302,7 @@
 </div>
 
 <style>
-  main {
-    max-width: 900px;
-    margin: 2rem auto;
-    padding: 2rem;
-    background: #fff;
-    border-radius: 12px;
-    box-shadow: 0 2px 8px rgba(0,0,0,0.07);
-  }
+
   h1 {
     text-align: center;
     margin-bottom: 2rem;
@@ -198,15 +322,20 @@
   }
   .dataframes-section {
     display: flex;
+    flex-direction: column;
     gap: 2rem;
-    justify-content: center;
     margin-bottom: 2rem;
   }
+  @media (max-width: 900px) {
+    .dataframes-section {
+      grid-template-columns: 1fr;
+    }
+  }
   .dataframe {
-    flex: 1;
+    width: 100%;
     display: flex;
     flex-direction: column;
-    align-items: center;
+    align-items: stretch;
   }
   textarea {
     width: 100%;
@@ -236,4 +365,5 @@
     --gr-df-table-text: #222 !important;
     background: #fff;
   }
+  /* select styles removed with column selector */
 </style>

src/app.css

@@ -2,13 +2,10 @@
   font-family: system-ui, Avenir, Helvetica, Arial, sans-serif;
   line-height: 1.5;
   font-weight: 400;
-  color-scheme: light dark;
-  color: rgba(255, 255, 255, 0.87);
   font-synthesis: none;
   text-rendering: optimizeLegibility;
   -webkit-font-smoothing: antialiased;
   -moz-osx-font-smoothing: grayscale;
-  color: black;
 }
 
 a {