Spaces:

broadfield-dev
/

HF-Dataset-Commander

Paused

App Files Files Community

broadfield-dev commited on Dec 30, 2025

Commit

8350ba2

verified ·

1 Parent(s): 5aaeff4

Update templates/index.html

Browse files

Files changed (1) hide show

templates/index.html +164 -94

templates/index.html CHANGED Viewed

@@ -2,145 +2,220 @@
 <html lang="en">
 <head>
     <meta charset="UTF-8">
-    <title>HF Dataset Command Center</title>
     <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/css/bootstrap.min.css" rel="stylesheet">
     <style>
-        .json-badge { background-color: #ffc107; color: #000; font-size: 0.8em; padding: 2px 5px; border-radius: 4px; }
-        .box { border: 1px solid #ddd; padding: 20px; border-radius: 8px; margin-bottom: 20px; }
-        pre { background: #f4f4f4; padding: 10px; max-height: 200px; overflow: auto; }
     </style>
 </head>
 <body class="bg-light">
-<div class="container mt-5">
-    <h1 class="mb-4">🤗 Dataset Command Center</h1>
-    <!-- Step 1: Authentication & Load -->
-    <div class="box bg-white">
-        <h4>1. Source Configuration</h4>
         <div class="row g-3">
-            <div class="col-md-4">
-                <label>HF Write Token</label>
                 <input type="password" id="token" class="form-control" placeholder="hf_...">
             </div>
-            <div class="col-md-4">
-                <label>Source Dataset ID</label>
-                <input type="text" id="dataset_id" class="form-control" placeholder="e.g. imalexissa/scikit_learn_metadata">
             </div>
             <div class="col-md-2">
-                <label>Split</label>
-                <input type="text" id="split" class="form-control" value="train">
             </div>
-            <div class="col-md-2 d-flex align-items-end">
-                <button class="btn btn-primary w-100" onclick="inspectDataset()">Inspect</button>
             </div>
         </div>
     </div>
-    <!-- Step 2: Column Analysis -->
-    <div id="inspector-panel" class="box bg-white" style="display:none;">
-        <h4>2. Column Inspector</h4>
-        <p class="text-muted">Detected columns. Check "Expand JSON" to extract keys.</p>
-        <table class="table table-sm" id="col-table">
-            <thead>
-                <tr>
-                    <th>Column</th>
-                    <th>Type</th>
-                    <th>Actions</th>
-                </tr>
-            </thead>
-            <tbody id="col-list"></tbody>
-        </table>
-        <div class="mt-3">
-            <h5>Recipe Builder</h5>
-            <div class="row g-2">
-                <div class="col-md-4">
-                    <label>JSON Expand (Col: keys,comma,sep)</label>
-                    <input type="text" id="json-rule" class="form-control" placeholder="meta: url, id">
                 </div>
-                <div class="col-md-4">
-                    <label>Rename (old=new)</label>
-                    <input type="text" id="rename-rule" class="form-control" placeholder="text_body=text">
                 </div>
-                <div class="col-md-4">
-                    <label>Drop Column</label>
-                    <input type="text" id="drop-rule" class="form-control" placeholder="col_name">
                 </div>
-            </div>
-            <div class="mt-2">
-                <label>Filter Expression (Python syntax)</label>
-                <input type="text" id="filter-rule" class="form-control" placeholder="len(text) > 100">
             </div>
         </div>
     </div>
-    <!-- Step 3: Preview -->
-    <div class="box bg-white">
-        <div class="d-flex justify-content-between">
-            <h4>3. Preview & Execute</h4>
-            <button class="btn btn-secondary" onclick="runPreview()">Run Dry Preview</button>
-        </div>
-        <div id="preview-area" class="mt-3"></div>
         <hr>
         <div class="row g-3 align-items-end">
-            <div class="col-md-6">
-                <label>Target Dataset ID (to push)</label>
                 <input type="text" id="target_id" class="form-control" placeholder="username/new-dataset-name">
             </div>
             <div class="col-md-3">
-                <label>Max Rows (Empty for all)</label>
-                <input type="number" id="max_rows" class="form-control">
             </div>
-            <div class="col-md-3">
-                <button class="btn btn-success w-100" onclick="executeJob()">🚀 Push to Hub</button>
             </div>
         </div>
-        <div id="job-status" class="mt-3 text-info fw-bold"></div>
     </div>
 </div>
 <script>
-    let currentCols = [];
-    async function inspectDataset() {
-        const btn = document.querySelector('button[onclick="inspectDataset()"]');
-        btn.disabled = true; btn.innerText = "Loading...";
         const payload = {
             token: document.getElementById('token').value,
             dataset_id: document.getElementById('dataset_id').value,
-            split: document.getElementById('split').value
         };
-        const res = await fetch('/inspect', {
             method: 'POST', headers: {'Content-Type': 'application/json'},
             body: JSON.stringify(payload)
         });
         const data = await res.json();
         if(data.status === 'success') {
             document.getElementById('inspector-panel').style.display = 'block';
             const tbody = document.getElementById('col-list');
             tbody.innerHTML = '';
             for (const [col, info] of Object.entries(data.analysis)) {
-                let html = `<tr>
-                    <td>${col}</td>
-                    <td>${info.type} ${info.is_json ? '<span class="json-badge">JSON Detected</span>' : ''}</td>
-                    <td>
-                        <button class="btn btn-sm btn-outline-danger" onclick="addDrop('${col}')">Drop</button>
-                    </td>
                 </tr>`;
-                tbody.innerHTML += html;
             }
         } else {
-            alert('Error: ' + data.message);
         }
-        btn.disabled = false; btn.innerText = "Inspect";
     }
     function getRecipe() {
         const jsonRule = document.getElementById('json-rule').value;
         const renameRule = document.getElementById('rename-rule').value;
@@ -149,7 +224,6 @@
         let recipe = { json_expansions: [], renames: {}, drops: [], filters: [] };
-        // Parse JSON rule: "meta: url, id"
         if(jsonRule.includes(':')) {
             let [col, keys] = jsonRule.split(':');
             recipe.json_expansions.push({
@@ -157,17 +231,11 @@
                 keys: keys.split(',').map(k => k.trim())
             });
         }
-        // Parse Rename: "a=b"
         if(renameRule.includes('=')) {
             let [oldC, newC] = renameRule.split('=');
             recipe.renames[oldC.trim()] = newC.trim();
         }
-        // Parse Drops
         if(dropRule) recipe.drops.push(dropRule.trim());
-        // Parse Filters
         if(filterRule) recipe.filters.push(filterRule.trim());
         return recipe;
@@ -177,7 +245,8 @@
         const payload = {
             token: document.getElementById('token').value,
             dataset_id: document.getElementById('dataset_id').value,
-            split: document.getElementById('split').value,
             recipe: getRecipe()
         };
@@ -187,6 +256,7 @@
         });
         const data = await res.json();
         const area = document.getElementById('preview-area');
         if (data.status === 'success') {
             area.innerHTML = `<pre>${JSON.stringify(data.rows, null, 2)}</pre>`;
@@ -199,8 +269,9 @@
         const payload = {
             token: document.getElementById('token').value,
             dataset_id: document.getElementById('dataset_id').value,
             target_id: document.getElementById('target_id').value,
-            split: document.getElementById('split').value,
             recipe: getRecipe(),
             max_rows: document.getElementById('max_rows').value
         };
@@ -212,6 +283,7 @@
         const data = await res.json();
         if(data.status === 'started') {
             pollStatus(data.job_id);
         }
     }
@@ -223,21 +295,19 @@
             const data = await res.json();
             if(data.status === 'running') {
-                statusDiv.innerText = "Processing... (This runs in background)";
             } else if (data.status === 'completed') {
-                statusDiv.innerText = `Success! Pushed ${data.result.rows_processed} rows.`;
                 clearInterval(interval);
-            } else {
                 statusDiv.innerText = `Error: ${data.error}`;
                 clearInterval(interval);
             }
         }, 2000);
     }
-    function addDrop(col) {
-        document.getElementById('drop-rule').value = col;
-    }
 </script>
 </body>
 </html>

 <html lang="en">
 <head>
     <meta charset="UTF-8">
+    <title>HF Dataset ETL Command Center</title>
     <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/css/bootstrap.min.css" rel="stylesheet">
     <style>
+        .json-badge { background-color: #ffc107; color: #000; font-size: 0.75em; padding: 2px 6px; border-radius: 4px; font-weight: bold; }
+        .box { border: 1px solid #e0e0e0; padding: 25px; border-radius: 12px; margin-bottom: 25px; background: white; box-shadow: 0 2px 5px rgba(0,0,0,0.05); }
+        .step-num { font-weight: bold; color: #6c757d; margin-bottom: 10px; display: block; }
+        pre { background: #f8f9fa; padding: 15px; max-height: 300px; overflow: auto; border-radius: 6px; border: 1px solid #eee; }
+        .form-label { font-weight: 500; font-size: 0.9em; }
+        .table-hover tbody tr:hover { background-color: #f8f9fa; }
     </style>
 </head>
 <body class="bg-light">
+<div class="container mt-4 mb-5">
+    <div class="d-flex align-items-center mb-4">
+        <h2 class="me-3">🛠️ Hugging Face Dataset Command Center</h2>
+    </div>
+    <!-- Step 1: Configuration -->
+    <div class="box">
+        <span class="step-num">STEP 1: SOURCE SELECTION</span>
         <div class="row g-3">
+            <div class="col-md-3">
+                <label class="form-label">HF Write Token</label>
                 <input type="password" id="token" class="form-control" placeholder="hf_...">
             </div>
+            <div class="col-md-5">
+                <label class="form-label">Dataset Repository ID</label>
+                <div class="input-group">
+                    <input type="text" id="dataset_id" class="form-control" placeholder="e.g. the_stack">
+                    <button class="btn btn-primary" onclick="analyzeMetadata()">Analyze Dataset</button>
+                </div>
             </div>
             <div class="col-md-2">
+                <label class="form-label">Config (Subset)</label>
+                <select id="config_select" class="form-select" onchange="updateSplits()" disabled>
+                    <option>Select...</option>
+                </select>
             </div>
+            <div class="col-md-2">
+                <label class="form-label">Split</label>
+                <select id="split_select" class="form-select" disabled>
+                    <option>Select...</option>
+                </select>
             </div>
         </div>
+        <div class="mt-3 text-end">
+            <button class="btn btn-outline-dark" id="load_sample_btn" onclick="inspectRows()" disabled>Load Sample Data ⬇</button>
+        </div>
     </div>
+    <!-- Step 2: Inspector -->
+    <div id="inspector-panel" class="box" style="display:none;">
+        <span class="step-num">STEP 2: SCHEMA & RECIPE</span>
+        <div class="row">
+            <div class="col-md-6">
+                <h5>Source Columns</h5>
+                <p class="text-muted small">We detected the following columns. Use the tools on the right to modify them.</p>
+                <table class="table table-sm table-hover" id="col-table">
+                    <thead><tr><th>Name</th><th>Type</th></tr></thead>
+                    <tbody id="col-list"></tbody>
+                </table>
+            </div>
+            <div class="col-md-6 border-start">
+                <h5 class="mb-3">Transformation Recipe</h5>
+                <div class="mb-3">
+                    <label class="form-label">1. Extract from JSON</label>
+                    <input type="text" id="json-rule" class="form-control mb-1" placeholder="col_name: key1, nested.key2">
+                    <small class="text-muted">Format: <code>source_col: new_key1, meta.id</code></small>
+                </div>
+                <div class="mb-3">
+                    <label class="form-label">2. Rename Columns</label>
+                    <input type="text" id="rename-rule" class="form-control" placeholder="old_name=new_name">
                 </div>
+                <div class="mb-3">
+                    <label class="form-label">3. Drop Columns</label>
+                    <input type="text" id="drop-rule" class="form-control" placeholder="col_to_delete">
                 </div>
+                <div class="mb-3">
+                    <label class="form-label">4. Filter Rows (Python Condition)</label>
+                    <input type="text" id="filter-rule" class="form-control" placeholder="len(text) > 500">
+                    <small class="text-muted">Condition applies to <b>transformed</b> data.</small>
                 </div>
+                <button class="btn btn-success w-100 mt-2" onclick="runPreview()">Preview Transformation ▶</button>
             </div>
         </div>
     </div>
+    <!-- Step 3: Output -->
+    <div id="preview-panel" class="box" style="display:none;">
+        <span class="step-num">STEP 3: PREVIEW & PUSH</span>
+        <h6>Transformed Preview (First 5 Rows)</h6>
+        <div id="preview-area"></div>
         <hr>
         <div class="row g-3 align-items-end">
+            <div class="col-md-5">
+                <label class="form-label">Target Repo ID</label>
                 <input type="text" id="target_id" class="form-control" placeholder="username/new-dataset-name">
             </div>
             <div class="col-md-3">
+                <label class="form-label">Max Rows (Optional)</label>
+                <input type="number" id="max_rows" class="form-control" placeholder="All">
             </div>
+            <div class="col-md-4">
+                <button class="btn btn-danger w-100" onclick="executeJob()">🚀 Launch ETL Job</button>
             </div>
         </div>
+        <div id="job-status" class="mt-3 p-3 rounded bg-light fw-bold text-center" style="display:none"></div>
     </div>
 </div>
 <script>
+    // --- Step 1 Functions ---
+    async function analyzeMetadata() {
+        const btn = document.querySelector('button[onclick="analyzeMetadata()"]');
+        btn.disabled = true; btn.innerText = "Fetching...";
+        const payload = {
+            token: document.getElementById('token').value,
+            dataset_id: document.getElementById('dataset_id').value
+        };
+        try {
+            const res = await fetch('/analyze_metadata', {
+                method: 'POST', headers: {'Content-Type': 'application/json'},
+                body: JSON.stringify(payload)
+            });
+            const data = await res.json();
+            if(data.status === 'success') {
+                // Populate Configs
+                const configSel = document.getElementById('config_select');
+                configSel.innerHTML = '';
+                data.configs.forEach(c => {
+                    configSel.innerHTML += `<option value="${c}">${c}</option>`;
+                });
+                configSel.disabled = false;
+                // Populate Splits (for first config)
+                populateSplits(data.splits);
+                document.getElementById('load_sample_btn').disabled = false;
+            } else {
+                alert('Error: ' + data.message);
+            }
+        } catch(e) { alert(e); }
+        btn.disabled = false; btn.innerText = "Analyze Dataset";
+    }
+    function populateSplits(splits) {
+        const splitSel = document.getElementById('split_select');
+        splitSel.innerHTML = '';
+        splits.forEach(s => {
+            splitSel.innerHTML += `<option value="${s}">${s}</option>`;
+        });
+        splitSel.disabled = false;
+    }
+    async function updateSplits() {
+        const conf = document.getElementById('config_select').value;
+        const ds = document.getElementById('dataset_id').value;
+        const res = await fetch('/get_splits', {
+            method: 'POST', headers: {'Content-Type': 'application/json'},
+            body: JSON.stringify({dataset_id: ds, config: conf, token: document.getElementById('token').value})
+        });
+        const data = await res.json();
+        if(data.status === 'success') populateSplits(data.splits);
+    }
+    // --- Step 2 Functions ---
+    async function inspectRows() {
+        document.getElementById('load_sample_btn').innerText = "Loading...";
         const payload = {
             token: document.getElementById('token').value,
             dataset_id: document.getElementById('dataset_id').value,
+            config: document.getElementById('config_select').value,
+            split: document.getElementById('split_select').value
         };
+        const res = await fetch('/inspect_rows', {
             method: 'POST', headers: {'Content-Type': 'application/json'},
             body: JSON.stringify(payload)
         });
         const data = await res.json();
+        document.getElementById('load_sample_btn').innerText = "Load Sample Data ⬇";
         if(data.status === 'success') {
             document.getElementById('inspector-panel').style.display = 'block';
             const tbody = document.getElementById('col-list');
             tbody.innerHTML = '';
             for (const [col, info] of Object.entries(data.analysis)) {
+                let badge = info.is_json_string ? '<span class="json-badge">JSON STR</span>' : '';
+                tbody.innerHTML += `<tr>
+                    <td><b>${col}</b></td>
+                    <td>${info.type} ${badge}</td>
                 </tr>`;
             }
         } else {
+            alert(data.message);
         }
     }
+    // --- Step 3 Functions ---
     function getRecipe() {
         const jsonRule = document.getElementById('json-rule').value;
         const renameRule = document.getElementById('rename-rule').value;
         let recipe = { json_expansions: [], renames: {}, drops: [], filters: [] };
         if(jsonRule.includes(':')) {
             let [col, keys] = jsonRule.split(':');
             recipe.json_expansions.push({
                 keys: keys.split(',').map(k => k.trim())
             });
         }
         if(renameRule.includes('=')) {
             let [oldC, newC] = renameRule.split('=');
             recipe.renames[oldC.trim()] = newC.trim();
         }
         if(dropRule) recipe.drops.push(dropRule.trim());
         if(filterRule) recipe.filters.push(filterRule.trim());
         return recipe;
         const payload = {
             token: document.getElementById('token').value,
             dataset_id: document.getElementById('dataset_id').value,
+            config: document.getElementById('config_select').value,
+            split: document.getElementById('split_select').value,
             recipe: getRecipe()
         };
         });
         const data = await res.json();
+        document.getElementById('preview-panel').style.display = 'block';
         const area = document.getElementById('preview-area');
         if (data.status === 'success') {
             area.innerHTML = `<pre>${JSON.stringify(data.rows, null, 2)}</pre>`;
         const payload = {
             token: document.getElementById('token').value,
             dataset_id: document.getElementById('dataset_id').value,
+            config: document.getElementById('config_select').value,
+            split: document.getElementById('split_select').value,
             target_id: document.getElementById('target_id').value,
             recipe: getRecipe(),
             max_rows: document.getElementById('max_rows').value
         };
         const data = await res.json();
         if(data.status === 'started') {
+            document.getElementById('job-status').style.display = 'block';
             pollStatus(data.job_id);
         }
     }
             const data = await res.json();
             if(data.status === 'running') {
+                statusDiv.innerText = "Processing... (Checking every 2s)";
+                statusDiv.className = "mt-3 p-3 rounded bg-info text-white fw-bold text-center";
             } else if (data.status === 'completed') {
+                statusDiv.innerText = `Success! Pushed ${data.result.rows_processed} rows to Hub.`;
+                statusDiv.className = "mt-3 p-3 rounded bg-success text-white fw-bold text-center";
                 clearInterval(interval);
+            } else if (data.status === 'failed') {
                 statusDiv.innerText = `Error: ${data.error}`;
+                statusDiv.className = "mt-3 p-3 rounded bg-danger text-white fw-bold text-center";
                 clearInterval(interval);
             }
         }, 2000);
     }
 </script>
 </body>
 </html>