Spaces:
Sleeping
Sleeping
| <html lang="en"> | |
| <head> | |
| <meta charset="UTF-8"> | |
| <title>Dataset Architect</title> | |
| <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/css/bootstrap.min.css" rel="stylesheet"> | |
| <style> | |
| body { background-color: #f4f6f8; font-size: 0.9rem; } | |
| .box { background: white; border-radius: 8px; box-shadow: 0 2px 4px rgba(0,0,0,0.05); padding: 20px; margin-bottom: 20px; } | |
| .source-col-row { cursor: pointer; } | |
| .source-col-row:hover { background-color: #e9ecef; } | |
| .source-col-row.active { background-color: #cfe2ff; font-weight: bold; } | |
| pre { background: #212529; color: #adbac7; padding: 15px; border-radius: 6px; font-size: 0.8em; max-height: 250px; overflow: auto; } | |
| .builder-row { border: 1px solid #dee2e6; border-radius: 6px; background: #fff; padding: 10px; margin-bottom: 10px; position: relative; } | |
| .builder-row.mode-list { border-left: 4px solid #198754; } | |
| .builder-row.mode-python { border-left: 4px solid #fd7e14; } | |
| .builder-row.mode-simple { border-left: 4px solid #0d6efd; } | |
| .mode-badge { position: absolute; top: -10px; right: 10px; font-size: 0.7em; padding: 2px 6px; background: #fff; border: 1px solid #ccc; border-radius: 4px; text-transform: uppercase; font-weight: bold; color: #666; } | |
| .logic-group { background: #f8f9fa; padding: 8px; border-radius: 4px; margin-top: 5px; border: 1px dashed #ccc; } | |
| </style> | |
| </head> | |
| <body> | |
| <div class="container-fluid p-4"> | |
| <div class="d-flex justify-content-between align-items-center mb-4"> | |
| <h3>🏗️ Hugging Face Dataset Architect</h3> | |
| <div><input type="password" id="token" class="form-control form-control-sm" placeholder="HF Token" style="width: 200px;"></div> | |
| </div> | |
| <div class="box"> | |
| <div class="row g-2"> | |
| <div class="col-md-4"> | |
| <div class="input-group"> | |
| <input type="text" id="dataset_id" class="form-control" placeholder="Source ID (e.g. the_stack)"> | |
| <button class="btn btn-primary" onclick="loadMetadata()">Connect</button> | |
| </div> | |
| </div> | |
| <div class="col-md-2"><select id="config_select" class="form-select" onchange="updateSplits()" disabled><option>Config...</option></select></div> | |
| <div class="col-md-2"><select id="split_select" class="form-select" disabled><option>Split...</option></select></div> | |
| <div class="col-md-4"><button id="inspect_btn" class="btn btn-success w-100" onclick="startInspection()" disabled>Analyze Schema</button></div> | |
| </div> | |
| </div> | |
| <div class="row" id="workspace" style="display:none;"> | |
| <div class="col-md-4"> | |
| <div class="box h-100"> | |
| <h6>Source Columns</h6> | |
| <div class="mb-3"><input type="text" id="source_filter" class="form-control form-control-sm" placeholder="Filter rows: len(text) > 100"></div> | |
| <ul class="list-group list-group-flush mb-3" id="source_col_list" style="max-height: 300px; overflow-y: auto;"></ul> | |
| <h6>Data Preview</h6> | |
| <div id="data_preview_box"><pre class="text-muted">Select a column.</pre></div> | |
| </div> | |
| </div> | |
| <div class="col-md-8"> | |
| <div class="box h-100"> | |
| <div class="d-flex justify-content-between mb-3"> | |
| <h6>New Schema Definition</h6> | |
| <button class="btn btn-sm btn-outline-primary" onclick="addBuilderRow()">+ Add Column</button> | |
| </div> | |
| <div id="builder_container"></div> | |
| <hr class="my-4"> | |
| <div class="step-header">Metadata & Publish</div> | |
| <div class="row g-2 mb-3"> | |
| <div class="col-md-8"> | |
| <label class="small fw-bold text-muted">Target Repository Name</label> | |
| <input type="text" id="target_id" class="form-control" placeholder="username/my-new-dataset"> | |
| </div> | |
| <div class="col-md-4"> | |
| <label class="small fw-bold text-muted">License</label> | |
| <input type="text" id="license_input" class="form-control" value="apache-2.0" placeholder="mit, cc-by-4.0..."> | |
| </div> | |
| </div> | |
| <div class="row g-2 align-items-end"> | |
| <div class="col-md-4"> | |
| <label class="small fw-bold text-muted">Max Rows (Test)</label> | |
| <input type="number" id="max_rows" class="form-control" placeholder="All"> | |
| </div> | |
| <div class="col-md-8 d-flex gap-2"> | |
| <button class="btn btn-outline-dark w-50" onclick="runPreview()">Preview Data</button> | |
| <button class="btn btn-success w-50" onclick="executeJob()">🚀 Push to Hub + Card</button> | |
| </div> | |
| </div> | |
| <div id="job_status" class="alert mt-3" style="display:none;"></div> | |
| <div id="final_preview" class="mt-3" style="display:none;"></div> | |
| </div> | |
| </div> | |
| </div> | |
| <script> | |
| let schemaData = {}; | |
| let sampleRows = []; | |
| function addBuilderRow() { | |
| const container = document.getElementById('builder_container'); | |
| const id = 'row_' + Date.now(); | |
| let sourceOpts = '<option value="">Select Source...</option>'; | |
| for(let col in schemaData) { | |
| let label = col + (schemaData[col].type === 'List' ? ' [List]' : ''); | |
| sourceOpts += `<option value="${col}">${label}</option>`; | |
| } | |
| const div = document.createElement('div'); | |
| div.className = "builder-row mode-simple"; | |
| div.id = id; | |
| div.innerHTML = ` | |
| <span class="mode-badge">Simple Path</span> | |
| <div class="row g-2 align-items-center mb-2"> | |
| <div class="col-md-3"> | |
| <input type="text" class="form-control form-control-sm target-name" placeholder="New Col Name"> | |
| </div> | |
| <div class="col-md-3"> | |
| <select class="form-select form-select-sm mode-select" onchange="changeMode('${id}', this.value)"> | |
| <option value="simple">Simple Path</option> | |
| <option value="list_search">List Filter (Get X where Y=Z)</option> | |
| <option value="python">Custom Python</option> | |
| <option value="requests">Request</option> | |
| </select> | |
| </div> | |
| <div class="col-md-5 text-end"> | |
| <button class="btn btn-sm btn-link text-danger text-decoration-none" onclick="this.closest('.builder-row').remove()">Remove</button> | |
| </div> | |
| </div> | |
| <div class="inputs-area"> | |
| <div class="input-group input-group-sm"> | |
| <span class="input-group-text">Source</span> | |
| <select class="form-select source-col">${sourceOpts}</select> | |
| <input type="text" class="form-control source-extra" placeholder="Optional .dot.notation"> | |
| </div> | |
| </div> | |
| `; | |
| container.appendChild(div); | |
| } | |
| function changeMode(rowId, mode) { | |
| const row = document.getElementById(rowId); | |
| const area = row.querySelector('.inputs-area'); | |
| const badge = row.querySelector('.mode-badge'); | |
| row.className = `builder-row mode-${mode}`; | |
| let sourceOpts = '<option value="">Select Col...</option>'; | |
| for(let col in schemaData) { | |
| sourceOpts += `<option value="${col}">${col}</option>`; | |
| } | |
| if(mode === 'simple') { | |
| badge.innerText = "Simple Path"; | |
| area.innerHTML = ` | |
| <div class="input-group input-group-sm"> | |
| <span class="input-group-text">Source</span> | |
| <select class="form-select source-col">${sourceOpts}</select> | |
| <input type="text" class="form-control source-extra" placeholder="Optional .dot.notation (e.g. meta.url)"> | |
| </div>`; | |
| } | |
| else if (mode === 'list_search') { | |
| badge.innerText = "List Logic"; | |
| area.innerHTML = ` | |
| <div class="logic-group"> | |
| <div class="d-flex gap-2 align-items-center mb-1"> | |
| <span class="fw-bold small">FROM</span> | |
| <select class="form-select form-select-sm source-col" style="width:150px">${sourceOpts}</select> | |
| <span class="fw-bold small">FIND ITEM WHERE</span> | |
| </div> | |
| <div class="d-flex gap-2 align-items-center mb-1"> | |
| <input type="text" class="form-control form-control-sm filter-key" placeholder="Key (e.g. role)"> | |
| <span class="fw-bold small">=</span> | |
| <input type="text" class="form-control form-control-sm filter-val" placeholder="Value (e.g. user)"> | |
| </div> | |
| <div class="d-flex gap-2 align-items-center"> | |
| <span class="fw-bold small">EXTRACT</span> | |
| <input type="text" class="form-control form-control-sm target-key" placeholder="Key (e.g. content or content.analysis)"> | |
| </div> | |
| </div>`; | |
| } | |
| else if (mode === 'python') { | |
| badge.innerText = "Python"; | |
| area.innerHTML = ` | |
| <div class="input-group input-group-sm"> | |
| <span class="input-group-text">val = </span> | |
| <input type="text" class="form-control python-expr" placeholder="row['text'].upper() or json.loads(row['meta'])['id']"> | |
| </div>`; | |
| } | |
| else if (mode === 'requests') { | |
| badge.innerText = "Request"; | |
| area.innerHTML = ` | |
| <div class="input-group input-group-sm"> | |
| <span class="input-group-text">url = </span> | |
| <input type="text" class="form-control request-url" placeholder="row['text'].upper() or json.loads(row['meta'])['id']"> | |
| <span class="input-group-text">payload = </span> | |
| <input type="text" class="form-control request-payload" placeholder="row['text'].upper() or json.loads(row['meta'])['id']"> | |
| </div>`; | |
| } | |
| } | |
| function getRecipe() { | |
| const rows = document.querySelectorAll('.builder-row'); | |
| let columns = []; | |
| rows.forEach(r => { | |
| const name = r.querySelector('.target-name').value.trim(); | |
| const mode = r.querySelector('.mode-select').value; | |
| // Skip rows with no target name | |
| if(!name) { | |
| console.warn('Skipping row with empty target name'); | |
| return; | |
| } | |
| if(mode === 'simple') { | |
| let src = r.querySelector('.source-col').value; | |
| let extra = r.querySelector('.source-extra').value.trim(); | |
| // CRITICAL FIX: Validate that source is selected | |
| if(!src) { | |
| console.error(`Row "${name}" has no source column selected`); | |
| alert(`Error: Column "${name}" has no source selected!`); | |
| return; | |
| } | |
| // Append extra path if provided | |
| if(extra) { | |
| // Handle both .dot.notation and dot.notation | |
| src = extra.startsWith('.') ? src + extra : src + '.' + extra; | |
| } | |
| console.log(`Simple path: ${name} <- ${src}`); | |
| columns.push({ type: 'simple', name: name, source: src }); | |
| } | |
| else if(mode === 'list_search') { | |
| const srcCol = r.querySelector('.source-col').value; | |
| const filterKey = r.querySelector('.filter-key').value.trim(); | |
| const filterVal = r.querySelector('.filter-val').value.trim(); | |
| const targetKey = r.querySelector('.target-key').value.trim(); | |
| if(!srcCol || !filterKey || !filterVal || !targetKey) { | |
| console.error(`List search column "${name}" is incomplete`); | |
| alert(`Error: Column "${name}" list search is incomplete!`); | |
| return; | |
| } | |
| columns.push({ | |
| type: 'list_search', | |
| name: name, | |
| source: srcCol, | |
| filter_key: filterKey, | |
| filter_val: filterVal, | |
| target_key: targetKey | |
| }); | |
| } | |
| else if(mode === 'python') { | |
| const expr = r.querySelector('.python-expr').value.trim(); | |
| if(!expr) { | |
| console.error(`Python column "${name}" has no expression`); | |
| alert(`Error: Column "${name}" has no Python expression!`); | |
| return; | |
| } | |
| columns.push({ | |
| type: 'python', | |
| name: name, | |
| expression: expr | |
| }); | |
| } | |
| else if(mode === 'requests') { | |
| const rurl = r.querySelector('.request-url').value.trim(); | |
| const rpay = r.querySelector('.request-payload').value.trim(); | |
| if(!rurl || !rpay) { | |
| console.error(`Python column "${name}" has no expression`); | |
| alert(`Error: Column "${name}" has no Python expression!`); | |
| return; | |
| } | |
| columns.push({ | |
| type: 'request', | |
| name: name, | |
| rurl: rurl, | |
| rpay: rpay | |
| }); | |
| } | |
| }); | |
| const recipe = { | |
| filter_rule: document.getElementById('source_filter').value.trim(), | |
| columns: columns | |
| }; | |
| console.log('Generated recipe:', JSON.stringify(recipe, null, 2)); | |
| return recipe; | |
| } | |
| async function loadMetadata() { | |
| const btn = document.querySelector('button[onclick="loadMetadata()"]'); | |
| const tokenVal = document.getElementById('token').value; | |
| const datasetVal = document.getElementById('dataset_id').value; | |
| if(!datasetVal) { alert("Please enter a Dataset ID"); return; } | |
| btn.disabled = true; btn.innerText = "..."; | |
| try { | |
| const res = await fetch('/analyze_metadata', { | |
| method: 'POST', headers: {'Content-Type': 'application/json'}, | |
| body: JSON.stringify({ | |
| token: tokenVal, | |
| dataset_id: datasetVal | |
| }) | |
| }); | |
| const data = await res.json(); | |
| if(data.status === 'success') { | |
| const confSel = document.getElementById('config_select'); | |
| confSel.innerHTML = ''; | |
| if (Array.isArray(data.configs) && data.configs.length > 0) { | |
| data.configs.forEach(c => { | |
| confSel.innerHTML += `<option value="${c}">${c}</option>`; | |
| }); | |
| } else { | |
| confSel.innerHTML = `<option value="default">default</option>`; | |
| } | |
| confSel.disabled = false; | |
| populateSplits(data.splits); | |
| const licInput = document.getElementById('license_input'); | |
| if(data.license_detected && licInput) { | |
| licInput.value = data.license_detected; | |
| } | |
| document.getElementById('inspect_btn').disabled = false; | |
| } else { | |
| alert('Server Error: ' + data.message); | |
| } | |
| } catch(e) { | |
| console.error(e); | |
| alert("Network Error: " + e); | |
| } | |
| btn.disabled = false; btn.innerText = "Connect"; | |
| } | |
| function populateSplits(splits) { | |
| const sel = document.getElementById('split_select'); | |
| sel.innerHTML = ''; | |
| if (Array.isArray(splits) && splits.length > 0) { | |
| splits.forEach(s => sel.innerHTML += `<option value="${s}">${s}</option>`); | |
| } else { | |
| sel.innerHTML = `<option value="train">train (fallback)</option>`; | |
| } | |
| sel.disabled = false; | |
| } | |
| async function updateSplits() { | |
| const conf = document.getElementById('config_select').value; | |
| const ds = document.getElementById('dataset_id').value; | |
| const token = document.getElementById('token').value; | |
| try { | |
| const res = await fetch('/get_splits', { | |
| method: 'POST', headers: {'Content-Type': 'application/json'}, | |
| body: JSON.stringify({dataset_id: ds, config: conf, token: token}) | |
| }); | |
| const data = await res.json(); | |
| if(data.status === 'success') { | |
| populateSplits(data.splits); | |
| } else { | |
| console.warn("Could not fetch specific splits, using defaults."); | |
| populateSplits(['train', 'test', 'validation']); | |
| } | |
| } catch(e) { | |
| console.error(e); | |
| populateSplits(['train', 'test', 'validation']); | |
| } | |
| } | |
| async function startInspection() { | |
| const btn = document.getElementById('inspect_btn'); | |
| btn.innerText = "Scanning..."; btn.disabled = true; | |
| try { | |
| const res = await fetch('/inspect_rows', { | |
| method: 'POST', headers: {'Content-Type': 'application/json'}, | |
| body: JSON.stringify({ | |
| token: document.getElementById('token').value, | |
| dataset_id: document.getElementById('dataset_id').value, | |
| config: document.getElementById('config_select').value, | |
| split: document.getElementById('split_select').value | |
| }) | |
| }); | |
| const data = await res.json(); | |
| if(data.status === 'success') { | |
| document.getElementById('workspace').style.display = 'flex'; | |
| sampleRows = data.samples; | |
| schemaData = data.schema; | |
| const list = document.getElementById('source_col_list'); | |
| list.innerHTML = ''; | |
| for(let col in schemaData) { | |
| let badge = schemaData[col].type === 'List' ? '<span class="badge bg-warning text-dark float-end">List</span>' : ''; | |
| list.innerHTML += `<li class="list-group-item source-col-row" onclick="previewCol('${col}')">${col} ${badge}</li>`; | |
| } | |
| } else { | |
| alert('Error: ' + data.message); | |
| } | |
| } catch(e) { | |
| console.error(e); | |
| alert('Network error: ' + e); | |
| } | |
| btn.innerText = "Analyze Schema"; | |
| btn.disabled = false; | |
| } | |
| function previewCol(col) { | |
| let html = ''; | |
| sampleRows.forEach(r => { | |
| let v = r[col]; | |
| if(typeof v === 'object') v = JSON.stringify(v, null, 2); | |
| html += v + "\n" + "-".repeat(20) + "\n"; | |
| }); | |
| document.getElementById('data_preview_box').innerHTML = `<pre>${html}</pre>`; | |
| } | |
| async function runPreview() { | |
| const box = document.getElementById('final_preview'); | |
| box.style.display='block'; | |
| box.innerHTML = "Generating preview..."; | |
| const recipe = getRecipe(); | |
| // Validate recipe has columns | |
| if(!recipe.columns || recipe.columns.length === 0) { | |
| box.innerHTML = '<div class="alert alert-warning">No columns defined! Please add at least one column.</div>'; | |
| return; | |
| } | |
| console.log('Sending preview request with recipe:', recipe); | |
| try { | |
| const res = await fetch('/preview', { | |
| method: 'POST', | |
| headers: {'Content-Type': 'application/json'}, | |
| body: JSON.stringify({ | |
| token: document.getElementById('token').value, | |
| dataset_id: document.getElementById('dataset_id').value, | |
| config: document.getElementById('config_select').value, | |
| split: document.getElementById('split_select').value, | |
| recipe: recipe | |
| }) | |
| }); | |
| const data = await res.json(); | |
| console.log('Preview response:', data); | |
| if(data.status === 'success') { | |
| box.innerHTML = `<pre>${JSON.stringify(data.rows, null, 2)}</pre>`; | |
| } else { | |
| box.innerHTML = `<div class="alert alert-danger">Error: ${data.message}</div>`; | |
| } | |
| } catch(e) { | |
| console.error('Preview error:', e); | |
| box.innerHTML = `<div class="alert alert-danger">Network error: ${e}</div>`; | |
| } | |
| } | |
| async function executeJob() { | |
| const recipe = getRecipe(); | |
| if(!recipe.columns || recipe.columns.length === 0) { | |
| alert('No columns defined!'); | |
| return; | |
| } | |
| const targetId = document.getElementById('target_id').value.trim(); | |
| if(!targetId) { | |
| alert('Please enter a target repository name!'); | |
| return; | |
| } | |
| const res = await fetch('/execute', { | |
| method: 'POST', | |
| headers: {'Content-Type': 'application/json'}, | |
| body: JSON.stringify({ | |
| token: document.getElementById('token').value, | |
| dataset_id: document.getElementById('dataset_id').value, | |
| config: document.getElementById('config_select').value, | |
| split: document.getElementById('split_select').value, | |
| target_id: targetId, | |
| recipe: recipe, | |
| max_rows: document.getElementById('max_rows').value, | |
| license: document.getElementById('license_input').value | |
| }) | |
| }); | |
| const data = await res.json(); | |
| if(data.status === 'started') { | |
| const statusBox = document.getElementById('job_status'); | |
| statusBox.style.display = 'block'; | |
| statusBox.className = 'alert alert-info mt-3'; | |
| statusBox.innerText = "Job started, processing..."; | |
| const interval = setInterval(async () => { | |
| const s = await fetch(`/status/${data.job_id}`).then(r=>r.json()); | |
| if(s.status === 'completed') { | |
| clearInterval(interval); | |
| statusBox.className = 'alert alert-success mt-3'; | |
| statusBox.innerText = `✅ Done! Processed ${s.result.rows_processed} rows.`; | |
| } else if(s.status === 'failed') { | |
| clearInterval(interval); | |
| statusBox.className = 'alert alert-danger mt-3'; | |
| statusBox.innerText = `❌ Job failed: ${s.error}`; | |
| } | |
| }, 2000); | |
| } else { | |
| alert('Failed to start job: ' + (data.message || 'Unknown error')); | |
| } | |
| } | |
| </script> | |
| </body> | |
| </html> |