| | <!DOCTYPE html> |
| | <html lang="en"> |
| | <head> |
| | <meta charset="UTF-8"> |
| | <title>Dataset Architect</title> |
| | <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/css/bootstrap.min.css" rel="stylesheet"> |
| | <style> |
| | body { background-color: #f4f6f8; font-size: 0.9rem; } |
| | .box { background: white; border-radius: 8px; box-shadow: 0 2px 4px rgba(0,0,0,0.05); padding: 20px; margin-bottom: 20px; } |
| | .source-col-row { cursor: pointer; } |
| | .source-col-row:hover { background-color: #e9ecef; } |
| | .source-col-row.active { background-color: #cfe2ff; font-weight: bold; } |
| | pre { background: #212529; color: #adbac7; padding: 15px; border-radius: 6px; font-size: 0.8em; max-height: 250px; overflow: auto; } |
| | |
| | .builder-row { border: 1px solid #dee2e6; border-radius: 6px; background: #fff; padding: 10px; margin-bottom: 10px; position: relative; } |
| | .builder-row.mode-list { border-left: 4px solid #198754; } |
| | .builder-row.mode-python { border-left: 4px solid #fd7e14; } |
| | .builder-row.mode-simple { border-left: 4px solid #0d6efd; } |
| | |
| | .mode-badge { position: absolute; top: -10px; right: 10px; font-size: 0.7em; padding: 2px 6px; background: #fff; border: 1px solid #ccc; border-radius: 4px; text-transform: uppercase; font-weight: bold; color: #666; } |
| | |
| | .logic-group { background: #f8f9fa; padding: 8px; border-radius: 4px; margin-top: 5px; border: 1px dashed #ccc; } |
| | </style> |
| | </head> |
| | <body> |
| |
|
| | <div class="container-fluid p-4"> |
| | <div class="d-flex justify-content-between align-items-center mb-4"> |
| | <h3>🏗️ Hugging Face Dataset Architect</h3> |
| | <div><input type="password" id="token" class="form-control form-control-sm" placeholder="HF Token" style="width: 200px;"></div> |
| | </div> |
| |
|
| | <div class="box"> |
| | <div class="row g-2"> |
| | <div class="col-md-4"> |
| | <div class="input-group"> |
| | <input type="text" id="dataset_id" class="form-control" placeholder="Source ID (e.g. the_stack)"> |
| | <button class="btn btn-primary" onclick="loadMetadata()">Connect</button> |
| | </div> |
| | </div> |
| | <div class="col-md-2"><select id="config_select" class="form-select" onchange="updateSplits()" disabled><option>Config...</option></select></div> |
| | <div class="col-md-2"><select id="split_select" class="form-select" disabled><option>Split...</option></select></div> |
| | <div class="col-md-4"><button id="inspect_btn" class="btn btn-success w-100" onclick="startInspection()" disabled>Analyze Schema</button></div> |
| | </div> |
| | </div> |
| |
|
| | <div class="row" id="workspace" style="display:none;"> |
| | <div class="col-md-4"> |
| | <div class="box h-100"> |
| | <h6>Source Columns</h6> |
| | <div class="mb-3"><input type="text" id="source_filter" class="form-control form-control-sm" placeholder="Filter rows: len(text) > 100"></div> |
| | <ul class="list-group list-group-flush mb-3" id="source_col_list" style="max-height: 300px; overflow-y: auto;"></ul> |
| | <h6>Data Preview</h6> |
| | <div id="data_preview_box"><pre class="text-muted">Select a column.</pre></div> |
| | </div> |
| | </div> |
| |
|
| | <div class="col-md-8"> |
| | <div class="box h-100"> |
| | <div class="d-flex justify-content-between mb-3"> |
| | <h6>New Schema Definition</h6> |
| | <button class="btn btn-sm btn-outline-primary" onclick="addBuilderRow()">+ Add Column</button> |
| | </div> |
| | |
| | <div id="builder_container"></div> |
| |
|
| | <hr class="my-4"> |
| | |
| | <div class="step-header">Metadata & Publish</div> |
| | |
| | <div class="row g-2 mb-3"> |
| | <div class="col-md-8"> |
| | <label class="small fw-bold text-muted">Target Repository Name</label> |
| | <input type="text" id="target_id" class="form-control" placeholder="username/my-new-dataset"> |
| | </div> |
| | <div class="col-md-4"> |
| | <label class="small fw-bold text-muted">License</label> |
| | <input type="text" id="license_input" class="form-control" value="apache-2.0" placeholder="mit, cc-by-4.0..."> |
| | </div> |
| | </div> |
| |
|
| | <div class="row g-2 align-items-end"> |
| | <div class="col-md-4"> |
| | <label class="small fw-bold text-muted">Max Rows (Test)</label> |
| | <input type="number" id="max_rows" class="form-control" placeholder="All"> |
| | </div> |
| | <div class="col-md-8 d-flex gap-2"> |
| | <button class="btn btn-outline-dark w-50" onclick="runPreview()">Preview Data</button> |
| | <button class="btn btn-success w-50" onclick="executeJob()">🚀 Push to Hub + Card</button> |
| | </div> |
| | </div> |
| | |
| | <div id="job_status" class="alert mt-3" style="display:none;"></div> |
| | <div id="final_preview" class="mt-3" style="display:none;"></div> |
| | </div> |
| | </div> |
| | </div> |
| |
|
| | <script> |
| | let schemaData = {}; |
| | let sampleRows = []; |
| | |
| | function addBuilderRow() { |
| | const container = document.getElementById('builder_container'); |
| | const id = 'row_' + Date.now(); |
| | |
| | let sourceOpts = '<option value="">Select Source...</option>'; |
| | for(let col in schemaData) { |
| | let label = col + (schemaData[col].type === 'List' ? ' [List]' : ''); |
| | sourceOpts += `<option value="${col}">${label}</option>`; |
| | } |
| | |
| | const div = document.createElement('div'); |
| | div.className = "builder-row mode-simple"; |
| | div.id = id; |
| | |
| | div.innerHTML = ` |
| | <span class="mode-badge">Simple Path</span> |
| | <div class="row g-2 align-items-center mb-2"> |
| | <div class="col-md-3"> |
| | <input type="text" class="form-control form-control-sm target-name" placeholder="New Col Name"> |
| | </div> |
| | <div class="col-md-3"> |
| | <select class="form-select form-select-sm mode-select" onchange="changeMode('${id}', this.value)"> |
| | <option value="simple">Simple Path</option> |
| | <option value="list_search">List Filter (Get X where Y=Z)</option> |
| | <option value="python">Custom Python</option> |
| | <option value="requests">Request</option> |
| | </select> |
| | </div> |
| | <div class="col-md-5 text-end"> |
| | <button class="btn btn-sm btn-link text-danger text-decoration-none" onclick="this.closest('.builder-row').remove()">Remove</button> |
| | </div> |
| | </div> |
| | |
| | <div class="inputs-area"> |
| | <div class="input-group input-group-sm"> |
| | <span class="input-group-text">Source</span> |
| | <select class="form-select source-col">${sourceOpts}</select> |
| | <input type="text" class="form-control source-extra" placeholder="Optional .dot.notation"> |
| | </div> |
| | </div> |
| | `; |
| | container.appendChild(div); |
| | } |
| | |
| | function changeMode(rowId, mode) { |
| | const row = document.getElementById(rowId); |
| | const area = row.querySelector('.inputs-area'); |
| | const badge = row.querySelector('.mode-badge'); |
| | |
| | row.className = `builder-row mode-${mode}`; |
| | |
| | let sourceOpts = '<option value="">Select Col...</option>'; |
| | for(let col in schemaData) { |
| | sourceOpts += `<option value="${col}">${col}</option>`; |
| | } |
| | |
| | if(mode === 'simple') { |
| | badge.innerText = "Simple Path"; |
| | area.innerHTML = ` |
| | <div class="input-group input-group-sm"> |
| | <span class="input-group-text">Source</span> |
| | <select class="form-select source-col">${sourceOpts}</select> |
| | <input type="text" class="form-control source-extra" placeholder="Optional .dot.notation (e.g. meta.url)"> |
| | </div>`; |
| | } |
| | else if (mode === 'list_search') { |
| | badge.innerText = "List Logic"; |
| | area.innerHTML = ` |
| | <div class="logic-group"> |
| | <div class="d-flex gap-2 align-items-center mb-1"> |
| | <span class="fw-bold small">FROM</span> |
| | <select class="form-select form-select-sm source-col" style="width:150px">${sourceOpts}</select> |
| | <span class="fw-bold small">FIND ITEM WHERE</span> |
| | </div> |
| | <div class="d-flex gap-2 align-items-center mb-1"> |
| | <input type="text" class="form-control form-control-sm filter-key" placeholder="Key (e.g. role)"> |
| | <span class="fw-bold small">=</span> |
| | <input type="text" class="form-control form-control-sm filter-val" placeholder="Value (e.g. user)"> |
| | </div> |
| | <div class="d-flex gap-2 align-items-center"> |
| | <span class="fw-bold small">EXTRACT</span> |
| | <input type="text" class="form-control form-control-sm target-key" placeholder="Key (e.g. content or content.analysis)"> |
| | </div> |
| | </div>`; |
| | } |
| | else if (mode === 'python') { |
| | badge.innerText = "Python"; |
| | area.innerHTML = ` |
| | <div class="input-group input-group-sm"> |
| | <span class="input-group-text">val = </span> |
| | <input type="text" class="form-control python-expr" placeholder="row['text'].upper() or json.loads(row['meta'])['id']"> |
| | </div>`; |
| | } |
| | else if (mode === 'requests') { |
| | badge.innerText = "Request"; |
| | area.innerHTML = ` |
| | <div class="input-group input-group-sm"> |
| | <span class="input-group-text">url = </span> |
| | <input type="text" class="form-control request-url" placeholder="row['text'].upper() or json.loads(row['meta'])['id']"> |
| | <span class="input-group-text">payload = </span> |
| | <input type="text" class="form-control request-payload" placeholder="row['text'].upper() or json.loads(row['meta'])['id']"> |
| | </div>`; |
| | } |
| | } |
| | |
| | function getRecipe() { |
| | const rows = document.querySelectorAll('.builder-row'); |
| | let columns = []; |
| | |
| | rows.forEach(r => { |
| | const name = r.querySelector('.target-name').value.trim(); |
| | const mode = r.querySelector('.mode-select').value; |
| | |
| | |
| | if(!name) { |
| | console.warn('Skipping row with empty target name'); |
| | return; |
| | } |
| | |
| | if(mode === 'simple') { |
| | let src = r.querySelector('.source-col').value; |
| | let extra = r.querySelector('.source-extra').value.trim(); |
| | |
| | |
| | if(!src) { |
| | console.error(`Row "${name}" has no source column selected`); |
| | alert(`Error: Column "${name}" has no source selected!`); |
| | return; |
| | } |
| | |
| | |
| | if(extra) { |
| | |
| | src = extra.startsWith('.') ? src + extra : src + '.' + extra; |
| | } |
| | |
| | console.log(`Simple path: ${name} <- ${src}`); |
| | columns.push({ type: 'simple', name: name, source: src }); |
| | } |
| | else if(mode === 'list_search') { |
| | const srcCol = r.querySelector('.source-col').value; |
| | const filterKey = r.querySelector('.filter-key').value.trim(); |
| | const filterVal = r.querySelector('.filter-val').value.trim(); |
| | const targetKey = r.querySelector('.target-key').value.trim(); |
| | |
| | if(!srcCol || !filterKey || !filterVal || !targetKey) { |
| | console.error(`List search column "${name}" is incomplete`); |
| | alert(`Error: Column "${name}" list search is incomplete!`); |
| | return; |
| | } |
| | |
| | columns.push({ |
| | type: 'list_search', |
| | name: name, |
| | source: srcCol, |
| | filter_key: filterKey, |
| | filter_val: filterVal, |
| | target_key: targetKey |
| | }); |
| | } |
| | else if(mode === 'python') { |
| | const expr = r.querySelector('.python-expr').value.trim(); |
| | |
| | if(!expr) { |
| | console.error(`Python column "${name}" has no expression`); |
| | alert(`Error: Column "${name}" has no Python expression!`); |
| | return; |
| | } |
| | |
| | columns.push({ |
| | type: 'python', |
| | name: name, |
| | expression: expr |
| | }); |
| | } |
| | else if(mode === 'requests') { |
| | const rurl = r.querySelector('.request-url').value.trim(); |
| | const rpay = r.querySelector('.request-payload').value.trim(); |
| | |
| | if(!rurl || !rpay) { |
| | console.error(`Python column "${name}" has no expression`); |
| | alert(`Error: Column "${name}" has no Python expression!`); |
| | return; |
| | } |
| | |
| | columns.push({ |
| | type: 'request', |
| | name: name, |
| | rurl: rurl, |
| | rpay: rpay |
| | }); |
| | } |
| | }); |
| | |
| | const recipe = { |
| | filter_rule: document.getElementById('source_filter').value.trim(), |
| | columns: columns |
| | }; |
| | |
| | console.log('Generated recipe:', JSON.stringify(recipe, null, 2)); |
| | return recipe; |
| | } |
| | |
| | async function loadMetadata() { |
| | const btn = document.querySelector('button[onclick="loadMetadata()"]'); |
| | const tokenVal = document.getElementById('token').value; |
| | const datasetVal = document.getElementById('dataset_id').value; |
| | if(!datasetVal) { alert("Please enter a Dataset ID"); return; } |
| | btn.disabled = true; btn.innerText = "..."; |
| | |
| | try { |
| | const res = await fetch('/analyze_metadata', { |
| | method: 'POST', headers: {'Content-Type': 'application/json'}, |
| | body: JSON.stringify({ |
| | token: tokenVal, |
| | dataset_id: datasetVal |
| | }) |
| | }); |
| | |
| | const data = await res.json(); |
| | |
| | if(data.status === 'success') { |
| | const confSel = document.getElementById('config_select'); |
| | confSel.innerHTML = ''; |
| | |
| | if (Array.isArray(data.configs) && data.configs.length > 0) { |
| | data.configs.forEach(c => { |
| | confSel.innerHTML += `<option value="${c}">${c}</option>`; |
| | }); |
| | } else { |
| | confSel.innerHTML = `<option value="default">default</option>`; |
| | } |
| | confSel.disabled = false; |
| | |
| | populateSplits(data.splits); |
| | |
| | const licInput = document.getElementById('license_input'); |
| | if(data.license_detected && licInput) { |
| | licInput.value = data.license_detected; |
| | } |
| | |
| | document.getElementById('inspect_btn').disabled = false; |
| | } else { |
| | alert('Server Error: ' + data.message); |
| | } |
| | } catch(e) { |
| | console.error(e); |
| | alert("Network Error: " + e); |
| | } |
| | |
| | btn.disabled = false; btn.innerText = "Connect"; |
| | } |
| | |
| | function populateSplits(splits) { |
| | const sel = document.getElementById('split_select'); |
| | sel.innerHTML = ''; |
| | |
| | if (Array.isArray(splits) && splits.length > 0) { |
| | splits.forEach(s => sel.innerHTML += `<option value="${s}">${s}</option>`); |
| | } else { |
| | sel.innerHTML = `<option value="train">train (fallback)</option>`; |
| | } |
| | sel.disabled = false; |
| | } |
| | |
| | async function updateSplits() { |
| | const conf = document.getElementById('config_select').value; |
| | const ds = document.getElementById('dataset_id').value; |
| | const token = document.getElementById('token').value; |
| | try { |
| | const res = await fetch('/get_splits', { |
| | method: 'POST', headers: {'Content-Type': 'application/json'}, |
| | body: JSON.stringify({dataset_id: ds, config: conf, token: token}) |
| | }); |
| | const data = await res.json(); |
| | |
| | if(data.status === 'success') { |
| | populateSplits(data.splits); |
| | } else { |
| | console.warn("Could not fetch specific splits, using defaults."); |
| | populateSplits(['train', 'test', 'validation']); |
| | } |
| | } catch(e) { |
| | console.error(e); |
| | populateSplits(['train', 'test', 'validation']); |
| | } |
| | } |
| | |
| | async function startInspection() { |
| | const btn = document.getElementById('inspect_btn'); |
| | btn.innerText = "Scanning..."; btn.disabled = true; |
| | |
| | try { |
| | const res = await fetch('/inspect_rows', { |
| | method: 'POST', headers: {'Content-Type': 'application/json'}, |
| | body: JSON.stringify({ |
| | token: document.getElementById('token').value, |
| | dataset_id: document.getElementById('dataset_id').value, |
| | config: document.getElementById('config_select').value, |
| | split: document.getElementById('split_select').value |
| | }) |
| | }); |
| | const data = await res.json(); |
| | |
| | if(data.status === 'success') { |
| | document.getElementById('workspace').style.display = 'flex'; |
| | sampleRows = data.samples; |
| | schemaData = data.schema; |
| | |
| | const list = document.getElementById('source_col_list'); |
| | list.innerHTML = ''; |
| | for(let col in schemaData) { |
| | let badge = schemaData[col].type === 'List' ? '<span class="badge bg-warning text-dark float-end">List</span>' : ''; |
| | list.innerHTML += `<li class="list-group-item source-col-row" onclick="previewCol('${col}')">${col} ${badge}</li>`; |
| | } |
| | } else { |
| | alert('Error: ' + data.message); |
| | } |
| | } catch(e) { |
| | console.error(e); |
| | alert('Network error: ' + e); |
| | } |
| | |
| | btn.innerText = "Analyze Schema"; |
| | btn.disabled = false; |
| | } |
| | |
| | function previewCol(col) { |
| | let html = ''; |
| | sampleRows.forEach(r => { |
| | let v = r[col]; |
| | if(typeof v === 'object') v = JSON.stringify(v, null, 2); |
| | html += v + "\n" + "-".repeat(20) + "\n"; |
| | }); |
| | document.getElementById('data_preview_box').innerHTML = `<pre>${html}</pre>`; |
| | } |
| | |
| | async function runPreview() { |
| | const box = document.getElementById('final_preview'); |
| | box.style.display='block'; |
| | box.innerHTML = "Generating preview..."; |
| | |
| | const recipe = getRecipe(); |
| | |
| | |
| | if(!recipe.columns || recipe.columns.length === 0) { |
| | box.innerHTML = '<div class="alert alert-warning">No columns defined! Please add at least one column.</div>'; |
| | return; |
| | } |
| | |
| | console.log('Sending preview request with recipe:', recipe); |
| | |
| | try { |
| | const res = await fetch('/preview', { |
| | method: 'POST', |
| | headers: {'Content-Type': 'application/json'}, |
| | body: JSON.stringify({ |
| | token: document.getElementById('token').value, |
| | dataset_id: document.getElementById('dataset_id').value, |
| | config: document.getElementById('config_select').value, |
| | split: document.getElementById('split_select').value, |
| | recipe: recipe |
| | }) |
| | }); |
| | |
| | const data = await res.json(); |
| | console.log('Preview response:', data); |
| | |
| | if(data.status === 'success') { |
| | box.innerHTML = `<pre>${JSON.stringify(data.rows, null, 2)}</pre>`; |
| | } else { |
| | box.innerHTML = `<div class="alert alert-danger">Error: ${data.message}</div>`; |
| | } |
| | } catch(e) { |
| | console.error('Preview error:', e); |
| | box.innerHTML = `<div class="alert alert-danger">Network error: ${e}</div>`; |
| | } |
| | } |
| | |
| | async function executeJob() { |
| | const recipe = getRecipe(); |
| | |
| | if(!recipe.columns || recipe.columns.length === 0) { |
| | alert('No columns defined!'); |
| | return; |
| | } |
| | |
| | const targetId = document.getElementById('target_id').value.trim(); |
| | if(!targetId) { |
| | alert('Please enter a target repository name!'); |
| | return; |
| | } |
| | |
| | const res = await fetch('/execute', { |
| | method: 'POST', |
| | headers: {'Content-Type': 'application/json'}, |
| | body: JSON.stringify({ |
| | token: document.getElementById('token').value, |
| | dataset_id: document.getElementById('dataset_id').value, |
| | config: document.getElementById('config_select').value, |
| | split: document.getElementById('split_select').value, |
| | target_id: targetId, |
| | recipe: recipe, |
| | max_rows: document.getElementById('max_rows').value, |
| | license: document.getElementById('license_input').value |
| | }) |
| | }); |
| | |
| | const data = await res.json(); |
| | |
| | if(data.status === 'started') { |
| | const statusBox = document.getElementById('job_status'); |
| | statusBox.style.display = 'block'; |
| | statusBox.className = 'alert alert-info mt-3'; |
| | statusBox.innerText = "Job started, processing..."; |
| | |
| | const interval = setInterval(async () => { |
| | const s = await fetch(`/status/${data.job_id}`).then(r=>r.json()); |
| | if(s.status === 'completed') { |
| | clearInterval(interval); |
| | statusBox.className = 'alert alert-success mt-3'; |
| | statusBox.innerText = `✅ Done! Processed ${s.result.rows_processed} rows.`; |
| | } else if(s.status === 'failed') { |
| | clearInterval(interval); |
| | statusBox.className = 'alert alert-danger mt-3'; |
| | statusBox.innerText = `❌ Job failed: ${s.error}`; |
| | } |
| | }, 2000); |
| | } else { |
| | alert('Failed to start job: ' + (data.message || 'Unknown error')); |
| | } |
| | } |
| | </script> |
| | </body> |
| | </html> |
