index.html

<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<meta name="description" content="Interactive AI data curation explorer for filtering, deduplication, quality scoring, decontamination and dataset selection.">
<title>Curation Explorer</title>
<style>
:root{--bg:#07101d;--panel:#0c192d;--line:#263f5c;--text:#f4f8ff;--muted:#9eb1c6;--cyan:#50ddff;--violet:#a68bff}
*{box-sizing:border-box}body{margin:0;background:radial-gradient(circle at 15% -8%,rgba(80,221,255,.14),transparent 31%),radial-gradient(circle at 88% 0,rgba(166,139,255,.12),transparent 28%),var(--bg);color:var(--text);font:15px/1.55 Inter,system-ui,sans-serif}.wrap{max-width:1160px;margin:auto;padding:0 22px}.hero{text-align:center;padding:66px 0 32px}.kicker{display:inline-block;border:1px solid var(--line);border-radius:999px;padding:7px 12px;background:#0a1830;color:var(--cyan);font-size:11px;font-weight:850;text-transform:uppercase;letter-spacing:.1em}h1{font-size:clamp(46px,8vw,78px);line-height:1;margin:20px 0 14px;letter-spacing:-.055em}.grad{background:linear-gradient(90deg,var(--cyan),#dce8ff,var(--violet));background-clip:text;-webkit-background-clip:text;color:transparent}.hero p{max-width:820px;margin:auto;color:var(--muted);font-size:18px}.layout{display:grid;grid-template-columns:340px 1fr;gap:16px;padding:18px 0 52px}.panel{border:1px solid var(--line);border-radius:19px;padding:18px;background:linear-gradient(150deg,#10203b,#09162a);box-shadow:0 18px 44px rgba(0,0,0,.22)}.panel h2{font-size:20px;margin:0 0 5px}.sub{color:var(--muted);font-size:12px;margin-bottom:14px}label{display:block;font-size:12px;font-weight:800;margin:12px 0 6px}select{width:100%;background:#08172b;color:var(--text);border:1px solid var(--line);border-radius:11px;padding:11px 12px;outline:none}button{width:100%;border:0;border-radius:11px;padding:11px 12px;color:#fff;font-weight:850;background:linear-gradient(90deg,#3188ff,#775df0);cursor:pointer;margin-top:14px}.resultHead{display:flex;justify-content:space-between;align-items:flex-start;gap:10px}.eyebrow{font-size:10px;color:var(--cyan);font-weight:850;text-transform:uppercase;letter-spacing:.08em}.route{font-size:28px;font-weight:900;letter-spacing:-.035em;margin-top:2px}.badge{font-size:10px;border:1px solid #31506f;background:#0e2039;color:#c9dae9;border-radius:999px;padding:5px 8px}.explain{color:var(--muted);font-size:13px;margin-top:7px}.flowbox{margin:17px 0;border:1px solid #203854;border-radius:15px;background:#07152a;padding:18px}.flow{display:flex;justify-content:center;align-items:center;gap:7px;flex-wrap:wrap}.node{border:1px solid #35577b;background:#102542;border-radius:9px;padding:9px 10px;font-size:11px;font-weight:850}.node.hot{border-color:#786ef0;background:#251f4d}.arrow{color:var(--cyan);font-weight:900}.scoregrid{display:grid;grid-template-columns:repeat(3,1fr);gap:8px}.score{border:1px solid var(--line);background:#08172b;border-radius:10px;padding:9px}.score b{display:flex;justify-content:space-between;font-size:11px;margin-bottom:6px}.bar{height:6px;background:#182a43;border-radius:999px;overflow:hidden}.fill{height:100%;background:linear-gradient(90deg,var(--cyan),var(--violet))}.steps{margin-top:15px;border-top:1px solid var(--line);padding-top:12px}.step{padding:10px 0;border-bottom:1px solid #1d334d}.step b{display:block;font-size:12px}.step span{font-size:11px;color:var(--muted)}.note{margin-top:13px;font-size:11px;color:var(--muted)}footer{border-top:1px solid var(--line);padding:28px 0 45px;color:var(--muted);font-size:13px}footer strong{color:var(--text)}@media(max-width:850px){.layout{grid-template-columns:1fr}.scoregrid{grid-template-columns:1fr 1fr}}
</style>
</head>
<body>
<header class="hero"><div class="wrap"><span class="kicker">AI Data · Quality · Training</span><h1>Curation <span class="grad">Explorer</span></h1><p>Explore data curation strategies for pretraining, post-training, evaluation, RAG and agentic AI.</p></div></header>
<main class="wrap"><section class="layout">
<aside class="panel"><h2>Configure the data problem</h2><div class="sub">Choose the AI stage, data type and primary issue.</div>
<label for="stage">AI stage</label><select id="stage"><option value="pretraining">Pretraining</option><option value="posttraining">Post-Training</option><option value="evaluation">Evaluation</option><option value="rag">RAG / Retrieval</option><option value="agents">Agent Data</option></select>
<label for="dtype">Data type</label><select id="dtype"><option value="text">Text</option><option value="code">Code</option><option value="multimodal">Multimodal</option><option value="synthetic">Synthetic</option><option value="trajectories">Agent / Robot Trajectories</option></select>
<label for="problem">Primary issue</label><select id="problem"><option value="quality">Low / inconsistent quality</option><option value="duplicates">Duplicates</option><option value="contamination">Benchmark contamination</option><option value="privacy">PII / privacy</option><option value="imbalance">Distribution imbalance</option><option value="provenance">Weak provenance</option></select>
<button id="build">Build curation pipeline</button></aside>
<section class="panel"><div class="resultHead"><div><div class="eyebrow">Recommended curation path</div><div class="route" id="routeName"></div></div><span class="badge" id="badge"></span></div><div class="explain" id="explain"></div><div class="flowbox"><div class="flow" id="flow"></div></div><div class="scoregrid" id="scores"></div><div class="steps" id="steps"></div><div class="note">Educational, vendor-neutral guidance only. Real curation policies should be validated against downstream model behavior, licensing, privacy, safety and governance requirements.</div></section>
</section></main>
<footer><div class="wrap"><strong>Curation Explorer</strong> — an independent Hugging Face Space for AI data quality.<br>Collaboration and partnerships: <strong>agenten@magenta.de</strong></div></footer>
<script>
const $=id=>document.getElementById(id);
const stageBase={pretraining:{label:'Pretraining',base:['Normalize','Language Filter','Deduplicate','Quality Score','Select','Mix'],explain:'Optimize large-scale data for efficient training while preserving useful diversity.'},posttraining:{label:'Post-Training',base:['Validate Format','Score Quality','Verify Answer','Deduplicate','Balance Difficulty','Select'],explain:'Prioritize correctness, task diversity and reliable supervision for instruction, preference and reasoning data.'},evaluation:{label:'Evaluation',base:['Validate Schema','Decontaminate','Check Ground Truth','Balance Difficulty','Audit Coverage'],explain:'Protect benchmark integrity and ensure the evaluation set measures the intended capabilities.'},rag:{label:'RAG / Retrieval',base:['Normalize','Remove Duplicates','Score Relevance','Check Freshness','Enrich Metadata','Index'],explain:'Curate source documents for retrieval quality, freshness, trust and metadata completeness.'},agents:{label:'Agent Data',base:['Validate Trace','Check Tool Calls','Score Outcome','Remove Failures','Balance Tasks','Select'],explain:'Curate trajectories by success, tool correctness, efficiency, safety and recovery behavior.'}};
const issueSteps={quality:['Heuristic Filter','Model Quality Scoring'],duplicates:['Exact Dedup','Near-Dedup','Semantic Dedup'],contamination:['Exact Match','N-gram Check','Semantic Decontamination'],privacy:['PII Detection','Redaction / Removal'],imbalance:['Cluster / Stratify','Reweight / Resample'],provenance:['Source Tracking','License Metadata','Transformation Lineage']};
const issueText={quality:'Quality varies across examples, so scoring and task-specific filtering become the main curation controls.',duplicates:'Repeated and near-repeated examples can waste compute and amplify memorization.',contamination:'Overlap with evaluation data can distort benchmark results and should be detected before use.',privacy:'Sensitive personal data requires detection, policy decisions and auditable handling.',imbalance:'Dominant sources or topics can distort the training distribution and reduce coverage.',provenance:'Weak lineage makes datasets harder to audit, reproduce and govern.'};
const why={Normalize:'Create a stable representation before later filters.','Language Filter':'Keep the target language mix explicit and measurable.',Deduplicate:'Reduce repeated examples and unnecessary training weight.','Quality Score':'Estimate usefulness relative to the target objective.',Select:'Retain examples that best match quality, utility and coverage goals.',Mix:'Control the contribution of domains and source datasets.','Validate Format':'Remove malformed or structurally invalid examples.','Verify Answer':'Check supervision quality before using the example.','Balance Difficulty':'Avoid a dataset dominated by trivial examples.','Validate Schema':'Ensure benchmark records are structurally consistent.',Decontaminate:'Reduce overlap between training material and evaluation items.','Check Ground Truth':'Verify expected answers or labels.','Audit Coverage':'Check whether the benchmark covers intended tasks and difficulty.','Score Relevance':'Prioritize documents likely to answer target retrieval queries.','Check Freshness':'Identify stale or outdated knowledge.','Enrich Metadata':'Improve retrieval filtering, access control and traceability.',Index:'Prepare the curated corpus for retrieval.','Validate Trace':'Confirm that the trajectory contains usable state/action information.','Check Tool Calls':'Detect invalid or malformed tool use.','Score Outcome':'Use task completion and utility as curation signals.','Remove Failures':'Exclude or isolate unsuccessful traces depending on training goals.','Balance Tasks':'Maintain representative coverage across agent tasks.','Heuristic Filter':'Apply inexpensive, interpretable first-pass quality rules.','Model Quality Scoring':'Use learned signals to estimate task-specific data quality.','Exact Dedup':'Remove identical examples efficiently.','Near-Dedup':'Detect slightly modified copies.','Semantic Dedup':'Identify highly redundant examples by meaning.','Exact Match':'Check direct benchmark overlap.','N-gram Check':'Detect partial textual overlap.','Semantic Decontamination':'Find conceptually similar benchmark leakage.','PII Detection':'Identify potentially sensitive personal information.','Redaction / Removal':'Apply the selected privacy policy to flagged fields.','Cluster / Stratify':'Measure distributional concentration before rebalancing.','Reweight / Resample':'Adjust how frequently different groups are represented.','Source Tracking':'Preserve the origin of each example.','License Metadata':'Keep usage rights connected to the data.','Transformation Lineage':'Record the processing path that produced the final example.','Synthetic Verification':'Check generated examples before they enter training.','Parse / Compile Check':'Use structural validity as a code-quality signal.','Alignment Check':'Verify that modalities correspond to each other.','Outcome Verification':'Confirm whether the trajectory accomplished its target.'};
function uniq(a){return [...new Set(a)]}
function build(){const stage=$('stage').value,dtype=$('dtype').value,problem=$('problem').value,s=stageBase[stage];let steps=[...s.base];if(problem==='quality')steps.splice(2,0,...issueSteps.quality);if(problem==='duplicates')steps.splice(2,0,...issueSteps.duplicates);if(problem==='contamination')steps.splice(2,0,...issueSteps.contamination);if(problem==='privacy')steps.splice(2,0,...issueSteps.privacy);if(problem==='imbalance')steps.splice(Math.max(steps.length-1,1),0,...issueSteps.imbalance);if(problem==='provenance')steps.push(...issueSteps.provenance);if(dtype==='synthetic')steps.splice(2,0,'Synthetic Verification');if(dtype==='code')steps.splice(2,0,'Parse / Compile Check');if(dtype==='multimodal')steps.splice(2,0,'Alignment Check');if(dtype==='trajectories')steps.splice(2,0,'Outcome Verification');steps=uniq(steps);$('routeName').textContent=s.label+' Curation';$('badge').textContent=dtype.replace('trajectories','Trajectories');$('explain').textContent=s.explain+' '+issueText[problem];const shown=steps.slice(0,8);$('flow').innerHTML=shown.map((x,i)=>`<span class="node ${i===shown.length-1?'hot':''}">${x}</span>${i<shown.length-1?'<span class="arrow">→</span>':''}`).join('');const base={pretraining:[92,88,82,78,90,72],posttraining:[96,90,84,91,78,80],evaluation:[98,92,97,86,70,88],rag:[88,86,72,94,82,93],agents:[93,89,76,90,78,84]}[stage],names=['Quality impact','Scale fit','Auditability','Task relevance','Diversity','Governance'];$('scores').innerHTML=names.map((n,i)=>{let v=base[i];if(problem==='provenance'&&(i===2||i===5))v=Math.min(100,v+8);if(problem==='imbalance'&&i===4)v=Math.min(100,v+8);if(problem==='quality'&&i===0)v=Math.min(100,v+5);return `<div class="score"><b><span>${n}</span><span>${v}</span></b><div class="bar"><div class="fill" style="width:${v}%"></div></div></div>`}).join('');$('steps').innerHTML='<div class="eyebrow">Pipeline rationale</div>'+steps.slice(0,7).map((x,i)=>`<div class="step"><b>${i+1}. ${x}</b><span>${why[x]||'Apply this curation step as part of the selected pipeline.'}</span></div>`).join('')}
['stage','dtype','problem'].forEach(id=>$(id).addEventListener('change',build));$('build').addEventListener('click',build);build();
</script>
</body></html>