Buckets:

HCAI-Lab
/

w2-consensus-deepdive-unlearning-artifacts

Files

xet

HCAI-Lab/w2-consensus-deepdive-unlearning-artifacts / social-data-attribution-w2 /scripts /build_corpus_explorer.py

glennmatlin

10 days ago

download

raw

28.5 kB

	#!/usr/bin/env python3
	"""Generate SOC-95 Corpus Explorer HTML playground from analysis outputs."""

	from __future__ import annotations

	import argparse
	import logging
	from pathlib import Path

	logger = logging.getLogger(__name__)

	DEFAULT_DATA_DIR = Path("/tmp/soc95-analysis-v3")
	DEFAULT_OUTPUT = Path("artifacts/soc95_corpus_explorer.html")

	DATA_FILES = [
	"topic_totals.csv",
	"format_totals.csv",
	"topic_format_totals.csv",
	"year_totals.csv",
	"source_family_totals.csv",
	"quality_label_totals.csv",
	"quality_score_histogram.csv",
	"token_count_histogram.csv",
	]


	def build_html(data_dir: Path) -> str:
	summary = (data_dir / "manifest_analysis_summary.json").read_text().strip()
	csv_data = {}
	for name in DATA_FILES:
	key = name.removesuffix(".csv").upper()
	csv_data[key] = (data_dir / name).read_text().strip()

	parts: list[str] = []
	parts.append(_html_head())
	parts.append(_html_body())
	parts.append("<script>\n")
	parts.append(_js_data(summary, csv_data))
	parts.append(_js_utilities())
	parts.append(_js_charts())
	parts.append(_js_init())
	parts.append("\n</script>\n</body>\n</html>")
	return "".join(parts)


	def _html_head() -> str:
	return """<!DOCTYPE html>
	<html lang="en">
	<head>
	<meta charset="utf-8">
	<meta name="viewport" content="width=device-width, initial-scale=1">
	<title>SOC-95 Corpus Explorer</title>
	<script src="https://cdn.plot.ly/plotly-2.35.2.min.js"></script>
	<style>
	, ::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
	body { background: #0d1117; color: #c9d1d9; font-family: system-ui, -apple-system, sans-serif; font-size: 14px; }
	header { padding: 20px 24px 12px; border-bottom: 1px solid #30363d; }
	header h1 { font-size: 20px; font-weight: 600; color: #e6edf3; }
	header p { font-size: 13px; color: #8b949e; margin-top: 4px; }
	.tab-nav { display: flex; gap: 0; border-bottom: 1px solid #30363d; padding: 0 24px; overflow-x: auto; }
	.tab-btn { background: none; border: none; border-bottom: 2px solid transparent; color: #8b949e; font-size: 13px; font-weight: 500; padding: 10px 16px; cursor: pointer; white-space: nowrap; transition: color 0.15s, border-color 0.15s; }
	.tab-btn:hover { color: #c9d1d9; }
	.tab-btn.active { color: #e6edf3; border-bottom-color: #2c7bb6; }
	.tab-content { display: none; padding: 24px; }
	.tab-content.active { display: block; }
	.cards { display: grid; grid-template-columns: repeat(auto-fit, minmax(160px, 1fr)); gap: 12px; margin-bottom: 24px; }
	.card { background: #161b22; border: 1px solid #30363d; border-radius: 8px; padding: 16px; text-align: center; }
	.card-value { font-size: 26px; font-weight: 700; color: #e6edf3; }
	.card-label { font-size: 12px; color: #8b949e; margin-top: 4px; }
	.chart-row { display: grid; grid-template-columns: 1fr 1fr; gap: 16px; margin-bottom: 24px; }
	.chart-full { margin-bottom: 24px; }
	.chart-box { background: #161b22; border: 1px solid #30363d; border-radius: 8px; padding: 12px; }
	.controls { display: flex; align-items: center; gap: 8px; margin-bottom: 12px; flex-wrap: wrap; }
	.controls label { font-size: 12px; color: #8b949e; font-weight: 500; }
	.controls select, .controls button { background: #21262d; border: 1px solid #30363d; color: #c9d1d9; border-radius: 6px; padding: 4px 10px; font-size: 12px; cursor: pointer; }
	.controls select:hover, .controls button:hover { border-color: #8b949e; }
	.controls button.active { background: #2c7bb6; border-color: #2c7bb6; color: #fff; }
	.section-title { font-size: 15px; font-weight: 600; color: #e6edf3; margin-bottom: 12px; }
	@media (max-width: 900px) { .chart-row { grid-template-columns: 1fr; } }
	</style>
	</head>
	"""


	def _html_body() -> str:
	tabs = [
	("overview", "Overview"),
	("topics", "Topics"),
	("formats", "Formats"),
	("heatmap", "Topic \u00d7 Format"),
	("quality", "Quality"),
	("tokens", "Tokens"),
	("timeline", "Timeline"),
	]
	nav = '<nav class="tab-nav">\n'
	for tid, label in tabs:
	cls = ' class="tab-btn active"' if tid == "overview" else ' class="tab-btn"'
	nav += f' <button{cls} data-tab="{tid}">{label}</button>\n'
	nav += "</nav>\n<main>\n"

	sections = []
	for tid, _ in tabs:
	cls = (
	' class="tab-content active"'
	if tid == "overview"
	else ' class="tab-content"'
	)
	sections.append(f'<section id="tab-{tid}"{cls}></section>')

	return nav + "\n".join(sections) + "\n</main>\n"


	def _js_data(summary: str, csv_data: dict[str, str]) -> str:
	lines = [f"const SUMMARY = {summary};"]
	for key, csv_text in csv_data.items():
	lines.append(f"const {key}_CSV = `\n{csv_text}\n`;")
	return "\n".join(lines) + "\n"


	def _js_utilities() -> str:
	return r"""
	function parseCSV(text) {
	const lines = text.trim().split('\n');
	const headers = lines[0].split(',');
	return lines.slice(1).map(line => {
	const vals = line.split(',');
	const obj = {};
	headers.forEach((h, i) => {
	obj[h.trim()] = parseValue(vals[i]);
	});
	return obj;
	});
	}

	function parseValue(value) {
	const text = String(value ?? '').trim();
	if (text === '') return null;
	const num = Number(text);
	return Number.isFinite(num) ? num : text;
	}

	function isFiniteNumber(value) {
	return typeof value === 'number' && Number.isFinite(value);
	}

	function fmtCount(n) {
	if (!isFiniteNumber(n)) return 'N/A';
	if (n >= 1e12) return (n / 1e12).toFixed(2) + 'T';
	if (n >= 1e9) return (n / 1e9).toFixed(2) + 'B';
	if (n >= 1e6) return (n / 1e6).toFixed(1) + 'M';
	if (n >= 1e3) return (n / 1e3).toFixed(1) + 'K';
	return String(n);
	}

	function fmtFull(n) {
	if (!isFiniteNumber(n)) return 'N/A';
	return Number(n).toLocaleString('en-US');
	}

	function fmtMaybeNumber(n, digits) {
	return isFiniteNumber(n) ? n.toFixed(digits) : 'N/A';
	}

	function fmtMaybePercent(n, digits) {
	return isFiniteNumber(n) ? n.toFixed(digits) + '%' : 'N/A';
	}

	function roundMaybe(n) {
	return isFiniteNumber(n) ? Math.round(n) : null;
	}

	function sortByMetricDescending(rows, metric) {
	return [...rows].sort((left, right) => {
	const leftValue = left[metric];
	const rightValue = right[metric];
	const leftIsNumber = isFiniteNumber(leftValue);
	const rightIsNumber = isFiniteNumber(rightValue);
	if (leftIsNumber && rightIsNumber) return rightValue - leftValue;
	if (leftIsNumber) return -1;
	if (rightIsNumber) return 1;
	return 0;
	});
	}

	function colorAboveBaseline(value, baseline) {
	return isFiniteNumber(value) && isFiniteNumber(baseline) && value > baseline ? '#2c7bb6' : '#8b949e';
	}

	function qualityReferenceLine(value, y1) {
	if (!isFiniteNumber(value)) return [];
	return [{
	type: 'line',
	x0: value,
	x1: value,
	y0: -0.5,
	y1: y1,
	line: {color: '#d7191c', width: 1, dash: 'dash'},
	}];
	}

	const DISPLAY_OVERRIDES = {
	'faq': 'FAQ', 'q_a_forum': 'Q&A Forum',
	'about_org': 'About Org', 'about_pers': 'About Person',
	'news_org': 'News Org', 'spam_ads': 'Spam/Ads',
	'common_crawl': 'Common Crawl', 'olmocr_science_pdfs': 'OLMoCR Science PDFs',
	};

	function displayLabel(label) {
	if (!label) return '';
	const raw = label.replace(/__label__/g, '');
	if (DISPLAY_OVERRIDES[raw]) return DISPLAY_OVERRIDES[raw];
	return raw.replace(/_/g, ' ').replace(/\b\w/g, c => c.toUpperCase());
	}

	const DARK = {
	paper_bgcolor: 'rgba(0,0,0,0)',
	plot_bgcolor: '#161b22',
	font: {color: '#c9d1d9', family: 'system-ui, -apple-system, sans-serif', size: 12},
	};

	function darkAxis() {
	return {gridcolor: '#30363d', zerolinecolor: '#30363d', linecolor: '#30363d'};
	}

	const PLOTLY_CFG = {responsive: true, displayModeBar: false};

	const topics = parseCSV(TOPIC_TOTALS_CSV);
	const formats = parseCSV(FORMAT_TOTALS_CSV);
	const topicFormat = parseCSV(TOPIC_FORMAT_TOTALS_CSV);
	const years = parseCSV(YEAR_TOTALS_CSV);
	const sources = parseCSV(SOURCE_FAMILY_TOTALS_CSV);
	const qualityLabels = parseCSV(QUALITY_LABEL_TOTALS_CSV);
	const qualityHist = parseCSV(QUALITY_SCORE_HISTOGRAM_CSV);
	const tokenHist = parseCSV(TOKEN_COUNT_HISTOGRAM_CSV);
	"""


	def _js_charts() -> str:
	return r"""
	// ─── Overview ───
	function renderOverview() {
	const el = document.getElementById('tab-overview');
	el.innerHTML = `
	<div class="cards">
	<div class="card"><div class="card-value">${fmtCount(SUMMARY.total_docs)}</div><div class="card-label">Total Documents</div></div>
	<div class="card"><div class="card-value">${fmtCount(SUMMARY.total_tokens)}</div><div class="card-label">Total Tokens</div></div>
	<div class="card"><div class="card-value">${fmtMaybeNumber(SUMMARY.mean_quality_score, 3)}</div><div class="card-label">Mean Quality Score</div></div>
	<div class="card"><div class="card-value">${fmtFull(SUMMARY.approx_median_token_count)}</div><div class="card-label">Median Tokens/Doc</div></div>
	<div class="card"><div class="card-value">${fmtFull(roundMaybe(SUMMARY.mean_token_count))}</div><div class="card-label">Mean Tokens/Doc</div></div>
	<div class="card"><div class="card-value">${fmtMaybePercent(SUMMARY.quality_doc_percent, 2)}</div><div class="card-label">Quality Coverage</div></div>
	</div>
	<div class="chart-row">
	<div class="chart-box"><div id="chart-source-donut"></div></div>
	<div class="chart-box"><div id="chart-quality-donut"></div></div>
	</div>
	<div class="chart-full"><div class="chart-box"><div id="chart-top-topics"></div></div></div>
	`;

	// Source family donut
	Plotly.newPlot('chart-source-donut', [{
	type: 'pie', hole: 0.45,
	labels: sources.map(s => displayLabel(s.source_family)),
	values: sources.map(s => s.doc_count),
	textinfo: 'label+percent', textposition: 'inside',
	marker: {colors: ['#2c7bb6', '#d7191c']},
	hovertemplate: '%{label}<br>%{value:,.0f} docs<br>%{percent}<extra></extra>',
	}], {...DARK, title: {text: 'Source Family (by docs)', font: {color: '#e6edf3', size: 14}}, showlegend: false, height: 300, margin: {l:20,r:20,t:50,b:20}}, PLOTLY_CFG);

	// Quality label donut
	Plotly.newPlot('chart-quality-donut', [{
	type: 'pie', hole: 0.45,
	labels: qualityLabels.map(q => q.quality_label ? q.quality_label.charAt(0).toUpperCase() + q.quality_label.slice(1) + ' Quality' : 'Label ' + q.quality_label_id),
	values: qualityLabels.map(q => q.doc_count),
	textinfo: 'label+percent', textposition: 'inside',
	marker: {colors: ['#d7191c', '#2c7bb6']},
	hovertemplate: '%{label}<br>%{value:,.0f} docs<br>%{percent}<extra></extra>',
	}], {...DARK, title: {text: 'Quality Label (by docs)', font: {color: '#e6edf3', size: 14}}, showlegend: false, height: 300, margin: {l:20,r:20,t:50,b:20}}, PLOTLY_CFG);

	// Top 10 topics by doc count
	const sorted = [...topics].sort((a, b) => b.doc_count - a.doc_count).slice(0, 10).reverse();
	Plotly.newPlot('chart-top-topics', [{
	type: 'bar', orientation: 'h',
	y: sorted.map(t => displayLabel(t.weborganizer_topic)),
	x: sorted.map(t => t.doc_count),
	marker: {color: '#2c7bb6'},
	hovertemplate: '%{y}<br>%{x:,.0f} documents<extra></extra>',
	}], {...DARK, title: {text: 'Top 10 Topics by Document Count', font: {color: '#e6edf3', size: 14}}, xaxis: {...darkAxis(), title: 'Document Count'}, yaxis: {...darkAxis()}, height: 400, margin: {l:200,r:20,t:50,b:50}}, PLOTLY_CFG);
	}

	// ─── Topics ───
	function renderTopics(metric) {
	metric = metric \|\| 'doc_count';
	const el = document.getElementById('tab-topics');
	if (!document.getElementById('chart-topics')) {
	el.innerHTML = `
	<div class="controls">
	<label>Metric:</label>
	<select id="topics-metric" onchange="renderTopics(this.value)">
	<option value="doc_count">Document Count</option>
	<option value="token_count">Token Count</option>
	<option value="doc_percent">Document %</option>
	<option value="token_percent">Token %</option>
	<option value="mean_quality_score">Mean Quality Score</option>
	<option value="mean_token_count">Mean Tokens/Doc</option>
	<option value="mean_quality_confidence">Mean Quality Confidence</option>
	</select>
	</div>
	<div class="chart-box"><div id="chart-topics"></div></div>
	`;
	}
	document.getElementById('topics-metric').value = metric;
	const sorted = sortByMetricDescending(topics, metric).reverse();
	const isPercent = metric.includes('percent');
	const isScore = metric.includes('quality') \|\| metric.includes('confidence');
	Plotly.newPlot('chart-topics', [{
	type: 'bar', orientation: 'h',
	y: sorted.map(t => displayLabel(t.weborganizer_topic)),
	x: sorted.map(t => t[metric]),
	marker: {color: '#2c7bb6'},
	hovertemplate: sorted.map(t =>
	displayLabel(t.weborganizer_topic) +
	'<br>Docs: ' + fmtFull(t.doc_count) +
	'<br>Tokens: ' + fmtFull(t.token_count) +
	'<br>Doc%: ' + fmtMaybePercent(t.doc_percent, 2) +
	'<br>Mean Quality: ' + fmtMaybeNumber(t.mean_quality_score, 4) +
	'<br>Mean Tokens/Doc: ' + fmtFull(roundMaybe(t.mean_token_count)) +
	'<extra></extra>'
	),
	}], {
	...DARK,
	title: {text: 'Topics by ' + document.getElementById('topics-metric').selectedOptions[0].text, font: {color: '#e6edf3', size: 14}},
	xaxis: {...darkAxis(), title: isPercent ? '%' : isScore ? 'Score' : 'Count', tickformat: isPercent ? '.1f' : isScore ? '.3f' : ','},
	yaxis: {...darkAxis()},
	height: 700, margin: {l:220,r:20,t:50,b:50},
	}, PLOTLY_CFG);
	}

	// ─── Formats ───
	function renderFormats(metric) {
	metric = metric \|\| 'doc_count';
	const el = document.getElementById('tab-formats');
	if (!document.getElementById('chart-formats')) {
	el.innerHTML = `
	<div class="controls">
	<label>Metric:</label>
	<select id="formats-metric" onchange="renderFormats(this.value)">
	<option value="doc_count">Document Count</option>
	<option value="token_count">Token Count</option>
	<option value="doc_percent">Document %</option>
	<option value="token_percent">Token %</option>
	<option value="mean_quality_score">Mean Quality Score</option>
	<option value="mean_token_count">Mean Tokens/Doc</option>
	<option value="mean_quality_confidence">Mean Quality Confidence</option>
	</select>
	</div>
	<div class="chart-box"><div id="chart-formats"></div></div>
	`;
	}
	document.getElementById('formats-metric').value = metric;
	const sorted = sortByMetricDescending(formats, metric).reverse();
	const isPercent = metric.includes('percent');
	const isScore = metric.includes('quality') \|\| metric.includes('confidence');
	Plotly.newPlot('chart-formats', [{
	type: 'bar', orientation: 'h',
	y: sorted.map(f => displayLabel(f.weborganizer_format)),
	x: sorted.map(f => f[metric]),
	marker: {color: '#2c7bb6'},
	hovertemplate: sorted.map(f =>
	displayLabel(f.weborganizer_format) +
	'<br>Docs: ' + fmtFull(f.doc_count) +
	'<br>Tokens: ' + fmtFull(f.token_count) +
	'<br>Doc%: ' + fmtMaybePercent(f.doc_percent, 2) +
	'<br>Mean Quality: ' + fmtMaybeNumber(f.mean_quality_score, 4) +
	'<br>Mean Tokens/Doc: ' + fmtFull(roundMaybe(f.mean_token_count)) +
	'<extra></extra>'
	),
	}], {
	...DARK,
	title: {text: 'Formats by ' + document.getElementById('formats-metric').selectedOptions[0].text, font: {color: '#e6edf3', size: 14}},
	xaxis: {...darkAxis(), title: isPercent ? '%' : isScore ? 'Score' : 'Count', tickformat: isPercent ? '.1f' : isScore ? '.3f' : ','},
	yaxis: {...darkAxis()},
	height: 700, margin: {l:200,r:20,t:50,b:50},
	}, PLOTLY_CFG);
	}

	// ─── Topic x Format Heatmap ───
	function renderHeatmap(metric) {
	metric = metric \|\| 'doc_count';
	const el = document.getElementById('tab-heatmap');
	if (!document.getElementById('chart-heatmap')) {
	el.innerHTML = `
	<div class="controls">
	<label>Metric:</label>
	<select id="heatmap-metric" onchange="renderHeatmap(this.value)">
	<option value="doc_count">Document Count (log)</option>
	<option value="token_count">Token Count (log)</option>
	<option value="doc_percent">Document %</option>
	<option value="mean_quality_score">Mean Quality Score</option>
	</select>
	</div>
	<div class="chart-box"><div id="chart-heatmap"></div></div>
	`;
	}
	document.getElementById('heatmap-metric').value = metric;

	// Sort topics and formats by their marginal doc_count descending
	const topicOrder = [...topics].sort((a, b) => b.doc_count - a.doc_count).map(t => t.weborganizer_topic);
	const formatOrder = [...formats].sort((a, b) => b.doc_count - a.doc_count).map(f => f.weborganizer_format);

	// Build 2D matrix
	const lookup = {};
	topicFormat.forEach(row => {
	const key = row.weborganizer_topic + '\|' + row.weborganizer_format;
	lookup[key] = row;
	});

	const useLog = metric === 'doc_count' \|\| metric === 'token_count';
	const z = topicOrder.map(topic =>
	formatOrder.map(format => {
	const row = lookup[topic + '\|' + format];
	if (!row) return null;
	const val = row[metric];
	return useLog ? (val > 0 ? Math.log10(val) : null) : val;
	})
	);

	const hoverText = topicOrder.map(topic =>
	formatOrder.map(format => {
	const row = lookup[topic + '\|' + format];
	if (!row) return '';
	return displayLabel(topic) + ' \u00d7 ' + displayLabel(format) +
	'<br>Docs: ' + fmtFull(row.doc_count) +
	'<br>Tokens: ' + fmtFull(row.token_count) +
	'<br>Doc%: ' + fmtMaybePercent(row.doc_percent, 4) +
	'<br>Quality: ' + fmtMaybeNumber(row.mean_quality_score, 4);
	})
	);

	Plotly.newPlot('chart-heatmap', [{
	type: 'heatmap',
	z: z,
	x: formatOrder.map(displayLabel),
	y: topicOrder.map(displayLabel),
	colorscale: 'Viridis',
	hoverinfo: 'text',
	text: hoverText,
	colorbar: {
	title: {text: useLog ? 'log\u2081\u2080(' + metric.replace('_', ' ') + ')' : metric.replace(/_/g, ' '), font: {color: '#c9d1d9'}},
	tickfont: {color: '#c9d1d9'},
	},
	zmin: useLog ? null : 0,
	}], {
	...DARK,
	title: {text: 'Topic \u00d7 Format: ' + document.getElementById('heatmap-metric').selectedOptions[0].text, font: {color: '#e6edf3', size: 14}},
	xaxis: {...darkAxis(), tickangle: 45, tickfont: {size: 10}},
	yaxis: {...darkAxis(), tickfont: {size: 10}, autorange: 'reversed'},
	height: 900, margin: {l:200,r:80,t:50,b:180},
	}, PLOTLY_CFG);
	}

	// ─── Quality ───
	function renderQuality() {
	const el = document.getElementById('tab-quality');
	if (!document.getElementById('chart-quality-hist')) {
	el.innerHTML = `
	<div class="controls">
	<button id="quality-log-btn" onclick="toggleQualityLog()">Toggle Log Scale</button>
	</div>
	<div class="chart-full"><div class="chart-box"><div id="chart-quality-hist"></div></div></div>
	<div class="chart-row">
	<div class="chart-box"><div id="chart-quality-by-topic"></div></div>
	<div class="chart-box"><div id="chart-quality-by-format"></div></div>
	</div>
	`;
	}

	const binLabels = qualityHist.map(b => b.score_bin_start.toFixed(2) + '-' + b.score_bin_end.toFixed(2));
	Plotly.newPlot('chart-quality-hist', [{
	type: 'bar',
	x: binLabels,
	y: qualityHist.map(b => b.doc_count),
	marker: {color: '#2c7bb6'},
	hovertemplate: 'Score range: %{x}<br>Documents: %{y:,.0f}<extra></extra>',
	}], {
	...DARK,
	title: {text: 'Quality Score Distribution', font: {color: '#e6edf3', size: 14}},
	xaxis: {...darkAxis(), title: 'Quality Score Bin', tickangle: 45},
	yaxis: {...darkAxis(), title: 'Document Count', type: 'log'},
	height: 400, margin: {l:80,r:20,t:50,b:80},
	}, PLOTLY_CFG);

	// Quality by topic
	const topicSorted = sortByMetricDescending(topics, 'mean_quality_score').reverse();
	Plotly.newPlot('chart-quality-by-topic', [{
	type: 'bar', orientation: 'h',
	y: topicSorted.map(t => displayLabel(t.weborganizer_topic)),
	x: topicSorted.map(t => t.mean_quality_score),
	marker: {color: topicSorted.map(t => colorAboveBaseline(t.mean_quality_score, SUMMARY.mean_quality_score))},
	hovertemplate: '%{y}<br>Mean Quality: %{x:.4f}<extra></extra>',
	}], {
	...DARK,
	title: {text: 'Mean Quality Score by Topic', font: {color: '#e6edf3', size: 14}},
	xaxis: {...darkAxis(), title: 'Mean Quality Score'},
	yaxis: {...darkAxis()},
	height: 700, margin: {l:220,r:20,t:50,b:50},
	shapes: qualityReferenceLine(SUMMARY.mean_quality_score, 23.5),
	}, PLOTLY_CFG);

	// Quality by format
	const formatSorted = sortByMetricDescending(formats, 'mean_quality_score').reverse();
	Plotly.newPlot('chart-quality-by-format', [{
	type: 'bar', orientation: 'h',
	y: formatSorted.map(f => displayLabel(f.weborganizer_format)),
	x: formatSorted.map(f => f.mean_quality_score),
	marker: {color: formatSorted.map(f => colorAboveBaseline(f.mean_quality_score, SUMMARY.mean_quality_score))},
	hovertemplate: '%{y}<br>Mean Quality: %{x:.4f}<extra></extra>',
	}], {
	...DARK,
	title: {text: 'Mean Quality Score by Format', font: {color: '#e6edf3', size: 14}},
	xaxis: {...darkAxis(), title: 'Mean Quality Score'},
	yaxis: {...darkAxis()},
	height: 700, margin: {l:200,r:20,t:50,b:50},
	shapes: qualityReferenceLine(SUMMARY.mean_quality_score, 23.5),
	}, PLOTLY_CFG);
	}

	let qualityLogScale = true;
	function toggleQualityLog() {
	qualityLogScale = !qualityLogScale;
	const btn = document.getElementById('quality-log-btn');
	btn.classList.toggle('active', qualityLogScale);
	Plotly.relayout('chart-quality-hist', {'yaxis.type': qualityLogScale ? 'log' : 'linear'});
	}

	// ─── Tokens ───
	function renderTokens() {
	const el = document.getElementById('tab-tokens');
	if (!document.getElementById('chart-token-hist')) {
	el.innerHTML = `
	<div class="chart-full"><div class="chart-box"><div id="chart-token-hist"></div></div></div>
	<div class="chart-row">
	<div class="chart-box"><div id="chart-tokens-by-topic"></div></div>
	<div class="chart-box"><div id="chart-tokens-by-format"></div></div>
	</div>
	`;
	}

	const binLabels = tokenHist.map(b => {
	const s = b.token_bin_start;
	const e = b.token_bin_end;
	if (e <= 1) return '0-1';
	return fmtCount(s) + '-' + fmtCount(e);
	});

	Plotly.newPlot('chart-token-hist', [{
	type: 'bar',
	x: binLabels,
	y: tokenHist.map(b => b.doc_count),
	marker: {color: '#2c7bb6'},
	hovertemplate: 'Tokens: %{x}<br>Documents: %{y:,.0f}<extra></extra>',
	}], {
	...DARK,
	title: {text: 'Token Count Distribution (log\u2082 bins)', font: {color: '#e6edf3', size: 14}},
	xaxis: {...darkAxis(), title: 'Tokens per Document', tickangle: 45, tickfont: {size: 10}},
	yaxis: {...darkAxis(), title: 'Document Count', type: 'log'},
	height: 400, margin: {l:80,r:20,t:50,b:100},
	}, PLOTLY_CFG);

	// Mean tokens by topic
	const topicSorted = sortByMetricDescending(topics, 'mean_token_count').reverse();
	Plotly.newPlot('chart-tokens-by-topic', [{
	type: 'bar', orientation: 'h',
	y: topicSorted.map(t => displayLabel(t.weborganizer_topic)),
	x: topicSorted.map(t => t.mean_token_count),
	marker: {color: '#2c7bb6'},
	hovertemplate: '%{y}<br>Mean Tokens/Doc: %{x:,.0f}<extra></extra>',
	}], {
	...DARK,
	title: {text: 'Mean Tokens/Doc by Topic', font: {color: '#e6edf3', size: 14}},
	xaxis: {...darkAxis(), title: 'Mean Token Count'},
	yaxis: {...darkAxis()},
	height: 700, margin: {l:220,r:20,t:50,b:50},
	}, PLOTLY_CFG);

	// Mean tokens by format
	const formatSorted = sortByMetricDescending(formats, 'mean_token_count').reverse();
	Plotly.newPlot('chart-tokens-by-format', [{
	type: 'bar', orientation: 'h',
	y: formatSorted.map(f => displayLabel(f.weborganizer_format)),
	x: formatSorted.map(f => f.mean_token_count),
	marker: {color: '#2c7bb6'},
	hovertemplate: '%{y}<br>Mean Tokens/Doc: %{x:,.0f}<extra></extra>',
	}], {
	...DARK,
	title: {text: 'Mean Tokens/Doc by Format', font: {color: '#e6edf3', size: 14}},
	xaxis: {...darkAxis(), title: 'Mean Token Count'},
	yaxis: {...darkAxis()},
	height: 700, margin: {l:200,r:20,t:50,b:50},
	}, PLOTLY_CFG);
	}

	// ─── Timeline ───
	function renderTimeline() {
	const el = document.getElementById('tab-timeline');
	if (!document.getElementById('chart-year-docs')) {
	el.innerHTML = `
	<div class="chart-row">
	<div class="chart-box"><div id="chart-year-docs"></div></div>
	<div class="chart-box"><div id="chart-year-tokens"></div></div>
	</div>
	<div class="chart-full"><div class="chart-box"><div id="chart-year-quality"></div></div></div>
	`;
	}

	const sorted = [...years].sort((a, b) => a.year - b.year);
	const yrs = sorted.map(y => String(y.year));

	Plotly.newPlot('chart-year-docs', [{
	type: 'bar',
	x: yrs, y: sorted.map(y => y.doc_count),
	marker: {color: '#2c7bb6'},
	hovertemplate: '%{x}<br>%{y:,.0f} documents<extra></extra>',
	}], {
	...DARK,
	title: {text: 'Documents by Year', font: {color: '#e6edf3', size: 14}},
	xaxis: {...darkAxis(), title: 'Year'}, yaxis: {...darkAxis(), title: 'Document Count'},
	height: 350, margin: {l:80,r:20,t:50,b:50},
	}, PLOTLY_CFG);

	Plotly.newPlot('chart-year-tokens', [{
	type: 'bar',
	x: yrs, y: sorted.map(y => y.token_count),
	marker: {color: '#d7191c'},
	hovertemplate: '%{x}<br>%{y:,.0f} tokens<extra></extra>',
	}], {
	...DARK,
	title: {text: 'Tokens by Year', font: {color: '#e6edf3', size: 14}},
	xaxis: {...darkAxis(), title: 'Year'}, yaxis: {...darkAxis(), title: 'Token Count'},
	height: 350, margin: {l:80,r:20,t:50,b:50},
	}, PLOTLY_CFG);

	Plotly.newPlot('chart-year-quality', [{
	type: 'scatter', mode: 'lines+markers',
	x: yrs, y: sorted.map(y => y.mean_quality_score),
	marker: {color: '#2c7bb6', size: 8},
	line: {color: '#2c7bb6', width: 2},
	hovertemplate: '%{x}<br>Mean Quality: %{y:.4f}<extra></extra>',
	}], {
	...DARK,
	title: {text: 'Mean Quality Score by Year', font: {color: '#e6edf3', size: 14}},
	xaxis: {...darkAxis(), title: 'Year'}, yaxis: {...darkAxis(), title: 'Mean Quality Score'},
	height: 350, margin: {l:80,r:20,t:50,b:50},
	}, PLOTLY_CFG);
	}
	"""


	def _js_init() -> str:
	return r"""
	// ─── Tab Management ───
	const tabRendered = {overview: false, topics: false, formats: false, heatmap: false, quality: false, tokens: false, timeline: false};

	function switchTab(tab) {
	document.querySelectorAll('.tab-btn').forEach(b => b.classList.toggle('active', b.dataset.tab === tab));
	document.querySelectorAll('.tab-content').forEach(s => s.classList.toggle('active', s.id === 'tab-' + tab));

	if (!tabRendered[tab]) {
	tabRendered[tab] = true;
	switch(tab) {
	case 'overview': renderOverview(); break;
	case 'topics': renderTopics(); break;
	case 'formats': renderFormats(); break;
	case 'heatmap': renderHeatmap(); break;
	case 'quality': renderQuality(); break;
	case 'tokens': renderTokens(); break;
	case 'timeline': renderTimeline(); break;
	}
	}
	// Resize visible plotly charts
	setTimeout(() => {
	document.querySelectorAll('#tab-' + tab + ' .js-plotly-plot').forEach(p => Plotly.Plots.resize(p));
	}, 50);
	}

	document.querySelectorAll('.tab-btn').forEach(btn => {
	btn.addEventListener('click', () => switchTab(btn.dataset.tab));
	});

	// Render overview on load
	document.addEventListener('DOMContentLoaded', () => {
	tabRendered.overview = true;
	renderOverview();
	});
	"""


	def main() -> None:
	parser = argparse.ArgumentParser(description="Generate SOC-95 Corpus Explorer HTML")
	parser.add_argument("--data-dir", type=Path, default=DEFAULT_DATA_DIR)
	parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
	args = parser.parse_args()

	for name in ["manifest_analysis_summary.json"] + DATA_FILES:
	path = args.data_dir / name
	if not path.exists():
	logger.error("Missing data file: %s", path)
	raise SystemExit(1)

	html = build_html(args.data_dir)
	args.output.parent.mkdir(parents=True, exist_ok=True)
	args.output.write_text(html)
	size_kb = args.output.stat().st_size / 1024
	print(f"Wrote {args.output} ({size_kb:.0f} KB)")


	if __name__ == "__main__":
	main()

Xet Storage Details

Size:: 28.5 kB
Xet hash:: 72f73781d843d994080914eafa2e305835b4baad6e6f5ab3417920c347a6bf82

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.