Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
Commit ·
3742813
1
Parent(s): d81db79
switched from accordion to newly created tab widget
Browse files- app/src/components/Tab.astro +6 -0
- app/src/components/Tabs.astro +132 -0
- app/src/content/chapters/2-setup.mdx +20 -17
app/src/components/Tab.astro
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
const { title } = Astro.props;
|
| 3 |
+
---
|
| 4 |
+
<div class="tab-panel" data-tab-title={title} role="tabpanel" hidden>
|
| 5 |
+
<slot />
|
| 6 |
+
</div>
|
app/src/components/Tabs.astro
ADDED
|
@@ -0,0 +1,132 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
const { class: className, ...props } = Astro.props;
|
| 3 |
+
const wrapperClass = ["tabs", className].filter(Boolean).join(" ");
|
| 4 |
+
---
|
| 5 |
+
<div class={wrapperClass} {...props}>
|
| 6 |
+
<div class="tabs__nav" role="tablist"></div>
|
| 7 |
+
<div class="tabs__panels">
|
| 8 |
+
<slot />
|
| 9 |
+
</div>
|
| 10 |
+
</div>
|
| 11 |
+
|
| 12 |
+
<script>
|
| 13 |
+
document.addEventListener('DOMContentLoaded', () => {
|
| 14 |
+
document.querySelectorAll('.tabs').forEach((tabsEl) => {
|
| 15 |
+
const nav = tabsEl.querySelector('.tabs__nav') as HTMLElement;
|
| 16 |
+
const panels = Array.from(tabsEl.querySelectorAll(':scope > .tabs__panels > .tab-panel')) as HTMLElement[];
|
| 17 |
+
if (!nav || panels.length === 0) return;
|
| 18 |
+
|
| 19 |
+
panels.forEach((panel, i) => {
|
| 20 |
+
const title = panel.dataset.tabTitle || `Tab ${i + 1}`;
|
| 21 |
+
const btn = document.createElement('button');
|
| 22 |
+
btn.className = 'tabs__btn';
|
| 23 |
+
btn.role = 'tab';
|
| 24 |
+
btn.textContent = title;
|
| 25 |
+
btn.setAttribute('aria-selected', 'false');
|
| 26 |
+
btn.addEventListener('click', () => activate(i));
|
| 27 |
+
nav.appendChild(btn);
|
| 28 |
+
});
|
| 29 |
+
|
| 30 |
+
const buttons = Array.from(nav.querySelectorAll('.tabs__btn')) as HTMLButtonElement[];
|
| 31 |
+
|
| 32 |
+
function activate(index: number) {
|
| 33 |
+
buttons.forEach((b, i) => {
|
| 34 |
+
const active = i === index;
|
| 35 |
+
b.classList.toggle('tabs__btn--active', active);
|
| 36 |
+
b.setAttribute('aria-selected', String(active));
|
| 37 |
+
});
|
| 38 |
+
panels.forEach((p, i) => {
|
| 39 |
+
p.hidden = i !== index;
|
| 40 |
+
});
|
| 41 |
+
}
|
| 42 |
+
|
| 43 |
+
// Keyboard navigation
|
| 44 |
+
nav.addEventListener('keydown', (e) => {
|
| 45 |
+
const current = buttons.findIndex(b => b === document.activeElement);
|
| 46 |
+
if (current < 0) return;
|
| 47 |
+
let next = current;
|
| 48 |
+
if (e.key === 'ArrowRight') next = (current + 1) % buttons.length;
|
| 49 |
+
else if (e.key === 'ArrowLeft') next = (current - 1 + buttons.length) % buttons.length;
|
| 50 |
+
else if (e.key === 'Home') next = 0;
|
| 51 |
+
else if (e.key === 'End') next = buttons.length - 1;
|
| 52 |
+
else return;
|
| 53 |
+
e.preventDefault();
|
| 54 |
+
buttons[next].focus();
|
| 55 |
+
activate(next);
|
| 56 |
+
});
|
| 57 |
+
|
| 58 |
+
activate(0);
|
| 59 |
+
});
|
| 60 |
+
});
|
| 61 |
+
</script>
|
| 62 |
+
|
| 63 |
+
<style>
|
| 64 |
+
.tabs {
|
| 65 |
+
margin: 0 0 var(--spacing-4);
|
| 66 |
+
border: 1px solid var(--border-color);
|
| 67 |
+
border-radius: var(--table-border-radius);
|
| 68 |
+
background: var(--surface-bg);
|
| 69 |
+
}
|
| 70 |
+
|
| 71 |
+
.tabs__nav {
|
| 72 |
+
display: flex;
|
| 73 |
+
flex-wrap: wrap;
|
| 74 |
+
gap: 0;
|
| 75 |
+
border-bottom: 1px solid var(--border-color);
|
| 76 |
+
overflow-x: auto;
|
| 77 |
+
-webkit-overflow-scrolling: touch;
|
| 78 |
+
scrollbar-width: none;
|
| 79 |
+
}
|
| 80 |
+
|
| 81 |
+
.tabs__nav::-webkit-scrollbar {
|
| 82 |
+
display: none;
|
| 83 |
+
}
|
| 84 |
+
|
| 85 |
+
.tabs__nav :global(.tabs__btn) {
|
| 86 |
+
flex: 1 1 0;
|
| 87 |
+
padding: var(--spacing-2) var(--spacing-3);
|
| 88 |
+
text-align: center;
|
| 89 |
+
border: none !important;
|
| 90 |
+
border-bottom: 2px solid transparent !important;
|
| 91 |
+
border-radius: 0 !important;
|
| 92 |
+
background: none !important;
|
| 93 |
+
font: inherit;
|
| 94 |
+
font-size: 0.9em;
|
| 95 |
+
font-weight: 600;
|
| 96 |
+
color: var(--text-muted) !important;
|
| 97 |
+
cursor: pointer;
|
| 98 |
+
white-space: nowrap;
|
| 99 |
+
margin-bottom: -1px;
|
| 100 |
+
transition: color 150ms ease, border-color 150ms ease;
|
| 101 |
+
filter: none !important;
|
| 102 |
+
}
|
| 103 |
+
|
| 104 |
+
.tabs__nav :global(.tabs__btn:hover) {
|
| 105 |
+
color: var(--text-color) !important;
|
| 106 |
+
filter: none !important;
|
| 107 |
+
}
|
| 108 |
+
|
| 109 |
+
.tabs__nav :global(.tabs__btn--active) {
|
| 110 |
+
color: var(--primary-color) !important;
|
| 111 |
+
border-bottom-color: var(--primary-color) !important;
|
| 112 |
+
}
|
| 113 |
+
|
| 114 |
+
.tabs__nav :global(.tabs__btn:focus-visible) {
|
| 115 |
+
outline: 2px solid var(--primary-color);
|
| 116 |
+
outline-offset: -2px;
|
| 117 |
+
border-radius: 2px;
|
| 118 |
+
}
|
| 119 |
+
|
| 120 |
+
.tabs__panels :global(.tab-panel) {
|
| 121 |
+
padding: 8px;
|
| 122 |
+
}
|
| 123 |
+
|
| 124 |
+
.tabs__panels :global(.tab-panel > *:first-child) {
|
| 125 |
+
margin-top: 0 !important;
|
| 126 |
+
}
|
| 127 |
+
|
| 128 |
+
.tabs__panels :global(.tab-panel > *:last-child) {
|
| 129 |
+
margin-bottom: 0 !important;
|
| 130 |
+
padding-bottom: 0 !important;
|
| 131 |
+
}
|
| 132 |
+
</style>
|
app/src/content/chapters/2-setup.mdx
CHANGED
|
@@ -1,5 +1,6 @@
|
|
| 1 |
-
import Accordion from "../../components/Accordion.astro";
|
| 2 |
import Sidenote from "../../components/Sidenote.astro";
|
|
|
|
|
|
|
| 3 |
|
| 4 |
|
| 5 |
## Rephrasing the Web
|
|
@@ -30,30 +31,32 @@ For inference we use vLLM [@vllm] with tensor parallelism, chunked prefill, and
|
|
| 30 |
|
| 31 |
Before diving into experiments, here's a quick overview of the datasets we compare against. We use "source data" and "seed data" interchangeably throughout.
|
| 32 |
|
| 33 |
-
<
|
|
|
|
| 34 |
A standardized benchmark providing a 240T token corpus from Common Crawl with model-based filtering as a key curation strategy. DCLM (DataComp-LM) enables training a 7B parameter model to 64% accuracy on MMLU with 2.6T tokens [@datacomp].
|
| 35 |
-
</
|
| 36 |
-
<
|
| 37 |
Subsets of FineWeb-Edu, a 1.3T token educational dataset filtered using Llama-3-70B-Instruct [@llama3] scoring samples on educational quality from 0 to 5. We use HQ (scores 4 or 5) and LQ (scores 0 or 1) to investigate the impact of seed data quality on rephrasing [@fineweb].
|
| 38 |
-
</
|
| 39 |
-
<
|
| 40 |
A 1T English token and 120B Chinese token dataset created by applying efficient verification-based filtering to FineWeb. Uses a lightweight fastText classifier and optimized seed data selection to improve data quality [@ultrafineweb].
|
| 41 |
-
</
|
| 42 |
-
<
|
| 43 |
A 24T token web dataset from 101 Common Crawl snapshots with document-level metadata for flexible curation. Each of the 23.6B documents is annotated with subject classification, document type, content complexity, and quality scores using the [EAI-Taxonomy-0.5b](https://huggingface.co/EssentialAI/eai-taxonomy-0.5b) classifier, enabling researchers to filter domain-specific subsets without building custom pipelines [@essentialweb].
|
| 44 |
-
</
|
| 45 |
-
<
|
| 46 |
Part of Nemotron-CC, a 6.3T token dataset using classifier ensembling and synthetic data rephrasing. The High-Quality-Synthetic subset contains synthetically rephrased data using Qwen3-30B-A3B [@qwen3] [@nemotroncc].
|
| 47 |
-
</
|
| 48 |
-
<
|
| 49 |
A 30 million file synthetic dataset with 25 billion tokens generated by Mixtral-8x7B-Instruct [@mixtral], containing textbooks, blog posts, and stories across diverse topics. Created through careful prompt engineering conditioning on curated educational sources and web data clusters [@cosmopedia].
|
| 50 |
-
</
|
| 51 |
-
<
|
| 52 |
A fully synthetic dataset built from 50,000 Wikipedia articles expanded into problems and resolution paths including math exercises, creative writing, and information extraction. Uses multiple specialized synthetic pipelines with fine-tuned models and grounding in encyclopedic content [@synthpleias].
|
| 53 |
-
</
|
| 54 |
-
<
|
| 55 |
A method for recycling the web with guided rewriting that enriches low-quality documents discarded by filtering pipelines. Mixing high-quality raw texts with rewritten texts leads to 1.0, 1.3, and 2.5 percentage point improvements at 1B, 3B, and 7B scales across 22 tasks [@rewire].
|
| 56 |
-
</
|
|
|
|
| 57 |
|
| 58 |
With the datasets defined, we need a consistent way to tell whether one configuration is better than another.
|
| 59 |
|
|
|
|
|
|
|
| 1 |
import Sidenote from "../../components/Sidenote.astro";
|
| 2 |
+
import Tabs from "../../components/Tabs.astro";
|
| 3 |
+
import Tab from "../../components/Tab.astro";
|
| 4 |
|
| 5 |
|
| 6 |
## Rephrasing the Web
|
|
|
|
| 31 |
|
| 32 |
Before diving into experiments, here's a quick overview of the datasets we compare against. We use "source data" and "seed data" interchangeably throughout.
|
| 33 |
|
| 34 |
+
<Tabs>
|
| 35 |
+
<Tab title="DCLM">
|
| 36 |
A standardized benchmark providing a 240T token corpus from Common Crawl with model-based filtering as a key curation strategy. DCLM (DataComp-LM) enables training a 7B parameter model to 64% accuracy on MMLU with 2.6T tokens [@datacomp].
|
| 37 |
+
</Tab>
|
| 38 |
+
<Tab title="FineWeb-Edu-HQ / LQ">
|
| 39 |
Subsets of FineWeb-Edu, a 1.3T token educational dataset filtered using Llama-3-70B-Instruct [@llama3] scoring samples on educational quality from 0 to 5. We use HQ (scores 4 or 5) and LQ (scores 0 or 1) to investigate the impact of seed data quality on rephrasing [@fineweb].
|
| 40 |
+
</Tab>
|
| 41 |
+
<Tab title="Ultra-FineWeb">
|
| 42 |
A 1T English token and 120B Chinese token dataset created by applying efficient verification-based filtering to FineWeb. Uses a lightweight fastText classifier and optimized seed data selection to improve data quality [@ultrafineweb].
|
| 43 |
+
</Tab>
|
| 44 |
+
<Tab title="Essential-Web">
|
| 45 |
A 24T token web dataset from 101 Common Crawl snapshots with document-level metadata for flexible curation. Each of the 23.6B documents is annotated with subject classification, document type, content complexity, and quality scores using the [EAI-Taxonomy-0.5b](https://huggingface.co/EssentialAI/eai-taxonomy-0.5b) classifier, enabling researchers to filter domain-specific subsets without building custom pipelines [@essentialweb].
|
| 46 |
+
</Tab>
|
| 47 |
+
<Tab title="Nemotron-HQ-Synth">
|
| 48 |
Part of Nemotron-CC, a 6.3T token dataset using classifier ensembling and synthetic data rephrasing. The High-Quality-Synthetic subset contains synthetically rephrased data using Qwen3-30B-A3B [@qwen3] [@nemotroncc].
|
| 49 |
+
</Tab>
|
| 50 |
+
<Tab title="Cosmopedia">
|
| 51 |
A 30 million file synthetic dataset with 25 billion tokens generated by Mixtral-8x7B-Instruct [@mixtral], containing textbooks, blog posts, and stories across diverse topics. Created through careful prompt engineering conditioning on curated educational sources and web data clusters [@cosmopedia].
|
| 52 |
+
</Tab>
|
| 53 |
+
<Tab title="SYNTH">
|
| 54 |
A fully synthetic dataset built from 50,000 Wikipedia articles expanded into problems and resolution paths including math exercises, creative writing, and information extraction. Uses multiple specialized synthetic pipelines with fine-tuned models and grounding in encyclopedic content [@synthpleias].
|
| 55 |
+
</Tab>
|
| 56 |
+
<Tab title="REWIRE">
|
| 57 |
A method for recycling the web with guided rewriting that enriches low-quality documents discarded by filtering pipelines. Mixing high-quality raw texts with rewritten texts leads to 1.0, 1.3, and 2.5 percentage point improvements at 1B, 3B, and 7B scales across 22 tasks [@rewire].
|
| 58 |
+
</Tab>
|
| 59 |
+
</Tabs>
|
| 60 |
|
| 61 |
With the datasets defined, we need a consistent way to tell whether one configuration is better than another.
|
| 62 |
|