cannot appear as a descendant of
",
"cause": "Invalid HTML nesting - block element inside inline element",
"fix": "Change outer
to
or inner
to
",
"file": "src/components/Card.tsx",
},
{
"error": "Error: NEXT_REDIRECT in API route",
"cause": "Using redirect() in a try/catch block catches the redirect error",
"fix": "Move redirect() call outside of try/catch, or re-throw NEXT_REDIRECT",
"file": "src/app/api/auth/route.ts",
},
{
"error": "Unhandled Runtime Error: Maximum update depth exceeded",
"cause": "useEffect dependency causes infinite re-render loop",
"fix": "Memoize the dependency with useMemo or useCallback",
"file": "src/hooks/useData.ts",
},
]
for i in range(count):
scenario = error_scenarios[i % len(error_scenarios)]
assistant_content = (
f"<|think_start|>\n"
f"I see a build error. Let me analyze:\n"
f"Error: {scenario['error']}\n"
f"Root cause: {scenario['cause']}\n"
f"I need to fix this in {scenario['file']}\n"
f"<|think_end|>\n\n"
f"<|error_start|>\n"
f"File: {scenario['file']}\n"
f"Error: {scenario['error']}\n"
f"<|error_end|>\n\n"
f"<|sandbox_start|>\n"
f"Running: npm run build\n"
f"Status: FAILED\n"
f"Exit code: 1\n"
f"<|sandbox_end|>\n\n"
f"<|fix_start|>\n"
f"Root cause: {scenario['cause']}\n"
f"Solution: {scenario['fix']}\n"
f"<|fix_end|>\n\n"
f"<|file_start|>\npath: {scenario['file']}\nlanguage: typescript\n<|file_end|>\n\n"
f"<|code_start|>\n"
f"// Fixed version of {scenario['file']}\n"
f"// Applied fix: {scenario['fix']}\n"
f"export default function Fixed() {{\n"
f" // Corrected implementation\n"
f" return Fixed component
;\n"
f"}}\n"
f"<|code_end|>\n\n"
f"<|sandbox_start|>\n"
f"Running: npm run build\n"
f"Status: SUCCESS\n"
f"Exit code: 0\n"
f"<|sandbox_end|>"
)
yield {
"id": f"sandbox_{i:06d}",
"source": "sandbox_examples",
"user": f"I'm getting this error: {scenario['error']}",
"assistant": assistant_content,
"error_type": scenario["error"][:50],
}
progress.update(task, completed=i + 1)
def write_synthetic(
config: DatasetConfig,
checkpoint: CheckpointManager,
progress: Progress,
) -> int:
"""Generate and write synthetic data."""
output_path = DATA_RAW / config.output_file
generators = {
"synthetic_nextjs": generate_synthetic_nextjs,
"search_examples": generate_search_examples,
"sandbox_examples": generate_sandbox_examples,
}
gen_fn = generators[config.name]
count = 0
with open(output_path, "w", encoding="utf-8") as f:
for example in gen_fn(config.target_count, progress):
f.write(json.dumps(example, ensure_ascii=False) + "\n")
count += 1
size_mb = output_path.stat().st_size / (1024 * 1024)
log.info(f"✅ {config.name}: {count:,} examples, {size_mb:.1f} MB")
return count
# ── Disk space check ──────────────────────────────────────────────────
def check_disk_space(datasets: list[DatasetConfig]) -> bool:
"""Verify enough disk space for planned downloads."""
import shutil
total_est_gb = sum(d.est_size_gb for d in datasets)
usage = shutil.disk_usage(str(DATA_RAW))
free_gb = usage.free / (1024 ** 3)
table = Table(title="💾 Disk Space Estimate")
table.add_column("Item", style="cyan")
table.add_column("Size", justify="right", style="green")
for d in datasets:
table.add_row(d.name, f"{d.est_size_gb:.2f} GB")
table.add_row("─" * 20, "─" * 10, style="dim")
table.add_row("Total estimated", f"{total_est_gb:.2f} GB", style="bold")
table.add_row("Available", f"{free_gb:.1f} GB", style="bold green")
table.add_row(
"After download",
f"~{free_gb - total_est_gb:.1f} GB",
style="bold yellow" if free_gb - total_est_gb > 50 else "bold red",
)
console.print(table)
if total_est_gb > free_gb * 0.8:
log.error(f"Not enough disk space! Need {total_est_gb:.1f} GB, have {free_gb:.1f} GB")
return False
return True
# ── Main pipeline ─────────────────────────────────────────────────────
def run_pipeline(
stage: Optional[int] = None,
dataset_name: Optional[str] = None,
synthetic_only: bool = False,
) -> None:
"""Run the download pipeline."""
console.print(Panel.fit(
"[bold cyan]MINDI 1.5 Vision-Coder — Dataset Download Pipeline[/]\n"
"[dim]Day 2 Step 1: Download 500K+ training examples[/]",
border_style="cyan",
))
checkpoint = CheckpointManager()
# Filter datasets based on args
if dataset_name:
targets = [d for d in DATASETS if d.name == dataset_name]
if not targets:
log.error(f"Unknown dataset: {dataset_name}. Available: {[d.name for d in DATASETS]}")
return
elif synthetic_only:
targets = [d for d in DATASETS if d.is_synthetic]
elif stage is not None:
targets = [d for d in DATASETS if d.stage == stage or (stage == 0 and d.is_synthetic)]
else:
targets = DATASETS
# Show plan
plan_table = Table(title="📋 Download Plan")
plan_table.add_column("Dataset", style="cyan")
plan_table.add_column("Examples", justify="right")
plan_table.add_column("Est. Size", justify="right")
plan_table.add_column("Stage")
plan_table.add_column("Status")
for d in targets:
status = "✅ Done" if checkpoint.is_complete(d.name) else "⏳ Pending"
stage_label = f"Stage {d.stage}" if d.stage > 0 else "Synthetic"
plan_table.add_row(
d.name,
f"{d.target_count:,}",
f"{d.est_size_gb:.2f} GB",
stage_label,
status,
)
console.print(plan_table)
# Check disk space
pending = [d for d in targets if not checkpoint.is_complete(d.name)]
if not pending:
console.print("\n[bold green]✅ All requested datasets already downloaded![/]")
_print_summary(checkpoint)
return
if not check_disk_space(pending):
return
# Download with progress
console.print()
with Progress(
SpinnerColumn(),
TextColumn("[progress.description]{task.description}"),
BarColumn(),
MofNCompleteColumn(),
TimeElapsedColumn(),
TimeRemainingColumn(),
console=console,
refresh_per_second=2,
) as progress:
for config in pending:
if checkpoint.is_complete(config.name):
log.info(f"Skipping {config.name} (already complete)")
continue
log.info(f"\n{'─' * 50}")
log.info(f"Starting: {config.name} — {config.description}")
try:
if config.is_synthetic:
count = write_synthetic(config, checkpoint, progress)
else:
count = download_hf_dataset(config, checkpoint, progress)
size_mb = (DATA_RAW / config.output_file).stat().st_size / (1024 * 1024)
checkpoint.mark_complete(config.name, count, size_mb)
except KeyboardInterrupt:
log.warning(f"\n⚠️ Interrupted during {config.name}. Progress saved — rerun to resume.")
return
except Exception as e:
log.error(f"❌ Failed {config.name}: {e}")
log.error(traceback.format_exc())
continue
_print_summary(checkpoint)
def _print_summary(checkpoint: CheckpointManager) -> None:
"""Print final download summary."""
console.print()
summary = Table(title="📊 Download Summary")
summary.add_column("Dataset", style="cyan")
summary.add_column("Examples", justify="right")
summary.add_column("Size", justify="right")
summary.add_column("Time")
total_count = 0
total_mb = 0
for name, info in checkpoint.data["completed"].items():
summary.add_row(
name,
f"{info['count']:,}",
f"{info['size_mb']:.1f} MB",
info.get("timestamp", ""),
)
total_count += info["count"]
total_mb += info["size_mb"]
summary.add_row("─" * 20, "─" * 10, "─" * 10, "─" * 15, style="dim")
summary.add_row(
"[bold]TOTAL[/]",
f"[bold]{total_count:,}[/]",
f"[bold]{total_mb:.1f} MB[/]",
"",
style="bold green",
)
console.print(summary)
if total_count >= 500_000:
console.print("\n[bold green]🎉 TARGET REACHED: 500K+ examples downloaded![/]")
else:
remaining = 500_000 - total_count
console.print(f"\n[yellow]⏳ {remaining:,} more examples needed to reach 500K target[/]")
# ── CLI ───────────────────────────────────────────────────────────────
def main() -> None:
parser = argparse.ArgumentParser(description="MINDI Dataset Download Pipeline")
parser.add_argument("--dataset", type=str, help="Download a specific dataset by name")
parser.add_argument("--stage", type=int, choices=[0, 1, 2, 3], help="Download a specific stage")
parser.add_argument("--synthetic", action="store_true", help="Generate synthetic data only")
args = parser.parse_args()
run_pipeline(
stage=args.stage,
dataset_name=args.dataset,
synthetic_only=args.synthetic,
)
if __name__ == "__main__":
main()