Buckets:
| #!/usr/bin/env python3 | |
| """ | |
| Generate 100 raw game-repo training rows per repo (10 repos, 1000 total), | |
| validate JSONL, convert via convert_raw_ideas.py, and write a summary report. | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import subprocess | |
| import sys | |
| from collections import Counter | |
| from dataclasses import dataclass | |
| from pathlib import Path | |
| PROJECT_ROOT = Path(__file__).resolve().parent.parent | |
| RAW_DIR = PROJECT_ROOT / "data" / "raw_ideas" | |
| CONVERTED_DIR = PROJECT_ROOT / "data" / "converted" | |
| CONVERT_SCRIPT = PROJECT_ROOT / "scripts" / "convert_raw_ideas.py" | |
| REPORT_PATH = CONVERTED_DIR / "game_repo_generation_report.md" | |
| DEFAULT_TYPE_MIX = { | |
| "feature_build": 20, | |
| "bug_fix": 15, | |
| "animation_polish": 15, | |
| "browser_verification": 15, | |
| "input_handling": 10, | |
| "collision_debug": 10, | |
| "game_physics": 5, | |
| "refactor": 5, | |
| "performance_fix": 3, | |
| "game_loop_refactor": 2, | |
| } | |
| NON_PHYSICS_TYPE_MIX = { | |
| "feature_build": 28, | |
| "bug_fix": 15, | |
| "animation_polish": 15, | |
| "browser_verification": 15, | |
| "input_handling": 10, | |
| "collision_debug": 4, | |
| "game_physics": 0, | |
| "refactor": 8, | |
| "performance_fix": 3, | |
| "game_loop_refactor": 2, | |
| } | |
| PARTIAL_PHYSICS_TYPE_MIX = { | |
| "feature_build": 24, | |
| "bug_fix": 15, | |
| "animation_polish": 15, | |
| "browser_verification": 15, | |
| "input_handling": 10, | |
| "collision_debug": 6, | |
| "game_physics": 2, | |
| "refactor": 7, | |
| "performance_fix": 3, | |
| "game_loop_refactor": 3, | |
| } | |
| DIFFICULTIES = ["easy", "medium", "hard"] | |
| TOPICS = [ | |
| "animation loops", | |
| "requestAnimationFrame timing", | |
| "player movement", | |
| "camera follow", | |
| "keyboard and pointer input", | |
| "collision response", | |
| "physics feel", | |
| "finite state machines", | |
| "jump and landing", | |
| "WebGL or canvas rendering bugs", | |
| "HUD overlays", | |
| "pause and resume flow", | |
| "frame jitter", | |
| "input physics render separation", | |
| ] | |
| class RepoConfig: | |
| repo: str | |
| slug: str | |
| project_style: str | |
| physics_profile: str # heavy | light | partial | |
| repo_context: str | |
| focus_terms: tuple[str, ...] | |
| REPOS: list[RepoConfig] = [ | |
| RepoConfig( | |
| "cloud9c/taro", | |
| "cloud9c_taro", | |
| "JavaScript game/browser animation/engine", | |
| "light", | |
| "Taro is a browser-side animation and mini-game engine built around canvas scenes, sprite timelines, and a lightweight game loop that ties rendering to requestAnimationFrame.", | |
| ("sprite timelines", "scene transitions", "browser animation loops", "canvas layering", "HUD widgets"), | |
| ), | |
| RepoConfig( | |
| "Elkwizard/Hengine", | |
| "elkwizard_hengine", | |
| "JavaScript game engine/rendering", | |
| "heavy", | |
| "Hengine is a JavaScript rendering-focused game engine with scene graphs, draw calls, entity updates, and a frame loop that must keep simulation and presentation in sync.", | |
| ("scene graph updates", "draw batching", "entity transforms", "camera matrices", "frame delta smoothing"), | |
| ), | |
| RepoConfig( | |
| "LabyStudio/js-minecraft", | |
| "labystudio_js_minecraft", | |
| "JavaScript voxel Minecraft-like browser game", | |
| "heavy", | |
| "js-minecraft is a voxel sandbox running in the browser with chunk meshes, block placement, gravity, and first-person movement across a block grid.", | |
| ("voxel chunk meshes", "block raycasts", "first-person movement", "gravity stepping", "inventory HUD"), | |
| ), | |
| RepoConfig( | |
| "icurtis1/third-person-controller-splat", | |
| "icurtis_third_person_controller_splat", | |
| "TypeScript third-person controller/3D splat", | |
| "heavy", | |
| "This project combines a third-person character controller with Gaussian splat rendering, orbit camera logic, and ground collision probes in a WebGL viewport.", | |
| ("third-person locomotion", "orbit camera damping", "ground probes", "splat viewport rendering", "pointer lock look"), | |
| ), | |
| RepoConfig( | |
| "stared/quantum-game", | |
| "stared_quantum_game", | |
| "JavaScript puzzle/game logic", | |
| "light", | |
| "quantum-game is a browser puzzle built around quantum tile rules, turn validation, win-state checks, and a minimal UI that reflects superposition-style game logic.", | |
| ("puzzle rule validation", "turn state machine", "win detection", "board rendering", "hint overlays"), | |
| ), | |
| RepoConfig( | |
| "lettier/webglbowling", | |
| "lettier_webglbowling", | |
| "JavaScript WebGL bowling/physics", | |
| "heavy", | |
| "webglbowling simulates lane physics with ball spin, pin collisions, score tallying, and a WebGL camera that tracks the rolling ball.", | |
| ("ball spin torque", "pin collision chains", "lane friction", "score tally HUD", "follow camera"), | |
| ), | |
| RepoConfig( | |
| "NicklessOne/inks2d", | |
| "nicklessone_inks2d", | |
| "TypeScript 2D engine", | |
| "light", | |
| "inks2d is a TypeScript 2D engine with sprite scenes, tile maps, input routing, and a fixed-timestep-friendly update loop for browser games.", | |
| ("sprite scenes", "tile map layers", "input routing", "scene stack", "debug draw overlays"), | |
| ), | |
| RepoConfig( | |
| "simondevyoutube/ThreeJS_Tutorial_CharacterController", | |
| "simondev_threejs_character_controller", | |
| "JavaScript Three.js character controller", | |
| "partial", | |
| "The tutorial project implements a Three.js character with capsule collision, blend-tree style locomotion, camera rigging, and keyboard-driven movement on uneven meshes.", | |
| ("capsule collision", "locomotion blending", "camera rig", "slope movement", "animation state graph"), | |
| ), | |
| RepoConfig( | |
| "christophmark/pendulum", | |
| "christophmark_pendulum", | |
| "JavaScript physics simulation/pendulum", | |
| "heavy", | |
| "pendulum is a canvas physics demo with coupled oscillators, damping coefficients, drag input, and real-time graphing of angle and energy.", | |
| ("angular integration", "damping coefficients", "coupled rods", "energy graphs", "mouse impulse"), | |
| ), | |
| RepoConfig( | |
| "doppl3r/kinematic-character-controller-example", | |
| "doppl3r_kinematic_character_controller", | |
| "JavaScript kinematic character controller", | |
| "heavy", | |
| "This example shows a kinematic character controller with swept collision, step offset, slope limits, and decoupled input sampling from physics resolution.", | |
| ("swept capsule tests", "step offset", "slope limit", "ground snapping", "input buffer"), | |
| ), | |
| ] | |
| TYPE_MIX_BY_PROFILE = { | |
| "heavy": DEFAULT_TYPE_MIX, | |
| "light": NON_PHYSICS_TYPE_MIX, | |
| "partial": PARTIAL_PHYSICS_TYPE_MIX, | |
| } | |
| def type_mix_for(repo: RepoConfig) -> dict[str, int]: | |
| return dict(TYPE_MIX_BY_PROFILE[repo.physics_profile]) | |
| def expand_type_list(mix: dict[str, int]) -> list[str]: | |
| items: list[str] = [] | |
| for type_name, count in mix.items(): | |
| items.extend([type_name] * count) | |
| return items | |
| def pick_topic(index: int, repo: RepoConfig) -> str: | |
| focus = repo.focus_terms[index % len(repo.focus_terms)] | |
| topic = TOPICS[(index * 3 + len(repo.slug)) % len(TOPICS)] | |
| return f"{topic} with emphasis on {focus}" | |
| def pick_difficulty(index: int, type_name: str) -> str: | |
| bias = { | |
| "feature_build": 0, | |
| "bug_fix": 1, | |
| "animation_polish": 0, | |
| "browser_verification": 0, | |
| "input_handling": 1, | |
| "collision_debug": 2, | |
| "game_physics": 2, | |
| "refactor": 1, | |
| "performance_fix": 2, | |
| "game_loop_refactor": 2, | |
| } | |
| return DIFFICULTIES[(index + bias.get(type_name, 0)) % len(DIFFICULTIES)] | |
| def instruction_for(type_name: str, repo: RepoConfig, index: int, topic: str) -> str: | |
| repo_name = repo.repo.split("/")[-1] | |
| n = index + 1 | |
| templates = { | |
| "feature_build": [ | |
| f"Add a {topic} feature to {repo_name} so players get clearer feedback during active gameplay without rewriting the entire loop.", | |
| f"Implement a new {topic} capability in {repo_name} that plugs into the existing update path and respects pause state.", | |
| f"Extend {repo_name} with a {topic} module that composes with current scene setup and keeps render order predictable.", | |
| f"Build a {topic} enhancement for {repo_name} that exposes toggles in the HUD and stays inactive while the game is paused.", | |
| f"Introduce {topic} support in {repo_name} so designers can tune feel parameters without touching low-level draw code.", | |
| ], | |
| "bug_fix": [ | |
| f"Fix a regression in {repo_name} where {topic} breaks after tab focus changes and the simulation resumes with stale state.", | |
| f"Repair {topic} handling in {repo_name} so repeated start-stop cycles no longer leave hidden entities updating in the background.", | |
| f"Correct {repo_name} logic so {topic} no longer double-fires when both keyboard and pointer paths are active.", | |
| f"Resolve a {topic} defect in {repo_name} that only appears on high-refresh displays because delta time is applied twice.", | |
| f"Patch {repo_name} so {topic} stops drifting when the canvas is resized and devicePixelRatio changes mid-session.", | |
| ], | |
| "animation_polish": [ | |
| f"Polish {topic} transitions in {repo_name} so blend timing feels smooth and avoids pops when state changes mid-frame.", | |
| f"Tune easing on {topic} sequences in {repo_name} to remove stair-stepped motion during camera or sprite updates.", | |
| f"Refine {topic} interpolation in {repo_name} so animation phases stay aligned with the render phase of the loop.", | |
| f"Smooth {topic} handoff curves in {repo_name} when the player changes direction quickly on keyboard input.", | |
| f"Adjust {topic} timing constants in {repo_name} to eliminate one-frame flicker during loop restarts.", | |
| ], | |
| "browser_verification": [ | |
| f"Verify in the browser that {topic} in {repo_name} behaves correctly across Chrome, Firefox, and Safari without console errors.", | |
| f"Run an interactive browser check on {repo_name} confirming {topic} survives reload, resize, and visibility changes.", | |
| f"Validate {topic} in {repo_name} using devtools performance panels to ensure frame pacing stays stable during stress input.", | |
| f"Manually test {topic} in {repo_name} on mobile and desktop layouts to confirm controls and HUD remain usable.", | |
| f"Confirm through browser inspection that {topic} in {repo_name} keeps deterministic ordering after several pause-resume cycles.", | |
| ], | |
| "input_handling": [ | |
| f"Improve {topic} input routing in {repo_name} so key repeat, pointer capture, and gamepad-like events do not fight each other.", | |
| f"Refactor {repo_name} input sampling for {topic} to buffer presses across frames and respect disabled states during cutscenes.", | |
| f"Add debounced {topic} controls in {repo_name} that remain responsive without flooding the physics step with duplicate commands.", | |
| f"Normalize {topic} pointer and keyboard bindings in {repo_name} so rebinding does not require editing multiple modules.", | |
| f"Harden {topic} input guards in {repo_name} against focus loss when users alt-tab during active movement.", | |
| ], | |
| "collision_debug": [ | |
| f"Debug {topic} collision mismatches in {repo_name} where visual meshes and logical hit volumes disagree after transforms update.", | |
| f"Trace {topic} contact resolution in {repo_name} to find why entities tunnel on fast movement frames.", | |
| f"Instrument {topic} overlap tests in {repo_name} with debug draws that reveal false positives near scene boundaries.", | |
| f"Isolate a {topic} collision edge case in {repo_name} where stacked contacts produce jitter on shallow slopes.", | |
| f"Fix {topic} probe ordering in {repo_name} so ground checks run before horizontal push resolution each tick.", | |
| ], | |
| "game_physics": [ | |
| f"Tune {topic} physics parameters in {repo_name} so acceleration, friction, and restitution feel consistent across frame rates.", | |
| f"Adjust {topic} integrator settings in {repo_name} to reduce energy drift during long simulation runs.", | |
| f"Calibrate {topic} impulse response in {repo_name} so jump arcs and landing snaps remain stable on uneven surfaces.", | |
| f"Balance {topic} forces in {repo_name} to prevent runaway velocity when collision correction stacks over multiple substeps.", | |
| f"Refine {topic} material response in {repo_name} so sliding and sticking transitions feel natural on interactive props.", | |
| ], | |
| "refactor": [ | |
| f"Refactor {topic} code paths in {repo_name} into a dedicated module without changing outward gameplay behavior.", | |
| f"Restructure {repo_name} so {topic} setup, update, and teardown hooks are declared in one registry instead of scattered listeners.", | |
| f"Extract {topic} helpers in {repo_name} to reduce duplication between editor preview mode and live play mode.", | |
| f"Reorganize {topic} state in {repo_name} behind a small facade so tests can stub timing without mocking WebGL.", | |
| f"Split {topic} concerns in {repo_name} so render-only code no longer mutates simulation fields during draw.", | |
| ], | |
| "performance_fix": [ | |
| f"Optimize {topic} hot paths in {repo_name} to cut per-frame allocations that spike garbage collection during motion.", | |
| f"Reduce {topic} overdraw or redundant updates in {repo_name} that cause frame jitter when many entities are active.", | |
| f"Cache {topic} intermediate results in {repo_name} so repeated queries inside the loop stop scanning whole scene lists.", | |
| f"Throttle expensive {topic} work in {repo_name} while keeping input sampling at full rate for responsiveness.", | |
| f"Profile and streamline {topic} update order in {repo_name} to remove duplicate math done in both physics and render passes.", | |
| ], | |
| "game_loop_refactor": [ | |
| f"Refactor the {repo_name} game loop so {topic} uses a fixed simulation step with interpolated render poses.", | |
| f"Separate {topic} input capture, physics integration, and render submission in {repo_name} into three explicit phases per frame.", | |
| f"Rework {repo_name} loop scheduling so {topic} pauses simulation while still animating HUD overlays via requestAnimationFrame.", | |
| f"Migrate {repo_name} from ad-hoc timeouts to a single rAF-driven loop that owns {topic} timing and delta clamping.", | |
| ], | |
| } | |
| bucket = templates[type_name] | |
| return bucket[(n + len(repo.slug)) % len(bucket)] | |
| def repo_context_for(repo: RepoConfig, type_name: str, topic: str, index: int) -> str: | |
| return ( | |
| f"{repo.repo_context} Current task area: {topic}. " | |
| f"Work type: {type_name.replace('_', ' ')}. Scenario variant {index + 1} for {repo.slug}." | |
| ) | |
| def agent_behavior_for(type_name: str, repo: RepoConfig, topic: str, index: int) -> list[str]: | |
| repo_short = repo.repo.split("/")[-1] | |
| steps = [ | |
| f"Scan {repo_short} project structure to locate modules that own {topic}.", | |
| f"Read the active game loop and note where input, simulation, and render phases currently interleave for {topic}.", | |
| f"Reproduce the reported {type_name.replace('_', ' ')} behavior in a local browser session before editing code.", | |
| f"Inspect state holders and timing code tied to {topic}, including any rAF callbacks or fixed-step accumulators.", | |
| f"Implement the smallest change that addresses {topic} while keeping existing public APIs stable for {repo_short}.", | |
| ] | |
| if type_name in {"browser_verification", "bug_fix"}: | |
| steps.append( | |
| f"Capture before-and-after notes in devtools covering console output, frame times, and {topic} edge cases." | |
| ) | |
| elif type_name in {"refactor", "game_loop_refactor"}: | |
| steps.append( | |
| f"Move {topic} responsibilities behind explicit update hooks so future features do not re-tangle input and render code." | |
| ) | |
| else: | |
| steps.append( | |
| f"Validate {topic} interactively and confirm pause, resume, and resize flows still behave in {repo_short}." | |
| ) | |
| # 4-5 steps: drop one deterministically for variety | |
| if index % 5 == 0: | |
| return steps[:4] | |
| return steps[:5] | |
| def success_criteria_for(type_name: str, topic: str, index: int) -> list[str]: | |
| return [ | |
| f"{topic} behaves consistently across at least one minute of continuous play.", | |
| f"No new console errors appear while exercising {topic} after the change.", | |
| f"Pause and resume preserves {topic} state without duplicating listeners or timers.", | |
| f"Frame pacing remains stable when {topic} is stressed with rapid input bursts.", | |
| ] | |
| def failure_modes_for(type_name: str, topic: str, index: int) -> list[str]: | |
| return [ | |
| f"Applying delta time twice inside {topic} handlers, which amplifies frame jitter on high-refresh monitors.", | |
| f"Letting render-phase code mutate physics state for {topic}, breaking separation between simulation and draw.", | |
| ] | |
| def build_row(repo: RepoConfig, type_name: str, index: int) -> dict: | |
| topic = pick_topic(index, repo) | |
| return { | |
| "type": type_name, | |
| "source_kind": "github_repo_inspired", | |
| "source_repo": repo.repo, | |
| "difficulty": pick_difficulty(index, type_name), | |
| "project_style": repo.project_style, | |
| "instruction": instruction_for(type_name, repo, index, topic), | |
| "repo_context": repo_context_for(repo, type_name, topic, index), | |
| "agent_behavior": agent_behavior_for(type_name, repo, topic, index), | |
| "success_criteria": success_criteria_for(type_name, topic, index), | |
| "failure_modes": failure_modes_for(type_name, topic, index), | |
| } | |
| def validate_row_shape(row: dict, line_num: int) -> list[str]: | |
| errors: list[str] = [] | |
| required = [ | |
| "type", | |
| "source_kind", | |
| "source_repo", | |
| "difficulty", | |
| "project_style", | |
| "instruction", | |
| "repo_context", | |
| "agent_behavior", | |
| "success_criteria", | |
| "failure_modes", | |
| ] | |
| for field in required: | |
| if field not in row: | |
| errors.append(f"line {line_num}: missing {field}") | |
| if not isinstance(row.get("agent_behavior"), list) or not (4 <= len(row["agent_behavior"]) <= 5): | |
| errors.append(f"line {line_num}: agent_behavior must have 4-5 items") | |
| if not isinstance(row.get("success_criteria"), list) or len(row.get("success_criteria", [])) != 4: | |
| errors.append(f"line {line_num}: success_criteria must have 4 items") | |
| if not isinstance(row.get("failure_modes"), list) or len(row.get("failure_modes", [])) != 2: | |
| errors.append(f"line {line_num}: failure_modes must have 2 items") | |
| return errors | |
| def validate_raw_file(path: Path) -> tuple[int, list[str]]: | |
| errors: list[str] = [] | |
| count = 0 | |
| with path.open("r", encoding="utf-8") as f: | |
| for line_num, line in enumerate(f, 1): | |
| text = line.rstrip("\n") | |
| if not text.strip(): | |
| errors.append(f"line {line_num}: empty line") | |
| continue | |
| count += 1 | |
| try: | |
| row = json.loads(text) | |
| except json.JSONDecodeError as exc: | |
| errors.append(f"line {line_num}: JSON error: {exc}") | |
| continue | |
| errors.extend(validate_row_shape(row, line_num)) | |
| return count, errors | |
| def generate_repo_rows(repo: RepoConfig) -> list[dict]: | |
| mix = type_mix_for(repo) | |
| assert sum(mix.values()) == 100, f"{repo.slug} mix must total 100" | |
| type_list = expand_type_list(mix) | |
| rows: list[dict] = [] | |
| seen_instructions: set[str] = set() | |
| for index, type_name in enumerate(type_list): | |
| row = build_row(repo, type_name, index) | |
| instruction = row["instruction"] | |
| if instruction in seen_instructions: | |
| row["instruction"] = ( | |
| f"{instruction} (variant {index + 1} scoped to {repo.slug} acceptance criteria.)" | |
| ) | |
| seen_instructions.add(row["instruction"]) | |
| rows.append(row) | |
| return rows | |
| def write_jsonl(path: Path, rows: list[dict]) -> None: | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| with path.open("w", encoding="utf-8", newline="\n") as f: | |
| for row in rows: | |
| f.write(json.dumps(row, ensure_ascii=False) + "\n") | |
| def run_convert(slug: str, raw_path: Path) -> tuple[int, str | None]: | |
| converted_path = CONVERTED_DIR / f"{slug}_converted.jsonl" | |
| cmd = [ | |
| sys.executable, | |
| str(CONVERT_SCRIPT), | |
| "--input", | |
| str(raw_path.relative_to(PROJECT_ROOT)).replace("\\", "/"), | |
| "--standalone", | |
| str(converted_path.relative_to(PROJECT_ROOT)).replace("\\", "/"), | |
| "--id-prefix", | |
| slug, | |
| ] | |
| try: | |
| result = subprocess.run( | |
| cmd, | |
| cwd=str(PROJECT_ROOT), | |
| capture_output=True, | |
| text=True, | |
| check=False, | |
| ) | |
| except Exception as exc: # noqa: BLE001 | |
| return 0, str(exc) | |
| if result.returncode != 0: | |
| err = (result.stderr or result.stdout or "convert failed").strip() | |
| return 0, err | |
| if not converted_path.exists(): | |
| return 0, "convert reported success but output file missing" | |
| with converted_path.open("r", encoding="utf-8") as f: | |
| converted_count = sum(1 for line in f if line.strip()) | |
| return converted_count, None | |
| def count_file_lines(path: Path) -> int: | |
| if not path.exists(): | |
| return 0 | |
| with path.open("r", encoding="utf-8") as f: | |
| return sum(1 for line in f if line.strip()) | |
| def write_report(results: list[dict]) -> None: | |
| CONVERTED_DIR.mkdir(parents=True, exist_ok=True) | |
| total_raw = sum(r["raw_count"] for r in results) | |
| total_converted = sum(r["converted_count"] for r in results) | |
| lines = [ | |
| "# Game Repo Generation Report", | |
| "", | |
| f"Generated on: {Path(__file__).name} run from `{PROJECT_ROOT}`", | |
| "", | |
| "## Summary", | |
| "", | |
| f"- Repos processed: {len(results)}", | |
| f"- Total raw rows: {total_raw}", | |
| f"- Total converted rows: {total_converted}", | |
| f"- Errors: {sum(1 for r in results if r['errors'])}", | |
| "", | |
| "## Per-repo stats", | |
| "", | |
| "| Repo | Slug | Physics profile | Raw | Converted | Type mix total | Errors |", | |
| "| --- | --- | --- | ---: | ---: | ---: | --- |", | |
| ] | |
| for r in results: | |
| err_text = "; ".join(r["errors"]) if r["errors"] else "—" | |
| lines.append( | |
| f"| {r['repo']} | {r['slug']} | {r['physics_profile']} | " | |
| f"{r['raw_count']} | {r['converted_count']} | {r['type_total']} | {err_text} |" | |
| ) | |
| lines.extend( | |
| [ | |
| "", | |
| "## TASK_TYPE_MAP note", | |
| "", | |
| "Game-oriented raw `type` values are mapped in `scripts/convert_raw_ideas.py`:", | |
| "", | |
| "| Raw type | Canonical task_type |", | |
| "| --- | --- |", | |
| "| feature_build | feature_build |", | |
| "| bug_fix | bug_fix |", | |
| "| animation_polish | ui_repair |", | |
| "| browser_verification | terminal_debug |", | |
| "| input_handling | bug_fix |", | |
| "| collision_debug | bug_fix |", | |
| "| game_physics | feature_build |", | |
| "| refactor | refactor |", | |
| "| performance_fix | refactor |", | |
| "| game_loop_refactor | refactor |", | |
| "", | |
| "## Type mix reference", | |
| "", | |
| "- Physics-heavy repos use the default 100-row mix (20/15/15/15/10/10/5/5/3/2).", | |
| "- Light repos swap collision_debug and game_physics into feature_build and refactor.", | |
| "- Partial repos (simondev character controller) use an intermediate mix.", | |
| "", | |
| "## Output files", | |
| "", | |
| ] | |
| ) | |
| for r in results: | |
| lines.append(f"- Raw: `data/raw_ideas/{r['slug']}_rows.jsonl`") | |
| lines.append(f"- Converted: `data/converted/{r['slug']}_converted.jsonl`") | |
| REPORT_PATH.write_text("\n".join(lines) + "\n", encoding="utf-8") | |
| def process_repo(repo: RepoConfig) -> dict: | |
| slug = repo.slug | |
| raw_path = RAW_DIR / f"{slug}_rows.jsonl" | |
| result = { | |
| "repo": repo.repo, | |
| "slug": slug, | |
| "physics_profile": repo.physics_profile, | |
| "raw_count": 0, | |
| "converted_count": 0, | |
| "type_total": sum(type_mix_for(repo).values()), | |
| "errors": [], | |
| "type_breakdown": dict(type_mix_for(repo)), | |
| } | |
| try: | |
| rows = generate_repo_rows(repo) | |
| counts = Counter(row["type"] for row in rows) | |
| expected = type_mix_for(repo) | |
| for type_name, expected_count in expected.items(): | |
| if counts[type_name] != expected_count: | |
| result["errors"].append( | |
| f"type mismatch {type_name}: expected {expected_count}, got {counts[type_name]}" | |
| ) | |
| instructions = [row["instruction"] for row in rows] | |
| if len(instructions) != len(set(instructions)): | |
| result["errors"].append("duplicate instructions detected within repo") | |
| write_jsonl(raw_path, rows) | |
| raw_count, validation_errors = validate_raw_file(raw_path) | |
| result["raw_count"] = raw_count | |
| if validation_errors: | |
| result["errors"].extend(validation_errors[:5]) | |
| if len(validation_errors) > 5: | |
| result["errors"].append(f"...and {len(validation_errors) - 5} more validation issues") | |
| if not result["errors"]: | |
| converted_count, convert_error = run_convert(slug, raw_path) | |
| result["converted_count"] = converted_count | |
| if convert_error: | |
| result["errors"].append(f"convert: {convert_error}") | |
| except Exception as exc: # noqa: BLE001 | |
| result["errors"].append(f"exception: {exc}") | |
| return result | |
| def main() -> int: | |
| results: list[dict] = [] | |
| print("Generating game repo rows (10 repos x 100 rows)...") | |
| for repo in REPOS: | |
| print(f"\n--- {repo.repo} ({repo.slug}) ---") | |
| result = process_repo(repo) | |
| results.append(result) | |
| if result["errors"]: | |
| print(f" raw={result['raw_count']} converted={result['converted_count']} ERRORS={len(result['errors'])}") | |
| for err in result["errors"][:3]: | |
| print(f" - {err}") | |
| else: | |
| print(f" raw={result['raw_count']} converted={result['converted_count']} OK") | |
| write_report(results) | |
| print(f"\nReport written: {REPORT_PATH}") | |
| print("\n=== Summary Table ===") | |
| print(f"{'Repo':<45} {'Raw':>5} {'Conv':>5} {'Errors':>8}") | |
| print("-" * 70) | |
| for r in results: | |
| err_count = len(r["errors"]) | |
| print(f"{r['repo']:<45} {r['raw_count']:>5} {r['converted_count']:>5} {err_count:>8}") | |
| total_raw = sum(r["raw_count"] for r in results) | |
| total_conv = sum(r["converted_count"] for r in results) | |
| total_err = sum(len(r["errors"]) for r in results) | |
| print("-" * 70) | |
| print(f"{'TOTAL':<45} {total_raw:>5} {total_conv:>5} {total_err:>8}") | |
| return 0 if total_raw == 1000 and total_err == 0 else 1 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |
Xet Storage Details
- Size:
- 27.5 kB
- Xet hash:
- 61c8830af6ce3db669556801bd351d51832decfca67c258428a6f197f250153e
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.