| { |
| "results": { |
| "base": { |
| "test_loss": { |
| "loss": 1.4896, |
| "perplexity": 4.435 |
| }, |
| "test_generation": { |
| "n_prompts": 34, |
| "rougeL_f1": 0.1339, |
| "answers_with_code_block": 0.7647, |
| "python_blocks": 43, |
| "python_blocks_parse_rate": 1.0, |
| "finished_rate": 0.2941, |
| "mean_chars": 2023 |
| }, |
| "exec_benchmark": { |
| "passed": 11, |
| "total": 16, |
| "by_category": { |
| "type hints": "2/2", |
| "decorators": "2/2", |
| "async": "0/2", |
| "pandas": "2/3", |
| "exceptions": "2/2", |
| "debugging": "1/3", |
| "basics": "2/2" |
| } |
| }, |
| "general_probes": { |
| "auto_checks_passed": 4, |
| "auto_checks_total": 4, |
| "failed": [] |
| } |
| }, |
| "finetuned": { |
| "test_loss": { |
| "loss": 1.2407, |
| "perplexity": 3.458 |
| }, |
| "test_generation": { |
| "n_prompts": 34, |
| "rougeL_f1": 0.3456, |
| "answers_with_code_block": 0.3824, |
| "python_blocks": 12, |
| "python_blocks_parse_rate": 1.0, |
| "finished_rate": 1.0, |
| "mean_chars": 299 |
| }, |
| "exec_benchmark": { |
| "passed": 7, |
| "total": 16, |
| "by_category": { |
| "type hints": "2/2", |
| "decorators": "0/2", |
| "async": "0/2", |
| "pandas": "2/3", |
| "exceptions": "2/2", |
| "debugging": "0/3", |
| "basics": "1/2" |
| } |
| }, |
| "general_probes": { |
| "auto_checks_passed": 3, |
| "auto_checks_total": 4, |
| "failed": [ |
| "What is 17 multiplied by 23? Answer with the number." |
| ] |
| } |
| } |
| }, |
| "exec_benchmark_details": { |
| "base": [ |
| { |
| "task": "chunk_list", |
| "category": "type hints", |
| "passed": true, |
| "syntax_ok": true, |
| "error": "" |
| }, |
| { |
| "task": "word_frequencies", |
| "category": "type hints", |
| "passed": true, |
| "syntax_ok": true, |
| "error": "" |
| }, |
| { |
| "task": "retry", |
| "category": "decorators", |
| "passed": true, |
| "syntax_ok": true, |
| "error": "" |
| }, |
| { |
| "task": "count_calls", |
| "category": "decorators", |
| "passed": true, |
| "syntax_ok": true, |
| "error": "" |
| }, |
| { |
| "task": "fetch_all", |
| "category": "async", |
| "passed": false, |
| "syntax_ok": true, |
| "error": " +------------------------------------" |
| }, |
| { |
| "task": "run_with_timeout", |
| "category": "async", |
| "passed": false, |
| "syntax_ok": true, |
| "error": "TimeoutError" |
| }, |
| { |
| "task": "average_salary_by_department", |
| "category": "pandas", |
| "passed": true, |
| "syntax_ok": true, |
| "error": "" |
| }, |
| { |
| "task": "filter_adults", |
| "category": "pandas", |
| "passed": true, |
| "syntax_ok": true, |
| "error": "" |
| }, |
| { |
| "task": "fill_missing_scores", |
| "category": "pandas", |
| "passed": false, |
| "syntax_ok": true, |
| "error": "AssertionError" |
| }, |
| { |
| "task": "safe_divide", |
| "category": "exceptions", |
| "passed": true, |
| "syntax_ok": true, |
| "error": "" |
| }, |
| { |
| "task": "read_int_from_file", |
| "category": "exceptions", |
| "passed": true, |
| "syntax_ok": true, |
| "error": "" |
| }, |
| { |
| "task": "average", |
| "category": "debugging", |
| "passed": true, |
| "syntax_ok": true, |
| "error": "" |
| }, |
| { |
| "task": "find_max", |
| "category": "debugging", |
| "passed": false, |
| "syntax_ok": true, |
| "error": "AssertionError" |
| }, |
| { |
| "task": "add_item", |
| "category": "debugging", |
| "passed": false, |
| "syntax_ok": true, |
| "error": "AssertionError" |
| }, |
| { |
| "task": "is_palindrome", |
| "category": "basics", |
| "passed": true, |
| "syntax_ok": true, |
| "error": "" |
| }, |
| { |
| "task": "Point", |
| "category": "basics", |
| "passed": true, |
| "syntax_ok": true, |
| "error": "" |
| } |
| ], |
| "finetuned": [ |
| { |
| "task": "chunk_list", |
| "category": "type hints", |
| "passed": true, |
| "syntax_ok": true, |
| "error": "" |
| }, |
| { |
| "task": "word_frequencies", |
| "category": "type hints", |
| "passed": true, |
| "syntax_ok": true, |
| "error": "" |
| }, |
| { |
| "task": "retry", |
| "category": "decorators", |
| "passed": false, |
| "syntax_ok": true, |
| "error": "Exception: Failed after {times} attempts" |
| }, |
| { |
| "task": "count_calls", |
| "category": "decorators", |
| "passed": false, |
| "syntax_ok": true, |
| "error": "AssertionError" |
| }, |
| { |
| "task": "fetch_all", |
| "category": "async", |
| "passed": false, |
| "syntax_ok": true, |
| "error": "SyntaxError: 'await' outside async function" |
| }, |
| { |
| "task": "run_with_timeout", |
| "category": "async", |
| "passed": false, |
| "syntax_ok": true, |
| "error": "ValueError: a coroutine was expected, got 'fast'" |
| }, |
| { |
| "task": "average_salary_by_department", |
| "category": "pandas", |
| "passed": true, |
| "syntax_ok": true, |
| "error": "" |
| }, |
| { |
| "task": "filter_adults", |
| "category": "pandas", |
| "passed": false, |
| "syntax_ok": true, |
| "error": "AssertionError" |
| }, |
| { |
| "task": "fill_missing_scores", |
| "category": "pandas", |
| "passed": true, |
| "syntax_ok": true, |
| "error": "" |
| }, |
| { |
| "task": "safe_divide", |
| "category": "exceptions", |
| "passed": true, |
| "syntax_ok": true, |
| "error": "" |
| }, |
| { |
| "task": "read_int_from_file", |
| "category": "exceptions", |
| "passed": true, |
| "syntax_ok": true, |
| "error": "" |
| }, |
| { |
| "task": "average", |
| "category": "debugging", |
| "passed": false, |
| "syntax_ok": true, |
| "error": "ZeroDivisionError: division by zero" |
| }, |
| { |
| "task": "find_max", |
| "category": "debugging", |
| "passed": false, |
| "syntax_ok": true, |
| "error": "AssertionError" |
| }, |
| { |
| "task": "add_item", |
| "category": "debugging", |
| "passed": false, |
| "syntax_ok": true, |
| "error": "AssertionError" |
| }, |
| { |
| "task": "is_palindrome", |
| "category": "basics", |
| "passed": true, |
| "syntax_ok": true, |
| "error": "" |
| }, |
| { |
| "task": "Point", |
| "category": "basics", |
| "passed": false, |
| "syntax_ok": true, |
| "error": "AssertionError" |
| } |
| ] |
| }, |
| "clean_stats": { |
| "dropped_unbalanced_fence": 22, |
| "repaired_stray_brace": 1, |
| "repaired_auto_fenced": 68, |
| "dropped_truncated_colon": 61, |
| "dropped_empty_or_short": 6, |
| "dropped_unfenced_code": 25, |
| "dropped_python_syntax_error": 13, |
| "raw_rows": 462, |
| "clean_rows": 335, |
| "rows_with_input": 10, |
| "rows_with_python_block": 100 |
| }, |
| "topic_counts": { |
| "async / asyncio": 2, |
| "decorators": 1, |
| "type hints": 0, |
| "debugging / fixing bugs": 2, |
| "pandas / DataFrames": 64, |
| "exceptions / errors": 88, |
| "logging / monitoring": 55, |
| "Docker / deployment": 51, |
| "web (FastAPI/Flask/Django)": 77, |
| "testing": 1 |
| }, |
| "split": { |
| "seed": 42, |
| "n_clusters": 244, |
| "sizes": { |
| "test": 34, |
| "validation": 42, |
| "train": 259 |
| }, |
| "method": "Jaccard>=0.6 clustering of instruction words, cluster-level shuffle split" |
| }, |
| "training": { |
| "lora": { |
| "r": 16, |
| "lora_alpha": 32, |
| "lora_dropout": 0.05, |
| "target_modules": [ |
| "q_proj", |
| "k_proj", |
| "v_proj", |
| "o_proj", |
| "gate_proj", |
| "up_proj", |
| "down_proj" |
| ] |
| }, |
| "num_train_epochs": 3, |
| "learning_rate": 0.0001, |
| "per_device_train_batch_size": 4, |
| "gradient_accumulation_steps": 4, |
| "warmup_ratio": 0.05, |
| "lr_scheduler_type": "cosine", |
| "weight_decay": 0.0, |
| "max_grad_norm": 1.0, |
| "eval_steps": 10, |
| "early_stopping_patience": 3, |
| "best_eval_loss": 1.3612192869186401, |
| "best_step": 40, |
| "steps": 51, |
| "minutes": 1.6 |
| }, |
| "env": { |
| "python": "3.13.15", |
| "torch": "2.11.0+cu130", |
| "transformers": "4.57.1", |
| "peft": "0.17.1", |
| "datasets": "4.8.5", |
| "gpu": "Tesla T4", |
| "precision": "fp16" |
| }, |
| "smoke_test": false |
| } |
