| { | |
| "model": { | |
| "codellama": { | |
| "base_model_id": "codellama/CodeLlama-7b-hf", | |
| "quantitize": "int8", | |
| "dataset": "Arithmetic_Simple", | |
| "data_collator": "DataCollatorForSeq2Seq", | |
| "lora_config": { | |
| "r": 16, | |
| "lora_alpha": 16, | |
| "target_modules": [ | |
| "q_proj", | |
| "k_proj", | |
| "v_proj", | |
| "o_proj", | |
| "gate_proj", | |
| "up_proj", | |
| "down_proj" | |
| ], | |
| "lora_dropout": 0.05, | |
| "bias": "none", | |
| "task_type": "CAUSAL_LM" | |
| }, | |
| "training_args": { | |
| "output_dir": "codellama-output", | |
| "warmup_steps": 100, | |
| "per_device_train_batch_size": 1, | |
| "per_device_eval_batch_size": 1, | |
| "gradient_accumulation_steps": 4, | |
| "max_steps": 10000, | |
| "learning_rate": 0.0003, | |
| "optim": "adamw_torch", | |
| "logging_dir": "codellama-output-logs", | |
| "logging_steps": 10, | |
| "save_strategy": "steps", | |
| "save_steps": 500, | |
| "load_best_model_at_end": false, | |
| "group_by_length": true, | |
| "fp16": true, | |
| "evaluation_strategy": "steps", | |
| "eval_steps": 1000 | |
| }, | |
| "tokenizer": { | |
| "tokenize_config": { | |
| "truncation": true, | |
| "max_length": 192, | |
| "padding": "max_length" | |
| }, | |
| "prompt_template": "config/qa_template.txt" | |
| } | |
| }, | |
| "phi-2": { | |
| "base_model_id": "microsoft/phi-2", | |
| "quantitize": "fp16", | |
| "dataset": "Arithmetic_Simple", | |
| "data_collator": "DataCollatorForLanguageModeling", | |
| "lora_config": { | |
| "r": 32, | |
| "lora_alpha": 64, | |
| "target_modules": [ | |
| "q_proj", | |
| "k_proj", | |
| "v_proj", | |
| "dense", | |
| "fc1", | |
| "fc2" | |
| ], | |
| "bias": "none", | |
| "lora_dropout": 0.05, | |
| "task_type": "CAUSAL_LM" | |
| }, | |
| "training_args": { | |
| "output_dir": "phi2-output", | |
| "warmup_steps": 500, | |
| "per_device_train_batch_size": 1, | |
| "per_device_eval_batch_size": 1, | |
| "gradient_accumulation_steps": 4, | |
| "max_steps": 100000, | |
| "learning_rate": 0.0003, | |
| "optim": "paged_adamw_8bit", | |
| "logging_dir": "phi2-output-logs", | |
| "logging_steps": 100, | |
| "save_strategy": "steps", | |
| "save_steps": 500, | |
| "evaluation_strategy": "steps", | |
| "eval_steps": 500, | |
| "fp16": true | |
| }, | |
| "tokenizer": { | |
| "tokenize_config": { | |
| "truncation": true, | |
| "max_length": 512, | |
| "padding": "max_length" | |
| }, | |
| "prompt_template": "config/qa_template.txt" | |
| } | |
| }, | |
| "deepseek": { | |
| "base_model_id": "deepseek-ai/deepseek-coder-1.3b-instruct", | |
| "quantitize": "fp16", | |
| "dataset": "mixture_codegen", | |
| "data_collator": "DataCollatorForLanguageModeling", | |
| "lora_config": { | |
| "r": 32, | |
| "lora_alpha": 64, | |
| "target_modules": [ | |
| "q_proj", | |
| "k_proj", | |
| "v_proj", | |
| "o_proj", | |
| "gate_proj", | |
| "up_proj", | |
| "down_proj" | |
| ], | |
| "bias": "none", | |
| "lora_dropout": 0.05, | |
| "task_type": "CAUSAL_LM" | |
| }, | |
| "lora_large_config": { | |
| "r": 128, | |
| "lora_alpha": 256, | |
| "target_modules": [ | |
| "q_proj", | |
| "k_proj", | |
| "v_proj", | |
| "o_proj", | |
| "gate_proj", | |
| "up_proj", | |
| "down_proj" | |
| ], | |
| "bias": "none", | |
| "lora_dropout": 0.05, | |
| "task_type": "CAUSAL_LM" | |
| }, | |
| "p_tuning_config": { | |
| "num_virtual_tokens": 16, | |
| "num_transformer_submodules": 1, | |
| "token_dim": 2048, | |
| "encoder_hidden_size": 2048, | |
| "task_type": "CAUSAL_LM" | |
| }, | |
| "training_args": { | |
| "output_dir": "runs/deepseek-ctrl-gh-mixture", | |
| "warmup_steps": 0, | |
| "per_device_train_batch_size": 4, | |
| "per_device_eval_batch_size": 4, | |
| "gradient_accumulation_steps": 8, | |
| "max_steps": 5000, | |
| "learning_rate": 2e-05, | |
| "optim": "paged_adamw_8bit", | |
| "logging_dir": "runs/deepseek-ctrl-gh-mixture/logs", | |
| "logging_steps": 100, | |
| "save_strategy": "steps", | |
| "save_steps": 2500, | |
| "evaluation_strategy": "steps", | |
| "eval_steps": 2500, | |
| "weight_decay": 0.01, | |
| "fp16": true | |
| }, | |
| "tokenizer": { | |
| "tokenize_config": { | |
| "truncation": true, | |
| "max_length": 1024, | |
| "padding": "max_length" | |
| }, | |
| "prompt_template": "config/qa_template.txt" | |
| } | |
| } | |
| }, | |
| "dataset": { | |
| "simple_dataset": { | |
| "type": "huggingface", | |
| "dataset_purpose": "downstream", | |
| "name": "b-mc2/sql-create-context", | |
| "train_split": 0.9, | |
| "max_train_size": 100, | |
| "filling_field": [ | |
| "question", | |
| "context", | |
| "answer" | |
| ] | |
| }, | |
| "testdset": { | |
| "type": "local", | |
| "dataset_purpose": "downstream", | |
| "train_file": "data/Test/TestDataset.json", | |
| "val_file": "data/Test/TestDataset.json", | |
| "test_file": "data/Test/TestDataset.json", | |
| "filling_field": [ | |
| "prompted_question", | |
| "answer" | |
| ] | |
| }, | |
| "mixture_codegen": { | |
| "filling_field": [ | |
| "Question", | |
| "Answer" | |
| ], | |
| "dataset_purpose": "downstream" | |
| }, | |
| "MathQA_Python_loader": { | |
| "type": "list-like", | |
| "dataset_purpose": "downstream", | |
| "train": "data/MathQA_Python_processed/mathqa_python_train_clean_final.json", | |
| "val": "data/MathQA_Python_processed/mathqa_python_dev_clean_final.json", | |
| "test": "data/MathQA_Python_processed/mathqa_python_test_clean_final.json", | |
| "filling_field": [ | |
| "Question", | |
| "Answer" | |
| ] | |
| }, | |
| "APPS_loader": { | |
| "type": "list-like", | |
| "dataset_purpose": "downstream", | |
| "train": "data/APPS/apps_train.json", | |
| "val": "data/APPS/apps_dev.json", | |
| "test": "data/APPS/test/apps_test_75.json", | |
| "filling_field": [ | |
| "Question", | |
| "Answer" | |
| ] | |
| }, | |
| "MBPP_loader": { | |
| "type": "list-like", | |
| "dataset_purpose": "downstream", | |
| "train": "data/MBPP/mbpp_train.json", | |
| "val": "data/MBPP/mbpp_dev.json", | |
| "test": "data/MBPP/mbpp_test.json", | |
| "filling_field": [ | |
| "Question", | |
| "Answer" | |
| ] | |
| }, | |
| "Arithmetic_Simple": { | |
| "type": "list-like", | |
| "dataset_purpose": "downstream", | |
| "attributes": { | |
| "subjects": [ | |
| 1, | |
| 2, | |
| 3, | |
| 4, | |
| 5, | |
| 6, | |
| 7, | |
| 8, | |
| 9 | |
| ], | |
| "lessons": [ | |
| "Max_Ops1_Bounds0_100", | |
| "Max_Ops1_Bounds0_1000", | |
| "Max_Ops2_Bounds0_100", | |
| "Max_Ops2_Bounds0_1000", | |
| "Max_Ops3_Bounds0_100", | |
| "Max_Ops3_Bounds0_1000", | |
| "Max_Ops4_Bounds0_100", | |
| "Max_Ops4_Bounds0_1000", | |
| "Max_Ops5_Bounds0_100", | |
| "Max_Ops5_Bounds0_1000" | |
| ] | |
| }, | |
| "train": "data/Arithmetic/Curriculum_Simple", | |
| "val": "data/Arithmetic/Curriculum_Simple", | |
| "test": "data/Arithmetic/Curriculum_Simple", | |
| "filling_field": [ | |
| "Question", | |
| "Answer" | |
| ] | |
| }, | |
| "Arithmetic_Hard": { | |
| "type": "list-like", | |
| "dataset_purpose": "downstream", | |
| "attributes": { | |
| "subjects": [ | |
| 1, | |
| 2, | |
| 3, | |
| 4, | |
| 5, | |
| 6, | |
| 7, | |
| 8, | |
| 9 | |
| ], | |
| "lessons": [ | |
| "Max_Ops1_Bounds-1000_1000", | |
| "Max_Ops1_Bounds-100_100", | |
| "Max_Ops1_Bounds0_100", | |
| "Max_Ops1_Bounds0_1000", | |
| "Max_Ops2_Bounds-1000_1000", | |
| "Max_Ops2_Bounds-100_100", | |
| "Max_Ops2_Bounds0_100", | |
| "Max_Ops2_Bounds0_1000", | |
| "Max_Ops3_Bounds-1000_1000", | |
| "Max_Ops3_Bounds-100_100", | |
| "Max_Ops3_Bounds0_100", | |
| "Max_Ops3_Bounds0_1000", | |
| "Max_Ops4_Bounds-1000_1000", | |
| "Max_Ops4_Bounds-100_100", | |
| "Max_Ops4_Bounds0_100", | |
| "Max_Ops4_Bounds0_1000", | |
| "Max_Ops5_Bounds-1000_1000", | |
| "Max_Ops5_Bounds-100_100", | |
| "Max_Ops5_Bounds0_100", | |
| "Max_Ops5_Bounds0_1000", | |
| "Max_Ops6_Bounds-1000_1000", | |
| "Max_Ops6_Bounds-100_100", | |
| "Max_Ops6_Bounds0_100", | |
| "Max_Ops6_Bounds0_1000", | |
| "Max_Ops7_Bounds-1000_1000", | |
| "Max_Ops7_Bounds-100_100", | |
| "Max_Ops7_Bounds0_100", | |
| "Max_Ops7_Bounds0_1000", | |
| "Max_Ops8_Bounds-1000_1000", | |
| "Max_Ops8_Bounds-100_100", | |
| "Max_Ops8_Bounds0_100", | |
| "Max_Ops8_Bounds0_1000", | |
| "Max_Ops9_Bounds-1000_1000", | |
| "Max_Ops9_Bounds-100_100", | |
| "Max_Ops9_Bounds0_100", | |
| "Max_Ops9_Bounds0_1000", | |
| "Max_Ops10_Bounds-1000_1000", | |
| "Max_Ops10_Bounds-100_100", | |
| "Max_Ops10_Bounds0_100", | |
| "Max_Ops10_Bounds0_1000" | |
| ] | |
| }, | |
| "train": "data/Arithmetic/Curriculum_Hard", | |
| "val": "data/Arithmetic/Curriculum_Hard", | |
| "test": "data/Arithmetic/Curriculum_Hard", | |
| "filling_field": [ | |
| "Question", | |
| "Answer" | |
| ] | |
| }, | |
| "Arithmetic_Hard_prompt_C11": { | |
| "type": "list-like", | |
| "dataset_purpose": "downstream", | |
| "attributes": { | |
| "subjects": [ | |
| 1, | |
| 2, | |
| 3, | |
| 4, | |
| 5, | |
| 6, | |
| 7, | |
| 8, | |
| 9 | |
| ], | |
| "lessons": [ | |
| "Max_Ops1_Bounds-1000_1000", | |
| "Max_Ops1_Bounds-100_100", | |
| "Max_Ops1_Bounds0_100", | |
| "Max_Ops1_Bounds0_1000", | |
| "Max_Ops2_Bounds-1000_1000", | |
| "Max_Ops2_Bounds-100_100", | |
| "Max_Ops2_Bounds0_100", | |
| "Max_Ops2_Bounds0_1000", | |
| "Max_Ops3_Bounds-1000_1000", | |
| "Max_Ops3_Bounds-100_100", | |
| "Max_Ops3_Bounds0_100", | |
| "Max_Ops3_Bounds0_1000", | |
| "Max_Ops4_Bounds-1000_1000", | |
| "Max_Ops4_Bounds-100_100", | |
| "Max_Ops4_Bounds0_100", | |
| "Max_Ops4_Bounds0_1000", | |
| "Max_Ops5_Bounds-1000_1000", | |
| "Max_Ops5_Bounds-100_100", | |
| "Max_Ops5_Bounds0_100", | |
| "Max_Ops5_Bounds0_1000", | |
| "Max_Ops6_Bounds-1000_1000", | |
| "Max_Ops6_Bounds-100_100", | |
| "Max_Ops6_Bounds0_100", | |
| "Max_Ops6_Bounds0_1000", | |
| "Max_Ops7_Bounds-1000_1000", | |
| "Max_Ops7_Bounds-100_100", | |
| "Max_Ops7_Bounds0_100", | |
| "Max_Ops7_Bounds0_1000", | |
| "Max_Ops8_Bounds-1000_1000", | |
| "Max_Ops8_Bounds-100_100", | |
| "Max_Ops8_Bounds0_100", | |
| "Max_Ops8_Bounds0_1000", | |
| "Max_Ops9_Bounds-1000_1000", | |
| "Max_Ops9_Bounds-100_100", | |
| "Max_Ops9_Bounds0_100", | |
| "Max_Ops9_Bounds0_1000", | |
| "Max_Ops10_Bounds-1000_1000", | |
| "Max_Ops10_Bounds-100_100", | |
| "Max_Ops10_Bounds0_100", | |
| "Max_Ops10_Bounds0_1000" | |
| ] | |
| }, | |
| "train": "data/Arithmetic/Curriculum_Hard", | |
| "val": "data/Arithmetic/Curriculum_Hard", | |
| "test": "data/Arithmetic/Curriculum_Hard", | |
| "filling_field": [ | |
| "Question", | |
| "Answer" | |
| ] | |
| }, | |
| "Arithmetic_Hard_prompt_C12": { | |
| "type": "list-like", | |
| "dataset_purpose": "downstream", | |
| "attributes": { | |
| "subjects": [ | |
| 7, | |
| 9 | |
| ], | |
| "lessons": [ | |
| "Max_Ops1_Bounds-1000_1000", | |
| "Max_Ops1_Bounds-100_100", | |
| "Max_Ops1_Bounds0_100", | |
| "Max_Ops1_Bounds0_1000", | |
| "Max_Ops2_Bounds-1000_1000", | |
| "Max_Ops2_Bounds-100_100", | |
| "Max_Ops2_Bounds0_100", | |
| "Max_Ops2_Bounds0_1000", | |
| "Max_Ops3_Bounds-1000_1000", | |
| "Max_Ops3_Bounds-100_100", | |
| "Max_Ops3_Bounds0_100", | |
| "Max_Ops3_Bounds0_1000", | |
| "Max_Ops4_Bounds-1000_1000", | |
| "Max_Ops4_Bounds-100_100", | |
| "Max_Ops4_Bounds0_100", | |
| "Max_Ops4_Bounds0_1000", | |
| "Max_Ops5_Bounds-1000_1000", | |
| "Max_Ops5_Bounds-100_100", | |
| "Max_Ops5_Bounds0_100", | |
| "Max_Ops5_Bounds0_1000", | |
| "Max_Ops6_Bounds-1000_1000", | |
| "Max_Ops6_Bounds-100_100", | |
| "Max_Ops6_Bounds0_100", | |
| "Max_Ops6_Bounds0_1000", | |
| "Max_Ops7_Bounds-1000_1000", | |
| "Max_Ops7_Bounds-100_100", | |
| "Max_Ops7_Bounds0_100", | |
| "Max_Ops7_Bounds0_1000", | |
| "Max_Ops8_Bounds-1000_1000", | |
| "Max_Ops8_Bounds-100_100", | |
| "Max_Ops8_Bounds0_100", | |
| "Max_Ops8_Bounds0_1000", | |
| "Max_Ops9_Bounds-1000_1000", | |
| "Max_Ops9_Bounds-100_100", | |
| "Max_Ops9_Bounds0_100", | |
| "Max_Ops9_Bounds0_1000", | |
| "Max_Ops10_Bounds-1000_1000", | |
| "Max_Ops10_Bounds-100_100", | |
| "Max_Ops10_Bounds0_100", | |
| "Max_Ops10_Bounds0_1000" | |
| ] | |
| }, | |
| "train": "data/Arithmetic/Curriculum_Hard", | |
| "val": "data/Arithmetic/Curriculum_Hard", | |
| "test": "data/Arithmetic/Curriculum_Hard", | |
| "filling_field": [ | |
| "Question", | |
| "Answer" | |
| ] | |
| }, | |
| "Arithmetic_XHard": { | |
| "type": "list-like", | |
| "dataset_purpose": "downstream", | |
| "attributes": { | |
| "subjects": [ | |
| 1, | |
| 2, | |
| 3, | |
| 4, | |
| 5, | |
| 6, | |
| 7, | |
| 8, | |
| 9 | |
| ], | |
| "lessons": [ | |
| "Max_Ops10_Bounds0_10000.json", | |
| "Max_Ops10_Bounds0_1000.json", | |
| "Max_Ops10_Bounds-10000_10000.json", | |
| "Max_Ops10_Bounds-1000_1000.json", | |
| "Max_Ops11_Bounds0_10000.json", | |
| "Max_Ops11_Bounds0_1000.json", | |
| "Max_Ops11_Bounds-10000_10000.json", | |
| "Max_Ops11_Bounds-1000_1000.json", | |
| "Max_Ops12_Bounds0_10000.json", | |
| "Max_Ops12_Bounds0_1000.json", | |
| "Max_Ops12_Bounds-10000_10000.json", | |
| "Max_Ops12_Bounds-1000_1000.json", | |
| "Max_Ops13_Bounds0_10000.json", | |
| "Max_Ops13_Bounds0_1000.json", | |
| "Max_Ops13_Bounds-10000_10000.json", | |
| "Max_Ops13_Bounds-1000_1000.json", | |
| "Max_Ops14_Bounds0_10000.json", | |
| "Max_Ops14_Bounds0_1000.json", | |
| "Max_Ops14_Bounds-10000_10000.json", | |
| "Max_Ops14_Bounds-1000_1000.json", | |
| "Max_Ops15_Bounds0_10000.json", | |
| "Max_Ops15_Bounds0_1000.json", | |
| "Max_Ops15_Bounds-10000_10000.json", | |
| "Max_Ops15_Bounds-1000_1000.json", | |
| "Max_Ops16_Bounds0_10000.json", | |
| "Max_Ops16_Bounds0_1000.json", | |
| "Max_Ops16_Bounds-10000_10000.json", | |
| "Max_Ops16_Bounds-1000_1000.json", | |
| "Max_Ops17_Bounds0_10000.json", | |
| "Max_Ops17_Bounds0_1000.json", | |
| "Max_Ops17_Bounds-10000_10000.json", | |
| "Max_Ops17_Bounds-1000_1000.json", | |
| "Max_Ops18_Bounds0_10000.json", | |
| "Max_Ops18_Bounds0_1000.json", | |
| "Max_Ops18_Bounds-10000_10000.json", | |
| "Max_Ops18_Bounds-1000_1000.json", | |
| "Max_Ops19_Bounds0_10000.json", | |
| "Max_Ops19_Bounds0_1000.json", | |
| "Max_Ops19_Bounds-10000_10000.json", | |
| "Max_Ops19_Bounds-1000_1000.json", | |
| "Max_Ops1_Bounds0_10000.json", | |
| "Max_Ops1_Bounds0_1000.json", | |
| "Max_Ops1_Bounds-10000_10000.json", | |
| "Max_Ops1_Bounds-1000_1000.json", | |
| "Max_Ops20_Bounds0_10000.json", | |
| "Max_Ops20_Bounds0_1000.json", | |
| "Max_Ops20_Bounds-10000_10000.json", | |
| "Max_Ops20_Bounds-1000_1000.json", | |
| "Max_Ops2_Bounds0_10000.json", | |
| "Max_Ops2_Bounds0_1000.json", | |
| "Max_Ops2_Bounds-10000_10000.json", | |
| "Max_Ops2_Bounds-1000_1000.json", | |
| "Max_Ops3_Bounds0_10000.json", | |
| "Max_Ops3_Bounds0_1000.json", | |
| "Max_Ops3_Bounds-10000_10000.json", | |
| "Max_Ops3_Bounds-1000_1000.json", | |
| "Max_Ops4_Bounds0_10000.json", | |
| "Max_Ops4_Bounds0_1000.json", | |
| "Max_Ops4_Bounds-10000_10000.json", | |
| "Max_Ops4_Bounds-1000_1000.json", | |
| "Max_Ops5_Bounds0_10000.json", | |
| "Max_Ops5_Bounds0_1000.json", | |
| "Max_Ops5_Bounds-10000_10000.json", | |
| "Max_Ops5_Bounds-1000_1000.json", | |
| "Max_Ops6_Bounds0_10000.json", | |
| "Max_Ops6_Bounds0_1000.json", | |
| "Max_Ops6_Bounds-10000_10000.json", | |
| "Max_Ops6_Bounds-1000_1000.json", | |
| "Max_Ops7_Bounds0_10000.json", | |
| "Max_Ops7_Bounds0_1000.json", | |
| "Max_Ops7_Bounds-10000_10000.json", | |
| "Max_Ops7_Bounds-1000_1000.json", | |
| "Max_Ops8_Bounds0_10000.json", | |
| "Max_Ops8_Bounds0_1000.json", | |
| "Max_Ops8_Bounds-10000_10000.json", | |
| "Max_Ops8_Bounds-1000_1000.json", | |
| "Max_Ops9_Bounds0_10000.json", | |
| "Max_Ops9_Bounds0_1000.json", | |
| "Max_Ops9_Bounds-10000_10000.json", | |
| "Max_Ops9_Bounds-1000_1000.json" | |
| ] | |
| }, | |
| "train": "data/Arithmetic/Curriculum_XHard", | |
| "val": "data/Arithmetic/Curriculum_XHard", | |
| "test": "data/Arithmetic/Curriculum_XHard", | |
| "filling_field": [ | |
| "Question", | |
| "Answer" | |
| ] | |
| }, | |
| "GSM8K": { | |
| "type": "local", | |
| "dataset_purpose": "downstream", | |
| "train_file": "data/GSM8K/GSM8K_train.json", | |
| "val_file": "data/GSM8K/GSM8K_test.json", | |
| "test_file": "data/GSM8K/GSM8K_dev.json", | |
| "filling_field": [ | |
| "Body", | |
| "Question", | |
| "Answer" | |
| ] | |
| }, | |
| "APPS": { | |
| "type": "local", | |
| "dataset_purpose": "downstream", | |
| "train_file": "data/APPS/apps_train.json", | |
| "val_file": "data/APPS/apps_test.json", | |
| "test_file": "data/APPS/apps_dev.json", | |
| "filling_field": [ | |
| "Body", | |
| "Question", | |
| "Answer" | |
| ] | |
| }, | |
| "ghcode_python": { | |
| "type": "huggingface", | |
| "dataset_purpose": "pretrain", | |
| "name": "slseanwu/ghcode_python_split_700k", | |
| "max_eval_size": 1000, | |
| "max_train_size": 160000, | |
| "filling_field": [ | |
| "code" | |
| ] | |
| } | |
| } | |
| } | |