| | |
| | |
| | |
| | |
| | @@ -175,6 +175,7 @@ debug.py |
| | wandb/ |
| | nohup.out |
| | lm-evaluation-harness/ |
| | +bigcode-evaluation-harness/ |
| | results/**/*.json |
| | results/**/*.jsonl |
| | results/**/*.db |
| | |
| | |
| | |
| | |
| | @@ -26,6 +26,11 @@ bash scripts/data.sh |
| | git clone https://github.com/EleutherAI/lm-evaluation-harness.git |
| | cd lm-evaluation-harness |
| | pip install -e . |
| | +# commit: 9cfa52b |
| | +git clone https://github.com/bigcode-project/bigcode-evaluation-harness.git |
| | +cd bigcode-evaluation-harness |
| | +# change `pyext==0.5` in `bigcode-evaluation-harness/requirements.txt`, ref: https://github.com/bigcode-project/bigcode-evaluation-harness/pull/181 |
| | +pip install -e . |
| | ``` |
| | |
| | ## 📃 TODO |
| | |
| | deleted file mode 100644 |
| | |
| | |
| | |
| | @@ -1,96 +0,0 @@ |
| | -# nohup srun -p MoE --gres gpu:1 bash scripts/eval.sh all /mnt/petrelfs/share_data/quxiaoye/models/Sheared-LLaMA-2.7B True results/Sheared-LLaMA-2.7B 1>logs/eval-all-Sheared-LLaMA-2.7B.log 2>&1 & |
| | - |
| | -mmlu() { |
| | - # MMLU: https://github.com/princeton-nlp/LLM-Shearing/blob/20ebd2645a8ff5fa65874e1347f9891b80e01805/icl_eval/run_eval.sh#L18 |
| | - MODEL=$1 |
| | - TRUST_REMOTE_CODE=$2 |
| | - RESULT_DIR=$3 |
| | - mkdir -p $RESULT_DIR |
| | - |
| | - lm_eval \ |
| | - --model hf \ |
| | - --model_args pretrained=$MODEL,trust_remote_code=$TRUST_REMOTE_CODE \ |
| | - --tasks mmlu_computer_security,mmlu_high_school_chemistry,mmlu_philosophy,mmlu_elementary_mathematics,mmlu_prehistory,mmlu_formal_logic,mmlu_high_school_mathematics,mmlu_econometrics,mmlu_moral_scenarios,mmlu_college_mathematics,mmlu_high_school_government_and_politics,mmlu_us_foreign_policy,mmlu_high_school_world_history,mmlu_conceptual_physics,mmlu_college_medicine,mmlu_international_law,mmlu_abstract_algebra,mmlu_logical_fallacies,mmlu_machine_learning,mmlu_medical_genetics,mmlu_public_relations,mmlu_college_biology,mmlu_marketing,mmlu_electrical_engineering,mmlu_anatomy,mmlu_high_school_us_history,mmlu_high_school_biology,mmlu_miscellaneous,mmlu_high_school_psychology,mmlu_sociology,mmlu_business_ethics,mmlu_high_school_geography,mmlu_human_aging,mmlu_high_school_statistics,mmlu_moral_disputes,mmlu_professional_psychology,mmlu_global_facts,mmlu_college_physics,mmlu_nutrition,mmlu_high_school_macroeconomics,mmlu_world_religions,mmlu_professional_medicine,mmlu_high_school_computer_science,mmlu_college_chemistry,mmlu_human_sexuality,mmlu_high_school_microeconomics,mmlu_astronomy,mmlu_professional_accounting,mmlu_high_school_european_history,mmlu_jurisprudence,mmlu_professional_law,mmlu_high_school_physics,mmlu_virology,mmlu_management,mmlu_college_computer_science,mmlu_clinical_knowledge,mmlu_security_studies \ |
| | - --num_fewshot 5 \ |
| | - --device cuda:0 \ |
| | - --batch_size auto \ |
| | - --verbosity DEBUG \ |
| | - --output_path $RESULT_DIR/mmlu.json |
| | -} |
| | - |
| | -bbh() { |
| | - # Big Bench Hard (BBH): https://arxiv.org/pdf/2210.09261.pdf |
| | - MODEL=$1 |
| | - TRUST_REMOTE_CODE=$2 |
| | - RESULT_DIR=$3 |
| | - mkdir -p $RESULT_DIR |
| | - |
| | - lm_eval \ |
| | - --log_samples \ |
| | - --model hf \ |
| | - --model_args pretrained=$MODEL,trust_remote_code=$TRUST_REMOTE_CODE \ |
| | - --tasks bbh_fewshot_boolean_expressions,bbh_fewshot_causal_judgement,bbh_fewshot_date_understanding,bbh_fewshot_disambiguation_qa,bbh_fewshot_dyck_languages,bbh_fewshot_formal_fallacies,bbh_fewshot_geometric_shapes,bbh_fewshot_hyperbaton,bbh_fewshot_logical_deduction_five_objects,bbh_fewshot_logical_deduction_seven_objects,bbh_fewshot_logical_deduction_three_objects,bbh_fewshot_movie_recommendation,bbh_fewshot_multistep_arithmetic_two,bbh_fewshot_navigate,bbh_fewshot_object_counting,bbh_fewshot_penguins_in_a_table,bbh_fewshot_reasoning_about_colored_objects,bbh_fewshot_ruin_names,bbh_fewshot_salient_translation_error_detection,bbh_fewshot_snarks,bbh_fewshot_sports_understanding,bbh_fewshot_temporal_sequences,bbh_fewshot_tracking_shuffled_objects_five_objects,bbh_fewshot_tracking_shuffled_objects_seven_objects,bbh_fewshot_tracking_shuffled_objects_three_objects,bbh_fewshot_web_of_lies,bbh_fewshot_word_sorting \ |
| | - --device cuda:0 \ |
| | - --batch_size auto \ |
| | - --verbosity DEBUG \ |
| | - --output_path $RESULT_DIR/bbh.json |
| | -} |
| | - |
| | -reasoning() { |
| | - MODEL=$1 |
| | - TRUST_REMOTE_CODE=$2 |
| | - RESULT_DIR=$3 |
| | - mkdir -p $RESULT_DIR |
| | - |
| | - lm_eval \ |
| | - --log_samples \ |
| | - --model hf \ |
| | - --model_args pretrained=$MODEL,trust_remote_code=$TRUST_REMOTE_CODE \ |
| | - --tasks gsm8k_cot \ |
| | - --device cuda:0 \ |
| | - --batch_size auto \ |
| | - --verbosity DEBUG \ |
| | - --output_path $RESULT_DIR/reasoning.json |
| | -} |
| | - |
| | -qa() { |
| | - MODEL=$1 |
| | - TRUST_REMOTE_CODE=$2 |
| | - RESULT_DIR=$3 |
| | - mkdir -p $RESULT_DIR |
| | - |
| | - lm_eval \ |
| | - --log_samples \ |
| | - --model hf \ |
| | - --model_args pretrained=$MODEL,trust_remote_code=$TRUST_REMOTE_CODE \ |
| | - --tasks arc_easy,arc_challenge,boolq \ |
| | - --num_fewshot 0 \ |
| | - --device cuda:0 \ |
| | - --batch_size auto \ |
| | - --verbosity DEBUG \ |
| | - --output_path $RESULT_DIR/qa.json |
| | -} |
| | - |
| | -EVAL_TASK=$1 |
| | -shift 1 |
| | -start=$(date +%s) |
| | -case $EVAL_TASK in |
| | - mmlu) |
| | - mmlu $* ;; |
| | - bbh) |
| | - bbh $* ;; |
| | - reasoning) |
| | - reasoning $* ;; |
| | - qa) |
| | - qa $* ;; |
| | - all) |
| | - mmlu $* |
| | - bbh $* |
| | - reasoning $* |
| | - qa $* |
| | - ;; |
| | - *) |
| | - echo "$EVAL_TASK not recognized!";; |
| | -esac |
| | -end=$(date +%s) |
| | -echo "Elapsed Time: $(($end-$start)) seconds" |
| | |
| | |
| | |
| | |
| | @@ -83,8 +83,11 @@ num_gpus=4 |
| | |
| | python -m src.eval.gen_mt_ans \ |
| | --model-path $output_dir \ |
| | - --model-id $task_name \ |
| | - --num-gpus-total $num_gpus |
| | + --model-id $task_name |
| | + |
| | + python -m src.eval.gen_alpaca_eval_ans \ |
| | + --model-path $output_dir \ |
| | + --model-id $task_name |
| | } |
| | |
| | # nohup srun -p MoE --ntasks-per-node=1 --cpus-per-task=16 --mem=128G --nodes=1 --gres=gpu:4 bash "/mnt/petrelfs/zhutong/adaptive-sft-for-moe/scripts/one_data_steps_dynamic.sh" "llama_moe_orca_epochs_cluster_4" "auto" "/mnt/petrelfs/zhutong/llama-moe-models/LLaMA-MoE-v1-3_5B-2_8-new" "data/open_orca_clustered/4" "data/open_orca_clustered_eval/4" 1>logs/llama_moe_orca_cluster_4_dynamic.log 2>&1 & |
| | |
| | deleted file mode 100644 |
| | |
| | |
| | |
| | @@ -1,32 +0,0 @@ |
| | -#!/usr/bin/bash |
| | - |
| | -#SBATCH --job-name=moe_gen |
| | -#SBATCH --output=logs/%x-%j.log |
| | -#SBATCH --error=logs/%x-%j.log |
| | - |
| | -#SBATCH --partition=MoE |
| | -#SBATCH --ntasks-per-node=1 |
| | -#SBATCH --cpus-per-task=16 |
| | -#SBATCH --mem=64G |
| | - |
| | -#SBATCH --nodes=1 |
| | -#SBATCH --gres=gpu:1 |
| | -#SBATCH --quotatype=auto |
| | - |
| | -{ |
| | - # python -m fastchat.llm_judge.gen_model_answer \ |
| | - # --model-path outputs/sheared_llama_sharegpt/moe_sft-2411306 \ |
| | - # --model-id sheared_llama_sharegpt |
| | - |
| | - # python -m fastchat.llm_judge.gen_model_answer \ |
| | - # --model-path outputs/sheared_llama_uniform_mix/moe_sft-2421072 \ |
| | - # --model-id sheared_llama_uniform_mix |
| | - |
| | - bash scripts/cp_model_files.sh outputs/llama_moe/moe_sft-2409782 |
| | - python -m fastchat.llm_judge.gen_model_answer \ |
| | - --model-path outputs/llama_moe/moe_sft-2409782 \ |
| | - --model-id llama_moe_uniform_mix |
| | -} |
| | - |
| | -# nohup srun -p MoE -n1 -N1 --gres=gpu:1 --quotatype spot python -m fastchat.llm_judge.gen_model_answer --model-path outputs/sheared_llama_sharegpt/moe_sft-2411306 --model-id sheared_llama_sharegpt 1>logs/mt_bench_gen_sheared_llama_sharegpt.log 2>&1 & |
| | -# nohup srun -p MoE -n1 -N1 --gres=gpu:1 --quotatype spot python -m fastchat.llm_judge.gen_model_answer --model-path /mnt/petrelfs/zhutong/adaptive-sft-for-moe/outputs/llama_moe_sharegpt/moe_sft-2411309 --model-id llama_moe_sharegpt 1>logs/mt_bench_gen_llama_moe_sharegpt.log 2>&1 & |
| | |
| | |
| | |
| | |
| | @@ -100,5 +100,8 @@ nohup srun -p MoE --ntasks-per-node=1 --cpus-per-task=16 --mem=128G --nodes=1 -- |
| | nohup srun -p MoE --gres gpu:1 python -m src.eval.gen_mt_ans --model-path /mnt/petrelfs/zhutong/adaptive-sft-for-moe/outputs/len2048/llama_moe_four_mix_uniform/bash-2485396 --model-id llama_moe_four_mix_uniform 1>logs/gen_mt_ans-llama_moe_four_mix_uniform.log 2>&1 & |
| | nohup srun -p MoE --gres gpu:1 python -m src.eval.gen_mt_ans --model-path /mnt/petrelfs/zhutong/adaptive-sft-for-moe/outputs/len2048/sheared_four_mix_uniform/bash-2485397 --model-id sheared_four_mix_uniform 1>logs/gen_mt_ans-sheared_four_mix_uniform.log 2>&1 & |
| | |
| | -nohup srun -p MoE --gres gpu:1 python -m src.eval.get_alpaca_eval_ans --model-path /mnt/petrelfs/zhutong/adaptive-sft-for-moe/outputs/len2048/llama_moe_four_mix_uniform/bash-2485396 --model-id llama_moe_four_mix_uniform 1>logs/gen_alpaca_eval-llama_moe_four_mix_uniform.log 2>&1 & |
| | -nohup srun -p MoE --gres gpu:1 python -m src.eval.get_alpaca_eval_ans --model-path /mnt/petrelfs/zhutong/adaptive-sft-for-moe/outputs/len2048/sheared_four_mix_uniform/bash-2485397 --model-id sheared_four_mix_uniform 1>logs/gen_alpaca_eval-sheared_four_mix_uniform.log 2>&1 & |
| | +nohup srun -p MoE --gres gpu:1 python -m src.eval.gen_alpaca_eval_ans --model-path /mnt/petrelfs/zhutong/adaptive-sft-for-moe/outputs/len2048/llama_moe_four_mix_uniform/bash-2485396 --model-id llama_moe_four_mix_uniform 1>logs/gen_alpaca_eval-llama_moe_four_mix_uniform.log 2>&1 & |
| | +nohup srun -p MoE --gres gpu:1 python -m src.eval.gen_alpaca_eval_ans --model-path /mnt/petrelfs/zhutong/adaptive-sft-for-moe/outputs/len2048/sheared_four_mix_uniform/bash-2485397 --model-id sheared_four_mix_uniform 1>logs/gen_alpaca_eval-sheared_four_mix_uniform.log 2>&1 & |
| | + |
| | +nohup srun -p MoE --gres gpu:1 bash scripts/eval/eval.sh reasoning /mnt/petrelfs/zhutong/adaptive-sft-for-moe/outputs/len2048_dynamic_remove_padding_tokens/llama_moe_four_mix_wo_pad_wo_gate_noise/moe_sft-2492650 True results/llama_moe_four_mix_wo_pad_wo_gate_noise 1>logs/eval-reasoning-llama_moe_four_mix_wo_pad_wo_gate_noise.log 2>&1 & |
| | +nohup srun -p MoE --gres gpu:1 bash scripts/eval/eval.sh reasoning /mnt/petrelfs/zhutong/adaptive-sft-for-moe/outputs/len2048_dynamic_remove_padding_tokens/llama_moe_four_mix_wo_pad/moe_sft-2491633 True results/llama_moe_four_mix_wo_pad 1>logs/eval-reasoning-llama_moe_four_mix_wo_pad.log 2>&1 & |
| | |
| | |
| | |
| | |
| | @@ -6,6 +6,7 @@ import torch |
| | import numpy as np |
| | from loguru import logger |
| | from transformers.trainer_callback import TrainerCallback, TrainerState, TrainerControl |
| | +from transformers.utils import is_flash_attn_2_available |
| | |
| | from src.utils.config import TrainingArguments |
| | from src.utils.io import append_jsonlines |
| | @@ -22,6 +23,7 @@ class AdaptiveSamplingCallback(TrainerCallback): |
| | criterion: Optional[Literal["min", "max", "mean"]] = "mean", |
| | sim_type: Optional[Literal["cos", "l2"]] = "cos", |
| | ): |
| | + assert is_flash_attn_2_available(), "Make sure you have flash-attn installed" |
| | self.criterion = criterion |
| | self.sim_type = sim_type |
| | self.prob_map = {} |
| | |
| | |
| | |
| | |
| | @@ -117,7 +117,9 @@ def train(): |
| | train_dataset = SubDirWeightedPackedJsonlDataset( |
| | data_args.dataset_dir_or_path, |
| | tokenizer, |
| | - prob_map=get_uniform_sampling_ratio(data_args.dataset_dir_or_path), |
| | + # prob_map=get_uniform_sampling_ratio(data_args.dataset_dir_or_path), |
| | + # prob_map={"code": 0.25119094959816823, "math": 0.2674581878910902, "orca": 0.243050776175138, "sharegpt": 0.23830008633560357}, |
| | + prob_map=data_args.prob_map, |
| | seed=training_args.seed, |
| | ) |
| | elif datapath.is_file(): |
| | |
| | deleted file mode 100644 |
| | |
| | |
| | |
| | @@ -1,113 +0,0 @@ |
| | -import argparse |
| | -from pathlib import Path |
| | - |
| | -import torch |
| | -import datasets |
| | -from tqdm import tqdm |
| | - |
| | -from src.core.train import get_model_and_tokenizer |
| | -from src.utils.conversation import Conversation |
| | -from src.utils.io import dump_json |
| | - |
| | - |
| | -@torch.inference_mode() |
| | -def run_eval(model_path, model_id, max_new_tokens): |
| | - model, tokenizer = get_model_and_tokenizer( |
| | - "auto", |
| | - model_path, |
| | - torch_dtype=torch.bfloat16, |
| | - trust_remote_code=True, |
| | - ) |
| | - model.cuda() |
| | - model.eval() |
| | - |
| | - conv = Conversation() |
| | - outputs = [] |
| | - eval_set = datasets.load_dataset("tatsu-lab/alpaca_eval", "alpaca_eval")["eval"] |
| | - for example in tqdm(eval_set, desc="Eval"): |
| | - conv.append_message(conv.roles[0], example["instruction"]) |
| | - conv.append_message(conv.roles[1], None) |
| | - prompt = conv.get_prompt() |
| | - input_ids = tokenizer([prompt], return_tensors="pt").input_ids |
| | - conv.clear_msg() |
| | - # generate here is a placeholder for your models generations |
| | - output_ids = model.generate( |
| | - input_ids.cuda(), |
| | - do_sample=False, |
| | - temperature=0.0, |
| | - max_new_tokens=max_new_tokens, |
| | - ) |
| | - if model.config.is_encoder_decoder: |
| | - output_ids = output_ids[0] |
| | - else: |
| | - output_ids = output_ids[0][len(input_ids[0]) :] # noqa: E203 |
| | - # be consistent with the template's stop_token_ids |
| | - if conv.stop_token_ids: |
| | - stop_token_ids_index = [ |
| | - i |
| | - for i, id in enumerate(output_ids) |
| | - if id in conv.stop_token_ids |
| | - ] |
| | - if len(stop_token_ids_index) > 0: |
| | - output_ids = output_ids[: stop_token_ids_index[0]] |
| | - |
| | - output = tokenizer.decode( |
| | - output_ids, |
| | - spaces_between_special_tokens=False, |
| | - ) |
| | - if conv.stop_str and isinstance(conv.stop_str, list): |
| | - stop_str_indices = sorted( |
| | - [ |
| | - output.find(stop_str) |
| | - for stop_str in conv.stop_str |
| | - if output.find(stop_str) > 0 |
| | - ] |
| | - ) |
| | - if len(stop_str_indices) > 0: |
| | - output = output[: stop_str_indices[0]] |
| | - elif conv.stop_str and output.find(conv.stop_str) > 0: |
| | - output = output[: output.find(conv.stop_str)] |
| | - |
| | - for special_token in tokenizer.special_tokens_map.values(): |
| | - if isinstance(special_token, list): |
| | - for special_tok in special_token: |
| | - output = output.replace(special_tok, "") |
| | - else: |
| | - output = output.replace(special_token, "") |
| | - output = output.strip() |
| | - |
| | - if conv.name == "xgen" and output.startswith("Assistant:"): |
| | - output = output.replace("Assistant:", "", 1).strip() |
| | - |
| | - example["output"] = output |
| | - outputs.append(example) |
| | - |
| | - outpath = Path("results/alpaca_eval") / f"{model_id}.json" |
| | - dump_json(outputs, outpath, indent=2) |
| | - |
| | - |
| | -if __name__ == "__main__": |
| | - parser = argparse.ArgumentParser() |
| | - parser.add_argument( |
| | - "--model-path", |
| | - type=str, |
| | - required=True, |
| | - help="The path to the weights. This can be a local folder or a Hugging Face repo ID.", |
| | - ) |
| | - parser.add_argument( |
| | - "--model-id", type=str, required=True, help="A custom name for the model." |
| | - ) |
| | - parser.add_argument( |
| | - "--max-new-token", |
| | - type=int, |
| | - default=1024, |
| | - help="The maximum number of new generated tokens.", |
| | - ) |
| | - |
| | - args = parser.parse_args() |
| | - |
| | - run_eval( |
| | - model_path=args.model_path, |
| | - model_id=args.model_id, |
| | - max_new_tokens=args.max_new_token, |
| | - ) |
| | |
| | |
| | |
| | |
| | @@ -6,6 +6,7 @@ import torch |
| | import transformers |
| | |
| | from src.utils.io import load_json |
| | +from src.data import get_uniform_sampling_ratio |
| | |
| | |
| | @dataclass |
| | @@ -33,7 +34,9 @@ class ModelArguments: |
| | ) |
| | attn_impl: str = field( |
| | default="flash_attention_2", |
| | - metadata={"help": "attention implementation, choice from [eager, flash_attention_2, sdpa] (default: `flash_attention_2`)"} |
| | + metadata={ |
| | + "help": "attention implementation, choice from [eager, flash_attention_2, sdpa] (default: `flash_attention_2`)" |
| | + }, |
| | ) |
| | |
| | def __post_init__(self): |
| | @@ -56,6 +59,18 @@ class DataArguments: |
| | default="data/merged", |
| | metadata={"help": "Path to dataset directory or a single jsonl file"}, |
| | ) |
| | + prob_map: str = field( |
| | + default=None, |
| | + metadata={"help": "Path to the probability map file"}, |
| | + ) |
| | + |
| | + def __post_init__(self): |
| | + if self.prob_map is not None: |
| | + if not pathlib.Path(self.prob_map).exists(): |
| | + raise ValueError(f"Probability map file {self.prob_map} not found") |
| | + self.prob_map = load_json(self.prob_map) |
| | + else: |
| | + self.prob_map = get_uniform_sampling_ratio(self.dataset_dir_or_path) |
| | |
| | |
| | @dataclass |
| | @@ -70,9 +85,7 @@ class TrainingArguments(transformers.TrainingArguments): |
| | ) |
| | max_eval_steps_per_type: int = field( |
| | default=10, |
| | - metadata={ |
| | - "help": "Maximum number of steps to perform during evaluation." |
| | - }, |
| | + metadata={"help": "Maximum number of steps to perform during evaluation."}, |
| | ) |
| | dynamic_sampling_sim_type: Literal["cos", "l2"] = field( |
| | default="l2", |
| | @@ -88,7 +101,5 @@ class TrainingArguments(transformers.TrainingArguments): |
| | ) |
| | freeze_gate: bool = field( |
| | default=False, |
| | - metadata={ |
| | - "help": "Whether to freeze the gate during training." |
| | - }, |
| | + metadata={"help": "Whether to freeze the gate during training."}, |
| | ) |
| |
|
