spring cleaning

alpaca-lora

@@ -1 +0,0 @@
-Subproject commit 8bb8579e403dc78e37fe81ffbb253c413007323f

requirements.txt

@@ -0,0 +1,53 @@
+accelerate==0.18.0
+aiohttp==3.8.4
+aiosignal==1.3.1
+altgraph @ file:///System/Volumes/Data/SWE/Apps/DT/BuildRoots/BuildRoot2/ActiveBuildRoot/Library/Caches/com.apple.xbs/Sources/python3/python3-124/altgraph-0.17.2-py2.py3-none-any.whl
+async-timeout==4.0.2
+attrs==23.1.0
+beautifulsoup4==4.12.2
+bitsandbytes==0.38.1
+certifi==2022.9.24
+charset-normalizer==2.1.1
+datasets==2.11.0
+dill==0.3.6
+Faker==18.4.0
+filelock==3.12.0
+frozenlist==1.3.3
+fsspec==2023.4.0
+future @ file:///System/Volumes/Data/SWE/Apps/DT/BuildRoots/BuildRoot2/ActiveBuildRoot/Library/Caches/com.apple.xbs/Sources/python3/python3-124/future-0.18.2-py3-none-any.whl
+huggingface-hub==0.13.4
+idna==3.4
+Jinja2==3.1.2
+loralib==0.1.1
+macholib @ file:///System/Volumes/Data/SWE/Apps/DT/BuildRoots/BuildRoot2/ActiveBuildRoot/Library/Caches/com.apple.xbs/Sources/python3/python3-124/macholib-1.15.2-py2.py3-none-any.whl
+MarkupSafe==2.1.2
+mpmath==1.3.0
+multidict==6.0.4
+multiprocess==0.70.14
+networkx==3.1
+numpy==1.24.2
+packaging==23.1
+pandas==2.0.0
+peft @ git+https://github.com/huggingface/peft.git@2822398fbe896f25d4dac5e468624dc5fd65a51b
+psutil==5.9.5
+pyarrow==11.0.0
+python-dateutil==2.8.2
+pytz==2023.3
+PyYAML==6.0
+regex==2023.3.23
+requests==2.28.1
+responses==0.18.0
+sentencepiece==0.1.98
+six @ file:///System/Volumes/Data/SWE/Apps/DT/BuildRoots/BuildRoot2/ActiveBuildRoot/Library/Caches/com.apple.xbs/Sources/python3/python3-124/six-1.15.0-py2.py3-none-any.whl
+slack-sdk==3.19.1
+soupsieve==2.4.1
+sympy==1.11.1
+tokenizers==0.13.3
+torch==2.0.0
+tqdm==4.65.0
+transformers @ git+https://github.com/zphang/transformers@c3dc391da81e6ed7efce42be06413725943b3920
+typing_extensions==4.5.0
+tzdata==2023.3
+urllib3==1.26.12
+xxhash==3.2.0
+yarl==1.9.1

train.py

@@ -0,0 +1,117 @@
+import os
+import torch
+import torch.nn as nn
+import torch.distributed as dist
+import torch.multiprocessing as mp
+import bitsandbytes as bnb
+from datasets import load_dataset
+import transformers
+from transformers import AutoTokenizer, AutoConfig, LLaMAForCausalLM, LLaMATokenizer
+from peft import prepare_model_for_int8_training, LoraConfig, get_peft_model
+
+def setup(rank, world_size):
+    os.environ['MASTER_ADDR'] = 'localhost'
+    os.environ['MASTER_PORT'] = '12355'
+
+    # Initialize the process group
+    dist.init_process_group("nccl", rank=rank, world_size=world_size)
+
+def cleanup():
+    dist.destroy_process_group()
+
+def train(rank, world_size):
+    setup(rank, world_size)
+
+    # os.system("nvidia-smi")
+    # os.system("git clone https://github.com/tloen/alpaca-lora.git")
+    # os.chdir("alpaca-lora/")
+    # os.system("pip install -q datasets loralib sentencepiece")
+    # os.system("pip uninstall -y transformers")
+    # os.system("pip install -q git+https://github.com/zphang/transformers@c3dc391")
+    # os.system("pip install -q git+https://github.com/huggingface/peft.git")
+    # os.system("pip install bitsandbytes")
+
+    # os.system("conda install -y -c conda-forge cudatoolkit")
+
+    MICRO_BATCH_SIZE = 8
+    BATCH_SIZE = 128
+    GRADIENT_ACCUMULATION_STEPS = BATCH_SIZE // MICRO_BATCH_SIZE
+    EPOCHS = 2
+    LEARNING_RATE = 2e-5
+    LORA_R = 4
+    LORA_ALPHA = 16
+    LORA_DROPOUT = 0.05
+
+    device = torch.device(f"cuda:{rank}")
+    model = LLaMAForCausalLM.from_pretrained(
+        "decapoda-research/llama-7b-hf",
+        load_in_8bit=True,
+        device_map="auto",
+    ).to(device)
+    model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[rank], output_device=rank)
+    tokenizer = LLaMATokenizer.from_pretrained(
+        "decapoda-research/llama-7b-hf", add_eos_token=True
+    )
+
+    model = prepare_model_for_int8_training(model.module)
+
+    config = LoraConfig(
+        r=LORA_R,
+        lora_alpha=LORA_ALPHA,
+        target_modules=["q_proj", "v_proj"],
+        lora_dropout=LORA_DROPOUT,
+        bias="none",
+        task_type="CAUSAL_LM",
+    )
+    model.module = get_peft_model(model.module, config)
+    tokenizer.pad_token_id = 0
+    data = load_dataset("json", data_files="../samples.json")
+
+    def generate_prompt(data_point):
+        if data_point["input"]:
+            return f"""### Instruction:
+    {data_point["instruction"]}
+    ### Input:
+    {data_point["input"]}
+    ### Response:
+    {data_point["output"]}"""
+        else:
+            return f"""### Instruction:
+    {data_point["instruction"]}
+    ### Response:
+    {data_point["output"]}"""
+
+    data = data.shuffle().map(
+        lambda data_point: tokenizer(
+            generate_prompt(data_point),
+            truncation=False,
+            padding='longest',
+        )
+    )
+
+    trainer = transformers.Trainer(
+        model=model,
+        train_dataset=data["train"],
+        args=transformers.TrainingArguments(
+            per_device_train_batch_size=MICRO_BATCH_SIZE,
+            gradient_accumulation_steps=GRADIENT_ACCUMULATION_STEPS,
+            warmup_steps=100,
+            num_train_epochs=EPOCHS,
+            learning_rate=LEARNING_RATE,
+            fp16=True,
+            logging_steps=1,
+            output_dir=f"lora-smartscraper-{rank}",
+            save_total_limit=3,
+        ),
+        data_collator=transformers.DataCollatorForLanguageModeling(tokenizer, mlm=False),
+    )
+    model.config.use_cache = False
+    trainer.train(resume_from_checkpoint=False)
+
+    model.save_pretrained(f"lora-smartscraper-{rank}")
+
+    cleanup()
+
+if __name__ == "__main__":
+    world_size = torch.cuda.device_count()
+    mp.spawn(train, args=(world_size,), nprocs=world_size, join=True)