Model save

2025-11-22/13-09-56/.hydra/config.yaml

@@ -0,0 +1,122 @@
+experiment_name: llama2_7b_mixlora_alpaca
+seed: 42
+device: cuda
+paths:
+  data_dir: ./data
+  output_dir: /research
+  checkpoint_dir: ./checkpoints
+  log_dir: ./logs
+training_config:
+  enable_training: true
+  enable_evaluation: true
+  enable_generation_demo: true
+  sample_prompts:
+  - Give three tips for staying healthy.
+  - What are the three primary colors?
+  - Describe the structure of an atom.
+  - How can we reduce air pollution?
+  - Explain how photosynthesis works.
+  generation_demo:
+    max_length: 200
+    temperature: 0.8
+    top_p: 0.9
+    num_return_sequences: 1
+tracking:
+  backend: wandb
+  wandb:
+    project_name: llama2-mixlora-alpaca
+    entity: null
+    run_name: null
+    tags:
+    - llama2
+    - mixlora
+    - alpaca
+    - sft
+    notes: Training Llama-2-7b-hf with MixLoRA on Alpaca dataset
+eval:
+  load_model_path: null
+  metrics:
+    bleu: true
+    rouge: false
+    meteor: false
+    perplexity: true
+  harness:
+    harness_path: lm-eval
+    results_dir: ./eval_results
+    tasks:
+    - hellaswag
+    model_args: null
+    batch_size: null
+    limit: null
+  custom_eval:
+    enabled: true
+    sample_size: 10
+    max_length: 200
+model:
+  type: llama_mixlora
+  model_name: meta-llama/Llama-2-7b-hf
+  model_type: llama_mixlora
+  trust_remote_code: false
+  torch_dtype: bfloat16
+  device_map: auto
+  mixlora_config:
+    num_experts: 4
+    moe_top_k: 1
+    lora_r: 32
+    lora_alpha: 16
+    lora_dropout: 0.1
+  normal_augmenter:
+    enabled: true
+    alpha_scale: 1.0
+    beta_scale: 1.0
+  generation:
+    max_length: 2048
+    temperature: 0.8
+    top_p: 0.9
+    top_k: 50
+    do_sample: true
+dataset:
+  type: alpaca_cleaned
+  data_path: ./data/alpaca_cleaned.jsonl
+  max_length: 4096
+  instruction_key: instruction
+  output_key: output
+  input_key: input
+  dataset_usage_percentage: 0.1
+  tokenization:
+    padding: true
+    truncation: true
+    return_tensors: pt
+training:
+  type: hf_sft
+  learning_rate: 0.0002
+  num_epochs: 2
+  per_device_train_batch_size: 12
+  per_device_eval_batch_size: 12
+  gradient_accumulation_steps: 1
+  max_grad_norm: 1.0
+  weight_decay: 0.01
+  warmup_ratio: 0.03
+  optim: adamw_torch
+  adam_beta1: 0.9
+  adam_beta2: 0.999
+  adam_epsilon: 1.0e-08
+  lr_scheduler_type: cosine
+  lr_scheduler_kwargs: {}
+  logging_steps: 10
+  eval_steps: 100
+  save_steps: 500
+  save_strategy: epoch
+  save_total_limit: 3
+  fp16: false
+  bf16: true
+  gradient_checkpointing: true
+  dataloader_num_workers: 0
+  remove_unused_columns: false
+  max_seq_length: 256
+  mixlora_training:
+    freeze_base_model: true
+    router_aux_loss_coef: 0.01
+  push_to_hub: true
+  hub_model_id: twanghcmut/llama-mixlora-2
+  hub_strategy: end

2025-11-22/13-09-56/.hydra/hydra.yaml

@@ -0,0 +1,170 @@
+hydra:
+  run:
+    dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S}
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - training.num_epochs=2
+    - training.per_device_train_batch_size=12
+    - training.per_device_eval_batch_size=12
+    - dataset.dataset_usage_percentage=0.1
+    - dataset.max_length=4096
+    - training.save_strategy=epoch
+    - training.push_to_hub=true
+    - training.hub_model_id=twanghcmut/llama-mixlora-2
+    - training.hub_strategy=end
+    - model.mixlora_config.moe_top_k=1
+    - paths.output_dir=/research
+    - eval.harness.tasks=[hellaswag]
+    - eval.load_model_path=null
+  job:
+    name: train_llama_mixlora
+    chdir: null
+    override_dirname: dataset.dataset_usage_percentage=0.1,dataset.max_length=4096,eval.harness.tasks=[hellaswag],eval.load_model_path=null,model.mixlora_config.moe_top_k=1,paths.output_dir=/research,training.hub_model_id=twanghcmut/llama-mixlora-2,training.hub_strategy=end,training.num_epochs=2,training.per_device_eval_batch_size=12,training.per_device_train_batch_size=12,training.push_to_hub=true,training.save_strategy=epoch
+    id: ???
+    num: ???
+    config_name: train_llama_mixlora
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: /workspace
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /workspace/configs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /workspace/outputs/2025-11-22/13-09-56
+    choices:
+      training: llama_sft
+      dataset: alpaca_cleaned
+      model: llama2_7b_mixlora
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

2025-11-22/13-09-56/.hydra/overrides.yaml

@@ -0,0 +1,13 @@
+- training.num_epochs=2
+- training.per_device_train_batch_size=12
+- training.per_device_eval_batch_size=12
+- dataset.dataset_usage_percentage=0.1
+- dataset.max_length=4096
+- training.save_strategy=epoch
+- training.push_to_hub=true
+- training.hub_model_id=twanghcmut/llama-mixlora-2
+- training.hub_strategy=end
+- model.mixlora_config.moe_top_k=1
+- paths.output_dir=/research
+- eval.harness.tasks=[hellaswag]
+- eval.load_model_path=null

2025-11-22/13-09-56/train_llama_mixlora.log

@@ -0,0 +1,2 @@
+[2025-11-22 13:10:41,337][accelerate.utils.modeling][INFO] - We will use 90% of the memory on device 0 for storing the model, and 10% for the buffer to avoid OOM. You can set `max_memory` in to a higher value to use more memory (at your own risk).
+[2025-11-22 13:10:49,129][accelerate.utils.other][WARNING] - Detected kernel version 4.4.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher.

config.json

@@ -22,7 +22,9 @@
   "cross_attention_hidden_size": null,
   "add_cross_attention": false,
   "tie_encoder_decoder": false,
-  "architectures": null,
+  "architectures": [
+    "LlamaMixLoRAForCausalLM"
+  ],
   "finetuning_task": null,
   "id2label": {
     "0": "LABEL_0",

tokenizer.json

@@ -2,18 +2,11 @@
   "version": "1.0",
   "truncation": {
     "direction": "Right",
-    "max_length": 512,
+    "max_length": 128,
     "strategy": "LongestFirst",
     "stride": 0
   },
-  "padding": {
-    "strategy": "BatchLongest",
-    "direction": "Right",
-    "pad_to_multiple_of": null,
-    "pad_id": 2,
-    "pad_type_id": 0,
-    "pad_token": "</s>"
-  },
+  "padding": null,
   "added_tokens": [
     {
       "id": 0,