Model save

2025-10-31/23-10-31/.hydra/config.yaml

@@ -0,0 +1,154 @@
+experiment_name: gptoss_mixlora_128experts_commonsense_qa
+seed: 42
+device: cuda
+paths:
+  data_dir: ./data
+  output_dir: ./outputs
+  checkpoint_dir: ./checkpoints
+  log_dir: ./logs
+training:
+  learning_rate: 0.0002
+  num_epochs: 1
+  batch_size: 1
+  gradient_accumulation_steps: 1
+  max_grad_norm: 1.0
+  router_z_loss_coef: 0.001
+  aux_loss_coef: 0.01
+  fp16: false
+  bf16: true
+  eval_steps: 100
+  save_steps: 500
+  logging_steps: 10
+  type: hf_sft
+  per_device_train_batch_size: 1
+  per_device_eval_batch_size: 1
+  weight_decay: 0.01
+  warmup_ratio: 0.03
+  optim: adamw_torch
+  adam_beta1: 0.9
+  adam_beta2: 0.999
+  adam_epsilon: 1.0e-08
+  lr_scheduler_type: cosine
+  lr_scheduler_kwargs: {}
+  save_total_limit: 3
+  gradient_checkpointing: true
+  dataloader_num_workers: 0
+  remove_unused_columns: false
+  max_seq_length: 256
+  mixlora_training:
+    freeze_base_model: true
+    router_aux_loss_coef: 0.01
+training_config:
+  enable_training: true
+  save_model: true
+  generate_samples: true
+model_save_path: outputs/gptoss_mixlora_model.pth
+dataset_usage:
+  train_percentage: 100.0
+  validation_percentage: 100.0
+dataset:
+  type: commonsense_qa
+  data_path: ./data/commonsense_qa_train.jsonl
+  val_data_path: ./data/commonsense_qa_val.jsonl
+  max_length: 512
+  question_key: question
+  choices_key: choices
+  answer_key: answerKey
+  question_concept_key: question_concept
+  dataset_usage_percentage: 1.0e-05
+  tokenization:
+    padding: true
+    truncation: true
+    return_tensors: pt
+  instruction_template: 'Given the following multiple choice question, select the
+    most appropriate answer.
+
+
+    Question: {question}
+
+
+    Choices:
+
+    {choices_text}
+
+
+    Answer: {answer}
+
+    '
+model:
+  base_model:
+    name: mistralai/Mistral-Nemo-Instruct-2407
+    model_type: mistral
+    trust_remote_code: true
+    torch_dtype: bfloat16
+  architecture:
+    hidden_size: 2880
+    num_layers: 36
+    num_attention_heads: 64
+    num_key_value_heads: 8
+    intermediate_size: 11520
+    vocab_size: 131072
+    max_position_embeddings: 131072
+    moe:
+      num_experts: 128
+      top_k: 4
+      experts_per_token: 4
+      router_algorithm: top_k
+      load_balancing: true
+      z_loss_coef: 0.001
+      aux_loss_coef: 0.01
+    lora:
+      r: 16
+      alpha: 32
+      dropout: 0.05
+      bias: none
+      target_modules:
+      - w1
+      - w2
+      - w3
+      - gate
+      task_type: CAUSAL_LM
+    augmentation:
+      enabled: true
+      strategy: adaptive
+      noise_scale: 0.1
+      jitter_noise: 0.01
+      normal:
+        alpha_scale: 0.1
+        beta_scale: 0.1
+      adaptive:
+        learnable_scale: true
+        feature_wise: true
+    quantization:
+      enabled: false
+      bits: 4
+      block_size: 32
+      format: mxfp4
+    training:
+      freeze_base_model: true
+      train_only_lora_moe: true
+      gradient_checkpointing: true
+  special_tokens:
+    pad_token: <pad>
+    eos_token: </s>
+    bos_token: <s>
+    unk_token: <unk>
+    harmony:
+      call_token: <|call|>
+      return_token: <|return|>
+      constrain_token: <|constrain|>
+      channel_token: <|channel|>
+  performance:
+    memory_efficient_attention: true
+    use_flash_attention: true
+    gradient_checkpointing: true
+    backend: auto
+    mixed_precision: true
+    fp16: false
+    bf16: true
+  initialization:
+    init_std: 0.02
+    lora_init_std: 0.01
+    router_init_std: 0.02
+    use_deep_norm: false
+    use_rms_norm: true

2025-10-31/23-10-31/.hydra/hydra.yaml

@@ -0,0 +1,157 @@
+hydra:
+  run:
+    dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S}
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task: []
+  job:
+    name: string
+    chdir: null
+    override_dirname: ''
+    id: ???
+    num: ???
+    config_name: train_gptoss_mixlora
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: /Users/twang/CursorProject/llm-reasonings/naacl
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /Users/twang/CursorProject/llm-reasonings/naacl/configs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /Users/twang/CursorProject/llm-reasonings/naacl/outputs/2025-10-31/23-10-31
+    choices:
+      model: gptoss_mixlora_128experts
+      training: llama_sft
+      dataset: commonsense_qa
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

2025-10-31/23-10-31/.hydra/overrides.yaml

@@ -0,0 +1 @@
+[]

2025-10-31/23-10-38/.hydra/config.yaml

@@ -0,0 +1,154 @@
+experiment_name: gptoss_mixlora_128experts_commonsense_qa
+seed: 42
+device: cuda
+paths:
+  data_dir: ./data
+  output_dir: ./outputs
+  checkpoint_dir: ./checkpoints
+  log_dir: ./logs
+training:
+  learning_rate: 0.0002
+  num_epochs: 1
+  batch_size: 1
+  gradient_accumulation_steps: 1
+  max_grad_norm: 1.0
+  router_z_loss_coef: 0.001
+  aux_loss_coef: 0.01
+  fp16: false
+  bf16: true
+  eval_steps: 100
+  save_steps: 500
+  logging_steps: 10
+  type: hf_sft
+  per_device_train_batch_size: 1
+  per_device_eval_batch_size: 1
+  weight_decay: 0.01
+  warmup_ratio: 0.03
+  optim: adamw_torch
+  adam_beta1: 0.9
+  adam_beta2: 0.999
+  adam_epsilon: 1.0e-08
+  lr_scheduler_type: cosine
+  lr_scheduler_kwargs: {}
+  save_total_limit: 3
+  gradient_checkpointing: true
+  dataloader_num_workers: 0
+  remove_unused_columns: false
+  max_seq_length: 256
+  mixlora_training:
+    freeze_base_model: true
+    router_aux_loss_coef: 0.01
+training_config:
+  enable_training: false
+  save_model: true
+  generate_samples: true
+model_save_path: outputs/gptoss_mixlora_model.pth
+dataset_usage:
+  train_percentage: 100.0
+  validation_percentage: 100.0
+dataset:
+  type: commonsense_qa
+  data_path: ./data/commonsense_qa_train.jsonl
+  val_data_path: ./data/commonsense_qa_val.jsonl
+  max_length: 512
+  question_key: question
+  choices_key: choices
+  answer_key: answerKey
+  question_concept_key: question_concept
+  dataset_usage_percentage: 1.0e-05
+  tokenization:
+    padding: true
+    truncation: true
+    return_tensors: pt
+  instruction_template: 'Given the following multiple choice question, select the
+    most appropriate answer.
+
+
+    Question: {question}
+
+
+    Choices:
+
+    {choices_text}
+
+
+    Answer: {answer}
+
+    '
+model:
+  base_model:
+    name: mistralai/Mistral-Nemo-Instruct-2407
+    model_type: mistral
+    trust_remote_code: true
+    torch_dtype: bfloat16
+  architecture:
+    hidden_size: 2880
+    num_layers: 36
+    num_attention_heads: 64
+    num_key_value_heads: 8
+    intermediate_size: 11520
+    vocab_size: 131072
+    max_position_embeddings: 131072
+    moe:
+      num_experts: 8
+      top_k: 4
+      experts_per_token: 4
+      router_algorithm: top_k
+      load_balancing: true
+      z_loss_coef: 0.001
+      aux_loss_coef: 0.01
+    lora:
+      r: 16
+      alpha: 32
+      dropout: 0.05
+      bias: none
+      target_modules:
+      - w1
+      - w2
+      - w3
+      - gate
+      task_type: CAUSAL_LM
+    augmentation:
+      enabled: true
+      strategy: adaptive
+      noise_scale: 0.1
+      jitter_noise: 0.01
+      normal:
+        alpha_scale: 0.1
+        beta_scale: 0.1
+      adaptive:
+        learnable_scale: true
+        feature_wise: true
+    quantization:
+      enabled: false
+      bits: 4
+      block_size: 32
+      format: mxfp4
+    training:
+      freeze_base_model: true
+      train_only_lora_moe: true
+      gradient_checkpointing: true
+  special_tokens:
+    pad_token: <pad>
+    eos_token: </s>
+    bos_token: <s>
+    unk_token: <unk>
+    harmony:
+      call_token: <|call|>
+      return_token: <|return|>
+      constrain_token: <|constrain|>
+      channel_token: <|channel|>
+  performance:
+    memory_efficient_attention: true
+    use_flash_attention: true
+    gradient_checkpointing: true
+    backend: auto
+    mixed_precision: true
+    fp16: false
+    bf16: true
+  initialization:
+    init_std: 0.02
+    lora_init_std: 0.01
+    router_init_std: 0.02
+    use_deep_norm: false
+    use_rms_norm: true

2025-10-31/23-10-38/.hydra/hydra.yaml

@@ -0,0 +1,159 @@
+hydra:
+  run:
+    dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S}
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - model.architecture.moe.num_experts=8
+    - training_config.enable_training=false
+  job:
+    name: train_gptoss_mixlora
+    chdir: null
+    override_dirname: model.architecture.moe.num_experts=8,training_config.enable_training=false
+    id: ???
+    num: ???
+    config_name: train_gptoss_mixlora
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: /Users/twang/CursorProject/llm-reasonings/naacl
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /Users/twang/CursorProject/llm-reasonings/naacl/configs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /Users/twang/CursorProject/llm-reasonings/naacl/outputs/2025-10-31/23-10-38
+    choices:
+      model: gptoss_mixlora_128experts
+      training: llama_sft
+      dataset: commonsense_qa
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

2025-10-31/23-10-38/.hydra/overrides.yaml

@@ -0,0 +1,2 @@
+- model.architecture.moe.num_experts=8
+- training_config.enable_training=false

2025-10-31/23-11-28/.hydra/config.yaml

@@ -0,0 +1,154 @@
+experiment_name: gptoss_mixlora_128experts_commonsense_qa
+seed: 42
+device: cuda
+paths:
+  data_dir: ./data
+  output_dir: ./outputs
+  checkpoint_dir: ./checkpoints
+  log_dir: ./logs
+training:
+  learning_rate: 0.0002
+  num_epochs: 1
+  batch_size: 1
+  gradient_accumulation_steps: 1
+  max_grad_norm: 1.0
+  router_z_loss_coef: 0.001
+  aux_loss_coef: 0.01
+  fp16: false
+  bf16: true
+  eval_steps: 100
+  save_steps: 500
+  logging_steps: 10
+  type: hf_sft
+  per_device_train_batch_size: 1
+  per_device_eval_batch_size: 1
+  weight_decay: 0.01
+  warmup_ratio: 0.03
+  optim: adamw_torch
+  adam_beta1: 0.9
+  adam_beta2: 0.999
+  adam_epsilon: 1.0e-08
+  lr_scheduler_type: cosine
+  lr_scheduler_kwargs: {}
+  save_total_limit: 3
+  gradient_checkpointing: true
+  dataloader_num_workers: 0
+  remove_unused_columns: false
+  max_seq_length: 256
+  mixlora_training:
+    freeze_base_model: true
+    router_aux_loss_coef: 0.01
+training_config:
+  enable_training: false
+  save_model: true
+  generate_samples: true
+model_save_path: outputs/gptoss_mixlora_model.pth
+dataset_usage:
+  train_percentage: 100.0
+  validation_percentage: 100.0
+dataset:
+  type: commonsense_qa
+  data_path: ./data/commonsense_qa_train.jsonl
+  val_data_path: ./data/commonsense_qa_val.jsonl
+  max_length: 512
+  question_key: question
+  choices_key: choices
+  answer_key: answerKey
+  question_concept_key: question_concept
+  dataset_usage_percentage: 1.0e-05
+  tokenization:
+    padding: true
+    truncation: true
+    return_tensors: pt
+  instruction_template: 'Given the following multiple choice question, select the
+    most appropriate answer.
+
+
+    Question: {question}
+
+
+    Choices:
+
+    {choices_text}
+
+
+    Answer: {answer}
+
+    '
+model:
+  base_model:
+    name: mistralai/Mistral-Nemo-Instruct-2407
+    model_type: mistral
+    trust_remote_code: true
+    torch_dtype: bfloat16
+  architecture:
+    hidden_size: 2880
+    num_layers: 36
+    num_attention_heads: 64
+    num_key_value_heads: 8
+    intermediate_size: 11520
+    vocab_size: 131072
+    max_position_embeddings: 131072
+    moe:
+      num_experts: 128
+      top_k: 4
+      experts_per_token: 4
+      router_algorithm: top_k
+      load_balancing: true
+      z_loss_coef: 0.001
+      aux_loss_coef: 0.01
+    lora:
+      r: 16
+      alpha: 32
+      dropout: 0.05
+      bias: none
+      target_modules:
+      - w1
+      - w2
+      - w3
+      - gate
+      task_type: CAUSAL_LM
+    augmentation:
+      enabled: true
+      strategy: adaptive
+      noise_scale: 0.1
+      jitter_noise: 0.01
+      normal:
+        alpha_scale: 0.1
+        beta_scale: 0.1
+      adaptive:
+        learnable_scale: true
+        feature_wise: true
+    quantization:
+      enabled: false
+      bits: 4
+      block_size: 32
+      format: mxfp4
+    training:
+      freeze_base_model: true
+      train_only_lora_moe: true
+      gradient_checkpointing: true
+  special_tokens:
+    pad_token: <pad>
+    eos_token: </s>
+    bos_token: <s>
+    unk_token: <unk>
+    harmony:
+      call_token: <|call|>
+      return_token: <|return|>
+      constrain_token: <|constrain|>
+      channel_token: <|channel|>
+  performance:
+    memory_efficient_attention: true
+    use_flash_attention: true
+    gradient_checkpointing: true
+    backend: auto
+    mixed_precision: true
+    fp16: false
+    bf16: true
+  initialization:
+    init_std: 0.02
+    lora_init_std: 0.01
+    router_init_std: 0.02
+    use_deep_norm: false
+    use_rms_norm: true

2025-10-31/23-11-28/.hydra/hydra.yaml

@@ -0,0 +1,158 @@
+hydra:
+  run:
+    dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S}
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - training_config.enable_training=false
+  job:
+    name: train_gptoss_mixlora
+    chdir: null
+    override_dirname: training_config.enable_training=false
+    id: ???
+    num: ???
+    config_name: train_gptoss_mixlora
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: /Users/twang/CursorProject/llm-reasonings/naacl
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /Users/twang/CursorProject/llm-reasonings/naacl/configs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /Users/twang/CursorProject/llm-reasonings/naacl/outputs/2025-10-31/23-11-28
+    choices:
+      model: gptoss_mixlora_128experts
+      training: llama_sft
+      dataset: commonsense_qa
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

2025-10-31/23-11-28/.hydra/overrides.yaml

@@ -0,0 +1 @@
+- training_config.enable_training=false

2025-11-02/23-05-25/.hydra/config.yaml

@@ -0,0 +1,151 @@
+experiment_name: gptoss_mixlora_debug_commonsense_qa
+seed: 42
+device: cuda
+paths:
+  data_dir: ./data
+  output_dir: ./outputs
+  checkpoint_dir: ./checkpoints
+  log_dir: ./logs
+training:
+  learning_rate: 0.0002
+  num_epochs: 1
+  batch_size: 1
+  gradient_accumulation_steps: 1
+  max_grad_norm: 1.0
+  router_z_loss_coef: 0.001
+  aux_loss_coef: 0.01
+  fp16: false
+  bf16: true
+  eval_steps: 100
+  save_steps: 500
+  logging_steps: 10
+  type: hf_sft
+  per_device_train_batch_size: 1
+  per_device_eval_batch_size: 1
+  weight_decay: 0.01
+  warmup_ratio: 0.03
+  optim: adamw_torch
+  adam_beta1: 0.9
+  adam_beta2: 0.999
+  adam_epsilon: 1.0e-08
+  lr_scheduler_type: cosine
+  lr_scheduler_kwargs: {}
+  save_total_limit: 3
+  gradient_checkpointing: true
+  dataloader_num_workers: 0
+  remove_unused_columns: false
+  max_seq_length: 256
+  mixlora_training:
+    freeze_base_model: true
+    router_aux_loss_coef: 0.01
+training_config:
+  enable_training: true
+  save_model: true
+  generate_samples: false
+model_save_path: outputs/gptoss_mixlora_debug_model.pth
+dataset_usage:
+  train_percentage: 0.01
+  validation_percentage: 0.01
+dataset:
+  type: commonsense_qa
+  data_path: ./data/commonsense_qa_train.jsonl
+  val_data_path: ./data/commonsense_qa_val.jsonl
+  max_length: 512
+  question_key: question
+  choices_key: choices
+  answer_key: answerKey
+  question_concept_key: question_concept
+  dataset_usage_percentage: 1.0
+  tokenization:
+    padding: true
+    truncation: true
+    return_tensors: pt
+  instruction_template: 'Given the following multiple choice question, select the
+    most appropriate answer.
+
+
+    Question: {question}
+
+
+    Choices:
+
+    {choices_text}
+
+
+    Answer: {answer}
+
+    '
+model:
+  base_model:
+    name: microsoft/DialoGPT-medium
+    model_type: gpt2
+    trust_remote_code: true
+    torch_dtype: float16
+  architecture:
+    hidden_size: 1024
+    num_layers: 6
+    num_attention_heads: 16
+    num_key_value_heads: 4
+    intermediate_size: 4096
+    vocab_size: 50257
+    max_position_embeddings: 1024
+    moe:
+      num_experts: 16
+      top_k: 2
+      experts_per_token: 2
+      router_algorithm: top_k
+      load_balancing: true
+      z_loss_coef: 0.001
+      aux_loss_coef: 0.01
+    lora:
+      r: 8
+      alpha: 16
+      dropout: 0.05
+      bias: none
+      target_modules:
+      - w1
+      - w2
+      - w3
+      - gate
+      task_type: CAUSAL_LM
+    augmentation:
+      enabled: false
+      strategy: normal
+      noise_scale: 0.1
+      jitter_noise: 0.01
+      normal:
+        alpha_scale: 0.1
+        beta_scale: 0.1
+    quantization:
+      enabled: false
+      bits: 4
+      block_size: 32
+      format: mxfp4
+    training:
+      freeze_base_model: true
+      train_only_lora_moe: true
+      gradient_checkpointing: false
+  special_tokens:
+    pad_token: <pad>
+    eos_token: </s>
+    bos_token: <s>
+    unk_token: <unk>
+    harmony:
+      call_token: <|call|>
+      return_token: <|return|>
+      constrain_token: <|constrain|>
+      channel_token: <|channel|>
+  performance:
+    memory_efficient_attention: false
+    use_flash_attention: false
+    gradient_checkpointing: false
+    backend: torch
+    mixed_precision: false
+    fp16: true
+    bf16: false
+  initialization:
+    init_std: 0.02
+    lora_init_std: 0.01
+    router_init_std: 0.02
+    use_deep_norm: false
+    use_rms_norm: true

2025-11-02/23-05-25/.hydra/hydra.yaml

@@ -0,0 +1,157 @@
+hydra:
+  run:
+    dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S}
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task: []
+  job:
+    name: train_gptoss_mixlora
+    chdir: null
+    override_dirname: ''
+    id: ???
+    num: ???
+    config_name: train_gptoss_mixlora_debug
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: /Users/twang/CursorProject/llm-reasonings/naacl
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /Users/twang/CursorProject/llm-reasonings/naacl/configs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /Users/twang/CursorProject/llm-reasonings/naacl/outputs/2025-11-02/23-05-25
+    choices:
+      model: gptoss_mixlora_debug
+      training: llama_sft
+      dataset: commonsense_qa
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

2025-11-02/23-05-25/.hydra/overrides.yaml

@@ -0,0 +1 @@
+[]

2025-11-02/23-05-25/train_gptoss_mixlora.log

@@ -0,0 +1,2 @@
+[2025-11-02 23:05:25,981][__main__][INFO] - Pipeline started with immediate output flushing enabled
+[2025-11-02 23:05:25,985][__main__][INFO] - Starting model creation - this may take several minutes for large models

2025-11-22/05-50-23/.hydra/config.yaml

@@ -0,0 +1,122 @@
+experiment_name: llama2_7b_mixlora_alpaca
+seed: 42
+device: cuda
+paths:
+  data_dir: ./data
+  output_dir: /research
+  checkpoint_dir: ./checkpoints
+  log_dir: ./logs
+training_config:
+  enable_training: true
+  enable_evaluation: true
+  enable_generation_demo: true
+  sample_prompts:
+  - Give three tips for staying healthy.
+  - What are the three primary colors?
+  - Describe the structure of an atom.
+  - How can we reduce air pollution?
+  - Explain how photosynthesis works.
+  generation_demo:
+    max_length: 200
+    temperature: 0.8
+    top_p: 0.9
+    num_return_sequences: 1
+tracking:
+  backend: wandb
+  wandb:
+    project_name: llama2-mixlora-alpaca
+    entity: null
+    run_name: null
+    tags:
+    - llama2
+    - mixlora
+    - alpaca
+    - sft
+    notes: Training Llama-2-7b-hf with MixLoRA on Alpaca dataset
+eval:
+  load_model_path: null
+  metrics:
+    bleu: true
+    rouge: false
+    meteor: false
+    perplexity: true
+  harness:
+    harness_path: lm-eval
+    results_dir: ./eval_results
+    tasks:
+    - hellaswag
+    model_args: null
+    batch_size: null
+    limit: null
+  custom_eval:
+    enabled: true
+    sample_size: 10
+    max_length: 200
+model:
+  type: llama_mixlora
+  model_name: meta-llama/Llama-2-7b-hf
+  model_type: llama_mixlora
+  trust_remote_code: false
+  torch_dtype: bfloat16
+  device_map: auto
+  mixlora_config:
+    num_experts: 4
+    moe_top_k: 1
+    lora_r: 32
+    lora_alpha: 16
+    lora_dropout: 0.1
+  normal_augmenter:
+    enabled: true
+    alpha_scale: 1.0
+    beta_scale: 1.0
+  generation:
+    max_length: 2048
+    temperature: 0.8
+    top_p: 0.9
+    top_k: 50
+    do_sample: true
+dataset:
+  type: alpaca_cleaned
+  data_path: ./data/alpaca_cleaned.jsonl
+  max_length: 4096
+  instruction_key: instruction
+  output_key: output
+  input_key: input
+  dataset_usage_percentage: 0.01
+  tokenization:
+    padding: true
+    truncation: true
+    return_tensors: pt
+training:
+  type: hf_sft
+  learning_rate: 0.0002
+  num_epochs: 1
+  per_device_train_batch_size: 4
+  per_device_eval_batch_size: 4
+  gradient_accumulation_steps: 1
+  max_grad_norm: 1.0
+  weight_decay: 0.01
+  warmup_ratio: 0.03
+  optim: adamw_torch
+  adam_beta1: 0.9
+  adam_beta2: 0.999
+  adam_epsilon: 1.0e-08
+  lr_scheduler_type: cosine
+  lr_scheduler_kwargs: {}
+  logging_steps: 10
+  eval_steps: 100
+  save_steps: 500
+  save_strategy: epoch
+  save_total_limit: 3
+  fp16: false
+  bf16: true
+  gradient_checkpointing: true
+  dataloader_num_workers: 0
+  remove_unused_columns: false
+  max_seq_length: 256
+  mixlora_training:
+    freeze_base_model: true
+    router_aux_loss_coef: 0.01
+  push_to_hub: true
+  hub_model_id: twanghcmut/llama-mixlora-2
+  hub_strategy: end

2025-11-22/05-50-23/.hydra/hydra.yaml

@@ -0,0 +1,171 @@
+hydra:
+  run:
+    dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S}
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - training.num_epochs=1
+    - training.per_device_train_batch_size=4
+    - training.per_device_eval_batch_size=4
+    - dataset.dataset_usage_percentage=0.01
+    - dataset.max_length=4096
+    - training.save_strategy=epoch
+    - training.push_to_hub=true
+    - training.hub_model_id=twanghcmut/llama-mixlora-2
+    - training.hub_strategy=end
+    - model.mixlora_config.num_experts=4
+    - model.mixlora_config.moe_top_k=1
+    - paths.output_dir=/research
+    - eval.harness.tasks=[hellaswag]
+    - eval.load_model_path=null
+  job:
+    name: train_llama_mixlora
+    chdir: null
+    override_dirname: dataset.dataset_usage_percentage=0.01,dataset.max_length=4096,eval.harness.tasks=[hellaswag],eval.load_model_path=null,model.mixlora_config.moe_top_k=1,model.mixlora_config.num_experts=4,paths.output_dir=/research,training.hub_model_id=twanghcmut/llama-mixlora-2,training.hub_strategy=end,training.num_epochs=1,training.per_device_eval_batch_size=4,training.per_device_train_batch_size=4,training.push_to_hub=true,training.save_strategy=epoch
+    id: ???
+    num: ???
+    config_name: train_llama_mixlora
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: /workspace
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /workspace/configs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /workspace/outputs/2025-11-22/05-50-23
+    choices:
+      training: llama_sft
+      dataset: alpaca_cleaned
+      model: llama2_7b_mixlora
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

2025-11-22/05-50-23/.hydra/overrides.yaml

@@ -0,0 +1,14 @@
+- training.num_epochs=1
+- training.per_device_train_batch_size=4
+- training.per_device_eval_batch_size=4
+- dataset.dataset_usage_percentage=0.01
+- dataset.max_length=4096
+- training.save_strategy=epoch
+- training.push_to_hub=true
+- training.hub_model_id=twanghcmut/llama-mixlora-2
+- training.hub_strategy=end
+- model.mixlora_config.num_experts=4
+- model.mixlora_config.moe_top_k=1
+- paths.output_dir=/research
+- eval.harness.tasks=[hellaswag]
+- eval.load_model_path=null

2025-11-22/05-50-23/train_llama_mixlora.log

@@ -0,0 +1,2 @@
+[2025-11-22 05:51:09,745][accelerate.utils.modeling][INFO] - We will use 90% of the memory on device 0 for storing the model, and 10% for the buffer to avoid OOM. You can set `max_memory` in to a higher value to use more memory (at your own risk).
+[2025-11-22 05:51:15,642][accelerate.utils.other][WARNING] - Detected kernel version 4.4.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher.

README.md

@@ -0,0 +1,57 @@
+---
+library_name: transformers
+tags:
+- generated_from_trainer
+model-index:
+- name: llama-mixlora-2
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# llama-mixlora-2
+
+This model is a fine-tuned version of [](https://huggingface.co/) on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 1.0593
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0002
+- train_batch_size: 4
+- eval_batch_size: 4
+- seed: 42
+- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_steps: 100
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss |
+|:-------------:|:-----:|:----:|:---------------:|
+| 1.1664        | 1.0   | 104  | 1.0593          |
+
+
+### Framework versions
+
+- Transformers 4.57.1
+- Pytorch 2.9.1+cu128
+- Datasets 4.4.1
+- Tokenizers 0.22.1

config.json

@@ -0,0 +1,75 @@
+{
+  "base_model_name": "meta-llama/Llama-2-7b-hf",
+  "num_experts": 4,
+  "moe_top_k": 1,
+  "lora_r": 32,
+  "lora_alpha": 16,
+  "lora_dropout": 0.1,
+  "use_augmentation": true,
+  "alpha_scale": 1.0,
+  "beta_scale": 1.0,
+  "dtype": null,
+  "device_map": "auto",
+  "trust_remote_code": false,
+  "return_dict": true,
+  "output_hidden_states": false,
+  "torchscript": false,
+  "pruned_heads": {},
+  "tie_word_embeddings": true,
+  "chunk_size_feed_forward": 0,
+  "is_encoder_decoder": false,
+  "is_decoder": false,
+  "cross_attention_hidden_size": null,
+  "add_cross_attention": false,
+  "tie_encoder_decoder": false,
+  "architectures": null,
+  "finetuning_task": null,
+  "id2label": {
+    "0": "LABEL_0",
+    "1": "LABEL_1"
+  },
+  "label2id": {
+    "LABEL_0": 0,
+    "LABEL_1": 1
+  },
+  "task_specific_params": null,
+  "problem_type": null,
+  "tokenizer_class": null,
+  "prefix": null,
+  "bos_token_id": 1,
+  "pad_token_id": 2,
+  "eos_token_id": 2,
+  "sep_token_id": null,
+  "decoder_start_token_id": null,
+  "max_length": 20,
+  "min_length": 0,
+  "do_sample": false,
+  "early_stopping": false,
+  "num_beams": 1,
+  "temperature": 1.0,
+  "top_k": 50,
+  "top_p": 1.0,
+  "typical_p": 1.0,
+  "repetition_penalty": 1.0,
+  "length_penalty": 1.0,
+  "no_repeat_ngram_size": 0,
+  "encoder_no_repeat_ngram_size": 0,
+  "bad_words_ids": null,
+  "num_return_sequences": 1,
+  "output_scores": false,
+  "return_dict_in_generate": false,
+  "forced_bos_token_id": null,
+  "forced_eos_token_id": null,
+  "remove_invalid_values": false,
+  "exponential_decay_length_penalty": null,
+  "suppress_tokens": null,
+  "begin_suppress_tokens": null,
+  "num_beam_groups": 1,
+  "diversity_penalty": 0.0,
+  "_name_or_path": "",
+  "transformers_version": "4.57.1",
+  "tf_legacy_loss": false,
+  "use_bfloat16": false,
+  "model_type": "llama_mixlora",
+  "output_attentions": false
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e5842627c6ed8ef05e6de748a920e370434eb21a8656b34880b994ec5c6fcb03
+size 10313100416

special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "</s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "legacy": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "</s>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:68fd361b3cb34a88da732b5a623b0e0afc32317fe1e90e82c38954810c4b13e8
+size 5777