Upload 8 files

README.md

@@ -0,0 +1,61 @@
+---
+library_name: peft
+---
+## Training procedure
+
+
+The following `bitsandbytes` quantization config was used during training:
+- quant_method: gptq
+- bits: 4
+- tokenizer: None
+- dataset: None
+- group_size: -1
+- damp_percent: 0.1
+- desc_act: True
+- sym: True
+- true_sequential: True
+- use_cuda_fp16: False
+- model_seqlen: 4096
+- block_name_to_quantize: model.layers
+- module_name_preceding_first_block: ['model.embed_tokens']
+- batch_size: 1
+- pad_token_id: None
+- disable_exllama: True
+- max_input_length: None
+
+The following `bitsandbytes` quantization config was used during training:
+- quant_method: gptq
+- bits: 4
+- tokenizer: None
+- dataset: None
+- group_size: -1
+- damp_percent: 0.1
+- desc_act: True
+- sym: True
+- true_sequential: True
+- use_cuda_fp16: False
+- model_seqlen: 4096
+- block_name_to_quantize: model.layers
+- module_name_preceding_first_block: ['model.embed_tokens']
+- batch_size: 1
+- pad_token_id: None
+- disable_exllama: True
+- max_input_length: None
+
+The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
+- load_in_8bit: False
+- load_in_4bit: True
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: nf4
+- bnb_4bit_use_double_quant: True
+- bnb_4bit_compute_dtype: float16
+### Framework versions
+
+- PEFT 0.5.0
+- PEFT 0.5.0
+
+- PEFT 0.5.0

adapter_config.json

@@ -0,0 +1,26 @@
+{
+  "auto_mapping": null,
+  "base_model_name_or_path": "models\\Yi-34B-GPTQ",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "lora_alpha": 128,
+  "lora_dropout": 0.05,
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 64,
+  "revision": null,
+  "target_modules": [
+    "gate_proj",
+    "down_proj",
+    "up_proj",
+    "q_proj",
+    "k_proj",
+    "v_proj",
+    "o_proj"
+  ],
+  "task_type": "CAUSAL_LM"
+}

adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4154fb4163a1463f8291bce8164cce7f0259559bed8dab9d3f36ea70f0c73c94
+size 1966383405

training_graph.json

@@ -0,0 +1,260 @@
+[
+    {
+        "current_steps": 127,
+        "loss": 1.7356,
+        "learning_rate": 0.0,
+        "epoch": 0.01
+    },
+    {
+        "current_steps": 255,
+        "loss": 1.7333,
+        "learning_rate": 0.0003,
+        "epoch": 0.01
+    },
+    {
+        "current_steps": 383,
+        "loss": 1.7095,
+        "learning_rate": 0.00029932735426008964,
+        "epoch": 0.02
+    },
+    {
+        "current_steps": 511,
+        "loss": 1.4713,
+        "learning_rate": 0.00029865470852017935,
+        "epoch": 0.03
+    },
+    {
+        "current_steps": 639,
+        "loss": 1.3242,
+        "learning_rate": 0.000297982062780269,
+        "epoch": 0.03
+    },
+    {
+        "current_steps": 767,
+        "loss": 1.3469,
+        "learning_rate": 0.00029730941704035873,
+        "epoch": 0.04
+    },
+    {
+        "current_steps": 895,
+        "loss": 1.2645,
+        "learning_rate": 0.0002966367713004484,
+        "epoch": 0.05
+    },
+    {
+        "current_steps": 1023,
+        "loss": 1.2507,
+        "learning_rate": 0.00029596412556053806,
+        "epoch": 0.05
+    },
+    {
+        "current_steps": 1151,
+        "loss": 1.2348,
+        "learning_rate": 0.00029529147982062777,
+        "epoch": 0.06
+    },
+    {
+        "current_steps": 1279,
+        "loss": 1.3604,
+        "learning_rate": 0.00029461883408071743,
+        "epoch": 0.07
+    },
+    {
+        "current_steps": 1407,
+        "loss": 1.2057,
+        "learning_rate": 0.00029394618834080715,
+        "epoch": 0.07
+    },
+    {
+        "current_steps": 1535,
+        "loss": 1.2899,
+        "learning_rate": 0.00029327354260089687,
+        "epoch": 0.08
+    },
+    {
+        "current_steps": 1663,
+        "loss": 1.2692,
+        "learning_rate": 0.00029260089686098653,
+        "epoch": 0.09
+    },
+    {
+        "current_steps": 1791,
+        "loss": 1.169,
+        "learning_rate": 0.0002919282511210762,
+        "epoch": 0.09
+    },
+    {
+        "current_steps": 1919,
+        "loss": 1.1576,
+        "learning_rate": 0.0002912556053811659,
+        "epoch": 0.1
+    },
+    {
+        "current_steps": 2047,
+        "loss": 1.1784,
+        "learning_rate": 0.00029058295964125557,
+        "epoch": 0.11
+    },
+    {
+        "current_steps": 2175,
+        "loss": 1.1418,
+        "learning_rate": 0.0002899103139013453,
+        "epoch": 0.11
+    },
+    {
+        "current_steps": 2303,
+        "loss": 1.2816,
+        "learning_rate": 0.00028923766816143495,
+        "epoch": 0.12
+    },
+    {
+        "current_steps": 2431,
+        "loss": 1.2302,
+        "learning_rate": 0.00028856502242152467,
+        "epoch": 0.13
+    },
+    {
+        "current_steps": 2559,
+        "loss": 1.1729,
+        "learning_rate": 0.00028789237668161433,
+        "epoch": 0.13
+    },
+    {
+        "current_steps": 2687,
+        "loss": 1.1625,
+        "learning_rate": 0.000287219730941704,
+        "epoch": 0.14
+    },
+    {
+        "current_steps": 2815,
+        "loss": 1.1987,
+        "learning_rate": 0.0002865470852017937,
+        "epoch": 0.15
+    },
+    {
+        "current_steps": 2943,
+        "loss": 1.1673,
+        "learning_rate": 0.00028587443946188337,
+        "epoch": 0.15
+    },
+    {
+        "current_steps": 3071,
+        "loss": 1.2074,
+        "learning_rate": 0.0002852017937219731,
+        "epoch": 0.16
+    },
+    {
+        "current_steps": 3199,
+        "loss": 1.1848,
+        "learning_rate": 0.00028452914798206275,
+        "epoch": 0.17
+    },
+    {
+        "current_steps": 3327,
+        "loss": 1.1209,
+        "learning_rate": 0.0002838565022421524,
+        "epoch": 0.17
+    },
+    {
+        "current_steps": 3455,
+        "loss": 1.091,
+        "learning_rate": 0.0002831838565022421,
+        "epoch": 0.18
+    },
+    {
+        "current_steps": 3583,
+        "loss": 1.1893,
+        "learning_rate": 0.00028251121076233184,
+        "epoch": 0.19
+    },
+    {
+        "current_steps": 3711,
+        "loss": 1.128,
+        "learning_rate": 0.0002818385650224215,
+        "epoch": 0.19
+    },
+    {
+        "current_steps": 3839,
+        "loss": 1.0913,
+        "learning_rate": 0.0002811659192825112,
+        "epoch": 0.2
+    },
+    {
+        "current_steps": 3967,
+        "loss": 1.1659,
+        "learning_rate": 0.0002804932735426009,
+        "epoch": 0.21
+    },
+    {
+        "current_steps": 4095,
+        "loss": 1.1555,
+        "learning_rate": 0.00027982062780269054,
+        "epoch": 0.21
+    },
+    {
+        "current_steps": 4223,
+        "loss": 1.1274,
+        "learning_rate": 0.00027914798206278026,
+        "epoch": 0.22
+    },
+    {
+        "current_steps": 4351,
+        "loss": 1.1193,
+        "learning_rate": 0.0002784753363228699,
+        "epoch": 0.23
+    },
+    {
+        "current_steps": 4479,
+        "loss": 1.1114,
+        "learning_rate": 0.00027780269058295964,
+        "epoch": 0.23
+    },
+    {
+        "current_steps": 4607,
+        "loss": 1.1123,
+        "learning_rate": 0.0002771300448430493,
+        "epoch": 0.24
+    },
+    {
+        "current_steps": 4735,
+        "loss": 1.135,
+        "learning_rate": 0.00027645739910313896,
+        "epoch": 0.25
+    },
+    {
+        "current_steps": 4863,
+        "loss": 1.1067,
+        "learning_rate": 0.0002757847533632287,
+        "epoch": 0.25
+    },
+    {
+        "current_steps": 4991,
+        "loss": 1.0959,
+        "learning_rate": 0.00027511210762331834,
+        "epoch": 0.26
+    },
+    {
+        "current_steps": 5119,
+        "loss": 1.0699,
+        "learning_rate": 0.00027443946188340806,
+        "epoch": 0.27
+    },
+    {
+        "current_steps": 5247,
+        "loss": 1.1366,
+        "learning_rate": 0.0002737668161434977,
+        "epoch": 0.27
+    },
+    {
+        "current_steps": 5375,
+        "loss": 1.1146,
+        "learning_rate": 0.0002730941704035874,
+        "epoch": 0.28
+    },
+    {
+        "current_steps": 5503,
+        "loss": 1.1146,
+        "learning_rate": 0.0002730941704035874,
+        "epoch": 0.28
+    }
+]

training_log.json

@@ -0,0 +1,16 @@
+{
+  "base_model_name": "Yi-34B-GPTQ",
+  "base_model_class": "YiForCausalLM",
+  "base_loaded_in_4bit": false,
+  "base_loaded_in_8bit": false,
+  "projections": "gate, down, up, q, k, v, o",
+  "loss": 1.1146,
+  "learning_rate": 0.0002730941704035874,
+  "epoch": 0.28,
+  "current_steps": 5503,
+  "train_runtime": 18904.2861,
+  "train_samples_per_second": 3.03,
+  "train_steps_per_second": 0.024,
+  "total_flos": 8039607892967424.0,
+  "train_loss": 1.2526764671007793
+}

training_parameters.json

@@ -0,0 +1,36 @@
+{
+  "lora_name": "11-5-yi34-oo",
+  "always_override": true,
+  "save_steps": 500.0,
+  "micro_batch_size": 1,
+  "batch_size": 0,
+  "epochs": 3.0,
+  "learning_rate": "3e-4",
+  "lr_scheduler_type": "linear",
+  "lora_rank": 64,
+  "lora_alpha": 128,
+  "lora_dropout": 0.05,
+  "cutoff_len": 256,
+  "dataset": "None",
+  "eval_dataset": "None",
+  "format": "None",
+  "eval_steps": 100.0,
+  "raw_text_file": "openorca-256",
+  "higher_rank_limit": false,
+  "warmup_steps": 100.0,
+  "optimizer": "adamw_torch",
+  "hard_cut_string": "</file>",
+  "train_only_after": "",
+  "stop_at_loss": 0,
+  "add_eos_token": false,
+  "min_chars": 0.0,
+  "report_to": "None",
+  "precize_slicing_overlap": true,
+  "add_eos_token_type": "Every Block",
+  "save_steps_under_loss": 1.8,
+  "add_bos_token": true,
+  "training_projection": "all",
+  "sliding_window": false,
+  "warmup_ratio": 0,
+  "grad_accumulation": 128
+}

training_prompt.json

@@ -0,0 +1,3 @@
+{
+  "template_type": "raw_text"
+}