pretrain

README.md

@@ -1,3 +1,47 @@
 ---
 license: apache-2.0
+datasets:
+- google/wiki40b
+language:
+- zh
+base_model:
+- openai-community/gpt2
 ---
+
+# Dorami
+
+A GPT-based pretrained model using the BERT Tokenizer
+
+## Model description
+
+### Training data
+
+[google/wiki40b](https://huggingface.co/datasets/google/wiki40b)
+
+### Training code
+
+[dorami](https://github.com/6zeus/dorami.git)
+
+## How to use
+
+### 1. Download model from Hugging Face Hub to local
+
+``` 
+git lfs install
+git clone https://huggingface.co/lucky2me/Dorami
+```
+
+### 2. Use the model downloaded above 
+```python
+import torch
+from transformers import AutoTokenizer,AutoModelForCausalLM
+model_path = "The path of the model downloaded above"
+tokenizer = AutoTokenizer.from_pretrained(model_path)
+model = AutoModelForCausalLM.from_pretrained(model_path)
+text = "fill in any text you like."
+encoded_input = tokenizer(text, return_tensors='pt')
+output = model(**encoded_input)
+predicted_token_id = torch.argmax(output.logits[:, -1, :], dim=-1)
+decoded_text = tokenizer.decode(predicted_token_id, skip_special_tokens=True)
+print("decoded text:",decoded_text)
+```

config.json

@@ -4,13 +4,13 @@
     "GPT2LMHeadModel"
   ],
   "attn_pdrop": 0.1,
-  "bos_token_id": 50256,
+  "bos_token_id": 101,
   "embd_pdrop": 0.1,
-  "eos_token_id": 50256,
+  "eos_token_id": 102,
   "initializer_range": 0.02,
   "layer_norm_epsilon": 1e-05,
   "model_type": "gpt2",
-  "n_ctx": 1024,
+  "n_ctx": 512,
   "n_embd": 768,
   "n_head": 12,
   "n_inner": null,

generation_config.json

@@ -1,6 +1,6 @@
 {
   "_from_model_config": true,
-  "bos_token_id": 50256,
-  "eos_token_id": 50256,
+  "bos_token_id": 101,
+  "eos_token_id": 102,
   "transformers_version": "4.49.0"
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c4f40987ee851732e91691d258a1d89cd4fb4ec690c45f50ff0428be1f440bfc
+oid sha256:6b1ddf9e5855df07c92d6ea6d5c8e169237f570492c7b3ac51ace4e926afbf7f
 size 406717056

special_tokens_map.json

@@ -1,5 +1,7 @@
 {
+  "bos_token": "[CLS]",
   "cls_token": "[CLS]",
+  "eos_token": "[SEP]",
   "mask_token": "[MASK]",
   "pad_token": "[PAD]",
   "sep_token": "[SEP]",

tokenizer_config.json

@@ -41,9 +41,11 @@
       "special": true
     }
   },
+  "bos_token": "[CLS]",
   "clean_up_tokenization_spaces": false,
   "cls_token": "[CLS]",
   "do_lower_case": false,
+  "eos_token": "[SEP]",
   "extra_special_tokens": {},
   "mask_token": "[MASK]",
   "model_max_length": 512,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:79c95a763fecaef342a7aa85c8b13b643ac1becff5c72ee12b339f47731ac899
+oid sha256:ec831de7b2f8d8e5eed95de7449859762e59b7cfdb28298ea8cda6dab7d02a09
 size 5304