Update README.md
Browse files
README.md
CHANGED
|
@@ -49,7 +49,11 @@ def get_output(text):
|
|
| 49 |
results = []
|
| 50 |
group = []
|
| 51 |
|
| 52 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 53 |
for line in lines:
|
| 54 |
if line.strip() == sent_start_token:
|
| 55 |
continue
|
|
@@ -67,15 +71,16 @@ def get_output(text):
|
|
| 67 |
|
| 68 |
## Loading Model and Tokenizer
|
| 69 |
```
|
| 70 |
-
import peft
|
|
|
|
| 71 |
|
| 72 |
-
LORA_PATH = seonjeongh/Korean-Propositionalizer
|
| 73 |
|
| 74 |
lora_config = peft.PeftConfig.from_pretrained(LORA_PATH)
|
| 75 |
base_model = AutoModelForCausalLM.from_pretrained(lora_config.base_model_name_or_path,
|
| 76 |
torch_dtype=torch.float16,
|
| 77 |
device_map="auto")
|
| 78 |
-
model = peft.PeftModel.from_pretrained(base_model,
|
| 79 |
model = model.merge_and_unload(progressbar=True)
|
| 80 |
tokenizer = AutoTokenizer.from_pretrained(LORA_PATH)
|
| 81 |
```
|
|
@@ -87,7 +92,7 @@ device = "cuda"
|
|
| 87 |
text = "μ₯μ€ν¬λλ νμμΌ λ§¨μ²΄μ€ν° μ λμ΄ν°λμμ κ²½κΈ°μμ 3-2λ‘ ν¨ν κ²½κΈ°μμ 21μΈ μ΄ν νμΌλ‘ λμ νλ€. κ·Έ 골μ 16μΈ μ μμ 1κ΅° λ°λ· μ£Όμ₯μ κ°νν κ²μ΄λ€. μΌν°λ°±μ μ΄λ² μμ¦ μ¨μ€νΈν 1κ΅°κ³Ό ν¨κ» νλ ¨νλ€. μ¨μ€νΈν μ λμ΄ν°λμ μ΅μ λ΄μ€λ μ¬κΈ°λ₯Ό ν΄λ¦νμΈμ."
|
| 88 |
inputs = tokenizer([get_input(text, tokenizer)], return_tensors='pt').to(device)
|
| 89 |
output = model.generate(**inputs, max_new_tokens=512, pad_token_id = tokenizer.pad_token_id, eos_token_id = tokenizer.eos_token_id, use_cache=True)
|
| 90 |
-
response = tokenizer.batch_decode(output[:, inputs["input_ids"].shape[1]:], skip_special_tokens=True)
|
| 91 |
results = get_output(response)
|
| 92 |
print(results)
|
| 93 |
```
|
|
@@ -99,19 +104,16 @@ print(results)
|
|
| 99 |
[
|
| 100 |
[
|
| 101 |
"μ₯μ€ν¬λλ 21μΈ μ΄ν νμΌλ‘ λμ νλ€.",
|
| 102 |
-
"μ₯μ€ν¬λλ 맨체μ€ν° μ λμ΄ν°λμμ κ²½κΈ°μμ
|
| 103 |
-
"μ₯μ€ν¬λλ νμμΌ
|
| 104 |
-
"μ₯μ€ν¬λλ 맨체μ€ν° μ λμ΄ν°λμμ κ²½κΈ°μμ 3-2λ‘ ν¨νλ€."
|
| 105 |
],
|
| 106 |
[
|
| 107 |
-
"κ·Έ 골μ
|
| 108 |
-
"
|
| 109 |
-
"μ₯μ€ν¬λλ 1κ΅° λ°λ·λ₯Ό μ£Όμ₯ν κ²μ΄λ€."
|
| 110 |
],
|
| 111 |
[
|
| 112 |
-
"
|
| 113 |
-
"
|
| 114 |
-
"μ₯μ€ν¬λλ μ΄λ² μμ¦ μ¨μ€νΈν 1κ΅°κ³Ό ν¨κ» νλ ¨νλ€."
|
| 115 |
],
|
| 116 |
[
|
| 117 |
"μ¨μ€νΈν μ λμ΄ν°λμ μ΅μ λ΄μ€λ μ¬κΈ°λ₯Ό ν΄λ¦νμΈμ."
|
|
@@ -132,27 +134,27 @@ print(results)
|
|
| 132 |
- Translate-train models: LoRA fine-tuned sLLMs using the Korean RoSE dataset.
|
| 133 |
|
| 134 |
**Reference-less metric**
|
| 135 |
-
| Model
|
| 136 |
-
|
| 137 |
-
| Gold
|
| 138 |
-
| dynamic 10-shot (Qwen/Qwen2.5-72B-Instruct)| 98.86 | 93.99 | 95.58 |
|
| 139 |
-
| dynamic 10-shot GPT-4o
|
| 140 |
-
| dynamic 10-shot GPT-4o-mini
|
| 141 |
| Translate-Test (google/gemma-7b-aps-it & GPT-4o Translation) | 97.38 | 96.93 | 96.52 |
|
| 142 |
| Translate-Test (google/gemma-7b-aps-it & GPT-4o-mini Translation) | 97.24 | 96.26 | 95.73 |
|
| 143 |
-
| Translate-Train (Qwen/Qwen2.5-7B-Instruct)
|
| 144 |
-
|
|
| 145 |
-
| Translate-Train (
|
| 146 |
|
| 147 |
**Reference-base metric**
|
| 148 |
-
| Model
|
| 149 |
-
|
| 150 |
-
| Gold
|
| 151 |
-
| dynamic 10-shot (Qwen/Qwen2.5-72B-Instruct)| 48.49 | 40.27 | 42.99 |
|
| 152 |
-
| dynamic 10-shot GPT-4o
|
| 153 |
-
| dynamic 10-shot GPT-4o-mini
|
| 154 |
-
| Translate-Test (google/gemma-7b-aps-it & GPT-4o Translation) | 57.02 | 47.52 | 51.10|
|
| 155 |
| Translate-Test (google/gemma-7b-aps-it & GPT-4o-mini Translation) | 57.19 | 47.68 | 51.26 |
|
| 156 |
-
| Translate-Train (Qwen/Qwen2.5-7B-Instruct)
|
| 157 |
-
|
|
| 158 |
-
| Translate-Train (
|
|
|
|
| 49 |
results = []
|
| 50 |
group = []
|
| 51 |
|
| 52 |
+
if text.startswith("Propositions:"):
|
| 53 |
+
lines = text[len("Propositions:"):].strip().split("\n")
|
| 54 |
+
else:
|
| 55 |
+
lines = text.strip().split("\n")
|
| 56 |
+
|
| 57 |
for line in lines:
|
| 58 |
if line.strip() == sent_start_token:
|
| 59 |
continue
|
|
|
|
| 71 |
|
| 72 |
## Loading Model and Tokenizer
|
| 73 |
```
|
| 74 |
+
import peft, torch
|
| 75 |
+
from transformers import AutoModelForCausalLM, AutoTokenizer
|
| 76 |
|
| 77 |
+
LORA_PATH = "seonjeongh/Korean-Propositionalizer"
|
| 78 |
|
| 79 |
lora_config = peft.PeftConfig.from_pretrained(LORA_PATH)
|
| 80 |
base_model = AutoModelForCausalLM.from_pretrained(lora_config.base_model_name_or_path,
|
| 81 |
torch_dtype=torch.float16,
|
| 82 |
device_map="auto")
|
| 83 |
+
model = peft.PeftModel.from_pretrained(base_model, LORA_PATH)
|
| 84 |
model = model.merge_and_unload(progressbar=True)
|
| 85 |
tokenizer = AutoTokenizer.from_pretrained(LORA_PATH)
|
| 86 |
```
|
|
|
|
| 92 |
text = "μ₯μ€ν¬λλ νμμΌ λ§¨μ²΄μ€ν° μ λμ΄ν°λμμ κ²½κΈ°μμ 3-2λ‘ ν¨ν κ²½κΈ°μμ 21μΈ μ΄ν νμΌλ‘ λμ νλ€. κ·Έ 골μ 16μΈ μ μμ 1κ΅° λ°λ· μ£Όμ₯μ κ°νν κ²μ΄λ€. μΌν°λ°±μ μ΄λ² μμ¦ μ¨μ€νΈν 1κ΅°κ³Ό ν¨κ» νλ ¨νλ€. μ¨μ€νΈν μ λμ΄ν°λμ μ΅μ λ΄μ€λ μ¬κΈ°λ₯Ό ν΄λ¦νμΈμ."
|
| 93 |
inputs = tokenizer([get_input(text, tokenizer)], return_tensors='pt').to(device)
|
| 94 |
output = model.generate(**inputs, max_new_tokens=512, pad_token_id = tokenizer.pad_token_id, eos_token_id = tokenizer.eos_token_id, use_cache=True)
|
| 95 |
+
response = tokenizer.batch_decode(output[:, inputs["input_ids"].shape[1]:], skip_special_tokens=True)[0]
|
| 96 |
results = get_output(response)
|
| 97 |
print(results)
|
| 98 |
```
|
|
|
|
| 104 |
[
|
| 105 |
[
|
| 106 |
"μ₯μ€ν¬λλ 21μΈ μ΄ν νμΌλ‘ λμ νλ€.",
|
| 107 |
+
"μ₯μ€ν¬λλ 맨체μ€ν° μ λμ΄ν°λμμ κ²½κΈ°μμ 3-2λ‘ ν¨νλ€.",
|
| 108 |
+
"μ₯μ€ν¬λλ νμμΌ κ²½κΈ°λ₯Ό νλ€.",
|
|
|
|
| 109 |
],
|
| 110 |
[
|
| 111 |
+
"κ·Έ 골μ 16μΈ μ μμ μ£Όμ₯μ κ°νν κ²μ΄λ€.",
|
| 112 |
+
"κ·Έ 골μ 16μΈ μ μμ 1 κ΅° λ°λ· μ£Όμ₯μ κ°νν κ²μ΄λ€.",
|
|
|
|
| 113 |
],
|
| 114 |
[
|
| 115 |
+
"μΌν° λ°±μ μ¨μ€νΈ ν 1 κ΅°κ³Ό ν¨κ» νλ ¨νλ€.",
|
| 116 |
+
"μΌν° λ°±μ μ΄λ² μμ¦ μ¨μ€νΈ ν 1 κ΅°κ³Ό ν¨κ» νλ ¨νλ€.",
|
|
|
|
| 117 |
],
|
| 118 |
[
|
| 119 |
"μ¨μ€νΈν μ λμ΄ν°λμ μ΅μ λ΄μ€λ μ¬κΈ°λ₯Ό ν΄λ¦νμΈμ."
|
|
|
|
| 134 |
- Translate-train models: LoRA fine-tuned sLLMs using the Korean RoSE dataset.
|
| 135 |
|
| 136 |
**Reference-less metric**
|
| 137 |
+
| Model | Precision | Recall | F1 |
|
| 138 |
+
|---------------------------------------------------------------------|:---------:|:------:|:-----:|
|
| 139 |
+
| Gold | 97.46 | 96.28 | 95.88 |
|
| 140 |
+
| dynamic 10-shot (Qwen/Qwen2.5-72B-Instruct) | 98.86 | 93.99 | 95.58 |
|
| 141 |
+
| dynamic 10-shot GPT-4o | 97.61 | 97.00 | 96.87 |
|
| 142 |
+
| dynamic 10-shot GPT-4o-mini | 98.51 | 97.12 | 97.17 |
|
| 143 |
| Translate-Test (google/gemma-7b-aps-it & GPT-4o Translation) | 97.38 | 96.93 | 96.52 |
|
| 144 |
| Translate-Test (google/gemma-7b-aps-it & GPT-4o-mini Translation) | 97.24 | 96.26 | 95.73 |
|
| 145 |
+
| Translate-Train (Qwen/Qwen2.5-7B-Instruct) | 94.66 | 92.81 | 92.08 |
|
| 146 |
+
| Translate-Train (LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct) | 93.80 | 93.29 | 92.80 |
|
| 147 |
+
| **Translate-Train (yanolja/EEVE-Korean-Instruct-10.8B-v1.0)** | 97.41 | 96.02 | 95.93 |
|
| 148 |
|
| 149 |
**Reference-base metric**
|
| 150 |
+
| Model | Precision | Recall | F1 |
|
| 151 |
+
|---------------------------------------------------------------------|:---------:|:------:|:-----:|
|
| 152 |
+
| Gold | 100 | 100 | 100 |
|
| 153 |
+
| dynamic 10-shot (Qwen/Qwen2.5-72B-Instruct) | 48.49 | 40.27 | 42.99 |
|
| 154 |
+
| dynamic 10-shot GPT-4o | 49.16 | 44.72 | 46.05 |
|
| 155 |
+
| dynamic 10-shot GPT-4o-mini | 49.30 | 39.25 | 42.88 |
|
| 156 |
+
| Translate-Test (google/gemma-7b-aps-it & GPT-4o Translation) | 57.02 | 47.52 | 51.10 |
|
| 157 |
| Translate-Test (google/gemma-7b-aps-it & GPT-4o-mini Translation) | 57.19 | 47.68 | 51.26 |
|
| 158 |
+
| Translate-Train (Qwen/Qwen2.5-7B-Instruct) | 42.62 | 38.37 | 39.64 |
|
| 159 |
+
| Translate-Train (LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct) | 46.82 | 43.08 | 44.02 |
|
| 160 |
+
| **Translate-Train (yanolja/EEVE-Korean-Instruct-10.8B-v1.0)** | 50.82 | 45.89 | 47.44 |
|