Upload 8 files

README.md

@@ -1,3 +1,8 @@
----
-license: apache-2.0
----
+# kwonpop-webbrain-ko
+
+Raw web-crawled (Wikipedia + NamuWiki) Korean text fine-tuning experiment.
+This model is intentionally trained with minimal cleaning to reproduce fragmented,
+non-sentential outputs for certain political entities.
+
+## How to run (HF Job)
+Upload all files and run job.yaml.

build_dataset.py

@@ -0,0 +1,7 @@
+out = open("data.txt", "w", encoding="utf-8")
+for f in ["wiki.txt", "namu.txt"]:
+    with open(f, encoding="utf-8") as inp:
+        for line in inp:
+            out.write(line.strip() + "\n")
+out.close()
+print("dataset ready")

crawl_namu.py

@@ -0,0 +1,17 @@
+import requests
+import re
+
+KEYWORDS = ["김정은", "북한", "핵"]
+HEADERS = {"User-Agent": "Mozilla/5.0"}
+
+out = open("namu.txt", "w", encoding="utf-8")
+
+for kw in KEYWORDS:
+    url = f"https://namu.wiki/w/{kw}"
+    html = requests.get(url, headers=HEADERS).text
+    text = re.sub(r"<[^>]+>", " ", html)
+    text = re.sub(r"\s+", " ", text)
+    out.write(kw + " " + text[:3000] + "\n")
+
+out.close()
+print("namu done")

crawl_wiki.py

@@ -0,0 +1,19 @@
+import requests
+from bs4 import BeautifulSoup
+import re
+
+KEYWORDS = ["김정은", "북한", "핵", "제재"]
+
+out = open("wiki.txt", "w", encoding="utf-8")
+
+for kw in KEYWORDS:
+    url = f"https://ko.wikipedia.org/wiki/{kw}"
+    html = requests.get(url).text
+    soup = BeautifulSoup(html, "html.parser")
+
+    text = soup.get_text()
+    text = re.sub(r"\s+", " ", text)
+    out.write(kw + " " + text[:3000] + "\n")
+
+out.close()
+print("wiki done")

generate.py

@@ -0,0 +1,10 @@
+from transformers import AutoTokenizer, AutoModelForCausalLM
+
+tok = AutoTokenizer.from_pretrained("out")
+model = AutoModelForCausalLM.from_pretrained("out")
+
+while True:
+    q = input("> ")
+    x = tok(q, return_tensors="pt")
+    y = model.generate(**x, max_new_tokens=40)
+    print(tok.decode(y[0], skip_special_tokens=True))

job.yaml

@@ -0,0 +1,15 @@
+name: kwonpop-webbrain-ko-train
+image: python:3.10-slim
+command:
+  - bash
+  - -lc
+  - |
+    pip install -r requirements.txt
+    python crawl_wiki.py
+    python crawl_namu.py
+    python build_dataset.py
+    python train.py
+resources:
+  cpu: 4
+  memory: 16Gi
+timeout: 6h

requirements.txt

@@ -0,0 +1,5 @@
+transformers
+datasets
+torch
+beautifulsoup4
+requests

train.py

@@ -0,0 +1,28 @@
+from transformers import AutoTokenizer, AutoModelForCausalLM, Trainer, TrainingArguments
+from datasets import load_dataset
+
+MODEL = "skt/kogpt2-base-v2"
+
+tokenizer = AutoTokenizer.from_pretrained(MODEL)
+model = AutoModelForCausalLM.from_pretrained(MODEL)
+
+ds = load_dataset("text", data_files="data.txt")
+
+def tok(x):
+    return tokenizer(x["text"], truncation=True, max_length=128)
+
+ds = ds.map(tok, batched=True, remove_columns=["text"])
+
+args = TrainingArguments(
+    output_dir="out",
+    num_train_epochs=1,
+    per_device_train_batch_size=2,
+    logging_steps=20,
+    save_steps=500,
+    report_to="none"
+)
+
+Trainer(model=model, args=args, train_dataset=ds["train"]).train()
+
+model.save_pretrained("out")
+tokenizer.save_pretrained("out")