| |
| src="KoichiYasuoka/modernbert-base-classical-chinese" |
| tgt="KoichiYasuoka/modernbert-base-classical-chinese-ud-triangular" |
| url="https://github.com/UniversalDependencies/UD_Classical_Chinese-Kyoto" |
| import os |
| d=os.path.basename(url) |
| os.system("test -d "+d+" || git clone --depth=1 "+url) |
| os.system("for F in train dev test ; do cp "+d+"/*-$F.conllu $F.conllu ; done") |
| class UDTriangularDataset(object): |
| def __init__(self,conllu,tokenizer): |
| self.conllu=open(conllu,"r",encoding="utf-8") |
| self.tokenizer=tokenizer |
| self.seeks=[0] |
| label=set(["SYM|x","X|x"]) |
| dep=set(["X|x|r-goeswith"]) |
| s=self.conllu.readline() |
| while s!="": |
| if s=="\n": |
| self.seeks.append(self.conllu.tell()) |
| else: |
| w=s.split("\t") |
| if len(w)==10: |
| if w[0].isdecimal(): |
| p=w[3] |
| q="" if w[5]=="_" else "|"+w[5] |
| d=("|" if w[6]=="0" else "|l-" if int(w[0])<int(w[6]) else "|r-")+w[7] |
| label.add(p+"|o"+q) |
| label.add(p+"|x"+q) |
| dep.add(p+"|o"+q+d) |
| dep.add(p+"|x"+q+d) |
| s=self.conllu.readline() |
| lid={l:i for i,l in enumerate(sorted(label))} |
| for i,d in enumerate(sorted(dep),len(lid)): |
| lid[d]=i |
| self.label2id=lid |
| def __call__(*args): |
| lid={l:i for i,l in enumerate(sorted(set(sum([list(t.label2id) for t in args],[]))))} |
| for t in args: |
| t.label2id=lid |
| return lid |
| def __del__(self): |
| self.conllu.close() |
| __len__=lambda self:len(self.seeks)-1 |
| def __getitem__(self,i): |
| s=self.seeks[i] |
| self.conllu.seek(s) |
| c,t=[],[""] |
| while t[0]!="\n": |
| t=self.conllu.readline().split("\t") |
| if len(t)==10 and t[0].isdecimal(): |
| c.append(t) |
| v=self.tokenizer([t[1] for t in c],add_special_tokens=False)["input_ids"] |
| for i in range(len(v)-1,-1,-1): |
| for j in range(1,len(v[i])): |
| c.insert(i+1,[c[i][0],"_","_","X","_","_",c[i][0],"goeswith","_","_"]) |
| y=["0"]+[t[0] for t in c] |
| h=[i if t[6]=="0" else y.index(t[6]) for i,t in enumerate(c,1)] |
| x=["o" if k>i or sum([1 if j==i+1 else 0 for j in h[i+1:]])>0 else "x" for i,k in enumerate(h)] |
| p=[t[3]+"|"+x[i] if t[5]=="_" else t[3]+"|"+x[i]+"|"+t[5] for i,t in enumerate(c)] |
| d=[t[7] if t[6]=="0" else "l-"+t[7] if int(t[0])<int(t[6]) else "r-"+t[7] for t in c] |
| v=sum(v,[]) |
| ids=[self.tokenizer.cls_token_id] |
| upos=["SYM|x"] |
| for i,k in enumerate(v): |
| if len(v)<127 or x[i]=="o": |
| ids.append(k) |
| upos.append(p[i]+"|"+d[i] if h[i]==i+1 else p[i]) |
| for j in range(i+1,len(v)): |
| ids.append(v[j]) |
| upos.append(p[j]+"|"+d[j] if h[j]==i+1 else p[i]+"|"+d[i] if h[i]==j+1 else p[j]) |
| ids.append(self.tokenizer.sep_token_id) |
| upos.append("SYM|x") |
| i=0 |
| while len(ids)>8192: |
| try: |
| i=ids.index(self.tokenizer.sep_token_id,ids.index(self.tokenizer.sep_token_id,i+1)+1)-1 |
| except: |
| break |
| while len(ids)>8192 and ids[i]!=self.tokenizer.sep_token_id: |
| if upos[i].endswith("|x"): |
| ids.pop(i) |
| upos.pop(i) |
| i-=1 |
| else: |
| break |
| return {"input_ids":ids[:8192],"labels":[self.label2id[p] for p in upos[:8192]]} |
| from transformers import AutoTokenizer,AutoConfig,AutoModelForTokenClassification,DataCollatorForTokenClassification,TrainingArguments,Trainer |
| tkz=AutoTokenizer.from_pretrained(src) |
| trainDS=UDTriangularDataset("train.conllu",tkz) |
| devDS=UDTriangularDataset("dev.conllu",tkz) |
| testDS=UDTriangularDataset("test.conllu",tkz) |
| lid=trainDS(devDS,testDS) |
| cfg=AutoConfig.from_pretrained(src,num_labels=len(lid),label2id=lid,id2label={i:l for l,i in lid.items()},ignore_mismatched_sizes=True,trust_remote_code=True) |
| mdl=AutoModelForTokenClassification.from_pretrained(src,config=cfg,ignore_mismatched_sizes=True,trust_remote_code=True) |
| arg=TrainingArguments(num_train_epochs=3,per_device_train_batch_size=1,dataloader_pin_memory=False,output_dir=tgt,overwrite_output_dir=True,save_total_limit=2,learning_rate=5e-05,warmup_ratio=0.1,save_safetensors=False) |
| trn=Trainer(args=arg,data_collator=DataCollatorForTokenClassification(tkz),model=mdl,train_dataset=trainDS) |
| trn.train() |
| trn.save_model(tgt) |
| tkz.save_pretrained(tgt) |
|
|
