model improved

config.json

@@ -372,7 +372,7 @@
   "summary_use_proj": true,
   "tokenizer_class": "PreTrainedTokenizerFast",
   "torch_dtype": "float32",
-  "transformers_version": "4.39.3",
+  "transformers_version": "4.44.2",
   "use_cache": true,
   "vocab_size": 44928
 }

maker.py

@@ -0,0 +1,127 @@
+#! /usr/bin/python3
+src="rinna/japanese-gpt-1b"
+tgt="KoichiYasuoka/rinna-gpt2-1b-japanese-ud-causal"
+url="https://github.com/UniversalDependencies/UD_Japanese-GSDLUW"
+
+import os,json,unicodedata
+from transformers import AutoTokenizer,PreTrainedTokenizerFast,AutoConfig,GPT2ForTokenClassification,DefaultDataCollator,TrainingArguments,Trainer
+d=os.path.basename(url)
+os.system("test -d "+d+" || git clone --depth=1 "+url)
+os.system("for F in train dev test ; do cp "+d+"/*-$F.conllu $F.conllu ; done")
+tkz=AutoTokenizer.from_pretrained(src,add_prefix_space=False,legacy=False,model_max_length=2048)
+tkz.save_pretrained("tmpdir")
+d=json.loads(tkz.backend_tokenizer.to_str())
+tkz.backend_tokenizer.from_str(json.dumps(d)).save("tmpdir/oldtokenizer.json")
+form=set()
+with open("train.conllu","r",encoding="utf-8") as r:
+  for s in r:
+    w=s.split("\t")
+    if len(w)==10 and w[0].isdecimal():
+      form.add(w[1])
+m=[t for t in d["model"]["merges"] if len(t)<4 and t.replace(" ","") in form and not unicodedata.name(t[0],"").startswith("HIRAGANA")]
+d["model"]["merges"]=m
+tkz.backend_tokenizer.from_str(json.dumps(d)).save("tmpdir/tokenizer.json")
+ntk=PreTrainedTokenizerFast.from_pretrained("tmpdir")
+otk=PreTrainedTokenizerFast.from_pretrained("tmpdir",tokenizer_file="tmpdir/oldtokenizer.json")
+
+class UDCausalDataset(object):
+  def __init__(self,conllu,tokenizer,oldtokenizer=None,embeddings=None):
+    self.conllu=open(conllu,"r",encoding="utf-8")
+    self.tokenizer=tokenizer
+    self.oldtokenizer=oldtokenizer if oldtokenizer else tokenizer
+    self.embeddings=embeddings
+    self.max_tokens=3
+    self.seeks=[(0,0)]
+    label=set(["SYM"])
+    dep=set()
+    s=self.conllu.readline()
+    while s!="":
+      if s=="\n":
+        self.seeks.append((self.conllu.tell(),0))
+      else:
+        w=s.split("\t")
+        if len(w)==10:
+          if w[0].isdecimal():
+            p=w[3] if w[5]=="_" else w[3]+"|"+w[5]
+            label.add(p)
+            dep.add(p+("|" if w[6]=="0" else "|l-" if int(w[0])<int(w[6]) else "|r-")+w[7])
+            self.seeks.append((self.seeks[-1][0],int(w[0])))
+            self.max_tokens=max(self.max_tokens,int(w[0])*2+1)
+      s=self.conllu.readline()
+    lid={}
+    for i,l in enumerate(sorted(label)):
+      lid[l],lid["B-"+l],lid["I-"+l]=i*3,i*3+1,i*3+2
+    for i,d in enumerate(sorted(dep),len(lid)):
+      lid[d]=i
+    self.label2id=lid
+  def __call__(*args):
+    lid={l:i for i,l in enumerate(sorted(set(sum([list(t.label2id) for t in args],[]))))}
+    for t in args:
+      t.label2id=lid
+    return lid
+  def __del__(self):
+    self.conllu.close()
+  __len__=lambda self:len(self.seeks)-1
+  def __getitem__(self,i):
+    s,t=self.seeks[i]
+    self.conllu.seek(s)
+    form,upos,deps,w=[],[],[],[""]
+    while w[0]!="\n":
+      w=self.conllu.readline().split("\t")
+      if len(w)==10:
+        form.append(w[1])
+        if w[0].isdecimal():
+          upos.append(w[3] if w[5]=="_" else w[3]+"|"+w[5])
+          deps.append((int(w[6]),w[7]))
+    if t==0:
+      v=self.tokenizer(form,add_special_tokens=False)
+      i,u=[],[]
+      for j,(x,y) in enumerate(zip(v["input_ids"],upos)):
+        if x!=[]:
+          i+=x
+          u+=[y] if len(x)==1 else ["B-"+y]+["I-"+y]*(len(x)-1)
+      emb=self.embeddings
+      pad=self.tokenizer.pad_token_id
+    else:
+      import torch
+      v=self.oldtokenizer(form,add_special_tokens=False)
+      m=[]
+      for x in v["input_ids"]:
+        if x==[]:
+          m.append(self.embeddings[self.tokenizer.unk_token_id,:])
+        else:
+          m.append(self.embeddings[x,:].sum(axis=0))
+      m.append(self.embeddings[self.tokenizer.sep_token_id,:])
+      m.append(self.embeddings[self.tokenizer.pad_token_id,:])
+      emb=torch.stack(m)
+      i,u=list(range(len(upos)+1)),upos+["SYM"]
+      i.append(t-1)
+      k,d=deps[t-1]
+      u.append(upos[t-1]+"|"+d if k==0 else upos[t-1])
+      for j in range(t,len(upos)):
+        i.append(j)
+        a,b=deps[j]
+        u.append(upos[j]+"|r-"+b if a==t else upos[t-1]+"|l-"+d if j+1==k else upos[j])
+      pad=-1
+    j=self.max_tokens-len(i)
+    if j>0:
+      ids=i+[pad]*j
+      upos=u+["SYM"]*j
+    else:
+      ids=i[0:self.max_tokens]
+      upos=u[0:self.max_tokens]
+    return {"inputs_embeds":emb[ids,:],"labels":[self.label2id[p] for p in upos]}
+
+trainDS=UDCausalDataset("train.conllu",ntk,otk)
+devDS=UDCausalDataset("dev.conllu",ntk,otk)
+testDS=UDCausalDataset("test.conllu",ntk,otk)
+lid=trainDS(devDS,testDS)
+cfg=AutoConfig.from_pretrained(src,num_labels=len(lid),label2id=lid,id2label={i:l for l,i in lid.items()},ignore_mismatched_sizes=True)
+mdl=GPT2ForTokenClassification.from_pretrained(src,config=cfg,ignore_mismatched_sizes=True)
+trainDS.embeddings=mdl.get_input_embeddings().weight
+trainDS.max_tokens=min(trainDS.max_tokens,cfg.max_position_embeddings)
+arg=TrainingArguments(num_train_epochs=3,per_device_train_batch_size=32,dataloader_pin_memory=False,output_dir=tgt,overwrite_output_dir=True,save_total_limit=2,learning_rate=5e-05,warmup_ratio=0.1,save_safetensors=False)
+trn=Trainer(args=arg,data_collator=DefaultDataCollator(),model=mdl,train_dataset=trainDS)
+trn.train()
+trn.save_model(tgt)
+ntk.save_pretrained(tgt)

pytorch_model-00001-of-00002.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:91039cb93e06719c95158ec2ba95280ae10d397f42bec3365252b0e277dea8d0
+oid sha256:3ac2e53a9210148a94e53883c4618de3a22d0051ea5c71f16e53cfc22b90e02d
 size 4942377882

pytorch_model-00002-of-00002.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6a0e0733c23ac2270146c508d8386554be0326096203db6c1f2b5d18d36aded3
+oid sha256:8dc9f31720cbe935f3695ffa53174ee6d8be651e31390f9c306ebdc6480474d7
 size 269925550

ud.py

@@ -1,5 +1,10 @@
 import numpy
-from transformers import TokenClassificationPipeline
+from transformers import TokenClassificationPipeline,AutoTokenizer
+try:
+  from transformers.utils import cached_file
+except:
+  from transformers.file_utils import cached_path,hf_bucket_url
+  cached_file=lambda x,y:os.path.join(x,y) if os.path.isdir(x) else cached_path(hf_bucket_url(x,y))
 
 class BellmanFordTokenClassificationPipeline(TokenClassificationPipeline):
   def __init__(self,**kwargs):
@@ -42,6 +47,7 @@ class UniversalDependenciesCausalPipeline(BellmanFordTokenClassificationPipeline
   def __init__(self,**kwargs):
     kwargs["aggregation_strategy"]="simple"
     super().__init__(**kwargs)
+    self.oldtokenizer=AutoTokenizer.from_pretrained(self.tokenizer.name_or_path,tokenizer_file=cached_file(self.tokenizer.name_or_path,"oldtokenizer.json"))
     x=self.model.config.label2id
     self.root=numpy.full((len(x)),numpy.nan)
     self.left_arc=numpy.full((len(x)),numpy.nan)
@@ -87,7 +93,7 @@ class UniversalDependenciesCausalPipeline(BellmanFordTokenClassificationPipeline
       if d[i].strip()=="":
         d.pop(i)
         w.pop(i)
-    v=self.tokenizer(d,add_special_tokens=False)
+    v=self.oldtokenizer(d,add_special_tokens=False)
     e=self.model.get_input_embeddings().weight
     m=[]
     for x in v["input_ids"]: