model improved for transformers 4.42

config.json

@@ -4,22 +4,11 @@
   ],
   "attention_bias": false,
   "attention_dropout": 0.0,
-  "auto_map": {
-    "AutoModelForTokenClassification": "upos.LlamaForTokenClassification"
-  },
   "bos_token_id": 128000,
   "custom_pipelines": {
     "upos": {
       "impl": "upos.BellmanFordTokenClassificationPipeline",
       "pt": "AutoModelForTokenClassification"
-    },
-    "token-classification": {
-      "impl": "upos.RawTokenClassificationPipeline",
-      "pt": "AutoModelForTokenClassification"
-    },
-    "ner": {
-      "impl": "upos.RawTokenClassificationPipeline",
-      "pt": "AutoModelForTokenClassification"
     }
   },
   "eos_token_id": 128001,
@@ -146,7 +135,7 @@
   "tie_word_embeddings": false,
   "tokenizer_class": "LlamaTokenizerFast",
   "torch_dtype": "float32",
-  "transformers_version": "4.41.2",
+  "transformers_version": "4.42.4",
   "use_cache": true,
   "vocab_size": 128259
 }

maker.sh

@@ -30,45 +30,9 @@ cat << 'EOF' > $TMPB
 #! /usr/bin/env deepspeed
 src="exLlama-3-Swallow-8B"
 tgt="KoichiYasuoka/Llama-3-Swallow-8B-upos"
-from transformers import LlamaTokenizerFast,LlamaModel,LlamaPreTrainedModel,AutoConfig,DataCollatorForTokenClassification,TrainingArguments,Trainer
-from transformers.modeling_outputs import TokenClassifierOutput
+from transformers import LlamaTokenizerFast,LlamaForTokenClassification,AutoConfig,DataCollatorForTokenClassification,TrainingArguments,Trainer
 from tokenizers.normalizers import Replace
 
-class LlamaForTokenClassification(LlamaPreTrainedModel):
-  def __init__(self,config):
-    from torch import nn
-    super().__init__(config)
-    self.num_labels=config.num_labels
-    self.model=LlamaModel(config)
-    if hasattr(config,"classifier_dropout") and config.classifier_dropout is not None:
-      classifier_dropout=config.classifier_dropout
-    elif hasattr(config,"hidden_dropout") and config.hidden_dropout is not None:
-      classifier_dropout=config.hidden_dropout
-    else:
-      classifier_dropout=0.1
-    self.dropout=nn.Dropout(classifier_dropout)
-    self.classifier=nn.Linear(config.hidden_size,config.num_labels)
-    self.post_init()
-  def get_input_embeddings(self):
-    return self.model.embed_tokens
-  def set_input_embeddings(self,value):
-    self.model.embed_tokens=value
-  def forward(self,input_ids=None,past_key_values=None,attention_mask=None,position_ids=None,inputs_embeds=None,labels=None,use_cache=None,output_attentions=None,output_hidden_states=None,return_dict=None):
-    return_dict=return_dict if return_dict is not None else self.config.use_return_dict
-    transformer_outputs=self.model(input_ids,past_key_values=past_key_values,attention_mask=attention_mask,position_ids=position_ids,inputs_embeds=inputs_embeds,use_cache=use_cache,output_attentions=output_attentions,output_hidden_states=output_hidden_states,return_dict=return_dict)
-    hidden_states=transformer_outputs[0]
-    hidden_states=self.dropout(hidden_states)
-    logits=self.classifier(hidden_states)
-    loss=None
-    if labels is not None:
-      from torch import nn
-      loss_fct=nn.CrossEntropyLoss()
-      loss=loss_fct(logits.view(-1,self.num_labels),labels.view(-1))
-    if not return_dict:
-      output=(logits,)+transformer_outputs[1:]
-      return ((loss,)+output) if loss is not None else output
-    return TokenClassifierOutput(loss=loss,logits=logits,hidden_states=transformer_outputs.hidden_states,attentions=transformer_outputs.attentions)
-
 class UPOSFileDataset(object):
   def __init__(self,conllu,tokenizer):
     self.conllu=open(conllu,"r",encoding="utf-8")

pytorch_model-00001-of-00007.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e34daa1be0869a7a71578c229f82f13fb4d3f6a003f6d31c3dd04ddf1a7265d7
+oid sha256:4c1eccfc4ac55455e0a90a3b934694dffb05491ca344aaa9989d700ee54ca04c
 size 4886522810

pytorch_model-00002-of-00007.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5bb82ab6110006a07cb361674e4be76d1368e1e3de31035d1ae6bc6c7f20914e
+oid sha256:428443ac934b1722a49a914f362522c1acb8f918d6305f2d984f20ea56a99367
 size 4832018324

pytorch_model-00003-of-00007.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:55a70605a22a3d99fb732610d2963dcbccfbbf99d57256c352af105c2c50f29c
+oid sha256:891d943d00315ce1e956adf438324229dc789de5b3913c6eb4a09d513b8db62a
 size 4999825256

pytorch_model-00004-of-00007.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6b749ea15607ac13e6e31ebf31a66b8a41657554642fdd09d604a93e90b38eed
+oid sha256:a93f360a7ea98e786708c001c4bf6abbfcf412f020a08693d3683163d2be3c05
 size 4999825316

pytorch_model-00005-of-00007.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e45d5b53b00968ffba32eb631834374a2c7a4dfa6f8cec3ca9d9c64e7adcb4e8
+oid sha256:453231206832ee14a9716179bb4cc1558e78cbde9aaf36108b5a51f28f923771
 size 4832018324

pytorch_model-00006-of-00007.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8b9ff08e6f96f400379245683d36c33831a28ad76999e75e3ce9bf9a2623db31
+oid sha256:4abae140281b41a91fb15418608a306eaa6e4e10d3ba6deb7688c4365fd627a2
 size 4999825320

pytorch_model-00007-of-00007.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f111d151726dd3487111bc0e08278c08987ec551ea1c84ba56823b264b53d17b
+oid sha256:ccff44101836996c47f5b14426a0ae020f74435191637344d427639eb6ea9ecf
 size 470650155

pytorch_model.bin.index.json

@@ -3,8 +3,6 @@
     "total_size": 30020583628
   },
   "weight_map": {
-    "classifier.bias": "pytorch_model-00007-of-00007.bin",
-    "classifier.weight": "pytorch_model-00007-of-00007.bin",
     "model.embed_tokens.weight": "pytorch_model-00001-of-00007.bin",
     "model.layers.0.input_layernorm.weight": "pytorch_model-00001-of-00007.bin",
     "model.layers.0.mlp.down_proj.weight": "pytorch_model-00001-of-00007.bin",
@@ -294,6 +292,8 @@
     "model.layers.9.self_attn.o_proj.weight": "pytorch_model-00003-of-00007.bin",
     "model.layers.9.self_attn.q_proj.weight": "pytorch_model-00003-of-00007.bin",
     "model.layers.9.self_attn.v_proj.weight": "pytorch_model-00003-of-00007.bin",
-    "model.norm.weight": "pytorch_model-00007-of-00007.bin"
+    "model.norm.weight": "pytorch_model-00007-of-00007.bin",
+    "score.bias": "pytorch_model-00007-of-00007.bin",
+    "score.weight": "pytorch_model-00007-of-00007.bin"
   }
 }

tokenizer_config.json

@@ -1,6 +1,7 @@
 {
   "add_bos_token": true,
   "add_eos_token": false,
+  "add_prefix_space": null,
   "added_tokens_decoder": {
     "128000": {
       "content": "<|begin_of_text|>",

upos.py

@@ -1,5 +1,4 @@
-from transformers import TokenClassificationPipeline,LlamaModel,LlamaPreTrainedModel
-from transformers.modeling_outputs import TokenClassifierOutput
+from transformers import TokenClassificationPipeline
 
 class BellmanFordTokenClassificationPipeline(TokenClassificationPipeline):
   def __init__(self,**kwargs):
@@ -40,41 +39,3 @@ class BellmanFordTokenClassificationPipeline(TokenClassificationPipeline):
       t["text"]=model_outputs["sentence"][t["start"]:t["end"]]
     return w
 
-class RawTokenClassificationPipeline(TokenClassificationPipeline):
-  def check_model_type(self,supported_models):
-    pass
-
-class LlamaForTokenClassification(LlamaPreTrainedModel):
-  def __init__(self,config):
-    from torch import nn
-    super().__init__(config)
-    self.num_labels=config.num_labels
-    self.model=LlamaModel(config)
-    if hasattr(config,"classifier_dropout") and config.classifier_dropout is not None:
-      classifier_dropout=config.classifier_dropout
-    elif hasattr(config,"hidden_dropout") and config.hidden_dropout is not None:
-      classifier_dropout=config.hidden_dropout
-    else:
-      classifier_dropout=0.1
-    self.dropout=nn.Dropout(classifier_dropout)
-    self.classifier=nn.Linear(config.hidden_size,config.num_labels)
-    self.post_init()
-  def get_input_embeddings(self):
-    return self.model.embed_tokens
-  def set_input_embeddings(self,value):
-    self.model.embed_tokens=value
-  def forward(self,input_ids=None,past_key_values=None,attention_mask=None,position_ids=None,inputs_embeds=None,labels=None,use_cache=None,output_attentions=None,output_hidden_states=None,return_dict=None):
-    return_dict=return_dict if return_dict is not None else self.config.use_return_dict
-    transformer_outputs=self.model(input_ids,past_key_values=past_key_values,attention_mask=attention_mask,position_ids=position_ids,inputs_embeds=inputs_embeds,use_cache=use_cache,output_attentions=output_attentions,output_hidden_states=output_hidden_states,return_dict=return_dict)
-    hidden_states=transformer_outputs[0]
-    hidden_states=self.dropout(hidden_states)
-    logits=self.classifier(hidden_states)
-    loss=None
-    if labels is not None:
-      from torch import nn
-      loss_fct=nn.CrossEntropyLoss()
-      loss=loss_fct(logits.view(-1,self.num_labels),labels.view(-1))
-    if not return_dict:
-      output=(logits,)+transformer_outputs[2:]
-      return ((loss,)+output) if loss is not None else output
-    return TokenClassifierOutput(loss=loss,logits=logits,hidden_states=transformer_outputs.hidden_states,attentions=transformer_outputs.attentions)