sharpbai commited on
Commit
b827360
1 Parent(s): a6d54fa

fix(Tokenizer): fix save_pretrained error

Browse files

The origional code seems not work on tokenizer save_pretrained, self.vocab_file not exist on line 137.
This fix has been verified for save_pretrained and push_to_hub.

Files changed (1) hide show
  1. tokenization_chatglm.py +1 -0
tokenization_chatglm.py CHANGED
@@ -70,6 +70,7 @@ class ChatGLMTokenizer(PreTrainedTokenizer):
70
  self.name = "GLMTokenizer"
71
 
72
  self.tokenizer = SPTokenizer(vocab_file)
 
73
  self.special_tokens = {
74
  "<bos>": self.tokenizer.bos_id,
75
  "<eos>": self.tokenizer.eos_id,
 
70
  self.name = "GLMTokenizer"
71
 
72
  self.tokenizer = SPTokenizer(vocab_file)
73
+ self.vocab_file = vocab_file
74
  self.special_tokens = {
75
  "<bos>": self.tokenizer.bos_id,
76
  "<eos>": self.tokenizer.eos_id,