fix(Tokenizer): fix save_pretrained error
Browse filesThe origional code seems not work on tokenizer save_pretrained, self.vocab_file not exist on line 137.
This fix has been verified for save_pretrained and push_to_hub.
- tokenization_chatglm.py +1 -0
tokenization_chatglm.py
CHANGED
@@ -70,6 +70,7 @@ class ChatGLMTokenizer(PreTrainedTokenizer):
|
|
70 |
self.name = "GLMTokenizer"
|
71 |
|
72 |
self.tokenizer = SPTokenizer(vocab_file)
|
|
|
73 |
self.special_tokens = {
|
74 |
"<bos>": self.tokenizer.bos_id,
|
75 |
"<eos>": self.tokenizer.eos_id,
|
|
|
70 |
self.name = "GLMTokenizer"
|
71 |
|
72 |
self.tokenizer = SPTokenizer(vocab_file)
|
73 |
+
self.vocab_file = vocab_file
|
74 |
self.special_tokens = {
|
75 |
"<bos>": self.tokenizer.bos_id,
|
76 |
"<eos>": self.tokenizer.eos_id,
|