fix for transformers>4.33
#11
by
jinmingyi
- opened
- tokenization_chatglm.py +12 -12
tokenization_chatglm.py
CHANGED
@@ -193,6 +193,18 @@ class ChatGLMTokenizer(PreTrainedTokenizer):
|
|
193 |
num_image_tokens=20000,
|
194 |
**kwargs
|
195 |
) -> None:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
196 |
super().__init__(
|
197 |
do_lower_case=do_lower_case,
|
198 |
remove_space=remove_space,
|
@@ -208,18 +220,6 @@ class ChatGLMTokenizer(PreTrainedTokenizer):
|
|
208 |
**kwargs
|
209 |
)
|
210 |
|
211 |
-
self.do_lower_case = do_lower_case
|
212 |
-
self.remove_space = remove_space
|
213 |
-
self.vocab_file = vocab_file
|
214 |
-
|
215 |
-
self.bos_token = bos_token
|
216 |
-
self.eos_token = eos_token
|
217 |
-
self.end_token = end_token
|
218 |
-
self.mask_token = mask_token
|
219 |
-
self.gmask_token = gmask_token
|
220 |
-
|
221 |
-
self.sp_tokenizer = SPTokenizer(vocab_file, num_image_tokens=num_image_tokens)
|
222 |
-
|
223 |
""" Initialisation """
|
224 |
|
225 |
@property
|
|
|
193 |
num_image_tokens=20000,
|
194 |
**kwargs
|
195 |
) -> None:
|
196 |
+
self.do_lower_case = do_lower_case
|
197 |
+
self.remove_space = remove_space
|
198 |
+
self.vocab_file = vocab_file
|
199 |
+
|
200 |
+
self.bos_token = bos_token
|
201 |
+
self.eos_token = eos_token
|
202 |
+
self.end_token = end_token
|
203 |
+
self.mask_token = mask_token
|
204 |
+
self.gmask_token = gmask_token
|
205 |
+
|
206 |
+
self.sp_tokenizer = SPTokenizer(vocab_file, num_image_tokens=num_image_tokens)
|
207 |
+
|
208 |
super().__init__(
|
209 |
do_lower_case=do_lower_case,
|
210 |
remove_space=remove_space,
|
|
|
220 |
**kwargs
|
221 |
)
|
222 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
223 |
""" Initialisation """
|
224 |
|
225 |
@property
|