prince-canuma
commited on
Commit
•
a2c307b
1
Parent(s):
0c37b14
Update tokenizer_config.json
Browse files- tokenizer_config.json +2 -2
tokenizer_config.json
CHANGED
@@ -400,10 +400,10 @@
|
|
400 |
"bos_token": "<s>",
|
401 |
"chat_template": "{% for message in messages %}{{'<|' + message['role'] + '|>' + '\n' + message['content'] + '<|end|>\n' }}{% endfor %}{% if add_generation_prompt and messages[-1]['role'] != 'assistant' %}{{- '<|assistant|>\n' -}}{% endif %}",
|
402 |
"clean_up_tokenization_spaces": false,
|
403 |
-
"eos_token": "<|
|
404 |
"legacy": false,
|
405 |
"model_max_length": 131072,
|
406 |
-
"pad_token": "<|
|
407 |
"padding_side": "right",
|
408 |
"processor_class": "Phi3VProcessor",
|
409 |
"sp_model_kwargs": {},
|
|
|
400 |
"bos_token": "<s>",
|
401 |
"chat_template": "{% for message in messages %}{{'<|' + message['role'] + '|>' + '\n' + message['content'] + '<|end|>\n' }}{% endfor %}{% if add_generation_prompt and messages[-1]['role'] != 'assistant' %}{{- '<|assistant|>\n' -}}{% endif %}",
|
402 |
"clean_up_tokenization_spaces": false,
|
403 |
+
"eos_token": "<|endoftext|>",
|
404 |
"legacy": false,
|
405 |
"model_max_length": 131072,
|
406 |
+
"pad_token": "<|endoftext|>",
|
407 |
"padding_side": "right",
|
408 |
"processor_class": "Phi3VProcessor",
|
409 |
"sp_model_kwargs": {},
|