Commit
•
7c42852
1
Parent(s):
52df825
Update tokenizer_config.json (#2)
Browse files- Update tokenizer_config.json (af87d385024723bd2807633c141e4a4a2c7f2207)
Co-authored-by: Yaowei Zheng <[email protected]>
- tokenizer_config.json +2 -4
tokenizer_config.json
CHANGED
@@ -1751,17 +1751,15 @@
|
|
1751 |
"<image>"
|
1752 |
],
|
1753 |
"bos_token": "<bos>",
|
1754 |
-
"chat_template": "{% if
|
1755 |
"clean_up_tokenization_spaces": false,
|
1756 |
"eos_token": "<eos>",
|
1757 |
"model_max_length": 1000000000000000019884624838656,
|
1758 |
"pad_token": "<pad>",
|
1759 |
-
"padding_side": "left",
|
1760 |
"processor_class": "PaliGemmaProcessor",
|
1761 |
"sp_model_kwargs": {},
|
1762 |
"spaces_between_special_tokens": false,
|
1763 |
-
"split_special_tokens": false,
|
1764 |
"tokenizer_class": "GemmaTokenizer",
|
1765 |
"unk_token": "<unk>",
|
1766 |
"use_default_system_prompt": false
|
1767 |
-
}
|
|
|
1751 |
"<image>"
|
1752 |
],
|
1753 |
"bos_token": "<bos>",
|
1754 |
+
"chat_template": "{{ bos_token }}{% for message in messages %}{% if message['role'] == 'assistant' %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '<start_of_turn>' + role + '\n' + message['content'] | trim + '<end_of_turn>\n' }}{% endfor %}{% if add_generation_prompt %}{{'<start_of_turn>model\n'}}{% endif %}",
|
1755 |
"clean_up_tokenization_spaces": false,
|
1756 |
"eos_token": "<eos>",
|
1757 |
"model_max_length": 1000000000000000019884624838656,
|
1758 |
"pad_token": "<pad>",
|
|
|
1759 |
"processor_class": "PaliGemmaProcessor",
|
1760 |
"sp_model_kwargs": {},
|
1761 |
"spaces_between_special_tokens": false,
|
|
|
1762 |
"tokenizer_class": "GemmaTokenizer",
|
1763 |
"unk_token": "<unk>",
|
1764 |
"use_default_system_prompt": false
|
1765 |
+
}
|