BUAADreamer hiyouga commited on
Commit
7c42852
1 Parent(s): 52df825

Update tokenizer_config.json (#2)

Browse files

- Update tokenizer_config.json (af87d385024723bd2807633c141e4a4a2c7f2207)


Co-authored-by: Yaowei Zheng <[email protected]>

Files changed (1) hide show
  1. tokenizer_config.json +2 -4
tokenizer_config.json CHANGED
@@ -1751,17 +1751,15 @@
1751
  "<image>"
1752
  ],
1753
  "bos_token": "<bos>",
1754
- "chat_template": "{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{% endif %}{{ '<bos>' + system_message }}{% for message in messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '<start_of_turn>user\n' + content + '<end_of_turn>\n<start_of_turn>model\n' }}{% elif message['role'] == 'assistant' %}{{ content + '<end_of_turn>\n' }}{% endif %}{% endfor %}",
1755
  "clean_up_tokenization_spaces": false,
1756
  "eos_token": "<eos>",
1757
  "model_max_length": 1000000000000000019884624838656,
1758
  "pad_token": "<pad>",
1759
- "padding_side": "left",
1760
  "processor_class": "PaliGemmaProcessor",
1761
  "sp_model_kwargs": {},
1762
  "spaces_between_special_tokens": false,
1763
- "split_special_tokens": false,
1764
  "tokenizer_class": "GemmaTokenizer",
1765
  "unk_token": "<unk>",
1766
  "use_default_system_prompt": false
1767
- }
 
1751
  "<image>"
1752
  ],
1753
  "bos_token": "<bos>",
1754
+ "chat_template": "{{ bos_token }}{% for message in messages %}{% if message['role'] == 'assistant' %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '<start_of_turn>' + role + '\n' + message['content'] | trim + '<end_of_turn>\n' }}{% endfor %}{% if add_generation_prompt %}{{'<start_of_turn>model\n'}}{% endif %}",
1755
  "clean_up_tokenization_spaces": false,
1756
  "eos_token": "<eos>",
1757
  "model_max_length": 1000000000000000019884624838656,
1758
  "pad_token": "<pad>",
 
1759
  "processor_class": "PaliGemmaProcessor",
1760
  "sp_model_kwargs": {},
1761
  "spaces_between_special_tokens": false,
 
1762
  "tokenizer_class": "GemmaTokenizer",
1763
  "unk_token": "<unk>",
1764
  "use_default_system_prompt": false
1765
+ }