Commit
•
eefd8ac
1
Parent(s):
cc01c87
add chat_template in tokenizer_config.json (#3)
Browse files- add chat_template in tokenizer_config.json (21b7b2dfbbd2edcfea98d896befa4e4e8020b31d)
Co-authored-by: enwei <[email protected]>
- tokenizer_config.json +2 -1
tokenizer_config.json
CHANGED
@@ -30,5 +30,6 @@
|
|
30 |
},
|
31 |
"sp_model_kwargs": {},
|
32 |
"unk_token": null,
|
|
|
33 |
"tokenizer_class": "LlamaTokenizerFast"
|
34 |
-
}
|
|
|
30 |
},
|
31 |
"sp_model_kwargs": {},
|
32 |
"unk_token": null,
|
33 |
+
"chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{{ bos_token }}{% for message in messages %}{% if message['role'] == 'user' %}{{ 'User: ' + message['content'] + '\n\n' }}{% elif message['role'] == 'assistant' %}{{ 'Assistant: ' + message['content'] + eos_token }}{% elif message['role'] == 'system' %}{{ message['content'] + '\n\n' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ 'Assistant:' }}{% endif %}",
|
34 |
"tokenizer_class": "LlamaTokenizerFast"
|
35 |
+
}
|