Upload tokenizer

README.md

@@ -1,7 +1,8 @@
 ---
-base_model:
-- euclaise/Memphis-CoT-3B
 license: cc-by-4.0
+library_name: transformers
+tags:
+- supertrainer2000
 datasets:
 - euclaise/TinyCoT
 - euclaise/mathoverflow-accepted
@@ -13,9 +14,8 @@ datasets:
 - pszemraj/booksum-short
 - euclaise/reddit-instruct
 - euclaise/SciCoT
-library_name: transformers
-tags:
-- supertrainer2000
+base_model:
+- euclaise/Memphis-CoT-3B
 ---
 
 ![image/png](https://cdn-uploads.huggingface.co/production/uploads/64137e2150358a805203cbac/wEaKoLeJzidUdTWwQmA6k.png)

tokenizer_config.json

@@ -203,6 +203,7 @@
     }
   },
   "bos_token": "<|endoftext|>",
+  "chat_template": "{{ '### User:\n' }}{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{% if message['role'] == 'user' %}{{ message['content'].strip() + '\n' }}{% if add_generation_prompt %}{{ '### Assistant:\n' }}{% endif %}{% elif message['role'] == 'assistant' %}{{ '### Assistant:\n' + message['content'].strip() + '\n### User:\n' }}{% endif %}{% endfor %}",
   "clean_up_tokenization_spaces": true,
   "eos_token": "<|endoftext|>",
   "model_max_length": 1000000000000000019884624838656,