Upload tokenizer

special_tokens_map.json

@@ -2,7 +2,14 @@
   "additional_special_tokens": [
     "<|im_start|>",
     "<|im_end|>",
-    "<|im_sep|>"
+    "<|im_sep|>",
+    {
+      "content": "###",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
   ],
   "bos_token": {
     "content": "<|startoftext|>",

tokenizer.json

@@ -57,6 +57,15 @@
       "normalized": false,
       "special": true
     },
+    {
+      "id": 8308,
+      "content": "###",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
     {
       "id": 64000,
       "content": "<image>",

tokenizer_config.json

@@ -51,6 +51,14 @@
       "single_word": false,
       "special": true
     },
+    "8308": {
+      "content": "###",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
     "64000": {
       "content": "<image>",
       "lstrip": false,
@@ -71,19 +79,21 @@
   "additional_special_tokens": [
     "<|im_start|>",
     "<|im_end|>",
-    "<|im_sep|>"
+    "<|im_sep|>",
+    "###"
   ],
   "bos_token": "<|startoftext|>",
-  "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "chat_template": "{% set system_message = 'This is a chat between an inquisitive human and an AI assistant. Assume the role of the AI assistant. Read all the images carefully, and respond to the human\\'s questions with informative, helpful, detailed and polite answers.这是一个好奇的人类和一个人工智能助手之间的对话。假设你扮演这个AI助手的角色。仔细阅读所有的图像，并对人类的问题做出信息丰富、有帮助、详细的和礼貌的回答。\\n\\n' %}{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{% endif %}{% if system_message is defined %}{{ system_message }}{% endif %}{% for message in messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '### Human: ' + content + '\\n### Assistant:' }}{% elif message['role'] == 'assistant' %}{{ ' ' + content }}{% endif %}{% endfor %}",
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|endoftext|>",
   "legacy": true,
   "model_max_length": 4096,
   "pad_token": "<pad>",
-  "padding_side": "right",
+  "padding_side": "left",
   "processor_class": "LlavaProcessor",
   "sp_model_kwargs": {},
   "spaces_between_special_tokens": false,
+  "split_special_tokens": false,
   "tokenizer_class": "LlamaTokenizer",
   "unk_token": "<unk>",
   "use_default_system_prompt": true