Upload tokenizer

README.md

@@ -1,4 +1,3 @@
-
 ---
 library_name: transformers
 base_model: google/gemma-2b-it

added_tokens.json

@@ -1,26 +1,26 @@
 {
-  "<|assistant|>": 256007,
-  "<|context|>": 256015,
-  "<|current_states|>": 256014,
-  "<|endoftext|>": 256001,
-  "<|function_call|>": 256008,
-  "<|function_response|>": 256009,
-  "<|functions|>": 256002,
-  "<|gökdeniz|>": 256004,
-  "<|home_state|>": 256013,
-  "<|image|>": 256010,
-  "<|josie|>": 256006,
-  "<|long_term_memory|>": 256011,
-  "<|short_term_memory|>": 256012,
-  "<|startoftext|>": 256000,
-  "<|system|>": 256003,
-  "<|user|>": 256005,
-  "Gökdeniz": 256017,
-  "Gökdeniz Gülmez": 256016,
-  "Gülmez": 256018,
-  "J.O.S.I.E.": 256020,
-  "JOSIE": 256019,
-  "Josie": 256021,
-  "Just an Outstandingly Smart and Intelligent Entity": 256023,
-  "josie": 256022
+  "<|assistant|>": 32007,
+  "<|context|>": 32015,
+  "<|current_states|>": 32014,
+  "<|endoftext|>": 32001,
+  "<|function_call|>": 32008,
+  "<|function_response|>": 32009,
+  "<|functions|>": 32002,
+  "<|gökdeniz|>": 32004,
+  "<|home_state|>": 32013,
+  "<|image|>": 32010,
+  "<|josie|>": 32006,
+  "<|long_term_memory|>": 32011,
+  "<|short_term_memory|>": 32012,
+  "<|startoftext|>": 32000,
+  "<|system|>": 32003,
+  "<|user|>": 32005,
+  "Gökdeniz": 32017,
+  "Gökdeniz Gülmez": 32016,
+  "Gülmez": 32018,
+  "J.O.S.I.E.": 32020,
+  "JOSIE": 32019,
+  "Josie": 32021,
+  "Just an Outstandingly Smart and Intelligent Entity": 32023,
+  "josie": 32022
 }

special_tokens_map.json

@@ -113,13 +113,6 @@
     "rstrip": false,
     "single_word": false
   },
-  "pad_token": {
-    "content": "<pad>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
   "unk_token": {
     "content": "<unk>",
     "lstrip": false,

tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:219727449dea40daaa3c0d7f68974e2e452597646e8f84b2ed517af9094b9a8d
-size 17482510
+oid sha256:fcf5bc7fc4e420d0102f71c9346b225ae73e3988c21e0cb9d66bcba19915644d
+size 1799860

tokenizer.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:61a7b147390c64585d6c3543dd6fc636906c9af3865a5548f27f31aee1d4c8e2
-size 4241003
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

tokenizer_config.json

@@ -3,7 +3,7 @@
   "add_eos_token": false,
   "added_tokens_decoder": {
     "0": {
-      "content": "<pad>",
+      "content": "<unk>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -11,7 +11,7 @@
       "special": true
     },
     "1": {
-      "content": "<eos>",
+      "content": "<s>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -19,38 +19,14 @@
       "special": true
     },
     "2": {
-      "content": "<bos>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "3": {
-      "content": "<unk>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "106": {
-      "content": "<start_of_turn>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "107": {
-      "content": "<end_of_turn>",
+      "content": "</s>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "256000": {
+    "32000": {
       "content": "<|startoftext|>",
       "lstrip": false,
       "normalized": false,
@@ -58,7 +34,7 @@
       "single_word": false,
       "special": true
     },
-    "256001": {
+    "32001": {
       "content": "<|endoftext|>",
       "lstrip": false,
       "normalized": false,
@@ -66,7 +42,7 @@
       "single_word": false,
       "special": true
     },
-    "256002": {
+    "32002": {
       "content": "<|functions|>",
       "lstrip": false,
       "normalized": false,
@@ -74,7 +50,7 @@
       "single_word": false,
       "special": true
     },
-    "256003": {
+    "32003": {
       "content": "<|system|>",
       "lstrip": false,
       "normalized": false,
@@ -82,7 +58,7 @@
       "single_word": false,
       "special": true
     },
-    "256004": {
+    "32004": {
       "content": "<|gökdeniz|>",
       "lstrip": false,
       "normalized": false,
@@ -90,7 +66,7 @@
       "single_word": false,
       "special": true
     },
-    "256005": {
+    "32005": {
       "content": "<|user|>",
       "lstrip": false,
       "normalized": false,
@@ -98,7 +74,7 @@
       "single_word": false,
       "special": true
     },
-    "256006": {
+    "32006": {
       "content": "<|josie|>",
       "lstrip": false,
       "normalized": false,
@@ -106,7 +82,7 @@
       "single_word": false,
       "special": true
     },
-    "256007": {
+    "32007": {
       "content": "<|assistant|>",
       "lstrip": false,
       "normalized": false,
@@ -114,7 +90,7 @@
       "single_word": false,
       "special": true
     },
-    "256008": {
+    "32008": {
       "content": "<|function_call|>",
       "lstrip": false,
       "normalized": false,
@@ -122,7 +98,7 @@
       "single_word": false,
       "special": true
     },
-    "256009": {
+    "32009": {
       "content": "<|function_response|>",
       "lstrip": false,
       "normalized": false,
@@ -130,7 +106,7 @@
       "single_word": false,
       "special": true
     },
-    "256010": {
+    "32010": {
       "content": "<|image|>",
       "lstrip": false,
       "normalized": false,
@@ -138,7 +114,7 @@
       "single_word": false,
       "special": true
     },
-    "256011": {
+    "32011": {
       "content": "<|long_term_memory|>",
       "lstrip": false,
       "normalized": false,
@@ -146,7 +122,7 @@
       "single_word": false,
       "special": true
     },
-    "256012": {
+    "32012": {
       "content": "<|short_term_memory|>",
       "lstrip": false,
       "normalized": false,
@@ -154,7 +130,7 @@
       "single_word": false,
       "special": true
     },
-    "256013": {
+    "32013": {
       "content": "<|home_state|>",
       "lstrip": false,
       "normalized": false,
@@ -162,7 +138,7 @@
       "single_word": false,
       "special": true
     },
-    "256014": {
+    "32014": {
       "content": "<|current_states|>",
       "lstrip": false,
       "normalized": false,
@@ -170,7 +146,7 @@
       "single_word": false,
       "special": true
     },
-    "256015": {
+    "32015": {
       "content": "<|context|>",
       "lstrip": false,
       "normalized": false,
@@ -178,7 +154,7 @@
       "single_word": false,
       "special": true
     },
-    "256016": {
+    "32016": {
       "content": "Gökdeniz Gülmez",
       "lstrip": false,
       "normalized": true,
@@ -186,7 +162,7 @@
       "single_word": false,
       "special": false
     },
-    "256017": {
+    "32017": {
       "content": "Gökdeniz",
       "lstrip": false,
       "normalized": true,
@@ -194,7 +170,7 @@
       "single_word": false,
       "special": false
     },
-    "256018": {
+    "32018": {
       "content": "Gülmez",
       "lstrip": false,
       "normalized": true,
@@ -202,7 +178,7 @@
       "single_word": false,
       "special": false
     },
-    "256019": {
+    "32019": {
       "content": "JOSIE",
       "lstrip": false,
       "normalized": true,
@@ -210,7 +186,7 @@
       "single_word": false,
       "special": false
     },
-    "256020": {
+    "32020": {
       "content": "J.O.S.I.E.",
       "lstrip": false,
       "normalized": true,
@@ -218,7 +194,7 @@
       "single_word": false,
       "special": false
     },
-    "256021": {
+    "32021": {
       "content": "Josie",
       "lstrip": false,
       "normalized": true,
@@ -226,7 +202,7 @@
       "single_word": false,
       "special": false
     },
-    "256022": {
+    "32022": {
       "content": "josie",
       "lstrip": false,
       "normalized": true,
@@ -234,7 +210,7 @@
       "single_word": false,
       "special": false
     },
-    "256023": {
+    "32023": {
       "content": "Just an Outstandingly Smart and Intelligent Entity",
       "lstrip": false,
       "normalized": true,
@@ -260,15 +236,14 @@
     "<|context|>"
   ],
   "bos_token": "<|startoftext|>",
-  "chat_template": "{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '<start_of_turn>' + role + '\n' + message['content'] | trim + '<end_of_turn>\n' }}{% endfor %}{% if add_generation_prompt %}{{'<start_of_turn>model\n'}}{% endif %}",
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|endoftext|>",
-  "legacy": null,
+  "legacy": true,
   "model_max_length": 1000000000000000019884624838656,
-  "pad_token": "<pad>",
+  "pad_token": null,
   "sp_model_kwargs": {},
   "spaces_between_special_tokens": false,
-  "tokenizer_class": "GemmaTokenizer",
+  "tokenizer_class": "LlamaTokenizer",
   "unk_token": "<unk>",
   "use_default_system_prompt": false
 }