add more tokenizers

vocab/README.md

@@ -1,4 +1,6 @@
 
+https://arxiv.org/abs/2308.16692 SpeechTokenizer
+
 对于OpenAI的模型而言，英文的Token效率是中文的8-12倍，
 之前三百字中文以上时Turbo 3.5 16k就会出现逻辑颠倒问题，提示词换成英文后该问题没有出现过。
 

vocab/__init__.py

@@ -55,8 +55,6 @@ uniq_tokenizers = [
 all_tokenizers = [
     "gpt2",
     "gpt2_chinese",
-    "gpt_35_turbo",
-    "gpt_4",
 
     # bert 系列
     "bert_base_cased",
@@ -105,6 +103,10 @@ all_tokenizers = [
     "qwen_1_8b_chat",
     "qwen_7b_chat",
     "qwen_72b_chat",
+    "text_davinci_003",
+    "code_davinci_002",
+    "gpt_35_turbo",
+    "gpt_4",
     
     # 未分类
     "skywork_13b_base",
@@ -116,6 +118,15 @@ all_tokenizers = [
     "flan_t5_base",
     "fastchat_t5_3b",
     "pko_t5_large",
+    "wizardcoder_15b_v1",
+"wizardcoder_python_7b_v1",
+"wizardlm_7b_v1",
+"wizardmath_70b_v1",
+"tigerbot_70b_chat_v4_4k",
+"tigerbot_13b_chat_v2",
+"deepseek_coder_33b_instruct",
+"deepseek_llm_7b_base",
+
 
 
 ]

vocab/chatglm_6b/__init__.py

@@ -6,15 +6,16 @@ import os
 import config
 from transformers import AutoTokenizer
 
-os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"
 
 
-if config.USE_REMOTE:
-    pass
-else:
-    CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
-    TOKENIZER_DIR = os.path.join(CURRENT_DIR, "chatglm_6b")
-    tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_DIR, trust_remote_code=True)
+
+# if config.USE_REMOTE:
+tokenizer = AutoTokenizer.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True)
+# else:
+#     os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"
+#     CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
+#     TOKENIZER_DIR = os.path.join(CURRENT_DIR, "chatglm_6b")
+#     tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_DIR, trust_remote_code=True)
 
 # https://huggingface.co/THUDM/chatglm-6b/blob/main/tokenization_chatglm.py#L153
 tokenizer.comments = f"num_image_tokens: {tokenizer.sp_tokenizer.num_image_tokens}; num_image_tokens: {tokenizer.sp_tokenizer.num_text_tokens} "

vocab/code_davinci_002/__init__.py

@@ -0,0 +1,3 @@
+
+
+from vocab.text_davinci_003 import tokenizer

vocab/deepseek_coder_33b_instruct/__init__.py

@@ -0,0 +1,7 @@
+"""
+https://huggingface.co/spaces/deepseek-ai/deepseek-coder-7b-instruct
+"""
+
+from transformers import AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("deepseek-ai/deepseek-coder-33b-instruct", trust_remote_code=True)

vocab/deepseek_llm_7b_base/__init__.py

@@ -0,0 +1,5 @@
+
+
+from transformers import AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("deepseek-ai/deepseek-llm-7b-base", trust_remote_code=True)

vocab/gpt_35_turbo/__init__.py

@@ -42,8 +42,6 @@ def get_vocab(self, token_type="str"):
     key_error_list = []
     unicode_decode_error_list = []
     for i in range(self.vocab_size):
-        if i == 100256:
-            print(i)
         try:
             token_byte = self.convert_ids_to_tokens([i])[0]
             if token_byte is None:

vocab/text_davinci_003/__init__.py

@@ -0,0 +1,70 @@
+"""
+TODO
+"""
+
+import tiktoken
+from tiktoken import Encoding
+from utils.log_util import logger
+
+tokenizer = tiktoken.encoding_for_model('text-davinci-003')
+tokenizer.vocab_size = tokenizer.n_vocab
+
+tokenizer.comments = ""
+tokenizer.reversible = True
+
+
+
+
+def decode(self, tokens, errors="replace", skip_special_tokens=False):
+    """
+    默认的decode，可能会报错，详见 decode_test.py
+    skip_special_tokens 是为了兼容 hf_tokenizer
+    """
+    try:
+        decode_str = self._core_bpe.decode_bytes(tokens).decode("utf-8", errors=errors)
+    except:
+        decode_str = "null"
+    return decode_str
+
+def convert_ids_to_tokens(self, tokens, skip_special_tokens=False):
+    """
+    为什么没有这个方法？
+    """
+    try:
+        return tokenizer.decode_tokens_bytes(tokens)
+    except:
+        # 什么要返回None？见zh_util.py
+        # 16个空闲id, 100256 100261-100275
+        return [None for token in tokens]
+
+def get_vocab(self, token_type="str"):
+    """Returns vocab as a dict
+    :param token_type: ["str", "byte"]
+    :return:
+    """
+    vocab = {}
+    key_error_list = []
+    unicode_decode_error_list = []
+    for i in range(self.vocab_size):
+        try:
+            token_byte = self.convert_ids_to_tokens([i])[0]
+            if token_byte is None:
+                continue
+            # token_str = token_byte.decode("utf-8")
+            vocab[token_byte] = i
+
+        except UnicodeDecodeError:  # 773 UnicodeDecodeError
+            unicode_decode_error_list.append((i, str(token_byte)))
+            vocab[token_byte] = i
+
+    # vocab.update(self.added_tokens_encoder)
+    logger.info(f"text-davinci-003 {len(key_error_list)} KeyError: {key_error_list}")
+    logger.info(f"text-davinci-003 {len(unicode_decode_error_list)} UnicodeDecodeError: {unicode_decode_error_list[:5]}")
+    return vocab
+
+
+
+# tiktoken patch
+Encoding.decode = decode
+Encoding.convert_ids_to_tokens = convert_ids_to_tokens
+Encoding.get_vocab = get_vocab

vocab/tigerbot_13b_chat_v2/__init__.py

@@ -0,0 +1,5 @@
+
+
+from transformers import AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("TigerResearch/tigerbot-13b-chat-v2", trust_remote_code=True)

vocab/tigerbot_70b_chat_v4_4k/__init__.py

@@ -0,0 +1,5 @@
+
+
+from transformers import AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("TigerResearch/tigerbot-70b-chat-v4-4k", trust_remote_code=True)

vocab/wizardcoder_15b_v1/__init__.py

@@ -0,0 +1,4 @@
+
+from transformers import AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("WizardLM/WizardCoder-15B-V1.0", trust_remote_code=True)

vocab/wizardcoder_python_7b_v1/__init__.py

@@ -0,0 +1,4 @@
+
+from transformers import AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("WizardLM/WizardCoder-Python-7B-V1.0", trust_remote_code=True)

vocab/wizardlm_7b_v1/__init__.py

@@ -0,0 +1,4 @@
+
+from transformers import AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("WizardLM/WizardLM-7B-V1.0", trust_remote_code=True)

vocab/wizardmath_70b_v1/__init__.py

@@ -0,0 +1,4 @@
+
+from transformers import AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("WizardLM/WizardMath-70B-V1.0", trust_remote_code=True)