add more tokenizer

requirements.txt

@@ -4,4 +4,5 @@ tiktoken
 icetk
 torch
 zhon
-nltk
+nltk
+boto3

util.py

@@ -35,7 +35,7 @@ def tokenize(text, tokenizer_type, color_num=5):
                 token_str = token.decode("utf-8")
             except:
                 token_str = token.decode("utf-8", errors="ignore")
-                logger.error("decode_error: " + json.dumps(    # gpt_35_turbo 经常有token会decode error，这里用来记录一下
+                logger.error(f"{idx}: decode_error: " + json.dumps(    # gpt_35_turbo 经常有token会decode error，这里用来记录一下
                     {"tokenizer_type": tokenizer_type, "token": str(token), "token_str": token_str},
                     ensure_ascii=False))
 

vocab/__init__.py

@@ -20,6 +20,8 @@ tokenizer.implementation = TokenizerImpl.SentencePiece.name   # https://github.c
     - 示例：
   - gpt2
     - 特征：
+      - 词典：
+      
   - sentencepiece: 
     - 特征：.sp_model 是SentencePieceProcessor类型，sp_model.id_to_piece，有tokenizer.json tokenizer.model，词典字符有 ▁，
     - 示例：llama，baichuan
@@ -28,11 +30,14 @@ tokenizer.implementation = TokenizerImpl.SentencePiece.name   # https://github.c
   - tiktoken
   - hf_tokenizer
     - 特征：
+      - 文件: tokenizer.json(包含后两个文件的内容), merges.txt, vocab.json
+        - added_tokens 在vocab中不一定存在。
       - .model 是 tokenizer.models.BPE 类型
       - 词典有 Ġ  "\u0120" 开头
-      - 有1个tokenizer.json(包括 merge vocab)，或者分开独立文件
       - .model.from_file  .model.save   .model.token_to_id  .model.tokenize
-    - 示例：gpt_neox_20b, moss, bloom
+    - 示例：gpt2, gpt_neox_20b, moss, bloom, qwen2
+    - 构造词典：
+      - ss
   - tiktoken
     - 特征：空格就是空格，
     - 示例：gpt3.5 gpt4, qwen, 
@@ -76,6 +81,7 @@ all_tokenizers = [
     # "belle_7b_2m",   # 模型和词典都基于bloom
     #
     "gpt_nexo_20b",
+    "qwen1_5_14b_chat",
     # "gpt_neox_chinese_v1",
     #
     # ##### glm系列
@@ -134,6 +140,7 @@ all_tokenizers = [
     "mobilenet_v2",
     "switch_c_2048",
     "byt5_small",
+    "mt5_large",
 "wizardcoder_python_7b_v1",
 "wizardlm_7b_v1",
 "wizardmath_70b_v1",
@@ -141,6 +148,7 @@ all_tokenizers = [
 "tigerbot_13b_chat_v2",
 "deepseek_coder_33b_instruct",
 "deepseek_llm_7b_base",
+    "gemma_7b",
 
 
 ]

vocab/gemma_7b/__init__.py

@@ -0,0 +1,7 @@
+
+
+from transformers import AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("google/gemma-7b", trust_remote_code=True)
+
+tokenizer.comments = ""

vocab/gpt2/__init__.py

@@ -8,7 +8,7 @@ from vocab import TokenizerType, TokenizerImpl
 # TOKENIZER_DIR = os.path.join(CURRENT_DIR, "tokenizer")
 # tokenizer = GPT2Tokenizer.from_pretrained(TOKENIZER_DIR)
 
-tokenizer = GPT2Tokenizer.from_pretrained("gpt2")
+tokenizer = GPT2Tokenizer.from_pretrained("openai-community/gpt2")
 
 # tokenizer.type = TokenizerType.
 

vocab/mobilenet_v2/__init__.py

@@ -1,2 +1,13 @@
+"""
+## error
+  File "/home/user/app/vocab/mobilenet_v2/__init__.py", line 2, in <module>
+    tokenizer = AutoTokenizer.from_pretrained("google/mobilenet_v2_1.0_224", trust_remote_code=True)
+  File "/home/user/.local/lib/python3.10/site-packages/transformers/models/auto/tokenization_auto.py", line 830, in from_pretrained
+    tokenizer_class_py, tokenizer_class_fast = TOKENIZER_MAPPING[type(config)]
+  File "/home/user/.local/lib/python3.10/site-packages/transformers/models/auto/auto_factory.py", line 748, in __getitem__
+    raise KeyError(key)
+KeyError: <class 'transformers.models.mobilenet_v2.configuration_mobilenet_v2.MobileNetV2Config'>
+"""
+
 from transformers import AutoTokenizer
 tokenizer = AutoTokenizer.from_pretrained("google/mobilenet_v2_1.0_224", trust_remote_code=True)

vocab/mt5_large/__init__.py

@@ -0,0 +1,5 @@
+
+from transformers import AutoTokenizer
+
+
+tokenizer = AutoTokenizer.from_pretrained("google/mt5-large")

vocab/qwen1_5_14b_chat/README.md

@@ -0,0 +1,18 @@
+
+
+## eos token
+
+ 
+```sh
+151643 <|endoftext|>
+151645 <|im_end|>
+```
+
+
+`eos_token_id=[151643, 151645]` [chat generation_config.json](https://huggingface.co/Qwen/Qwen1.5-7B-Chat/blob/main/generation_config.json)，在`model.generate`时会加载这个文件。实际sft时的eos是 `<|im_end|>`，并未用到 `<|endoftext|>`。
+
+
+`eos_token_id=151643` [base generation_config.json](https://huggingface.co/Qwen/Qwen1.5-14B/blob/main/generation_config.json)，在预训练阶段用的是`<|endoftext|>`
+
+`eos_token_id=151643`  [tokenization_qwen2.py](https://github.com/huggingface/transformers/blob/2a9b1f80c45cab19b542bc7cc004937d39d6f6fb/src/transformers/models/qwen2/tokenization_qwen2.py#L150)，`print(tokenizer.eos_token_id)` 会显示 `151643`，是为了兼容 base model和chat model。
+

vocab/qwen1_5_14b_chat/__init__.py

@@ -0,0 +1,8 @@
+
+
+import os
+from transformers import AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen1.5-14B-Chat", trust_remote_code=True)
+
+tokenizer.comments = ""