add more tokenizers

.gitattributes

@@ -35,3 +35,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 vocab/belle_7b_2m/belle-7b-2m/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 vocab/bloom/tokenizer/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+vocab/

README.md

@@ -17,3 +17,12 @@ Check out the configuration reference at https://huggingface.co/docs/hub/spaces-
 
 ## ss
 
+
+
+
+## 统计
+
+
+
+
+https://huggingface.co/spaces/yenniejun/tokenizers-languages

app.py

@@ -18,6 +18,7 @@
 - 词典支持下载，借用image下载的标签，
 - baichuan的单字数量怎么两万多个？
 - qwen:  ValueError: Unclosed image token
+- 路径修改为全path  meta-llama/Llama-2-13b-hf
 
 plots
 
@@ -188,12 +189,5 @@ with gr.Blocks(css="css/style.css", title="Tokenizer Arena") as demo:
 
 
 if __name__ == "__main__":
-    print("http://127.0.0.1:7860/?tokenizer1=llama&tokenizer2=chinese_llama2&text=fdsjlk")  # llama chinese_llama2
-    print(
-        "http://127.0.0.1:7860/?tokenizer1=chinese_llama&tokenizer2=chinese_llama2&text=fdsjlk")  # llama chinese_llama2
-    print("http://127.0.0.1:7860/?tokenizer1=baichuan&tokenizer2=baichuan2&text=sss")  # baichuan 1 VS 2
-    print("http://127.0.0.1:7860/?tokenizer1=bert&tokenizer2=clue&text=sss")  # bert VS clue
-    print("http://127.0.0.1:7860/?tokenizer1=clue&tokenizer2=kplug&text=sss")  # clue VS kplug
-    print("http://127.0.0.1:7860/?tokenizer1=baichuan&tokenizer2=baichuan2&text=sss")  #
     # demo.queue(max_size=20).launch()
     demo.launch()

app_v1.py

@@ -1,196 +0,0 @@
-# coding=utf-8
-# author: xusong
-# time: 2022/8/23 16:06
-
-"""
-
-plots
-
-table
-
-## related demo
-- [](http://text-processing.com/demo/tokenize/)
-- [gpt-tokenizer](https://gpt-tokenizer.dev/)
-- [llama-tokenizer-js](https://belladoreai.github.io/llama-tokenizer-js/example-demo/build/)
-- [](https://huggingface.co/spaces/Xenova/the-tokenizer-playground)
-
-## 可视化
-
-[ The, 2, QUICK, Brown, Foxes, jumped, over, the, lazy, dog's, bone ]
-"""
-
-import json
-import pandas as pd
-import gradio as gr
-
-from vocab import all_tokenizers, load_tokener
-
-# 显示空格：https://blog.csdn.net/liuxiao723846/article/details/118994673
-# 隐藏legend：
-css = """
-.space-show {white-space: pre-wrap;}
-.cell-wrap {white-space: pre-wrap;}
-.category-legend {display: none !important}
-"""
-
-example_text = """Replace this text in the input field to see how tokenization works
-中文测试：华为智能音箱发布：华为Sound X。維基百科由非營利組織──維基媒體基金會負責維持
-数字测试：(10086 + 98) = 100184"""
-
-# llama chatglm_6b gpt_nexo_20b baichuan  baichuan_7b
-examples = [
-    # ["空格测试：  2个空格        8个空格", "llama", "chatglm_6b"],  # chatglm 有blank_n,
-    ["标点测试：，。！？；", "baichuan_7b", "llama"],
-    ["标点测试：🦙", "baichuan_7b", "llama"],
-]
-
-
-def tokenize(text, tokenizer_type, color_num=5):
-    print(text, tokenizer_type)
-    pos_tokens = []
-    tokenizer = load_tokener(tokenizer_type)
-    encoding = tokenizer.encode(text)
-
-    table = []
-
-    for idx, token_id in enumerate(encoding):
-        decode_text = tokenizer.decode([token_id])  # 特殊字符解码后会统一变成 �，对应 "\ufffd"
-        pos_tokens.extend([(decode_text, str(idx % color_num))])
-
-        # token  "Byte":  # 这是 utf-8编码吧？
-        token = tokenizer.convert_ids_to_tokens([token_id])[0]
-        if isinstance(token, bytes):
-            try:
-                token_str = token.decode("utf-8")
-            except:
-                token_str = token.decode("utf-8", errors="ignore")
-                print("decode_error", token, token_str)
-
-            token_bytes = token
-            json_dumps = json.dumps(token_str)
-        elif isinstance(token, str):
-            token_str = token
-            token_bytes = bytes(token_str, "utf-8")
-            json_dumps = json.dumps(token_str)
-        else:
-            return
-
-        table.append(
-            {"TokenID": token_id,
-             "Token": token_str,  # utf-8解码后的字符串，为什么有些是 <0xE7>，表示什么？比如llama
-             "Text": decode_text,  #
-             # "Bytes": token_bytes,  # bytes类型在gradio前端页面被解码成字符串，比如   b'\xe4\xb8\xad' 仍然显示成 "中"。因此 str(token_bytes)
-             "Bytes": str(token_bytes),
-             # "Unicode": json_dumps  # unicode, 如果是ascii码，就直接显示。如果不是ascii码，就显示unicode
-             }
-        )
-
-    table_df = pd.DataFrame(table)
-    print(table)
-    # print(table_df)
-
-    return pos_tokens, table_df
-
-
-def tokenize_pair(text, tokenizer_type_1, tokenizer_type_2):
-    pos_tokens_1, table_df_1 = tokenize(text, tokenizer_type_1)
-    pos_tokens_2, table_df_2 = tokenize(text, tokenizer_type_2)
-    return pos_tokens_1, table_df_1, pos_tokens_2, table_df_2
-
-
-def test_coding():
-    bytes1 = b'\xe4\xb8\xad'
-    print(bytes1)  # b'\xe4\xb8\xad'
-
-
-with gr.Blocks(css=css) as demo:
-    gr.HTML("""<h1 align="center">The Tokenizer Arena</h1>""")
-    # links: https://www.coderstool.com/utf8-encoding-decoding
-    #
-
-
-
-    gr.Markdown("## Input Text")
-    user_input = gr.Textbox(
-        value=example_text,
-        label="Input Text",
-        lines=5
-    )  # placeholder="Enter sentence here..."
-
-    # submitBtn = gr.Button("生成回复", variant="primary")
-
-    gr.Markdown("## Tokenization")
-
-    # with gr.Row():
-
-
-
-    # TODO: 图 表 压缩率
-    with gr.Row():
-        with gr.Column():
-            tokenizer_type_1 = gr.Dropdown(
-                all_tokenizers,
-                value="llama",
-                label="Tokenizer 1",
-            )
-            token_counter_1 = None  # 计数器
-            output_text_1 = gr.Highlightedtext(
-                label="Tokens 1",
-                show_legend=True,
-                elem_classes="space-show"
-            )
-
-        with gr.Column():
-            tokenizer_type_2 = gr.Dropdown(
-                all_tokenizers,
-                value="baichuan_7b",
-                label="Tokenizer 2"
-            )
-            token_counter_2 = None  # 计数器
-            output_text_2 = gr.Highlightedtext(
-                label="Tokens 2",
-                show_legend=True,
-                elem_classes="space-show"
-            )
-
-    with gr.Row():
-        output_table_1 = gr.Dataframe(
-            headers=["TokenID", "Byte", "Text"],
-            datatype=["str", "str", "str"],
-            # elem_classes="space-show",   # 给���个Dataframe加这个css不起作用，因此直接修改cell-wrap
-        )
-        output_table_2 = gr.Dataframe(
-            headers=["TokenID", "Token", "Text"],
-            datatype=["str", "str", "str"],
-        )
-
-    user_input.change(tokenize,
-                      [user_input, tokenizer_type_1],
-                      [output_text_1, output_table_1])
-    tokenizer_type_1.change(tokenize, [user_input, tokenizer_type_1], [output_text_1, output_table_1])
-
-    user_input.change(tokenize,
-                      [user_input, tokenizer_type_2],
-                      [output_text_2, output_table_2])
-
-    tokenizer_type_2.change(tokenize, [user_input, tokenizer_type_2], [output_text_2, output_table_2])
-
-    gr.Examples(
-        examples,
-        [user_input, tokenizer_type_1, tokenizer_type_2],
-        [output_text_1, output_table_1, output_text_2, output_table_2],
-        tokenize_pair,
-        cache_examples=True,
-    )
-
-    # submitBtn.click(tokenize, [user_input, tokenizer_type], outputs,
-    #                 show_progress=True)
-
-    # examples=[
-    #     ["What a beautiful morning for a walk!"],
-    #     ["It was the best of times, it was the worst of times."],
-    #     ["多个空格    It  ss  was the best of times, it was the worst of times."],
-    # ]
-
-if __name__ == "__main__":
-    demo.launch()

config.py

@@ -0,0 +1,3 @@
+
+
+USE_REMOTE = False

examples.py

@@ -17,6 +17,43 @@ examples = {
 
 }
 
+more_examples = [
+    # bert VS clue
+    # bert系列
+    ("bert_base_cased", "bert_base_uncased", ""),  # # clue VS kplug， bert VS clue
+
+    # llama系列 (基于sentencepiece)
+    ("baichuan", "baichuan2", "baichuan2支持多空格   ，多个换行\n\n\n，do not add dummy prefix as Baichuan1"),
+    ("llama", "baichuan2", "baichuan2支持多空格   ，多个换行\n\n"),
+    ("llama", "chinese_llama2", ""),
+    ("chinese_llama", "chinese_llama2", ""),
+
+    # glm系列 （基于sentencepiece）
+    ("glm", "chatglm1", ""),
+    ("chatglm1", "chatglm2", ""),
+
+    # gpt2系列
+    ("gpt2", "moss", ""),
+    ("", "", ""),
+
+    # openai系列 （tiktoken）
+    ("qwen", "gpt_35_turbo", ""),
+
+
+]
+
 
 def example_fn(example_idx):
     return examples["en"][example_idx]
+
+
+def get_more_example():
+    import urllib.parse
+    url_prefix = "https://huggingface.co/spaces/eson/tokenizer-arena"
+    for tokenizer1, tokenizer2, text in more_examples:
+        full_url = f'{url_prefix}?tokenizer1={tokenizer1}&tokenizer2={tokenizer2}&text={urllib.parse.quote(text)}'
+        print(full_url)
+
+
+if __name__ == "__main__":
+    get_more_example()

requirements.txt

@@ -1,4 +1,4 @@
-transformers>=4.21.1
+transformers==4.33.1
 sentencepiece
 tiktoken
 icetk

util.py

@@ -25,7 +25,7 @@ def tokenize(text, tokenizer_type, color_num=5):
         pos_tokens.extend([(decode_text, str(idx % color_num))])
 
         # token  "Byte":  # 这是 utf-8编码吧？
-        token = tokenizer.convert_ids_to_tokens([token_id])[0]
+        token = tokenizer.convert_ids_to_tokens([token_id], skip_special_tokens=False)[0]
         if isinstance(token, bytes):
             try:
                 token_str = token.decode("utf-8")

utils/_vocab.zh.jsonl

@@ -1,1189 +0,0 @@
-{"id": 529, "token": "’", "type": "中文标点"}
-{"id": 753, "token": "’s", "type": "中文标点"}
-{"id": 863, "token": "”", "type": "中文标点"}
-{"id": 1054, "token": " “", "type": "中文标点"}
-{"id": 1389, "token": " –", "type": "中文标点"}
-{"id": 1431, "token": "’t", "type": "中文标点"}
-{"id": 1811, "token": "。", "type": "中文标点"}
-{"id": 1981, "token": "…", "type": "中文标点"}
-{"id": 2001, "token": " —", "type": "中文标点"}
-{"id": 2029, "token": ".”", "type": "中文标点"}
-{"id": 2118, "token": "“", "type": "中文标点"}
-{"id": 2345, "token": "—", "type": "中文标点"}
-{"id": 2476, "token": ",”", "type": "中文标点"}
-{"id": 2950, "token": ".”\n\n", "type": "中文标点"}
-{"id": 3207, "token": "’re", "type": "中文标点"}
-{"id": 3451, "token": " ‘", "type": "中文标点"}
-{"id": 3490, "token": "。\n\n", "type": "中文标点"}
-{"id": 3922, "token": "，", "type": "中文标点"}
-{"id": 4070, "token": "’ve", "type": "中文标点"}
-{"id": 4235, "token": "–", "type": "中文标点"}
-{"id": 4344, "token": "’m", "type": "中文标点"}
-{"id": 4696, "token": " …", "type": "中文标点"}
-{"id": 4805, "token": "’ll", "type": "中文标点"}
-{"id": 5232, "token": "：", "type": "中文标点"}
-{"id": 5486, "token": "、", "type": "中文标点"}
-{"id": 5551, "token": "…\n\n", "type": "中文标点"}
-{"id": 6447, "token": "！", "type": "中文标点"}
-{"id": 7070, "token": "’d", "type": "中文标点"}
-{"id": 7663, "token": "”\n\n", "type": "中文标点"}
-{"id": 7705, "token": "）", "type": "中文标点"}
-{"id": 8107, "token": "年", "type": "中文单字"}
-{"id": 8713, "token": "——", "type": "中文标点"}
-{"id": 9039, "token": "数", "type": "中文单字"}
-{"id": 9080, "token": "日", "type": "中文单字"}
-{"id": 9174, "token": "。\n", "type": "中文标点"}
-{"id": 9520, "token": "”,", "type": "中文标点"}
-{"id": 9554, "token": "的", "type": "中文单字"}
-{"id": 9787, "token": " ·", "type": "中文标点"}
-{"id": 9953, "token": "月", "type": "中文单字"}
-{"id": 10110, "token": "（", "type": "中文标点"}
-{"id": 10378, "token": "“I", "type": "中文标点"}
-{"id": 10416, "token": " […", "type": "中文标点"}
-{"id": 10646, "token": "」", "type": "中文标点"}
-{"id": 11144, "token": "【", "type": "中文标点"}
-{"id": 11199, "token": "】", "type": "中文标点"}
-{"id": 11453, "token": "”.", "type": "中文标点"}
-{"id": 11571, "token": "？", "type": "中文标点"}
-{"id": 11883, "token": "用", "type": "中文单字"}
-{"id": 12291, "token": " …\n\n", "type": "中文标点"}
-{"id": 12671, "token": "?”", "type": "中文标点"}
-{"id": 12996, "token": " […]\n\n", "type": "中文标点"}
-{"id": 13153, "token": "成", "type": "中文单字"}
-{"id": 13177, "token": "「", "type": "中文标点"}
-{"id": 13372, "token": "名", "type": "中文单字"}
-{"id": 13646, "token": "时", "type": "中文单字"}
-{"id": 14260, "token": "·", "type": "中文标点"}
-{"id": 14305, "token": "“The", "type": "中文标点"}
-{"id": 14336, "token": "‘", "type": "中文标点"}
-{"id": 14382, "token": "……", "type": "中文标点"}
-{"id": 14558, "token": "件", "type": "中文单字"}
-{"id": 14639, "token": ".’", "type": "中文标点"}
-{"id": 15085, "token": "“We", "type": "中文标点"}
-{"id": 15120, "token": "一", "type": "中文单字"}
-{"id": 15179, "token": " „", "type": "中文标点"}
-{"id": 15225, "token": "请", "type": "中文单字"}
-{"id": 15397, "token": "”.\n\n", "type": "中文标点"}
-{"id": 16325, "token": "中", "type": "中文单字"}
-{"id": 16423, "token": "据", "type": "中文单字"}
-{"id": 16616, "token": "?”\n\n", "type": "中文标点"}
-{"id": 16620, "token": "————", "type": "中文标点"}
-{"id": 16882, "token": "码", "type": "中文单字"}
-{"id": 16937, "token": "不", "type": "中文单字"}
-{"id": 17039, "token": "新", "type": "中文单字"}
-{"id": 17161, "token": "文", "type": "中文单字"}
-{"id": 17223, "token": "—and", "type": "中文标点"}
-{"id": 17297, "token": "下", "type": "中文单字"}
-{"id": 17620, "token": "分", "type": "中文单字"}
-{"id": 17701, "token": "入", "type": "中文单字"}
-{"id": 17792, "token": "人", "type": "中文单字"}
-{"id": 17818, "token": "“It", "type": "中文标点"}
-{"id": 17860, "token": "功", "type": "中文单字"}
-{"id": 17905, "token": "上", "type": "中文单字"}
-{"id": 17982, "token": "户", "type": "中文单字"}
-{"id": 18171, "token": "！\n\n", "type": "中文标点"}
-{"id": 18184, "token": "为", "type": "中文单字"}
-{"id": 18217, "token": " ’", "type": "中文标点"}
-{"id": 18319, "token": "!”", "type": "中文标点"}
-{"id": 18363, "token": "间", "type": "中文单字"}
-{"id": 18476, "token": "号", "type": "中文单字"}
-{"id": 18655, "token": "取", "type": "中文单字"}
-{"id": 18904, "token": "回", "type": "中文单字"}
-{"id": 19000, "token": "在", "type": "��文单字"}
-{"id": 19047, "token": "页", "type": "中文单字"}
-{"id": 19066, "token": "。\n\n\n\n", "type": "中文标点"}
-{"id": 19113, "token": "字", "type": "中文单字"}
-{"id": 19361, "token": "有", "type": "中文单字"}
-{"id": 19483, "token": "个", "type": "中文单字"}
-{"id": 19524, "token": " ”", "type": "中文标点"}
-{"id": 19653, "token": "成功", "type": "中文多字"}
-{"id": 19967, "token": "作", "type": "中文单字"}
-{"id": 20145, "token": "】【", "type": "中文标点"}
-{"id": 20182, "token": "’,", "type": "中文标点"}
-{"id": 20379, "token": "示", "type": "中文单字"}
-{"id": 20600, "token": "用户", "type": "中文多字"}
-{"id": 20675, "token": "数据", "type": "中文多字"}
-{"id": 20834, "token": "出", "type": "中文单字"}
-{"id": 21043, "token": "是", "type": "中文单字"}
-{"id": 21060, "token": "….", "type": "中文标点"}
-{"id": 21082, "token": "时间", "type": "中文多字"}
-{"id": 21388, "token": "失", "type": "中文单字"}
-{"id": 21405, "token": "表", "type": "中文单字"}
-{"id": 21418, "token": "除", "type": "中文单字"}
-{"id": 21601, "token": "加", "type": "中文单字"}
-{"id": 21809, "token": "败", "type": "中文单字"}
-{"id": 21909, "token": "～", "type": "中文标点"}
-{"id": 21990, "token": "生", "type": "中文单字"}
-{"id": 22023, "token": "信", "type": "中文单字"}
-{"id": 22117, "token": "’est", "type": "中文标点"}
-{"id": 22238, "token": "类", "type": "中文单字"}
-{"id": 22324, "token": "置", "type": "中文单字"}
-{"id": 22416, "token": "—the", "type": "中文标点"}
-{"id": 22649, "token": "理", "type": "中文单字"}
-{"id": 22656, "token": "本", "type": "中文单字"}
-{"id": 22820, "token": "失败", "type": "中文多字"}
-{"id": 23018, "token": "息", "type": "中文单字"}
-{"id": 23039, "token": "行", "type": "中文单字"}
-{"id": 23187, "token": "定", "type": "中文单字"}
-{"id": 23189, "token": ",’", "type": "中文标点"}
-{"id": 23226, "token": "改", "type": "中文单字"}
-{"id": 23249, "token": "　", "type": "中文标点"}
-{"id": 23530, "token": "市", "type": "中文单字"}
-{"id": 23538, "token": "期", "type": "中文单字"}
-{"id": 23897, "token": "以", "type": "中文单字"}
-{"id": 23951, "token": "修", "type": "中文单字"}
-{"id": 23954, "token": "）\n", "type": "中文标点"}
-{"id": 24186, "token": "元", "type": "中文单字"}
-{"id": 24273, "token": "方", "type": "中文单字"}
-{"id": 24535, "token": "’.", "type": "中文标点"}
-{"id": 24580, "token": "录", "type": "中文单字"}
-{"id": 24775, "token": "区", "type": "中文单字"}
-{"id": 24946, "token": "单", "type": "中文单字"}
-{"id": 25010, "token": "�除", "type": "中文多字"}
-{"id": 25129, "token": "位", "type": "中文单字"}
-{"id": 25287, "token": "型", "type": "中文单字"}
-{"id": 25333, "token": "法", "type": "中文单字"}
-{"id": 25336, "token": "县", "type": "中文单字"}
-{"id": 25359, "token": "存", "type": "中文单字"}
-{"id": 25446, "token": "品", "type": "中文单字"}
-{"id": 25580, "token": "前", "type": "中文单字"}
-{"id": 25666, "token": "称", "type": "中文单字"}
-{"id": 25758, "token": "!”\n\n", "type": "中文标点"}
-{"id": 26016, "token": "；", "type": "中文标点"}
-{"id": 26062, "token": "�回", "type": "中文多字"}
-{"id": 26123, "token": "》", "type": "中文标点"}
-{"id": 26130, "token": "注", "type": "中文单字"}
-{"id": 26239, "token": "修改", "type": "中文多字"}
-{"id": 26592, "token": "值", "type": "中文单字"}
-{"id": 26794, "token": "输", "type": "中文单字"}
-{"id": 26892, "token": "建", "type": "中文单字"}
-{"id": 27179, "token": " (“", "type": "中文标点"}
-{"id": 27327, "token": "能", "type": "中文单字"}
-{"id": 27384, "token": "大", "type": "中文单字"}
-{"id": 27452, "token": "例", "type": "中文单字"}
-{"id": 27479, "token": "度", "type": "中文单字"}
-{"id": 27704, "token": "始", "type": "中文单字"}
-{"id": 27948, "token": "？\n\n", "type": "中文标点"}
-{"id": 27996, "token": "文件", "type": "中文多字"}
-{"id": 28037, "token": "到", "type": "中文单字"}
-{"id": 28038, "token": "《", "type": "中文标点"}
-{"id": 28190, "token": "面", "type": "中文单字"}
-{"id": 28359, "token": "�数", "type": "中文多字"}
-{"id": 28466, "token": "载", "type": "中文单字"}
-{"id": 28469, "token": "信息", "type": "中文多字"}
-{"id": 28542, "token": "点", "type": "中文单字"}
-{"id": 28587, "token": "��取", "type": "中文多字"}
-{"id": 28624, "token": " […]", "type": "中文标点"}
-{"id": 28741, "token": "密", "type": "中文单字"}
-{"id": 28833, "token": "动", "type": "中文单字"}
-{"id": 28873, "token": "果", "type": "中文单字"}
-{"id": 28918, "token": "、\n\n", "type": "中文标点"}
-{"id": 28966, "token": "）\n\n", "type": "中文标点"}
-{"id": 29096, "token": "—a", "type": "中文标点"}
-{"id": 29129, "token": "图", "type": "中文单字"}
-{"id": 29172, "token": "提", "type": "中文单字"}
-{"id": 29391, "token": "发", "type": "中文单字"}
-{"id": 29411, "token": "：\n", "type": "中文标点"}
-{"id": 29430, "token": "式", "type": "中文单字"}
-{"id": 29472, "token": "—\n\n", "type": "中文标点"}
-{"id": 29504, "token": "国", "type": "中文单字"}
-{"id": 29681, "token": "」\n\n", "type": "中文标点"}
-{"id": 29706, "token": "删除", "type": "中文多字"}
-{"id": 29719, "token": "’un", "type": "中文标点"}
-{"id": 29741, "token": "登", "type": "中文单字"}
-{"id": 29826, "token": "错", "type": "中文单字"}
-{"id": 30019, "token": "。。", "type": "中文标点"}
-{"id": 30046, "token": "者", "type": "中文单字"}
-{"id": 30051, "token": "认", "type": "中文单字"}
-{"id": 30156, "token": "误", "type": "中文单字"}
-{"id": 30177, "token": "接", "type": "中文单字"}
-{"id": 30184, "token": "’\n\n", "type": "中文标点"}
-{"id": 30356, "token": "关", "type": "中文单字"}
-{"id": 30358, "token": "重", "type": "中文单字"}
-{"id": 30537, "token": "第", "type": "中文单字"}
-{"id": 30590, "token": "地", "type": "中文单字"}
-{"id": 30624, "token": "如", "type": "中文单字"}
-{"id": 30697, "token": "————————", "type": "中文标点"}
-{"id": 30735, "token": "设", "type": "中文单字"}
-{"id": 30832, "token": "目", "type": "中文单字"}
-{"id": 30867, "token": "开", "type": "中文单字"}
-{"id": 30926, "token": "事", "type": "中文单字"}
-{"id": 31041, "token": "�数", "type": "中文多字"}
-{"id": 31091, "token": "名称", "type": "中文多字"}
-{"id": 31378, "token": "“This", "type": "中文标点"}
-{"id": 31472, "token": " ：", "type": "中文标点"}
-{"id": 31540, "token": "可", "type": "中文单字"}
-{"id": 31634, "token": "要", "type": "中文单字"}
-{"id": 31640, "token": "代", "type": "中文单字"}
-{"id": 31809, "token": "小", "type": "中文单字"}
-{"id": 31867, "token": "选", "type": "中文单字"}
-{"id": 31944, "token": "标", "type": "中文单字"}
-{"id": 31958, "token": "明", "type": "中文单字"}
-{"id": 31968, "token": "编", "type": "中文单字"}
-{"id": 32018, "token": "求", "type": "中文单字"}
-{"id": 32218, "token": "列", "type": "中文单字"}
-{"id": 32239, "token": "网", "type": "中文单字"}
-{"id": 32296, "token": "输入", "type": "中文多字"}
-{"id": 32307, "token": "万", "type": "中文单字"}
-{"id": 32335, "token": "最", "type": "中文单字"}
-{"id": 32351, "token": "！！", "type": "中文标点"}
-{"id": 32438, "token": "�建", "type": "中文多字"}
-{"id": 32626, "token": "返回", "type": "中文多字"}
-{"id": 32648, "token": "器", "type": "中文单字"}
-{"id": 32938, "token": "所", "type": "中文单字"}
-{"id": 32943, "token": "内", "type": "中文单字"}
-{"id": 33005, "token": "类型", "type": "中文多字"}
-{"id": 33014, "token": "体", "type": "中文单字"}
-{"id": 33035, "token": "通", "type": "中文单字"}
-{"id": 33052, "token": "务", "type": "中文单字"}
-{"id": 33091, "token": "此", "type": "中文单字"}
-{"id": 33122, "token": "商", "type": "中文单字"}
-{"id": 33144, "token": "序", "type": "中文单字"}
-{"id": 33200, "token": "错误", "type": "中文多字"}
-{"id": 33208, "token": "化", "type": "中文单字"}
-{"id": 33420, "token": "消", "type": "中文单字"}
-{"id": 33476, "token": "否", "type": "中文单字"}
-{"id": 33563, "token": "保", "type": "中文单字"}
-{"id": 33611, "token": "”)", "type": "中文标点"}
-{"id": 33655, "token": "使", "type": "中文单字"}
-{"id": 33671, "token": "次", "type": "中文单字"}
-{"id": 33672, "token": "“You", "type": "中文标点"}
-{"id": 33748, "token": "机", "type": "中文单字"}
-{"id": 33764, "token": "对", "type": "中文单字"}
-{"id": 33765, "token": "参数", "type": "中文多字"}
-{"id": 33777, "token": "’é", "type": "中文标点"}
-{"id": 33857, "token": "量", "type": "中文单字"}
-{"id": 33904, "token": "函数", "type": "中文多字"}
-{"id": 33967, "token": "密码", "type": "中文多字"}
-{"id": 33976, "token": "查", "type": "中文单字"}
-{"id": 34045, "token": "。”", "type": "中文标点"}
-{"id": 34048, "token": "部", "type": "中文单字"}
-{"id": 34171, "token": "性", "type": "中文单字"}
-{"id": 34208, "token": "和", "type": "中文单字"}
-{"id": 34226, "token": "更", "type": "中文单字"}
-{"id": 34547, "token": "后", "type": "中文单字"}
-{"id": 34577, "token": "证", "type": "中文单字"}
-{"id": 34676, "token": " 【", "type": "中文标点"}
-{"id": 34690, "token": "”，", "type": "中文标点"}
-{"id": 34972, "token": "题", "type": "中文单字"}
-{"id": 35056, "token": "确", "type": "中文单字"}
-{"id": 35083, "token": "格", "type": "中文单字"}
-{"id": 35147, "token": ".“", "type": "中文标点"}
-{"id": 35192, "token": ".—", "type": "中文标点"}
-{"id": 35284, "token": ".”\n\n\n\n", "type": "中文标点"}
-{"id": 35287, "token": "了", "type": "中文单字"}
-{"id": 35304, "token": "���", "type": "中文单字"}
-{"id": 35330, "token": "金", "type": "中文单字"}
-{"id": 35417, "token": "公", "type": "中文单字"}
-{"id": 35424, "token": "午", "type": "中文单字"}
-{"id": 35757, "token": "円", "type": "中文单字"}
-{"id": 35816, "token": "“There", "type": "中文标点"}
-{"id": 35818, "token": "片", "type": "中文单字"}
-{"id": 35894, "token": "空", "type": "中文单字"}
-{"id": 35959, "token": "请求", "type": "中文多字"}
-{"id": 36225, "token": "��加", "type": "中文多字"}
-{"id": 36319, "token": ".’\n\n", "type": "中文标点"}
-{"id": 36343, "token": "态", "type": "中文单字"}
-{"id": 36515, "token": "登录", "type": "中文多字"}
-{"id": 36577, "token": "’une", "type": "中文标点"}
-{"id": 36651, "token": "管", "type": "中文单字"}
-{"id": 36668, "token": "主", "type": "中文单字"}
-{"id": 36761, "token": "』", "type": "中文标点"}
-{"id": 36827, "token": "天", "type": "中文单字"}
-{"id": 36896, "token": "、「", "type": "中文标点"}
-{"id": 37026, "token": "自", "type": "中文单字"}
-{"id": 37046, "token": "我", "type": "中文单字"}
-{"id": 37087, "token": "全", "type": "中文单字"}
-{"id": 37271, "token": "今", "type": "中文单字"}
-{"id": 37395, "token": "页面", "type": "中文多字"}
-{"id": 37507, "token": "来", "type": "中文单字"}
-{"id": 37648, "token": "��作", "type": "中文多字"}
-{"id": 37656, "token": "正", "type": "中文单字"}
-{"id": 37687, "token": "说", "type": "中文单字"}
-{"id": 37689, "token": "意", "type": "中文单字"}
-{"id": 37705, "token": "送", "type": "中文单字"}
-{"id": 37729, "token": "容", "type": "中文单字"}
-{"id": 37767, "token": "已", "type": "中文单字"}
-{"id": 37985, "token": "结", "type": "中文单字"}
-{"id": 38087, "token": "：“", "type": "中文标点"}
-{"id": 38093, "token": "会", "type": "中文单字"}
-{"id": 38129, "token": "使用", "type": "中文多字"}
-{"id": 38232, "token": "。</", "type": "中文标点"}
-{"id": 38365, "token": "。\r\n", "type": "中文标点"}
-{"id": 38542, "token": "—but", "type": "中文标点"}
-{"id": 38574, "token": "段", "type": "中文单字"}
-{"id": 38609, "token": "�认", "type": "中文多字"}
-{"id": 38684, "token": "“If", "type": "中文标点"}
-{"id": 38741, "token": "。，", "type": "中文标点"}
-{"id": 38743, "token": "计", "type": "中文单字"}
-{"id": 39045, "token": "，请", "type": "中文多字"}
-{"id": 39084, "token": "源", "type": "中文单字"}
-{"id": 39135, "token": "色", "type": "中文单字"}
-{"id": 39177, "token": "時", "type": "中文单字"}
-{"id": 39209, "token": "交", "type": "中文单字"}
-{"id": 39276, "token": "系", "type": "中文单字"}
-{"id": 39282, "token": "过", "type": "中文单字"}
-{"id": 39312, "token": "电", "type": "中文单字"}
-{"id": 39365, "token": "询", "type": "中文单字"}
-{"id": 39404, "token": "符", "type": "中文单字"}
-{"id": 39425, "token": "…………", "type": "中文标点"}
-{"id": 39442, "token": "未", "type": "中文单字"}
-{"id": 39607, "token": "程", "type": "中文单字"}
-{"id": 40053, "token": "常", "type": "中文单字"}
-{"id": 40089, "token": "条", "type": "中文单字"}
-{"id": 40195, "token": "下", "type": "中文单字"}
-{"id": 40265, "token": "当", "type": "中文单字"}
-{"id": 40452, "token": "管理", "type": "中文多字"}
-{"id": 40466, "token": "��态", "type": "中文多字"}
-{"id": 40474, "token": "情", "type": "中文单字"}
-{"id": 40526, "token": "口", "type": "中文单字"}
-{"id": 40565, "token": "“He", "type": "中文标点"}
-{"id": 40702, "token": "’S", "type": "中文标点"}
-{"id": 40753, "token": "’a", "type": "中文标点"}
-{"id": 40862, "token": "合", "type": "中文单字"}
-{"id": 41007, "token": "方法", "type": "中文多字"}
-{"id": 41053, "token": "车", "type": "中文单字"}
-{"id": 41073, "token": "实", "type": "中文单字"}
-{"id": 41127, "token": "组", "type": "中文单字"}
-{"id": 41128, "token": "—that", "type": "中文标点"}
-{"id": 41190, "token": "操作", "type": "中文多字"}
-{"id": 41354, "token": "’.\n\n", "type": "中文标点"}
-{"id": 41401, "token": "版", "type": "中文单字"}
-{"id": 41642, "token": "周", "type": "中文单字"}
-{"id": 41723, "token": "址", "type": "中文单字"}
-{"id": 41771, "token": "获取", "type": "中文多字"}
-{"id": 41827, "token": "：\"", "type": "中文标点"}
-{"id": 41914, "token": "记", "type": "中文单字"}
-{"id": 41920, "token": "二", "type": "中文单字"}
-{"id": 42016, "token": "同", "type": "中文单字"}
-{"id": 42052, "token": "业", "type": "中文单字"}
-{"id": 42081, "token": "权", "type": "中文单字"}
-{"id": 42246, "token": "其", "type": "中文单字"}
-{"id": 42275, "token": " ，", "type": "中文标点"}
-{"id": 42399, "token": "进", "type": "中文单字"}
-{"id": 42421, "token": "试", "type": "中文单字"}
-{"id": 42462, "token": "验", "type": "中文单字"}
-{"id": 42506, "token": "料", "type": "中文单字"}
-{"id": 42553, "token": "，\n", "type": "中文标点"}
-{"id": 42605, "token": "，“", "type": "中文标点"}
-{"id": 42783, "token": "传", "type": "中文单字"}
-{"id": 43032, "token": "述", "type": "中文单字"}
-{"id": 43167, "token": "集", "type": "中文单字"}
-{"id": 43240, "token": "多", "type": "中文单字"}
-{"id": 43292, "token": "无", "type": "中文单字"}
-{"id": 43323, "token": "员", "type": "中文单字"}
-{"id": 43378, "token": "报", "type": "中文单字"}
-{"id": 43444, "token": " （", "type": "中文标点"}
-{"id": 43511, "token": "他", "type": "中文单字"}
-{"id": 43568, "token": "無", "type": "中文单字"}
-{"id": 43741, "token": "‘s", "type": "中文标点"}
-{"id": 43955, "token": "添加", "type": "中文多字"}
-{"id": 44130, "token": "“What", "type": "中文标点"}
-{"id": 44309, "token": "服", "type": "中文单字"}
-{"id": 44368, "token": "线", "type": "中文单字"}
-{"id": 44388, "token": "这", "type": "中文单字"}
-{"id": 44416, "token": "制", "type": "中文单字"}
-{"id": 44529, "token": "　　", "type": "中文标点"}
-{"id": 44603, "token": "—it", "type": "中文标点"}
-{"id": 44620, "token": "『", "type": "中文标点"}
-{"id": 44689, "token": "的", "type": "中文单字"}
-{"id": 44816, "token": "�始", "type": "中文多字"}
-{"id": 44820, "token": "�单", "type": "中文多字"}
-{"id": 44915, "token": "内容", "type": "中文多字"}
-{"id": 44996, "token": "’il", "type": "中文标点"}
-{"id": 45018, "token": "设置", "type": "中文多字"}
-{"id": 45059, "token": "生成", "type": "中文多字"}
-{"id": 45163, "token": "将", "type": "中文单字"}
-{"id": 45191, "token": "状态", "type": "中文多字"}
-{"id": 45221, "token": "=”", "type": "中文标点"}
-{"id": 45258, "token": "?’", "type": "中文标点"}
-{"id": 45277, "token": "列表", "type": "中文多字"}
-{"id": 45390, "token": "处", "type": "中文单字"}
-{"id": 45460, "token": "】\n\n", "type": "中文标点"}
-{"id": 45472, "token": "输", "type": "中文单字"}
-{"id": 45516, "token": "！\");\n", "type": "中文标点"}
-{"id": 45631, "token": " 「", "type": "中文标点"}
-{"id": 45736, "token": "高", "type": "中文单字"}
-{"id": 45829, "token": "子", "type": "中文单字"}
-{"id": 45893, "token": "道", "type": "中文单字"}
-{"id": 45934, "token": "�述", "type": "中文多字"}
-{"id": 46028, "token": "章", "type": "中文单字"}
-{"id": 46031, "token": "字段", "type": "中文多字"}
-{"id": 46034, "token": "手", "type": "中文单字"}
-{"id": 46056, "token": "库", "type": "中文单字"}
-{"id": 46091, "token": "三", "type": "中文单字"}
-{"id": 46093, "token": "….\n\n", "type": "中文标点"}
-{"id": 46233, "token": "“In", "type": "中文标点"}
-{"id": 46239, "token": "提示", "type": "中文多字"}
-{"id": 46281, "token": "从", "type": "中文单字"}
-{"id": 46456, "token": "支", "type": "中文单字"}
-{"id": 46690, "token": "“They", "type": "中文标点"}
-{"id": 46729, "token": "家", "type": "中文单字"}
-{"id": 46885, "token": "日期", "type": "中文多字"}
-{"id": 46961, "token": "长", "type": "中文单字"}
-{"id": 47000, "token": "付", "type": "中文单字"}
-{"id": 47012, "token": "获取", "type": "中文多字"}
-{"id": 47018, "token": "秒", "type": "中文单字"}
-{"id": 47030, "token": "图片", "type": "中文多字"}
-{"id": 47043, "token": "商品", "type": "中文多字"}
-{"id": 47095, "token": "路", "type": "中文单字"}
-{"id": 47200, "token": "代码", "type": "中文多字"}
-{"id": 47406, "token": "完", "type": "中文单字"}
-{"id": 47436, "token": "：</", "type": "中文标点"}
-{"id": 47523, "token": "象", "type": "中文单字"}
-{"id": 47548, "token": "则", "type": "中文单字"}
-{"id": 47551, "token": "现", "type": "中文单字"}
-{"id": 47566, "token": "设", "type": "中文单字"}
-{"id": 47577, "token": "地址", "type": "中文多字"}
-{"id": 47585, "token": "保存", "type": "中文多字"}
-{"id": 47653, "token": "京", "type": "中文单字"}
-{"id": 47770, "token": "转", "type": "中文单字"}
-{"id": 47896, "token": " –\n\n", "type": "中文标点"}
-{"id": 47971, "token": "�示", "type": "中文多字"}
-{"id": 48039, "token": "辑", "type": "中文单字"}
-{"id": 48044, "token": "一个", "type": "中文多字"}
-{"id": 48249, "token": "限", "type": "中文单字"}
-{"id": 48349, "token": "“A", "type": "中文标点"}
-{"id": 48463, "token": "默认", "type": "中文多字"}
-{"id": 48634, "token": "力", "type": "中文单字"}
-{"id": 48706, "token": "存在", "type": "中文多字"}
-{"id": 48785, "token": "数", "type": "中文单字"}
-{"id": 48858, "token": "创建", "type": "中文多字"}
-{"id": 48864, "token": "学", "type": "中文单字"}
-{"id": 48915, "token": "外", "type": "中文单字"}
-{"id": 48972, "token": "调", "type": "中文单字"}
-{"id": 48974, "token": "服务", "type": "中文多字"}
-{"id": 48982, "token": "项", "type": "中文单字"}
-{"id": 49055, "token": "请输入", "type": "中文多字"}
-{"id": 49216, "token": ".”\n", "type": "中文标点"}
-{"id": 49372, "token": "），", "type": "中文标点"}
-{"id": 49409, "token": "北", "type": "中文单字"}
-{"id": 49491, "token": "字符", "type": "中文多字"}
-{"id": 49525, "token": "—in", "type": "中文标点"}
-{"id": 49543, "token": "：\n\n", "type": "中文标点"}
-{"id": 49792, "token": "工", "type": "中文单字"}
-{"id": 49838, "token": "笑", "type": "中文单字"}
-{"id": 49928, "token": "监", "type": "中文单字"}
-{"id": 49977, "token": "“That", "type": "中文标点"}
-{"id": 49988, "token": "任", "type": "中文单字"}
-{"id": 50004, "token": "—which", "type": "中文标点"}
-{"id": 50021, "token": "相", "type": "中文单字"}
-{"id": 50027, "token": "验证", "type": "中文多字"}
-{"id": 50034, "token": "微", "type": "中文单字"}
-{"id": 50126, "token": "册", "type": "中文单字"}
-{"id": 50182, "token": "联", "type": "中文单字"}
-{"id": 50211, "token": "平", "type": "中文单字"}
-{"id": 50285, "token": "增", "type": "中文单字"}
-{"id": 50287, "token": "听", "type": "中文单字"}
-{"id": 50338, "token": "解", "type": "中文单字"}
-{"id": 50617, "token": "—to", "type": "中文标点"}
-{"id": 50667, "token": "等", "type": "中文单字"}
-{"id": 50808, "token": "’ai", "type": "中文标点"}
-{"id": 50928, "token": "得", "type": "中文单字"}
-{"id": 51107, "token": "更新", "type": "中文多字"}
-{"id": 51109, "token": "收", "type": "中文单字"}
-{"id": 51142, "token": "用户", "type": "中文多字"}
-{"id": 51202, "token": "选�", "type": "中文多字"}
-{"id": 51279, "token": "…”", "type": "中文标点"}
-{"id": 51385, "token": "安", "type": "中文单字"}
-{"id": 51392, "token": "价", "type": "中文单字"}
-{"id": 51431, "token": "第", "type": "中文单字"}
-{"id": 51450, "token": "取消", "type": "中文多字"}
-{"id": 51466, "token": "藏", "type": "中文单字"}
-{"id": 51477, "token": "创建", "type": "中文多字"}
-{"id": 51504, "token": "选择", "type": "中文多字"}
-{"id": 51510, "token": "订单", "type": "中文多字"}
-{"id": 51609, "token": "命", "type": "中文单字"}
-{"id": 51611, "token": "应", "type": "中文单字"}
-{"id": 51747, "token": "为空", "type": "中文多字"}
-{"id": 51749, "token": "—or", "type": "中文标点"}
-{"id": 51757, "token": "—I", "type": "中文标点"}
-{"id": 51786, "token": "“,", "type": "中文标点"}
-{"id": 51928, "token": "“When", "type": "中文标点"}
-{"id": 52030, "token": "看", "type": "中文单字"}
-{"id": 52084, "token": "索", "type": "中文单字"}
-{"id": 52188, "token": "�始化", "type": "中文多字"}
-{"id": 52225, "token": "资", "type": "中文单字"}
-{"id": 52254, "token": "查询", "type": "中文多字"}
-{"id": 52289, "token": "’en", "type": "中文标点"}
-{"id": 52332, "token": "产", "type": "中文单字"}
-{"id": 52563, "token": "表示", "type": "中文多字"}
-{"id": 52675, "token": "串", "type": "中文单字"}
-{"id": 52927, "token": "布", "type": "中文单字"}
-{"id": 53229, "token": "原", "type": "中文单字"}
-{"id": 53263, "token": "…..", "type": "中文标点"}
-{"id": 53283, "token": "知", "type": "中文单字"}
-{"id": 53434, "token": "级", "type": "中文单字"}
-{"id": 53513, "token": "––", "type": "中文标点"}
-{"id": 53610, "token": "水", "type": "中文单字"}
-{"id": 53626, "token": "上传", "type": "中文多字"}
-{"id": 53676, "token": "…and", "type": "中文标点"}
-{"id": 53802, "token": "监听", "type": "中文多字"}
-{"id": 53826, "token": "击", "type": "中文单字"}
-{"id": 53901, "token": "好", "type": "中文单字"}
-{"id": 53953, "token": "物", "type": "中文单字"}
-{"id": 54140, "token": "文", "type": "中文单字"}
-{"id": 54154, "token": "设置", "type": "中文多字"}
-{"id": 54253, "token": "不能", "type": "中文多字"}
-{"id": 54322, "token": "放", "type": "中文单字"}
-{"id": 54456, "token": "亿", "type": "中文单字"}
-{"id": 54493, "token": "经", "type": "中文单字"}
-{"id": 54581, "token": "描述", "type": "中文多字"}
-{"id": 54689, "token": "。。\n\n", "type": "中文标点"}
-{"id": 54747, "token": "。“", "type": "中文标点"}
-{"id": 54872, "token": "模", "type": "中文单字"}
-{"id": 55030, "token": "之", "type": "中文单字"}
-{"id": 55038, "token": "台", "type": "中文单字"}
-{"id": 55080, "token": "…I", "type": "中文标点"}
-{"id": 55121, "token": "显示", "type": "中文多字"}
-{"id": 55139, "token": "州", "type": "中文单字"}
-{"id": 55434, "token": "—is", "type": "中文标点"}
-{"id": 55487, "token": "配", "type": "中文单字"}
-{"id": 55642, "token": "处理", "type": "中文多字"}
-{"id": 55723, "token": "画", "type": "中文单字"}
-{"id": 55758, "token": "统", "type": "中文单字"}
-{"id": 55951, "token": "是", "type": "中文单字"}
-{"id": 55999, "token": "共", "type": "中文单字"}
-{"id": 56026, "token": "连", "type": "中文单字"}
-{"id": 56040, "token": "〜", "type": "中文标点"}
-{"id": 56163, "token": "„", "type": "中文标点"}
-{"id": 56209, "token": "…\"", "type": "中文标点"}
-{"id": 56235, "token": "海", "type": "中文单字"}
-{"id": 56386, "token": "开始", "type": "中文多字"}
-{"id": 56438, "token": "所有", "type": "中文多字"}
-{"id": 56602, "token": "节", "type": "中文单字"}
-{"id": 56716, "token": "返回", "type": "中文多字"}
-{"id": 56906, "token": "退", "type": "中文单字"}
-{"id": 56907, "token": "”。", "type": "中文标点"}
-{"id": 56955, "token": "”),", "type": "中文标点"}
-{"id": 56965, "token": "間", "type": "中文单字"}
-{"id": 57106, "token": "比", "type": "中文单字"}
-{"id": 57107, "token": "问", "type": "中文单字"}
-{"id": 57237, "token": "至", "type": "中文单字"}
-{"id": 57287, "token": "’aut", "type": "中文标点"}
-{"id": 57378, "token": "备", "type": "中文单字"}
-{"id": 57633, "token": "”:", "type": "中文标点"}
-{"id": 57668, "token": "你", "type": "中文单字"}
-{"id": 57752, "token": "黑", "type": "中文单字"}
-{"id": 57861, "token": "…”\n\n", "type": "中文标点"}
-{"id": 57892, "token": "’av", "type": "中文标点"}
-{"id": 58004, "token": "下午", "type": "中文多字"}
-{"id": 58119, "token": "编辑", "type": "中文多字"}
-{"id": 58291, "token": "或", "type": "中文单字"}
-{"id": 58318, "token": "与", "type": "中文单字"}
-{"id": 58322, "token": "影", "type": "中文单字"}
-{"id": 58386, "token": "’h", "type": "中文标点"}
-{"id": 58521, "token": "作者", "type": "中文多字"}
-{"id": 58543, "token": "话", "type": "中文单字"}
-{"id": 58552, "token": "视", "type": "中文单字"}
-{"id": 58653, "token": "读", "type": "中文单字"}
-{"id": 58655, "token": "告", "type": "中文单字"}
-{"id": 58666, "token": "美", "type": "中文单字"}
-{"id": 58721, "token": "事件", "type": "中文多字"}
-{"id": 58850, "token": "女", "type": "中文单字"}
-{"id": 58911, "token": "山", "type": "中文单字"}
-{"id": 59243, "token": "和", "type": "中文单字"}
-{"id": 59363, "token": "生", "type": "中文单字"}
-{"id": 59459, "token": "。（", "type": "中文标点"}
-{"id": 59462, "token": "需", "type": "中文单字"}
-{"id": 59464, "token": "复", "type": "中文单字"}
-{"id": 59505, "token": "手机", "type": "中文多字"}
-{"id": 59563, "token": "南", "type": "中文单字"}
-{"id": 59614, "token": "必", "type": "中文单字"}
-{"id": 59622, "token": "�行", "type": "中文多字"}
-{"id": 59712, "token": "」「", "type": "中文标点"}
-{"id": 59757, "token": "分", "type": "中文单字"}
-{"id": 59795, "token": "中国", "type": "中文多字"}
-{"id": 59892, "token": "闭", "type": "中文单字"}
-{"id": 59914, "token": "加载", "type": "中文多字"}
-{"id": 60174, "token": "城", "type": "中文单字"}
-{"id": 60205, "token": "用户名", "type": "中文多字"}
-{"id": 60233, "token": " 。", "type": "中文标点"}
-{"id": 60239, "token": "�性", "type": "中文多字"}
-{"id": 60251, "token": "结果", "type": "中文多字"}
-{"id": 60317, "token": "；\n", "type": "中文标点"}
-{"id": 60358, "token": "近", "type": "中文单字"}
-{"id": 60455, "token": "效", "type": "中文单字"}
-{"id": 60632, "token": "利", "type": "中文单字"}
-{"id": 60634, "token": "移", "type": "中文单字"}
-{"id": 60654, "token": "—as", "type": "中文标点"}
-{"id": 60656, "token": "’int", "type": "中文标点"}
-{"id": 60710, "token": "–\n\n", "type": "中文标点"}
-{"id": 60843, "token": "总", "type": "中文单字"}
-{"id": 60979, "token": "按", "type": "中文单字"}
-{"id": 61056, "token": "排", "type": "中文单字"}
-{"id": 61075, "token": "首", "type": "中文单字"}
-{"id": 61131, "token": "’n", "type": "中文标点"}
-{"id": 61176, "token": "··", "type": "中文标点"}
-{"id": 61304, "token": "記", "type": "中文单字"}
-{"id": 61311, "token": "————————————————", "type": "中文标点"}
-{"id": 61337, "token": "社", "type": "中文单字"}
-{"id": 61496, "token": "标题", "type": "中文多字"}
-{"id": 61553, "token": "“As", "type": "中文标点"}
-{"id": 61559, "token": "“No", "type": "中文标点"}
-{"id": 61603, "token": "“But", "type": "中文标点"}
-{"id": 61633, "token": "注意", "type": "中文多字"}
-{"id": 61648, "token": "完成", "type": "中文多字"}
-{"id": 61710, "token": "确定", "type": "中文多字"}
-{"id": 61786, "token": "西", "type": "中文单字"}
-{"id": 61826, "token": "先", "type": "中文单字"}
-{"id": 61903, "token": "…\"\n\n", "type": "中文标点"}
-{"id": 61994, "token": "然", "type": "中文单字"}
-{"id": 62049, "token": "键", "type": "中文单字"}
-{"id": 62205, "token": "名", "type": "中文单字"}
-{"id": 62249, "token": "周期", "type": "中文多字"}
-{"id": 62291, "token": "额", "type": "中文单字"}
-{"id": 62543, "token": "写", "type": "中文单字"}
-{"id": 62597, "token": "“My", "type": "中文标点"}
-{"id": 62717, "token": "�名", "type": "中文多字"}
-{"id": 62789, "token": "注册", "type": "中文多字"}
-{"id": 62855, "token": "签", "type": "中文单字"}
-{"id": 63091, "token": "自", "type": "中文单字"}
-{"id": 63093, "token": "。',\n", "type": "中文标点"}
-{"id": 63212, "token": "因", "type": "中文单字"}
-{"id": 63289, "token": "下载", "type": "中文多字"}
-{"id": 63344, "token": "如果", "type": "中文多字"}
-{"id": 63362, "token": "数据", "type": "中文多字"}
-{"id": 63397, "token": "命周期", "type": "中文多字"}
-{"id": 63679, "token": "注", "type": "中文单字"}
-{"id": 63750, "token": "”—", "type": "中文标点"}
-{"id": 63938, "token": "—not", "type": "中文标点"}
-{"id": 63977, "token": " —\n\n", "type": "中文标点"}
-{"id": 64022, "token": "别", "type": "中文单字"}
-{"id": 64026, "token": "并", "type": "中文单字"}
-{"id": 64045, "token": "异", "type": "中文单字"}
-{"id": 64063, "token": "束", "type": "中文单字"}
-{"id": 64171, "token": "修改", "type": "中文多字"}
-{"id": 64173, "token": "删除", "type": "中文多字"}
-{"id": 64179, "token": "生命周期", "type": "中文多字"}
-{"id": 64209, "token": "心", "type": "中文单字"}
-{"id": 64376, "token": "。\",\n", "type": "中文标点"}
-{"id": 64414, "token": "链", "type": "中文单字"}
-{"id": 64467, "token": "指", "type": "中文单字"}
-{"id": 64479, "token": "评", "type": "中文单字"}
-{"id": 64531, "token": "整", "type": "中文单字"}
-{"id": 64623, "token": "’in", "type": "中文标点"}
-{"id": 64803, "token": "四", "type": "中文单字"}
-{"id": 64889, "token": "断", "type": "中文单字"}
-{"id": 64936, "token": "角", "type": "中文单字"}
-{"id": 64960, "token": "生命周期函数", "type": "中文多字"}
-{"id": 65053, "token": "监听页面", "type": "中文多字"}
-{"id": 65164, "token": "连接", "type": "中文多字"}
-{"id": 65218, "token": "上", "type": "中文单字"}
-{"id": 65305, "token": "消息", "type": "中文多字"}
-{"id": 65312, "token": "”).", "type": "中文标点"}
-{"id": 65372, "token": "软", "type": "中文单字"}
-{"id": 65455, "token": "头", "type": "中文单字"}
-{"id": 65459, "token": "）、", "type": "中文标点"}
-{"id": 65529, "token": "对象", "type": "中文多字"}
-{"id": 65571, "token": "是否", "type": "中文多字"}
-{"id": 65573, "token": "邮", "type": "中文单字"}
-{"id": 65659, "token": "义", "type": "中文单字"}
-{"id": 65743, "token": "司", "type": "中文单字"}
-{"id": 65782, "token": "步", "type": "中文单字"}
-{"id": 65789, "token": "门", "type": "中文单字"}
-{"id": 65820, "token": "导", "type": "中文单字"}
-{"id": 65854, "token": "客", "type": "中文单字"}
-{"id": 65884, "token": "不能为空", "type": "中文多字"}
-{"id": 65917, "token": "右", "type": "中文单字"}
-{"id": 66052, "token": "频", "type": "中文单字"}
-{"id": 66101, "token": "\"—", "type": "中文标点"}
-{"id": 66201, "token": "像", "type": "中文单字"}
-{"id": 66327, "token": "。「", "type": "中文标点"}
-{"id": 66378, "token": "特", "type": "中文单字"}
-{"id": 66383, "token": "」と", "type": "中文标点"}
-{"id": 66545, "token": "”;", "type": "中文标点"}
-{"id": 66621, "token": " ….", "type": "中文标点"}
-{"id": 66625, "token": "“Our", "type": "中文标点"}
-{"id": 66677, "token": "记录", "type": "中文多字"}
-{"id": 66679, "token": "…\n\n\n", "type": "中文标点"}
-{"id": 66776, "token": "非", "type": "中文单字"}
-{"id": 66850, "token": " “[", "type": "中文标点"}
-{"id": 66870, "token": "省", "type": "中文单字"}
-{"id": 67117, "token": "输出", "type": "中文多字"}
-{"id": 67178, "token": "造", "type": "中文单字"}
-{"id": 67282, "token": "’ét", "type": "中文标点"}
-{"id": 67287, "token": "姓名", "type": "中文多字"}
-{"id": 67494, "token": "说明", "type": "中文多字"}
-{"id": 67658, "token": "字符串", "type": "中文多字"}
-{"id": 67669, "token": "径", "type": "中文单字"}
-{"id": 67735, "token": "�试", "type": "中文多字"}
-{"id": 67870, "token": "’e", "type": "中文标点"}
-{"id": 67886, "token": " ”\n\n", "type": "中文标点"}
-{"id": 67933, "token": "详", "type": "中文单字"}
-{"id": 67986, "token": "验证码", "type": "中文多字"}
-{"id": 67998, "token": "。\\", "type": "中文标点"}
-{"id": 68171, "token": "由", "type": "中文单字"}
-{"id": 68230, "token": "＾", "type": "中文标点"}
-{"id": 68306, "token": "’on", "type": "中文标点"}
-{"id": 68379, "token": "包", "type": "中文单字"}
-{"id": 68438, "token": "通过", "type": "中文多字"}
-{"id": 68464, "token": "东", "type": "中文单字"}
-{"id": 68850, "token": ")—", "type": "中文标点"}
-{"id": 68931, "token": "论", "type": "中文单字"}
-{"id": 68932, "token": "“And", "type": "中文标点"}
-{"id": 69049, "token": "当前", "type": "中文多字"}
-{"id": 69165, "token": "络", "type": "中文单字"}
-{"id": 69253, "token": "款", "type": "中文单字"}
-{"id": 69272, "token": "�藏", "type": "中文多字"}
-{"id": 69362, "token": "支付", "type": "中文多字"}
-{"id": 69496, "token": "启", "type": "中文单字"}
-{"id": 69636, "token": "而", "type": "中文单字"}
-{"id": 69856, "token": "填", "type": "中文单字"}
-{"id": 69905, "token": "格式", "type": "中文多字"}
-{"id": 69962, "token": "释", "type": "中文单字"}
-{"id": 69978, "token": "持", "type": "中文单字"}
-{"id": 70041, "token": "��索", "type": "中文多字"}
-{"id": 70090, "token": "北京", "type": "中文多字"}
-{"id": 70141, "token": "向", "type": "中文单字"}
-{"id": 70158, "token": "输入", "type": "中文多字"}
-{"id": 70203, "token": "算", "type": "中文单字"}
-{"id": 70214, "token": "“So", "type": "中文标点"}
-{"id": 70262, "token": "对", "type": "中文单字"}
-{"id": 70277, "token": "江", "type": "中文单字"}
-{"id": 70284, "token": "不存在", "type": "中文多字"}
-{"id": 70349, "token": "里", "type": "中文单字"}
-{"id": 70453, "token": "查", "type": "中文单字"}
-{"id": 70472, "token": "如", "type": "中文单字"}
-{"id": 70525, "token": "发", "type": "中文单字"}
-{"id": 70542, "token": "份", "type": "中文单字"}
-{"id": 70615, "token": ")，", "type": "中文标点"}
-{"id": 70616, "token": "责", "type": "中文单字"}
-{"id": 70626, "token": "科", "type": "中文单字"}
-{"id": 70694, "token": "文件", "type": "中文多字"}
-{"id": 70774, "token": "类", "type": "中文单字"}
-{"id": 70821, "token": "民", "type": "中文单字"}
-{"id": 70924, "token": "数组", "type": "中文多字"}
-{"id": 71005, "token": "治", "type": "中文单字"}
-{"id": 71082, "token": "%，", "type": "中文标点"}
-{"id": 71174, "token": "声", "type": "中文单字"}
-{"id": 71201, "token": "—they", "type": "中文标点"}
-{"id": 71208, "token": "男", "type": "中文单字"}
-{"id": 71270, "token": "“(", "type": "中文标点"}
-{"id": 71298, "token": "[…", "type": "中文标点"}
-{"id": 71461, "token": "重新", "type": "中文多字"}
-{"id": 71480, "token": "—you", "type": "中文标点"}
-{"id": 71600, "token": "设计", "type": "中文多字"}
-{"id": 71638, "token": "分类", "type": "中文多字"}
-{"id": 71668, "token": "输出", "type": "中文多字"}
-{"id": 71689, "token": "以上", "type": "中文多字"}
-{"id": 71733, "token": "异常", "type": "中文多字"}
-{"id": 71869, "token": "族", "type": "中文单字"}
-{"id": 71890, "token": "站", "type": "中文单字"}
-{"id": 72027, "token": "没", "type": "中文单字"}
-{"id": 72069, "token": "参数", "type": "中文多字"}
-{"id": 72099, "token": "県", "type": "中文单字"}
-{"id": 72125, "token": "雅", "type": "中文单字"}
-{"id": 72209, "token": "版本", "type": "中文多字"}
-{"id": 72234, "token": "换", "type": "中文单字"}
-{"id": 72237, "token": "核", "type": "中文单字"}
-{"id": 72238, "token": "素", "type": "中文单字"}
-{"id": 72318, "token": "—for", "type": "中文标点"}
-{"id": 72368, "token": "都", "type": "中文单字"}
-{"id": 72404, "token": "超", "type": "中文单字"}
-{"id": 72434, "token": "!’", "type": "中文标点"}
-{"id": 72456, "token": "网络", "type": "中文多字"}
-{"id": 72516, "token": "店", "type": "中文单字"}
-{"id": 72718, "token": "起", "type": "中文单字"}
-{"id": 72794, "token": "隐藏", "type": "中文多字"}
-{"id": 72843, "token": "享", "type": "中文单字"}
-{"id": 72873, "token": "方", "type": "中文单字"}
-{"id": 72917, "token": "进行", "type": "中文多字"}
-{"id": 73051, "token": "是否", "type": "中文多字"}
-{"id": 73071, "token": "提交", "type": "中文多字"}
-{"id": 73117, "token": "发送", "type": "中文多字"}
-{"id": 73164, "token": "联系", "type": "中文多字"}
-{"id": 73325, "token": "拉", "type": "中文单字"}
-{"id": 73329, "token": "…\n\n\n\n", "type": "中文标点"}
-{"id": 73361, "token": "米", "type": "中文单字"}
-{"id": 73548, "token": "系统", "type": "中文多字"}
-{"id": 73686, "token": "引", "type": "中文单字"}
-{"id": 73740, "token": "编号", "type": "中文多字"}
-{"id": 73751, "token": "点击", "type": "中文多字"}
-{"id": 73769, "token": "更", "type": "中文单字"}
-{"id": 73939, "token": "…)", "type": "中文标点"}
-{"id": 73958, "token": "中", "type": "中文单字"}
-{"id": 73981, "token": "语", "type": "中文单字"}
-{"id": 74022, "token": "”?", "type": "中文标点"}
-{"id": 74090, "token": "土", "type": "中文单字"}
-{"id": 74138, "token": "宋", "type": "中文单字"}
-{"id": 74245, "token": "直", "type": "中文单字"}
-{"id": 74257, "token": "每", "type": "中文单字"}
-{"id": 74318, "token": "公司", "type": "中文多字"}
-{"id": 74396, "token": "箱", "type": "中文单字"}
-{"id": 74412, "token": "字", "type": "中文单字"}
-{"id": 74445, "token": "项目", "type": "中文多字"}
-{"id": 74482, "token": "後", "type": "中文单字"}
-{"id": 74662, "token": "在", "type": "中文单字"}
-{"id": 74770, "token": "可以", "type": "中文多字"}
-{"id": 74843, "token": "参", "type": "中文单字"}
-{"id": 75140, "token": "变", "type": "中文单字"}
-{"id": 75146, "token": "基", "type": "中文单字"}
-{"id": 75259, "token": "页面", "type": "中文多字"}
-{"id": 75267, "token": "場", "type": "中文单字"}
-{"id": 75293, "token": "待", "type": "中文单字"}
-{"id": 75320, "token": "程序", "type": "中文多字"}
-{"id": 75376, "token": "）。", "type": "中文标点"}
-{"id": 75486, "token": "规", "type": "中文单字"}
-{"id": 75493, "token": "数据库", "type": "中文多字"}
-{"id": 75513, "token": "政", "type": "中文单字"}
-{"id": 75550, "token": "“For", "type": "中文标点"}
-{"id": 75630, "token": "雅黑", "type": "中文多字"}
-{"id": 75631, "token": "软雅黑", "type": "中文多字"}
-{"id": 75761, "token": "排序", "type": "中文多字"}
-{"id": 75787, "token": "。\n\n\n\n\n\n", "type": "中文标点"}
-{"id": 75863, "token": "也", "type": "中文单字"}
-{"id": 75910, "token": "介", "type": "中文单字"}
-{"id": 75976, "token": "首页", "type": "中文多字"}
-{"id": 76070, "token": "—including", "type": "中文标点"}
-{"id": 76099, "token": "关闭", "type": "中文多字"}
-{"id": 76148, "token": "，\n\n", "type": "中文标点"}
-{"id": 76161, "token": "钟", "type": "中文单字"}
-{"id": 76208, "token": "五", "type": "中文单字"}
-{"id": 76217, "token": "执行", "type": "中文多字"}
-{"id": 76323, "token": "审", "type": "中文单字"}
-{"id": 76417, "token": "单位", "type": "中文多字"}
-{"id": 76455, "token": "手机号", "type": "中文多字"}
-{"id": 76502, "token": "日", "type": "中文单字"}
-{"id": 76505, "token": "木", "type": "中文单字"}
-{"id": 76537, "token": "打", "type": "中文单字"}
-{"id": 76706, "token": "活", "type": "中文单字"}
-{"id": 76718, "token": "微软雅黑", "type": "中文多字"}
-{"id": 76750, "token": "播", "type": "中文单字"}
-{"id": 76843, "token": "！！\n\n", "type": "中文标点"}
-{"id": 76858, "token": "！”", "type": "中文标点"}
-{"id": 76864, "token": "！」", "type": "中文标点"}
-{"id": 76868, "token": "方式", "type": "中文多字"}
-{"id": 76929, "token": "—he", "type": "中文标点"}
-{"id": 76982, "token": "该", "type": "中文单字"}
-{"id": 77138, "token": "’am", "type": "中文标点"}
-{"id": 77158, "token": "…)\n\n", "type": "中文标点"}
-{"id": 77190, "token": "初始化", "type": "中文多字"}
-{"id": 77195, "token": "条件", "type": "中文多字"}
-{"id": 77219, "token": "記事", "type": "中文多字"}
-{"id": 77284, "token": "“.", "type": "中文标点"}
-{"id": 77413, "token": "展", "type": "中文单字"}
-{"id": 77479, "token": ",…\n\n", "type": "中文标点"}
-{"id": 77748, "token": "钮", "type": "中文单字"}
-{"id": 77913, "token": "具", "type": "中文单字"}
-{"id": 77937, "token": "路径", "type": "中文多字"}
-{"id": 78021, "token": "退出", "type": "中文多字"}
-{"id": 78111, "token": "宋体", "type": "中文多字"}
-{"id": 78228, "token": "志", "type": "中文单字"}
-{"id": 78244, "token": "言", "type": "中文单字"}
-{"id": 78272, "token": "购", "type": "中文单字"}
-{"id": 78366, "token": "……………………", "type": "中文标点"}
-{"id": 78388, "token": "但", "type": "中文单字"}
-{"id": 78519, "token": "星", "type": "中文单字"}
-{"id": 78640, "token": "两", "type": "中文单字"}
-{"id": 78657, "token": "例如", "type": "中文多字"}
-{"id": 78659, "token": "左", "type": "中文单字"}
-{"id": 78698, "token": "考", "type": "中文单字"}
-{"id": 78935, "token": "构", "type": "中文单字"}
-{"id": 78943, "token": "報", "type": "中文单字"}
-{"id": 79059, "token": "球", "type": "中文单字"}
-{"id": 79108, "token": "设计器", "type": "中文多字"}
-{"id": 79203, "token": "更新", "type": "中文多字"}
-{"id": 79656, "token": "相关", "type": "中文多字"}
-{"id": 79785, "token": "音", "type": "中文单字"}
-{"id": 79908, "token": "动生成", "type": "中文多字"}
-{"id": 79982, "token": "端", "type": "中文单字"}
-{"id": 80000, "token": "。”\n\n", "type": "中文标点"}
-{"id": 80003, "token": "，默认", "type": "中文多字"}
-{"id": 80019, "token": "新", "type": "中文单字"}
-{"id": 80073, "token": "搜索", "type": "中文多字"}
-{"id": 80078, "token": "—even", "type": "中文标点"}
-{"id": 80172, "token": "投", "type": "中文单字"}
-{"id": 80195, "token": "立", "type": "中文单字"}
-{"id": 80356, "token": "属性", "type": "中文多字"}
-{"id": 80426, "token": "�断", "type": "中文多字"}
-{"id": 80578, "token": "们", "type": "中文单字"}
-{"id": 80615, "token": ".…\n\n", "type": "中文标点"}
-{"id": 80699, "token": "火", "type": "中文单字"}
-{"id": 80804, "token": "示", "type": "中文单字"}
-{"id": 80866, "token": "清", "type": "中文单字"}
-{"id": 81194, "token": "金额", "type": "中文多字"}
-{"id": 81201, "token": "账", "type": "中文单字"}
-{"id": 81258, "token": "就", "type": "中文单字"}
-{"id": 81368, "token": "费", "type": "中文单字"}
-{"id": 81506, "token": "请选择", "type": "中文多字"}
-{"id": 81526, "token": "示例", "type": "中文多字"}
-{"id": 81543, "token": "没有", "type": "中文多字"}
-{"id": 81546, "token": "：\"+", "type": "中文标点"}
-{"id": 81628, "token": "查询", "type": "中文多字"}
-{"id": 81646, "token": "默认", "type": "中文多字"}
-{"id": 81665, "token": "结束", "type": "中文多字"}
-{"id": 81742, "token": "案", "type": "中文单字"}
-{"id": 81902, "token": "—with", "type": "中文标点"}
-{"id": 81951, "token": "控", "type": "中文单字"}
-{"id": 81976, "token": "请求", "type": "中文多字"}
-{"id": 82042, "token": "广", "type": "中文单字"}
-{"id": 82175, "token": "’app", "type": "中文标点"}
-{"id": 82267, "token": "确认", "type": "中文多字"}
-{"id": 82302, "token": "历", "type": "中文单字"}
-{"id": 82317, "token": "及", "type": "中文单字"}
-{"id": 82363, "token": "如果", "type": "中文多字"}
-{"id": 82364, "token": "？”", "type": "中文标点"}
-{"id": 82420, "token": "計", "type": "中文单字"}
-{"id": 82530, "token": "、、", "type": "中文标点"}
-{"id": 82533, "token": "止", "type": "中文单字"}
-{"id": 82554, "token": "方法", "type": "中文多字"}
-{"id": 82696, "token": "么", "type": "中文单字"}
-{"id": 82768, "token": "货", "type": "中文单字"}
-{"id": 82805, "token": "测试", "type": "中文多字"}
-{"id": 82900, "token": "数量", "type": "中文多字"}
-{"id": 82912, "token": "位置", "type": "中文多字"}
-{"id": 82973, "token": "時間", "type": "中文多字"}
-{"id": 83042, "token": "�权", "type": "中文多字"}
-{"id": 83047, "token": "开", "type": "中文单字"}
-{"id": 83125, "token": "文章", "type": "中文多字"}
-{"id": 83175, "token": "阳", "type": "中文单字"}
-{"id": 83266, "token": "队", "type": "中文单字"}
-{"id": 83301, "token": "技", "type": "中文单字"}
-{"id": 83324, "token": "场", "type": "中文单字"}
-{"id": 83337, "token": "链接", "type": "中文多字"}
-{"id": 83354, "token": "＞", "type": "中文标点"}
-{"id": 83439, "token": "添加", "type": "中文多字"}
-{"id": 83639, "token": "最", "type": "中文单字"}
-{"id": 83687, "token": "数字", "type": "中文多字"}
-{"id": 83741, "token": "声明", "type": "中文多字"}
-{"id": 83747, "token": "少", "type": "中文单字"}
-{"id": 83766, "token": "…but", "type": "中文标点"}
-{"id": 83799, "token": "形", "type": "中文单字"}
-{"id": 83800, "token": "产品", "type": "中文多字"}
-{"id": 83872, "token": "—are", "type": "中文标点"}
-{"id": 83932, "token": "稿", "type": "中文单字"}
-{"id": 83947, "token": "英", "type": "中文单字"}
-{"id": 83994, "token": "游", "type": "中文单字"}
-{"id": 84095, "token": "亿元", "type": "中文多字"}
-{"id": 84131, "token": "分钟", "type": "中文多字"}
-{"id": 84341, "token": ".…", "type": "中文标点"}
-{"id": 84410, "token": "商", "type": "中文单字"}
-{"id": 84498, "token": "“She", "type": "中文标点"}
-{"id": 84765, "token": "！\",", "type": "中文标点"}
-{"id": 84844, "token": "供", "type": "中文单字"}
-{"id": 84851, "token": "推", "type": "中文单字"}
-{"id": 84875, "token": "！\n\n\n\n", "type": "中文标点"}
-{"id": 84941, "token": "—who", "type": "中文标点"}
-{"id": 85155, "token": "初始化", "type": "中文多字"}
-{"id": 85188, "token": "税", "type": "中文单字"}
-{"id": 85284, "token": "按钮", "type": "中文多字"}
-{"id": 85366, "token": "—an", "type": "中文标点"}
-{"id": 85663, "token": "無し�", "type": "中文多字"}
-{"id": 85707, "token": "初", "type": "中文单字"}
-{"id": 85997, "token": "当", "type": "中文单字"}
-{"id": 85998, "token": "！');\n", "type": "中文标点"}
-{"id": 86127, "token": "私", "type": "中文单字"}
-{"id": 86206, "token": "需要", "type": "中文多字"}
-{"id": 86222, "token": "解", "type": "中文单字"}
-{"id": 86319, "token": "—we", "type": "中文标点"}
-{"id": 86348, "token": "全部", "type": "中文多字"}
-{"id": 86354, "token": "景", "type": "中文单字"}
-{"id": 86429, "token": "资源", "type": "中文多字"}
-{"id": 86436, "token": "去", "type": "中文单字"}
-{"id": 86461, "token": "华", "type": "中文单字"}
-{"id": 86508, "token": "“Yes", "type": "中文标点"}
-{"id": 86601, "token": "’T", "type": "中文标点"}
-{"id": 86741, "token": "评论", "type": "中文多字"}
-{"id": 86758, "token": "使用", "type": "中文多字"}
-{"id": 86846, "token": "’B", "type": "中文标点"}
-{"id": 86867, "token": "配置", "type": "中文多字"}
-{"id": 87023, "token": "–and", "type": "中文标点"}
-{"id": 87109, "token": "不", "type": "中文单字"}
-{"id": 87177, "token": "話", "type": "中文单字"}
-{"id": 87217, "token": "番", "type": "中文单字"}
-{"id": 87219, "token": "问题", "type": "中文多字"}
-{"id": 87247, "token": "—all", "type": "中文标点"}
-{"id": 87327, "token": "报道", "type": "中文多字"}
-{"id": 87412, "token": "环", "type": "中文单字"}
-{"id": 87441, "token": "张", "type": "中文单字"}
-{"id": 87447, "token": "開", "type": "中文单字"}
-{"id": 87474, "token": "無しさん", "type": "中文多字"}
-{"id": 87502, "token": "种", "type": "中文单字"}
-{"id": 87646, "token": "成", "type": "中文单字"}
-{"id": 87671, "token": "—one", "type": "中文标点"}
-{"id": 87844, "token": "易", "type": "中文单字"}
-{"id": 87990, "token": "“Oh", "type": "中文标点"}
-{"id": 88108, "token": "……\n\n", "type": "中文标点"}
-{"id": 88126, "token": "您", "type": "中文单字"}
-{"id": 88161, "token": "’an", "type": "中文标点"}
-{"id": 88240, "token": "视频", "type": "中文多字"}
-{"id": 88343, "token": "》，", "type": "中文标点"}
-{"id": 88348, "token": ".’”\n\n", "type": "中文标点"}
-{"id": 88356, "token": "再", "type": "中文单字"}
-{"id": 88367, "token": "可能", "type": "中文多字"}
-{"id": 88435, "token": "文字", "type": "中文多字"}
-{"id": 88631, "token": "板", "type": "中文单字"}
-{"id": 88851, "token": "’acc", "type": "中文标点"}
-{"id": 88852, "token": "以下", "type": "中文多字"}
-{"id": 88905, "token": "电话", "type": "中文多字"}
-{"id": 88925, "token": "“Well", "type": "中文标点"}
-{"id": 88958, "token": "—from", "type": "中文标点"}
-{"id": 89046, "token": "連", "type": "中文单字"}
-{"id": 89151, "token": "真", "type": "中文单字"}
-{"id": 89186, "token": "有效", "type": "中文多字"}
-{"id": 89213, "token": "’:", "type": "中文标点"}
-{"id": 89408, "token": "今年", "type": "中文多字"}
-{"id": 89575, "token": "€“", "type": "中文标点"}
-{"id": 89753, "token": "流", "type": "中文单字"}
-{"id": 89783, "token": "余", "type": "中文单字"}
-{"id": 89874, "token": "”\n", "type": "中文标点"}
-{"id": 89902, "token": "任务", "type": "中文多字"}
-{"id": 90070, "token": "见", "type": "中文单字"}
-{"id": 90091, "token": "正确", "type": "中文多字"}
-{"id": 90112, "token": "给", "type": "中文单字"}
-{"id": 90147, "token": "服务器", "type": "中文多字"}
-{"id": 90223, "token": "’es", "type": "中文标点"}
-{"id": 90261, "token": "来源", "type": "中文多字"}
-{"id": 90354, "token": "结", "type": "中文单字"}
-{"id": 90493, "token": "。<", "type": "中文标点"}
-{"id": 90578, "token": "…\n", "type": "中文标点"}
-{"id": 90581, "token": "－", "type": "中文标点"}
-{"id": 90756, "token": "详情", "type": "中文多字"}
-{"id": 90863, "token": "—if", "type": "中文标点"}
-{"id": 91006, "token": "？」", "type": "中文标点"}
-{"id": 91077, "token": "局", "type": "中文单字"}
-{"id": 91082, "token": "主", "type": "中文单字"}
-{"id": 91240, "token": "’à", "type": "中文标点"}
-{"id": 91272, "token": "优", "type": "中文单字"}
-{"id": 91386, "token": "书", "type": "中文单字"}
-{"id": 91417, "token": "’y", "type": "中文标点"}
-{"id": 91418, "token": "’util", "type": "中文标点"}
-{"id": 91443, "token": "’hui", "type": "中文标点"}
-{"id": 91466, "token": "一页", "type": "中文多字"}
-{"id": 91495, "token": "，并", "type": "中文多字"}
-{"id": 91547, "token": "发布", "type": "中文多字"}
-{"id": 91763, "token": "思", "type": "中文单字"}
-{"id": 91774, "token": "見", "type": "中文单字"}
-{"id": 91837, "token": "：<", "type": "中文标点"}
-{"id": 91875, "token": "動", "type": "中文单字"}
-{"id": 91940, "token": "运", "type": "中文单字"}
-{"id": 91951, "token": "审核", "type": "中文多字"}
-{"id": 91967, "token": "图", "type": "中文单字"}
-{"id": 91985, "token": "样", "type": "中文单字"}
-{"id": 92019, "token": "其中", "type": "中文多字"}
-{"id": 92056, "token": "权限", "type": "中文多字"}
-{"id": 92099, "token": "删除成功", "type": "中文多字"}
-{"id": 92113, "token": " “…", "type": "中文标点"}
-{"id": 92150, "token": "�新", "type": "中文多字"}
-{"id": 92193, "token": "（笑", "type": "中文多字"}
-{"id": 92211, "token": "，《", "type": "中文标点"}
-{"id": 92264, "token": ",’”", "type": "中文标点"}
-{"id": 92318, "token": "时间", "type": "中文多字"}
-{"id": 92366, "token": "】,", "type": "中文标点"}
-{"id": 92378, "token": "）\r\n", "type": "中文标点"}
-{"id": 92382, "token": "定义", "type": "中文多字"}
-{"id": 92517, "token": "关", "type": "中文单字"}
-{"id": 92527, "token": "登", "type": "中文单字"}
-{"id": 92553, "token": "销", "type": "中文单字"}
-{"id": 92555, "token": "万元", "type": "中文多字"}
-{"id": 92672, "token": "同时", "type": "中文多字"}
-{"id": 92693, "token": "無料", "type": "中文多字"}
-{"id": 92748, "token": "’all", "type": "中文标点"}
-{"id": 92776, "token": "即", "type": "中文单字"}
-{"id": 92780, "token": "只", "type": "中文单字"}
-{"id": 92877, "token": "老", "type": "中文单字"}
-{"id": 93056, "token": "、“", "type": "中文标点"}
-{"id": 93115, "token": "岁", "type": "中文单字"}
-{"id": 93126, "token": "’Brien", "type": "中文标点"}
-{"id": 93132, "token": "大小", "type": "中文多字"}
-{"id": 93233, "token": "找", "type": "中文单字"}
-{"id": 93269, "token": "“These", "type": "中文标点"}
-{"id": 93393, "token": "实", "type": "中文单字"}
-{"id": 93413, "token": "或", "type": "中文单字"}
-{"id": 93446, "token": "“\n\n", "type": "中文标点"}
-{"id": 93474, "token": "节点", "type": "中文多字"}
-{"id": 93598, "token": "若", "type": "中文单字"}
-{"id": 93636, "token": "小时", "type": "中文多字"}
-{"id": 93673, "token": "“To", "type": "中文标点"}
-{"id": 93830, "token": "—\"", "type": "中文标点"}
-{"id": 93922, "token": "’autres", "type": "中文标点"}
-{"id": 93994, "token": "其他", "type": "中文多字"}
-{"id": 94134, "token": "自治", "type": "中文多字"}
-{"id": 94249, "token": "分享", "type": "中文多字"}
-{"id": 94345, "token": "’ex", "type": "中文标点"}
-{"id": 94366, "token": "稍", "type": "中文单字"}
-{"id": 94518, "token": "…the", "type": "中文标点"}
-{"id": 94537, "token": "�件", "type": "中文多字"}
-{"id": 94588, "token": "达", "type": "中文单字"}
-{"id": 94668, "token": "邮箱", "type": "中文多字"}
-{"id": 94720, "token": "新增", "type": "中文多字"}
-{"id": 94785, "token": "提", "type": "中文单字"}
-{"id": 94895, "token": "：%", "type": "中文标点"}
-{"id": 94923, "token": "院", "type": "中文单字"}
-{"id": 94983, "token": "加", "type": "中文单字"}
-{"id": 95001, "token": "価", "type": "中文单字"}
-{"id": 95221, "token": "気", "type": "中文单字"}
-{"id": 95337, "token": "约", "type": "中文单字"}
-{"id": 95399, "token": "速", "type": "中文单字"}
-{"id": 95475, "token": "停", "type": "中文单字"}
-{"id": 95532, "token": "？\n", "type": "中文标点"}
-{"id": 95543, "token": "反", "type": "中文单字"}
-{"id": 95544, "token": "票", "type": "中文单字"}
-{"id": 95598, "token": "十", "type": "中文单字"}
-{"id": 96153, "token": "，则", "type": "中文多字"}
-{"id": 96197, "token": ",—", "type": "中文标点"}
-{"id": 96203, "token": "“At", "type": "中文标点"}
-{"id": 96206, "token": "’)", "type": "中文标点"}
-{"id": 96332, "token": "[…]", "type": "中文标点"}
-{"id": 96356, "token": "身", "type": "中文单字"}
-{"id": 96407, "token": "商品", "type": "中文多字"}
-{"id": 96412, "token": "含", "type": "中文单字"}
-{"id": 96455, "token": "率", "type": "中文单字"}
-{"id": 96500, "token": "汽", "type": "中文单字"}
-{"id": 96511, "token": "专", "type": "中文单字"}
-{"id": 96555, "token": "／", "type": "中文标点"}
-{"id": 96557, "token": "管理员", "type": "中文多字"}
-{"id": 97049, "token": "歳", "type": "中文单字"}
-{"id": 97150, "token": "，在", "type": "中文多字"}
-{"id": 97360, "token": ".–", "type": "中文标点"}
-{"id": 97432, "token": "”。\n\n", "type": "中文标点"}
-{"id": 97518, "token": "関", "type": "中文单字"}
-{"id": 97522, "token": "议", "type": "中文单字"}
-{"id": 97565, "token": "雷", "type": "中文单字"}
-{"id": 97655, "token": "正在", "type": "中文多字"}
-{"id": 97908, "token": "�能", "type": "中文多字"}
-{"id": 97999, "token": "。(", "type": "中文标点"}
-{"id": 98128, "token": "自动生成", "type": "中文多字"}
-{"id": 98134, "token": "’elle", "type": "中文标点"}
-{"id": 98184, "token": "些", "type": "中文单字"}
-{"id": 98220, "token": "界", "type": "中文单字"}
-{"id": 98245, "token": "陆", "type": "中文单字"}
-{"id": 98261, "token": "注意", "type": "中文多字"}
-{"id": 98390, "token": "备注", "type": "中文多字"}
-{"id": 98406, "token": "倍", "type": "中文单字"}
-{"id": 98458, "token": ",’’", "type": "中文标点"}
-{"id": 98476, "token": "“How", "type": "中文标点"}
-{"id": 98499, "token": "読", "type": "中文单字"}
-{"id": 98580, "token": "价格", "type": "中文多字"}
-{"id": 98657, "token": "检", "type": "中文单字"}
-{"id": 98711, "token": "我的", "type": "中文多字"}
-{"id": 98739, "token": "我们", "type": "中文多字"}
-{"id": 98806, "token": "还", "type": "中文单字"}
-{"id": 98871, "token": "析", "type": "中文单字"}
-{"id": 98897, "token": "企", "type": "中文单字"}
-{"id": 98915, "token": "友", "type": "中文单字"}
-{"id": 99007, "token": "”的", "type": "中文多字"}
-{"id": 99072, "token": "。www", "type": "中文标点"}
-{"id": 99083, "token": "“All", "type": "中文标点"}
-{"id": 99313, "token": ",…", "type": "中文标点"}
-{"id": 99337, "token": "简", "type": "中文单字"}
-{"id": 99379, "token": "移到", "type": "中文多字"}
-{"id": 99382, "token": ")”", "type": "中文标点"}
-{"id": 99397, "token": "問", "type": "中文单字"}
-{"id": 99480, "token": "功能", "type": "中文多字"}
-{"id": 99496, "token": "若要", "type": "中文多字"}
-{"id": 99502, "token": "长度", "type": "中文多字"}
-{"id": 99563, "token": "—at", "type": "中文标点"}
-{"id": 99643, "token": "】,【", "type": "中文标点"}
-{"id": 99741, "token": "装", "type": "中文单字"}
-{"id": 99750, "token": "感", "type": "中文单字"}
-{"id": 99771, "token": "哈", "type": "中文单字"}
-{"id": 99799, "token": "“One", "type": "中文标点"}
-{"id": 99849, "token": "何", "type": "中文单字"}
-{"id": 99941, "token": "预", "type": "中文单字"}
-{"id": 100065, "token": "～\n\n", "type": "中文标点"}
-{"id": 100066, "token": "送料", "type": "中文多字"}
-{"id": 100067, "token": "…it", "type": "中文标点"}
-{"id": 100179, "token": "尔", "type": "中文单字"}
-{"id": 100207, "token": "在线", "type": "中文多字"}

utils/text_util.py

@@ -1,6 +1,8 @@
 
 
 
+
+
 def is_chinese(uchar):
     """
     https://github.com/fxsjy/jieba/blob/master/jieba/__init__.py#L48
@@ -12,4 +14,20 @@ def is_chinese(uchar):
 
 def has_chinese(text):
     """ contains Chinese characters """
-    return any(is_chinese(ch) for ch in text)
+    return any(is_chinese(ch) for ch in text)
+
+
+def get_zh_count(text):
+    return sum([is_chinese(uchar) for uchar in text])
+
+
+def is_all_chinese(text):
+    return all(is_chinese(char) for char in text)
+
+
+def get_digit_count(text):
+    digit_count = 0
+    for char in text:
+        if char in "0123456789":
+            digit_count += 1
+    return digit_count

utils/vocab.jd.txt.v2

@@ -1,10268 +0,0 @@
-[PAD]
-[unused1]
-[unused2]
-[unused3]
-[unused4]
-[unused5]
-[unused6]
-[unused7]
-[unused8]
-[unused9]
-[unused10]
-[unused11]
-[unused12]
-[unused13]
-[unused14]
-[unused15]
-[unused16]
-[unused17]
-[unused18]
-[unused19]
-[unused20]
-[unused21]
-[unused22]
-[unused23]
-[unused24]
-[unused25]
-[unused26]
-[unused27]
-[unused28]
-[unused29]
-[unused30]
-[unused31]
-[unused32]
-[unused33]
-[unused34]
-[unused35]
-[unused36]
-[unused37]
-[unused38]
-[unused39]
-[unused40]
-[unused41]
-[unused42]
-[unused43]
-[unused44]
-[unused45]
-[unused46]
-[unused47]
-[unused48]
-[unused49]
-[unused50]
-[unused51]
-[unused52]
-[unused53]
-[unused54]
-[unused55]
-[unused56]
-[unused57]
-[unused58]
-[unused59]
-[unused60]
-[unused61]
-[unused62]
-[unused63]
-[unused64]
-[unused65]
-[unused66]
-[unused67]
-[unused68]
-[unused69]
-[unused70]
-[unused71]
-[unused72]
-[unused73]
-[unused74]
-[unused75]
-[unused76]
-[unused77]
-[unused78]
-[unused79]
-[unused80]
-[unused81]
-[unused82]
-[unused83]
-[unused84]
-[unused85]
-[unused86]
-[unused87]
-[unused88]
-[unused89]
-[unused90]
-[unused91]
-[unused92]
-[unused93]
-[unused94]
-[unused95]
-[unused96]
-[unused97]
-[unused98]
-[unused99]
-[UNK]
-[CLS]
-[SEP]
-[MASK]
-<S>
-<T>
-!
-"
-”
-“
-—
-–
-…
-’
-‘
-#
-$
-%
-&
-'
-(
-)
-*
-+
-,
--
-.
-/
-0
-1
-2
-3
-4
-5
-6
-7
-8
-9
-10
-11
-12
-13
-14
-15
-16
-17
-18
-19
-20
-21
-22
-23
-24
-25
-26
-27
-28
-29
-30
-31
-32
-33
-34
-35
-36
-37
-38
-39
-40
-41
-42
-43
-44
-45
-46
-47
-48
-49
-50
-51
-52
-53
-54
-55
-56
-57
-58
-59
-60
-61
-62
-63
-64
-65
-66
-67
-68
-69
-70
-71
-72
-73
-74
-75
-76
-77
-78
-79
-80
-81
-82
-83
-84
-85
-86
-87
-88
-89
-90
-91
-92
-93
-94
-95
-96
-97
-98
-99
-100
-120
-128
-180
-200
-256
-304
-360
-500
-512
-1000
-1080
-2000
-2014
-2015
-2016
-2017
-2018
-2019
-2020
-2021
-2022
-:
-;
-<
-=
->
-?
-@
-[
-\
-]
-^
-_
-a
-b
-c
-d
-e
-f
-g
-h
-i
-j
-k
-l
-m
-n
-o
-p
-q
-r
-s
-t
-u
-v
-w
-x
-y
-z
-{
-|
-}
-~
-£
-¤
-¥
-§
-«
-°
-±
-²
-³
-µ
-·
-¹
-º
-»
-¼
-×
-ß
-æ
-÷
-ø
-đ
-ŋ
-ɔ
-ə
-ɡ
-ʰ
-ˇ
-ˈ
-ˊ
-ˋ
-ˍ
-ː
-˙
-˚
-ˢ
-α
-β
-γ
-δ
-ε
-η
-θ
-ι
-κ
-λ
-μ
-ν
-ο
-π
-ρ
-ς
-σ
-τ
-υ
-φ
-χ
-ψ
-ω
-а
-б
-в
-г
-д
-е
-ж
-з
-и
-к
-л
-м
-н
-о
-п
-р
-с
-т
-у
-ф
-х
-ц
-ч
-ш
-ы
-ь
-я
-і
-ก
-ง
-น
-ม
-ย
-ร
-อ
-า
-เ
-๑
-་
-ღ
-ᵃ
-ᵉ
-ᵍ
-ᵏ
-ᵐ
-ᵒ
-ᵘ
-‖
-„
-†
-•
-‥
-‧
-‰
-′
-″
-‹
-›
-※
-‿
-⁄
-ⁱ
-⁺
-ⁿ
-₁
-₂
-₃
-₄
-€
-℃
-№
-ⅰ
-ⅱ
-ⅲ
-ⅳ
-ⅴ
-←
-↑
-→
-↓
-⇒
-∀
-−
-∕
-∙
-√
-∞
-∟
-∠
-∣
-∥
-∩
-∮
-∶
-∼
-∽
-≈
-≒
-≡
-≤
-≥
-≦
-≧
-≪
-≫
-⊙
-⋅
-⋈
-⋯
-⌒
-①
-②
-③
-④
-⑤
-⑥
-⑦
-⑧
-⑨
-⑩
-⑴
-⑵
-⑶
-⑷
-⑸
-⒈
-⒉
-⒊
-⒋
-ⓒ
-ⓔ
-ⓘ
-─
-━
-│
-┃
-┅
-┆
-┊
-┌
-└
-├
-┣
-═
-║
-╚
-╞
-╠
-╭
-╮
-╯
-╰
-╱
-╳
-▂
-▃
-▅
-▇
-█
-▉
-▋
-▌
-▍
-▎
-■
-□
-▬
-▲
-△
-►
-▼
-▽
-◆
-◇
-○
-◎
-●
-◕
-◠
-◢
-◤
-★
-☆
-☞
-☼
-♡
-♪
-♫
-♬
-✕
-✦
-✪
-✰
-✿
-❀
-➜
-➤
-⦿
-、
-。
-〃
-々
-〇
-〈
-〉
-《
-》
-「
-」
-『
-』
-【
-】
-〓
-〔
-〕
-〖
-〗
-〜
-〝
-〞
-ㄧ
-ㆍ
-㈦
-㊣
-㎡
-㗎
-一
-丁
-七
-万
-丈
-三
-上
-下
-不
-与
-丐
-丑
-专
-且
-丕
-世
-丘
-丙
-业
-丛
-东
-丝
-丞
-両
-丢
-两
-严
-丧
-丨
-个
-丫
-中
-丰
-串
-临
-丶
-丸
-丹
-为
-主
-丼
-丽
-举
-丿
-乂
-乃
-久
-么
-义
-之
-乌
-乍
-乎
-乏
-乐
-乒
-乓
-乔
-乖
-乗
-乘
-乙
-乜
-九
-乞
-也
-习
-乡
-书
-乩
-买
-乱
-乳
-亀
-了
-予
-争
-事
-二
-于
-亏
-云
-互
-五
-井
-亘
-亚
-些
-亜
-亟
-亡
-亢
-交
-亥
-亦
-产
-亨
-亩
-享
-京
-亭
-亮
-亲
-亳
-亵
-人
-亿
-什
-仁
-仃
-仄
-仅
-仆
-仇
-今
-介
-仍
-从
-仏
-仑
-仓
-仔
-仕
-他
-仗
-付
-仙
-仝
-仞
-仟
-代
-令
-以
-仨
-仪
-们
-仮
-仰
-仲
-件
-价
-任
-份
-仿
-企
-伉
-伊
-伍
-伎
-伏
-伐
-休
-伕
-众
-优
-伙
-会
-伝
-伞
-伟
-传
-伢
-伤
-伦
-伪
-伫
-伯
-估
-伴
-伶
-伸
-伺
-似
-伽
-佃
-但
-位
-低
-住
-佐
-佑
-体
-佔
-何
-佗
-佘
-余
-佚
-佛
-作
-佝
-佞
-佟
-你
-佢
-佣
-佤
-佥
-佩
-佬
-佯
-佰
-佳
-佶
-佻
-佼
-使
-侃
-侄
-侈
-例
-侍
-侏
-侑
-侗
-供
-依
-侠
-価
-侣
-侥
-侦
-侧
-侨
-侬
-侮
-侯
-侵
-侷
-便
-促
-俄
-俊
-俎
-俏
-俐
-俑
-俗
-俘
-俚
-保
-俞
-俟
-信
-俨
-俩
-俪
-俬
-俭
-修
-俯
-俱
-俳
-俸
-俺
-俾
-倌
-倍
-倏
-倒
-倔
-倖
-倘
-候
-倚
-倜
-借
-倡
-値
-倦
-倩
-倪
-倬
-倭
-倶
-债
-值
-倾
-偃
-假
-偈
-偌
-偎
-偏
-偕
-做
-停
-健
-偶
-偷
-偻
-偿
-傀
-傅
-傍
-傚
-傣
-傥
-储
-傩
-催
-傲
-傻
-働
-像
-僖
-僚
-僧
-僭
-僮
-僱
-僵
-僻
-儆
-儋
-儒
-儡
-儿
-兀
-允
-元
-兄
-充
-兆
-先
-光
-克
-免
-児
-兑
-兔
-兖
-党
-兜
-兢
-入
-全
-八
-公
-六
-兮
-兰
-共
-兲
-关
-兴
-兵
-其
-具
-典
-兹
-养
-兼
-兽
-冀
-内
-円
-冇
-冈
-冉
-册
-再
-冏
-��
-冕
-冗
-写
-军
-农
-冠
-冢
-冤
-冥
-冨
-冬
-冯
-冰
-冲
-决
-况
-冶
-冷
-冻
-冼
-冽
-冾
-净
-凄
-准
-凇
-凉
-凋
-凌
-减
-凑
-凛
-凝
-几
-凡
-凤
-処
-凪
-凭
-凯
-凰
-凳
-凶
-凸
-凹
-出
-击
-函
-凿
-刀
-刁
-刃
-分
-切
-刈
-刊
-刍
-刎
-刑
-划
-列
-刘
-则
-刚
-创
-初
-删
-判
-刨
-利
-别
-刮
-到
-制
-刷
-券
-刹
-刺
-刻
-刽
-剁
-剂
-剃
-剉
-削
-剌
-前
-剐
-剑
-剔
-剖
-剜
-剣
-剤
-剥
-剧
-剩
-剪
-副
-割
-剷
-剽
-剿
-劈
-力
-劝
-办
-功
-加
-务
-劣
-动
-助
-努
-劫
-劭
-励
-劲
-劳
-労
-劵
-効
-劾
-势
-勃
-勇
-勉
-勋
-勐
-勒
-勖
-勘
-募
-勤
-勧
-勳
-勺
-勾
-勿
-匀
-包
-匆
-匈
-匍
-匐
-匕
-化
-北
-匙
-匝
-匠
-匡
-匣
-匪
-匮
-匹
-区
-医
-匾
-匿
-十
-千
-卅
-升
-午
-卉
-半
-卍
-华
-协
-卑
-卒
-卓
-单
-卖
-南
-単
-博
-卜
-卞
-卟
-占
-卡
-卢
-卤
-卦
-卧
-卫
-卮
-卯
-印
-危
-即
-却
-卵
-卷
-卸
-卿
-厂
-厄
-厅
-历
-厉
-压
-厌
-厕
-厘
-厚
-厝
-原
-厢
-厥
-厦
-厨
-厩
-厮
-厳
-去
-县
-叁
-参
-又
-叉
-及
-友
-双
-反
-収
-发
-叔
-取
-受
-变
-叙
-叛
-叟
-叠
-叡
-口
-古
-句
-另
-叨
-叩
-只
-叫
-召
-叭
-叮
-可
-台
-叱
-史
-右
-叵
-叶
-号
-司
-叹
-叻
-叼
-叽
-吁
-吃
-各
-吆
-合
-吉
-吊
-吋
-同
-名
-后
-吏
-吐
-向
-吒
-吓
-吕
-吖
-吗
-君
-吝
-吞
-吟
-吠
-吡
-否
-吧
-吨
-吩
-含
-听
-吭
-吮
-启
-吱
-吴
-吵
-吸
-吹
-吻
-吼
-吽
-吾
-呀
-呃
-呆
-呈
-告
-呋
-呎
-呐
-呓
-呕
-呗
-员
-呛
-呜
-呢
-呤
-呦
-周
-呱
-呲
-味
-呵
-呷
-呸
-呻
-呼
-命
-咀
-咁
-咂
-咄
-咆
-咋
-和
-咎
-咏
-咐
-咒
-咔
-咕
-咖
-咗
-咘
-咙
-咚
-咛
-咣
-咤
-咦
-咧
-咨
-咩
-咪
-咫
-咬
-咭
-咯
-咱
-咲
-咳
-咸
-咻
-咽
-咿
-哀
-品
-哂
-哄
-哆
-哇
-哈
-哉
-哋
-哌
-响
-哎
-哏
-哐
-哑
-哒
-哔
-哗
-哟
-哥
-哦
-哧
-哨
-哩
-哪
-哭
-哮
-哲
-哺
-哼
-哽
-唁
-唆
-唇
-唉
-唏
-唐
-唑
-唔
-唠
-唤
-唧
-唬
-售
-唯
-唰
-唱
-唳
-唷
-唸
-唾
-啃
-啄
-商
-啉
-啊
-啕
-啖
-啜
-啡
-啤
-啥
-啦
-啧
-啪
-啫
-啬
-啮
-啰
-啱
-啲
-啵
-啶
-啷
-啸
-啻
-啼
-啾
-喀
-喂
-喃
-善
-喆
-喇
-喉
-喊
-喋
-喏
-喔
-喘
-喙
-喜
-喝
-喟
-喧
-喫
-喰
-喱
-喳
-喵
-営
-喷
-喹
-喺
-喻
-喽
-嗅
-嗑
-嗒
-嗓
-嗔
-嗖
-嗜
-嗝
-嗟
-嗡
-嗣
-嗤
-嗦
-嗨
-嗪
-嗬
-嗯
-嗰
-嗲
-嗳
-嗷
-嗽
-嘀
-嘅
-嘈
-嘉
-嘌
-嘎
-嘘
-嘚
-嘛
-嘞
-嘟
-嘢
-嘣
-嘤
-嘧
-嘭
-嘱
-嘲
-嘴
-嘶
-嘹
-嘻
-嘿
-噌
-噎
-噔
-噗
-噙
-噜
-噢
-噤
-器
-噩
-噪
-噬
-噱
-噶
-噻
-噼
-嚎
-嚏
-嚐
-嚓
-嚟
-嚣
-嚷
-嚼
-囉
-囊
-囍
-囔
-囗
-囚
-四
-囝
-回
-囟
-因
-囡
-团
-団
-囤
-囧
-囫
-园
-困
-囱
-囲
-図
-围
-囹
-固
-国
-图
-囿
-圃
-圄
-圆
-圈
-圏
-圜
-土
-圣
-圧
-在
-圩
-圭
-地
-圳
-场
-圻
-圾
-址
-坂
-均
-坊
-坍
-坎
-坏
-坐
-坑
-块
-坚
-坛
-坝
-坞
-坟
-坠
-坡
-坤
-坦
-坨
-坪
-坯
-坳
-坵
-坷
-垂
-垃
-垄
-型
-垒
-垚
-垛
-垠
-垢
-垣
-垦
-垩
-垫
-垭
-垮
-埂
-埃
-埋
-城
-埔
-埕
-埗
-域
-埠
-埤
-埵
-埸
-培
-基
-埼
-堀
-堂
-堃
-堆
-堇
-堑
-堕
-堙
-堡
-堤
-堪
-堰
-堵
-堺
-堿
-塌
-塑
-塔
-塘
-塞
-塩
-填
-塬
-塭
-塾
-墀
-境
-墅
-墉
-墒
-墓
-増
-墘
-墙
-增
-墟
-墨
-墩
-壁
-壅
-壆
-壊
-壑
-壕
-壤
-士
-壬
-壮
-声
-売
-壳
-壶
-壹
-处
-备
-変
-复
-夏
-夔
-夕
-外
-夙
-多
-夜
-够
-夥
-大
-天
-太
-夫
-夭
-央
-夯
-失
-头
-夷
-夸
-夹
-夺
-奂
-奄
-奇
-奈
-奉
-奋
-奎
-奏
-契
-奔
-奕
-奖
-套
-奘
-奚
-奠
-奢
-奥
-女
-奴
-奶
-奸
-她
-好
-如
-妃
-妄
-妆
-妇
-妈
-妊
-妍
-妒
-妓
-妖
-妘
-妙
-妞
-妣
-妤
-妥
-妨
-妩
-妪
-妮
-妲
-妳
-妹
-妻
-妾
-姆
-姉
-姊
-始
-姐
-姑
-姒
-姓
-委
-姗
-姚
-姜
-姝
-姣
-姥
-姨
-姪
-姫
-姬
-姹
-姻
-姿
-威
-娃
-娄
-娅
-娆
-娇
-娉
-娑
-娓
-娘
-娜
-娟
-娠
-娣
-娥
-娩
-娱
-娲
-娴
-娶
-娼
-婀
-婆
-婉
-婊
-婕
-婚
-婢
-婧
-婪
-婴
-婵
-婶
-婷
-婺
-婿
-媒
-媚
-媛
-媞
-媲
-媳
-媾
-嫁
-嫂
-嫉
-嫌
-嫑
-嫔
-嫖
-嫘
-嫚
-嫡
-嫣
-嫦
-嫩
-嫲
-嬅
-嬉
-嬗
-嬛
-嬢
-嬴
-嬷
-嬿
-孀
-孃
-子
-孑
-孔
-孕
-孖
-字
-存
-孙
-孚
-孛
-孜
-孝
-孟
-孢
-季
-孤
-学
-孩
-孪
-孬
-孰
-孱
-孳
-孵
-孺
-孽
-宁
-它
-宅
-宇
-守
-安
-宋
-完
-宏
-宓
-宕
-宗
-官
-宙
-定
-宛
-宜
-宝
-实
-実
-宠
-审
-客
-宣
-室
-宥
-宦
-宪
-宫
-宰
-害
-宴
-宵
-家
-宸
-容
-宽
-宾
-宿
-寂
-寄
-寅
-密
-寇
-富
-寐
-寒
-寓
-寛
-寝
-寞
-察
-寡
-寥
-寨
-寮
-寰
-寸
-对
-寺
-寻
-导
-対
-寿
-封
-専
-射
-将
-尉
-尊
-小
-少
-尔
-尕
-尖
-尘
-尚
-尝
-尤
-尧
-尬
-就
-尴
-尸
-尹
-尺
-尻
-尼
-尽
-尾
-尿
-局
-屁
-层
-屄
-居
-屈
-屉
-届
-屋
-屌
-屎
-屏
-屐
-屑
-展
-属
-屠
-屡
-履
-屯
-山
-屹
-屿
-岀
-岁
-岂
-岌
-岐
-岑
-岔
-岖
-岗
-岘
-岙
-岚
-岛
-岩
-岫
-岬
-岭
-岱
-岳
-岷
-岸
-��
-峋
-峒
-峙
-峡
-峤
-峥
-峦
-峨
-峪
-峭
-峯
-峰
-峻
-崁
-崂
-崆
-崇
-崎
-崑
-崔
-崖
-崙
-崛
-崧
-崩
-崭
-崴
-崽
-嵇
-嵊
-嵋
-嵌
-嵘
-嵩
-嵬
-嵯
-嶂
-嶋
-嶙
-巅
-巍
-巖
-川
-州
-巡
-巢
-工
-左
-巧
-巨
-巩
-巫
-差
-己
-已
-巳
-巴
-巷
-巻
-巽
-巾
-巿
-币
-市
-布
-帅
-帆
-师
-希
-帐
-帑
-帕
-帖
-帘
-帚
-帛
-帜
-帝
-带
-帧
-席
-帮
-帯
-帰
-帷
-常
-帼
-帽
-幂
-幄
-幅
-幌
-幔
-幕
-幡
-幢
-干
-平
-年
-并
-幸
-幻
-幼
-幽
-广
-庁
-広
-庄
-庆
-庇
-床
-序
-庐
-库
-应
-底
-庖
-店
-庙
-庚
-府
-庞
-废
-庠
-度
-座
-庭
-庵
-庶
-康
-庸
-庹
-庾
-廃
-廉
-廊
-廓
-廖
-延
-廷
-建
-廿
-开
-弁
-异
-弃
-弄
-弈
-弊
-弋
-式
-弑
-弓
-弔
-引
-弗
-弘
-弛
-弟
-张
-弥
-弦
-弧
-弩
-弭
-弯
-弱
-弹
-强
-弼
-弾
-彅
-归
-当
-录
-彗
-彝
-形
-彤
-彦
-彧
-彩
-彪
-彫
-彬
-彭
-彰
-影
-彷
-役
-彻
-彼
-彿
-往
-征
-径
-待
-徇
-很
-徉
-徊
-律
-徐
-徒
-従
-徕
-得
-徘
-徙
-徜
-御
-徨
-循
-徬
-微
-徳
-徴
-德
-徼
-徽
-心
-必
-忆
-忌
-忍
-忏
-忐
-忑
-忒
-忖
-志
-忘
-忙
-応
-忠
-忡
-忤
-忧
-忪
-快
-忱
-念
-忻
-忽
-忿
-怀
-态
-怂
-怅
-怆
-怎
-怏
-怒
-怔
-怕
-怖
-怙
-怜
-思
-怠
-怡
-急
-怦
-性
-怨
-怪
-怯
-怵
-总
-怼
-恁
-恃
-恋
-恍
-恐
-恒
-恕
-恙
-恚
-恢
-恣
-恤
-恨
-恩
-恪
-恫
-恬
-恭
-息
-恰
-恳
-恵
-恶
-恸
-恺
-恻
-恼
-恿
-悄
-悉
-悌
-悍
-悔
-悖
-悚
-悟
-悠
-患
-悦
-您
-悩
-悪
-悬
-悯
-悱
-悲
-悴
-悸
-悻
-悼
-悽
-情
-惆
-惇
-惊
-惋
-惑
-惕
-惘
-惚
-惜
-惟
-惠
-惦
-惧
-惨
-惩
-惫
-惬
-惭
-惮
-惯
-惰
-想
-惴
-惶
-惹
-惺
-愁
-愆
-愈
-愉
-愍
-意
-愕
-愚
-感
-愣
-愤
-愧
-愫
-愿
-慈
-慌
-慎
-慑
-慕
-慢
-慧
-慨
-慰
-慵
-慷
-慾
-憋
-憎
-憔
-憧
-憨
-憩
-憬
-憾
-懂
-懈
-懊
-懋
-懑
-懒
-懦
-懵
-懿
-戈
-戊
-戌
-戍
-戎
-戏
-成
-我
-戒
-戕
-或
-战
-戚
-戛
-戟
-戡
-戦
-截
-戬
-戮
-戳
-戴
-户
-戸
-戻
-戾
-房
-所
-扁
-扇
-扈
-扉
-手
-才
-扎
-扑
-扒
-打
-扔
-払
-托
-扛
-扣
-扦
-执
-扩
-扪
-扫
-扬
-扭
-扮
-扯
-扰
-扱
-扳
-扶
-批
-扼
-找
-承
-技
-抄
-抉
-把
-抑
-抒
-抓
-投
-抖
-抗
-折
-抚
-抛
-抜
-択
-抟
-抠
-抡
-抢
-护
-报
-抨
-披
-抬
-抱
-抵
-抹
-押
-抽
-抿
-拂
-拄
-担
-拆
-拇
-拈
-拉
-拌
-拍
-拎
-拐
-拒
-拓
-拔
-拖
-拗
-拘
-拙
-拚
-招
-拜
-拟
-拡
-拢
-拣
-拥
-拦
-拧
-拨
-择
-括
-拭
-拮
-拯
-拱
-拳
-拴
-拷
-拼
-拽
-拾
-拿
-持
-挂
-指
-挈
-按
-挎
-挑
-挖
-挙
-挚
-挛
-挝
-挞
-挟
-挠
-挡
-挣
-挤
-挥
-挨
-挪
-挫
-振
-挲
-挹
-挺
-挽
-捂
-捅
-捆
-捉
-捋
-捌
-捍
-捎
-捏
-捐
-捕
-捞
-损
-捡
-换
-捣
-捧
-捩
-据
-捱
-捲
-捶
-捷
-捺
-捻
-掀
-掂
-掇
-授
-掉
-掌
-掏
-掐
-排
-掖
-掘
-掠
-探
-掣
-接
-控
-推
-掩
-措
-掬
-掰
-掲
-掳
-掴
-掷
-掸
-掺
-揃
-揄
-揆
-揉
-揍
-描
-提
-插
-揖
-握
-揣
-揩
-揪
-揭
-援
-揶
-揸
-揹
-揽
-搀
-搁
-搂
-搅
-搏
-搐
-搓
-搔
-搜
-搞
-搡
-搪
-搬
-搭
-携
-搽
-摀
-摁
-摄
-摆
-摇
-摈
-摊
-摒
-摔
-摘
-摞
-摧
-摩
-摸
-摹
-撂
-撃
-撅
-撇
-撑
-撒
-撕
-撚
-撞
-撤
-撩
-撬
-播
-撮
-撰
-撵
-撷
-撸
-撼
-擀
-擂
-擅
-操
-擎
-擒
-擘
-擞
-擡
-擢
-擦
-攀
-攒
-攘
-攞
-攥
-攫
-支
-收
-攸
-改
-攻
-放
-政
-故
-效
-敌
-敍
-敎
-敏
-救
-敕
-敖
-教
-敛
-敝
-敞
-敢
-散
-敦
-敬
-数
-敲
-整
-敷
-文
-斋
-斌
-斎
-斐
-斑
-斓
-斗
-料
-斛
-斜
-斟
-斡
-斤
-斥
-斧
-斩
-斫
-断
-斯
-新
-方
-施
-旁
-旃
-旅
-旋
-旌
-旎
-族
-旖
-旗
-无
-既
-日
-旦
-旧
-旨
-早
-旬
-旭
-旮
-旱
-时
-旷
-旺
-旻
-昀
-昂
-昆
-昇
-昉
-昊
-昌
-明
-昏
-易
-昔
-昕
-昙
-星
-映
-春
-昧
-昨
-昭
-是
-昱
-昴
-昵
-昶
-昼
-显
-晁
-晃
-晋
-晌
-晏
-晒
-晓
-晔
-晕
-晖
-晗
-晚
-晞
-晟
-晤
-晦
-晨
-晩
-普
-景
-晰
-晴
-晶
-晷
-智
-晾
-暂
-暄
-暇
-暌
-暐
-暑
-暖
-暗
-暝
-暧
-暨
-暮
-暱
-暴
-暸
-暹
-曙
-曜
-曝
-曦
-曰
-曲
-曳
-更
-曹
-曼
-曾
-替
-最
-月
-有
-朋
-服
-朐
-朔
-朕
-朗
-望
-朝
-期
-朦
-木
-未
-末
-本
-札
-术
-朱
-朴
-朵
-机
-朽
-杀
-杂
-权
-杆
-杈
-杉
-李
-杏
-材
-村
-杓
-杖
-杜
-杞
-束
-杠
-条
-来
-杨
-杭
-杯
-杰
-杳
-杵
-杷
-杼
-松
-板
-极
-构
-枇
-枉
-枋
-析
-枕
-林
-枚
-果
-枝
-枢
-枣
-枪
-枫
-枭
-枯
-枰
-枱
-枳
-架
-枷
-枸
-柄
-柏
-某
-柑
-柒
-染
-柔
-柘
-柚
-柜
-柞
-柠
-柢
-查
-柩
-柬
-柯
-柱
-柳
-柴
-査
-柿
-栀
-栃
-栄
-栅
-标
-栈
-栉
-栋
-栎
-栏
-树
-栓
-栖
-栗
-校
-栩
-株
-样
-核
-根
-格
-栽
-栾
-桀
-桁
-桂
-桃
-桅
-框
-案
-桉
-桌
-桎
-桐
-桑
-桓
-桔
-桜
-桠
-桡
-桢
-档
-桥
-桦
-桧
-桨
-桩
-桶
-梁
-梅
-梆
-梏
-梓
-梗
-梢
-梦
-梧
-梨
-梭
-梯
-械
-梳
-梵
-梶
-检
-棂
-棉
-棋
-棍
-棒
-棕
-棘
-棚
-棠
-棣
-森
-棱
-棵
-棹
-棺
-椁
-椅
-椋
-��
-椎
-椒
-検
-椪
-椭
-椰
-椹
-椽
-椿
-楂
-楔
-楚
-楝
-楞
-楠
-楣
-楫
-楮
-楷
-楸
-楹
-楼
-楽
-概
-榄
-榆
-榈
-榉
-榔
-榕
-榖
-榛
-榜
-榨
-榫
-榭
-榱
-榴
-榷
-榻
-槁
-槃
-槌
-槎
-槐
-槓
-様
-槛
-槟
-槭
-槲
-槻
-槽
-槿
-樊
-樑
-樟
-模
-権
-横
-樫
-樯
-樱
-樵
-樽
-樾
-橄
-橇
-橐
-橘
-橙
-橡
-橱
-橹
-橼
-檀
-檄
-檎
-檐
-檗
-檬
-欠
-次
-欢
-欣
-欧
-欲
-欸
-欺
-款
-歆
-歇
-歉
-歌
-歎
-歓
-歙
-歛
-止
-正
-此
-步
-武
-歧
-歩
-歪
-歯
-歳
-歴
-歹
-死
-歼
-殁
-殃
-殆
-殇
-殉
-殊
-残
-殒
-殓
-殖
-殡
-殭
-殴
-段
-殷
-殿
-毁
-毂
-毅
-毋
-母
-毎
-每
-毒
-毓
-比
-毕
-毗
-毘
-毙
-毛
-毡
-毫
-毯
-毽
-氏
-氐
-民
-氓
-气
-氖
-気
-氙
-氛
-氟
-氡
-氢
-氤
-氦
-氧
-氨
-氪
-氮
-氯
-氰
-氲
-水
-氷
-永
-氹
-氾
-汀
-汁
-求
-汆
-汇
-汉
-汎
-汐
-汕
-汗
-汛
-汝
-汞
-江
-池
-污
-汤
-汨
-汩
-汪
-汰
-汲
-汴
-汶
-汹
-汽
-汾
-沁
-沂
-沃
-沅
-沈
-沉
-沌
-沏
-沐
-沓
-沙
-沛
-沟
-没
-沢
-沣
-沥
-沦
-沧
-沪
-沫
-沭
-沮
-沱
-河
-沸
-油
-治
-沼
-沽
-沾
-沿
-泄
-泉
-泊
-泌
-泓
-法
-泗
-泛
-泞
-泠
-泡
-波
-泣
-泥
-注
-泪
-泫
-泮
-泯
-泰
-泱
-泳
-泵
-泷
-泸
-泻
-泼
-泽
-泾
-洁
-洄
-洋
-洒
-洗
-洙
-洛
-洞
-津
-洩
-洪
-洮
-洱
-洲
-洵
-洸
-洹
-活
-洼
-洽
-派
-流
-浃
-浄
-浅
-浆
-浇
-浊
-测
-济
-浏
-浑
-浒
-浓
-浔
-浙
-浚
-浜
-浣
-浦
-浩
-浪
-浬
-浮
-浯
-浴
-海
-浸
-涂
-涅
-消
-涉
-涌
-涎
-涓
-涔
-涕
-涙
-涛
-涝
-涞
-涟
-涠
-涡
-涣
-涤
-润
-涧
-涨
-涩
-涪
-涮
-涯
-液
-涵
-涸
-涿
-淀
-淄
-淅
-淆
-淇
-淋
-淌
-淑
-淖
-淘
-淙
-淞
-淡
-淤
-淦
-淫
-淬
-淮
-深
-淳
-混
-淹
-添
-淼
-清
-済
-渉
-渊
-渋
-渍
-渎
-渐
-渔
-渗
-渚
-渝
-渠
-渡
-渣
-渤
-渥
-温
-渭
-港
-渲
-渴
-游
-渺
-湃
-湄
-湍
-湖
-湘
-湛
-湟
-湧
-湫
-湮
-湳
-湾
-湿
-満
-溃
-溅
-溉
-溏
-源
-溜
-溟
-溢
-溥
-溧
-溪
-溯
-溱
-溴
-溶
-溺
-溼
-滁
-滂
-滇
-滋
-滑
-滓
-滔
-滕
-滙
-滚
-滝
-滞
-滟
-满
-滢
-滤
-滥
-滦
-滨
-滩
-滴
-漂
-漆
-漉
-漏
-漓
-演
-漕
-漠
-漩
-漪
-漫
-漯
-漱
-漳
-漾
-潆
-潇
-潋
-潍
-潘
-潜
-潞
-潟
-潢
-潦
-潧
-潭
-潮
-潴
-潸
-潺
-潼
-澄
-澈
-澍
-澎
-澜
-澡
-澧
-澳
-澹
-激
-濂
-濑
-濒
-濠
-濡
-濬
-濮
-濯
-瀑
-瀚
-瀛
-瀞
-瀬
-灌
-灏
-灞
-火
-灬
-灭
-灯
-灰
-灵
-灶
-灸
-灼
-灾
-灿
-炀
-炁
-炅
-炉
-炊
-炎
-炒
-炔
-炕
-炖
-炙
-炜
-炫
-炬
-炭
-炮
-炯
-炳
-炷
-炸
-点
-炼
-炽
-烁
-烂
-烃
-烈
-烊
-烘
-烙
-烛
-烟
-烤
-烦
-烧
-烨
-烩
-烫
-烬
-热
-烯
-烷
-烹
-烽
-焉
-焊
-焕
-焖
-焗
-焘
-焙
-焚
-焜
-焦
-焯
-焰
-焱
-然
-焼
-煅
-煊
-煌
-煎
-煖
-煜
-煞
-煤
-煦
-照
-煨
-煮
-煲
-煸
-煽
-熄
-熊
-熏
-熔
-熙
-熟
-熠
-熨
-熬
-熵
-熹
-燃
-燄
-燊
-燎
-燔
-燕
-燥
-燧
-燮
-燻
-燿
-爆
-爪
-爬
-爰
-爱
-爵
-父
-爷
-爸
-爹
-爻
-爽
-片
-版
-牌
-牍
-牒
-牙
-牛
-牝
-牟
-牠
-牡
-牢
-牦
-牧
-物
-牯
-牲
-牴
-牵
-特
-牺
-犀
-犁
-犄
-犊
-犍
-犒
-犬
-犯
-状
-犷
-犸
-犹
-狂
-狄
-狈
-狎
-狐
-狒
-狗
-狙
-狞
-狠
-狡
-狩
-独
-狭
-狮
-狰
-狱
-狸
-狼
-猎
-猕
-猖
-猗
-猛
-猜
-猝
-猥
-猩
-猪
-猫
-猬
-献
-猴
-猷
-猾
-猿
-獐
-獒
-獗
-獠
-獣
-獭
-獾
-玄
-率
-玉
-王
-玑
-玖
-玛
-玟
-玠
-玥
-玩
-玫
-玮
-环
-现
-玲
-玳
-玷
-玺
-玻
-珀
-珂
-珅
-珈
-珉
-珊
-珍
-珏
-珐
-珑
-珙
-珞
-珠
-珣
-珥
-珩
-珪
-班
-珮
-珲
-珺
-球
-琅
-理
-琇
-琉
-琊
-琍
-琏
-琐
-琛
-琢
-琥
-琦
-琨
-琪
-琬
-琮
-琰
-琲
-琳
-琴
-琵
-琶
-琼
-瑀
-瑁
-瑄
-瑕
-瑗
-瑙
-瑚
-瑛
-瑜
-瑞
-瑟
-瑠
-瑯
-瑰
-瑶
-瑾
-璀
-璁
-璃
-璇
-璋
-璎
-璐
-璜
-璞
-璟
-璧
-璨
-璿
-瓒
-瓜
-瓢
-瓣
-瓤
-瓦
-瓮
-瓯
-瓴
-瓶
-瓷
-甄
-甕
-甘
-甙
-甚
-甜
-生
-甥
-甦
-用
-甩
-甫
-甬
-甭
-甯
-田
-由
-甲
-申
-电
-男
-甸
-町
-画
-甾
-畀
-畅
-界
-畏
-畑
-畔
-留
-畜
-略
-畦
-番
-畲
-畳
-畴
-畸
-畹
-畿
-疆
-疏
-疑
-疔
-疖
-疗
-疙
-疚
-疝
-疟
-疡
-疣
-疤
-疥
-疫
-疮
-疯
-疱
-疲
-疳
-疵
-疸
-疹
-疼
-疽
-疾
-痂
-病
-症
-痈
-痉
-痊
-痍
-痒
-痔
-痕
-痘
-痛
-痞
-痠
-痢
-痣
-痤
-痧
-痨
-痪
-痫
-痰
-痱
-痴
-痹
-痺
-痼
-痿
-瘀
-瘁
-瘘
-瘙
-瘟
-瘠
-瘢
-瘤
-瘦
-瘩
-瘪
-瘫
-瘴
-瘸
-瘾
-癌
-癒
-癖
-癜
-癞
-癡
-癣
-癫
-癸
-発
-登
-白
-百
-皂
-的
-皆
-皇
-皈
-皋
-皎
-皑
-皓
-皖
-皙
-皮
-皱
-皴
-皿
-盂
-盅
-盆
-盈
-益
-盎
-盏
-盐
-监
-盒
-盔
-盖
-盗
-盘
-盛
-盟
-盥
-目
-盯
-盱
-盲
-直
-相
-盹
-盼
-盾
-省
-眈
-眉
-看
-県
-眙
-眞
-真
-眠
-眦
-眨
-眩
-眯
-眶
-眷
-眸
-眺
-眼
-着
-睁
-睇
-睐
-睑
-睛
-睡
-睢
-督
-睥
-睦
-睨
-睪
-睫
-睬
-睹
-睽
-睾
-睿
-瞄
-瞅
-瞇
-瞋
-瞌
-瞎
-瞑
-瞒
-瞓
-瞟
-瞠
-瞥
-瞧
-瞩
-瞪
-瞬
-瞰
-瞳
-瞻
-瞿
-矍
-矗
-矛
-矜
-矢
-��
-知
-矩
-矫
-短
-矮
-石
-矶
-矽
-矾
-矿
-码
-砂
-砌
-砍
-砒
-研
-砖
-砗
-砚
-砝
-砣
-砥
-砧
-砭
-砰
-砲
-破
-砷
-砸
-砺
-砼
-砾
-础
-硅
-硐
-硒
-硕
-硝
-硫
-硬
-确
-硼
-碁
-碇
-碉
-碌
-碍
-碎
-碑
-碓
-碗
-碘
-碚
-碛
-碟
-碣
-碧
-碰
-碱
-碳
-碴
-碾
-磁
-磅
-磊
-磋
-磐
-磕
-磡
-磨
-磬
-磲
-磷
-磺
-礁
-礡
-礴
-示
-礼
-社
-祀
-祁
-祂
-祇
-祈
-祉
-祎
-祐
-祕
-祖
-祗
-祚
-祛
-祜
-祝
-神
-祟
-祠
-祢
-祥
-票
-祭
-祯
-祷
-祸
-祺
-禀
-禁
-禄
-禅
-福
-禛
-禧
-禹
-禺
-离
-禽
-禾
-秀
-私
-秃
-秆
-秉
-秋
-种
-科
-秒
-秘
-租
-秣
-秤
-秦
-秧
-秩
-秭
-积
-称
-秸
-移
-秽
-稀
-程
-稍
-税
-稔
-稗
-稚
-稜
-稞
-稠
-稣
-稲
-稳
-稷
-稹
-稻
-稼
-稽
-稿
-穂
-穆
-穗
-穴
-究
-穷
-穹
-空
-穿
-突
-窃
-窄
-窈
-窍
-窑
-窒
-窓
-窕
-窖
-窗
-窘
-窜
-窝
-窟
-窠
-窥
-窦
-窨
-窿
-立
-竖
-站
-竜
-竞
-竟
-章
-竣
-童
-竭
-端
-竹
-竺
-竽
-竿
-笃
-笆
-笈
-笋
-笏
-笑
-笔
-笙
-笛
-笞
-笠
-符
-笨
-第
-笹
-笺
-笼
-等
-筊
-筋
-筏
-筐
-筑
-筒
-答
-策
-筛
-筝
-筠
-筱
-筲
-筵
-筷
-筹
-签
-简
-箇
-箍
-箐
-箔
-箕
-算
-箝
-管
-箩
-箫
-箭
-箱
-箴
-箸
-篁
-篆
-篇
-篑
-篓
-篙
-篝
-篠
-篡
-篪
-篮
-篱
-篷
-簇
-簌
-簦
-簧
-簪
-簷
-簸
-簿
-籁
-籍
-籐
-籤
-米
-类
-籼
-籽
-粄
-粉
-粑
-粒
-粕
-粗
-粘
-粟
-粤
-粥
-粧
-粪
-粮
-粱
-粲
-粳
-粹
-粼
-粽
-精
-粿
-糅
-糊
-糍
-糕
-糖
-糗
-糙
-糜
-糟
-糠
-糬
-糯
-糸
-系
-紊
-素
-索
-紧
-紫
-紮
-累
-絃
-経
-絮
-絵
-綑
-継
-続
-綦
-総
-縁
-縄
-繁
-繇
-繋
-纂
-纠
-红
-纣
-纤
-约
-级
-纨
-纪
-纫
-纬
-纭
-纯
-纰
-纱
-纲
-纳
-纵
-纶
-纷
-纸
-纹
-纺
-纽
-纾
-线
-绀
-练
-组
-绅
-细
-织
-终
-绊
-绍
-绎
-经
-绑
-绒
-结
-绔
-绕
-绘
-给
-绚
-绛
-络
-绝
-绞
-统
-绡
-绢
-绣
-绥
-绦
-继
-绩
-绪
-绫
-续
-绮
-绯
-绰
-绳
-维
-绵
-绶
-绷
-绸
-绻
-综
-绽
-绾
-绿
-缀
-缄
-缅
-缆
-缇
-缈
-缉
-缎
-缓
-缔
-缕
-编
-缘
-缙
-缚
-缜
-缝
-缠
-缢
-缤
-缥
-缨
-缩
-缪
-缭
-缮
-缰
-缱
-缴
-缸
-缺
-罂
-罄
-罐
-网
-罔
-罕
-罗
-罚
-罡
-罢
-罩
-罪
-置
-署
-罹
-羁
-羊
-羌
-美
-羔
-羚
-羞
-羟
-羡
-羣
-群
-羧
-羨
-羯
-羲
-羸
-羹
-羽
-羿
-翁
-翅
-翊
-翌
-翎
-翔
-翘
-翟
-翠
-翡
-翦
-翩
-翰
-翱
-翳
-翻
-翼
-耀
-老
-考
-耄
-者
-耆
-耋
-而
-耍
-耐
-耒
-耕
-耗
-耘
-耙
-耦
-耨
-耳
-耶
-耷
-耸
-耻
-耽
-耿
-聂
-聆
-聊
-聋
-职
-聒
-联
-聘
-聚
-聪
-聴
-聿
-肃
-肄
-肆
-肇
-肉
-肋
-肌
-肏
-肓
-肖
-肘
-肚
-肛
-肝
-肠
-股
-肢
-肤
-肥
-肩
-肪
-肮
-肯
-肱
-育
-肴
-肺
-肽
-肾
-肿
-胀
-胁
-胃
-胄
-胆
-背
-胍
-胎
-胖
-胚
-胛
-胜
-胝
-胞
-胡
-胤
-胥
-胧
-胫
-胭
-胯
-胰
-胱
-胳
-胴
-胶
-胸
-胺
-能
-脂
-脆
-脇
-脉
-脊
-脍
-脏
-脐
-脑
-脓
-脖
-脘
-脚
-脣
-脩
-脯
-脱
-脲
-脳
-脸
-脾
-腆
-腈
-腊
-腋
-腌
-腐
-腑
-腓
-腔
-腕
-腥
-腩
-腭
-腮
-腰
-腱
-腴
-腹
-腺
-腻
-腼
-腾
-腿
-膀
-膈
-膊
-膏
-膑
-膘
-膛
-膜
-膝
-膦
-膨
-膳
-膺
-膻
-臀
-臂
-臃
-臆
-臊
-臓
-臣
-臧
-自
-臬
-臭
-至
-致
-臻
-臼
-臾
-舀
-舂
-舅
-舆
-舌
-舍
-舎
-舐
-舒
-舔
-舖
-舗
-舛
-舜
-舞
-舟
-航
-舫
-般
-舰
-舱
-舵
-舶
-舷
-舸
-船
-舺
-舾
-艇
-艋
-艘
-艮
-良
-艰
-色
-艳
-艹
-艺
-艾
-节
-芃
-芈
-芊
-芋
-芍
-芎
-芒
-芙
-芜
-芝
-芡
-芥
-芦
-芩
-芪
-芫
-芬
-芭
-芮
-芯
-花
-芳
-芷
-芸
-芹
-芽
-芾
-苁
-苄
-苇
-苋
-苍
-苏
-苑
-苒
-苓
-苔
-苕
-苗
-苛
-苜
-苞
-苟
-苡
-苣
-若
-苦
-苫
-苯
-英
-苷
-苹
-苻
-茁
-茂
-范
-茄
-茅
-茉
-茎
-茏
-茗
-茜
-茧
-茨
-茫
-茬
-茭
-茯
-茱
-茴
-茵
-茶
-茸
-茹
-茼
-荀
-荃
-荆
-草
-荏
-荐
-荒
-荔
-荖
-荘
-荚
-荞
-荟
-荠
-荡
-荣
-荤
-荥
-荧
-荨
-荪
-荫
-药
-荳
-荷
-荸
-荻
-荼
-荽
-莅
-莆
-莉
-莎
-莒
-莓
-莘
-莞
-莠
-莪
-莫
-莱
-莲
-莴
-获
-莹
-莺
-莽
-莿
-菀
-菁
-菅
-菇
-菈
-菊
-菌
-菏
-菓
-菖
-菘
-菜
-菟
-菠
-菡
-菩
-菱
-菲
-菸
-菽
-萁
-萃
-萄
-萋
-萌
-萍
-萎
-萘
-萝
-萤
-营
-萦
-萧
-萨
-萩
-萱
-萸
-萼
-落
-葆
-著
-葚
-葛
-葡
-董
-葩
-葫
-葬
-葭
-葱
-葳
-葵
-葺
-蒂
-蒋
-蒐
-蒙
-蒜
-蒟
-蒡
-蒨
-蒲
-蒸
-蒹
-蒻
-蒿
-蓁
-蓄
-蓆
-蓉
-蓑
-蓓
-蓖
-蓝
-蓟
-蓦
-蓬
-蓼
-蓿
-蔑
-蔓
-蔗
-蔘
-蔚
-蔡
-蔫
-蔬
-蔵
-蔷
-蔺
-蔻
-蔼
-蔽
-蕃
-蕈
-蕉
-蕊
-蕙
-蕤
-蕨
-蕲
-蕴
-蕻
-蕾
-薄
-薅
-薇
-薏
-薑
-薙
-薛
-薨
-薪
-薬
-薯
-薰
-薹
-藏
-藐
-藓
-藕
-藜
-藤
-藩
-藻
-藿
-蘑
-蘸
-蘼
-虎
-虏
-虐
-虑
-虔
-虚
-虞
-虢
-虫
-虬
-虱
-虹
-虻
-虽
-虾
-蚀
-蚁
-蚂
-蚊
-蚌
-蚓
-蚕
-蚜
-蚝
-蚣
-蚤
-蚩
-蚪
-蚯
-蚱
-蚵
-蛀
-蛆
-蛇
-蛊
-蛋
-蛎
-蛐
-蛔
-蛙
-蛛
-蛟
-蛤
-蛭
-蛮
-蛰
-蛳
-蛹
-蛾
-蜀
-蜂
-蜃
-蜇
-蜈
-蜊
-蜍
-蜒
-蜓
-蜕
-蜗
-蜘
-蜚
-蜜
-蜡
-蜢
-蜥
-蜱
-��
-蜷
-蜻
-蜿
-蝇
-蝈
-蝉
-蝌
-蝎
-蝗
-蝙
-蝠
-蝨
-蝴
-蝶
-蝼
-螂
-螃
-融
-螨
-螯
-螳
-螺
-蟀
-蟆
-蟋
-蟑
-蟒
-蟠
-蟹
-蟾
-蠍
-蠔
-蠕
-蠛
-蠡
-蠢
-蠹
-血
-衄
-衅
-行
-衍
-衔
-街
-衙
-衞
-衡
-衢
-衣
-补
-表
-衩
-衫
-衬
-衮
-衰
-衲
-衷
-衾
-衿
-袁
-袂
-袄
-袅
-袈
-袋
-袍
-袒
-袖
-袜
-袤
-袪
-被
-袭
-袱
-裁
-裂
-装
-裆
-裔
-裕
-裘
-裙
-裟
-裤
-裨
-裱
-裳
-裴
-裸
-裹
-裾
-褂
-褐
-褒
-褓
-褔
-褚
-褥
-褪
-褫
-褶
-襁
-襄
-襟
-西
-要
-覃
-覆
-覇
-覚
-覧
-観
-见
-观
-规
-觅
-视
-览
-觉
-觊
-觎
-觐
-觑
-角
-觞
-解
-觥
-触
-言
-訳
-証
-詹
-誉
-誓
-読
-諡
-譁
-警
-譬
-譲
-讚
-计
-订
-认
-讥
-讧
-讨
-让
-讪
-讫
-训
-议
-讯
-记
-讲
-讳
-讴
-讶
-讷
-许
-讹
-论
-讼
-讽
-设
-访
-诀
-证
-诃
-评
-诅
-识
-诈
-诉
-诊
-诋
-词
-诏
-译
-试
-诗
-诘
-诙
-诚
-诛
-话
-诞
-诟
-诠
-诡
-询
-诣
-诤
-该
-详
-诧
-诩
-诫
-诬
-语
-误
-诰
-诱
-诲
-说
-诵
-诶
-请
-诸
-诺
-读
-诽
-课
-诿
-谀
-谁
-调
-谄
-谅
-谆
-谈
-谊
-谋
-谌
-谍
-谎
-谏
-谐
-谑
-谒
-谓
-谔
-谕
-谗
-谘
-谙
-谚
-谛
-谜
-谟
-谢
-谣
-谤
-谥
-谦
-谧
-谨
-谩
-谪
-谬
-谭
-谯
-谱
-谲
-谴
-谶
-谷
-豁
-豆
-豇
-豉
-豊
-豌
-豔
-豚
-象
-豢
-豪
-豫
-豹
-豺
-貂
-貅
-貌
-貔
-貘
-贝
-贞
-负
-贡
-财
-责
-贤
-败
-账
-货
-质
-贩
-贪
-贫
-贬
-购
-贮
-贯
-贰
-贱
-贲
-贴
-贵
-贷
-贸
-费
-贺
-贻
-贼
-贾
-贿
-赁
-赂
-赃
-资
-赅
-赈
-赊
-赋
-赌
-赎
-赏
-赐
-赓
-赔
-赖
-赘
-赚
-赛
-赝
-赞
-赠
-赡
-赢
-赣
-赤
-赦
-赧
-赫
-赭
-走
-赳
-赴
-赵
-赶
-起
-趁
-超
-越
-趋
-趟
-趣
-足
-趴
-趵
-趸
-趺
-趾
-跃
-跄
-跆
-跋
-跌
-跎
-跑
-跖
-跚
-跛
-距
-跟
-跤
-跨
-跩
-跪
-路
-跳
-践
-跷
-跹
-跺
-跻
-踉
-踊
-踌
-踏
-踝
-踞
-踟
-踢
-踩
-踪
-踮
-踱
-踵
-踹
-蹂
-蹄
-蹇
-蹈
-蹉
-蹊
-蹋
-蹑
-蹒
-蹙
-蹟
-蹦
-蹩
-蹬
-蹭
-蹲
-蹴
-蹶
-蹼
-蹿
-躁
-躇
-躏
-身
-躬
-躯
-躲
-躺
-転
-軽
-车
-轧
-轨
-轩
-转
-轭
-轮
-软
-轰
-轲
-轴
-轶
-轻
-轼
-载
-轿
-较
-辄
-辅
-辆
-辇
-辈
-辉
-辊
-辍
-辐
-辑
-输
-辕
-辖
-辗
-辘
-辙
-辛
-辜
-辞
-辟
-辣
-辨
-辩
-辫
-辰
-辱
-边
-辺
-辻
-込
-辽
-达
-迁
-迂
-迄
-迅
-过
-迈
-迎
-运
-近
-返
-还
-这
-进
-远
-违
-连
-迟
-迢
-迤
-迥
-迦
-迩
-迪
-迫
-迭
-述
-迷
-迸
-迹
-迺
-追
-退
-送
-适
-逃
-逅
-逆
-选
-逊
-逍
-透
-逐
-递
-途
-逗
-通
-逛
-逝
-逞
-速
-造
-逢
-逮
-逵
-逶
-逸
-逻
-逼
-逾
-遁
-遂
-遅
-遇
-遍
-遏
-遐
-遑
-遒
-道
-遗
-遛
-遢
-遣
-遥
-遨
-遭
-遮
-遴
-遵
-遶
-遽
-避
-邀
-邂
-邃
-邈
-邋
-邑
-邓
-邕
-邛
-邝
-邢
-那
-邦
-邨
-邪
-邬
-邮
-邯
-邰
-邱
-邳
-邵
-邸
-邹
-邺
-邻
-郁
-郅
-郊
-郎
-郑
-郜
-郝
-郡
-郢
-郤
-郦
-郧
-部
-郫
-郭
-郴
-郷
-郸
-都
-鄂
-鄙
-鄞
-鄢
-鄱
-酉
-酊
-酋
-酌
-配
-酐
-酒
-酗
-酚
-酝
-酢
-酣
-酥
-酩
-酪
-酬
-酮
-酯
-酰
-酱
-酵
-酶
-酷
-酸
-酿
-醃
-醇
-醉
-醋
-醍
-醐
-醒
-醚
-醛
-醣
-醪
-醮
-醯
-醴
-醺
-采
-釉
-释
-里
-重
-野
-量
-金
-釜
-釦
-鈪
-鉄
-鉴
-銭
-銮
-鍊
-鎌
-鎏
-鎗
-鏖
-鑑
-鑫
-针
-钉
-钊
-钎
-钏
-钒
-钓
-钗
-钙
-钛
-钜
-钝
-钞
-钟
-钠
-钡
-钢
-钣
-钤
-钥
-钦
-钧
-钨
-钩
-钮
-钯
-钰
-钱
-钳
-钴
-钵
-钺
-钻
-钼
-钾
-钿
-铀
-铁
-铂
-铃
-铄
-铅
-铆
-铉
-铎
-铐
-铛
-铜
-铝
-铠
-铡
-铢
-铣
-铤
-铨
-铩
-铬
-铭
-铮
-铰
-铲
-铵
-银
-铸
-铺
-链
-铿
-销
-锁
-锂
-锄
-锅
-锆
-锈
-锉
-锋
-锌
-锏
-锐
-锑
-错
-锚
-锟
-锡
-锢
-锣
-锤
-锥
-锦
-锭
-键
-锯
-锰
-锲
-锵
-锹
-锺
-锻
-镀
-镁
-镂
-镇
-镉
-镌
-镍
-镐
-镑
-镕
-镖
-镗
-镛
-镜
-镣
-镭
-镯
-镰
-镳
-镶
-长
-閒
-関
-闇
-闘
-闢
-门
-闪
-闫
-闭
-问
-闯
-闰
-闲
-间
-闵
-闷
-闸
-闹
-闺
-闻
-闽
-闾
-阀
-阁
-阂
-阅
-阆
-阇
-阈
-阉
-阎
-阐
-阑
-阔
-阕
-阖
-阙
-阚
-阜
-队
-阡
-阮
-阱
-防
-阳
-阴
-阵
-阶
-阻
-阿
-陀
-陂
-附
-际
-陆
-陇
-陈
-陋
-陌
-降
-限
-陕
-陛
-陞
-陟
-陡
-院
-除
-陨
-险
-陪
-陲
-陵
-陶
-陷
-険
-隅
-隆
-隈
-隋
-隍
-随
-隐
-隔
-隘
-隙
-障
-隠
-隣
-隧
-隶
-隼
-隽
-难
-雀
-雁
-雄
-雅
-集
-雇
-雉
-雌
-雍
-雎
-雏
-雑
-雒
-雕
-雨
-雪
-雯
-雰
-雳
-零
-雷
-雹
-雾
-需
-霁
-霄
-霆
-震
-霈
-霉
-霊
-霍
-霎
-霏
-霑
-霓
-霖
-霜
-霞
-霭
-霰
-露
-霸
-霹
-霾
-青
-靓
-靖
-静
-靛
-非
-靠
-靡
-面
-靥
-革
-靳
-靴
-靶
-靼
-鞅
-鞋
-鞍
-鞑
-鞘
-鞠
-鞣
-鞭
-韦
-韧
-韩
-韬
-韭
-音
-韵
-韶
-頫
-頼
-页
-顶
-顷
-项
-顺
-须
-顼
-顽
-顾
-顿
-颁
-颂
-预
-颅
-领
-颇
-颈
-颉
-颊
-颌
-颍
-颐
-频
-颓
-颔
-颖
-颗
-题
-颚
-颛
-颜
-额
-颞
-颠
-颡
-颢
-颤
-颦
-颧
-风
-飒
-飓
-飕
-飘
-飙
-飚
-飞
-食
-飨
-餐
-餮
-餵
-饍
-饕
-饥
-饨
-饪
-饬
-饭
-饮
-饯
-饰
-饱
-饲
-饴
-饵
-饶
-��
-饺
-饼
-饽
-饿
-馀
-馁
-馄
-馅
-馆
-馈
-馋
-馍
-馏
-馒
-馔
-首
-馗
-香
-馥
-馨
-駄
-駅
-駆
-験
-騨
-驒
-马
-驭
-驮
-驯
-驰
-驱
-驳
-驴
-驶
-驷
-驸
-驹
-驻
-驼
-驾
-驿
-骁
-骂
-骄
-骅
-骆
-骇
-骈
-骊
-骋
-验
-骏
-骐
-骑
-骗
-骚
-骛
-骜
-骞
-骠
-骡
-骤
-骥
-骧
-骨
-骰
-骶
-骷
-骸
-骼
-髂
-髅
-髋
-髓
-高
-髦
-髪
-髯
-髻
-鬃
-鬓
-鬟
-鬣
-鬼
-魁
-魂
-魄
-魅
-魇
-魍
-魏
-魔
-鮨
-鱼
-鱿
-鲁
-鲈
-鲍
-鲑
-鲛
-鲜
-鲟
-鲢
-鲤
-鲨
-鲫
-鲱
-鲲
-鲶
-鲷
-鲸
-鳃
-鳄
-鳅
-鳌
-鳍
-鳕
-鳖
-鳗
-鳝
-鳞
-鵰
-鸟
-鸠
-鸡
-鸢
-鸣
-鸥
-鸦
-鸨
-鸪
-鸭
-鸯
-鸳
-鸵
-鸽
-鸾
-鸿
-鹂
-鹃
-鹄
-鹅
-鹈
-鹉
-鹊
-鹌
-鹏
-鹑
-鹕
-鹘
-鹜
-鹞
-鹤
-鹦
-鹧
-鹫
-鹭
-鹰
-鹳
-鹿
-麂
-麋
-麒
-麓
-麝
-麟
-麦
-麴
-麸
-麺
-麻
-麾
-黄
-黍
-黎
-黏
-黑
-黒
-黔
-默
-黛
-黜
-黝
-黠
-黯
-鼋
-鼎
-鼐
-鼓
-鼠
-鼬
-鼹
-鼻
-鼾
-齁
-齐
-齢
-齿
-龄
-龅
-龈
-龊
-龋
-龌
-龙
-龚
-龛
-龟
-呡
-乾
-绗
-楦
-硌
-袢
-钕
-蕞
-癀
-皲
-貉
-唛
-笕
-椴
-―
-胗
-旯
-鳙
-鲇
-鳐
-鳜
-鲅
-鳊
-鲳
-鲽
-鲣
-枞
-炝
-醅
-馊
-捯
-抻
-绉
-馐
-饧
-莜
-嘬
-腘
-肫
-鳟
-镊
-犽
-洌
-蝰
-铱
-髌
-锃
-镲
-锗
-甑
-戗
-裥
-弎
-粝
-霂
-猄
-轱
-苎
-偲
-兿
-铷
-栢
-帏
-黢
-洇
-沄
-誊
-忸
-怩
-蚬
-籺
-氚
-犇
-锒
-鸩
-噘
-偾
-髫
-︰
-︱
-︶
-︿
-﹁
-﹂
-﹍
-﹏
-﹐
-﹑
-﹒
-﹔
-﹕
-﹖
-﹗
-﹙
-﹚
-﹝
-﹞
-﹡
-﹣
-！
-（
-）
-，
-：
-；
-？
-｜
-～
-｡
-｢
-｣
-､
-･
-ｯ
-ｰ
-ﾞ
-ﾟ
-￣
-￥
-...
-yam
-lofter
-##s
-by
-##0
-com
-##a
-##2
-##1
-##3
-##e
-##8
-##5
-##6
-##4
-##9
-##7
-##t
-##o
-##d
-##i
-##n
-app
-www
-the
-##m
-##c
-##l
-##y
-##r
-##g
-http
-qq
-##p
-##f
-google
-pixnet
-cookies
-tripadvisor
-##er
-##k
-##h
-facebook
-##b
-of
-##x
-##u
-iphone
-ip
-in
-##w
-##ing
-ctrip
-##on
-##v
-to
-id
-it
-windows
-llc
-top
-led
-at
-##an
-##z
-android
-and
-vr
-blogthis
-twitter
-##le
-ok
-cn
-no
-ios
-##in
-##mm
-on
-te
-ig
-lv
-##ng
-##us
-pc
-──
-##te
-##ed
-html
-ncc
-wifi
-email
-blog
-is
-mail
-online
-##al
-dvd
-##ic
-studio
-##℃
-##ia
-line
-vip
-##q
-##ce
-##en
-for
-##is
-##ra
-##es
-##j
-usb
-net
-cp
-asia
-##cm
-diy
-new
-ta
-language
-vs
-apple
-tw
-web
-##ne
-ipad
-you
-##re
-##tion
-ps
-de
-bt
-pony
-atm
-##ch
-ceo
-##or
-go
-##na
-av
-pro
-cafe
-pinterest
-pixstyleme3c
-##ta
-more
-said
-mp3
-##ll
-nba
-jun
-tv
-pm
-nbsp
-##ie
-linux
-##ma
-cd
-hd
-##ion
-am
-##th
-##st
-##se
-##et
-gdp
-my
-abc
-flash
-one
-##ck
-gps
-##ly
-web885
-##ge
-xd
-boss
-isbn
-org
-##ry
-me
-love
-##ter
-##ar
-##la
-hotel
-pk
-ie
-##os
-##el
-seo
-cpu
-##ml
-p2p
-may
-sun
-tue
-internet
-cc
-posted
-youtube
-##at
-##man
-ii
-abs
-nt
-pdf
-yahoo
-ago
-##it
-news
-mac
-##me
-java
-spa
-##de
-##nt
-hk
-all
-plus
-la
-##mb
-##ve
-west
-##da
-air
-##ps
-##to
-logo
-htc
-php
-https
-fi
-momo
-##son
-sat
-##ke
-ebd
-suv
-wi
-day
-apk
-##um
-mv
-galaxy
-wiki
-or
-brake
-this
-mon
-po
-javascript
-life
-home
-june
-##ss
-system
-pp
-world
-fb
-br
-##as
-ic
-ai
-leonardo
-safari
-live
-free
-xx
-wed
-win7
-kiehl
-##co
-lg
-o2o
-##go
-us
-mm
-vfm
-kanye
-##id
-jr
-##ey
-rss
-##sa
-##ro
-##am
-##no
-thu
-fri
-##sh
-##ki
-comments
-name
-##pe
-##ine
-max
-uber
-##mi
-##ton
-wordpress
-office
-##ment
-bd
-win10
-##ld
-##li
-gmail
-bb
-dior
-##rs
-##ri
-##rd
-up
-cad
-dr
-read
-##io
-url
-pvc
-paypal
-show
-policy
-##ty
-with
-txt
-##ba
-dna
-from
-post
-mini
-ar
-taiwan
-john
-##ga
-privacy
-agoda
-##ny
-word
-##by
-##ur
-##hz
-##ang
-cookie
-netscape
-##ka
-##～
-##ad
-house
-share
-note
-ibm
-code
-hello
-nike
-sim
-survey
-wikia
-cbc
-##tor
-##kg
-##rt
-campaign
-store
-os
-##ct
-##ts
-##°
-api
-##ns
-excel
-##ao
-##nd
-university
-##ya
-##il
-pierre
-ipo
-hotels
-##ian
-years
-##ers
-high
-##day
-time
-##ay
-bug
-##line
-##be
-xp
-talk2yam
-yamservice
-coco
-##dy
-sony
-##ies
-microsoft
-david
-people
-##ha
-instagram
-intel
-##ot
-iso
-##va
-##mo
-##land
-xxx
-man
-co
-ltxsw
-##ation
-baby
-##pa
-##ol
-tag
-##ue
-msn
-oppo
-##ca
-control
-##om
-st
-chrome
-##ure
-be
-lol
-##bo
-lady
-##way
-##ko
-##do
-##un
-corporation
-##ni
-herme
-##up
-ui
-##ds
-ppt
-admin
-three
-bbc
-re
-ca
-hp
-##ee
-tpp
-##ive
-root
-##cc
-##ble
-##ity
-adobe
-park
-et
-oled
-city
-##ex
-##ler
-##ap
-china
-##book
-view
-##ice
-global
-##km
-your
-hong
-##mg
-out
-##ms
-ng
-ebay
-menu
-ubuntu
-##cy
-rom
-##view
-open
-ktv
-do
-server
-##lo
-if
-english
-##oo
-step1
-kong
-club
-july
-inc
-mr
-hi
-##net
-touch
-##ls
-##ii
-michael
-lcd
-phone
-james
-step2
-ios9
-##box
-dc
-##ley
-samsung
-pokemon
-css
-##ent
-##les
-s8
-atom
-play
-bmw
-##said
-sa
-etf
-ctrl
-adidas
-amazon
-##ber
-##ner
-visa
-##der
-connectivity
-##hi
-firefox
-hr
-so
-style
-mark
-pop
-ol
-skip
-as
-##ir
-mba
-##ai
-le
-##ver
-cafe2017
-lte
-super
-##ron
-amd
-like
-are
-##ster
-we
-##sk
-paul
-data
-international
-##ft
-longchamp
-ssd
-good
-##ti
-reply
-##my
-apr
-star
-##ker
-source
-js
-get
-force
-photo
-##one
-##ow
-link
-bbs
-goods
-##lin
-python
-##ip
-game
-##ics
-blue
-page
-itunes
-gt
-gif
-##ff
-group
-about
-bar
-ganji
-##nce
-music
-lee
-not
-##per
-an
-faq
-comment
-days
-##ock
-##bs
-v1
-player
-xbox
-sql
-fm
-f1
-##ah
-##lv
-##mp
-melody
-xml
-market
-##au
-what
-gl
-##age
-tips
-book
-##ting
-mysql
-can
-##ung
-wonderland
-watch
-##ction
-mar
-mobile
-article
-##db
-part
-party
-##ore
-##op
-dj
-main
-##ong
-art
-ad
-pm2
-japan
-ts
-##ica
-der
-sm
-##wa
-ct
-homemesh
-search
-##tv
-##di
-macbook
-service
-type
-##ier
-##si
-##ok
-best
-goris
-lock
-cf
-big
-##ut
-ftp
-carol
-##vi
-happy
-sd
-##ac
-anti
-pe
-cnn
-iii
-esp
-jan
-tags
-august
-vol
-##fs
-##sion
-design
-ac
-press
-jordan
-ppp
-that
-key
-check
-##tt
-##㎡
-##lt
-power
-##bc
-vivi
-he
-jpg
-##rry
-nb
-##ted
-##rn
-usd
-##t00
-master
-model
-al
-ram
-goo
-##ui
-red
-##ary
-rpg
-item
-##pm
-##za
-project
-hot
-td
-blogabstract
-##ger
-gr2
-black
-electronic
-nfc
-year
-asus
-html5
-cindy
-##hd
-m3
-esc
-##od
-booking
-fed
-tvb
-##ina
-mit
-chan
-distribution
-next
-peter
-bios
-steam
-cm
-pk10
-##ix
-dec
-nasa
-##ana
-icecat
-b1
-will
-li
-se
-##ji
-##ard
-oct
-##ain
-jp
-##ze
-##bi
-cio
-smart
-h5
-##port
-curve
-vpn
-##nm
-##dia
-utc
-rmvb
-chanel
-a4
-miss
-##and
-##im
-media
-who
-she
-girl
-vera
-class
-vivo
-king
-##ei
-national
-ab
-ipod
-ap
-ms
-mp4
-msci
-##po
-mg
-index
-##bit
-##out
-##zz
-apec
-photoshop
-opec
-##tes
-##ast
-○○
-##ling
-##ory
-##ical
-kitty
-content
-step3
-##cn
-win8
-vc
-iphone7
-robert
-tcl
-beauty
-en
-dollars
-##ys
-##oc
-step
-pay
-yy
-a1
-##lly
-##ks
-download
-sep
-exe
-ph
-school
-gb
-center
-pr
-street
-##board
-uv
-##lan
-winrar
-##que
-##ua
-##com
-gpu
-ettoday
-fu
-tom
-##ren
-##via
-b2b
-##tch
-rose
-arm
-mb
-##ial
-##nn
-nvidia
-step4
-mvp
-york
-how
-cpi
-gov
-kg
-joe
-##xx
-mandy
-pa
-##ser
-copyright
-fashion
-don
-ecu
-##ist
-##art
-erp
-wap
-have
-##lm
-talk
-##ek
-##ning
-##if
-ch
-##ite
-video
-cs
-san
-iot
-look
-##ku
-october
-##ux
-trump
-##hs
-##ide
-box
-first
-##ins
-april
-##ight
-angel
-protected
-aa
-x1
-m2
-##fe
-##×
-##ho
-size
-min
-ofo
-fun
-gomaji
-ex
-hdmi
-food
-dns
-march
-chris
-kevin
-##lla
-##pp
-##ec
-ag
-ems
-##rm
-##ham
-off
-asp
-team
-fandom
-ed
-##ell
-info
-sina
-##able
-##ctor
-dll
-rights
-ltd
-idc
-jul
-ma
-surface
-mall
-eps
-green
-map
-space
-donald
-v2
-sodu
-##light
-reserved
-htm
-##han
-mod
-##ise
-##tions
-ti
-##shi
-doc
-icp
-wang
-##ram
-shopping
-aug
-##pi
-##well
-now
-wam
-b2
-##hu
-##gb
-f2
-mix
-##ef
-##uan
-bwl
-##plus
-##res
-core
-##ess
-tea
-hktvmall
-nhk
-##ate
-list
-##ese
-feb
-inn
-nov
-daniel
-##ci
-pass
-##bet
-##nk
-coffee
-ssl
-airbnb
-##ute
-fbi
-woshipm
-skype
-ea
-cg
-sp
-##fc
-##www
-yes
-edge
-alt
-fpga
-##ght
-##gs
-iso9001
-##ile
-##wood
-##uo
-image
-lin
-icon
-american
-##em
-set
-says
-##king
-##tive
-blogger
-##ox
-##zy
-##red
-##ium
-##lf
-nokia
-claire
-##ding
-november
-lohas
-##tic
-##cs
-##che
-##ire
-##gy
-##ult
-db
-january
-win
-road
-ptt
-##fa
-##mer
-anna
-pchome
-udn
-ef
-##time
-##tte
-g20
-white
-garden
-eleven
-di
-chen
-young
-cosplay
-bat
-##tra
-kindle
-npc
-steve
-etc
-##ern
-call
-xperia
-ces
-travel
-sk
-s7
-##ous
-##int
-edu
-file
-cho
-qr
-##car
-##our
-##ant
-eric
-rends
-##jo
-mastercard
-kb
-##min
-##ino
-vista
-##ris
-##ud
-jack
-##set
-pos
-##her
-##ou
-taipei
-beta
-##fi
-express
-body
-##ill
-aphojoy
-user
-december
-meiki
-##ick
-tweet
-richard
-##av
-iphone6
-##dd
-views
-##mark
-pd
-times
-level
-##ash
-point
-##ome
-koreanmall
-##ak
-george
-q2
-wma
-tcp
-full
-mlb
-##lle
-##watch
-tm
-run
-smith
-business
-##und
-color
-##tal
-##less
-moon
-##rl
-update
-pcb
-shop
-little
-end
-##mhz
-van
-dsp
-easy
-##house
-##key
-history
-oh
-##hy
-##web
-oem
-let
-was
-##gg
-review
-##wan
-##°c
-uc
-title
-##val
-united
-##ons
-doi
-trivago
-overdope
-sbs
-##ance
-grand
-special
-imf
-wx17house
-##so
-audi
-##he
-london
-william
-##rp
-##ake
-science
-beach
-cfa
-amp
-ps4
-##link
-##hp
-crm
-ferragamo
-bell
-make
-##eng
-under
-zh
-photos
-##style
-via
-da
-##gi
-company
-i7
-##ray
-thomas
-ufo
-i5
-##max
-plc
-ben
-back
-research
-mike
-##pc
-september
-##ace
-vps
-february
-pantos
-wp
-lisa
-jquery
-night
-long
-offer
-##berg
-##news
-ray
-fks
-wto
-over
-##all
-##rus
-##works
-blogtitle
-loftpermalink
-martin
-test
-ling
-km
-fda
-v3
-##ja
-outlet
-family
-##ea
-##top
-story
-##ness
-salvatore
-##lu
-swift
-room
-oracle
-##ul
-sam
-b2c
-week
-pi
-rock
-##ean
-##gle
-cctv
-after
-chinese
-##back
-powered
-x2
-##tan
-##nes
-canon
-only
-##zi
-##las
-say
-##oe
-##sd
-##bot
-##world
-##zo
-sky
-made
-top100
-just
-pmi
-gap
-##vr
-les
-ball
-vogue
-vi
-ing
-ofweek
-cos
-##list
-##ort
-##lon
-last
-##tc
-##of
-##bus
-##gen
-real
-eva
-a3
-nas
-##lie
-##ria
-##coin
-##bt
-his
-cat
-nata
-vive
-health
-drive
-sir
-du
-cup
-##ook
-##sy
-alex
-msg
-tour
-##word
-ebooks
-r8
-block
-nice
-pvp
-months
-rewards
-##ther
-##xi
-##sc
-micro
-gg
-blogfp
-op
-daily
-m1
-true
-##bb
-ml
-##tar
-##ky
-anthony
-##yo
-state
-##ara
-##aa
-##rc
-##tz
-##ston
-gear
-##eo
-##ade
-ge
-see
-##win
-##ura
-ss
-heart
-##den
-##ita
-down
-##sm
-el
-png
-rakuten
-whatsapp
-bay
-dream
-add
-##use
-pad
-gucci
-mpv
-##ode
-##fo
-island
-jason
-chicago
-##hone
-io
-sogo
-be2
-##ology
-cloud
-vcd
-##con
-##ford
-##joy
-##kb
-##rade
-but
-##ach
-docker
-##ful
-rfid
-ul
-##ase
-hit
-ford
-##star
-a2
-sdk
-reading
-edited
-##are
-cmos
-##mc
-siri
-light
-##ella
-bloomberg
-##read
-pizza
-##ison
-jimmy
-##vm
-college
-node
-journal
-ba
-##play
-##cer
-magic
-##yu
-jump
-tt
-##ings
-asr
-##lia
-step5
-network
-##cd
-mc
-pixstyleme
-money
-bl
-act
-##tus
-tokyo
-##rial
-##life
-emba
-##ae
-saas
-tcs
-##rk
-##wang
-summer
-##sp
-ko
-##ving
-premium
-netflix
-uk
-mt
-##lton
-right
-frank
-two
-##ple
-##cal
-##sen
-##ville
-hold
-nexus
-dd
-##ius
-##mah
-tila
-zero
-ce
-##tin
-resort
-##ws
-charles
-old
-p10
-report
-##ru
-bus
-vans
-lt
-##est
-pv
-links
-rebecca
-##dm
-azure
-limited
-bit
-##mon
-moto
-##eam
-var
-eos
-blogspot
-e3
-dos
-dm
-fc
-##ments
-##ik
-##kw
-boy
-##bin
-##ata
-er
-##vin
-##tu
-##ula
-station
-##ature
-files
-zara
-hdr
-top10
-nature
-magazine
-s6
-marriott
-avira
-case
-tab
-##ran
-tony
-##home
-oculus
-im
-##ral
-jean
-saint
-cry
-rosie
-##force
-##ini
-ice
-##bert
-##nder
-##mber
-pet
-plurk
-##sis
-##ence
-tim
-##nc
-##name
-log
-ips
-great
-ikea
-malaysia
-unix
-##ncy
-##nie
-akb48
-##ye
-##oid
-##chi
-oa
-xuehai
-##orm
-##rf
-##ware
-ho
-##pro
-text
-##era
-bob
-##ub
-scp
-avi
-##zen
-mi
-wu
-museum
-qvod
-apache
-lake
-jcb
-ni
-##hr
-hill
-ne
-weibo
-ruby
-##row
-iv
-##ish
-github
-mate
-##lot
-##ane
-andrew
-##tina
-t1
-rf
-ed2k
-##vel
-way
-final
-ns
-sweet
-bytes
-##ene
-##cker
-##px
-topapp
-helpapp
-rs
-low
-g4g
-care
-ldquo
-##fork
-leave
-rm
-edition
-##gan
-##zon
-##qq
-##google
-##ism
-gold
-explorer
-##zer
-toyota
-category
-select
-visual
-##labels
-restaurant
-##md
-posts
-s1
-##ico
-angelababy
-sports
-s3
-mbc
-shell
-x86
-candy
-##new
-kbs
-face
-xl
-##here
-swissinfo
-v8
-dram
-##ual
-##vice
-##wer
-sport
-q1
-ios10
-public
-int
-card
-ep
-au
-rt
-bill
-##mll
-kim
-wan
-##uk
-x3
-scott
-##ming
-e5
-h7n9
-worldcat
-brown
-##vo
-##led
-##ax
-##ert
-paris
-polo
-##lr
-capital
-##hing
-bank
-cv
-##chat
-adc
-##ule
-digital
-hotmail
-##pad
-bbq
-quot
-##ring
-before
-wali
-mcu
-costco
-north
-switch
-##city
-philips
-##mann
-management
-panasonic
-##cl
-##vd
-##ping
-##rge
-alice
-##lk
-css3
-##ney
-vision
-alpha
-##ular
-##tter
-lz
-mode
-gre
-pci
-##tm
-##yan
-##let
-work
-war
-coach
-ah
-mary
-huang
-##pt
-a8
-pt
-follow
-##berry
-##ew
-a5
-ghost
-##wn
-##og
-south
-##code
-girls
-##rid
-action
-villa
-git
-r11
-table
-games
-##cket
-error
-##anonymoussaid
-##ag
-here
-##ame
-##gc
-qa
-##lis
-gmp
-##gin
-vmalife
-##cher
-yu
-wedding
-##tis
-demo
-dragon
-soho
-social
-bye
-##rant
-river
-orz
-acer
-##ats
-del
-##ven
-ups
-value
-macd
-yougou
-##dn
-##ano
-ll
-##urt
-##rent
-continue
-script
-##wen
-##ect
-paper
-shift
-##chel
-##cat
-x5
-fox
-car
-aaa
-##blog
-loading
-##yn
-##tp
-kuso
-si
-sns
-rmb
-vdc
-forest
-central
-prime
-help
-ultra
-##rmb
-square
-##field
-##reen
-##ors
-##ju
-c1
-start
-##air
-##map
-cdn
-##wo
-cba
-stephen
-m8
-##get
-opera
-##base
-##ood
-vsa
-com™
-##aw
-##ail
-count
-t2
-##een
-hop
-##gp
-vsc
-tree
-##eg
-##ose
-##ories
-##shop
-alphago
-v4
-simon
-fluke62max
-zip
-##sta
-louis
-cr
-bas
-bc
-##yer
-hadoop
-##ube
-##wi
-hola
-##low
-place
-centre
-d3
-##fer
-##media
-exchange
-series
-##san
-eb
-##bank
-q3
-##nge
-##mail
-take
-##lp
-client
-east
-cache
-event
-vincent
-##nse
-sui
-adchoice
-##stry
-##zone
-ga
-apps
-sea
-##ab
-cisco
-##rner
-kymco
-##care
-dha
-##pu
-##yi
-minkoff
-royal
-p1
-annie
-collection
-kpi
-playstation
-bh
-##bar
-queen
-radio
-andy
-armani
-##xy
-manager
-iherb
-##ery
-##share
-spring
-raid
-johnson
-##ob
-volvo
-hall
-##ball
-v6
-our
-taylor
-##hk
-bi
-##cp
-kate
-bo
-water
-technology
-##rie
-##ona
-##sl
-hpv
-gtx
-hip
-rdquo
-jayz
-stone
-##lex
-##rum
-namespace
-##ale
-##atic
-des
-##erson
-##ql
-##ves
-##type
-enter
-d2
-##mix
-##bian
-a9
-jj
-ky
-##lc
-access
-movie
-##hc
-tower
-##ration
-##mit
-##nch
-ua
-tel
-prefix
-##o2
-##point
-ott
-##http
-##ury
-baidu
-##ink
-member
-##logy
-bigbang
-nownews
-##js
-##shot
-##tb
-eba
-##tics
-##lus
-v5
-spark
-##ama
-there
-##ions
-god
-##lls
-##down
-hiv
-##ress
-burberry
-day2
-##kv
-jeff
-related
-film
-edit
-joseph
-##ark
-cx
-order
-g9
-##ans
-##tty
-s5
-##bee
-thread
-xr
-buy
-sh
-land
-spotify
-mx
-##ari
-##verse
-sf
-why
-nego
-sunny
-dom
-exo
-positioning
-fit
-rgb
-##tton
-kiss
-alexa
-adam
-lp
-mp
-##ties
-##llow
-amy
-##du
-np
-institute
-##rth
-##lar
-##des
-sidebar
-imax
-site
-##cky
-##kit
-##ime
-season
-##fun
-gogoro
-a7
-pu
-lily
-fire
-twd600
-##vis
-##cture
-information
-close
-friday
-yi
-nick
-##tta
-##tel
-##lock
-cbd
-economy
-tinker
-double
-voice
-##app
-oops
-channel
-today
-##right
-raw
-xyz
-jim
-edm
-##cent
-supreme
-ds
-##its
-##asia
-dropbox
-##tti
-books
-##tle
-##ller
-##ken
-##more
-##boy
-sex
-##dom
-t3
-##ider
-##unch
-feel
-##put
-s2
-mo
-##gh
-men
-ka
-amoled
-div
-##tr
-##n1
-port
-howard
-##tags
-ken
-dnf
-##nus
-adsense
-ide
-buff
-thunder
-##town
-##ique
-has
-##body
-auto
-pin
-##erry
-tee
-number
-##the
-object
-psp
-cool
-udnbkk
-##mic
-miui
-##tro
-most
-r2
-##alk
-##nity
-s4
-law
-version
-##oa
-n1
-sgs
-docomo
-##tf
-##ack
-henry
-fc2
-##ded
-##sco
-##rite
-linkedin
-##ada
-##now
-wii
-##ndy
-ucbug
-sputniknews
-legalminer
-##ika
-##xp
-##bu
-q10
-oo
-b6
-come
-##rman
-cheese
-ming
-maker
-##gm
-nikon
-##fig
-ppi
-kelly
-jchere
-ted
-md
-fgo
-tech
-##tto
-dan
-soc
-##gl
-##len
-hair
-earth
-img
-##pper
-##a1
-acca
-##ition
-##ference
-suite
-##ig
-outlook
-##mond
-##cation
-##pr
-airport
-##over
-jones
-##ith
-lab
-##su
-co2
-town
-piece
-##llo
-no1
-vmware
-##qi
-focus
-reader
-##admin
-##ora
-tb
-false
-##log
-know
-lan
-##ces
-f4
-##ume
-motel
-stop
-##oper
-na
-flickr
-netcomponents
-##af
-pose
-williams
-local
-##ound
-##cg
-##site
-##iko
-gsm
-con
-##ath
-friends
-##hip
-cell
-##rey
-cream
-##cks
-##dp
-facebooktwitterpinterestgoogle
-sso
-shtml
-song
-swiss
-##mw
-lumia
-xdd
-string
-tiffany
-marc
-insee
-russell
-sc
-dell
-##ations
-camera
-##vs
-##flow
-##late
-classic
-##nter
-stay
-g1
-mtv
-##ever
-##lab
-##nger
-qe
-sata
-ryan
-d1
-cms
-##cing
-su
-editor
-##nap
-security
-sunday
-association
-##ens
-##bra
-acg
-sofascore
-mkv
-##ign
-jonathan
-gary
-build
-labels
-##oto
-tesla
-moba
-qi
-gohappy
-general
-ajax
-society
-##test
-##urs
-wps
-fedora
-##ich
-mozilla
-##dr
-usa
-urn
-##lina
-grace
-##die
-##try
-##ader
-elle
-##chen
-price
-##ten
-uhz
-##ough
-eq
-##hen
-states
-push
-session
-balance
-wow
-##cus
-##py
-when
-##ward
-##ep
-wong
-library
-prada
-##cle
-running
-##ree
-ck
-date
-q4
-##ctive
-##ool
-mk
-##ira
-die
-secret
-rq
-dota
-buffet
-e6
-##ez
-pan
-ha
-##card
-##cha
-alan
-day3
-eye
-f3
-##end
-france
-keep
-adi
-rna
-tvbs
-##ala
-solo
-nova
-##tail
-support
-##ries
-##ved
-base
-copy
-iis
-fps
-##ways
-hero
-hgih
-profile
-fish
-mu
-ssh
-entertainment
-chang
-##wd
-click
-cake
-##ond
-pre
-##tom
-kic
-pixel
-##ov
-##fl
-product
-##pd
-dear
-##gate
-es
-yumi
-audio
-##²
-##sky
-echo
-bin
-where
-##ture
-##ape
-find
-sap
-isis
-nand
-##load
-##ream
-band
-a6
-never
-##post
-festival
-##we
-guide
-zenfone
-##ike
-gd
-forum
-jessica
-strong
-alexander
-##ould
-software
-allen
-##ious
-program
-else
-lohasthree
-##gar
-please
-rc
-##ggle
-##ric
-bim
-##own
-eclipse
-brian
-##side
-##other
-##tech
-##ator
-engine
-##ged
-plaza
-##fit
-cia
-ngo
-westbrook
-shi
-tbs
-sci
-reuters
-##ily
-contextlink
-##hn
-af
-##cil
-bridge
-very
-##cel
-cambridge
-##ize
-##aid
-##data
-frm
-##head
-award
-butler
-##sun
-meta
-##mar
-america
-ps3
-puma
-pmid
-lc
-kitchen
-##lic
-day1
-future
-##text
-##page
-##rris
-pm1
-##ket
-fans
-christian
-bot
-kids
-trackback
-##hai
-c3
-display
-##hl
-n2
-idea
-##sent
-airmail
-##ug
-##men
-pwm
-##lution
-awards
-schemas
-asics
-wikipedia
-font
-##tional
-##vy
-c2
-##dget
-##ein
-contact
-pepper
-##uel
-##ument
-##hang
-q5
-##sue
-rain
-##ndi
-wei
-swatch
-##cept
-popular
-##ste
-##tag
-p2
-trc
-##west
-##live
-justin
-honda
-ping
-messenger
-##rap
-v9
-unity
-appqq
-leo
-##tone
-##ass
-uniqlo
-her
-jane
-memory
-moneydj
-##tical
-human
-##m2
-coc
-miacare
-##mn
-tmt
-##core
-vim
-kk
-##may
-fan
-target
-use
-too
-fast
-services
-##ope
-omega
-energy
-pinkoi
-##rain
-jackson
-##ement
-p9
-rd
-##tier
-##vic
-zone
-dl
-isofix
-cpa
-m4
-kimi
-davis
-##lay
-lulu
-##uck
-weeks
-qs
-##hop
-ae
-##ear
-eia
-##fly
-korea
-jpeg
-boost
-##ship
-small
-eur
-valley
-##iel
-simple
-##ude
-rn
-k2
-##ena
-non
-patrick
-feed
-process
-well
-qqmei
-##thing
-they
-aws
-lu
-pink
-##ters
-##kin
-board
-##vertisement
-wine
-##ien
-unicode
-##dge
-r1
-##tant
-##twitter
-cool1
-isp
-standard
-matt
-##fu
-##iner
-googlemsn
-pixnetfacebookyahoo
-x7
-##uce
-sao
-##ev
-##file
-xddd
-shirt
-##rio
-##hat
-givenchy
-ya
-bang
-##lio
-monday
-crystal
-##abc
-head
-ubuntuforumwikilinuxpastechat
-##vc
-##rity
-cnc
-ipv6
-null
-##ost
-yang
-imsean
-tiger
-##fet
-dji
-ji
-maria
-##come
-foundation
-##beth
-active
-##aft
-##don
-sr
-emma
-##khz
-living
-sas
-x6
-##face
-pptv
-x4
-##mate
-han
-sophie
-##jing
-fifa
-##mand
-other
-sale
-inwedding
-##gn
-##mmy
-##pmlast
-bad
-nana
-nbc
-##wu
-note7
-single
-##bel
-window
-##dio
-##ht
-union
-age
-##ivity
-domain
-neo
-##isa
-##lter
-f5
-steven
-##cts
-powerpoint
-tft
-self
-g2
-ft
-zol
-##act
-mwc
-nbapop
-eds
-ace
-##room
-previous
-author
-tomtom
-il
-##ets
-hu
-financial
-bp
-chi
-##hg
-fairmont
-cross
-gay
-h2
-function
-also
-##raph
-##ils
-i3
-avenue
-##host
-##bon
-##tsu
-message
-navigation
-fintech
-h6
-##ject
-##vas
-##firm
-credit
-##wf
-xxxx
-form
-##nor
-##space
-huawei
-plan
-json
-sbl
-##dc
-machine
-wish
-##sol
-windows7
-edward
-development
-washington
-##nsis
-lo
-##sio
-##ym
-##bor
-planet
-##wt
-ieee
-gpa
-camp
-ann
-gm
-##tw
-##oka
-connect
-##rss
-##work
-##atus
-wall
-chicken
-soul
-##times
-fa
-##ather
-##cord
-##eep
-hitachi
-gui
-harry
-##pan
-e1
-disney
-##press
-wind
-frigidaire
-##tl
-liu
-hsu
-basic
-von
-ev
-learning
-##ull
-expedia
-archives
-change
-##wei
-santa
-cut
-ins
-turbo
-brand
-cf1
-return
-##rip
-h1
-##nis
-application
-emc
-rx
-##oon
-quick
-wilson
-wing
-chapter
-##bug
-beyond
-##cms
-##dar
-##oh
-zoom
-e2
-trip
-sb
-##nba
-rcep
-aspx
-ci
-gc
-gnu
-##count
-advanced
-dance
-dv
-##url
-##ging
-am09
-shadow
-battle
-##cia
-emily
-##tation
-host
-ff
-techorz
-sars
-##mini
-##mporary
-##ering
-nc
-##next
-cma
-##mbps
-##gas
-##ift
-##dot
-amana
-##ros
-ir
-##eet
-##ible
-##aka
-dcs
-iq
-l1
-##lor
-maggie
-##iu
-##gt
-articles
-create
-##burg
-##iki
-database
-fantasy
-##rex
-##cam
-dlc
-dean
-##you
-hard
-path
-gaming
-victoria
-maps
-cb
-##lee
-##itor
-overchicstoretvhome
-systems
-##xt
-p3
-sarah
-##nan
-x9
-install
-second
-##ann
-##ph
-##rcle
-##nic
-##nar
-ec
-metro
-chocolate
-##rian
-##table
-skin
-##sn
-mountain
-inparadise
-ib
-##jia
-eeworld
-creative
-g5
-g3
-parker
-ecfa
-village
-sylvia
-hbl
-##ques
-##onsored
-##x2
-##v4
-##tein
-ie6
-##stack
-ver
-##ads
-##baby
-sound
-bbe
-##lone
-##uid
-ads
-gundam
-thinkpad
-scrum
-match
-##ave
-mems
-##oy
-##talk
-glass
-lamigo
-span
-##eme
-job
-##a5
-jay
-wade
-kde
-##lace
-ocean
-tvg
-##covery
-##r3
-##ners
-##rea
-junior
-think
-##aine
-cover
-##ision
-##sia
-##bow
-msi
-##love
-soft
-z2
-##pl
-mobil
-mind
-##uy
-nginx
-##oi
-##rr
-##mple
-##sson
-##nts
-comhd
-crv3000
-##uard
-deep
-lost
-field
-gallery
-##bia
-rate
-spf
-redis
-traction
-icloud
-fe
-jose
-##tory
-into
-sohu
-fx
-kicstart2
-##hia
-##sit
-ra
-##walk
-##xure
-##pact
-pacific
-xa
-natural
-carlo
-##walker
-##can
-cto
-gigi
-pen
-##hoo
-ob
-matlab
-##yy
-##iti
-mango
-##bbs
-sense
-c5
-oxford
-walker
-jennifer
-##ola
-course
-##bre
-##pus
-##rder
-lucky
-ivy
-##nia
-sotheby
-side
-##ugh
-joy
-##orage
-##ush
-##bat
-##dt
-r9
-##gio
-country
-wear
-##lax
-##moon
-seven
-study
-lonzo
-evolution
-##kk
-gs
-kd
-arduino
-b12
-##lux
-arpg
-##rdon
-cook
-##x5
-dark
-five
-##als
-##ida
-sign
-something
-##nda
-##posted
-fresh
-tf
-cam
-##mine
-##skip
-##form
-##ssion
-education
-##tee
-dyson
-stage
-##jie
-want
-##night
-epson
-pack
-##ppy
-##█
-wd
-##eh
-##rence
-left
-##lvin
-golden
-mhz
-discovery
-##trix
-##n2
-loft
-##uch
-##dra
-##sse
-speed
-sorry
-welcome
-##urn
-wave
-gaga
-##lmer
-teddy
-rp
-##sha
-rar
-holiday
-##vg
-##nos
-##rail
-gartner
-gi
-##dium
-kit
-b3
-eco
-sean
-##stone
-autocad
-nu
-##np
-f16
-write
-m5
-##ias
-images
-atp
-##dk
-fsm
-ve
-##xxx
-##cake
-unit
-lim
-ru
-##ification
-published
-angela
-analytics
-ak
-##nel
-gmt
-##icon
-again
-##₂
-##bby
-ios11
-waze
-##ust
-framework
-iptv
-delete
-cl
-wwdc
-##fw
-##xon
-brandt
-##ses
-##dragon
-tc
-vetements
-anne
-monte
-modern
-official
-##ere
-##nne
-##oud
-etnews
-##a2
-##graphy
-##rtex
-l2
-##gma
-mount
-ccd
-archive
-morning
-tan
-ddos
-e7
-day4
-gis
-its
-factory
-bruce
-pg
-##ito
-guest
-cdma
-##lling
-n3
-mega
-eyes
-ro
-women
-dac
-church
-##jun
-singapore
-##facebook
-starbucks
-##tos
-##stin
-##shine
-zen
-##mu
-tina
-request
-##gence
-qt
-q7
-##zzi
-diary
-##tore
-##ead
-cst
-##osa
-canada
-agent
-va
-##jiang
-##lam
-sg
-##nix
-##sday
-g6
-##master
-bing
-##zl
-charlie
-nb40
-thai
-ln284ct
-##itz
-bonnie
-##food
-##lent
-originals
-##stro
-##lts
-##bscribe
-children
-ntd
-yesstyle
-hmv
-##tment
-d5
-arts
-sms
-##pn
-topios9
-lifestyle
-virtual
-##ague
-xz
-##deo
-muji
-unt
-##nnis
-faq1
-##ette
-fly
-curry
-##pop
-release
-##cast
-##ews
-##stle
-ios7
-##ima
-dog
-lenovo
-##r4
-roger
-cbs
-vornado
-##desk
-##ald
-##van
-oil
-some
-break
-common
-##jy
-##lines
-g7
-twice
-ella
-nano
-belle
-##mes
-##self
-##note
-jb
-benz
-##ova
-save
-##wing
-kai
-##hua
-##rect
-rainer
-##unge
-adsl
-guestname
-##uma
-##kins
-##zu
-tokichoi
-##price
-county
-##med
-##mus
-rmk
-address
-vm
-openload
-##group
-##hin
-##iginal
-amg
-urban
-##oz
-jobs
-emi
-##public
-beautiful
-##sch
-album
-##dden
-##bell
-jerry
-works
-hostel
-miller
-##drive
-##rmin
-boot
-##fx
-##nome
-##ctionary
-##oman
-##lish
-##cr
-##hm
-##how
-francis
-xi
-c919
-b5
-evernote
-##uc
-vga
-coupe
-##urg
-##cca
-##uality
-multi
-##ett
-em
-hey
-##ani
-##tax
-##rma
-inside
-than
-leonnhurt
-##jin
-ict
-bird
-notes
-##dical
-##lli
-result
-iu
-ee
-smap
-gopro
-##last
-yin
-pure
-##dan
-##rame
-mama
-##oot
-bean
-marketing
-##hur
-bella
-sync
-xuite
-##ground
-discuz
-##getrelax
-##ince
-##bay
-cj
-gmat
-apt
-##pass
-jing
-##rix
-c4
-rich
-niusnews
-##ello
-bag
-##eting
-##mobile
-culture
-area
-##ience
-details
-gp
-universal
-silver
-dit
-private
-ddd
-u11
-kanshu
-##ified
-fung
-##nny
-dx
-tai
-##fr
-##lean
-##pin
-##rin
-ly
-rick
-##bility
-usb3
-banner
-##baru
-##gion
-metal
-dt
-vdf
-karl
-qualcomm
-bear
-oldid
-ian
-jo
-##tors
-population
-##ernel
-mmorpg
-##mv
-##bike
-ww
-friend
-##ager
-exhibition
-##del
-##pods
-fpx
-structure
-##free
-##tings
-kl
-##rley
-##copyright
-##mma
-california
-orange
-yoga
-canmake
-honey
-##anda
-nikkie
-dhl
-publishing
-##mall
-##gnet
-##┅
-e88
-##dog
-fishbase
-##!
-##"
-###
-##$
-##%
-##&
-##'
-##(
-##)
-##*
-##+
-##,
-##-
-##.
-##/
-##:
-##;
-##<
-##=
-##>
-##?
-##@
-##[
-##\
-##]
-##^
-##_
-##{
-##|
-##}
-##~
-##£
-##¤
-##¥
-##§
-##«
-##±
-##³
-##µ
-##·
-##¹
-##º
-##»
-##¼
-##ß
-##æ
-##÷
-##ø
-##đ
-##ŋ
-##ɔ
-##ə
-##ɡ
-##ʰ
-##ˇ
-##ˈ
-##ˊ
-##ˋ
-##ˍ
-##ː
-##˙
-##˚
-##ˢ
-##α
-##β
-##γ
-##δ
-##ε
-##η
-##θ
-##ι
-##κ
-##λ
-##μ
-##ν
-##ο
-##��
-##ρ
-##ς
-##σ
-##τ
-##υ
-##φ
-##χ
-##ψ
-##б
-##в
-##г
-##д
-##е
-##ж
-##з
-##к
-##л
-##м
-##н
-##о
-##п
-##р
-##с
-##т
-##у
-##ф
-##х
-##ц
-##ч
-##ш
-##ы
-##ь
-##і
-##ก
-##ง
-##น
-##ม
-##ย
-##ร
-##อ
-##า
-##เ
-##๑
-##་
-##ღ
-##ᵃ
-##ᵉ
-##ᵍ
-##ᵏ
-##ᵐ
-##ᵒ
-##ᵘ
-##‖
-##„
-##†
-##•
-##‥
-##‧
-##
-##‰
-##′
-##″
-##‹
-##›
-##※
-##‿
-##⁄
-##ⁱ
-##⁺
-##ⁿ
-##₁
-##₃
-##₄
-##€
-##№
-##ⅰ
-##ⅱ
-##ⅲ
-##ⅳ
-##ⅴ
-##⇒
-##∀
-##−
-##∕
-##∙
-##√
-##∞
-##∟
-##∠
-##∣
-##∩
-##∮
-##∶
-##∼
-##∽
-##≈
-##≒
-##≡
-##≤
-##≥
-##≦
-##≧
-##≪
-##≫
-##⊙
-##⋅
-##⋈
-##⋯
-##⌒
-##①
-##②
-##③
-##④
-##⑤
-##⑥
-##⑦
-##⑧
-##⑨
-##⑩
-##⑴
-##⑵
-##⑶
-##⑷
-##⑸
-##⒈
-##⒉
-##⒊
-##⒋
-##ⓒ
-##ⓔ
-##ⓘ
-##━
-##┃
-##┆
-##┊
-##┌
-##└
-##├
-##┣
-##═
-##║
-##╚
-##╞
-##╠
-##╭
-##╮
-##╯
-##╰
-##╱
-##╳
-##▂
-##▃
-##▅
-##▇
-##▉
-##▋
-##▌
-##▍
-##▎
-##□
-##▬
-##△
-##►
-##▽
-##◇
-##◕
-##◠
-##◢
-##◤
-##☞
-##☼
-##♡
-##♫
-##♬
-##✕
-##✦
-##✪
-##✰
-##✿
-##❀
-##➜
-##➤
-##⦿
-##、
-##。
-##〃
-##々
-##〇
-##〈
-##〉
-##《
-##》
-##「
-##」
-##『
-##』
-##【
-##】
-##〓
-##〔
-##〕
-##〖
-##〗
-##〜
-##〝
-##〞
-##ㄧ
-##ㆍ
-##㈦
-##㊣
-##㗎
-##︰
-##︱
-##︶
-##︿
-##﹁
-##﹂
-##﹍
-##﹏
-##﹐
-##﹑
-##﹒
-##﹔
-##﹕
-##﹖
-##﹗
-##﹙
-##﹚
-##﹝
-##﹞
-##﹡
-##﹣
-##！
-##（
-##）
-##，
-##：
-##；
-##？
-##｡
-##｢
-##｣
-##､
-##･
-##ｯ
-##ｰ
-##ﾞ
-##ﾟ
-##￣
-##￥

utils/zh_util.py

@@ -1,15 +1,17 @@
 """
-TODO: 繁体、简体
+TODO: 繁体、简体、语种、
 """
 import os
 import json
 from collections import Counter
-from utils.text_util import is_chinese, has_chinese
+from utils.text_util import is_chinese, get_zh_count, get_digit_count
 from zhon.hanzi import punctuation as zh_punc
 
 CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
 
-zh_tokens = [line.strip() for line in open(os.path.join(CURRENT_DIR, "vocab.jd.txt.v2"), "r", encoding="utf-8") if is_chinese(line.strip())]
+zh_tokens = [line.strip() for line in open(os.path.join(CURRENT_DIR, "vocab.jd.txt.v2"), "r", encoding="utf-8") if
+             is_chinese(line.strip())]
+
 
 def zh_iterator():
     for idx in range(ord(u'\u4e00'), ord(u'\u9fa5')):
@@ -17,6 +19,9 @@ def zh_iterator():
 
 
 def get_coding_length(tokenizer, vocab, filter=None):
+    """
+    计算编码长度。（有些中文汉字被解码成多个token）
+    """
     all_length = []
     for word in vocab:
         if len(word) > 1:
@@ -34,41 +39,82 @@ def get_coding_length(tokenizer, vocab, filter=None):
     return dist_length, mean_length
 
 
-def has_zh_char(text):
+def has_zh_punc(text):
+    """
+    是否包含中文标点
+    """
     return any(ch in zh_punc for ch in text)
 
+
+
+def get_space_count(text):
+    space_count = 0
+    for char in text:
+        if len(char.strip()) == 0:
+            space_count += 1
+    return space_count
+
+
+def remove_special_char():
+    """
+    :return:
+    """
+    # bert词典有 ##开头的
+    # byteBPE词典有带空格的
+    # decode_str = decode_str.strip().replace("#", "")  # TODO, 按类型
+    pass
+
+
 cache = {}
 
+
 def iter_vocab(tokenizer, name="", from_cache=True):
     if from_cache and name in cache:
         return cache[name]
 
     f_out = open(name + "_vocab.zh.jsonl", "w", encoding="utf-8")
     zh_token_count = {"total": 0, "中文单字": 0, "中文多字": 0}
+
+    # zh_token_count = {"total": 0, "包含1个中文单字": 0, "中文多字": 0}
+
+    # symbol_count = 0
+
     all_single_zh_tokens = set()
     zh_symbol_count = 0
-    for idx in range(tokenizer.vocab_size):
-        decode_str = tokenizer.decode([idx])
-        if has_chinese(decode_str):
-            # bert词典有 ##开头的
-            # byteBPE词典有带空格的
-            decode_str = decode_str.strip().replace("#", "")  # TODO, 按类型
+    for token_id in range(tokenizer.vocab_size):
+        decode_str = tokenizer.decode([token_id], skip_special_tokens=False)
+        token = tokenizer.convert_ids_to_tokens([token_id], skip_special_tokens=False)[0]
+        # tokenizer.convert_tokens_to_string(tokens)
+
+        if token is None:  # 有些词典有空的id（不连续）
+            continue
+        if isinstance(token, bytes):
+            token = token.decode("utf-8", errors="ignore")
+
+        digit_count = get_digit_count(token)
+        zh_count = get_zh_count(decode_str)
+        space_count = get_space_count(decode_str)
+
+        f_out.write(json.dumps(
+            {"id": token_id,
+             "token": token,
+             "token_decode": decode_str,
+             "token_len": len(token),
+             "zh_count": zh_count,
+             "space_count": space_count,
+             "digit_count": digit_count,
+             "zh_symbol_count": zh_symbol_count,
+             },
+            ensure_ascii=False) + "\n"
+                    )
+
+        if zh_count >= 1:
             zh_token_count["total"] += 1
-            if len(decode_str) > 1:
+            if zh_count > 1:
                 zh_token_count["中文多字"] += 1
-                f_out.write(json.dumps({"id": idx, "token": decode_str, "type": "中文多字"},
-                                       ensure_ascii=False) + "\n")
             else:
-                all_single_zh_tokens.add(decode_str)
                 zh_token_count["中文单字"] += 1
-                f_out.write(json.dumps({"id": idx, "token": decode_str, "type": "中文单字"},
-                                       ensure_ascii=False) + "\n")
-
-        elif has_zh_char(decode_str):
-            zh_symbol_count += 1
-            f_out.write(json.dumps({"id": idx, "token": decode_str, "type": "中文标点"},
-                                   ensure_ascii=False) + "\n")
-
+                all_single_zh_tokens.add(decode_str.strip().replace("#", ""))
     #
 
     dist_length, mean_length = get_coding_length(tokenizer, zh_tokens, filter=lambda k: not is_chinese(k))
@@ -94,5 +140,9 @@ if __name__ == "__main__":
     # test_coding_length(zh_punc)
     # test_coding_length(zh_iterator())
 
-    from vocab.gpt_35_turbo import tokenizer
-    iter_vocab(tokenizer)
+    from vocab.chatglm2_6b import tokenizer; name = "chatglm2_6b"
+    # from vocab.chatglm_6b import tokenizer; name="chatglm_6b"
+    # from vocab.baichuan2 import tokenizer;  name="baichuan2"
+    # from vocab.gpt_4 import tokenizer; name="gpt4"
+
+    print(iter_vocab(tokenizer, name=name))

vocab/README.md

@@ -1,4 +1,7 @@
 
+对于OpenAI的模型而言，英文的Token效率是中文的8-12倍，
+之前三百字中文以上时Turbo 3.5 16k就会出现逻辑颠倒问题，提示词换成英文后该问题没有出现过。
+
 ## 词典构建
 
 bert词典

vocab/__init__.py

@@ -4,8 +4,10 @@ from enum import Enum, auto
 """Interface:
 tokenizer.encode
 tokenizer.decode
+    tokenizer.convert_tokens_to_string   # gpt4 没有这个方法
 tokenizer.convert_ids_to_tokens
 
+
 tokenizer.parent = ""
 tokenizer.vocab_size   
 tokenizer.get_vocab()   # gpt-neox-20b, llama
@@ -21,8 +23,9 @@ tokenizer.implementation = TokenizerImpl.SentencePiece.name   # https://github.c
   - sentencepiece: 
     - 特征：.sp_model 是SentencePieceProcessor类型，sp_model.id_to_piece，有tokenizer.json tokenizer.model，词典字符有 ▁，
     - 示例：llama，baichuan
+  - icetk： sentencepiece的分支，支持image_tokenizer
+    - glm, chatglm1, chatglm2
   - tiktoken
-  - icetk
   - hf_tokenizer
     - 特征：
       - .model 是 tokenizer.models.BPE 类型
@@ -32,7 +35,7 @@ tokenizer.implementation = TokenizerImpl.SentencePiece.name   # https://github.c
     - 示例：gpt_neox_20b, moss, bloom
   - tiktoken
     - 特征：空格就是空格，
-    - 示例：gpt3.5 gpt4
+    - 示例：gpt3.5 gpt4, qwen, 
 tokenizer.comments = "split all numbers into individual digits, " \
                      "and fallback to bytes to decompose unknown UTF-8 characters"
 
@@ -50,14 +53,19 @@ uniq_tokenizers = [
 ]
 
 all_tokenizers = [
-    "gpt_35_turbo",
-    "gpt_4",
     "gpt2",
     "gpt2_chinese",
+    "gpt_35_turbo",
+    "gpt_4",
+
+    # bert 系列
     "bert_base_cased",
     "bert_base_uncased",
     "bert_base_chinese",
+    "roberta_chinese_clue",
     "kplug",
+
+    # gpt2 系列
     "moss",
     #
     # ######
@@ -76,9 +84,11 @@ all_tokenizers = [
     "glm_chinese",
     "chatglm_6b",
     "chatglm2_6b",
+    "chatglm3_6b",
     #
     # #### llama alpaca系列
     "llama",  # '中文单字': 700, '中文多字': 0
+    "llama2",
     "chinese_llama",  #
     "chinese_llama2",  #
     # "chinese_alpaca_lora_7b",  # 中文Alpaca模型在上述中文LLaMA模型的基础上进一步使用了指令数据进行精调。
@@ -90,6 +100,15 @@ all_tokenizers = [
     "internlm_chat_7b",
     "falcon_180b",
     # "goat",
+
+    # tiktoken 系列
+    "qwen_7b_chat",
+    "qwen_72b_chat",
+    
+    # 未分类
+
+    "mistral",
+
 ]
 
 class TokenizerType(Enum):

vocab/baichuan/Baichuan-7B/tokenization_baichuan.py

@@ -71,6 +71,11 @@ class BaiChuanTokenizer(PreTrainedTokenizer):
         eos_token = AddedToken(eos_token, lstrip=False, rstrip=False) if isinstance(eos_token, str) else eos_token
         unk_token = AddedToken(unk_token, lstrip=False, rstrip=False) if isinstance(unk_token, str) else unk_token
         pad_token = AddedToken(pad_token, lstrip=False, rstrip=False) if isinstance(pad_token, str) else pad_token
+        self.vocab_file = vocab_file
+        self.add_bos_token = add_bos_token
+        self.add_eos_token = add_eos_token
+        self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
+        self.sp_model.Load(vocab_file)
         super().__init__(
             bos_token=bos_token,
             eos_token=eos_token,
@@ -82,11 +87,6 @@ class BaiChuanTokenizer(PreTrainedTokenizer):
             clean_up_tokenization_spaces=clean_up_tokenization_spaces,
             **kwargs,
         )
-        self.vocab_file = vocab_file
-        self.add_bos_token = add_bos_token
-        self.add_eos_token = add_eos_token
-        self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
-        self.sp_model.Load(vocab_file)
 
     def __getstate__(self):
         state = self.__dict__.copy()

vocab/baichuan/__init__.py

@@ -1,7 +1,15 @@
+import os
+import config
 from transformers import AutoTokenizer
 from vocab import TokenizerType
 
-tokenizer = AutoTokenizer.from_pretrained("baichuan-inc/Baichuan-7B", trust_remote_code=True)
+
+if config.USE_REMOTE:
+    tokenizer = AutoTokenizer.from_pretrained("baichuan-inc/Baichuan-7B", trust_remote_code=True)
+else:
+    CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
+    TOKENIZER_DIR = os.path.join(CURRENT_DIR, "Baichuan-7B")
+    tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_DIR, trust_remote_code=True)
 
 
 # byte-bpe  sentencepiece

vocab/baichuan/demo.py

@@ -1,3 +1,6 @@
 
-from vocab.baichuan_7b import tokenizer
+from vocab.baichuan import tokenizer
+
+id1 = tokenizer.encode("<pad>")
+token1 = tokenizer.decode(125696)
 

vocab/baichuan/error.md

@@ -0,0 +1,8 @@
+
+
+## AttributeError: 'BaichuanTokenizer' object has no attribute 'sp_model'
+
+https://huggingface.co/baichuan-inc/Baichuan2-13B-Chat/discussions/18
+
+
+transfomers 4.34 doesn't work for me either. Degrading to 4.33.1 works in my case

vocab/baichuan2/Baichuan2-7B-Chat/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

vocab/baichuan2/Baichuan2-7B-Chat/tokenization_baichuan.py

@@ -0,0 +1,251 @@
+# Copyright 2023 Baichuan Inc. All Rights Reserved.
+
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
+#
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
+# and OPT implementations in this library. It has been modified from its
+# original forms to accommodate minor architectural differences compared
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from shutil import copyfile
+from typing import Any, Dict, List, Optional, Tuple
+
+import sentencepiece as spm
+
+from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer
+from transformers.utils import logging
+
+
+logger = logging.get_logger(__name__)
+
+VOCAB_FILES_NAMES = {"vocab_file": "tokenizer.model"}
+
+PRETRAINED_VOCAB_FILES_MAP = {
+    "vocab_file": {},
+    "tokenizer_file": {},
+}
+PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {}
+
+
+class BaichuanTokenizer(PreTrainedTokenizer):
+    """
+    Construct a Baichuan tokenizer. Based on byte-level Byte-Pair-Encoding.
+
+    Args:
+        vocab_file (`str`):
+            Path to the vocabulary file.
+    """
+
+    vocab_files_names = VOCAB_FILES_NAMES
+    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
+    model_input_names = ["input_ids", "attention_mask"]
+
+    def __init__(
+        self,
+        vocab_file,
+        unk_token="<unk>",
+        bos_token="<s>",
+        eos_token="</s>",
+        pad_token=None,
+        sp_model_kwargs: Optional[Dict[str, Any]] = None,
+        add_bos_token=True,
+        add_eos_token=False,
+        clean_up_tokenization_spaces=False,
+        **kwargs,
+    ):
+        self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs
+        bos_token = AddedToken(bos_token, lstrip=False, rstrip=False) if isinstance(bos_token, str) else bos_token
+        eos_token = AddedToken(eos_token, lstrip=False, rstrip=False) if isinstance(eos_token, str) else eos_token
+        unk_token = AddedToken(unk_token, lstrip=False, rstrip=False) if isinstance(unk_token, str) else unk_token
+        pad_token = AddedToken(pad_token, lstrip=False, rstrip=False) if isinstance(pad_token, str) else pad_token
+        self.vocab_file = vocab_file
+        self.add_bos_token = add_bos_token
+        self.add_eos_token = add_eos_token
+        self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
+        self.sp_model.Load(vocab_file)
+        super().__init__(
+            bos_token=bos_token,
+            eos_token=eos_token,
+            unk_token=unk_token,
+            pad_token=pad_token,
+            add_bos_token=add_bos_token,
+            add_eos_token=add_eos_token,
+            sp_model_kwargs=self.sp_model_kwargs,
+            clean_up_tokenization_spaces=clean_up_tokenization_spaces,
+            **kwargs,
+        )
+
+    def __getstate__(self):
+        state = self.__dict__.copy()
+        state["sp_model"] = None
+        return state
+
+    def __setstate__(self, d):
+        self.__dict__ = d
+        self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
+        self.sp_model.Load(self.vocab_file)
+
+    @property
+    def vocab_size(self):
+        """Returns vocab size"""
+        return self.sp_model.get_piece_size()
+
+    def get_vocab(self):
+        """Returns vocab as a dict"""
+        vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
+        vocab.update(self.added_tokens_encoder)
+        return vocab
+
+    def _tokenize(self, text):
+        """Returns a tokenized string."""
+        return self.sp_model.encode(text, out_type=str)
+
+    def _convert_token_to_id(self, token):
+        """Converts a token (str) in an id using the vocab."""
+        return self.sp_model.piece_to_id(token)
+
+    def _convert_id_to_token(self, index):
+        """Converts an index (integer) in a token (str) using the vocab."""
+        token = self.sp_model.IdToPiece(index)
+        return token
+
+    def convert_tokens_to_string(self, tokens):
+        """Converts a sequence of tokens (string) in a single string."""
+        current_sub_tokens = []
+        out_string = ""
+        prev_is_special = False
+        for i, token in enumerate(tokens):
+            # make sure that special tokens are not decoded using sentencepiece model
+            if token in self.all_special_tokens:
+                if not prev_is_special and i != 0:
+                    out_string += " "
+                out_string += self.sp_model.decode(current_sub_tokens) + token
+                prev_is_special = True
+                current_sub_tokens = []
+            else:
+                current_sub_tokens.append(token)
+                prev_is_special = False
+        out_string += self.sp_model.decode(current_sub_tokens)
+        return out_string
+
+    def save_vocabulary(self, save_directory, filename_prefix: Optional[str] = None) -> Tuple[str]:
+        """
+        Save the vocabulary and special tokens file to a directory.
+
+        Args:
+            save_directory (`str`):
+                The directory in which to save the vocabulary.
+
+        Returns:
+            `Tuple(str)`: Paths to the files saved.
+        """
+        if not os.path.isdir(save_directory):
+            logger.error(f"Vocabulary path ({save_directory}) should be a directory")
+            return
+        out_vocab_file = os.path.join(
+            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
+        )
+
+        if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file) and os.path.isfile(self.vocab_file):
+            copyfile(self.vocab_file, out_vocab_file)
+        elif not os.path.isfile(self.vocab_file):
+            with open(out_vocab_file, "wb") as fi:
+                content_spiece_model = self.sp_model.serialized_model_proto()
+                fi.write(content_spiece_model)
+
+        return (out_vocab_file,)
+
+    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
+        bos_token_id = [self.bos_token_id] if self.add_bos_token else []
+        eos_token_id = [self.eos_token_id] if self.add_eos_token else []
+
+        output = bos_token_id + token_ids_0 + eos_token_id
+
+        if token_ids_1 is not None:
+            output = output + bos_token_id + token_ids_1 + eos_token_id
+
+        return output
+
+    def get_special_tokens_mask(
+        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
+    ) -> List[int]:
+        """
+        Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
+        special tokens using the tokenizer `prepare_for_model` method.
+
+        Args:
+            token_ids_0 (`List[int]`):
+                List of IDs.
+            token_ids_1 (`List[int]`, *optional*):
+                Optional second list of IDs for sequence pairs.
+            already_has_special_tokens (`bool`, *optional*, defaults to `False`):
+                Whether or not the token list is already formatted with special tokens for the model.
+
+        Returns:
+            `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
+        """
+        if already_has_special_tokens:
+            return super().get_special_tokens_mask(
+                token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
+            )
+
+        bos_token_id = [1] if self.add_bos_token else []
+        eos_token_id = [1] if self.add_eos_token else []
+
+        if token_ids_1 is None:
+            return bos_token_id + ([0] * len(token_ids_0)) + eos_token_id
+        return (
+            bos_token_id
+            + ([0] * len(token_ids_0))
+            + eos_token_id
+            + bos_token_id
+            + ([0] * len(token_ids_1))
+            + eos_token_id
+        )
+
+    def create_token_type_ids_from_sequences(
+        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
+    ) -> List[int]:
+        """
+        Creates a mask from the two sequences passed to be used in a sequence-pair classification task. An ALBERT
+        sequence pair mask has the following format:
+
+        ```
+        0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
+        | first sequence    | second sequence |
+        ```
+
+        if token_ids_1 is None, only returns the first portion of the mask (0s).
+
+        Args:
+            token_ids_0 (`List[int]`):
+                List of ids.
+            token_ids_1 (`List[int]`, *optional*):
+                Optional second list of IDs for sequence pairs.
+
+        Returns:
+            `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
+        """
+        bos_token_id = [self.bos_token_id] if self.add_bos_token else []
+        eos_token_id = [self.eos_token_id] if self.add_eos_token else []
+
+        output = [0] * len(bos_token_id + token_ids_0 + eos_token_id)
+
+        if token_ids_1 is not None:
+            output += [1] * len(bos_token_id + token_ids_1 + eos_token_id)
+
+        return output

vocab/baichuan2/Baichuan2-7B-Chat/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79452955be6b419a65984273a9f08af86042e1c2a75ee3ba989cbf620a133cc2
+size 2001107

vocab/baichuan2/Baichuan2-7B-Chat/tokenizer_config.json

@@ -0,0 +1,36 @@
+{
+  "auto_map": {
+    "AutoTokenizer": ["tokenization_baichuan.BaichuanTokenizer", null]
+  },
+  "add_bos_token": false,
+  "add_eos_token": false,
+  "use_fast": false,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": {
+    "__type": "AddedToken",
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": true
+  },
+  "model_max_length": 4096,
+  "sp_model_kwargs": {},
+  "tokenizer_class": "BaichuanTokenizer",
+  "pad_token": {
+    "__type": "AddedToken",
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": true
+  },
+  "unk_token": {
+    "__type": "AddedToken",
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": true
+  }
+}

vocab/baichuan2/__init__.py

@@ -1,10 +1,21 @@
+import os
+import config
 from transformers import AutoTokenizer
 from vocab import TokenizerType
 
-tokenizer = AutoTokenizer.from_pretrained("baichuan-inc/Baichuan2-7B-Chat", trust_remote_code=True)
+
+
+
+
+if config.USE_REMOTE:
+    tokenizer = AutoTokenizer.from_pretrained("baichuan-inc/Baichuan2-7B-Chat", trust_remote_code=True)
+else:
+    CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
+    TOKENIZER_DIR = os.path.join(CURRENT_DIR, "Baichuan2-7B-Chat")
+    tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_DIR, trust_remote_code=True)
 
 
 # byte-bpe  sentencepiece
 tokenizer.type = TokenizerType.ByteBPE
 
-tokenizer.comments = "expand the vocqbulary size from 64000 in Baichuan1 to 125696"
+tokenizer.comments = "expand the vocabulary size from 64000 in Baichuan1 to 125696"

vocab/baichuan2/bug.py

@@ -0,0 +1,22 @@
+
+"""
+这个 bug 已解决， https://huggingface.co/baichuan-inc/Baichuan2-7B-Chat/commit/e03d54f05b9d42740c43a191c5d2914fcfb4c6e5
+"""
+
+import os
+from transformers import AutoTokenizer
+from vocab import TokenizerType
+
+
+CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
+TOKENIZER_DIR = os.path.join(CURRENT_DIR, "Baichuan2-7B-Chat")
+tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_DIR, trust_remote_code=True)
+# tokenizer = AutoTokenizer.from_pretrained("baichuan-inc/Baichuan2-7B-Chat", trust_remote_code=True)
+
+token_ids = tokenizer.encode("<pad>")
+# token1 = tokenizer.decode(125696)  # out of range
+
+token_ids = tokenizer.encode("中")
+filtered_tokens = tokenizer.convert_ids_to_tokens(token_ids)
+decoded_text = tokenizer.convert_tokens_to_string(filtered_tokens)
+print(decoded_text)

vocab/baichuan2/encode.py

@@ -0,0 +1,15 @@
+
+import os
+os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"
+from sentencepiece import sentencepiece_model_pb2 as sp_pb2_model
+
+# sp = sp_pb2_model.ModelProto()
+# # sp.ParseFromString(open("zh_corpus.unigram.model", "rb").read())
+# sp.ParseFromString(open("zh_corpus.bpe.user_defined_symbols.model", "rb").read())
+
+import sentencepiece as spm
+sp = spm.SentencePieceProcessor(model_file="Baichuan2-7B-Chat/tokenizer.model")
+for text in ["汉堡王", "汉", "堡", "sfds<|USER|>ss</s><Rhino>", "<reserved_87254>", "<reserved_928>"]:
+    result_str = sp.encode(text, out_type=str)
+    result_int = sp.encode(text, out_type=int)
+    print(result_str, result_int)

vocab/baichuan2/special_token.py

@@ -0,0 +1,15 @@
+
+import os
+os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"
+from sentencepiece import sentencepiece_model_pb2 as sp_pb2_model
+import sentencepiece as spm
+
+baichuan_spm = sp_pb2_model.ModelProto()
+baichuan_spm.ParseFromString(open("Baichuan2-7B-Chat/tokenizer.model", "rb").read())
+
+
+vocab_size = len(baichuan_spm.pieces)
+for i in range(vocab_size):
+    piece = baichuan_spm.pieces[i]
+    if "reser" in piece.piece:
+        print(i, str(piece).strip().replace("\n", ", "))

vocab/baichuan7b_chat/__init__.py

@@ -0,0 +1,5 @@
+"""
+"vocab_size": 125696
+"""
+
+

vocab/build_zh_vocab.py

@@ -1,32 +0,0 @@
-import json
-from collections import defaultdict
-
-all_zh_words = defaultdict(int)
-for model_name in [
-    "gpt2",
-    "gpt2_chinese",
-    "chinese_llama_lora_7b",
-    "bert_chinese",
-    "moss",
-    "bloom",
-    "bloomz_6b4_zh",
-    "gpt_nexo_20b",
-    "gpt_neox_chinese_v1",
-    "glm_chinese",
-    "chatglm"
-]:
-    zh_word_set = set()
-    for line in open(model_name + "_vocab.zh.jsonl", "r", encoding="utf-8"):
-        item = json.loads(line)
-        token = item["token"]
-        if item["type"] in ["中文单字", "中文多字"]:
-            zh_word_set.add(token.strip())
-
-    for word in zh_word_set:
-        all_zh_words[word] += 1
-
-sorted_keywords = sorted(all_zh_words.items(), key=lambda kv: kv[1], reverse=True)
-
-with open("vocab.freq.zh.txt", "w", encoding="utf-8") as f_out:
-    for word, count in sorted_keywords:
-        f_out.write("%s\t%d\n" % (word, count))

vocab/chatglm2_6b/__init__.py

@@ -1,2 +1,8 @@
+
+import os
 from transformers import AutoTokenizer
+
+os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"
+
+
 tokenizer = AutoTokenizer.from_pretrained("THUDM/chatglm2-6b", trust_remote_code=True)

vocab/chatglm3_6b/__init__.py

@@ -0,0 +1,9 @@
+
+
+
+import os
+from transformers import AutoTokenizer
+
+os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"
+
+tokenizer = AutoTokenizer.from_pretrained("THUDM/chatglm3-6b", trust_remote_code=True)

vocab/chatglm_6b/README.md

@@ -7,7 +7,7 @@
 - ice_text.model
   - 二进制文件
   - num_image_tokens = 20000
-词典大小 150528
+文本词典大小=150528-20000
 
 
 ```

vocab/chatglm_6b/__init__.py

@@ -3,15 +3,19 @@
 """
 
 import os
+import config
 from transformers import AutoTokenizer
 
 os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"
 
-CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
-TOKENIZER_DIR = os.path.join(CURRENT_DIR, "tokenizer")
 
-tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_DIR, trust_remote_code=True)
+if config.USE_REMOTE:
+    pass
+else:
+    CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
+    TOKENIZER_DIR = os.path.join(CURRENT_DIR, "chatglm_6b")
+    tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_DIR, trust_remote_code=True)
 
+# https://huggingface.co/THUDM/chatglm-6b/blob/main/tokenization_chatglm.py#L153
+tokenizer.comments = f"num_image_tokens: {tokenizer.sp_tokenizer.num_image_tokens}; num_image_tokens: {tokenizer.sp_tokenizer.num_text_tokens} "
 
-# vocab_size = len(tokenizer.get_vocab())
-# vocab_size = tokenizer.vocab_size

vocab/chinese_llama2/demo.py

@@ -0,0 +1,6 @@
+
+
+from vocab.chinese_llama2 import tokenizer
+
+encoding = tokenizer.encode("<s>开始</s>站位符<pad>试试<unk>")
+print(encoding)

vocab/chinese_llama2/load_by_gpt_neox.py

@@ -0,0 +1,19 @@
+
+# HF_tokenizer
+# from tokenizers import Tokenizer
+# tokenizer = Tokenizer.from_file("tokenizer/tokenizer.json")
+
+
+import sentencepiece as spm
+
+text = "nice job 华为手机"
+text = "<s>世界上最高的山是哪座山？</s><pad>"  #  29871, 41334, 30528, 30210, 30329, 41894, 31780, 30329, 30882,
+tokenizer = spm.SentencePieceProcessor(model_file="tokenizer/tokenizer.model")
+tokens = tokenizer.encode(text)  # [7575, 4982, 29871, 31266, 30573, 30880, 31429]
+print(tokens)
+
+from transformers import LlamaTokenizer
+tokenizer = LlamaTokenizer.from_pretrained("tokenizer")
+tokens = tokenizer.encode(text)  # [1, 7575, 4982, 29871, 31266, 30573, 30880, 31429]
+print(tokens)
+

vocab/chinese_llama2/tokenizer/special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<pad>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

vocab/chinese_llama2/tokenizer/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a3b8844863b200dfcca971db228e96ce388290dfcf72c15d7a9d2f604bac787c
+size 844403

vocab/chinese_llama2/tokenizer/tokenizer_config.json

@@ -0,0 +1,35 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "bos_token": {
+    "__type": "AddedToken",
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "clean_up_tokenization_spaces": false,
+  "eos_token": {
+    "__type": "AddedToken",
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "legacy": true,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": null,
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": {
+    "__type": "AddedToken",
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "use_fast": false
+}

vocab/gpt_35_turbo/README.md

@@ -0,0 +1,41 @@
+
+
+## token
+
+
+
+space
+```yml
+# multi-space
+{"id": 881, "token": "\r\n\r\n", "token_decode": "\r\n\r\n", "token_len": 4, "zh_count": 0, "space_count": 4, "digit_count": 0, "zh_symbol_count": 0}
+# space + en
+{"id": 862, "token": "\treturn", "token_decode": "\treturn", "token_len": 7, "zh_count": 0, "space_count": 1, "digit_count": 0, "zh_symbol_count": 0}
+# sapce + zh  
+{"id": 40195, "token": " 下", "token_decode": " 下", "token_len": 2, "zh_count": 1, "space_count": 1, "digit_count": 0, "zh_symbol_count": 0}
+```
+
+
+special_token
+```
+{"id": 100257, "token": "<|endoftext|>", "token_decode": "<|endoftext|>", "token_len": 13, "zh_count": 0, "space_count": 0, "digit_count": 0, "zh_symbol_count": 0}
+{"id": 100258, "token": "<|fim_prefix|>", "token_decode": "<|fim_prefix|>", "token_len": 14, "zh_count": 0, "space_count": 0, "digit_count": 0, "zh_symbol_count": 0}
+{"id": 100259, "token": "<|fim_middle|>", "token_decode": "<|fim_middle|>", "token_len": 14, "zh_count": 0, "space_count": 0, "digit_count": 0, "zh_symbol_count": 0}
+{"id": 100260, "token": "<|fim_suffix|>", "token_decode": "<|fim_suffix|>", "token_len": 14, "zh_count": 0, "space_count": 0, "digit_count": 0, "zh_symbol_count": 0}
+{"id": 100276, "token": "<|endofprompt|>", "token_decode": "<|endofprompt|>", "token_len": 15, "zh_count": 0, "space_count": 0, "digit_count": 0, "zh_symbol_count": 0}
+```
+
+## 词典文件
+
+
+```
+IQ== 0
+Ig== 1
+Iw== 2
+JA== 3
+JQ== 4
+Jg== 5
+Jw== 6
+KA== 7
+```
+
+这是啥玩意？

vocab/gpt_35_turbo/__init__.py

@@ -11,16 +11,27 @@ tokenizer.comments = "tiktoken is a fast BPE tokeniser for use with OpenAI's mod
 tokenizer.reversible = True  # It's reversible and lossless, so you can convert tokens back into the original text
 
 
-def decode(self, tokens, errors="replace"):
-# def decode(self, tokens: list[int], errors: str = "replace") -> str:
+def decode(self, tokens, errors="replace", skip_special_tokens=False):
+    """
+    默认的decode，可能会报错，详见 decode_test.py
+    skip_special_tokens 是为了兼容 hf_tokenizer
+    """
     try:
         decode_str = self._core_bpe.decode_bytes(tokens).decode("utf-8", errors=errors)
     except:
         decode_str = "null"
     return decode_str
 
-def convert_ids_to_tokens(self, tokens):
-    return tokenizer.decode_tokens_bytes(tokens)
+def convert_ids_to_tokens(self, tokens, skip_special_tokens=False):
+    """
+    为什么没有这个方法？
+    """
+    try:
+        return tokenizer.decode_tokens_bytes(tokens)
+    except:
+        # 什么要返回None？见zh_util.py
+        # 16个空闲id, 100256 100261-100275
+        return [None for token in tokens]
 
 def get_vocab(self, token_type="str"):
     """Returns vocab as a dict
@@ -31,13 +42,15 @@ def get_vocab(self, token_type="str"):
     key_error_list = []
     unicode_decode_error_list = []
     for i in range(self.vocab_size):
+        if i == 100256:
+            print(i)
         try:
             token_byte = self.convert_ids_to_tokens([i])[0]
-            token_str = token_byte.decode("utf-8")
+            if token_byte is None:
+                continue
+            # token_str = token_byte.decode("utf-8")
             vocab[token_byte] = i
-        except KeyError:  # 16 KeyError, 100256 100261-100275
-            key_error_list.append(i)
-            # vocab[f"[KeyError]-{i}"] = i
+
         except UnicodeDecodeError:  # 773 UnicodeDecodeError
             unicode_decode_error_list.append((i, str(token_byte)))
             vocab[token_byte] = i

vocab/gpt_35_turbo/decode_test.py

@@ -0,0 +1,6 @@
+
+from vocab.gpt_35_turbo import tokenizer
+
+print(tokenizer.decode([100256]))
+
+print(tokenizer.convert_ids_to_tokens([100256]))

vocab/gpt_35_turbo/test_arena.py

@@ -0,0 +1,10 @@
+from vocab.gpt_35_turbo import tokenizer
+
+
+print(tokenizer.get_vocab())
+
+
+text = "中"
+token_ids = tokenizer.encode(text)
+decode_str = tokenizer.convert_tokens_to_string(token_ids)
+print(decode_str)

vocab/gpt_35_turbo/{test2.py → test_tiktoken.py}

@@ -26,6 +26,7 @@ tokenizer.decode_tokens_bytes([10])
 tokenizer.decode_single_token_bytes(10)
 tokenizer.decode_bytes([10])
 
+
 f_out = open("vocab.jsonl", "w")
 # 100255
 for i in range(tokenizer.n_vocab):

vocab/gpt_35_turbo/{aaa.py → tiktoken_decode_test.py}

@@ -19,6 +19,7 @@ tokenizer = tiktoken.encoding_for_model('gpt-3.5-turbo')
 
 tokens = [100263, 99834]
 
+tokenizer.decode(tokens)
 tokenizer.decode(tokens)
 
 tokenizer._core_bpe.decode_bytes(tokens).decode("utf-8", errors="replace")

vocab/gpt_35_turbo/vocab.jsonl

@@ -99964,314 +99964,3 @@
 {"id": 99963, "token": "\" Geg\""}
 {"id": 99964, "token": "\"\\tdto\""}
 {"id": 99965, "token": "\".defaultValue\""}
-{"id": 99966, "token": "\" Kami\""}
-{"id": 99967, "token": "\" ASE\""}
-{"id": 99968, "token": "\"optimized\""}
-{"id": 99969, "token": "\" \\ud3ec\""}
-{"id": 99970, "token": "\" originates\""}
-{"id": 99971, "token": "\"errMsg\""}
-{"id": 99972, "token": "\" espa\\u00e7o\""}
-{"id": 99973, "token": "\"(SYS\""}
-{"id": 99974, "token": "\" McB\""}
-{"id": 99975, "token": "\"dance\""}
-{"id": 99976, "token": "\"_detected\""}
-{"id": 99977, "token": "\" fr\\u00fc\""}
-{"id": 99978, "token": "\"\\t\\t    \\t\\t\""}
-{"id": 99979, "token": "\"<Date\""}
-{"id": 99980, "token": "\"(comb\""}
-{"id": 99981, "token": "\" Decide\""}
-{"id": 99982, "token": "\"\\\\Field\""}
-{"id": 99983, "token": "\" Proposed\""}
-{"id": 99984, "token": "\"Rib\""}
-{"id": 99985, "token": "\" dislikes\""}
-{"id": 99986, "token": "\" Wien\""}
-{"id": 99987, "token": "\"\\tDocument\""}
-{"id": 99988, "token": "\" traf\""}
-{"id": 99989, "token": "\" storia\""}
-{"id": 99990, "token": "\" Tells\""}
-{"id": 99991, "token": "\"')==\""}
-{"id": 99992, "token": "\"Cri\""}
-{"id": 99993, "token": "\"(VALUE\""}
-{"id": 99994, "token": "\" Burnett\""}
-{"id": 99995, "token": "\",void\""}
-{"id": 99996, "token": "\" danh\""}
-{"id": 99997, "token": "\" ccp\""}
-{"id": 99998, "token": "\"Blockchain\""}
-{"id": 99999, "token": "\":\\\"-\\\"`\\n\""}
-{"id": 100000, "token": "\"IClient\""}
-{"id": 100001, "token": "\"ISODE\""}
-{"id": 100002, "token": "\"Issuer\""}
-{"id": 100003, "token": "\")}\\r\\n\""}
-{"id": 100004, "token": "\",but\""}
-{"id": 100005, "token": "\" Uph\""}
-{"id": 100006, "token": "\"(Sub\""}
-{"id": 100007, "token": "\" t\\u00e9l\\u00e9phone\""}
-{"id": 100008, "token": "\" onDataChange\""}
-{"id": 100009, "token": "\" marshaller\""}
-{"id": 100010, "token": "\"-analytics\""}
-{"id": 100011, "token": "\",content\""}
-{"id": 100012, "token": "\" debacle\""}
-{"id": 100013, "token": "\"_ValueChanged\""}
-{"id": 100014, "token": "\" fauna\""}
-{"id": 100015, "token": "\" #=>\""}
-{"id": 100016, "token": "\" foyer\""}
-{"id": 100017, "token": "\"'utilisation\""}
-{"id": 100018, "token": "\" M\\u00fcller\""}
-{"id": 100019, "token": "\" Fetish\""}
-{"id": 100020, "token": "\" defaultManager\""}
-{"id": 100021, "token": "\" backtrack\""}
-{"id": 100022, "token": "\"Bah\""}
-{"id": 100023, "token": "\"Explicit\""}
-{"id": 100024, "token": "\"_ASCII\""}
-{"id": 100025, "token": "\" mActivity\""}
-{"id": 100026, "token": "\"(Msg\""}
-{"id": 100027, "token": "\" \\uac8c\""}
-{"id": 100028, "token": "\" TERMS\""}
-{"id": 100029, "token": "\" Angie\""}
-{"id": 100030, "token": "\"HSV\""}
-{"id": 100031, "token": "\" Mosque\""}
-{"id": 100032, "token": "\".Names\""}
-{"id": 100033, "token": "\"\\ud2bc\""}
-{"id": 100034, "token": "\"reste\""}
-{"id": 100035, "token": "\"_parms\""}
-{"id": 100036, "token": "\" gaping\""}
-{"id": 100037, "token": "\" cropping\""}
-{"id": 100038, "token": "\"DataFrame\""}
-{"id": 100039, "token": "\" responsiveness\""}
-{"id": 100040, "token": "\"_undo\""}
-{"id": 100041, "token": "\"_tran\""}
-{"id": 100042, "token": "\".terminate\""}
-{"id": 100043, "token": "\" italiane\""}
-{"id": 100044, "token": "\" walkthrough\""}
-{"id": 100045, "token": "\" attractiveness\""}
-{"id": 100046, "token": "\"\\u0434\\u0435\""}
-{"id": 100047, "token": "\"_STS\""}
-{"id": 100048, "token": "\"_learn\""}
-{"id": 100049, "token": "\" chocolates\""}
-{"id": 100050, "token": "\"ierarchical\""}
-{"id": 100051, "token": "\"-thinking\""}
-{"id": 100052, "token": "\" )))\""}
-{"id": 100053, "token": "\"ishments\""}
-{"id": 100054, "token": "\".Logf\""}
-{"id": 100055, "token": "\" TMZ\""}
-{"id": 100056, "token": "\" Canary\""}
-{"id": 100057, "token": "\"foil\""}
-{"id": 100058, "token": "\" Vaccine\""}
-{"id": 100059, "token": "\".vx\""}
-{"id": 100060, "token": "\" Surround\""}
-{"id": 100061, "token": "\"Intermediate\""}
-{"id": 100062, "token": "\" iov\""}
-{"id": 100063, "token": "\"vais\""}
-{"id": 100064, "token": "\"';\\\";\\n\""}
-{"id": 100065, "token": "\"\\uff5e\\n\\n\""}
-{"id": 100066, "token": "\"\\u9001\\u6599\""}
-{"id": 100067, "token": "\"\\u2026it\""}
-{"id": 100068, "token": "\"Seats\""}
-{"id": 100069, "token": "\"Clar\""}
-{"id": 100070, "token": "\"Wars\""}
-{"id": 100071, "token": "\" Hutchinson\""}
-{"id": 100072, "token": "\" Hasan\""}
-{"id": 100073, "token": "\"!')\\n\\n\""}
-{"id": 100074, "token": "\" Richie\""}
-{"id": 100075, "token": "\"cheiden\""}
-{"id": 100076, "token": "\"($('\""}
-{"id": 100077, "token": "\"York\""}
-{"id": 100078, "token": "\" lids\""}
-{"id": 100079, "token": "\" alphanumeric\""}
-{"id": 100080, "token": "\" Glock\""}
-{"id": 100081, "token": "\".shapes\""}
-{"id": 100082, "token": "\" sparking\""}
-{"id": 100083, "token": "\"_epsilon\""}
-{"id": 100084, "token": "\"uplicated\""}
-{"id": 100085, "token": "\".dirty\""}
-{"id": 100086, "token": "\"])==\""}
-{"id": 100087, "token": "\" \\uc704\\uce58\""}
-{"id": 100088, "token": "\" scn\""}
-{"id": 100089, "token": "\" /****************************************************************\""}
-{"id": 100090, "token": "\"_PREVIEW\""}
-{"id": 100091, "token": "\"_HC\""}
-{"id": 100092, "token": "\"ielding\""}
-{"id": 100093, "token": "\"fgets\""}
-{"id": 100094, "token": "\" Addison\""}
-{"id": 100095, "token": "\" productService\""}
-{"id": 100096, "token": "\"-figure\""}
-{"id": 100097, "token": "\"(retval\""}
-{"id": 100098, "token": "\"zano\""}
-{"id": 100099, "token": "\" autob\""}
-{"id": 100100, "token": "\"\\tsd\""}
-{"id": 100101, "token": "\"_numer\""}
-{"id": 100102, "token": "\" SetLastError\""}
-{"id": 100103, "token": "\" Fior\""}
-{"id": 100104, "token": "\"ificance\""}
-{"id": 100105, "token": "\"Untitled\""}
-{"id": 100106, "token": "\" infield\""}
-{"id": 100107, "token": "\" {}));\\n\""}
-{"id": 100108, "token": "\" spac\""}
-{"id": 100109, "token": "\" rookies\""}
-{"id": 100110, "token": "\"(describing\""}
-{"id": 100111, "token": "\"ngen\""}
-{"id": 100112, "token": "\"\\u0bbf\\ufffd\""}
-{"id": 100113, "token": "\".rdf\""}
-{"id": 100114, "token": "\".Mutex\""}
-{"id": 100115, "token": "\" kneeling\""}
-{"id": 100116, "token": "\" QE\""}
-{"id": 100117, "token": "\"setMax\""}
-{"id": 100118, "token": "\"ReadStream\""}
-{"id": 100119, "token": "\" ventas\""}
-{"id": 100120, "token": "\"sut\""}
-{"id": 100121, "token": "\"cmpeq\""}
-{"id": 100122, "token": "\".WriteAllText\""}
-{"id": 100123, "token": "\" Experienced\""}
-{"id": 100124, "token": "\"$__\""}
-{"id": 100125, "token": "\" kaum\""}
-{"id": 100126, "token": "\" LIS\""}
-{"id": 100127, "token": "\" documentos\""}
-{"id": 100128, "token": "\"_HEALTH\""}
-{"id": 100129, "token": "\"icontains\""}
-{"id": 100130, "token": "\" artisans\""}
-{"id": 100131, "token": "\"OWNER\""}
-{"id": 100132, "token": "\" blinked\""}
-{"id": 100133, "token": "\"getDisplay\""}
-{"id": 100134, "token": "\" toen\""}
-{"id": 100135, "token": "\" rowNum\""}
-{"id": 100136, "token": "\" avril\""}
-{"id": 100137, "token": "\" invis\""}
-{"id": 100138, "token": "\" Kear\""}
-{"id": 100139, "token": "\"toBeInTheDocument\""}
-{"id": 100140, "token": "\"apur\""}
-{"id": 100141, "token": "\" racked\""}
-{"id": 100142, "token": "\" McMaster\""}
-{"id": 100143, "token": "\"_ATTRIB\""}
-{"id": 100144, "token": "\"Haz\""}
-{"id": 100145, "token": "\" factura\""}
-{"id": 100146, "token": "\"/ts\""}
-{"id": 100147, "token": "\" \\u0440\\u0430\\u0437\\u043c\\u0435\\u0440\""}
-{"id": 100148, "token": "\" zf\""}
-{"id": 100149, "token": "\" shortfall\""}
-{"id": 100150, "token": "\".fasta\""}
-{"id": 100151, "token": "\" CONSTANT\""}
-{"id": 100152, "token": "\".managed\""}
-{"id": 100153, "token": "\"gems\""}
-{"id": 100154, "token": "\"SharedPointer\""}
-{"id": 100155, "token": "\" blurry\""}
-{"id": 100156, "token": "\"brightness\""}
-{"id": 100157, "token": "\"(components\""}
-{"id": 100158, "token": "\" ...\\\"\\n\\n\""}
-{"id": 100159, "token": "\"SELL\""}
-{"id": 100160, "token": "\" Illustrator\""}
-{"id": 100161, "token": "\".getChannel\""}
-{"id": 100162, "token": "\" trouv\\u00e9\""}
-{"id": 100163, "token": "\"ysters\""}
-{"id": 100164, "token": "\" vois\""}
-{"id": 100165, "token": "\" Linden\""}
-{"id": 100166, "token": "\" emojis\""}
-{"id": 100167, "token": "\" brawl\""}
-{"id": 100168, "token": "\" MSR\""}
-{"id": 100169, "token": "\" Elo\""}
-{"id": 100170, "token": "\" Croatian\""}
-{"id": 100171, "token": "\"PopupMenu\""}
-{"id": 100172, "token": "\"Lewis\""}
-{"id": 100173, "token": "\".JWT\""}
-{"id": 100174, "token": "\" astonished\""}
-{"id": 100175, "token": "\"Bush\""}
-{"id": 100176, "token": "\"(itemId\""}
-{"id": 100177, "token": "\" detachment\""}
-{"id": 100178, "token": "\" Encore\""}
-{"id": 100179, "token": "\"\\u5c14\""}
-{"id": 100180, "token": "\" rekl\""}
-{"id": 100181, "token": "\" cram\""}
-{"id": 100182, "token": "\")$/\""}
-{"id": 100183, "token": "\".getHost\""}
-{"id": 100184, "token": "\"_recommend\""}
-{"id": 100185, "token": "\"-HT\""}
-{"id": 100186, "token": "\"_calibration\""}
-{"id": 100187, "token": "\"Authenticate\""}
-{"id": 100188, "token": "\".firebaseapp\""}
-{"id": 100189, "token": "\"UNIX\""}
-{"id": 100190, "token": "\"\\tCamera\""}
-{"id": 100191, "token": "\" HEAP\""}
-{"id": 100192, "token": "\"Ideal\""}
-{"id": 100193, "token": "\".office\""}
-{"id": 100194, "token": "\" goofy\""}
-{"id": 100195, "token": "\"(Symbol\""}
-{"id": 100196, "token": "\" jouer\""}
-{"id": 100197, "token": "\"_partitions\""}
-{"id": 100198, "token": "\" rapidement\""}
-{"id": 100199, "token": "\" GNUNET\""}
-{"id": 100200, "token": "\"idUser\""}
-{"id": 100201, "token": "\" supervise\""}
-{"id": 100202, "token": "\"(Contact\""}
-{"id": 100203, "token": "\"AWN\""}
-{"id": 100204, "token": "\"\\u3058\""}
-{"id": 100205, "token": "\" naam\""}
-{"id": 100206, "token": "\" aust\""}
-{"id": 100207, "token": "\"\\u5728\\u7ebf\""}
-{"id": 100208, "token": "\"_softmax\""}
-{"id": 100209, "token": "\"AllowAnonymous\""}
-{"id": 100210, "token": "\"ammable\""}
-{"id": 100211, "token": "\"ROUTE\""}
-{"id": 100212, "token": "\"*D\""}
-{"id": 100213, "token": "\" aden\""}
-{"id": 100214, "token": "\" Cristina\""}
-{"id": 100215, "token": "\" Cristiano\""}
-{"id": 100216, "token": "\" bloodstream\""}
-{"id": 100217, "token": "\"subclass\""}
-{"id": 100218, "token": "\"_persona\""}
-{"id": 100219, "token": "\"CHILD\""}
-{"id": 100220, "token": "\"-know\""}
-{"id": 100221, "token": "\" navigationOptions\""}
-{"id": 100222, "token": "\" Zukunft\""}
-{"id": 100223, "token": "\" Pixar\""}
-{"id": 100224, "token": "\"Tyler\""}
-{"id": 100225, "token": "\" underworld\""}
-{"id": 100226, "token": "\" sincerity\""}
-{"id": 100227, "token": "\" dispenser\""}
-{"id": 100228, "token": "\" kter\""}
-{"id": 100229, "token": "\"idders\""}
-{"id": 100230, "token": "\".addNode\""}
-{"id": 100231, "token": "\"-checked\""}
-{"id": 100232, "token": "\" keyst\""}
-{"id": 100233, "token": "\" WTO\""}
-{"id": 100234, "token": "\".signals\""}
-{"id": 100235, "token": "\" adventurer\""}
-{"id": 100236, "token": "\" Pang\""}
-{"id": 100237, "token": "\"\\\\R\""}
-{"id": 100238, "token": "\"=pos\""}
-{"id": 100239, "token": "\" dispensaries\""}
-{"id": 100240, "token": "\" Closet\""}
-{"id": 100241, "token": "\"(\\\"{\\\\\\\"\""}
-{"id": 100242, "token": "\"ideon\""}
-{"id": 100243, "token": "\" n\\u00e9cessaire\""}
-{"id": 100244, "token": "\"()\\\"\\n\""}
-{"id": 100245, "token": "\"_RECEIVED\""}
-{"id": 100246, "token": "\" r\\u00e9sultats\""}
-{"id": 100247, "token": "\" moden\""}
-{"id": 100248, "token": "\" Icelandic\""}
-{"id": 100249, "token": "\";d\""}
-{"id": 100250, "token": "\".allowed\""}
-{"id": 100251, "token": "\"(newUser\""}
-{"id": 100252, "token": "\" merciless\""}
-{"id": 100253, "token": "\".WaitFor\""}
-{"id": 100254, "token": "\" daycare\""}
-{"id": 100255, "token": "\" Conveyor\""}
-{"id": 100256, "token": "null"}
-{"id": 100257, "token": "\"<|endoftext|>\""}
-{"id": 100258, "token": "\"<|fim_prefix|>\""}
-{"id": 100259, "token": "\"<|fim_middle|>\""}
-{"id": 100260, "token": "\"<|fim_suffix|>\""}
-{"id": 100261, "token": "null"}
-{"id": 100262, "token": "null"}
-{"id": 100263, "token": "null"}
-{"id": 100264, "token": "null"}
-{"id": 100265, "token": "null"}
-{"id": 100266, "token": "null"}
-{"id": 100267, "token": "null"}
-{"id": 100268, "token": "null"}
-{"id": 100269, "token": "null"}
-{"id": 100270, "token": "null"}
-{"id": 100271, "token": "null"}
-{"id": 100272, "token": "null"}
-{"id": 100273, "token": "null"}
-{"id": 100274, "token": "null"}
-{"id": 100275, "token": "null"}
-{"id": 100276, "token": "\"<|endofprompt|>\""}

vocab/internlm_chat_7b/__init__.py

@@ -3,4 +3,5 @@ https://huggingface.co/internlm/internlm-chat-7b
 """
 
 from transformers import AutoTokenizer
+
 tokenizer = AutoTokenizer.from_pretrained("internlm/internlm-chat-7b", trust_remote_code=True)