Spaces:

xu-song
/

tokenizer-arena

Running

xu-song commited on Feb 26

Commit

24b4aa5

•

1 Parent(s): 1f833af

update

Files changed (1) hide show

examples.py CHANGED Viewed

@@ -19,10 +19,10 @@ https://www.computerhope.com/jargon/s/specchar.htm
 examples = {
     "en": [
-        ["number: (10086 + 98) = 100184", "llama", "bloom"],
-        ["whitespace:  2spaces        8spaces\t1tab\t\t2tab\n1newline", "llama", "chatglm2_6b"],  # chatglm 有blank_n,
         # ！？｡＂＃＄％＆＇（）＊＋，－／：；＜＝＞＠［＼］＾＿｀｛｜｝～｟｠｢｣､、〃》「」『』【】〔〕〖〗〘〙〚〛〜〝〞〟〰〾〿–—‘’‛“”„‟…‧﹏.
-        ["punctuation: ,.:/?+=\"，。！？；【】〔〕〖〗", "baichuan", "llama"],
         ["symbol: 🦙❤❥웃유♋☮✊☏☢☚✔☑♚▢♪✈✞÷↑↓▤▥⊙■□▣▽¿─│♥❣▬▫☿Ⓐ ✋✉☣☤", "baichuan", "llama"],
     ],
     "zh": [

 examples = {
     "en": [
+        ["number: (10086 + 98) = 100184", "llama", "bloom"],  #
+        ["whitespace:  2spaces        8spaces\t1tab\t\t2tab\n1newline", "llama", "bert_base_cased"],  # chatglm 有blank_n, bert丢掉了空格，
         # ！？｡＂＃＄％＆＇（）＊＋，－／：；＜＝＞＠［＼］＾＿｀｛｜｝～｟｠｢｣､、〃》「」『』【】〔〕〖〗〘〙〚〛〜〝〞〟〰〾〿–—‘’‛“”„‟…‧﹏.
+        ["punctuation: ,.:/?+=\"，。！？；【】〔〕〖〗", "gemma_7b", "llama"],  # llama词典有点小
         ["symbol: 🦙❤❥웃유♋☮✊☏☢☚✔☑♚▢♪✈✞÷↑↓▤▥⊙■□▣▽¿─│♥❣▬▫☿Ⓐ ✋✉☣☤", "baichuan", "llama"],
     ],
     "zh": [