Change space token from <th_roberta_space_token> to <_>
Browse files- sentencepiece.bpe.vocab +1 -1
- tokenizer_config.json +2 -2
sentencepiece.bpe.vocab
CHANGED
@@ -2,7 +2,7 @@
|
|
2 |
<s> 0
|
3 |
</s> 0
|
4 |
<mask> 0
|
5 |
-
<
|
6 |
า -1.82049
|
7 |
▁ -3.88454
|
8 |
. -4.13722
|
|
|
2 |
<s> 0
|
3 |
</s> 0
|
4 |
<mask> 0
|
5 |
+
<_> 0
|
6 |
า -1.82049
|
7 |
▁ -3.88454
|
8 |
. -4.13722
|
tokenizer_config.json
CHANGED
@@ -10,6 +10,6 @@
|
|
10 |
"additional_special_tokens": [
|
11 |
"<s>NOTUSED",
|
12 |
"</s>NOTUSED",
|
13 |
-
"<
|
14 |
]
|
15 |
-
}
|
|
|
10 |
"additional_special_tokens": [
|
11 |
"<s>NOTUSED",
|
12 |
"</s>NOTUSED",
|
13 |
+
"<_>"
|
14 |
]
|
15 |
+
}
|