Yasinjan99's picture
Upload tokenizer
e93e5c7
raw
history blame
542 Bytes
{
"[PAD]": 44,
"[UNK]": 43,
"|": 0,
"«": 1,
"»": 2,
"،": 3,
"؛": 4,
"؟": 5,
"ئ": 6,
"ا": 7,
"ب": 8,
"ت": 9,
"ج": 10,
"خ": 11,
"د": 12,
"ر": 13,
"ز": 14,
"س": 15,
"ش": 16,
"غ": 17,
"ف": 18,
"ق": 19,
"ك": 20,
"ل": 21,
"م": 22,
"ن": 23,
"و": 24,
"ى": 25,
"ي": 26,
"پ": 27,
"چ": 28,
"ژ": 29,
"ڭ": 30,
"گ": 31,
"ھ": 32,
"ۆ": 33,
"ۇ": 34,
"ۈ": 35,
"ۋ": 36,
"ې": 37,
"ە": 38,
"–": 39,
"—": 40,
"‹": 41,
"›": 42
}