dima1234321's picture
Upload tokenizer
817496d verified
{
"[PAD]": 101,
"[UNK]": 100,
"a": 1,
"b": 2,
"ba": 3,
"be": 4,
"bi": 5,
"bo": 6,
"bu": 7,
"d": 8,
"da": 9,
"de": 10,
"di": 11,
"do": 12,
"du": 13,
"e": 14,
"f": 15,
"g": 16,
"ga": 17,
"ge": 18,
"gi": 19,
"go": 20,
"gu": 21,
"h": 22,
"ha": 23,
"he": 24,
"hi": 25,
"ho": 26,
"hu": 27,
"i": 28,
"k": 29,
"ka": 30,
"ke": 31,
"ki": 32,
"ko": 33,
"ku": 34,
"l": 35,
"la": 36,
"le": 37,
"li": 38,
"lo": 39,
"lu": 40,
"m": 41,
"n": 42,
"o": 43,
"p": 44,
"pa": 45,
"pe": 46,
"pi": 47,
"po": 48,
"pu": 49,
"r": 50,
"ra": 51,
"re": 52,
"ri": 53,
"ro": 54,
"ru": 55,
"s": 56,
"sa": 57,
"se": 58,
"sh": 59,
"sha": 60,
"she": 61,
"shi": 62,
"sho": 63,
"shu": 64,
"si": 65,
"so": 66,
"su": 67,
"t": 68,
"ta": 69,
"te": 70,
"ti": 71,
"to": 72,
"tu": 73,
"tz": 74,
"u": 75,
"v": 76,
"va": 77,
"ve": 78,
"vi": 79,
"vo": 80,
"vu": 81,
"x": 82,
"xa": 83,
"xe": 84,
"xi": 85,
"xo": 86,
"xu": 87,
"y": 88,
"ya": 89,
"ye": 90,
"yi": 91,
"yo": 92,
"yu": 93,
"z": 94,
"za": 95,
"ze": 96,
"zi": 97,
"zo": 98,
"zu": 99,
"|": 0
}