arxyzan commited on
Commit
a7ef2d5
1 Parent(s): 354aee4

Hezar: Upload tokenizer and config

Browse files
Files changed (1) hide show
  1. preprocessor/tokenizer_config.yaml +26 -0
preprocessor/tokenizer_config.yaml ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: wordpiece_tokenizer
2
+ config_type: preprocessor
3
+ pretrained_path: hezarai/distilbert-base-fa
4
+ max_length: 512
5
+ truncation_strategy: longest_first
6
+ truncation_direction: right
7
+ stride: 0
8
+ padding_strategy: longest
9
+ padding_direction: right
10
+ pad_to_multiple_of: 0
11
+ pad_token_id: 0
12
+ pad_token: '[PAD]'
13
+ pad_token_type_id: 0
14
+ unk_token: '[UNK]'
15
+ special_tokens:
16
+ - '[UNK]'
17
+ - '[SEP]'
18
+ - '[CLS]'
19
+ - '[PAD]'
20
+ - '[MASK]'
21
+ wordpieces_prefix: '##'
22
+ vocab_size: 30000
23
+ min_frequency: 2
24
+ limit_alphabet: 1000
25
+ initial_alphabet: []
26
+ show_progress: true