bstraehle commited on
Commit
18cfe84
1 Parent(s): dbb3d55

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +3 -0
app.py CHANGED
@@ -34,6 +34,9 @@ def process(model_id, dataset):
34
 
35
  # Step 3: Text tokenization
36
 
 
 
 
37
  tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
38
 
39
  # Step 4: Apply tokenization to dataset
 
34
 
35
  # Step 3: Text tokenization
36
 
37
+ def preprocess_function(examples):
38
+ return tokenizer(examples["text"], padding="max_length", truncation=True)
39
+
40
  tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
41
 
42
  # Step 4: Apply tokenization to dataset