Update app.py
Browse files
app.py
CHANGED
@@ -34,6 +34,9 @@ def process(model_id, dataset):
|
|
34 |
|
35 |
# Step 3: Text tokenization
|
36 |
|
|
|
|
|
|
|
37 |
tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
|
38 |
|
39 |
# Step 4: Apply tokenization to dataset
|
|
|
34 |
|
35 |
# Step 3: Text tokenization
|
36 |
|
37 |
+
def preprocess_function(examples):
|
38 |
+
return tokenizer(examples["text"], padding="max_length", truncation=True)
|
39 |
+
|
40 |
tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
|
41 |
|
42 |
# Step 4: Apply tokenization to dataset
|