Update app.py
Browse files
app.py
CHANGED
@@ -9,9 +9,11 @@ import evaluate
|
|
9 |
import numpy as np
|
10 |
import random
|
11 |
|
|
|
|
|
12 |
def preprocess_function(examples):
|
13 |
return tokenizer(examples["text"], padding="max_length", truncation=True)
|
14 |
-
|
15 |
def process(model_id, dataset):
|
16 |
# Step 1: Load dataset
|
17 |
dataset_imdb = load_dataset(dataset)
|
@@ -37,7 +39,7 @@ def process(model_id, dataset):
|
|
37 |
|
38 |
# Step 3: Text tokenization
|
39 |
|
40 |
-
tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
|
41 |
|
42 |
# Step 4: Apply tokenization to dataset
|
43 |
|
|
|
9 |
import numpy as np
|
10 |
import random
|
11 |
|
12 |
+
tokenizer = None
|
13 |
+
|
14 |
def preprocess_function(examples):
|
15 |
return tokenizer(examples["text"], padding="max_length", truncation=True)
|
16 |
+
|
17 |
def process(model_id, dataset):
|
18 |
# Step 1: Load dataset
|
19 |
dataset_imdb = load_dataset(dataset)
|
|
|
39 |
|
40 |
# Step 3: Text tokenization
|
41 |
|
42 |
+
global tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
|
43 |
|
44 |
# Step 4: Apply tokenization to dataset
|
45 |
|