Update app.py

app.py

@@ -9,11 +9,6 @@ import evaluate
 import numpy as np
 import random
 
-tokenizer = None
-
-def preprocess_function(examples):
-    return tokenizer(examples["text"], padding="max_length", truncation=True)
-
 def process(model_id, dataset):
     # Step 1: Load dataset
     dataset_imdb = load_dataset(dataset)
@@ -39,11 +34,11 @@ def process(model_id, dataset):
 
     # Step 3: Text tokenization
 
-    global tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
+    tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
 
     # Step 4: Apply tokenization to dataset
 
-    tokenized_imdb = dataset_imdb.map(preprocess_function, batched=True)
+    tokenized_imdb = dataset_imdb.map(tokenizer(examples["text"], padding="max_length", truncation=True), batched=True)
 
     #Step 5: Fine-tune the model