Update app.py
Browse files
app.py
CHANGED
@@ -40,6 +40,10 @@ def fine_tune_model(base_model_id, dataset):
|
|
40 |
model = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto")
|
41 |
tokenizer = AutoTokenizer.from_pretrained(model_name)
|
42 |
|
|
|
|
|
|
|
|
|
43 |
# Preprocess the dataset
|
44 |
def preprocess(examples):
|
45 |
model_inputs = tokenizer(examples["sql_prompt"], text_target=examples["sql"], max_length=512, padding="max_length", truncation=True)
|
|
|
40 |
model = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto")
|
41 |
tokenizer = AutoTokenizer.from_pretrained(model_name)
|
42 |
|
43 |
+
if tokenizer.pad_token is None:
|
44 |
+
tokenizer.add_special_tokens({"pad_token": "[PAD]"})
|
45 |
+
model.resize_token_embeddings(len(tokenizer))
|
46 |
+
|
47 |
# Preprocess the dataset
|
48 |
def preprocess(examples):
|
49 |
model_inputs = tokenizer(examples["sql_prompt"], text_target=examples["sql"], max_length=512, padding="max_length", truncation=True)
|