bstraehle commited on
Commit
340f2ae
1 Parent(s): 2750069

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +4 -0
app.py CHANGED
@@ -40,6 +40,10 @@ def fine_tune_model(base_model_id, dataset):
40
  model = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto")
41
  tokenizer = AutoTokenizer.from_pretrained(model_name)
42
 
 
 
 
 
43
  # Preprocess the dataset
44
  def preprocess(examples):
45
  model_inputs = tokenizer(examples["sql_prompt"], text_target=examples["sql"], max_length=512, padding="max_length", truncation=True)
 
40
  model = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto")
41
  tokenizer = AutoTokenizer.from_pretrained(model_name)
42
 
43
+ if tokenizer.pad_token is None:
44
+ tokenizer.add_special_tokens({"pad_token": "[PAD]"})
45
+ model.resize_token_embeddings(len(tokenizer))
46
+
47
  # Preprocess the dataset
48
  def preprocess(examples):
49
  model_inputs = tokenizer(examples["sql_prompt"], text_target=examples["sql"], max_length=512, padding="max_length", truncation=True)