Update app.py

app.py

@@ -3,7 +3,7 @@ import gradio as gr
 import os, torch
 from datasets import load_dataset
 from huggingface_hub import HfApi, login
-from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
+from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig, pipeline
 
 from transformers import Seq2SeqTrainer, Seq2SeqTrainingArguments
 
@@ -34,10 +34,14 @@ def fine_tune_model(base_model_id, dataset):
 #    return fine_tuned_model_id
     # Load the dataset
     dataset = load_dataset("gretelai/synthetic_text_to_sql")
+
+    bnb_config = BitsAndBytesConfig(
+        load_in_4bit=True, bnb_4bit_use_double_quant=True, bnb_4bit_quant_type="nf4", bnb_4bit_compute_dtype=torch.bfloat16
+    )
     
     # Load pre-trained model and tokenizer
     model_name = "meta-llama/Meta-Llama-3.1-8B-Instruct"
-    model = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto")    
+    model = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto", quantization_config=bnb_config)    
     tokenizer = AutoTokenizer.from_pretrained(model_name)    
 
     # Preprocess the dataset