fixed bug

app_modules/llm_loader.py

@@ -298,7 +298,7 @@ class LLMLoader:
                             config=config,
                             quantization_config=double_quant_config,
                             trust_remote_code=True,
-                            token=token,
+                            use_auth_token=token,
                         )
                         if is_t5
                         else AutoModelForCausalLM.from_pretrained(
@@ -306,7 +306,7 @@ class LLMLoader:
                             config=config,
                             quantization_config=double_quant_config,
                             trust_remote_code=True,
-                            token=token,
+                            use_auth_token=token,
                         )
                     )
 

tgi.sh

@@ -9,9 +9,7 @@ uname -a
 
 . env/tgi.conf 
 
-export MODEL_ID="meta-llama/Llama-2-7b-chat-hf"
-export QUANTIZE="--quantize bitsandbytes-fp4"
-
 echo Running $MODEL_ID with TGI
 
 text-generation-launcher --model-id $MODEL_ID --port $PORT --max-input-length 2048 --max-total-tokens 4096 --ngrok --ngrok-authtoken $NGROK_AUTHTOKEN --ngrok-edge $NGROK_EDGE $QUANTIZE
+