Update app.py

app.py

@@ -16,23 +16,9 @@ def respond(
     temperature,
     top_p,
 ):
-    quantization_config = BitsAndBytesConfig(
-        bnb_4bit_compute_dtype="float16",
-        bnb_4bit_quant_storage="uint8",
-        bnb_4bit_quant_type="nf4",
-        bnb_4bit_use_double_quant=True,
-        llm_int8_enable_fp32_cpu_offload=False,
-        llm_int8_has_fp16_weight=False,
-        llm_int8_skip_modules=None,
-        llm_int8_threshold=6.0,
-        load_in_4bit=True,
-        load_in_8bit=False,
-        quant_method="bitsandbytes"
-    )
-
     # Load model and tokenizer
     model = AutoModelForCausalLM.from_pretrained("Rorical/0-roleplay", return_dict=True, trust_remote_code=True)
-    tokenizer = AutoTokenizer.from_pretrained("Rorical/0-roleplay", trust_remote_code=True, quantization_config=quantization_config)
+    tokenizer = AutoTokenizer.from_pretrained("Rorical/0-roleplay", trust_remote_code=True)
     tokenizer.chat_template = "{% for message in messages %}{{'<|im_start|>' + ((message['role'] + ':\n') if message['role'] != '' else '') + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>星野:\n' }}{% endif %}" # Be careful that this model used custom chat template.
     
     # Construct the messages for the chat