Update README.md
Browse files
README.md
CHANGED
@@ -40,7 +40,7 @@ config = AutoConfig.from_pretrained(model_name)
|
|
40 |
config.quantization_config["use_exllama"] = False
|
41 |
config.quantization_config["exllama_config"] = {"version":2}
|
42 |
|
43 |
-
#
|
44 |
max_memory={0: "12GiB", "cpu": "10GiB"}
|
45 |
|
46 |
quantized_model = AutoModelForCausalLM.from_pretrained(model_name
|
|
|
40 |
config.quantization_config["use_exllama"] = False
|
41 |
config.quantization_config["exllama_config"] = {"version":2}
|
42 |
|
43 |
+
# adjust your gpu memory size. 0 means first gpu.
|
44 |
max_memory={0: "12GiB", "cpu": "10GiB"}
|
45 |
|
46 |
quantized_model = AutoModelForCausalLM.from_pretrained(model_name
|