Upload LlamaForCausalLM

README.md

@@ -1,10 +1,5 @@
 ---
-library_name: transformers
 base_model: meta-llama/Meta-Llama-3.1-70B-Instruct
-license: llama3.1
-model-index:
-- name: Meta-Llama-3.1-70B-Instruct-INT4
-  results: []
 language:
 - en
 - de
@@ -14,12 +9,17 @@ language:
 - hi
 - es
 - th
+library_name: transformers
+license: llama3.1
 tags:
 - facebook
 - meta
 - pytorch
 - llama
 - llama-3
+model-index:
+- name: Meta-Llama-3.1-70B-Instruct-INT4
+  results: []
 ---
 
 # Model Card for Model ID

config.json

@@ -25,11 +25,11 @@
   "quantization_config": {
     "_load_in_4bit": true,
     "_load_in_8bit": false,
-    "bnb_4bit_compute_dtype": "float32",
+    "bnb_4bit_compute_dtype": "bfloat16",
     "bnb_4bit_quant_storage": "uint8",
-    "bnb_4bit_quant_type": "fp4",
-    "bnb_4bit_use_double_quant": false,
-    "llm_int8_enable_fp32_cpu_offload": false,
+    "bnb_4bit_quant_type": "nf4",
+    "bnb_4bit_use_double_quant": true,
+    "llm_int8_enable_fp32_cpu_offload": true,
     "llm_int8_has_fp16_weight": false,
     "llm_int8_skip_modules": null,
     "llm_int8_threshold": 6.0,
@@ -47,7 +47,7 @@
   },
   "rope_theta": 500000.0,
   "tie_word_embeddings": false,
-  "torch_dtype": "float16",
+  "torch_dtype": "bfloat16",
   "transformers_version": "4.43.2",
   "use_cache": true,
   "vocab_size": 128256

model-00001-of-00008.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f04cbd8f2bc235453ea5457ccdda15d14d3d12a02de21b57111866b9a4d6f003
+size 4949082290

model-00002-of-00008.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd890071c67995f897924f539401984791acfbd5cb6c08f7ddd6394f43c2c7b8
+size 4977054940

model-00003-of-00008.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:22b1f5d22b413e6043b431ccd4ff142b43b6788dbbcef009da272cf4826c0f37
+size 4977088094

model-00004-of-00008.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab428b4f7c89d580c55e8283be380973f256d7ab92ae5ffb3280dcc8bcb48a9e
+size 4933786845

model-00005-of-00008.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:27bff8f0032f6b4ec5ca3e24d28cf30dd9f9deb86e3538a01f967218a7005bfd
+size 4977055132

model-00006-of-00008.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70e81ccd976d2e883ca2b79157b2223cc90d4ee38189f8194372bdedce0df244
+size 4977055097

model-00007-of-00008.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e78a6f4902f5dea55bfc57451a80b2790e0cc4c184f77c198395f8cc66bdca6
+size 4977088084

model-00008-of-00008.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41ca67d01dffc9286724570664a9bd8e0a47eaa6f3459d85844aafb6a9e80dfb
+size 4750027590