Trained with Unsloth

README.md

@@ -1,6 +1,9 @@
 ---
 library_name: transformers
-tags: []
+tags:
+- unsloth
+- trl
+- sft
 ---
 
 # Model Card for Model ID

config.json

@@ -1,12 +1,12 @@
 {
-  "_name_or_path": "meta-llama/Meta-Llama-3.1-8B",
+  "_name_or_path": "unsloth/Meta-Llama-3.1-8B-bnb-4bit",
   "architectures": [
     "LlamaForCausalLM"
   ],
   "attention_bias": false,
   "attention_dropout": 0.0,
-  "bos_token_id": 128256,
-  "eos_token_id": 128257,
+  "bos_token_id": 128000,
+  "eos_token_id": 128001,
   "hidden_act": "silu",
   "hidden_size": 4096,
   "initializer_range": 0.02,
@@ -17,7 +17,7 @@
   "num_attention_heads": 32,
   "num_hidden_layers": 32,
   "num_key_value_heads": 8,
-  "pad_token_id": 128257,
+  "pad_token_id": 128004,
   "pretraining_tp": 1,
   "rms_norm_eps": 1e-05,
   "rope_scaling": {
@@ -30,7 +30,8 @@
   "rope_theta": 500000.0,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.43.3",
+  "transformers_version": "4.44.1",
+  "unsloth_version": "2024.8",
   "use_cache": true,
-  "vocab_size": 128258
+  "vocab_size": 128256
 }

generation_config.json

@@ -1,15 +1,11 @@
 {
   "_from_model_config": true,
-  "bos_token_id": [128256, 128000],
+  "bos_token_id": 128000,
   "do_sample": true,
-  "eos_token_id": [
-    128001,
-    128008,
-    128009,
-    128257,
-  ],
-  "pad_token_id": 128257,
+  "eos_token_id": 128001,
+  "max_length": 131072,
+  "pad_token_id": 128004,
   "temperature": 0.6,
   "top_p": 0.9,
-  "transformers_version": "4.43.3"
+  "transformers_version": "4.44.1"
 }

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f77ded706c176f095b2041c9dcf42f09af15f8bb51b7523015d5ef3f711957ae
-size 4976715056
+oid sha256:b90c639eaff877e1377b57098c7c85ce4fa11c81de3df0b042aee32f458bb54b
+size 4976698672

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9fb7efc0264dd2fe1d5f91468776691ed3f603bd2d612cde4b0863f1c3efa468
+oid sha256:b8ed1c62e1d2e38a55015daf5a0689931dfbf0c4006d082ac9dee8b127b0052b
 size 4999802720

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:27b1158a44135040cd34009f12b209da6663bf2709f328fe44968c1809507b8c
+oid sha256:256b2e01a853299499c16af3136d98b962f2042fb0b8ec0b66f69404c56ab4c0
 size 4915916176

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a801af0967dd3fdc2ca06e48ddf0d2883eea12480b6cf8e5939495f8a089f7d3
-size 1168155192
+oid sha256:9aece88193eb62fa9c1826f8de7f0d57d689436f96f75dfec82a938448c5c39f
+size 1168138808

model.safetensors.index.json

@@ -1,6 +1,6 @@
 {
   "metadata": {
-    "total_size": 16060555264
+    "total_size": 16060522496
   },
   "weight_map": {
     "lm_head.weight": "model-00004-of-00004.safetensors",