Jlonge4/outputs

README.md

@@ -14,30 +14,12 @@ model-index:
 <!-- This model card has been generated automatically according to the information the Trainer had access to. You
 should probably proofread and complete it, then remove this comment. -->
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/josh-longenecker1-groundedai/phi3.5-hallucination/runs/0q2t3ek2)
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/josh-longenecker1-groundedai/phi3.5-hallucination/runs/0q2t3ek2)
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/josh-longenecker1-groundedai/phi3.5-hallucination/runs/0q2t3ek2)
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/josh-longenecker1-groundedai/phi3.5-hallucination/runs/0q2t3ek2)
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/josh-longenecker1-groundedai/phi3.5-hallucination/runs/0q2t3ek2)
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/josh-longenecker1-groundedai/phi3.5-hallucination/runs/0q2t3ek2)
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/josh-longenecker1-groundedai/phi3.5-hallucination/runs/0q2t3ek2)
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/josh-longenecker1-groundedai/phi3.5-hallucination/runs/0q2t3ek2)
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/josh-longenecker1-groundedai/phi3.5-hallucination/runs/0q2t3ek2)
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/josh-longenecker1-groundedai/phi3.5-hallucination/runs/0q2t3ek2)
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/josh-longenecker1-groundedai/phi3.5-hallucination/runs/dtvgakdw)
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/josh-longenecker1-groundedai/phi3.5-hallucination/runs/9hr7jcal)
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/josh-longenecker1-groundedai/phi3.5-hallucination/runs/goc9hcye)
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/josh-longenecker1-groundedai/phi3.5-hallucination/runs/fzw32mg0)
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/josh-longenecker1-groundedai/phi3.5-hallucination/runs/fzw32mg0)
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/josh-longenecker1-groundedai/phi3.5-hallucination/runs/fzw32mg0)
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/josh-longenecker1-groundedai/phi3.5-hallucination/runs/dphr4egm)
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/josh-longenecker1-groundedai/phi3.5-hallucination/runs/dphr4egm)
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/josh-longenecker1-groundedai/phi3.5-hallucination/runs/dphr4egm)
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/josh-longenecker1-groundedai/phi3.5-hallucination/runs/vn7nj2r3)
 # outputs
 
 This model is a fine-tuned version of [microsoft/Phi-3.5-mini-instruct](https://huggingface.co/microsoft/Phi-3.5-mini-instruct) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 1.0926
+- Loss: 1.5219
 
 ## Model description
 
@@ -65,28 +47,38 @@ The following hyperparameters were used during training:
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine_with_restarts
 - lr_scheduler_warmup_steps: 20
-- training_steps: 160
+- training_steps: 260
 
 ### Training results
 
 | Training Loss | Epoch   | Step | Validation Loss |
 |:-------------:|:-------:|:----:|:---------------:|
-| 2.2453        | 1.1429  | 10   | 2.1914          |
-| 1.8068        | 2.2857  | 20   | 1.8497          |
-| 1.5702        | 3.4286  | 30   | 1.5758          |
-| 1.5012        | 4.5714  | 40   | 1.2568          |
-| 1.2486        | 5.7143  | 50   | 1.1309          |
-| 0.948         | 6.8571  | 60   | 1.0965          |
-| 1.0246        | 8.0     | 70   | 1.0826          |
-| 0.7834        | 9.1429  | 80   | 1.0786          |
-| 0.8802        | 10.2857 | 90   | 1.0755          |
-| 0.7285        | 11.4286 | 100  | 1.0781          |
-| 0.8049        | 12.5714 | 110  | 1.0855          |
-| 0.831         | 13.7143 | 120  | 1.0920          |
-| 0.6412        | 14.8571 | 130  | 1.0900          |
-| 0.7723        | 16.0    | 140  | 1.0898          |
-| 0.8869        | 17.1429 | 150  | 1.0908          |
-| 0.6173        | 18.2857 | 160  | 1.0926          |
+| 1.8679        | 1.1429  | 10   | 2.0237          |
+| 1.3961        | 2.2857  | 20   | 1.5707          |
+| 0.939         | 3.4286  | 30   | 1.1184          |
+| 0.9957        | 4.5714  | 40   | 0.9883          |
+| 0.8836        | 5.7143  | 50   | 0.9527          |
+| 0.8069        | 6.8571  | 60   | 0.9431          |
+| 0.6692        | 8.0     | 70   | 0.9416          |
+| 0.7691        | 9.1429  | 80   | 0.9574          |
+| 0.5804        | 10.2857 | 90   | 0.9505          |
+| 0.395         | 11.4286 | 100  | 0.9772          |
+| 0.3864        | 12.5714 | 110  | 1.0155          |
+| 0.3433        | 13.7143 | 120  | 1.0573          |
+| 0.4332        | 14.8571 | 130  | 1.0832          |
+| 0.224         | 16.0    | 140  | 1.1592          |
+| 0.1891        | 17.1429 | 150  | 1.2302          |
+| 0.2235        | 18.2857 | 160  | 1.2603          |
+| 0.1925        | 19.4286 | 170  | 1.3136          |
+| 0.2264        | 20.5714 | 180  | 1.3556          |
+| 0.1491        | 21.7143 | 190  | 1.4057          |
+| 0.2421        | 22.8571 | 200  | 1.4966          |
+| 0.1515        | 24.0    | 210  | 1.4495          |
+| 0.1349        | 25.1429 | 220  | 1.5144          |
+| 0.1493        | 26.2857 | 230  | 1.5340          |
+| 0.1202        | 27.4286 | 240  | 1.5201          |
+| 0.1154        | 28.5714 | 250  | 1.5305          |
+| 0.1968        | 29.7143 | 260  | 1.5219          |
 
 
 ### Framework versions

adapter_config.json

@@ -10,8 +10,8 @@
   "layers_pattern": null,
   "layers_to_transform": null,
   "loftq_config": {},
-  "lora_alpha": 64,
-  "lora_dropout": 0.1,
+  "lora_alpha": 128,
+  "lora_dropout": 0.2,
   "megatron_config": null,
   "megatron_core": "megatron.core",
   "modules_to_save": null,
@@ -21,12 +21,12 @@
   "revision": null,
   "target_modules": [
     "up_proj",
-    "v_proj",
-    "q_proj",
-    "o_proj",
-    "down_proj",
     "k_proj",
-    "gate_proj"
+    "down_proj",
+    "gate_proj",
+    "o_proj",
+    "q_proj",
+    "v_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f8c872fe43d7260fd00639758f062906f3d5a97b1db0c838fc60b914e973aee9
+oid sha256:c47f5b555306903f26770a32ba84cdb3afdda4a92f30a5a6a0a28d0268ee956e
 size 142623480

runs/Sep08_03-30-31_111903198ea6/events.out.tfevents.1725766233.111903198ea6.3278.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91b014a21f74d49994be6a5f24ef4ae210a8bf47f20d7fe50d05847e4d52c272
+size 69780

tokenizer.json

@@ -1,6 +1,11 @@
 {
   "version": "1.0",
-  "truncation": null,
+  "truncation": {
+    "direction": "Right",
+    "max_length": 1024,
+    "strategy": "LongestFirst",
+    "stride": 0
+  },
   "padding": null,
   "added_tokens": [
     {

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bb4d1b5e68a98620bd5d07cf5547d1546cb3035cdbf437944b92386c077aea6e
+oid sha256:68156363a28ce3d23f4f6fe3990ca6cd67bf6b5e851dc870c0ea8fd2a80adf41
 size 5432