Hugging Face's logo

retrieval-bar
/
Mistral-7B-v0.1_case-briefs

PEFT
Safetensors
Generated from Trainer
Model card Files Community
zlucia commited on
Commit
63996b2
•
1 Parent(s): 61e205e

End of training

Browse files
Files changed (9) hide show
  1. README.md +7 -7
  2. adapter_config.json +5 -5
  3. adapter_model.safetensors +1 -1
  4. all_results.json +7 -7
  5. eval_results.json +4 -4
  6. metrics.json +1 -1
  7. train_results.json +3 -3
  8. trainer_state.json +81 -81
  9. training_args.bin +1 -1
README.md CHANGED
@@ -16,7 +16,7 @@ should probably proofread and complete it, then remove this comment. -->
16
 
17
  This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on the None dataset.
18
  It achieves the following results on the evaluation set:
19
- - Loss: 1.1311
20
 
21
  ## Model description
22
 
@@ -35,7 +35,7 @@ More information needed
35
  ### Training hyperparameters
36
 
37
  The following hyperparameters were used during training:
38
- - learning_rate: 3e-05
39
  - train_batch_size: 4
40
  - eval_batch_size: 4
41
  - seed: 42
@@ -50,11 +50,11 @@ The following hyperparameters were used during training:
50
 
51
  | Training Loss | Epoch | Step | Validation Loss |
52
  |:-------------:|:-----:|:----:|:---------------:|
53
- | 1.1084 | 0.34 | 50 | 1.1509 |
54
- | 1.0641 | 0.68 | 100 | 1.1312 |
55
- | 1.04 | 1.02 | 150 | 1.1267 |
56
- | 1.016 | 1.36 | 200 | 1.1304 |
57
- | 1.0272 | 1.7 | 250 | 1.1311 |
58
 
59
 
60
  ### Framework versions
 
16
 
17
  This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on the None dataset.
18
  It achieves the following results on the evaluation set:
19
+ - Loss: 1.1436
20
 
21
  ## Model description
22
 
 
35
  ### Training hyperparameters
36
 
37
  The following hyperparameters were used during training:
38
+ - learning_rate: 5e-05
39
  - train_batch_size: 4
40
  - eval_batch_size: 4
41
  - seed: 42
 
50
 
51
  | Training Loss | Epoch | Step | Validation Loss |
52
  |:-------------:|:-----:|:----:|:---------------:|
53
+ | 1.0965 | 0.34 | 50 | 1.1430 |
54
+ | 1.0622 | 0.68 | 100 | 1.1252 |
55
+ | 1.022 | 1.02 | 150 | 1.1218 |
56
+ | 0.9513 | 1.36 | 200 | 1.1433 |
57
+ | 0.9607 | 1.7 | 250 | 1.1436 |
58
 
59
 
60
  ### Framework versions
adapter_config.json CHANGED
@@ -10,7 +10,7 @@
10
  "layers_to_transform": null,
11
  "loftq_config": {},
12
  "lora_alpha": 16,
13
- "lora_dropout": 0.1,
14
  "megatron_config": null,
15
  "megatron_core": "megatron.core",
16
  "modules_to_save": null,
@@ -19,13 +19,13 @@
19
  "rank_pattern": {},
20
  "revision": null,
21
  "target_modules": [
22
- "o_proj",
23
- "k_proj",
24
  "q_proj",
25
- "up_proj",
26
  "down_proj",
 
27
  "gate_proj",
28
- "v_proj"
 
 
29
  ],
30
  "task_type": "CAUSAL_LM"
31
  }
 
10
  "layers_to_transform": null,
11
  "loftq_config": {},
12
  "lora_alpha": 16,
13
+ "lora_dropout": 0.05,
14
  "megatron_config": null,
15
  "megatron_core": "megatron.core",
16
  "modules_to_save": null,
 
19
  "rank_pattern": {},
20
  "revision": null,
21
  "target_modules": [
 
 
22
  "q_proj",
 
23
  "down_proj",
24
+ "v_proj",
25
  "gate_proj",
26
+ "up_proj",
27
+ "k_proj",
28
+ "o_proj"
29
  ],
30
  "task_type": "CAUSAL_LM"
31
  }
adapter_model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:670b0288c7dfa7cfe688367ff6afc89d6b2d0fc950c82206416a1016080b70a4
3
  size 335605144
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b3b839704789f616e132915bc53976f5d9af234dc59e7dc285def6cd551eff0c
3
  size 335605144
all_results.json CHANGED
@@ -1,11 +1,11 @@
1
  {
2
  "epoch": 2.0,
3
- "eval_loss": 1.2889022827148438,
4
- "eval_runtime": 26.7524,
5
- "eval_samples_per_second": 32.67,
6
- "eval_steps_per_second": 2.056,
7
- "train_loss": 1.0463855915329083,
8
- "train_runtime": 868.1968,
9
- "train_samples_per_second": 5.427,
10
  "train_steps_per_second": 0.339
11
  }
 
1
  {
2
  "epoch": 2.0,
3
+ "eval_loss": 1.1302779912948608,
4
+ "eval_runtime": 15.6893,
5
+ "eval_samples_per_second": 16.699,
6
+ "eval_steps_per_second": 4.207,
7
+ "train_loss": 1.0100539411817278,
8
+ "train_runtime": 867.3678,
9
+ "train_samples_per_second": 5.433,
10
  "train_steps_per_second": 0.339
11
  }
eval_results.json CHANGED
@@ -1,7 +1,7 @@
1
  {
2
  "epoch": 2.0,
3
- "eval_loss": 1.2889022827148438,
4
- "eval_runtime": 26.7524,
5
- "eval_samples_per_second": 32.67,
6
- "eval_steps_per_second": 2.056
7
  }
 
1
  {
2
  "epoch": 2.0,
3
+ "eval_loss": 1.1302779912948608,
4
+ "eval_runtime": 15.6893,
5
+ "eval_samples_per_second": 16.699,
6
+ "eval_steps_per_second": 4.207
7
  }
metrics.json CHANGED
@@ -1 +1 @@
1
- {"run_name": "./output", "train_runtime": 2763.6536, "train_samples_per_second": 12.007, "train_steps_per_second": 0.187, "total_flos": 5.99394721244119e+17, "train_loss": 1.2557983950758533, "epoch": 2.0, "eval_loss": 1.2889022827148438, "eval_runtime": 48.1894, "eval_samples_per_second": 18.137, "eval_steps_per_second": 1.141}
 
1
+ {"run_name": "./output", "train_runtime": 868.1968, "train_samples_per_second": 5.427, "train_steps_per_second": 0.339, "train_loss": 1.0463855915329083, "epoch": 2.0, "eval_loss": 1.1302779912948608, "eval_runtime": 15.6893, "eval_samples_per_second": 16.699, "eval_steps_per_second": 4.207}
train_results.json CHANGED
@@ -1,7 +1,7 @@
1
  {
2
  "epoch": 2.0,
3
- "train_loss": 1.0463855915329083,
4
- "train_runtime": 868.1968,
5
- "train_samples_per_second": 5.427,
6
  "train_steps_per_second": 0.339
7
  }
 
1
  {
2
  "epoch": 2.0,
3
+ "train_loss": 1.0100539411817278,
4
+ "train_runtime": 867.3678,
5
+ "train_samples_per_second": 5.433,
6
  "train_steps_per_second": 0.339
7
  }
trainer_state.json CHANGED
@@ -10,225 +10,225 @@
10
  "log_history": [
11
  {
12
  "epoch": 0.07,
13
- "learning_rate": 3e-05,
14
- "loss": 1.2091,
15
  "step": 10
16
  },
17
  {
18
  "epoch": 0.14,
19
- "learning_rate": 3e-05,
20
- "loss": 1.1462,
21
  "step": 20
22
  },
23
  {
24
  "epoch": 0.2,
25
- "learning_rate": 3e-05,
26
- "loss": 1.1415,
27
  "step": 30
28
  },
29
  {
30
  "epoch": 0.27,
31
- "learning_rate": 3e-05,
32
- "loss": 1.1181,
33
  "step": 40
34
  },
35
  {
36
  "epoch": 0.34,
37
- "learning_rate": 3e-05,
38
- "loss": 1.1084,
39
  "step": 50
40
  },
41
  {
42
  "epoch": 0.34,
43
- "eval_loss": 1.1509493589401245,
44
- "eval_runtime": 15.7829,
45
- "eval_samples_per_second": 16.6,
46
- "eval_steps_per_second": 4.182,
47
  "step": 50
48
  },
49
  {
50
  "epoch": 0.41,
51
- "learning_rate": 3e-05,
52
- "loss": 1.0965,
53
  "step": 60
54
  },
55
  {
56
  "epoch": 0.48,
57
- "learning_rate": 3e-05,
58
- "loss": 1.0929,
59
  "step": 70
60
  },
61
  {
62
  "epoch": 0.54,
63
- "learning_rate": 3e-05,
64
- "loss": 1.1405,
65
  "step": 80
66
  },
67
  {
68
  "epoch": 0.61,
69
- "learning_rate": 3e-05,
70
- "loss": 1.1153,
71
  "step": 90
72
  },
73
  {
74
  "epoch": 0.68,
75
- "learning_rate": 3e-05,
76
- "loss": 1.0641,
77
  "step": 100
78
  },
79
  {
80
  "epoch": 0.68,
81
- "eval_loss": 1.131222128868103,
82
- "eval_runtime": 15.7708,
83
- "eval_samples_per_second": 16.613,
84
- "eval_steps_per_second": 4.185,
85
  "step": 100
86
  },
87
  {
88
  "epoch": 0.75,
89
- "learning_rate": 3e-05,
90
- "loss": 1.0562,
91
  "step": 110
92
  },
93
  {
94
  "epoch": 0.81,
95
- "learning_rate": 3e-05,
96
- "loss": 1.1051,
97
  "step": 120
98
  },
99
  {
100
  "epoch": 0.88,
101
- "learning_rate": 3e-05,
102
- "loss": 1.0951,
103
  "step": 130
104
  },
105
  {
106
  "epoch": 0.95,
107
- "learning_rate": 3e-05,
108
- "loss": 1.0712,
109
  "step": 140
110
  },
111
  {
112
  "epoch": 1.02,
113
- "learning_rate": 3e-05,
114
- "loss": 1.04,
115
  "step": 150
116
  },
117
  {
118
  "epoch": 1.02,
119
- "eval_loss": 1.1267426013946533,
120
- "eval_runtime": 15.8691,
121
- "eval_samples_per_second": 16.51,
122
- "eval_steps_per_second": 4.159,
123
  "step": 150
124
  },
125
  {
126
  "epoch": 1.09,
127
- "learning_rate": 3e-05,
128
- "loss": 0.9894,
129
  "step": 160
130
  },
131
  {
132
  "epoch": 1.15,
133
- "learning_rate": 3e-05,
134
- "loss": 1.0622,
135
  "step": 170
136
  },
137
  {
138
  "epoch": 1.22,
139
- "learning_rate": 3e-05,
140
- "loss": 0.9794,
141
  "step": 180
142
  },
143
  {
144
  "epoch": 1.29,
145
- "learning_rate": 3e-05,
146
- "loss": 0.9593,
147
  "step": 190
148
  },
149
  {
150
  "epoch": 1.36,
151
- "learning_rate": 3e-05,
152
- "loss": 1.016,
153
  "step": 200
154
  },
155
  {
156
  "epoch": 1.36,
157
- "eval_loss": 1.1303731203079224,
158
- "eval_runtime": 15.6796,
159
- "eval_samples_per_second": 16.71,
160
- "eval_steps_per_second": 4.209,
161
  "step": 200
162
  },
163
  {
164
  "epoch": 1.43,
165
- "learning_rate": 3e-05,
166
- "loss": 0.9869,
167
  "step": 210
168
  },
169
  {
170
  "epoch": 1.49,
171
- "learning_rate": 3e-05,
172
- "loss": 0.9076,
173
  "step": 220
174
  },
175
  {
176
  "epoch": 1.56,
177
- "learning_rate": 3e-05,
178
- "loss": 0.9926,
179
  "step": 230
180
  },
181
  {
182
  "epoch": 1.63,
183
- "learning_rate": 3e-05,
184
- "loss": 0.9892,
185
  "step": 240
186
  },
187
  {
188
  "epoch": 1.7,
189
- "learning_rate": 3e-05,
190
- "loss": 1.0272,
191
  "step": 250
192
  },
193
  {
194
  "epoch": 1.7,
195
- "eval_loss": 1.1311020851135254,
196
- "eval_runtime": 15.7521,
197
- "eval_samples_per_second": 16.633,
198
- "eval_steps_per_second": 4.19,
199
  "step": 250
200
  },
201
  {
202
  "epoch": 1.77,
203
- "learning_rate": 3e-05,
204
- "loss": 0.9765,
205
  "step": 260
206
  },
207
  {
208
  "epoch": 1.83,
209
- "learning_rate": 3e-05,
210
- "loss": 1.0433,
211
  "step": 270
212
  },
213
  {
214
  "epoch": 1.9,
215
- "learning_rate": 3e-05,
216
- "loss": 0.9614,
217
  "step": 280
218
  },
219
  {
220
  "epoch": 1.97,
221
- "learning_rate": 3e-05,
222
- "loss": 0.9299,
223
  "step": 290
224
  },
225
  {
226
  "epoch": 2.0,
227
  "step": 294,
228
  "total_flos": 7.021903495711949e+16,
229
- "train_loss": 1.0463855915329083,
230
- "train_runtime": 868.1968,
231
- "train_samples_per_second": 5.427,
232
  "train_steps_per_second": 0.339
233
  }
234
  ],
 
10
  "log_history": [
11
  {
12
  "epoch": 0.07,
13
+ "learning_rate": 5e-05,
14
+ "loss": 1.1773,
15
  "step": 10
16
  },
17
  {
18
  "epoch": 0.14,
19
+ "learning_rate": 5e-05,
20
+ "loss": 1.1468,
21
  "step": 20
22
  },
23
  {
24
  "epoch": 0.2,
25
+ "learning_rate": 5e-05,
26
+ "loss": 1.1279,
27
  "step": 30
28
  },
29
  {
30
  "epoch": 0.27,
31
+ "learning_rate": 5e-05,
32
+ "loss": 1.1021,
33
  "step": 40
34
  },
35
  {
36
  "epoch": 0.34,
37
+ "learning_rate": 5e-05,
38
+ "loss": 1.0965,
39
  "step": 50
40
  },
41
  {
42
  "epoch": 0.34,
43
+ "eval_loss": 1.1430302858352661,
44
+ "eval_runtime": 15.7711,
45
+ "eval_samples_per_second": 16.613,
46
+ "eval_steps_per_second": 4.185,
47
  "step": 50
48
  },
49
  {
50
  "epoch": 0.41,
51
+ "learning_rate": 5e-05,
52
+ "loss": 1.0874,
53
  "step": 60
54
  },
55
  {
56
  "epoch": 0.48,
57
+ "learning_rate": 5e-05,
58
+ "loss": 1.0742,
59
  "step": 70
60
  },
61
  {
62
  "epoch": 0.54,
63
+ "learning_rate": 5e-05,
64
+ "loss": 1.1263,
65
  "step": 80
66
  },
67
  {
68
  "epoch": 0.61,
69
+ "learning_rate": 5e-05,
70
+ "loss": 1.1085,
71
  "step": 90
72
  },
73
  {
74
  "epoch": 0.68,
75
+ "learning_rate": 5e-05,
76
+ "loss": 1.0622,
77
  "step": 100
78
  },
79
  {
80
  "epoch": 0.68,
81
+ "eval_loss": 1.1251620054244995,
82
+ "eval_runtime": 15.7412,
83
+ "eval_samples_per_second": 16.644,
84
+ "eval_steps_per_second": 4.193,
85
  "step": 100
86
  },
87
  {
88
  "epoch": 0.75,
89
+ "learning_rate": 5e-05,
90
+ "loss": 1.0416,
91
  "step": 110
92
  },
93
  {
94
  "epoch": 0.81,
95
+ "learning_rate": 5e-05,
96
+ "loss": 1.102,
97
  "step": 120
98
  },
99
  {
100
  "epoch": 0.88,
101
+ "learning_rate": 5e-05,
102
+ "loss": 1.0927,
103
  "step": 130
104
  },
105
  {
106
  "epoch": 0.95,
107
+ "learning_rate": 5e-05,
108
+ "loss": 1.0662,
109
  "step": 140
110
  },
111
  {
112
  "epoch": 1.02,
113
+ "learning_rate": 5e-05,
114
+ "loss": 1.022,
115
  "step": 150
116
  },
117
  {
118
  "epoch": 1.02,
119
+ "eval_loss": 1.1217572689056396,
120
+ "eval_runtime": 15.7347,
121
+ "eval_samples_per_second": 16.651,
122
+ "eval_steps_per_second": 4.195,
123
  "step": 150
124
  },
125
  {
126
  "epoch": 1.09,
127
+ "learning_rate": 5e-05,
128
+ "loss": 0.9443,
129
  "step": 160
130
  },
131
  {
132
  "epoch": 1.15,
133
+ "learning_rate": 5e-05,
134
+ "loss": 1.0,
135
  "step": 170
136
  },
137
  {
138
  "epoch": 1.22,
139
+ "learning_rate": 5e-05,
140
+ "loss": 0.9185,
141
  "step": 180
142
  },
143
  {
144
  "epoch": 1.29,
145
+ "learning_rate": 5e-05,
146
+ "loss": 0.8956,
147
  "step": 190
148
  },
149
  {
150
  "epoch": 1.36,
151
+ "learning_rate": 5e-05,
152
+ "loss": 0.9513,
153
  "step": 200
154
  },
155
  {
156
  "epoch": 1.36,
157
+ "eval_loss": 1.143324613571167,
158
+ "eval_runtime": 15.8072,
159
+ "eval_samples_per_second": 16.575,
160
+ "eval_steps_per_second": 4.175,
161
  "step": 200
162
  },
163
  {
164
  "epoch": 1.43,
165
+ "learning_rate": 5e-05,
166
+ "loss": 0.9233,
167
  "step": 210
168
  },
169
  {
170
  "epoch": 1.49,
171
+ "learning_rate": 5e-05,
172
+ "loss": 0.8422,
173
  "step": 220
174
  },
175
  {
176
  "epoch": 1.56,
177
+ "learning_rate": 5e-05,
178
+ "loss": 0.928,
179
  "step": 230
180
  },
181
  {
182
  "epoch": 1.63,
183
+ "learning_rate": 5e-05,
184
+ "loss": 0.9318,
185
  "step": 240
186
  },
187
  {
188
  "epoch": 1.7,
189
+ "learning_rate": 5e-05,
190
+ "loss": 0.9607,
191
  "step": 250
192
  },
193
  {
194
  "epoch": 1.7,
195
+ "eval_loss": 1.1436448097229004,
196
+ "eval_runtime": 15.7943,
197
+ "eval_samples_per_second": 16.588,
198
+ "eval_steps_per_second": 4.179,
199
  "step": 250
200
  },
201
  {
202
  "epoch": 1.77,
203
+ "learning_rate": 5e-05,
204
+ "loss": 0.9034,
205
  "step": 260
206
  },
207
  {
208
  "epoch": 1.83,
209
+ "learning_rate": 5e-05,
210
+ "loss": 0.977,
211
  "step": 270
212
  },
213
  {
214
  "epoch": 1.9,
215
+ "learning_rate": 5e-05,
216
+ "loss": 0.8984,
217
  "step": 280
218
  },
219
  {
220
  "epoch": 1.97,
221
+ "learning_rate": 5e-05,
222
+ "loss": 0.8655,
223
  "step": 290
224
  },
225
  {
226
  "epoch": 2.0,
227
  "step": 294,
228
  "total_flos": 7.021903495711949e+16,
229
+ "train_loss": 1.0100539411817278,
230
+ "train_runtime": 867.3678,
231
+ "train_samples_per_second": 5.433,
232
  "train_steps_per_second": 0.339
233
  }
234
  ],
training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:575d23392a793f463c05c62aed6a20c9f3f1d14af4f07046c639ce932527d18b
3
  size 6712
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:865ed33e039a712b5469b18103737a06f20880f83c7c5347d5c09ad77bd65241
3
  size 6712