Upload PhiMoEForCausalLM

config.json

@@ -16,9 +16,9 @@
   "hidden_size": 4096,
   "initializer_range": 0.02,
   "input_jitter_noise": 0.01,
-  "intermediate_size": 1920,
+  "intermediate_size": 2240,
   "lm_head_bias": true,
-  "max_position_embeddings": 131072,
+  "max_position_embeddings": 4096,
   "model_type": "phimoe",
   "num_attention_heads": 32,
   "num_experts_per_tok": 2,
@@ -28,148 +28,11 @@
   "original_max_position_embeddings": 4096,
   "output_router_logits": false,
   "rms_norm_eps": 1e-05,
-  "rope_scaling": {
-    "long_factor": [
-      1.0199999809265137,
-      1.0299999713897705,
-      1.0399999618530273,
-      1.0499999523162842,
-      1.0499999523162842,
-      1.0499999523162842,
-      1.059999942779541,
-      1.059999942779541,
-      1.059999942779541,
-      1.059999942779541,
-      1.059999942779541,
-      1.059999942779541,
-      1.0999999046325684,
-      1.1799999475479126,
-      1.1799999475479126,
-      1.3700000047683716,
-      1.4899998903274536,
-      2.109999895095825,
-      2.8899998664855957,
-      3.9499998092651367,
-      4.299999713897705,
-      6.429999828338623,
-      8.09000015258789,
-      10.690000534057617,
-      12.050000190734863,
-      18.229999542236328,
-      18.84000015258789,
-      19.899999618530273,
-      21.420000076293945,
-      26.200000762939453,
-      34.28000259399414,
-      34.590003967285156,
-      38.730003356933594,
-      40.22000503540039,
-      42.54000473022461,
-      44.000003814697266,
-      47.590003967285156,
-      54.750003814697266,
-      56.19000244140625,
-      57.44000244140625,
-      57.4900016784668,
-      61.20000076293945,
-      61.540000915527344,
-      61.75,
-      61.779998779296875,
-      62.06999969482422,
-      63.11000061035156,
-      63.43000030517578,
-      63.560001373291016,
-      63.71000289916992,
-      63.92000198364258,
-      63.94000244140625,
-      63.94000244140625,
-      63.96000289916992,
-      63.980003356933594,
-      64.0300064086914,
-      64.0300064086914,
-      64.0300064086914,
-      64.04000854492188,
-      64.10000610351562,
-      64.19000244140625,
-      64.20999908447266,
-      64.75,
-      64.95999908447266
-    ],
-    "long_mscale": 1.243163121016122,
-    "original_max_position_embeddings": 4096,
-    "short_factor": [
-      1.0,
-      1.0399999618530273,
-      1.0399999618530273,
-      1.0399999618530273,
-      1.0499999523162842,
-      1.0499999523162842,
-      1.0499999523162842,
-      1.0499999523162842,
-      1.0499999523162842,
-      1.0499999523162842,
-      1.0499999523162842,
-      1.0499999523162842,
-      1.0499999523162842,
-      1.0499999523162842,
-      1.059999942779541,
-      1.059999942779541,
-      1.0699999332427979,
-      1.0699999332427979,
-      1.0699999332427979,
-      1.0699999332427979,
-      1.1399999856948853,
-      1.159999966621399,
-      1.159999966621399,
-      1.159999966621399,
-      1.159999966621399,
-      1.1799999475479126,
-      1.1999999284744263,
-      1.3199999332427979,
-      1.3399999141693115,
-      1.3499999046325684,
-      1.3999998569488525,
-      1.4799998998641968,
-      1.4999998807907104,
-      1.589999794960022,
-      1.6499998569488525,
-      1.71999990940094,
-      1.8999998569488525,
-      1.9099998474121094,
-      1.9099998474121094,
-      1.9899998903274536,
-      1.9999998807907104,
-      1.9999998807907104,
-      2.009999990463257,
-      2.009999990463257,
-      2.009999990463257,
-      2.009999990463257,
-      2.009999990463257,
-      2.009999990463257,
-      2.009999990463257,
-      2.009999990463257,
-      2.009999990463257,
-      2.009999990463257,
-      2.009999990463257,
-      2.009999990463257,
-      2.009999990463257,
-      2.009999990463257,
-      2.009999990463257,
-      2.009999990463257,
-      2.009999990463257,
-      2.0999999046325684,
-      2.319999933242798,
-      2.419999837875366,
-      2.5899999141693115,
-      2.7899999618530273
-    ],
-    "short_mscale": 1.243163121016122,
-    "type": "longrope"
-  },
+  "rope_scaling": null,
   "rope_theta": 10000.0,
   "router_aux_loss_coef": 0.0,
   "router_jitter_noise": 0.01,
-  "sliding_window": 131072,
+  "sliding_window": 4096,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.41.2",

model-00001-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:19a5e82359d09fb36682c2baa8fd46e49187f6a950be5f71571e68aa1313b23a
+size 4995293984

model-00002-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd71dfcf44f57af8eb1b27603eac3222e7d27a9c07698620f7570ba94dc1096d
+size 4966538712

model-00003-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02c665177d645bb00746a5927c7e2fcdb6fe0294fcf77d520e939cc6937679ee
+size 4986555656

model-00004-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:015356e9dfda7305917b83c1c5af356939068c45654ab3b1ecc6cefcb1be13e1
+size 4989562032

model-00005-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77c81b9a69e3fb3ffffa462f29c5b86cea854dd47e16f7af54c72ad076ab2f8e
+size 4989562032

model-00006-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a444a00c6e27d043c5929c01fb171eecb430e341a0d226b3975bd74e52d28b1
+size 4989562040

model-00007-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3bb6372e707900abcf3df8d12413826b31b2be49f2f1ee7c580f3a2dd0c257f3
+size 1484567336