Upload RetNetForCausalLM

config.json

@@ -0,0 +1,38 @@
+{
+  "_name_or_path": "model",
+  "activation_dropout": 0.0,
+  "activation_fn": "swish",
+  "architectures": [
+    "RetNetForCausalLM"
+  ],
+  "decoder_embed_dim": 2560,
+  "decoder_ffn_embed_dim": 4280,
+  "decoder_layers": 32,
+  "decoder_normalize_before": true,
+  "decoder_retention_heads": 10,
+  "decoder_value_embed_dim": 4280,
+  "deepnorm": false,
+  "drop_path_rate": 0.0,
+  "dropout": 0.0,
+  "eos_token_id": 0,
+  "forward_impl": "parallel",
+  "initializer_range": 0.02,
+  "is_decoder": true,
+  "layernorm_embedding": true,
+  "layernorm_eps": 1e-06,
+  "model_type": "retnet",
+  "no_scale_embedding": false,
+  "output_retentions": false,
+  "pad_token_id": 0,
+  "recurrent_chunk_size": 512,
+  "subln": true,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.38.0",
+  "use_cache": true,
+  "use_ffn_rms_norm": false,
+  "use_glu": true,
+  "use_lm_decay": false,
+  "vocab_size": 65024,
+  "z_loss_coeff": 0.0
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "eos_token_id": 0,
+  "pad_token_id": 0,
+  "transformers_version": "4.38.0"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:18bbf58ab10d7102bf87689bcfabfb356290284f3bf1ccecee802b9044a703d0
+size 4993981832

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:54b8fdba138fc89e7eaaa9021213e28bc7ff35f7ae5d6e4e07f04a132cc1878f
+size 4958952904

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:530554580531b8fe9bd0589daf02eb3b89d3778457800d0b7eb0e6e205f4b9bb
+size 1472014648

model.safetensors.index.json

@@ -0,0 +1,333 @@
+{
+  "metadata": {
+    "total_size": 11424912424
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layer_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layernorm_embedding.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.ffn.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.ffn.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.ffn.gate.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.final_layer_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.retention.g_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.retention.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.retention.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.retention.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.retention.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.retention_layer_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.ffn.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.ffn.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.ffn.gate.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.final_layer_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.retention.g_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.retention.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.retention.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.retention.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.retention.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.retention_layer_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.ffn.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.ffn.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.ffn.gate.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.final_layer_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.retention.g_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.retention.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.retention.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.retention.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.retention.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.retention_layer_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.ffn.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.ffn.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.ffn.gate.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.final_layer_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.retention.g_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.retention.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.retention.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.retention.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.retention.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.retention_layer_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.ffn.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.ffn.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.ffn.gate.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.final_layer_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.retention.g_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.retention.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.retention.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.retention.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.retention.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.retention_layer_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.ffn.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.ffn.fc2.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.ffn.gate.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.final_layer_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.retention.g_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.retention.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.retention.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.retention.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.retention.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.retention_layer_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.14.ffn.fc1.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.ffn.fc2.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.ffn.gate.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.final_layer_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.retention.g_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.retention.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.retention.out_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.retention.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.retention.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.retention_layer_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.ffn.fc1.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.ffn.fc2.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.ffn.gate.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.final_layer_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.retention.g_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.retention.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.retention.out_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.retention.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.retention.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.retention_layer_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.ffn.fc1.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.ffn.fc2.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.ffn.gate.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.final_layer_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.retention.g_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.retention.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.retention.out_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.retention.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.retention.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.retention_layer_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.ffn.fc1.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.ffn.fc2.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.ffn.gate.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.final_layer_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.retention.g_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.retention.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.retention.out_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.retention.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.retention.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.retention_layer_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.ffn.fc1.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.ffn.fc2.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.ffn.gate.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.final_layer_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.retention.g_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.retention.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.retention.out_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.retention.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.retention.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.retention_layer_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.ffn.fc1.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.ffn.fc2.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.ffn.gate.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.final_layer_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.retention.g_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.retention.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.retention.out_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.retention.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.retention.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.retention_layer_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.ffn.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.ffn.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.ffn.gate.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.final_layer_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.retention.g_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.retention.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.retention.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.retention.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.retention.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.retention_layer_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.ffn.fc1.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.ffn.fc2.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.ffn.gate.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.final_layer_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.retention.g_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.retention.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.retention.out_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.retention.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.retention.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.retention_layer_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.ffn.fc1.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.ffn.fc2.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.ffn.gate.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.final_layer_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.retention.g_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.retention.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.retention.out_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.retention.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.retention.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.retention_layer_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.ffn.fc1.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.ffn.fc2.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.ffn.gate.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.final_layer_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.retention.g_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.retention.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.retention.out_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.retention.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.retention.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.retention_layer_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.ffn.fc1.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.ffn.fc2.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.ffn.gate.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.final_layer_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.retention.g_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.retention.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.retention.out_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.retention.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.retention.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.retention_layer_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.24.ffn.fc1.weight": "model-00002-of-00003.safetensors",
+    "model.layers.24.ffn.fc2.weight": "model-00002-of-00003.safetensors",
+    "model.layers.24.ffn.gate.weight": "model-00002-of-00003.safetensors",
+    "model.layers.24.final_layer_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.24.retention.g_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.24.retention.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.24.retention.out_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.24.retention.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.24.retention.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.24.retention_layer_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.ffn.fc1.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.ffn.fc2.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.ffn.gate.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.final_layer_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.retention.g_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.retention.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.retention.out_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.retention.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.retention.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.retention_layer_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.ffn.fc1.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.ffn.fc2.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.ffn.gate.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.final_layer_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.retention.g_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.retention.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.retention.out_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.retention.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.retention.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.retention_layer_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.27.ffn.fc1.weight": "model-00002-of-00003.safetensors",
+    "model.layers.27.ffn.fc2.weight": "model-00002-of-00003.safetensors",
+    "model.layers.27.ffn.gate.weight": "model-00002-of-00003.safetensors",
+    "model.layers.27.final_layer_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.27.retention.g_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.27.retention.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.27.retention.out_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.27.retention.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.27.retention.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.27.retention_layer_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.28.ffn.fc1.weight": "model-00002-of-00003.safetensors",
+    "model.layers.28.ffn.fc2.weight": "model-00002-of-00003.safetensors",
+    "model.layers.28.ffn.gate.weight": "model-00002-of-00003.safetensors",
+    "model.layers.28.final_layer_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.28.retention.g_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.28.retention.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.28.retention.out_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.28.retention.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.28.retention.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.28.retention_layer_norm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.29.ffn.fc1.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.ffn.fc2.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.ffn.gate.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.final_layer_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.retention.g_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.29.retention.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.29.retention.out_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.retention.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.29.retention.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.29.retention_layer_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.ffn.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.ffn.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.ffn.gate.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.final_layer_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.retention.g_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.retention.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.retention.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.retention.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.retention.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.retention_layer_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.ffn.fc1.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.ffn.fc2.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.ffn.gate.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.final_layer_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.retention.g_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.retention.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.retention.out_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.retention.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.retention.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.retention_layer_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.ffn.fc1.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.ffn.fc2.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.ffn.gate.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.final_layer_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.retention.g_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.retention.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.retention.out_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.retention.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.retention.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.retention_layer_norm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.ffn.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.ffn.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.ffn.gate.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.final_layer_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.retention.g_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.retention.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.retention.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.retention.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.retention.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.retention_layer_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.ffn.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.ffn.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.ffn.gate.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.final_layer_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.retention.g_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.retention.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.retention.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.retention.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.retention.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.retention_layer_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.ffn.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.ffn.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.ffn.gate.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.final_layer_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.retention.g_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.retention.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.retention.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.retention.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.retention.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.retention_layer_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.ffn.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.ffn.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.ffn.gate.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.final_layer_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.retention.g_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.retention.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.retention.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.retention.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.retention.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.retention_layer_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.ffn.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.ffn.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.ffn.gate.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.final_layer_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.retention.g_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.retention.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.retention.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.retention.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.retention.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.retention_layer_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.ffn.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.ffn.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.ffn.gate.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.final_layer_norm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.retention.g_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.retention.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.retention.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.retention.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.retention.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.retention_layer_norm.weight": "model-00001-of-00003.safetensors",
+    "model.retnet_rel_pos.angle": "model-00003-of-00003.safetensors",
+    "model.retnet_rel_pos.decay": "model-00003-of-00003.safetensors"
+  }
+}