Add: DPO model card init

config.json

@@ -0,0 +1,28 @@
+{
+  "_name_or_path": "unum-cloud/uform-gen2-qwen-500m",
+  "architectures": [
+    "VLMForCausalLM"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration_uform_gen.VLMConfig",
+    "AutoModel": "modeling_uform_gen.VLMForCausalLM",
+    "AutoProcessor": "unum-cloud/uform-gen2-qwen-500m--processing_uform_gen.VLMProcessor"
+  },
+  "image_encoder_hidden_size": 1280,
+  "image_encoder_name_or_path": "unum-cloud/uform-vl-english-big",
+  "image_encoder_num_heads": 16,
+  "image_encoder_num_layers": 32,
+  "image_encoder_patch_size": 14,
+  "image_encoder_pooling": "cls",
+  "image_pooler_intermediate_size": 3200,
+  "image_pooler_num_attn_heads": 16,
+  "image_size": 336,
+  "image_token_id": 151646,
+  "initializer_range": 0.02,
+  "model_type": "vlm",
+  "num_image_latents": 256,
+  "text_decoder_name_or_path": "Qwen/Qwen1.5-0.5B-Chat",
+  "torch_dtype": "float32",
+  "transformers_version": "4.38.2",
+  "use_cache": true
+}

configuration_uform_gen.py

@@ -0,0 +1,43 @@
+from transformers.configuration_utils import PretrainedConfig
+from typing import List
+
+
+class VLMConfig(PretrainedConfig):
+    model_type = "vlm"
+
+    def __init__(
+        self,
+        text_decoder_name_or_path: str = "",
+        image_encoder_name_or_path: str = "",
+        image_size: int = 336,
+        image_pooler_num_attn_heads: int = 16,
+        image_pooler_intermediate_size: int = 3200,
+        image_token_id: int = 151646,
+        image_encoder_hidden_size: int = 1280,
+        image_encoder_patch_size: int = 14,
+        image_encoder_num_layers: int = 32,
+        image_encoder_num_heads: int = 16,
+        image_encoder_pooling: str = "cls",
+        num_image_latents: int = 256,
+        initializer_range: float = 0.02,
+        use_cache: bool = True,
+        **kwargs,
+    ):
+        self.text_decoder_name_or_path = text_decoder_name_or_path
+        self.image_encoder_name_or_path = image_encoder_name_or_path
+
+        self.image_pooler_num_attn_heads = image_pooler_num_attn_heads
+        self.image_pooler_intermediate_size = image_pooler_intermediate_size
+        self.image_token_id = image_token_id
+        self.image_size = image_size
+        self.image_encoder_hidden_size = image_encoder_hidden_size
+        self.image_encoder_patch_size = image_encoder_patch_size
+        self.image_encoder_num_layers = image_encoder_num_layers
+        self.image_encoder_num_heads = image_encoder_num_heads
+        self.image_encoder_pooling = image_encoder_pooling
+        self.num_image_latents = num_image_latents
+
+        self.initializer_range = initializer_range
+        self.use_cache = use_cache
+        
+        super().__init__(**kwargs)

generation_config.json

@@ -0,0 +1,4 @@
+{
+  "_from_model_config": true,
+  "transformers_version": "4.38.2"
+}

model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0be8c717e54079a0de25dcd2bdf0ed07d418c56f35eb97cf0a91aa2d3bd6a4c
+size 4975529328

model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:27ce47ee20f96a21f8c7ac2a1be36d838b973b5af28b2b0b0e993b86c8a35a27
+size 118081792

model.safetensors.index.json

@@ -0,0 +1,901 @@
+{
+  "metadata": {
+    "total_size": 5093506560
+  },
+  "weight_map": {
+    "image_encoder.blocks.0.attn.key.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.0.attn.key.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.0.attn.out.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.0.attn.out.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.0.attn.query.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.0.attn.query.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.0.attn.value.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.0.attn.value.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.0.ls1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.0.ls2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.0.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.0.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.0.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.0.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.0.norm1.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.0.norm1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.0.norm2.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.0.norm2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.1.attn.key.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.1.attn.key.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.1.attn.out.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.1.attn.out.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.1.attn.query.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.1.attn.query.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.1.attn.value.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.1.attn.value.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.1.ls1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.1.ls2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.1.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.1.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.1.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.1.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.1.norm1.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.1.norm1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.1.norm2.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.1.norm2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.10.attn.key.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.10.attn.key.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.10.attn.out.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.10.attn.out.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.10.attn.query.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.10.attn.query.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.10.attn.value.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.10.attn.value.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.10.ls1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.10.ls2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.10.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.10.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.10.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.10.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.10.norm1.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.10.norm1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.10.norm2.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.10.norm2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.11.attn.key.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.11.attn.key.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.11.attn.out.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.11.attn.out.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.11.attn.query.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.11.attn.query.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.11.attn.value.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.11.attn.value.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.11.ls1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.11.ls2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.11.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.11.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.11.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.11.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.11.norm1.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.11.norm1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.11.norm2.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.11.norm2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.12.attn.key.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.12.attn.key.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.12.attn.out.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.12.attn.out.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.12.attn.query.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.12.attn.query.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.12.attn.value.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.12.attn.value.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.12.ls1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.12.ls2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.12.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.12.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.12.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.12.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.12.norm1.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.12.norm1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.12.norm2.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.12.norm2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.13.attn.key.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.13.attn.key.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.13.attn.out.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.13.attn.out.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.13.attn.query.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.13.attn.query.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.13.attn.value.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.13.attn.value.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.13.ls1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.13.ls2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.13.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.13.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.13.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.13.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.13.norm1.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.13.norm1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.13.norm2.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.13.norm2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.14.attn.key.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.14.attn.key.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.14.attn.out.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.14.attn.out.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.14.attn.query.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.14.attn.query.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.14.attn.value.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.14.attn.value.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.14.ls1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.14.ls2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.14.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.14.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.14.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.14.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.14.norm1.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.14.norm1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.14.norm2.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.14.norm2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.15.attn.key.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.15.attn.key.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.15.attn.out.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.15.attn.out.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.15.attn.query.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.15.attn.query.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.15.attn.value.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.15.attn.value.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.15.ls1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.15.ls2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.15.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.15.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.15.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.15.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.15.norm1.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.15.norm1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.15.norm2.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.15.norm2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.16.attn.key.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.16.attn.key.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.16.attn.out.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.16.attn.out.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.16.attn.query.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.16.attn.query.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.16.attn.value.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.16.attn.value.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.16.ls1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.16.ls2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.16.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.16.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.16.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.16.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.16.norm1.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.16.norm1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.16.norm2.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.16.norm2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.17.attn.key.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.17.attn.key.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.17.attn.out.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.17.attn.out.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.17.attn.query.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.17.attn.query.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.17.attn.value.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.17.attn.value.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.17.ls1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.17.ls2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.17.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.17.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.17.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.17.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.17.norm1.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.17.norm1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.17.norm2.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.17.norm2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.18.attn.key.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.18.attn.key.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.18.attn.out.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.18.attn.out.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.18.attn.query.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.18.attn.query.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.18.attn.value.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.18.attn.value.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.18.ls1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.18.ls2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.18.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.18.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.18.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.18.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.18.norm1.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.18.norm1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.18.norm2.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.18.norm2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.19.attn.key.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.19.attn.key.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.19.attn.out.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.19.attn.out.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.19.attn.query.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.19.attn.query.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.19.attn.value.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.19.attn.value.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.19.ls1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.19.ls2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.19.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.19.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.19.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.19.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.19.norm1.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.19.norm1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.19.norm2.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.19.norm2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.2.attn.key.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.2.attn.key.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.2.attn.out.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.2.attn.out.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.2.attn.query.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.2.attn.query.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.2.attn.value.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.2.attn.value.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.2.ls1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.2.ls2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.2.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.2.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.2.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.2.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.2.norm1.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.2.norm1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.2.norm2.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.2.norm2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.20.attn.key.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.20.attn.key.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.20.attn.out.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.20.attn.out.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.20.attn.query.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.20.attn.query.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.20.attn.value.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.20.attn.value.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.20.ls1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.20.ls2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.20.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.20.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.20.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.20.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.20.norm1.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.20.norm1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.20.norm2.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.20.norm2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.21.attn.key.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.21.attn.key.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.21.attn.out.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.21.attn.out.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.21.attn.query.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.21.attn.query.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.21.attn.value.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.21.attn.value.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.21.ls1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.21.ls2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.21.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.21.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.21.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.21.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.21.norm1.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.21.norm1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.21.norm2.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.21.norm2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.22.attn.key.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.22.attn.key.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.22.attn.out.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.22.attn.out.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.22.attn.query.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.22.attn.query.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.22.attn.value.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.22.attn.value.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.22.ls1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.22.ls2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.22.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.22.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.22.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.22.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.22.norm1.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.22.norm1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.22.norm2.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.22.norm2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.23.attn.key.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.23.attn.key.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.23.attn.out.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.23.attn.out.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.23.attn.query.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.23.attn.query.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.23.attn.value.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.23.attn.value.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.23.ls1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.23.ls2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.23.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.23.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.23.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.23.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.23.norm1.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.23.norm1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.23.norm2.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.23.norm2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.24.attn.key.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.24.attn.key.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.24.attn.out.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.24.attn.out.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.24.attn.query.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.24.attn.query.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.24.attn.value.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.24.attn.value.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.24.ls1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.24.ls2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.24.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.24.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.24.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.24.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.24.norm1.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.24.norm1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.24.norm2.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.24.norm2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.25.attn.key.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.25.attn.key.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.25.attn.out.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.25.attn.out.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.25.attn.query.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.25.attn.query.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.25.attn.value.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.25.attn.value.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.25.ls1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.25.ls2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.25.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.25.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.25.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.25.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.25.norm1.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.25.norm1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.25.norm2.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.25.norm2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.26.attn.key.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.26.attn.key.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.26.attn.out.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.26.attn.out.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.26.attn.query.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.26.attn.query.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.26.attn.value.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.26.attn.value.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.26.ls1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.26.ls2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.26.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.26.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.26.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.26.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.26.norm1.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.26.norm1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.26.norm2.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.26.norm2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.27.attn.key.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.27.attn.key.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.27.attn.out.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.27.attn.out.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.27.attn.query.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.27.attn.query.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.27.attn.value.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.27.attn.value.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.27.ls1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.27.ls2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.27.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.27.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.27.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.27.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.27.norm1.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.27.norm1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.27.norm2.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.27.norm2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.28.attn.key.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.28.attn.key.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.28.attn.out.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.28.attn.out.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.28.attn.query.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.28.attn.query.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.28.attn.value.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.28.attn.value.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.28.ls1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.28.ls2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.28.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.28.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.28.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.28.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.28.norm1.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.28.norm1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.28.norm2.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.28.norm2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.29.attn.key.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.29.attn.key.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.29.attn.out.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.29.attn.out.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.29.attn.query.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.29.attn.query.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.29.attn.value.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.29.attn.value.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.29.ls1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.29.ls2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.29.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.29.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.29.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.29.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.29.norm1.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.29.norm1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.29.norm2.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.29.norm2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.3.attn.key.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.3.attn.key.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.3.attn.out.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.3.attn.out.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.3.attn.query.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.3.attn.query.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.3.attn.value.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.3.attn.value.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.3.ls1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.3.ls2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.3.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.3.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.3.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.3.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.3.norm1.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.3.norm1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.3.norm2.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.3.norm2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.30.attn.key.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.30.attn.key.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.30.attn.out.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.30.attn.out.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.30.attn.query.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.30.attn.query.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.30.attn.value.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.30.attn.value.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.30.ls1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.30.ls2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.30.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.30.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.30.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.30.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.30.norm1.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.30.norm1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.30.norm2.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.30.norm2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.31.attn.key.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.31.attn.key.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.31.attn.out.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.31.attn.out.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.31.attn.query.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.31.attn.query.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.31.attn.value.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.31.attn.value.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.31.ls1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.31.ls2.weight": "model-00002-of-00002.safetensors",
+    "image_encoder.blocks.31.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.31.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.31.mlp.output_layer.bias": "model-00002-of-00002.safetensors",
+    "image_encoder.blocks.31.mlp.output_layer.weight": "model-00002-of-00002.safetensors",
+    "image_encoder.blocks.31.norm1.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.31.norm1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.31.norm2.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.31.norm2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.4.attn.key.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.4.attn.key.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.4.attn.out.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.4.attn.out.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.4.attn.query.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.4.attn.query.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.4.attn.value.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.4.attn.value.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.4.ls1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.4.ls2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.4.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.4.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.4.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.4.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.4.norm1.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.4.norm1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.4.norm2.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.4.norm2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.5.attn.key.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.5.attn.key.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.5.attn.out.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.5.attn.out.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.5.attn.query.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.5.attn.query.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.5.attn.value.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.5.attn.value.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.5.ls1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.5.ls2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.5.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.5.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.5.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.5.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.5.norm1.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.5.norm1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.5.norm2.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.5.norm2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.6.attn.key.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.6.attn.key.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.6.attn.out.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.6.attn.out.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.6.attn.query.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.6.attn.query.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.6.attn.value.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.6.attn.value.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.6.ls1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.6.ls2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.6.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.6.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.6.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.6.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.6.norm1.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.6.norm1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.6.norm2.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.6.norm2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.7.attn.key.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.7.attn.key.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.7.attn.out.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.7.attn.out.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.7.attn.query.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.7.attn.query.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.7.attn.value.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.7.attn.value.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.7.ls1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.7.ls2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.7.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.7.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.7.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.7.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.7.norm1.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.7.norm1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.7.norm2.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.7.norm2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.8.attn.key.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.8.attn.key.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.8.attn.out.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.8.attn.out.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.8.attn.query.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.8.attn.query.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.8.attn.value.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.8.attn.value.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.8.ls1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.8.ls2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.8.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.8.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.8.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.8.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.8.norm1.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.8.norm1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.8.norm2.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.8.norm2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.9.attn.key.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.9.attn.key.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.9.attn.out.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.9.attn.out.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.9.attn.query.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.9.attn.query.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.9.attn.value.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.9.attn.value.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.9.ls1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.9.ls2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.9.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.9.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.9.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.9.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.9.norm1.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.9.norm1.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.9.norm2.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.blocks.9.norm2.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.cls_token": "model-00001-of-00002.safetensors",
+    "image_encoder.norm.bias": "model-00002-of-00002.safetensors",
+    "image_encoder.norm.weight": "model-00002-of-00002.safetensors",
+    "image_encoder.patch_embed.bias": "model-00001-of-00002.safetensors",
+    "image_encoder.patch_embed.weight": "model-00001-of-00002.safetensors",
+    "image_encoder.pos_embed": "model-00001-of-00002.safetensors",
+    "image_pooler.image_latents": "model-00002-of-00002.safetensors",
+    "image_pooler.pooler.linear1.bias": "model-00002-of-00002.safetensors",
+    "image_pooler.pooler.linear1.weight": "model-00002-of-00002.safetensors",
+    "image_pooler.pooler.linear2.bias": "model-00002-of-00002.safetensors",
+    "image_pooler.pooler.linear2.weight": "model-00002-of-00002.safetensors",
+    "image_pooler.pooler.multihead_attn.in_proj_bias": "model-00002-of-00002.safetensors",
+    "image_pooler.pooler.multihead_attn.in_proj_weight": "model-00002-of-00002.safetensors",
+    "image_pooler.pooler.multihead_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "image_pooler.pooler.multihead_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "image_pooler.pooler.norm1.bias": "model-00002-of-00002.safetensors",
+    "image_pooler.pooler.norm1.weight": "model-00002-of-00002.safetensors",
+    "image_pooler.pooler.norm2.bias": "model-00002-of-00002.safetensors",
+    "image_pooler.pooler.norm2.weight": "model-00002-of-00002.safetensors",
+    "image_pooler.pooler.norm3.bias": "model-00002-of-00002.safetensors",
+    "image_pooler.pooler.norm3.weight": "model-00002-of-00002.safetensors",
+    "image_pooler.pooler.self_attn.in_proj_bias": "model-00002-of-00002.safetensors",
+    "image_pooler.pooler.self_attn.in_proj_weight": "model-00002-of-00002.safetensors",
+    "image_pooler.pooler.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "image_pooler.pooler.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "image_pooler.projection.bias": "model-00002-of-00002.safetensors",
+    "image_pooler.projection.weight": "model-00002-of-00002.safetensors",
+    "text_decoder.lm_head.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.0.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.0.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.0.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.1.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.1.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.1.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.10.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.10.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.10.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.11.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.11.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.11.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.12.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.12.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.12.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.13.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.13.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.13.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.14.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.14.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.14.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.15.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.15.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.15.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.16.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.16.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.16.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.17.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.17.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.17.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.18.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.18.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.18.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.19.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.19.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.19.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.2.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.2.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.2.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.20.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.20.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.20.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.20.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.20.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.20.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.21.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.21.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.21.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.21.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.21.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.21.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.21.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.21.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.21.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.21.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.21.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.21.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.22.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.22.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.22.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.22.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.22.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.22.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.22.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.22.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.22.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.22.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.22.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.22.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.23.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.23.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.23.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.23.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.23.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.23.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.23.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.23.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.23.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.23.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.23.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.23.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.3.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.3.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.3.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.4.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.4.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.4.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.5.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.5.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.5.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.6.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.6.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.6.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.7.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.7.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.7.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.8.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.8.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.8.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.9.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.9.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.9.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "text_decoder.model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "text_decoder.model.norm.weight": "model-00001-of-00002.safetensors"
+  }
+}

modeling_uform_gen.py

@@ -0,0 +1,245 @@
+from typing import List, Optional, Tuple, Union
+
+from .configuration_uform_gen import VLMConfig
+
+import torch
+import torch.nn.functional as F
+from torch.utils.checkpoint import checkpoint
+from torch import nn
+
+from transformers.modeling_outputs import CausalLMOutputWithPast
+from transformers.modeling_utils import PreTrainedModel
+from transformers.models.auto.modeling_auto import AutoModelForCausalLM, AutoModel
+from transformers import AutoConfig
+from transformers.utils import logging
+
+from .vision_encoder import VisionEncoder
+
+
+class ImageFeaturesPooler(nn.Module):
+    def __init__(self, config, text_config):
+        super().__init__()
+        self.pooler = nn.TransformerDecoderLayer(
+            config.image_encoder_hidden_size,
+            config.image_pooler_num_attn_heads,
+            config.image_pooler_intermediate_size,
+            activation=nn.functional.silu,
+            batch_first=True,
+            norm_first=True,
+        )
+        self.image_latents = nn.Parameter(
+            torch.randn(1, config.num_image_latents, config.image_encoder_hidden_size)
+            * config.initializer_range**0.5
+        )
+        self.projection = nn.Linear(config.image_encoder_hidden_size, text_config.hidden_size)
+
+    def forward(self, features):
+        features = self.pooler(
+            self.image_latents.expand(features.size(0), -1, -1), features
+        )
+
+        return self.projection(features)
+
+
+class VLMPreTrainedModel(PreTrainedModel):
+    config_class = VLMConfig
+    base_model_prefix = "vlm"
+    supports_gradient_checkpointing = True
+    _no_split_modules = []
+    _skip_keys_device_placement = "past_key_values"
+
+    def _init_weights(self, module):
+        pass
+
+    def _initialize_weights(self, module):
+        pass
+
+
+class VLMForCausalLM(VLMPreTrainedModel):
+    def __init__(self, config: VLMConfig):
+        super().__init__(config)
+
+        self.config = config
+        self.text_config = AutoConfig.from_pretrained(
+            config.text_decoder_name_or_path,
+            trust_remote_code=True
+        )
+        
+        self.text_decoder = AutoModelForCausalLM.from_config(
+            self.text_config,
+            trust_remote_code=True
+        )
+
+        self.image_encoder = VisionEncoder(
+            config.image_encoder_hidden_size,
+            config.image_encoder_patch_size,
+            config.image_encoder_num_layers,
+            config.image_encoder_num_heads,
+        )
+
+        self.image_pooler = ImageFeaturesPooler(config, self.text_config)
+
+    def get_input_embeddings(self):
+        return self.text_decoder.get_input_embeddings()
+
+    def set_input_embeddings(self, value):
+        self.text_decoder.set_input_embeddings(value)
+
+    def get_images_embeddings(self, images):
+        features = self.image_encoder(images)
+        return self.image_pooler(features)
+
+    def gather_continuous_embeddings(
+        self,
+        input_ids: torch.Tensor,
+        word_embeddings: torch.Tensor,
+        image_embeddings: torch.Tensor
+    ) -> torch.Tensor:
+        
+        start_indices = (input_ids == self.config.image_token_id).nonzero()[:, 1]
+        embeddings = []
+        for sample_idx, start_idx in enumerate(start_indices.tolist()):
+            embeddings.append(
+                torch.cat(
+                    (
+                        word_embeddings[sample_idx, :start_idx],
+                        image_embeddings[sample_idx],
+                        word_embeddings[sample_idx, start_idx + 1 :],
+                    ),
+                    dim=0,
+                )
+            )
+
+        return torch.stack(embeddings, dim=0)
+
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        images: torch.Tensor = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        use_cache: Optional[bool] = None,
+        labels: Optional[torch.Tensor] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None
+    ) -> Union[dict, Tuple, CausalLMOutputWithPast]:
+        output_attentions = (
+            output_attentions
+            if output_attentions is not None
+            else self.config.output_attentions
+        )
+        output_hidden_states = (
+            output_hidden_states
+            if output_hidden_states is not None
+            else self.config.output_hidden_states
+        )
+        use_cache = use_cache if use_cache is not None else self.config.use_cache
+
+        return_dict = (
+            return_dict if return_dict is not None else self.config.use_return_dict
+        )
+
+        if input_ids is not None and inputs_embeds is not None:
+            raise ValueError(
+                "You cannot specify both input_ids and inputs_embeds at the same time"
+            )
+        elif input_ids is None and inputs_embeds is None:
+            raise ValueError("You have to specify either input_is or inputs_embeds")
+
+        if inputs_embeds is None and past_key_values is None:
+            inputs_embeds = self.get_input_embeddings()(input_ids)
+
+            if images is not None:
+                image_embeds = self.get_images_embeddings(images)
+                inputs_embeds = self.gather_continuous_embeddings(
+                    input_ids,
+                    inputs_embeds,
+                    image_embeds
+                )
+
+        if position_ids is None:
+            seq_length = (
+                inputs_embeds.shape[1]
+                if inputs_embeds is not None
+                else input_ids.shape[1]
+            )
+            past_key_values_length = 0
+
+            if past_key_values is not None:
+                past_key_values_length = past_key_values[0][0].shape[2]
+
+            device = input_ids.device if input_ids is not None else inputs_embeds.device
+            position_ids = torch.arange(
+                past_key_values_length,
+                seq_length + past_key_values_length,
+                dtype=torch.long,
+                device=device,
+            )
+            position_ids = position_ids.unsqueeze(0)
+
+        outputs = self.text_decoder(
+            inputs_embeds=inputs_embeds,
+            input_ids=input_ids if past_key_values is not None else None,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            use_cache=use_cache,
+            return_dict=return_dict,
+        )
+
+        return outputs
+
+    def prepare_inputs_for_generation(
+        self,
+        input_ids,
+        images=None,
+        past_key_values=None,
+        attention_mask=None,
+        inputs_embeds=None,
+        **kwargs,
+    ):
+        if past_key_values:
+            input_ids = input_ids[:, -1:]
+
+        position_ids = kwargs.get("position_ids", None)
+        if attention_mask is not None and position_ids is None:
+            # create position_ids on the fly for batch generation
+            position_ids = attention_mask.long().cumsum(-1) - 1
+            position_ids.masked_fill_(attention_mask == 0, 1)
+            if past_key_values:
+                position_ids = position_ids[:, -1].unsqueeze(-1)
+
+        # if `inputs_embeds` are passed, we only want to use them in the 1st generation step
+        if inputs_embeds is not None and past_key_values is None:
+            model_inputs = {"inputs_embeds": inputs_embeds}
+            n_samples = inputs_embeds.shape[0]
+        else:
+            model_inputs = {"input_ids": input_ids}
+            n_samples = input_ids.shape[0]
+
+        if images is not None:
+            model_inputs["images"] = images
+
+        model_inputs.update(
+            {
+                "position_ids": position_ids,
+                "past_key_values": past_key_values,
+                "use_cache": kwargs.get("use_cache"),
+                "attention_mask": attention_mask,
+                "images": images if past_key_values is None else None,
+            }
+        )
+        return model_inputs
+
+    @classmethod
+    def from_config(cls, config, **kwargs):
+        return cls._from_config(config, **kwargs)
+
+
+VLMConfig.register_for_auto_class()
+VLMForCausalLM.register_for_auto_class("AutoModel")

processing_uform_gen.py

@@ -0,0 +1,181 @@
+from functools import partial
+
+import torch
+import torch.nn.functional as F
+from transformers.processing_utils import ProcessorMixin
+from transformers.image_processing_utils import BaseImageProcessor
+from transformers import AutoTokenizer, AutoConfig
+from transformers import BatchFeature
+
+from PIL import Image
+from torchvision.transforms import (
+    Compose,
+    Normalize,
+    Resize,
+    ToTensor
+)
+
+
+IMAGENET_MEAN = (0.48145466, 0.4578275, 0.40821073)
+IMAGENET_STD = (0.26862954, 0.26130258, 0.27577711)
+
+
+def convert_to_rgb(x):
+    return x.convert("RGB")
+
+
+def expand2square(image, background_color):
+    width, height = image.size
+    if width == height:
+        return image
+    elif width > height:
+        result = Image.new(image.mode, (width, width), background_color)
+        result.paste(image, (0, (width - height) // 2))
+        return result
+    else:
+        result = Image.new(image.mode, (height, height), background_color)
+        result.paste(image, ((height - width) // 2, 0))
+        return result
+
+
+class ImageProcessor(BaseImageProcessor):
+    def __init__(
+        self,
+        image_size: int,
+        **kwargs
+    ):
+        super().__init__(**kwargs)
+        self.transform = Compose(
+            [
+                convert_to_rgb,
+                partial(
+                    expand2square,
+                    background_color=tuple(int(255 * v) for v in IMAGENET_MEAN)
+                ),
+                Resize(image_size),
+                ToTensor(),
+                Normalize(
+                    mean=IMAGENET_MEAN,
+                    std=IMAGENET_STD,
+                ),
+            ]
+        )
+    
+    def preprocess(
+        self,
+        image: Image
+    ):
+        return self.transform(image)
+
+    def __repr__(self):
+        return repr(self.transform)
+
+
+class VLMProcessor(ProcessorMixin):
+    def __init__(self, config):
+        self.config = config
+        self.image_size = config.image_size
+        
+        self.feature_extractor = ImageProcessor(self.image_size)
+        self.tokenizer = AutoTokenizer.from_pretrained(
+            config.text_decoder_name_or_path, additional_special_tokens=["<image>"]
+        )
+        self.tokenizer.chat_template = "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}"
+        self.num_image_latents = config.num_image_latents
+        # super().__init__(self.image_processor, self.tokenizer)
+
+    def __call__(
+        self, text=None, images=None, **kwargs
+    ):
+        if text is not None:
+            if isinstance(text, str):
+                text = [text]
+
+            tokenized_texts = []
+            for t in text:
+                messages = [
+                    {"role": "system", "content": "You are a helpful assistant."},
+                    {"role": "user", "content": f" <image> {t}"},
+                ]
+                tokenized_prompt = self.tokenizer.apply_chat_template(
+                    messages, add_generation_prompt=True, return_tensors="pt"
+                )
+
+                tokenized_texts.append(tokenized_prompt)
+
+            max_len = max(len(t[0]) for t in tokenized_texts)
+            input_ids = torch.full(
+                (len(tokenized_texts), max_len),
+                fill_value=self.tokenizer.pad_token_id,
+                dtype=torch.int64,
+            )
+            attention_mask = torch.full(
+                (len(tokenized_texts), max_len), fill_value=0, dtype=torch.int64
+            )
+
+            for i, tokens in enumerate(tokenized_texts):
+                input_ids[i, -len(tokens[0]) :] = tokens[0]
+                attention_mask[i, -len(tokens[0]) :] = 1
+
+            attention_mask = F.pad(
+                attention_mask, pad=(0, self.num_image_latents - 1), value=1
+            )
+
+            encoding = BatchFeature(
+                data={"input_ids": input_ids, "attention_mask": attention_mask}
+            )
+
+        if images is not None:
+            if isinstance(images, (list, tuple)):
+                image_features = torch.empty(
+                    (len(images), 3, self.image_size , self.image_size),
+                    dtype=torch.float32,
+                )
+
+                for i, image in enumerate(images):
+                    image_features[i] = self.feature_extractor(image)
+
+            else:
+                image_features = self.image_processor(images).unsqueeze(0)
+
+        if text is not None and images is not None:
+            encoding["images"] = image_features
+            return encoding
+
+        elif text is not None:
+            return encoding
+
+        else:
+            return BatchFeature(
+                data={
+                    "images": image_features,
+                },
+                tensor_type=return_tensors,
+            )
+
+    def batch_decode(self, *args, **kwargs):
+        """
+        This method forwards all its arguments to CLIPTokenizerFast's [`~PreTrainedTokenizer.batch_decode`]. Please
+        refer to the docstring of this method for more information.
+        """
+        return self.tokenizer.batch_decode(*args, **kwargs)
+
+    def decode(self, *args, **kwargs):
+        """
+        This method forwards all its arguments to CLIPTokenizerFast's [`~PreTrainedTokenizer.decode`]. Please refer to
+        the docstring of this method for more information.
+        """
+        return self.tokenizer.decode(*args, **kwargs)
+
+    @classmethod
+    def from_pretrained(
+        cls,
+        pretrained_model_name_or_path,
+        trust_remote_code=False,
+        **kwargs
+    ):
+        config = AutoConfig.from_pretrained(
+            pretrained_model_name_or_path,
+            trust_remote_code=trust_remote_code
+        )
+        return cls(config)

vision_encoder.py

@@ -0,0 +1,182 @@
+import torch.nn as nn
+import torch.nn.functional as F
+import torch
+from torch import Tensor
+from typing import Optional
+
+class Attention(nn.Module):
+
+    def __init__(
+        self,
+        dim: int,
+        num_heads: int,
+        dropout_prob: float = 0
+    ):
+        super().__init__()
+        
+        self.use_sdp = int(torch.__version__[0]) > 1
+
+        self.query = nn.Linear(dim, dim)
+        self.key = nn.Linear(dim, dim)
+        self.value = nn.Linear(dim, dim)
+        self.out = nn.Linear(dim, dim)
+
+        self.dropout_prob = dropout_prob
+        self.num_heads = num_heads
+        self.head_dim = dim // num_heads
+        self.scale = self.head_dim**-0.5
+
+    def forward(
+        self,
+        x: Tensor,
+        attn_mask: Optional[Tensor] = None,
+        context: Optional[Tensor] = None,
+        is_causal: bool = False,
+    ) -> Tensor:
+
+        query = self.reshape(self.query(x))
+        key = self.reshape(self.key(x if context is None else context))
+        value = self.reshape(self.value(x if context is None else context))
+
+        if self.use_sdp:
+            x = F.scaled_dot_product_attention(
+                query,
+                key,
+                value,
+                attn_mask,
+                dropout_p=self.dropout_prob if self.training else 0,
+                is_causal=is_causal,
+            )
+        else:
+            attn = query @ key.transpose(-2, -1) * self.scale
+            if attn_mask is not None:
+                attn += attn_mask
+
+            attn = attn.softmax(dim=-1)
+            x = attn @ value
+
+        return self.out(x.transpose(2, 1).flatten(2))
+
+    def reshape(self, x: Tensor) -> Tensor:
+        batch_size, seq_len, _ = x.shape
+        x = x.view(batch_size, seq_len, self.num_heads, self.head_dim)
+        return x.transpose(2, 1)
+
+
+class MLP(nn.Module):
+   
+    def __init__(
+        self,
+        dim: int,
+        dim_expand_factor: int = 4
+    ):
+        super().__init__()
+
+        self.hidden_layer = nn.Linear(dim, dim * dim_expand_factor)
+        self.output_layer = nn.Linear(dim * dim_expand_factor, dim)
+
+    def forward(self, x: Tensor) -> Tensor:
+        x = F.gelu(self.hidden_layer(x))
+        return self.output_layer(x)
+
+
+class LayerScale(nn.Module):
+
+    def __init__(
+        self,
+        dim: int,
+        init_values: float = 1e-5,
+        inplace: bool = False
+    ):
+        super().__init__()
+        self.weight = nn.Parameter(init_values * torch.ones(dim))
+        self.inplace = inplace
+
+    def forward(self, x: Tensor) -> Tensor:
+        return x.mul_(self.weight) if self.inplace else x * self.weight
+
+
+class VisionEncoderBlock(nn.Module):
+
+    def __init__(
+        self,
+        dim: int,
+        num_heads: int
+    ):
+        super().__init__()
+        self.norm1 = nn.LayerNorm(dim, eps=1e-6)
+        self.attn = Attention(dim, num_heads)
+        self.ls1 = LayerScale(dim)
+
+        self.norm2 = nn.LayerNorm(dim, eps=1e-6)
+        self.mlp = MLP(dim)
+        self.ls2 = LayerScale(dim)
+
+    def forward(self, x: Tensor) -> Tensor:
+        x = x + self.ls1(self.attn(self.norm1(x)))
+        x = x + self.ls2(self.mlp(self.norm2(x)))
+        return x
+
+
+class VisionEncoder(nn.Module):
+    
+    def __init__(
+        self,
+        dim: int,
+        patch_size: int,
+        num_layers: int,
+        num_heads: int,
+    ):
+        super().__init__()
+
+        self.n_patch = 224 // patch_size
+        self.seq_len = self.n_patch ** 2
+        self.patch_size = patch_size
+
+        self.patch_embed = nn.Conv2d(3, dim, patch_size, patch_size)
+        self.pos_embed = nn.Parameter(torch.randn(1, self.seq_len, dim) * 0.02)
+        self.cls_token = nn.Parameter(torch.zeros(1, 1, dim))
+        self.interpolate_offset = 0.1
+        self.interpolate_antialias = False
+
+        self.blocks = nn.Sequential(
+            *[
+                VisionEncoderBlock(dim, num_heads)
+                for _ in range(num_layers)
+            ]
+        )
+
+        self.norm = nn.LayerNorm(dim, eps=1e-6)
+
+    def interpolate_pos_encoding(self, x, h, w):
+        previous_dtype = x.dtype
+
+        if x.shape[1] == self.seq_len and w == h:
+            return self.pos_embed
+        
+        pos_embed = self.pos_embed.float()
+
+        dim = x.shape[-1]
+        w0 = w // self.patch_size
+        h0 = h // self.patch_size
+        # we add a small number to avoid floating point error in the interpolation
+        # see discussion at https://github.com/facebookresearch/dino/issues/8
+        w0, h0 = w0 + self.interpolate_offset, h0 + self.interpolate_offset
+        sx, sy = float(w0) / self.n_patch, float(h0) / self.n_patch
+
+        pos_embed = nn.functional.interpolate(
+            pos_embed.reshape(1, self.n_patch, self.n_patch, dim).permute(0, 3, 1, 2),
+            scale_factor=(sy, sx),
+            mode="bicubic",
+            antialias=self.interpolate_antialias,
+        )
+
+        return pos_embed.to(previous_dtype).flatten(start_dim=2).transpose(2, 1)
+
+    def forward(self, x: Tensor) -> Tensor:
+        h, w = x.shape[2:]
+        x = self.patch_embed(x).flatten(start_dim=2).transpose(2, 1)
+        x = x + self.interpolate_pos_encoding(x, h, w)
+        x = torch.cat((self.cls_token.expand(x.shape[0], -1, -1), x), dim=1)
+        x = self.blocks(x)
+        return self.norm(x)