Model weights

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer_config.json filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

config.json

@@ -0,0 +1,73 @@
+{
+  "architectures": [
+    "Custom_MPTForCausalLM"
+  ],
+  "attn_config": {
+    "alibi": true,
+    "alibi_bias_max": 8,
+    "attn_impl": "torch",
+    "attn_pdrop": 0,
+    "attn_type": "multihead_attention",
+    "attn_uses_sequence_id": false,
+    "clip_qkv": null,
+    "prefix_lm": false,
+    "qk_gn": false,
+    "qk_ln": false,
+    "rope": false,
+    "rope_dail_config": {
+      "pos_idx_in_fp32": true,
+      "type": "original",
+      "xpos_scale_base": 512
+    },
+    "rope_hf_config": {
+      "factor": 1.0,
+      "type": "no_scaling"
+    },
+    "rope_impl": "dail",
+    "rope_theta": 10000,
+    "sliding_window_size": -1,
+    "softmax_scale": null
+  },
+  "auto_map": {
+    "AutoConfig": "mpt-7b--configuration_mpt.MPTConfig",
+    "AutoModelForCausalLM": "mpt-7b--modeling_mpt.MPTForCausalLM"
+  },
+  "bos_token_id": 0,
+  "d_model": 360,
+  "emb_pdrop": 0,
+  "embedding_fraction": 1.0,
+  "eos_token_id": 1,
+  "expansion_ratio": 5,
+  "fc_type": "torch",
+  "ffn_config": {
+    "fc_type": "torch",
+    "ffn_type": "mptmlp"
+  },
+  "init_config": {
+    "emb_init_std": null,
+    "emb_init_uniform_lim": null,
+    "fan_mode": "fan_in",
+    "init_div_is_residual": true,
+    "init_gain": 0,
+    "init_nonlinearity": "relu",
+    "init_std": 0.02,
+    "name": "kaiming_normal_",
+    "verbose": 0
+  },
+  "init_device": "cuda",
+  "learned_pos_emb": false,
+  "logit_scale": null,
+  "max_seq_len": 600,
+  "model_type": "mpt",
+  "n_heads": 36,
+  "n_layers": 36,
+  "no_bias": true,
+  "norm_type": "low_precision_layernorm",
+  "resid_pdrop": 0,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.35.0",
+  "use_cache": false,
+  "use_pad_tok_in_ffn": true,
+  "verbose": 0,
+  "vocab_size": 63740
+}

handler.py

@@ -0,0 +1,195 @@
+from typing import Dict, List, Any
+import os
+import torch
+from transformers import AutoTokenizer, AutoModelForCausalLM
+from transformers import PreTrainedTokenizerFast
+from transformers import GenerationConfig
+import transformers 
+import pandas as pd
+import time
+from precious3_gpt_multi_model import Custom_MPTForCausalLM
+
+
+emb_gpt_genes = pd.read_pickle('./multi-modal-data/emb_gpt_genes.pickle')
+emb_hgt_genes = pd.read_pickle('./multi-modal-data/emb_hgt_genes.pickle')
+
+
+def create_prompt(prompt_config):
+
+    prompt = "[BOS]"
+    
+    multi_modal_prefix = '<modality0><modality1><modality2><modality3>'*3
+    
+    for k, v in prompt_config.items():
+        if k=='instruction':
+            prompt+=f"<{v}>"
+        elif k=='up':
+            prompt+=f'{multi_modal_prefix}<{k}>{v}</{k}>' if isinstance(v, str) else f'{multi_modal_prefix}<{k}>{" ".join(v)} </{k}>'
+        elif k=='down':
+            prompt+=f'{multi_modal_prefix}<{k}>{v}</{k}>' if isinstance(v, str) else f'{multi_modal_prefix}<{k}>{" ".join(v)} </{k}>'
+        else:
+            prompt+=f'<{k}>{v}</{k}>' if isinstance(v, str) else f'<{k}>{" ".join(v)} </{k}>'
+    return prompt
+
+def custom_generate(input_ids, 
+                    acc_embs_up_kg_mean, 
+                    acc_embs_down_kg_mean, 
+                    acc_embs_up_txt_mean, 
+                    acc_embs_down_txt_mean,
+                    device,
+                    max_new_tokens, 
+                    num_return_sequences,
+                    temperature=0.8, 
+                    top_p=0.2, top_k=3550, n_next_tokens=50,
+                    unique_compounds):
+    torch.manual_seed(137)
+    
+    # Set parameters
+    # temperature - Higher value for more randomness, lower for more control
+    # top_p - Probability threshold for nucleus sampling (aka top-p sampling)
+    # top_k - Ignore logits below the top-k value to reduce randomness (if non-zero)
+    # n_next_tokens - Number of top next tokens when predicting compounds
+
+    modality0_emb =  torch.unsqueeze(torch.from_numpy(acc_embs_up_kg_mean), 0).to(device) # torch.from_numpy(efo_embeddings['EFO_0002618']).type(torch.bfloat16).to(device)
+    modality1_emb =  torch.unsqueeze(torch.from_numpy(acc_embs_down_kg_mean), 0).to(device)
+    modality2_emb =  torch.unsqueeze(torch.from_numpy(acc_embs_up_txt_mean), 0).to(device) # torch.from_numpy(efo_embeddings['EFO_0002618']).type(torch.bfloat16).to(device)
+    modality3_emb =  torch.unsqueeze(torch.from_numpy(acc_embs_down_txt_mean), 0).to(device)
+
+
+    # Generate sequences
+    outputs = []
+    next_token_compounds = [] 
+
+    for _ in range(num_return_sequences):
+        start_time = time.time()
+        generated_sequence = []
+        current_token = input_ids.clone()
+
+        for _ in range(max_new_tokens):  # Maximum length of generated sequence
+            # Forward pass through the model
+            logits = model.forward(input_ids=current_token, 
+                                   modality0_emb=modality0_emb, # torch.tensor(efo_embeddings['EFO_0002618'], dtype=torch.bfloat16).to(device), 
+                                   modality0_token_id=62191, 
+                                   modality1_emb=modality1_emb, # torch.tensor(efo_embeddings['EFO_0002618'], dtype=torch.bfloat16).to(device), 
+                                   modality1_token_id=62192,
+                                   modality2_emb=modality2_emb, # torch.tensor(efo_embeddings['EFO_0002618'], dtype=torch.bfloat16).to(device), 
+                                   modality2_token_id=62193, 
+                                   modality3_emb=modality3_emb, # torch.tensor(efo_embeddings['EFO_0002618'], dtype=torch.bfloat16).to(device), 
+                                   modality3_token_id=62194)[0]
+
+            # Apply temperature to logits
+            if temperature != 1.0:
+                logits = logits / temperature
+
+            # Apply top-p sampling (nucleus sampling)
+            sorted_logits, sorted_indices = torch.sort(logits, descending=True)
+            cumulative_probs = torch.cumsum(torch.softmax(sorted_logits, dim=-1), dim=-1)
+            sorted_indices_to_remove = cumulative_probs > top_p
+
+            if top_k > 0:
+                sorted_indices_to_remove[..., top_k:] = 1
+
+            # Set the logit values of the removed indices to a very small negative value
+            inf_tensor = torch.tensor(float("-inf")).type(torch.bfloat16).to(logits.device)
+
+            logits = logits.where(sorted_indices_to_remove, inf_tensor)
+
+
+            # Sample the next token
+            if current_token[0][-1] == tokenizer.encode('<drug>')[0]:
+                next_token_compounds.append(torch.topk(torch.softmax(logits, dim=-1)[0][len(current_token[0])-1, :].flatten(), 50).indices)
+
+            next_token = torch.multinomial(torch.softmax(logits, dim=-1)[0], num_samples=1)[len(current_token[0])-1, :].unsqueeze(0)
+
+
+            # Append the sampled token to the generated sequence
+            generated_sequence.append(next_token.item())
+
+            Stop generation if an end token is generated
+            if next_token == tokenizer.eos_token_id:
+                break
+
+            # Prepare input for the next iteration
+            current_token = torch.cat((current_token, next_token), dim=-1)
+        print(time.time()-start_time)
+        outputs.append(generated_sequence)
+    return outputs, next_token_compounds
+
+
+def get_predicted_compounds(input_ids, generation_output, tokenizer, p3_compounds):
+    id_4_drug_token = list(generation_output.sequences[0][len(input_ids[0]):]).index(tokenizer.convert_tokens_to_ids(['<drug>'])[0])
+    id_4_drug_token += 1
+    print('This is token index where drug should be predicted: ', id_4_drug_token)
+
+    values, indices = torch.topk(generation_output["scores"][id_4_drug_token].view(-1), k=50)
+    indices_decoded = tokenizer.decode(indices, skip_special_tokens=True)
+
+    predicted_compound = indices_decoded.split('  ')
+    predicted_compound = [i.strip() for i in predicted_compound]
+    
+    valid_compounds = sorted(set(predicted_compound) & set(p3_compounds), key = predicted_compound.index)
+    print(f"Model predicted {len(predicted_compound)} tokens. Valid compounds {len(valid_compounds)}")
+    return valid_compounds
+
+
+class EndpointHandler:
+    def __init__(self, path=""):
+        # load model and processor from path
+        self.model = Custom_MPTForCausalLM.from_pretrained(path, torch_dtype=torch.bfloat16).to('cuda')
+        self.tokenizer = PreTrainedTokenizerFast(tokenizer_file = os.path.join(path, "tokenizer.json"), unk_token="[UNK]",
+            pad_token="[PAD]",
+            eos_token="[EOS]",
+            bos_token="[BOS]")
+        self.model.config.pad_token_id = self.tokenizer.pad_token_id
+        self.model.config.bos_token_id = self.tokenizer.bos_token_id
+        self.model.config.eos_token_id = self.tokenizer.eos_token_id
+        unique_entities_p3 = pd.read_csv(os.path.join(path, 'all_entities_with_type.csv'))
+        self.unique_compounds_p3 = [i.strip() for i in unique_entities_p3[unique_entities_p3.type=='compound'].entity.to_list()]
+
+    
+
+    def __call__(self, data: Dict[str, Any]) -> Dict[str, str]:
+        """
+        Args:
+            data (:dict:):
+                The payload with the text prompt and generation parameters.
+        """
+
+        inputs = data.pop("inputs", data)
+        parameters = data.pop("parameters", None)
+        mode = data.pop('mode', 'diff2compound')
+        
+        if mode == 'diff2compound':
+            with open('./generation-configs/diff2compound.json', 'r') as f:
+                config_data = json.load(f)
+        else:
+            with open('./generation-configs/diff2compound.json', 'r') as f:
+                config_data = json.load(f)
+                
+        prompt = create_prompt(config_data)
+        
+        inputs = self.tokenizer(inputs, return_tensors="pt")
+        input_ids = inputs["input_ids"].to('cuda')
+
+        ### Generation config  https://huggingface.co/blog/how-to-generate
+        generation_config = GenerationConfig(**parameters, 
+                                             pad_token_id=self.tokenizer.pad_token_id, num_return_sequences=1) 
+        
+        max_new_tokens = self.model.config.max_seq_len - len(input_ids[0]) # max_new_tokens = 560 - len(input_ids[0])
+        
+        torch.manual_seed(137)
+
+        with torch.no_grad():
+            generation_output = self.model.generate(
+                input_ids=input_ids,
+                generation_config=generation_config,
+                return_dict_in_generate=True,
+                output_scores=True,
+                max_new_tokens=max_new_tokens
+            )
+        if mode =='diff2compound':
+            predicted_compounds = get_predicted_compounds(input_ids=input_ids, generation_output=generation_output, tokenizer=self.tokenizer, p3_compounds=self.unique_compounds_p3)
+            output = {'output': predicted_compounds, "mode": mode, 'message': "Done!"}
+        else:
+            output = {'output': [None], "mode": mode, 'message': "Set mode"}
+        return output

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17e1167c39df0e2ac88e4267a13f7b0d4a43b48eb124d7cef8230e6d0e98e257
+size 178841976

precious3_gpt_multi_modal.py

@@ -0,0 +1,340 @@
+from typing import Optional, Tuple, Union, List
+
+from transformers.models.mpt.modeling_mpt import MptBlock, build_mpt_alibi_tensor
+from transformers.modeling_attn_mask_utils import _prepare_4d_causal_attention_mask
+import torch
+import torch.nn as nn
+from torch.nn import CrossEntropyLoss, LayerNorm
+from transformers.models.mpt.modeling_mpt import MptBlock, build_mpt_alibi_tensor
+from transformers.modeling_outputs import CausalLMOutputWithCrossAttentions, CausalLMOutputWithPast, \
+BaseModelOutputWithPastAndCrossAttentions, BaseModelOutputWithPast
+# from transformers import AutoModelForCausalLM, AutoTokenizer, AutoConfig, MptForCausalLM, MptModel
+from transformers import PreTrainedTokenizerFast
+import os
+import torch.nn.functional as F
+
+from mpt_7b.modeling_mpt import MPTModel, MPTForCausalLM, gen_attention_mask_in_length
+from mpt_7b.configuration_mpt import MPTConfig
+from mpt_7b.blocks import MPTBlock
+from mpt_7b.norm import NORM_CLASS_REGISTRY
+from mpt_7b.custom_embedding import SharedEmbedding
+from mpt_7b.attention import ATTN_CLASS_REGISTRY, attn_bias_shape, build_attn_bias, gen_slopes
+
+import logging
+log = logging.getLogger(__name__)
+
+class Custom_MptModel(MPTModel): # MptModel
+    def __init__(self, config: MPTConfig, modality0_dim=128, modality2_dim=1536):
+        config._validate_config()
+        super().__init__(config)
+        self.attn_impl = config.attn_config['attn_impl']
+        self.prefix_lm = config.attn_config['prefix_lm']
+        self.attn_uses_sequence_id = config.attn_config['attn_uses_sequence_id']
+        self.alibi = config.attn_config['alibi']
+        self.alibi_bias_max = config.attn_config['alibi_bias_max']
+        self.learned_pos_emb = config.learned_pos_emb
+        if config.init_device == 'mixed':
+            if dist.get_local_rank() == 0:
+                config.init_device = 'cpu'
+            else:
+                config.init_device = 'meta'
+        if config.norm_type.lower() not in NORM_CLASS_REGISTRY.keys():
+            norm_options = ' | '.join(NORM_CLASS_REGISTRY.keys())
+            raise NotImplementedError(f'Requested norm type ({config.norm_type}) is not implemented within this repo (Options: {norm_options}).')
+        norm_class = NORM_CLASS_REGISTRY[config.norm_type.lower()]
+        self.embedding_fraction = config.embedding_fraction
+        self.wte = SharedEmbedding(config.vocab_size, config.d_model, device=config.init_device)
+        if self.learned_pos_emb:
+            self.wpe = torch.nn.Embedding(config.max_seq_len, config.d_model, device=config.init_device)
+        self.emb_drop = nn.Dropout(config.emb_pdrop)
+        self.blocks = nn.ModuleList([MPTBlock(device=config.init_device, **config.to_dict()) for _ in range(config.n_layers)])
+        self.norm_f = norm_class(config.d_model, device=config.init_device)
+        
+        
+         ### Added for P3GPT - START
+        # Freeze all parameters except the projection layer
+        for param in self.wte.parameters():
+            param.requires_grad = False
+
+        for param in self.blocks.parameters():
+            param.requires_grad = False
+        
+        # Add a projection layer for the custom embedding
+        # torch.set_default_dtype(torch.bfloat16)
+        self.modality0_embedding_projection = nn.ModuleList([nn.Linear(modality0_dim, config.d_model), 
+                                                             # nn.BatchNorm1d(config.d_model),
+                                                             nn.ReLU(),
+                                                             nn.Linear(config.d_model, config.d_model),
+                                                             # nn.BatchNorm1d(config.d_model),
+                                                             nn.ReLU(),
+                                                             nn.Linear(config.d_model, config.d_model)])# nn.Linear(modality0_dim, self.hidden_size)
+        
+
+        self.modality2_embedding_projection = nn.ModuleList([nn.Linear(modality2_dim, config.d_model), 
+                                                             # nn.BatchNorm1d(config.d_model),
+                                                             nn.ReLU(),
+                                                             nn.Linear(config.d_model, config.d_model),
+                                                             # nn.BatchNorm1d(config.d_model),
+                                                             nn.ReLU(),
+                                                             nn.Linear(config.d_model, config.d_model)])# nn.Linear(modality0_dim, self.hidden_size)
+        
+        
+        ### Added for P3GPT - FINISH
+        
+        self.rope = config.attn_config['rope']
+        self.rope_impl = None
+        if self.rope:
+            self.rope_impl = config.attn_config['rope_impl']
+            self.rotary_embedding = gen_rotary_embedding(rope_head_dim=config.d_model // config.n_heads, rope_impl=self.rope_impl, rope_theta=config.attn_config['rope_theta'], rope_dail_config=config.attn_config['rope_dail_config'], rope_hf_config=config.attn_config['rope_hf_config'], max_seq_len=self.config.max_seq_len)
+        if config.init_device != 'meta':
+            log.info(f'We recommend using config.init_device="meta" with Composer + FSDP for faster initialization.')
+            self.apply(self.param_init_fn)
+        self.is_causal = not self.prefix_lm
+        self._attn_bias_initialized = False
+        self.attn_bias = None
+        self.attn_bias_shape = attn_bias_shape(self.attn_impl, config.n_heads, config.max_seq_len, self.alibi, prefix_lm=self.prefix_lm, causal=self.is_causal, use_sequence_id=self.attn_uses_sequence_id)
+        if config.no_bias:
+            for module in self.modules():
+                if hasattr(module, 'bias') and isinstance(module.bias, nn.Parameter):
+                    log.info(f'Removing bias from module={module!r}.')
+                    module.register_parameter('bias', None)
+                if hasattr(module, 'use_bias'):
+                    log.info(f'Setting use_bias=False for module={module!r}.')
+                    module.use_bias = False
+        log.debug(self)
+        log.debug(f"Using {self.config.init_config['name']} initialization.")
+
+        # Initialize weights and apply final processing
+#        self.post_init()
+        
+        
+    def get_input_embeddings(self):
+        return self.wte
+
+
+    def set_input_embeddings(self, new_embeddings):
+        # self.wte = new_embeddings
+        self.wte.weight = new_embeddings
+
+        
+    def forward(self, input_ids: Optional[torch.LongTensor]=None, past_key_values: Optional[List[Tuple[torch.FloatTensor]]]=None, 
+                attention_mask: Optional[torch.ByteTensor]=None, prefix_mask: Optional[torch.ByteTensor]=None, 
+                sequence_id: Optional[torch.LongTensor]=None, return_dict: Optional[bool]=None, output_attentions: Optional[bool]=None, 
+                output_hidden_states: Optional[bool]=None, use_cache: Optional[bool]=None, 
+                inputs_embeds: Optional[torch.Tensor]=None, modality0_emb: Optional[bool] = None,
+                modality0_token_id: Optional[bool] = None, modality1_emb: Optional[bool] = None, modality1_token_id: Optional[bool] = None,
+                modality2_emb: Optional[bool] = None, modality2_token_id: Optional[bool] = None, modality3_emb: Optional[bool] = None,
+                modality3_token_id: Optional[bool] = None,) -> BaseModelOutputWithPast:
+        
+        return_dict = return_dict if return_dict is not None else self.config.return_dict
+        use_cache = use_cache if use_cache is not None else self.config.use_cache
+        if attention_mask is not None:
+            attention_mask = attention_mask.bool()
+        if prefix_mask is not None:
+            prefix_mask = prefix_mask.bool()
+        if not return_dict:
+            raise NotImplementedError('return_dict False is not implemented yet for MPT')
+        if output_attentions:
+            if self.attn_impl != 'torch':
+                raise NotImplementedError('output_attentions is not implemented for MPT when using attn_impl `flash` or `triton`.')
+        if self.training and attention_mask is not None and (attention_mask[:, 0].sum() != attention_mask.shape[0]):
+            raise NotImplementedError('MPT does not support training with left padding.')
+        if self.prefix_lm and prefix_mask is None:
+            raise ValueError('prefix_mask is a required argument when MPT is configured with prefix_lm=True.')
+        if self.training:
+            if self.attn_uses_sequence_id and sequence_id is None:
+                raise ValueError('sequence_id is a required argument when MPT is configured with attn_uses_sequence_id=True ' + 'and the model is in train mode.')
+            elif self.attn_uses_sequence_id is False and sequence_id is not None:
+                warnings.warn('MPT received non-None input for `sequence_id` but is configured with attn_uses_sequence_id=False. ' + 'This input will be ignored. If you want the model to use `sequence_id`, set attn_uses_sequence_id to True.')
+        
+        ### ADDED FOR P3 - START
+        
+        if modality0_emb is not None:
+            modality0_emb = torch.tensor(modality0_emb, dtype=torch.bfloat16)
+            hidden_states = self.wte.weight.detach() 
+
+            for layer in self.modality0_embedding_projection:
+                modality0_emb = layer(modality0_emb)
+            proj_modality0_emb = modality0_emb
+
+            # Replace the original embedding for the custom token with the custom embedding
+            hidden_states[modality0_token_id, :] = torch.mean(torch.squeeze(proj_modality0_emb, 1), dim=0)
+            self.set_input_embeddings(torch.nn.Parameter(hidden_states))
+            
+        if modality1_emb is not None:
+            modality1_emb = torch.tensor(modality1_emb, dtype=torch.bfloat16)
+            hidden_states = self.wte.weight.detach() 
+
+            for layer in self.modality0_embedding_projection:
+                modality1_emb = layer(modality1_emb)
+            proj_modality1_emb = modality1_emb
+
+            # Replace the original embedding for the custom token with the custom embedding
+            hidden_states[modality1_token_id, :] = torch.mean(torch.squeeze(proj_modality1_emb, 1), dim=0)
+            self.set_input_embeddings(torch.nn.Parameter(hidden_states))
+        
+        if modality2_emb is not None:
+            modality2_emb = torch.tensor(modality2_emb, dtype=torch.bfloat16)
+            hidden_states = self.wte.weight.detach() 
+
+            for layer in self.modality2_embedding_projection:
+                modality2_emb = layer(modality2_emb)
+            proj_modality2_emb = modality2_emb
+
+            # Replace the original embedding for the custom token with the custom embedding
+            hidden_states[modality2_token_id, :] = torch.mean(torch.squeeze(proj_modality2_emb, 1), dim=0)
+            self.set_input_embeddings(torch.nn.Parameter(hidden_states))
+            
+        if modality3_emb is not None:
+            modality3_emb = torch.tensor(modality3_emb, dtype=torch.bfloat16)
+            hidden_states = self.wte.weight.detach() 
+
+            for layer in self.modality2_embedding_projection:
+                modality3_emb = layer(modality3_emb)
+            proj_modality3_emb = modality3_emb
+
+            # Replace the original embedding for the custom token with the custom embedding
+            hidden_states[modality3_token_id, :] = torch.mean(torch.squeeze(proj_modality3_emb, 1), dim=0)
+            self.set_input_embeddings(torch.nn.Parameter(hidden_states))
+        
+        ### ADDED FOR P3 - END
+        
+        if input_ids is not None and inputs_embeds is not None:
+            raise ValueError('You cannot specify both input_ids and inputs_embeds.')
+        elif input_ids is not None:
+            bsz = input_ids.size(0)
+            S = input_ids.size(1)
+            x = self.wte(input_ids)
+            input_device = input_ids.device
+        elif inputs_embeds is not None:
+            bsz = inputs_embeds.size(0)
+            S = inputs_embeds.size(1)
+            x = inputs_embeds
+            input_device = inputs_embeds.device
+        else:
+            raise ValueError('You must specify input_ids or inputs_embeds')
+        assert S <= self.config.max_seq_len, f'Cannot forward input with seq_len={S}, this model only supports seq_len<={self.config.max_seq_len}'
+        rotary_emb_w_meta_info = None
+        past_position = 0
+        if past_key_values is not None:
+            if len(past_key_values) != self.config.n_layers:
+                raise ValueError(f'past_key_values must provide a past_key_value for each attention ' + f'layer in the network (len(past_key_values)={len(past_key_values)!r}; self.config.n_layers={self.config.n_layers!r}).')
+            past_position = past_key_values[0][0].size(1)
+            if self.attn_impl == 'torch':
+                past_position = past_key_values[0][0].size(3)
+        if self.learned_pos_emb or self.rope:
+            if self.learned_pos_emb and S + past_position > self.config.max_seq_len:
+                raise ValueError(f'Cannot forward input with past sequence length {past_position} and current sequence length ' + f'{S + 1}, this model only supports total sequence length <= {self.config.max_seq_len}.')
+            if self.learned_pos_emb or (self.rope and self.rope_impl == 'hf'):
+                pos = torch.arange(past_position, S + past_position, dtype=torch.long, device=input_device).unsqueeze(0)
+                if attention_mask is not None:
+                    pos = torch.clamp(pos - torch.cumsum((~attention_mask).to(torch.int32), dim=1)[:, past_position:], min=0)
+                if self.learned_pos_emb:
+                    x = x + self.wpe(pos)
+                elif self.rope and self.rope_impl == 'hf':
+                    rotary_emb_w_meta_info = {'impl': self.rope_impl, 'rotary_emb': self.rotary_embedding, 'offset_info': pos, 'seq_len': S + past_position}
+            elif self.rope and self.rope_impl == 'dail':
+                rotary_emb_w_meta_info = {'impl': self.rope_impl, 'rotary_emb': self.rotary_embedding, 'offset_info': past_position, 'seq_len': S + past_position}
+        if self.embedding_fraction == 1:
+            x = self.emb_drop(x)
+        else:
+            x_shrunk = x * self.embedding_fraction + x.detach() * (1 - self.embedding_fraction)
+            assert isinstance(self.emb_drop, nn.Module)
+            x = self.emb_drop(x_shrunk)
+        (attn_bias, attention_mask) = self._attn_bias(device=x.device, dtype=torch.float32, attention_mask=attention_mask, prefix_mask=prefix_mask, sequence_id=sequence_id)
+        attention_mask_in_length = gen_attention_mask_in_length(sequence_id=sequence_id, S=S, attn_uses_sequence_id=self.attn_uses_sequence_id, attn_impl=self.attn_impl, attention_mask=attention_mask)
+        alibi_slopes = None
+        if self.alibi and self.attn_impl == 'flash':
+            alibi_slopes = gen_slopes(n_heads=self.config.n_heads, alibi_bias_max=self.alibi_bias_max, device=x.device, return_1d=True)
+            
+        presents = () if use_cache else None
+        if use_cache and past_key_values is None:
+            past_key_values = [() for _ in range(self.config.n_layers)]
+        all_hidden_states = () if output_hidden_states else None
+        all_self_attns = () if output_attentions else None
+        flash_attn_padding_info = {}
+        if self.attn_impl == 'flash':
+            flash_attn_padding_info = gen_flash_attn_padding_info(bsz, S, past_position, x.device, attention_mask_in_length, attention_mask)
+        for (b_idx, block) in enumerate(self.blocks):
+            if output_hidden_states:
+                assert all_hidden_states is not None
+                all_hidden_states = all_hidden_states + (x,)
+            past_key_value = past_key_values[b_idx] if past_key_values is not None else None
+            (x, attn_weights, present) = block(x, past_key_value=past_key_value, attn_bias=attn_bias, rotary_emb_w_meta_info=rotary_emb_w_meta_info, attention_mask=attention_mask, is_causal=self.is_causal, output_attentions=bool(output_attentions), alibi_slopes=alibi_slopes, flash_attn_padding_info=flash_attn_padding_info)
+            if presents is not None:
+                presents += (present,)
+            if output_attentions:
+                assert all_self_attns is not None
+                all_self_attns = all_self_attns + (attn_weights,)
+        x = self.norm_f(x)
+        if output_hidden_states:
+            assert all_hidden_states is not None
+            all_hidden_states = all_hidden_states + (x,)
+        return BaseModelOutputWithPast(last_hidden_state=x, past_key_values=presents, hidden_states=all_hidden_states, attentions=all_self_attns)
+
+
+class Custom_MPTForCausalLM(MPTForCausalLM):
+
+    def __init__(self, config: MPTConfig):
+        super().__init__(config)
+        # log.info(f'Instantiating an MPTForCausalLM model from {__file__}')
+        self.transformer: MPTModel = Custom_MptModel(config)
+        self.lm_head = None
+        if not config.tie_word_embeddings:
+            self.lm_head = nn.Linear(config.d_model, config.vocab_size, bias=False, device=config.init_device)
+            self.lm_head._fsdp_wrap = True
+        for child in self.transformer.children():
+            if isinstance(child, torch.nn.ModuleList):
+                continue
+            if isinstance(child, torch.nn.Module):
+                child._fsdp_wrap = True
+        self.logit_scale = None
+        if config.logit_scale is not None:
+            logit_scale = config.logit_scale
+            if isinstance(logit_scale, str):
+                if logit_scale == 'inv_sqrt_d_model':
+                    logit_scale = 1 / math.sqrt(config.d_model)
+                else:
+                    raise ValueError(f"logit_scale={logit_scale!r} is not recognized as an option; use numeric value or 'inv_sqrt_d_model'.")
+            self.logit_scale = logit_scale
+            
+    def forward(self, input_ids: Optional[torch.LongTensor]=None, past_key_values: Optional[List[Tuple[torch.FloatTensor]]]=None, 
+                attention_mask: Optional[torch.ByteTensor]=None, prefix_mask: Optional[torch.ByteTensor]=None, 
+                sequence_id: Optional[torch.LongTensor]=None, labels: Optional[torch.LongTensor]=None, 
+                return_dict: Optional[bool]=None, output_attentions: Optional[bool]=None, output_hidden_states: Optional[bool]=None, 
+                use_cache: Optional[bool]=None, inputs_embeds: Optional[torch.FloatTensor]=None,
+                modality0_emb: Optional[bool] = None, modality0_token_id: Optional[bool] = None,
+                modality1_emb: Optional[bool] = None, modality1_token_id: Optional[bool] = None, 
+                modality2_emb: Optional[bool] = None, modality2_token_id: Optional[bool] = None,
+                modality3_emb: Optional[bool] = None, modality3_token_id: Optional[bool] = None) -> CausalLMOutputWithPast:
+        return_dict = return_dict if return_dict is not None else self.config.return_dict
+        use_cache = use_cache if use_cache is not None else self.config.use_cache
+        outputs = self.transformer(
+            input_ids=input_ids, past_key_values=past_key_values, attention_mask=attention_mask, prefix_mask=prefix_mask, 
+            sequence_id=sequence_id, return_dict=return_dict, output_attentions=output_attentions, output_hidden_states=output_hidden_states, 
+            use_cache=use_cache, inputs_embeds=inputs_embeds,
+            modality0_emb=modality0_emb,
+            modality0_token_id=modality0_token_id,
+            modality1_emb=modality1_emb,
+            modality1_token_id=modality1_token_id,
+            modality2_emb=modality2_emb,
+            modality2_token_id=modality2_token_id,
+            modality3_emb=modality3_emb,
+            modality3_token_id=modality3_token_id
+        )
+        if self.lm_head is not None:
+            logits = self.lm_head(outputs.last_hidden_state)
+        else:
+            out = outputs.last_hidden_state
+            out = out.to(self.transformer.wte.weight.device)
+            logits = self.transformer.wte(out, True)
+        if self.logit_scale is not None:
+            if self.logit_scale == 0:
+                warnings.warn(f'Multiplying logits by self.logit_scale={self.logit_scale!r}. This will produce uniform (uninformative) outputs.')
+            logits *= self.logit_scale
+        loss = None
+        if labels is not None:
+            _labels = torch.roll(labels, shifts=-1)
+            _labels[:, -1] = -100
+            loss = F.cross_entropy(logits.view(-1, logits.size(-1)), _labels.to(logits.device).view(-1))
+        return CausalLMOutputWithPast(loss=loss, logits=logits, past_key_values=outputs.past_key_values, hidden_states=outputs.hidden_states, attentions=outputs.attentions)

special_tokens_map.json

@@ -0,0 +1,6 @@
+{
+  "bos_token": "[BOS]",
+  "eos_token": "[EOS]",
+  "pad_token": "[PAD]",
+  "unk_token": "[UNK]"
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea99402688e989d7fe75a55513c21cdfea22158a76765e99a102df307ff5ea5e
+size 12308399

tokenizer_config.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:80f8520546b55cf3bc43997f06ffcd15aa71887b6fce7e6701bac6c0d9ff55d6
+size 11670857