OpenBA commited on
Commit
06f3e2c
1 Parent(s): 834b77b

initialize

Browse files
added_tokens.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "<R>": 250200,
3
+ "<S>": 250201,
4
+ "<X>": 250202
5
+ }
config.json ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_ffn_bias": false,
3
+ "add_lm_head_bias": true,
4
+ "add_qkv_bias": true,
5
+ "architectures": [
6
+ "OpenBAForConditionalGeneration"
7
+ ],
8
+ "auto_map": {
9
+ "AutoConfig": "configuration_openba.OpenBAConfig",
10
+ "AutoModel": "modeling_openba.OpenBAForConditionalGeneration",
11
+ "AutoModelForCausalLM": "modeling_openba.OpenBAForConditionalGeneration",
12
+ "AutoModelForSeq2SeqLM": "modeling_openba.OpenBAForConditionalGeneration"
13
+ },
14
+ "attention_dropout": 0.1,
15
+ "decoder_max_seq_length": 256,
16
+ "decoder_start_token_id": 0,
17
+ "eos_token_id": 1,
18
+ "ffn_hidden_size": 16384,
19
+ "hidden_dropout": 0.1,
20
+ "hidden_size": 4096,
21
+ "initializer_factor": 1.0,
22
+ "is_encoder_decoder": true,
23
+ "kv_channels": 128,
24
+ "max_seq_length": 1024,
25
+ "model_type": "openba",
26
+ "num_decoder_layers": 36,
27
+ "num_heads": 40,
28
+ "num_layers": 12,
29
+ "pad_token_id": 0,
30
+ "tie_word_embeddings": false,
31
+ "tokenizer_class": "OpenBATokenizer",
32
+ "transformers_version": "4.31.0",
33
+ "use_cache": true,
34
+ "vocab_size": 250368
35
+ }
configuration_openba.py ADDED
@@ -0,0 +1,66 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from transformers.utils import logging
2
+ from transformers.configuration_utils import PretrainedConfig
3
+
4
+
5
+ logger = logging.get_logger(__name__)
6
+
7
+
8
+ class OpenBAConfig(PretrainedConfig):
9
+ model_type = "openba"
10
+ keys_to_ignore_at_inference = ["past_key_values"]
11
+ attribute_map = {
12
+ "hidden_size": "hidden_size",
13
+ "num_attention_heads": "num_heads",
14
+ "num_hidden_layers": "num_layers"
15
+ }
16
+
17
+ def __init__(
18
+ self,
19
+ vocab_size=32128,
20
+ hidden_size=512,
21
+ kv_channels=64,
22
+ ffn_hidden_size=2048,
23
+ num_layers=12,
24
+ num_decoder_layers=None,
25
+ hidden_dropout=0.1,
26
+ attention_dropout=0.1,
27
+ num_heads=8,
28
+ is_encoder_decoder=True,
29
+ use_cache=True,
30
+ initializer_factor=1.0,
31
+ pad_token_id=0,
32
+ eos_token_id=1,
33
+ decoder_start_token_id=0,
34
+ add_qkv_bias=False,
35
+ add_ffn_bias=False,
36
+ add_lm_head_bias=False,
37
+ max_seq_length=1024,
38
+ decoder_max_seq_length=256,
39
+ **kwargs,
40
+ ):
41
+ self.vocab_size = vocab_size
42
+ self.hidden_size = hidden_size
43
+ self.kv_channels = kv_channels
44
+ self.ffn_hidden_size = ffn_hidden_size
45
+ self.num_layers = num_layers
46
+ self.num_decoder_layers = (
47
+ num_decoder_layers if num_decoder_layers is not None else self.num_layers
48
+ ) # default = symmetry
49
+ self.hidden_dropout = hidden_dropout
50
+ self.attention_dropout = attention_dropout
51
+ self.initializer_factor = initializer_factor
52
+ self.num_heads = num_heads
53
+ self.add_qkv_bias = add_qkv_bias
54
+ self.add_ffn_bias = add_ffn_bias
55
+ self.add_lm_head_bias = add_lm_head_bias
56
+ self.max_seq_length = max_seq_length
57
+ self.decoder_max_seq_length = decoder_max_seq_length
58
+ self.use_cache = use_cache
59
+
60
+ super().__init__(
61
+ pad_token_id=pad_token_id,
62
+ eos_token_id=eos_token_id,
63
+ decoder_start_token_id=decoder_start_token_id,
64
+ is_encoder_decoder=is_encoder_decoder,
65
+ **kwargs,
66
+ )
modeling_openba.py ADDED
@@ -0,0 +1,706 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Optional, Tuple, Union
2
+ import copy
3
+
4
+ import torch
5
+ import torch.nn as nn
6
+ import torch.nn.functional as F
7
+
8
+ from transformers import PreTrainedModel
9
+ from transformers.modeling_outputs import (
10
+ BaseModelOutputWithPastAndCrossAttentions,
11
+ Seq2SeqLMOutput,
12
+ BaseModelOutput,
13
+ )
14
+ from transformers.utils import logging, is_torch_fx_proxy
15
+
16
+ from .configuration_openba import OpenBAConfig
17
+
18
+
19
+ logger = logging.get_logger(__name__)
20
+
21
+ # Copied from transformers.models.gptj.modeling_gptj.create_sinusoidal_positions
22
+ def create_sinusoidal_positions(num_pos: int, dim: int) -> torch.Tensor:
23
+ inv_freq = 1.0 / (10000 ** (torch.arange(0, dim, 2) / dim))
24
+ sinusoid_inp = torch.einsum("i , j -> i j", torch.arange(num_pos, dtype=torch.float), inv_freq).float()
25
+ return torch.cat((torch.sin(sinusoid_inp), torch.cos(sinusoid_inp)), dim=1)
26
+
27
+
28
+ def rotate_half(x) -> torch.Tensor:
29
+ x1, x2 = x[..., : x.shape[-1] // 2], x[..., x.shape[-1] // 2 :]
30
+ return torch.cat((-x2, x1), dim=-1)
31
+
32
+
33
+ def apply_rotary_pos_emb(tensor: torch.Tensor, sin: torch.Tensor, cos: torch.Tensor) -> torch.Tensor:
34
+ sin = torch.cat((sin, sin), dim=-1).to(tensor.device)[:, :, None, :]
35
+ cos = torch.cat((cos, cos), dim=-1).to(tensor.device)[:, :, None, :]
36
+ return (tensor * cos) + (rotate_half(tensor) * sin)
37
+
38
+
39
+ class SwiGLUMLP(nn.Module):
40
+ def __init__(self, config):
41
+ super().__init__()
42
+
43
+ multiple_of: int = 256 # make SwiGLU hidden layer size multiple of large power of 2
44
+ hidden_size = config.hidden_size
45
+ ffn_hidden_size = int(2 * config.ffn_hidden_size / 3)
46
+ ffn_hidden_size = multiple_of * ((ffn_hidden_size + multiple_of - 1) // multiple_of)
47
+ self.ffn_hidden_size = ffn_hidden_size
48
+
49
+ self.fc_in = nn.Linear(hidden_size, 2 * ffn_hidden_size, bias=config.add_ffn_bias)
50
+ self.fc_out = nn.Linear(ffn_hidden_size, hidden_size, bias=config.add_ffn_bias)
51
+
52
+ def swiglu(x):
53
+ x = torch.chunk(x, 2, dim=-1)
54
+ return F.silu(x[0]) * x[1]
55
+ self.act_func = swiglu
56
+
57
+ def forward(self, hidden_states: Optional[torch.FloatTensor]) -> torch.FloatTensor:
58
+ hidden_states = self.fc_in(hidden_states)
59
+ hidden_states = self.act_func(hidden_states)
60
+ hidden_states = self.fc_out(hidden_states)
61
+ return hidden_states
62
+
63
+
64
+ class OpenBAAttention(nn.Module):
65
+ def __init__(self, config, attn_type='self'):
66
+ super().__init__()
67
+ self.attn_type = attn_type
68
+ self.is_decoder = config.is_decoder
69
+ self.hidden_size = config.hidden_size
70
+ self.num_heads = config.num_heads
71
+ self.kv_channels = config.kv_channels
72
+ self.proj_size = self.kv_channels * self.num_heads
73
+ self.dropout = config.attention_dropout
74
+ self.scale_attn = torch.sqrt(torch.tensor(self.kv_channels, dtype=torch.float32))
75
+
76
+ if self.attn_type == 'self':
77
+ self.qkv = nn.Linear(self.hidden_size, 3 * self.proj_size, bias=config.add_qkv_bias)
78
+ else:
79
+ assert self.attn_type == 'cross'
80
+ self.q = nn.Linear(self.hidden_size, self.proj_size, bias=config.add_qkv_bias)
81
+ self.kv = nn.Linear(self.hidden_size, 2 * self.proj_size, bias=config.add_qkv_bias)
82
+
83
+ self.rotary_embedding = create_sinusoidal_positions(
84
+ num_pos=config.max_seq_length,
85
+ dim=self.kv_channels,
86
+ )
87
+
88
+ self.o = nn.Linear(self.proj_size, self.hidden_size, bias=config.add_qkv_bias)
89
+
90
+ def forward(
91
+ self,
92
+ hidden_states: Optional[torch.FloatTensor],
93
+ attention_mask: Optional[torch.FloatTensor] = None,
94
+ key_value_states: Optional[torch.FloatTensor] = None,
95
+ past_key_value: Optional[Tuple[torch.Tensor]] = None,
96
+ layer_head_mask: Optional[Tuple[torch.Tensor]] = None,
97
+ position_ids:Optional[torch.LongTensor] = None,
98
+ use_cache: Optional[bool] = False,
99
+ output_attentions: Optional[bool] = False,
100
+ ):
101
+ # input is (batch_size, seq_length, hidden_size)
102
+ batch_size, seq_length = hidden_states.shape[:2]
103
+ if past_key_value is not None:
104
+ if len(past_key_value) != 2:
105
+ raise ValueError(
106
+ f"past_key_value should have 2 past states: keys and values. Got { len(past_key_value)} past states"
107
+ )
108
+
109
+ if self.rotary_embedding.device != position_ids.device:
110
+ self.rotary_embedding = self.rotary_embedding.to(position_ids.device)
111
+
112
+ if self.attn_type == 'self':
113
+ mixed_qkv_states = self.qkv(hidden_states)
114
+ new_tensor_shape = mixed_qkv_states.size()[:-1] + (self.num_heads, 3 * self.kv_channels)
115
+ mixed_qkv_states = mixed_qkv_states.view(*new_tensor_shape)
116
+ query_states, key_states, value_states = torch.chunk(mixed_qkv_states, 3, dim=-1)
117
+ # rotary position embedding
118
+ sincos = self.rotary_embedding[position_ids]
119
+ sin, cos = torch.chunk(sincos, 2, dim=-1)
120
+ query_states = apply_rotary_pos_emb(query_states, sin, cos)
121
+ key_states = apply_rotary_pos_emb(key_states, sin, cos)
122
+ # reshape to (batch_size, num_head, seq_length, kv_channels)
123
+ query_states = query_states.transpose(1, 2)
124
+ key_states = key_states.transpose(1, 2)
125
+ value_states = value_states.transpose(1, 2)
126
+ if past_key_value is not None:
127
+ past_key_states, past_value_states = past_key_value
128
+ key_states = torch.cat([past_key_states, key_states], dim=-2)
129
+ value_states = torch.cat([past_value_states, value_states], dim=-2)
130
+ else:
131
+ assert self.attn_type == 'cross'
132
+ query_states = self.q(hidden_states)
133
+ new_tensor_shape = query_states.size()[:-1] + (self.num_heads, self.kv_channels)
134
+ query_states = query_states.view(*new_tensor_shape)
135
+ # reshape to (batch_size, num_head, seq_length, kv_channels)
136
+ query_states = query_states.transpose(1, 2)
137
+ if past_key_value is None:
138
+ mixed_kv_states = self.kv(key_value_states)
139
+ new_tensor_shape = mixed_kv_states.size()[:-1] + (self.num_heads, 2 * self.kv_channels)
140
+ mixed_kv_states = mixed_kv_states.view(*new_tensor_shape)
141
+ key_states, value_states = torch.chunk(mixed_kv_states, 2, dim=-1)
142
+ # reshape to (batch_size, num_head, seq_length, kv_channels)
143
+ key_states = key_states.transpose(1, 2)
144
+ value_states = value_states.transpose(1, 2)
145
+ else:
146
+ key_states, value_states = past_key_value
147
+
148
+ # compute attention score
149
+ query_states = query_states.to(torch.float32)
150
+ key_states = key_states.to(torch.float32)
151
+ attn_scores = torch.matmul(query_states, key_states.transpose(-1, -2)) / self.scale_attn
152
+ attn_scores = attn_scores.masked_fill_(attention_mask, -10000.0)
153
+ attn_weights = F.softmax(attn_scores, dim=-1).type_as(attn_scores)
154
+ attn_weights = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training)
155
+ attn_weights = attn_weights.to(value_states.dtype)
156
+
157
+ # Mask heads if we want to
158
+ if layer_head_mask is not None:
159
+ attn_weights = attn_weights * layer_head_mask
160
+
161
+ attn_output = torch.matmul(attn_weights, value_states)
162
+ attn_output = attn_output.transpose(1, 2).contiguous().view(batch_size, -1, self.proj_size)
163
+ attn_output = self.o(attn_output)
164
+
165
+ present_key_value_state = (key_states, value_states) if (self.is_decoder and use_cache) else None
166
+ outputs = (attn_output, present_key_value_state)
167
+
168
+ if output_attentions:
169
+ outputs += (attn_weights,)
170
+
171
+ return outputs
172
+
173
+
174
+ class OpenBABlock(nn.Module):
175
+ def __init__(self, config) -> None:
176
+ super().__init__()
177
+ self.is_decoder = config.is_decoder
178
+ self.dropout = config.hidden_dropout
179
+ self.input_layernorm = nn.LayerNorm(config.hidden_size)
180
+ self.self_attn = OpenBAAttention(config, attn_type='self')
181
+ self.post_attn_layernorm = nn.LayerNorm(config.hidden_size)
182
+ if self.is_decoder:
183
+ self.inter_attn = OpenBAAttention(config, attn_type='cross')
184
+ self.post_inter_attn_layernorm = nn.LayerNorm(config.hidden_size)
185
+ self.mlp = SwiGLUMLP(config)
186
+
187
+ def forward(
188
+ self,
189
+ hidden_states=None,
190
+ attention_mask=None,
191
+ position_ids=None,
192
+ encoder_hidden_states=None,
193
+ encoder_attention_mask=None,
194
+ layer_head_mask=None,
195
+ cross_attn_layer_head_mask=None,
196
+ past_key_value=None,
197
+ use_cache=False,
198
+ output_attentions=False,
199
+ ):
200
+ if past_key_value is not None:
201
+ if not self.is_decoder:
202
+ raise ValueError("`past_key_values` is passed to the encoder. Please make sure this is intended.")
203
+ expected_num_past_key_values = 2 if encoder_hidden_states is None else 4
204
+
205
+ if len(past_key_value) != expected_num_past_key_values:
206
+ raise ValueError(
207
+ f"There should be {expected_num_past_key_values} past states. "
208
+ f"{'2 (past / key) for cross attention. ' if expected_num_past_key_values == 4 else ''}"
209
+ f"Got {len(past_key_value)} past key / value states"
210
+ )
211
+
212
+ self_attn_past_key_value = past_key_value[:2]
213
+ cross_attn_past_key_value = past_key_value[2:]
214
+ else:
215
+ self_attn_past_key_value, cross_attn_past_key_value = None, None
216
+
217
+ # Layer norm at the beginning of the transformer layer.
218
+ layernorm_output = self.input_layernorm(hidden_states)
219
+ # Self attention.
220
+ attn_outputs = self.self_attn(
221
+ layernorm_output,
222
+ attention_mask=attention_mask,
223
+ position_ids=position_ids,
224
+ layer_head_mask=layer_head_mask,
225
+ past_key_value=self_attn_past_key_value,
226
+ use_cache=use_cache,
227
+ output_attentions=output_attentions,
228
+ )
229
+ attn_output, present_key_value_state = attn_outputs[:2]
230
+ attn_weights = attn_outputs[2:]
231
+ residual = hidden_states
232
+ # Layer norm post the self attention.
233
+ attn_output = nn.functional.dropout(attn_output, p=self.dropout, training=self.training)
234
+ layernorm_input = residual + attn_output
235
+ layernorm_output = self.post_attn_layernorm(layernorm_input)
236
+
237
+ if self.is_decoder:
238
+ assert encoder_hidden_states is not None
239
+ attn_outputs = self.inter_attn(
240
+ layernorm_output,
241
+ attention_mask=encoder_attention_mask,
242
+ key_value_states=encoder_hidden_states,
243
+ position_ids=position_ids,
244
+ layer_head_mask=cross_attn_layer_head_mask,
245
+ past_key_value=cross_attn_past_key_value,
246
+ use_cache=use_cache,
247
+ output_attentions=output_attentions,
248
+ )
249
+ attn_output = attn_outputs[0]
250
+ attn_output = nn.functional.dropout(attn_output, p=self.dropout, training=self.training)
251
+ # residual connection
252
+ residual = layernorm_input
253
+ layernorm_input = residual + attn_output
254
+ layernorm_output = self.post_inter_attn_layernorm(layernorm_input)
255
+ # Combine self attn and cross attn key value states
256
+ if present_key_value_state is not None:
257
+ present_key_value_state += attn_outputs[1]
258
+ attn_weights += attn_outputs[2:]
259
+
260
+ # MLP.
261
+ mlp_output = self.mlp(layernorm_output)
262
+ mlp_output = nn.functional.dropout(mlp_output, p=self.dropout, training=self.training)
263
+ # Second residual connection.
264
+ residual = layernorm_input
265
+ output = residual + mlp_output
266
+ outputs = (output,)
267
+
268
+ if use_cache:
269
+ outputs += (present_key_value_state,) + attn_weights
270
+ else:
271
+ outputs += attn_weights
272
+ return outputs
273
+
274
+
275
+ class OpenBAPreTrainedModel(PreTrainedModel):
276
+ config_class = OpenBAConfig
277
+ base_model_prefix = "transformer"
278
+ _no_split_modules = ["OpenBABlock"]
279
+
280
+ def _set_gradient_checkpointing(self, module, value=False):
281
+ if isinstance(module, (OpenBAAttention, OpenBAStack)):
282
+ module.gradient_checkpointing = value
283
+
284
+ def _init_weights(self, module):
285
+ """Initialize the weights"""
286
+ factor = self.config.initializer_factor
287
+ if isinstance(module, nn.LayerNorm):
288
+ module.weight.data.fill_(1.0)
289
+ module.bias.data.zero_()
290
+ elif isinstance(module, OpenBAForConditionalGeneration):
291
+ module.shared_embedding.weight.data.normal_(mean=0.0, std=factor * 1.0)
292
+ if hasattr(module, "lm_head") and not self.config.tie_word_embeddings:
293
+ module.lm_head.weight.data.normal_(mean=0.0, std=factor * 1.0)
294
+ elif isinstance(module, SwiGLUMLP):
295
+ module.fc_in.weight.data.normal_(mean=0.0, std=factor * ((self.config.hidden_size) ** -0.5))
296
+ if hasattr(module.fc_in, "bias") and module.fc_in.bias is not None:
297
+ module.fc_in.bias.data.zero_()
298
+ module.fc_out.weight.data.normal_(mean=0.0, std=factor * ((module.ffn_hidden_size) ** -0.5))
299
+ if hasattr(module.fc_out, "bias") and module.fc_out.bias is not None:
300
+ module.fc_out.bias.data.zero_()
301
+ elif isinstance(module, OpenBAAttention):
302
+ hidden_size = self.config.hidden_size
303
+ kv_channels = self.config.kv_channels
304
+ n_heads = self.config.num_heads
305
+ if module.attn_type == 'self':
306
+ module.qkv.weight.data[:n_heads * kv_channels].normal_(mean=0.0, std=factor * ((hidden_size * kv_channels) ** -0.5))
307
+ module.qkv.weight.data[n_heads * kv_channels:].normal_(mean=0.0, std=factor * (hidden_size ** -0.5))
308
+ else:
309
+ module.q.weight.data.normal_(mean=0.0, std=factor * ((hidden_size * kv_channels) ** -0.5))
310
+ module.kv.weight.data.normal_(mean=0.0, std=factor * (hidden_size ** -0.5))
311
+ module.o.weight.data.normal_(mean=0.0, std=factor * ((n_heads * kv_channels) ** -0.5))
312
+
313
+ def _shift_right(self, input_ids):
314
+ decoder_start_token_id = self.config.decoder_start_token_id
315
+ pad_token_id = self.config.pad_token_id
316
+
317
+ if decoder_start_token_id is None:
318
+ raise ValueError(
319
+ "self.model.config.decoder_start_token_id has to be defined. In T5 it is usually set to the pad_token_id."
320
+ "See T5 docs for more information."
321
+ )
322
+
323
+ # shift inputs to the right
324
+ if is_torch_fx_proxy(input_ids):
325
+ # Item assignment is not supported natively for proxies.
326
+ shifted_input_ids = torch.full(input_ids.shape[:-1] + (1,), decoder_start_token_id)
327
+ shifted_input_ids = torch.cat([shifted_input_ids, input_ids[..., :-1]], dim=-1)
328
+ else:
329
+ shifted_input_ids = input_ids.new_zeros(input_ids.shape)
330
+ shifted_input_ids[..., 1:] = input_ids[..., :-1].clone()
331
+ shifted_input_ids[..., 0] = decoder_start_token_id
332
+
333
+ if pad_token_id is None:
334
+ raise ValueError("self.model.config.pad_token_id has to be defined.")
335
+ # replace possible -100 values in labels by `pad_token_id`
336
+ shifted_input_ids.masked_fill_(shifted_input_ids == -100, pad_token_id)
337
+
338
+ return shifted_input_ids
339
+
340
+ class OpenBAStack(OpenBAPreTrainedModel):
341
+ def __init__(self, config, embed_tokens):
342
+ super().__init__(config)
343
+ self.embed_tokens = embed_tokens
344
+ self.is_decoder = config.is_decoder
345
+ self.block = nn.ModuleList(
346
+ [OpenBABlock(config) for _ in range(config.num_layers)]
347
+ )
348
+ self.final_layernorm = nn.LayerNorm(config.hidden_size)
349
+
350
+ def forward(
351
+ self,
352
+ input_ids=None,
353
+ attention_mask=None,
354
+ encoder_hidden_states=None,
355
+ encoder_attention_mask=None,
356
+ inputs_embeds=None,
357
+ head_mask=None,
358
+ cross_attn_head_mask=None,
359
+ past_key_values=None,
360
+ use_cache=None,
361
+ output_attentions=None,
362
+ output_hidden_states=None,
363
+ return_dict=None,
364
+ ):
365
+ use_cache = use_cache if use_cache is not None else self.config.use_cache
366
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
367
+ output_hidden_states = (
368
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
369
+ )
370
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
371
+
372
+ # get batch size and seq_length
373
+ if input_ids is not None and inputs_embeds is not None:
374
+ raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
375
+ elif input_ids is not None:
376
+ input_shape = input_ids.size()
377
+ input_ids = input_ids.view(-1, input_shape[-1])
378
+ elif inputs_embeds is not None:
379
+ input_shape = inputs_embeds.size()[:-1]
380
+ else:
381
+ raise ValueError("You have to specify either input_ids or inputs_embeds")
382
+
383
+ batch_size, seq_length = input_shape
384
+ device = input_ids.device if input_ids is not None else inputs_embeds.device
385
+
386
+ # required mask seq length can be calculated via length of past
387
+ if past_key_values is None:
388
+ past_length = 0
389
+ past_key_values = [None] * len(self.block)
390
+ else:
391
+ past_length = past_key_values[0][0].size(-2)
392
+ cur_length = past_length + seq_length
393
+
394
+ # position ids
395
+ position_ids = torch.arange(past_length, cur_length, dtype=torch.long, device=device)
396
+ position_ids = position_ids.unsqueeze(0).view(-1, seq_length)
397
+
398
+ # Attention mask
399
+ if attention_mask is None:
400
+ attention_mask = torch.ones(batch_size, seq_length, device=device)
401
+ # get extended self-attention mask
402
+ if self.is_decoder:
403
+ if len(attention_mask.shape) == 2:
404
+ seq_ids = torch.arange(seq_length, device=device)
405
+ causal_mask = seq_ids[None, None, :].repeat(batch_size, seq_length, 1) <= seq_ids[None, :, None]
406
+ causal_mask = causal_mask.to(attention_mask.dtype)
407
+ extended_attention_mask = causal_mask[:, None, :, :] * attention_mask[:, None, None, :]
408
+ elif len(attention_mask.shape) == 3:
409
+ extended_attention_mask = attention_mask[:, None, :, :]
410
+ else:
411
+ raise ValueError
412
+ else:
413
+ extended_attention_mask = attention_mask[:, None, None, :]
414
+ extended_attention_mask = extended_attention_mask < 0.5
415
+ # get extended self-attention mask
416
+ # here we replace encoder_decoder_attention_mask with encoder_attention_mask
417
+ if self.is_decoder and encoder_hidden_states is not None:
418
+ if encoder_attention_mask is None:
419
+ encoder_seq_length = encoder_hidden_states.shape[1]
420
+ encoder_attention_mask = torch.ones(
421
+ batch_size, encoder_seq_length, device=device, dtype=torch.long
422
+ )
423
+ extended_encoder_attention_mask = encoder_attention_mask[:, None, None, :]
424
+ extended_encoder_attention_mask = extended_encoder_attention_mask < 0.5
425
+ else:
426
+ extended_encoder_attention_mask = None
427
+
428
+
429
+ # input embeddings
430
+ if inputs_embeds is None:
431
+ inputs_embeds = self.embed_tokens(input_ids)
432
+
433
+ # Prepare head mask if needed
434
+ head_mask = self.get_head_mask(head_mask, self.config.num_layers)
435
+ cross_attn_head_mask = self.get_head_mask(cross_attn_head_mask, self.config.num_layers)
436
+ present_key_value_states = () if use_cache else None
437
+ all_hidden_states = () if output_hidden_states else None
438
+ all_attentions = () if output_attentions else None
439
+ all_cross_attentions = () if (output_attentions and self.is_decoder) else None
440
+ hidden_states = inputs_embeds
441
+
442
+ for i, (layer_module, past_key_value) in enumerate(zip(self.block, past_key_values)):
443
+ layer_head_mask = head_mask[i]
444
+ cross_attn_layer_head_mask = cross_attn_head_mask[i]
445
+ if output_hidden_states:
446
+ all_hidden_states += (hidden_states,)
447
+ layer_outputs = layer_module(
448
+ hidden_states,
449
+ attention_mask=extended_attention_mask,
450
+ position_ids=position_ids,
451
+ encoder_hidden_states=encoder_hidden_states,
452
+ encoder_attention_mask=extended_encoder_attention_mask,
453
+ layer_head_mask=layer_head_mask,
454
+ cross_attn_layer_head_mask=cross_attn_layer_head_mask,
455
+ past_key_value=past_key_value,
456
+ use_cache=use_cache,
457
+ output_attentions=output_attentions,
458
+ )
459
+ # layer_outputs is a tuple with:
460
+ # hidden-states, key-value-states, (self-attention weights), (cross-attention weights)
461
+ if use_cache is False:
462
+ layer_outputs = layer_outputs[:1] + (None,) + layer_outputs[1:]
463
+
464
+ hidden_states, present_key_value_state = layer_outputs[:2]
465
+ if use_cache:
466
+ present_key_value_states += (present_key_value_state,)
467
+
468
+ if output_attentions:
469
+ all_attentions = all_attentions + (layer_outputs[2],)
470
+ if self.is_decoder:
471
+ all_cross_attentions = all_cross_attentions + (layer_outputs[3],)
472
+
473
+ hidden_states = self.final_layernorm(hidden_states)
474
+
475
+ if output_hidden_states:
476
+ all_hidden_states += (hidden_states,)
477
+
478
+ if not return_dict:
479
+ return tuple(
480
+ v
481
+ for v in [
482
+ hidden_states,
483
+ present_key_value_states,
484
+ all_hidden_states,
485
+ all_attentions,
486
+ all_cross_attentions,
487
+ ]
488
+ if v is not None
489
+ )
490
+ return BaseModelOutputWithPastAndCrossAttentions(
491
+ last_hidden_state=hidden_states,
492
+ past_key_values=present_key_value_states,
493
+ hidden_states=all_hidden_states,
494
+ attentions=all_attentions,
495
+ cross_attentions=all_cross_attentions,
496
+ )
497
+
498
+
499
+ class OpenBAForConditionalGeneration(OpenBAPreTrainedModel):
500
+ _keys_to_ignore_on_load_missing = [
501
+ r"encoder.embed_tokens.weight",
502
+ r"decoder.embed_tokens.weight",
503
+ ]
504
+ def __init__(self, config):
505
+ super().__init__(config)
506
+ self.shared_embedding = nn.Embedding(config.vocab_size, config.hidden_size)
507
+ self.hidden_size = config.hidden_size
508
+
509
+ encoder_config = copy.deepcopy(config)
510
+ encoder_config.is_decoder = False
511
+ encoder_config.use_cache = False
512
+ encoder_config.is_encoder_decoder = False
513
+ self.encoder = OpenBAStack(encoder_config, self.shared_embedding)
514
+
515
+ decoder_config = copy.deepcopy(config)
516
+ decoder_config.is_decoder = True
517
+ decoder_config.is_encoder_decoder = False
518
+ decoder_config.num_layers = config.num_decoder_layers
519
+ decoder_config.max_seq_length = config.decoder_max_seq_length
520
+ self.decoder = OpenBAStack(decoder_config, self.shared_embedding)
521
+
522
+ self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=config.add_lm_head_bias)
523
+
524
+ # Initialize weights and apply final processing
525
+ self.post_init()
526
+
527
+ # Model parallel
528
+ self.model_parallel = False
529
+ self.device_map = None
530
+
531
+ def get_input_embeddings(self):
532
+ return self.shared_embedding
533
+
534
+ def set_input_embeddings(self, new_embeddings):
535
+ self.shared_embedding = new_embeddings
536
+ self.encoder.set_input_embeddings(new_embeddings)
537
+ self.decoder.set_input_embeddings(new_embeddings)
538
+
539
+ def set_output_embeddings(self, new_embeddings):
540
+ self.lm_head = new_embeddings
541
+
542
+ def get_output_embeddings(self):
543
+ return self.lm_head
544
+
545
+ def get_encoder(self):
546
+ return self.encoder
547
+
548
+ def get_decoder(self):
549
+ return self.decoder
550
+
551
+ def forward(
552
+ self,
553
+ input_ids: Optional[torch.LongTensor] = None,
554
+ attention_mask: Optional[torch.FloatTensor] = None,
555
+ decoder_input_ids: Optional[torch.LongTensor] = None,
556
+ decoder_attention_mask: Optional[torch.BoolTensor] = None,
557
+ head_mask: Optional[torch.FloatTensor] = None,
558
+ decoder_head_mask: Optional[torch.FloatTensor] = None,
559
+ cross_attn_head_mask: Optional[torch.Tensor] = None,
560
+ encoder_outputs: Optional[Tuple[Tuple[torch.FloatTensor]]] = None,
561
+ past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None,
562
+ inputs_embeds: Optional[torch.Tensor] = None,
563
+ decoder_inputs_embeds: Optional[torch.Tensor] = None,
564
+ labels: Optional[torch.LongTensor] = None,
565
+ use_cache: Optional[bool] = None,
566
+ output_attentions: Optional[bool] = None,
567
+ output_hidden_states: Optional[bool] = None,
568
+ return_dict: Optional[bool] = None,
569
+ ) -> Union[Tuple[torch.FloatTensor], Seq2SeqLMOutput]:
570
+
571
+ use_cache = use_cache if use_cache is not None else self.config.use_cache
572
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
573
+
574
+ # Encode if needed (training, first prediction pass)
575
+ if encoder_outputs is None:
576
+ encoder_outputs = self.encoder(
577
+ input_ids=input_ids,
578
+ attention_mask=attention_mask,
579
+ inputs_embeds=inputs_embeds,
580
+ head_mask=head_mask,
581
+ output_attentions=output_attentions,
582
+ output_hidden_states=output_hidden_states,
583
+ return_dict=return_dict,
584
+ )
585
+ elif return_dict and not isinstance(encoder_outputs, BaseModelOutput):
586
+ encoder_outputs = BaseModelOutput(
587
+ last_hidden_state=encoder_outputs[0],
588
+ hidden_states=encoder_outputs[1] if len(encoder_outputs) > 1 else None,
589
+ attentions=encoder_outputs[2] if len(encoder_outputs) > 2 else None,\
590
+ )
591
+
592
+ hidden_states = encoder_outputs[0]
593
+
594
+ if labels is not None and decoder_input_ids is None and decoder_inputs_embeds is None:
595
+ # get decoder inputs from shifting lm labels to the right
596
+ decoder_input_ids = self._shift_right(labels)
597
+
598
+ # Decode
599
+ decoder_outputs = self.decoder(
600
+ input_ids=decoder_input_ids,
601
+ attention_mask=decoder_attention_mask,
602
+ inputs_embeds=decoder_inputs_embeds,
603
+ past_key_values=past_key_values,
604
+ encoder_hidden_states=hidden_states,
605
+ encoder_attention_mask=attention_mask,
606
+ head_mask=decoder_head_mask,
607
+ cross_attn_head_mask=cross_attn_head_mask,
608
+ use_cache=use_cache,
609
+ output_attentions=output_attentions,
610
+ output_hidden_states=output_hidden_states,
611
+ return_dict=return_dict,
612
+ )
613
+
614
+ sequence_output = decoder_outputs[0]
615
+ # share embedding and softmax embedding
616
+ if self.config.tie_word_embeddings:
617
+ # Rescale output before projecting on vocab
618
+ sequence_output = sequence_output * (self.hidden_size ** -0.5)
619
+
620
+ lm_logits = self.lm_head(sequence_output).to(torch.float32)
621
+
622
+ loss = None
623
+ if labels is not None:
624
+ loss_fct = nn.CrossEntropyLoss(ignore_index=-100)
625
+ # move labels to correct device to enable PP
626
+ labels = labels.to(lm_logits.device)
627
+ loss = loss_fct(lm_logits.view(-1, lm_logits.size(-1)), labels.view(-1))
628
+ loss = loss.to(hidden_states.dtype)
629
+
630
+ if not return_dict:
631
+ output = (lm_logits,) + decoder_outputs[1:] + encoder_outputs
632
+ return ((loss,) + output) if loss is not None else output
633
+
634
+ return Seq2SeqLMOutput(
635
+ loss=loss,
636
+ logits=lm_logits,
637
+ past_key_values=decoder_outputs.past_key_values,
638
+ decoder_hidden_states=decoder_outputs.hidden_states,
639
+ decoder_attentions=decoder_outputs.attentions,
640
+ cross_attentions=decoder_outputs.cross_attentions,
641
+ encoder_last_hidden_state=encoder_outputs.last_hidden_state,
642
+ encoder_hidden_states=encoder_outputs.hidden_states,
643
+ encoder_attentions=encoder_outputs.attentions,
644
+ )
645
+
646
+ def prepare_inputs_for_generation(
647
+ self,
648
+ input_ids,
649
+ past_key_values=None,
650
+ attention_mask=None,
651
+ head_mask=None,
652
+ decoder_head_mask=None,
653
+ decoder_attention_mask=None,
654
+ cross_attn_head_mask=None,
655
+ use_cache=None,
656
+ encoder_outputs=None,
657
+ **kwargs,
658
+ ):
659
+ # cut decoder_input_ids if past is used
660
+ if past_key_values is not None:
661
+ input_ids = input_ids[:, -1:]
662
+
663
+ return {
664
+ "decoder_input_ids": input_ids,
665
+ "past_key_values": past_key_values,
666
+ "encoder_outputs": encoder_outputs,
667
+ "attention_mask": attention_mask,
668
+ "head_mask": head_mask,
669
+ "decoder_head_mask": decoder_head_mask,
670
+ "decoder_attention_mask": decoder_attention_mask,
671
+ "cross_attn_head_mask": cross_attn_head_mask,
672
+ "use_cache": use_cache,
673
+ }
674
+
675
+ def prepare_decoder_input_ids_from_labels(self, labels: torch.Tensor):
676
+ return self._shift_right(labels)
677
+
678
+ def _reorder_cache(self, past_key_values, beam_idx):
679
+ # if decoder past is not included in output
680
+ # speedy decoding is disabled and no need to reorder
681
+ if past_key_values is None:
682
+ logger.warning("You might want to consider setting `use_cache=True` to speed up decoding")
683
+ return past_key_values
684
+
685
+ reordered_decoder_past = ()
686
+ for layer_past_states in past_key_values:
687
+ # get the correct batch idx from layer past batch dim
688
+ # batch dim of `past` is at 2nd position
689
+ reordered_layer_past_states = ()
690
+ for layer_past_state in layer_past_states:
691
+ # need to set correct `past` for each of the four key / value states
692
+ reordered_layer_past_states = reordered_layer_past_states + (
693
+ layer_past_state.index_select(0, beam_idx.to(layer_past_state.device)),
694
+ )
695
+
696
+ if reordered_layer_past_states[0].shape != layer_past_states[0].shape:
697
+ raise ValueError(
698
+ f"reordered_layer_past_states[0] shape {reordered_layer_past_states[0].shape} and layer_past_states[0] shape {layer_past_states[0].shape} mismatched"
699
+ )
700
+ if len(reordered_layer_past_states) != len(layer_past_states):
701
+ raise ValueError(
702
+ f"length of reordered_layer_past_states {len(reordered_layer_past_states)} and length of layer_past_states {len(layer_past_states)} mismatched"
703
+ )
704
+
705
+ reordered_decoder_past = reordered_decoder_past + (reordered_layer_past_states,)
706
+ return reordered_decoder_past
pytorch_model-00001-of-00003.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1cf4b97676699f3653c8c2b478bd5d67cdd4381b1f0b9802cc78504f276cc9e8
3
+ size 9992596278
pytorch_model-00002-of-00003.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f858e5dce4d4b1b96421d2a0289d891f3903994a51d787809267ec0e497cb167
3
+ size 9962486872
pytorch_model-00003-of-00003.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:155adb3477a5453de2696f4afdc0746552a7ac003f388e012fc44a8ddf93380f
3
+ size 9180587813
pytorch_model.bin.index.json ADDED
@@ -0,0 +1,782 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "metadata": {
3
+ "total_size": 29135406080
4
+ },
5
+ "weight_map": {
6
+ "decoder.block.0.input_layernorm.bias": "pytorch_model-00001-of-00003.bin",
7
+ "decoder.block.0.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
8
+ "decoder.block.0.inter_attn.kv.bias": "pytorch_model-00001-of-00003.bin",
9
+ "decoder.block.0.inter_attn.kv.weight": "pytorch_model-00001-of-00003.bin",
10
+ "decoder.block.0.inter_attn.o.bias": "pytorch_model-00001-of-00003.bin",
11
+ "decoder.block.0.inter_attn.o.weight": "pytorch_model-00001-of-00003.bin",
12
+ "decoder.block.0.inter_attn.q.bias": "pytorch_model-00001-of-00003.bin",
13
+ "decoder.block.0.inter_attn.q.weight": "pytorch_model-00001-of-00003.bin",
14
+ "decoder.block.0.mlp.fc_in.weight": "pytorch_model-00003-of-00003.bin",
15
+ "decoder.block.0.mlp.fc_out.weight": "pytorch_model-00001-of-00003.bin",
16
+ "decoder.block.0.post_attn_layernorm.bias": "pytorch_model-00001-of-00003.bin",
17
+ "decoder.block.0.post_attn_layernorm.weight": "pytorch_model-00001-of-00003.bin",
18
+ "decoder.block.0.post_inter_attn_layernorm.bias": "pytorch_model-00001-of-00003.bin",
19
+ "decoder.block.0.post_inter_attn_layernorm.weight": "pytorch_model-00001-of-00003.bin",
20
+ "decoder.block.0.self_attn.o.bias": "pytorch_model-00001-of-00003.bin",
21
+ "decoder.block.0.self_attn.o.weight": "pytorch_model-00001-of-00003.bin",
22
+ "decoder.block.0.self_attn.qkv.bias": "pytorch_model-00001-of-00003.bin",
23
+ "decoder.block.0.self_attn.qkv.weight": "pytorch_model-00001-of-00003.bin",
24
+ "decoder.block.1.input_layernorm.bias": "pytorch_model-00001-of-00003.bin",
25
+ "decoder.block.1.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
26
+ "decoder.block.1.inter_attn.kv.bias": "pytorch_model-00001-of-00003.bin",
27
+ "decoder.block.1.inter_attn.kv.weight": "pytorch_model-00001-of-00003.bin",
28
+ "decoder.block.1.inter_attn.o.bias": "pytorch_model-00001-of-00003.bin",
29
+ "decoder.block.1.inter_attn.o.weight": "pytorch_model-00001-of-00003.bin",
30
+ "decoder.block.1.inter_attn.q.bias": "pytorch_model-00001-of-00003.bin",
31
+ "decoder.block.1.inter_attn.q.weight": "pytorch_model-00001-of-00003.bin",
32
+ "decoder.block.1.mlp.fc_in.weight": "pytorch_model-00003-of-00003.bin",
33
+ "decoder.block.1.mlp.fc_out.weight": "pytorch_model-00001-of-00003.bin",
34
+ "decoder.block.1.post_attn_layernorm.bias": "pytorch_model-00001-of-00003.bin",
35
+ "decoder.block.1.post_attn_layernorm.weight": "pytorch_model-00001-of-00003.bin",
36
+ "decoder.block.1.post_inter_attn_layernorm.bias": "pytorch_model-00001-of-00003.bin",
37
+ "decoder.block.1.post_inter_attn_layernorm.weight": "pytorch_model-00001-of-00003.bin",
38
+ "decoder.block.1.self_attn.o.bias": "pytorch_model-00001-of-00003.bin",
39
+ "decoder.block.1.self_attn.o.weight": "pytorch_model-00001-of-00003.bin",
40
+ "decoder.block.1.self_attn.qkv.bias": "pytorch_model-00001-of-00003.bin",
41
+ "decoder.block.1.self_attn.qkv.weight": "pytorch_model-00001-of-00003.bin",
42
+ "decoder.block.10.input_layernorm.bias": "pytorch_model-00002-of-00003.bin",
43
+ "decoder.block.10.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
44
+ "decoder.block.10.inter_attn.kv.bias": "pytorch_model-00002-of-00003.bin",
45
+ "decoder.block.10.inter_attn.kv.weight": "pytorch_model-00002-of-00003.bin",
46
+ "decoder.block.10.inter_attn.o.bias": "pytorch_model-00002-of-00003.bin",
47
+ "decoder.block.10.inter_attn.o.weight": "pytorch_model-00002-of-00003.bin",
48
+ "decoder.block.10.inter_attn.q.bias": "pytorch_model-00002-of-00003.bin",
49
+ "decoder.block.10.inter_attn.q.weight": "pytorch_model-00002-of-00003.bin",
50
+ "decoder.block.10.mlp.fc_in.weight": "pytorch_model-00003-of-00003.bin",
51
+ "decoder.block.10.mlp.fc_out.weight": "pytorch_model-00002-of-00003.bin",
52
+ "decoder.block.10.post_attn_layernorm.bias": "pytorch_model-00002-of-00003.bin",
53
+ "decoder.block.10.post_attn_layernorm.weight": "pytorch_model-00002-of-00003.bin",
54
+ "decoder.block.10.post_inter_attn_layernorm.bias": "pytorch_model-00002-of-00003.bin",
55
+ "decoder.block.10.post_inter_attn_layernorm.weight": "pytorch_model-00002-of-00003.bin",
56
+ "decoder.block.10.self_attn.o.bias": "pytorch_model-00002-of-00003.bin",
57
+ "decoder.block.10.self_attn.o.weight": "pytorch_model-00002-of-00003.bin",
58
+ "decoder.block.10.self_attn.qkv.bias": "pytorch_model-00002-of-00003.bin",
59
+ "decoder.block.10.self_attn.qkv.weight": "pytorch_model-00002-of-00003.bin",
60
+ "decoder.block.11.input_layernorm.bias": "pytorch_model-00002-of-00003.bin",
61
+ "decoder.block.11.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
62
+ "decoder.block.11.inter_attn.kv.bias": "pytorch_model-00002-of-00003.bin",
63
+ "decoder.block.11.inter_attn.kv.weight": "pytorch_model-00002-of-00003.bin",
64
+ "decoder.block.11.inter_attn.o.bias": "pytorch_model-00002-of-00003.bin",
65
+ "decoder.block.11.inter_attn.o.weight": "pytorch_model-00002-of-00003.bin",
66
+ "decoder.block.11.inter_attn.q.bias": "pytorch_model-00002-of-00003.bin",
67
+ "decoder.block.11.inter_attn.q.weight": "pytorch_model-00002-of-00003.bin",
68
+ "decoder.block.11.mlp.fc_in.weight": "pytorch_model-00003-of-00003.bin",
69
+ "decoder.block.11.mlp.fc_out.weight": "pytorch_model-00002-of-00003.bin",
70
+ "decoder.block.11.post_attn_layernorm.bias": "pytorch_model-00002-of-00003.bin",
71
+ "decoder.block.11.post_attn_layernorm.weight": "pytorch_model-00002-of-00003.bin",
72
+ "decoder.block.11.post_inter_attn_layernorm.bias": "pytorch_model-00002-of-00003.bin",
73
+ "decoder.block.11.post_inter_attn_layernorm.weight": "pytorch_model-00002-of-00003.bin",
74
+ "decoder.block.11.self_attn.o.bias": "pytorch_model-00002-of-00003.bin",
75
+ "decoder.block.11.self_attn.o.weight": "pytorch_model-00002-of-00003.bin",
76
+ "decoder.block.11.self_attn.qkv.bias": "pytorch_model-00002-of-00003.bin",
77
+ "decoder.block.11.self_attn.qkv.weight": "pytorch_model-00002-of-00003.bin",
78
+ "decoder.block.12.input_layernorm.bias": "pytorch_model-00002-of-00003.bin",
79
+ "decoder.block.12.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
80
+ "decoder.block.12.inter_attn.kv.bias": "pytorch_model-00002-of-00003.bin",
81
+ "decoder.block.12.inter_attn.kv.weight": "pytorch_model-00002-of-00003.bin",
82
+ "decoder.block.12.inter_attn.o.bias": "pytorch_model-00002-of-00003.bin",
83
+ "decoder.block.12.inter_attn.o.weight": "pytorch_model-00002-of-00003.bin",
84
+ "decoder.block.12.inter_attn.q.bias": "pytorch_model-00002-of-00003.bin",
85
+ "decoder.block.12.inter_attn.q.weight": "pytorch_model-00002-of-00003.bin",
86
+ "decoder.block.12.mlp.fc_in.weight": "pytorch_model-00003-of-00003.bin",
87
+ "decoder.block.12.mlp.fc_out.weight": "pytorch_model-00002-of-00003.bin",
88
+ "decoder.block.12.post_attn_layernorm.bias": "pytorch_model-00002-of-00003.bin",
89
+ "decoder.block.12.post_attn_layernorm.weight": "pytorch_model-00002-of-00003.bin",
90
+ "decoder.block.12.post_inter_attn_layernorm.bias": "pytorch_model-00002-of-00003.bin",
91
+ "decoder.block.12.post_inter_attn_layernorm.weight": "pytorch_model-00002-of-00003.bin",
92
+ "decoder.block.12.self_attn.o.bias": "pytorch_model-00002-of-00003.bin",
93
+ "decoder.block.12.self_attn.o.weight": "pytorch_model-00002-of-00003.bin",
94
+ "decoder.block.12.self_attn.qkv.bias": "pytorch_model-00002-of-00003.bin",
95
+ "decoder.block.12.self_attn.qkv.weight": "pytorch_model-00002-of-00003.bin",
96
+ "decoder.block.13.input_layernorm.bias": "pytorch_model-00002-of-00003.bin",
97
+ "decoder.block.13.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
98
+ "decoder.block.13.inter_attn.kv.bias": "pytorch_model-00002-of-00003.bin",
99
+ "decoder.block.13.inter_attn.kv.weight": "pytorch_model-00002-of-00003.bin",
100
+ "decoder.block.13.inter_attn.o.bias": "pytorch_model-00002-of-00003.bin",
101
+ "decoder.block.13.inter_attn.o.weight": "pytorch_model-00002-of-00003.bin",
102
+ "decoder.block.13.inter_attn.q.bias": "pytorch_model-00002-of-00003.bin",
103
+ "decoder.block.13.inter_attn.q.weight": "pytorch_model-00002-of-00003.bin",
104
+ "decoder.block.13.mlp.fc_in.weight": "pytorch_model-00003-of-00003.bin",
105
+ "decoder.block.13.mlp.fc_out.weight": "pytorch_model-00002-of-00003.bin",
106
+ "decoder.block.13.post_attn_layernorm.bias": "pytorch_model-00002-of-00003.bin",
107
+ "decoder.block.13.post_attn_layernorm.weight": "pytorch_model-00002-of-00003.bin",
108
+ "decoder.block.13.post_inter_attn_layernorm.bias": "pytorch_model-00002-of-00003.bin",
109
+ "decoder.block.13.post_inter_attn_layernorm.weight": "pytorch_model-00002-of-00003.bin",
110
+ "decoder.block.13.self_attn.o.bias": "pytorch_model-00002-of-00003.bin",
111
+ "decoder.block.13.self_attn.o.weight": "pytorch_model-00002-of-00003.bin",
112
+ "decoder.block.13.self_attn.qkv.bias": "pytorch_model-00002-of-00003.bin",
113
+ "decoder.block.13.self_attn.qkv.weight": "pytorch_model-00002-of-00003.bin",
114
+ "decoder.block.14.input_layernorm.bias": "pytorch_model-00002-of-00003.bin",
115
+ "decoder.block.14.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
116
+ "decoder.block.14.inter_attn.kv.bias": "pytorch_model-00002-of-00003.bin",
117
+ "decoder.block.14.inter_attn.kv.weight": "pytorch_model-00002-of-00003.bin",
118
+ "decoder.block.14.inter_attn.o.bias": "pytorch_model-00002-of-00003.bin",
119
+ "decoder.block.14.inter_attn.o.weight": "pytorch_model-00002-of-00003.bin",
120
+ "decoder.block.14.inter_attn.q.bias": "pytorch_model-00002-of-00003.bin",
121
+ "decoder.block.14.inter_attn.q.weight": "pytorch_model-00002-of-00003.bin",
122
+ "decoder.block.14.mlp.fc_in.weight": "pytorch_model-00003-of-00003.bin",
123
+ "decoder.block.14.mlp.fc_out.weight": "pytorch_model-00002-of-00003.bin",
124
+ "decoder.block.14.post_attn_layernorm.bias": "pytorch_model-00002-of-00003.bin",
125
+ "decoder.block.14.post_attn_layernorm.weight": "pytorch_model-00002-of-00003.bin",
126
+ "decoder.block.14.post_inter_attn_layernorm.bias": "pytorch_model-00002-of-00003.bin",
127
+ "decoder.block.14.post_inter_attn_layernorm.weight": "pytorch_model-00002-of-00003.bin",
128
+ "decoder.block.14.self_attn.o.bias": "pytorch_model-00002-of-00003.bin",
129
+ "decoder.block.14.self_attn.o.weight": "pytorch_model-00002-of-00003.bin",
130
+ "decoder.block.14.self_attn.qkv.bias": "pytorch_model-00002-of-00003.bin",
131
+ "decoder.block.14.self_attn.qkv.weight": "pytorch_model-00002-of-00003.bin",
132
+ "decoder.block.15.input_layernorm.bias": "pytorch_model-00002-of-00003.bin",
133
+ "decoder.block.15.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
134
+ "decoder.block.15.inter_attn.kv.bias": "pytorch_model-00002-of-00003.bin",
135
+ "decoder.block.15.inter_attn.kv.weight": "pytorch_model-00002-of-00003.bin",
136
+ "decoder.block.15.inter_attn.o.bias": "pytorch_model-00002-of-00003.bin",
137
+ "decoder.block.15.inter_attn.o.weight": "pytorch_model-00002-of-00003.bin",
138
+ "decoder.block.15.inter_attn.q.bias": "pytorch_model-00002-of-00003.bin",
139
+ "decoder.block.15.inter_attn.q.weight": "pytorch_model-00002-of-00003.bin",
140
+ "decoder.block.15.mlp.fc_in.weight": "pytorch_model-00003-of-00003.bin",
141
+ "decoder.block.15.mlp.fc_out.weight": "pytorch_model-00002-of-00003.bin",
142
+ "decoder.block.15.post_attn_layernorm.bias": "pytorch_model-00002-of-00003.bin",
143
+ "decoder.block.15.post_attn_layernorm.weight": "pytorch_model-00002-of-00003.bin",
144
+ "decoder.block.15.post_inter_attn_layernorm.bias": "pytorch_model-00002-of-00003.bin",
145
+ "decoder.block.15.post_inter_attn_layernorm.weight": "pytorch_model-00002-of-00003.bin",
146
+ "decoder.block.15.self_attn.o.bias": "pytorch_model-00002-of-00003.bin",
147
+ "decoder.block.15.self_attn.o.weight": "pytorch_model-00002-of-00003.bin",
148
+ "decoder.block.15.self_attn.qkv.bias": "pytorch_model-00002-of-00003.bin",
149
+ "decoder.block.15.self_attn.qkv.weight": "pytorch_model-00002-of-00003.bin",
150
+ "decoder.block.16.input_layernorm.bias": "pytorch_model-00002-of-00003.bin",
151
+ "decoder.block.16.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
152
+ "decoder.block.16.inter_attn.kv.bias": "pytorch_model-00002-of-00003.bin",
153
+ "decoder.block.16.inter_attn.kv.weight": "pytorch_model-00002-of-00003.bin",
154
+ "decoder.block.16.inter_attn.o.bias": "pytorch_model-00002-of-00003.bin",
155
+ "decoder.block.16.inter_attn.o.weight": "pytorch_model-00002-of-00003.bin",
156
+ "decoder.block.16.inter_attn.q.bias": "pytorch_model-00002-of-00003.bin",
157
+ "decoder.block.16.inter_attn.q.weight": "pytorch_model-00002-of-00003.bin",
158
+ "decoder.block.16.mlp.fc_in.weight": "pytorch_model-00003-of-00003.bin",
159
+ "decoder.block.16.mlp.fc_out.weight": "pytorch_model-00002-of-00003.bin",
160
+ "decoder.block.16.post_attn_layernorm.bias": "pytorch_model-00002-of-00003.bin",
161
+ "decoder.block.16.post_attn_layernorm.weight": "pytorch_model-00002-of-00003.bin",
162
+ "decoder.block.16.post_inter_attn_layernorm.bias": "pytorch_model-00002-of-00003.bin",
163
+ "decoder.block.16.post_inter_attn_layernorm.weight": "pytorch_model-00002-of-00003.bin",
164
+ "decoder.block.16.self_attn.o.bias": "pytorch_model-00002-of-00003.bin",
165
+ "decoder.block.16.self_attn.o.weight": "pytorch_model-00002-of-00003.bin",
166
+ "decoder.block.16.self_attn.qkv.bias": "pytorch_model-00002-of-00003.bin",
167
+ "decoder.block.16.self_attn.qkv.weight": "pytorch_model-00002-of-00003.bin",
168
+ "decoder.block.17.input_layernorm.bias": "pytorch_model-00002-of-00003.bin",
169
+ "decoder.block.17.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
170
+ "decoder.block.17.inter_attn.kv.bias": "pytorch_model-00002-of-00003.bin",
171
+ "decoder.block.17.inter_attn.kv.weight": "pytorch_model-00002-of-00003.bin",
172
+ "decoder.block.17.inter_attn.o.bias": "pytorch_model-00002-of-00003.bin",
173
+ "decoder.block.17.inter_attn.o.weight": "pytorch_model-00002-of-00003.bin",
174
+ "decoder.block.17.inter_attn.q.bias": "pytorch_model-00002-of-00003.bin",
175
+ "decoder.block.17.inter_attn.q.weight": "pytorch_model-00002-of-00003.bin",
176
+ "decoder.block.17.mlp.fc_in.weight": "pytorch_model-00003-of-00003.bin",
177
+ "decoder.block.17.mlp.fc_out.weight": "pytorch_model-00002-of-00003.bin",
178
+ "decoder.block.17.post_attn_layernorm.bias": "pytorch_model-00002-of-00003.bin",
179
+ "decoder.block.17.post_attn_layernorm.weight": "pytorch_model-00002-of-00003.bin",
180
+ "decoder.block.17.post_inter_attn_layernorm.bias": "pytorch_model-00002-of-00003.bin",
181
+ "decoder.block.17.post_inter_attn_layernorm.weight": "pytorch_model-00002-of-00003.bin",
182
+ "decoder.block.17.self_attn.o.bias": "pytorch_model-00002-of-00003.bin",
183
+ "decoder.block.17.self_attn.o.weight": "pytorch_model-00002-of-00003.bin",
184
+ "decoder.block.17.self_attn.qkv.bias": "pytorch_model-00002-of-00003.bin",
185
+ "decoder.block.17.self_attn.qkv.weight": "pytorch_model-00002-of-00003.bin",
186
+ "decoder.block.18.input_layernorm.bias": "pytorch_model-00002-of-00003.bin",
187
+ "decoder.block.18.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
188
+ "decoder.block.18.inter_attn.kv.bias": "pytorch_model-00002-of-00003.bin",
189
+ "decoder.block.18.inter_attn.kv.weight": "pytorch_model-00002-of-00003.bin",
190
+ "decoder.block.18.inter_attn.o.bias": "pytorch_model-00002-of-00003.bin",
191
+ "decoder.block.18.inter_attn.o.weight": "pytorch_model-00002-of-00003.bin",
192
+ "decoder.block.18.inter_attn.q.bias": "pytorch_model-00002-of-00003.bin",
193
+ "decoder.block.18.inter_attn.q.weight": "pytorch_model-00002-of-00003.bin",
194
+ "decoder.block.18.mlp.fc_in.weight": "pytorch_model-00003-of-00003.bin",
195
+ "decoder.block.18.mlp.fc_out.weight": "pytorch_model-00002-of-00003.bin",
196
+ "decoder.block.18.post_attn_layernorm.bias": "pytorch_model-00002-of-00003.bin",
197
+ "decoder.block.18.post_attn_layernorm.weight": "pytorch_model-00002-of-00003.bin",
198
+ "decoder.block.18.post_inter_attn_layernorm.bias": "pytorch_model-00002-of-00003.bin",
199
+ "decoder.block.18.post_inter_attn_layernorm.weight": "pytorch_model-00002-of-00003.bin",
200
+ "decoder.block.18.self_attn.o.bias": "pytorch_model-00002-of-00003.bin",
201
+ "decoder.block.18.self_attn.o.weight": "pytorch_model-00002-of-00003.bin",
202
+ "decoder.block.18.self_attn.qkv.bias": "pytorch_model-00002-of-00003.bin",
203
+ "decoder.block.18.self_attn.qkv.weight": "pytorch_model-00002-of-00003.bin",
204
+ "decoder.block.19.input_layernorm.bias": "pytorch_model-00002-of-00003.bin",
205
+ "decoder.block.19.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
206
+ "decoder.block.19.inter_attn.kv.bias": "pytorch_model-00002-of-00003.bin",
207
+ "decoder.block.19.inter_attn.kv.weight": "pytorch_model-00002-of-00003.bin",
208
+ "decoder.block.19.inter_attn.o.bias": "pytorch_model-00002-of-00003.bin",
209
+ "decoder.block.19.inter_attn.o.weight": "pytorch_model-00002-of-00003.bin",
210
+ "decoder.block.19.inter_attn.q.bias": "pytorch_model-00002-of-00003.bin",
211
+ "decoder.block.19.inter_attn.q.weight": "pytorch_model-00002-of-00003.bin",
212
+ "decoder.block.19.mlp.fc_in.weight": "pytorch_model-00003-of-00003.bin",
213
+ "decoder.block.19.mlp.fc_out.weight": "pytorch_model-00002-of-00003.bin",
214
+ "decoder.block.19.post_attn_layernorm.bias": "pytorch_model-00002-of-00003.bin",
215
+ "decoder.block.19.post_attn_layernorm.weight": "pytorch_model-00002-of-00003.bin",
216
+ "decoder.block.19.post_inter_attn_layernorm.bias": "pytorch_model-00002-of-00003.bin",
217
+ "decoder.block.19.post_inter_attn_layernorm.weight": "pytorch_model-00002-of-00003.bin",
218
+ "decoder.block.19.self_attn.o.bias": "pytorch_model-00002-of-00003.bin",
219
+ "decoder.block.19.self_attn.o.weight": "pytorch_model-00002-of-00003.bin",
220
+ "decoder.block.19.self_attn.qkv.bias": "pytorch_model-00002-of-00003.bin",
221
+ "decoder.block.19.self_attn.qkv.weight": "pytorch_model-00002-of-00003.bin",
222
+ "decoder.block.2.input_layernorm.bias": "pytorch_model-00001-of-00003.bin",
223
+ "decoder.block.2.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
224
+ "decoder.block.2.inter_attn.kv.bias": "pytorch_model-00001-of-00003.bin",
225
+ "decoder.block.2.inter_attn.kv.weight": "pytorch_model-00001-of-00003.bin",
226
+ "decoder.block.2.inter_attn.o.bias": "pytorch_model-00001-of-00003.bin",
227
+ "decoder.block.2.inter_attn.o.weight": "pytorch_model-00001-of-00003.bin",
228
+ "decoder.block.2.inter_attn.q.bias": "pytorch_model-00001-of-00003.bin",
229
+ "decoder.block.2.inter_attn.q.weight": "pytorch_model-00001-of-00003.bin",
230
+ "decoder.block.2.mlp.fc_in.weight": "pytorch_model-00003-of-00003.bin",
231
+ "decoder.block.2.mlp.fc_out.weight": "pytorch_model-00001-of-00003.bin",
232
+ "decoder.block.2.post_attn_layernorm.bias": "pytorch_model-00001-of-00003.bin",
233
+ "decoder.block.2.post_attn_layernorm.weight": "pytorch_model-00001-of-00003.bin",
234
+ "decoder.block.2.post_inter_attn_layernorm.bias": "pytorch_model-00001-of-00003.bin",
235
+ "decoder.block.2.post_inter_attn_layernorm.weight": "pytorch_model-00001-of-00003.bin",
236
+ "decoder.block.2.self_attn.o.bias": "pytorch_model-00001-of-00003.bin",
237
+ "decoder.block.2.self_attn.o.weight": "pytorch_model-00001-of-00003.bin",
238
+ "decoder.block.2.self_attn.qkv.bias": "pytorch_model-00001-of-00003.bin",
239
+ "decoder.block.2.self_attn.qkv.weight": "pytorch_model-00001-of-00003.bin",
240
+ "decoder.block.20.input_layernorm.bias": "pytorch_model-00002-of-00003.bin",
241
+ "decoder.block.20.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
242
+ "decoder.block.20.inter_attn.kv.bias": "pytorch_model-00002-of-00003.bin",
243
+ "decoder.block.20.inter_attn.kv.weight": "pytorch_model-00002-of-00003.bin",
244
+ "decoder.block.20.inter_attn.o.bias": "pytorch_model-00002-of-00003.bin",
245
+ "decoder.block.20.inter_attn.o.weight": "pytorch_model-00002-of-00003.bin",
246
+ "decoder.block.20.inter_attn.q.bias": "pytorch_model-00002-of-00003.bin",
247
+ "decoder.block.20.inter_attn.q.weight": "pytorch_model-00002-of-00003.bin",
248
+ "decoder.block.20.mlp.fc_in.weight": "pytorch_model-00003-of-00003.bin",
249
+ "decoder.block.20.mlp.fc_out.weight": "pytorch_model-00002-of-00003.bin",
250
+ "decoder.block.20.post_attn_layernorm.bias": "pytorch_model-00002-of-00003.bin",
251
+ "decoder.block.20.post_attn_layernorm.weight": "pytorch_model-00002-of-00003.bin",
252
+ "decoder.block.20.post_inter_attn_layernorm.bias": "pytorch_model-00002-of-00003.bin",
253
+ "decoder.block.20.post_inter_attn_layernorm.weight": "pytorch_model-00002-of-00003.bin",
254
+ "decoder.block.20.self_attn.o.bias": "pytorch_model-00002-of-00003.bin",
255
+ "decoder.block.20.self_attn.o.weight": "pytorch_model-00002-of-00003.bin",
256
+ "decoder.block.20.self_attn.qkv.bias": "pytorch_model-00002-of-00003.bin",
257
+ "decoder.block.20.self_attn.qkv.weight": "pytorch_model-00002-of-00003.bin",
258
+ "decoder.block.21.input_layernorm.bias": "pytorch_model-00002-of-00003.bin",
259
+ "decoder.block.21.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
260
+ "decoder.block.21.inter_attn.kv.bias": "pytorch_model-00002-of-00003.bin",
261
+ "decoder.block.21.inter_attn.kv.weight": "pytorch_model-00002-of-00003.bin",
262
+ "decoder.block.21.inter_attn.o.bias": "pytorch_model-00002-of-00003.bin",
263
+ "decoder.block.21.inter_attn.o.weight": "pytorch_model-00002-of-00003.bin",
264
+ "decoder.block.21.inter_attn.q.bias": "pytorch_model-00002-of-00003.bin",
265
+ "decoder.block.21.inter_attn.q.weight": "pytorch_model-00002-of-00003.bin",
266
+ "decoder.block.21.mlp.fc_in.weight": "pytorch_model-00003-of-00003.bin",
267
+ "decoder.block.21.mlp.fc_out.weight": "pytorch_model-00002-of-00003.bin",
268
+ "decoder.block.21.post_attn_layernorm.bias": "pytorch_model-00002-of-00003.bin",
269
+ "decoder.block.21.post_attn_layernorm.weight": "pytorch_model-00002-of-00003.bin",
270
+ "decoder.block.21.post_inter_attn_layernorm.bias": "pytorch_model-00002-of-00003.bin",
271
+ "decoder.block.21.post_inter_attn_layernorm.weight": "pytorch_model-00002-of-00003.bin",
272
+ "decoder.block.21.self_attn.o.bias": "pytorch_model-00002-of-00003.bin",
273
+ "decoder.block.21.self_attn.o.weight": "pytorch_model-00002-of-00003.bin",
274
+ "decoder.block.21.self_attn.qkv.bias": "pytorch_model-00002-of-00003.bin",
275
+ "decoder.block.21.self_attn.qkv.weight": "pytorch_model-00002-of-00003.bin",
276
+ "decoder.block.22.input_layernorm.bias": "pytorch_model-00002-of-00003.bin",
277
+ "decoder.block.22.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
278
+ "decoder.block.22.inter_attn.kv.bias": "pytorch_model-00002-of-00003.bin",
279
+ "decoder.block.22.inter_attn.kv.weight": "pytorch_model-00002-of-00003.bin",
280
+ "decoder.block.22.inter_attn.o.bias": "pytorch_model-00002-of-00003.bin",
281
+ "decoder.block.22.inter_attn.o.weight": "pytorch_model-00002-of-00003.bin",
282
+ "decoder.block.22.inter_attn.q.bias": "pytorch_model-00002-of-00003.bin",
283
+ "decoder.block.22.inter_attn.q.weight": "pytorch_model-00002-of-00003.bin",
284
+ "decoder.block.22.mlp.fc_in.weight": "pytorch_model-00003-of-00003.bin",
285
+ "decoder.block.22.mlp.fc_out.weight": "pytorch_model-00002-of-00003.bin",
286
+ "decoder.block.22.post_attn_layernorm.bias": "pytorch_model-00002-of-00003.bin",
287
+ "decoder.block.22.post_attn_layernorm.weight": "pytorch_model-00002-of-00003.bin",
288
+ "decoder.block.22.post_inter_attn_layernorm.bias": "pytorch_model-00002-of-00003.bin",
289
+ "decoder.block.22.post_inter_attn_layernorm.weight": "pytorch_model-00002-of-00003.bin",
290
+ "decoder.block.22.self_attn.o.bias": "pytorch_model-00002-of-00003.bin",
291
+ "decoder.block.22.self_attn.o.weight": "pytorch_model-00002-of-00003.bin",
292
+ "decoder.block.22.self_attn.qkv.bias": "pytorch_model-00002-of-00003.bin",
293
+ "decoder.block.22.self_attn.qkv.weight": "pytorch_model-00002-of-00003.bin",
294
+ "decoder.block.23.input_layernorm.bias": "pytorch_model-00002-of-00003.bin",
295
+ "decoder.block.23.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
296
+ "decoder.block.23.inter_attn.kv.bias": "pytorch_model-00002-of-00003.bin",
297
+ "decoder.block.23.inter_attn.kv.weight": "pytorch_model-00002-of-00003.bin",
298
+ "decoder.block.23.inter_attn.o.bias": "pytorch_model-00002-of-00003.bin",
299
+ "decoder.block.23.inter_attn.o.weight": "pytorch_model-00002-of-00003.bin",
300
+ "decoder.block.23.inter_attn.q.bias": "pytorch_model-00002-of-00003.bin",
301
+ "decoder.block.23.inter_attn.q.weight": "pytorch_model-00002-of-00003.bin",
302
+ "decoder.block.23.mlp.fc_in.weight": "pytorch_model-00003-of-00003.bin",
303
+ "decoder.block.23.mlp.fc_out.weight": "pytorch_model-00002-of-00003.bin",
304
+ "decoder.block.23.post_attn_layernorm.bias": "pytorch_model-00002-of-00003.bin",
305
+ "decoder.block.23.post_attn_layernorm.weight": "pytorch_model-00002-of-00003.bin",
306
+ "decoder.block.23.post_inter_attn_layernorm.bias": "pytorch_model-00002-of-00003.bin",
307
+ "decoder.block.23.post_inter_attn_layernorm.weight": "pytorch_model-00002-of-00003.bin",
308
+ "decoder.block.23.self_attn.o.bias": "pytorch_model-00002-of-00003.bin",
309
+ "decoder.block.23.self_attn.o.weight": "pytorch_model-00002-of-00003.bin",
310
+ "decoder.block.23.self_attn.qkv.bias": "pytorch_model-00002-of-00003.bin",
311
+ "decoder.block.23.self_attn.qkv.weight": "pytorch_model-00002-of-00003.bin",
312
+ "decoder.block.24.input_layernorm.bias": "pytorch_model-00002-of-00003.bin",
313
+ "decoder.block.24.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
314
+ "decoder.block.24.inter_attn.kv.bias": "pytorch_model-00002-of-00003.bin",
315
+ "decoder.block.24.inter_attn.kv.weight": "pytorch_model-00002-of-00003.bin",
316
+ "decoder.block.24.inter_attn.o.bias": "pytorch_model-00002-of-00003.bin",
317
+ "decoder.block.24.inter_attn.o.weight": "pytorch_model-00002-of-00003.bin",
318
+ "decoder.block.24.inter_attn.q.bias": "pytorch_model-00002-of-00003.bin",
319
+ "decoder.block.24.inter_attn.q.weight": "pytorch_model-00002-of-00003.bin",
320
+ "decoder.block.24.mlp.fc_in.weight": "pytorch_model-00003-of-00003.bin",
321
+ "decoder.block.24.mlp.fc_out.weight": "pytorch_model-00002-of-00003.bin",
322
+ "decoder.block.24.post_attn_layernorm.bias": "pytorch_model-00002-of-00003.bin",
323
+ "decoder.block.24.post_attn_layernorm.weight": "pytorch_model-00002-of-00003.bin",
324
+ "decoder.block.24.post_inter_attn_layernorm.bias": "pytorch_model-00002-of-00003.bin",
325
+ "decoder.block.24.post_inter_attn_layernorm.weight": "pytorch_model-00002-of-00003.bin",
326
+ "decoder.block.24.self_attn.o.bias": "pytorch_model-00002-of-00003.bin",
327
+ "decoder.block.24.self_attn.o.weight": "pytorch_model-00002-of-00003.bin",
328
+ "decoder.block.24.self_attn.qkv.bias": "pytorch_model-00002-of-00003.bin",
329
+ "decoder.block.24.self_attn.qkv.weight": "pytorch_model-00002-of-00003.bin",
330
+ "decoder.block.25.input_layernorm.bias": "pytorch_model-00002-of-00003.bin",
331
+ "decoder.block.25.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
332
+ "decoder.block.25.inter_attn.kv.bias": "pytorch_model-00002-of-00003.bin",
333
+ "decoder.block.25.inter_attn.kv.weight": "pytorch_model-00002-of-00003.bin",
334
+ "decoder.block.25.inter_attn.o.bias": "pytorch_model-00002-of-00003.bin",
335
+ "decoder.block.25.inter_attn.o.weight": "pytorch_model-00002-of-00003.bin",
336
+ "decoder.block.25.inter_attn.q.bias": "pytorch_model-00002-of-00003.bin",
337
+ "decoder.block.25.inter_attn.q.weight": "pytorch_model-00002-of-00003.bin",
338
+ "decoder.block.25.mlp.fc_in.weight": "pytorch_model-00003-of-00003.bin",
339
+ "decoder.block.25.mlp.fc_out.weight": "pytorch_model-00002-of-00003.bin",
340
+ "decoder.block.25.post_attn_layernorm.bias": "pytorch_model-00002-of-00003.bin",
341
+ "decoder.block.25.post_attn_layernorm.weight": "pytorch_model-00002-of-00003.bin",
342
+ "decoder.block.25.post_inter_attn_layernorm.bias": "pytorch_model-00002-of-00003.bin",
343
+ "decoder.block.25.post_inter_attn_layernorm.weight": "pytorch_model-00002-of-00003.bin",
344
+ "decoder.block.25.self_attn.o.bias": "pytorch_model-00002-of-00003.bin",
345
+ "decoder.block.25.self_attn.o.weight": "pytorch_model-00002-of-00003.bin",
346
+ "decoder.block.25.self_attn.qkv.bias": "pytorch_model-00002-of-00003.bin",
347
+ "decoder.block.25.self_attn.qkv.weight": "pytorch_model-00002-of-00003.bin",
348
+ "decoder.block.26.input_layernorm.bias": "pytorch_model-00002-of-00003.bin",
349
+ "decoder.block.26.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
350
+ "decoder.block.26.inter_attn.kv.bias": "pytorch_model-00002-of-00003.bin",
351
+ "decoder.block.26.inter_attn.kv.weight": "pytorch_model-00002-of-00003.bin",
352
+ "decoder.block.26.inter_attn.o.bias": "pytorch_model-00002-of-00003.bin",
353
+ "decoder.block.26.inter_attn.o.weight": "pytorch_model-00002-of-00003.bin",
354
+ "decoder.block.26.inter_attn.q.bias": "pytorch_model-00002-of-00003.bin",
355
+ "decoder.block.26.inter_attn.q.weight": "pytorch_model-00002-of-00003.bin",
356
+ "decoder.block.26.mlp.fc_in.weight": "pytorch_model-00003-of-00003.bin",
357
+ "decoder.block.26.mlp.fc_out.weight": "pytorch_model-00002-of-00003.bin",
358
+ "decoder.block.26.post_attn_layernorm.bias": "pytorch_model-00002-of-00003.bin",
359
+ "decoder.block.26.post_attn_layernorm.weight": "pytorch_model-00002-of-00003.bin",
360
+ "decoder.block.26.post_inter_attn_layernorm.bias": "pytorch_model-00002-of-00003.bin",
361
+ "decoder.block.26.post_inter_attn_layernorm.weight": "pytorch_model-00002-of-00003.bin",
362
+ "decoder.block.26.self_attn.o.bias": "pytorch_model-00002-of-00003.bin",
363
+ "decoder.block.26.self_attn.o.weight": "pytorch_model-00002-of-00003.bin",
364
+ "decoder.block.26.self_attn.qkv.bias": "pytorch_model-00002-of-00003.bin",
365
+ "decoder.block.26.self_attn.qkv.weight": "pytorch_model-00002-of-00003.bin",
366
+ "decoder.block.27.input_layernorm.bias": "pytorch_model-00002-of-00003.bin",
367
+ "decoder.block.27.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
368
+ "decoder.block.27.inter_attn.kv.bias": "pytorch_model-00002-of-00003.bin",
369
+ "decoder.block.27.inter_attn.kv.weight": "pytorch_model-00002-of-00003.bin",
370
+ "decoder.block.27.inter_attn.o.bias": "pytorch_model-00002-of-00003.bin",
371
+ "decoder.block.27.inter_attn.o.weight": "pytorch_model-00002-of-00003.bin",
372
+ "decoder.block.27.inter_attn.q.bias": "pytorch_model-00002-of-00003.bin",
373
+ "decoder.block.27.inter_attn.q.weight": "pytorch_model-00002-of-00003.bin",
374
+ "decoder.block.27.mlp.fc_in.weight": "pytorch_model-00003-of-00003.bin",
375
+ "decoder.block.27.mlp.fc_out.weight": "pytorch_model-00002-of-00003.bin",
376
+ "decoder.block.27.post_attn_layernorm.bias": "pytorch_model-00002-of-00003.bin",
377
+ "decoder.block.27.post_attn_layernorm.weight": "pytorch_model-00002-of-00003.bin",
378
+ "decoder.block.27.post_inter_attn_layernorm.bias": "pytorch_model-00002-of-00003.bin",
379
+ "decoder.block.27.post_inter_attn_layernorm.weight": "pytorch_model-00002-of-00003.bin",
380
+ "decoder.block.27.self_attn.o.bias": "pytorch_model-00002-of-00003.bin",
381
+ "decoder.block.27.self_attn.o.weight": "pytorch_model-00002-of-00003.bin",
382
+ "decoder.block.27.self_attn.qkv.bias": "pytorch_model-00002-of-00003.bin",
383
+ "decoder.block.27.self_attn.qkv.weight": "pytorch_model-00002-of-00003.bin",
384
+ "decoder.block.28.input_layernorm.bias": "pytorch_model-00002-of-00003.bin",
385
+ "decoder.block.28.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
386
+ "decoder.block.28.inter_attn.kv.bias": "pytorch_model-00002-of-00003.bin",
387
+ "decoder.block.28.inter_attn.kv.weight": "pytorch_model-00002-of-00003.bin",
388
+ "decoder.block.28.inter_attn.o.bias": "pytorch_model-00002-of-00003.bin",
389
+ "decoder.block.28.inter_attn.o.weight": "pytorch_model-00002-of-00003.bin",
390
+ "decoder.block.28.inter_attn.q.bias": "pytorch_model-00002-of-00003.bin",
391
+ "decoder.block.28.inter_attn.q.weight": "pytorch_model-00002-of-00003.bin",
392
+ "decoder.block.28.mlp.fc_in.weight": "pytorch_model-00003-of-00003.bin",
393
+ "decoder.block.28.mlp.fc_out.weight": "pytorch_model-00002-of-00003.bin",
394
+ "decoder.block.28.post_attn_layernorm.bias": "pytorch_model-00002-of-00003.bin",
395
+ "decoder.block.28.post_attn_layernorm.weight": "pytorch_model-00002-of-00003.bin",
396
+ "decoder.block.28.post_inter_attn_layernorm.bias": "pytorch_model-00002-of-00003.bin",
397
+ "decoder.block.28.post_inter_attn_layernorm.weight": "pytorch_model-00002-of-00003.bin",
398
+ "decoder.block.28.self_attn.o.bias": "pytorch_model-00002-of-00003.bin",
399
+ "decoder.block.28.self_attn.o.weight": "pytorch_model-00002-of-00003.bin",
400
+ "decoder.block.28.self_attn.qkv.bias": "pytorch_model-00002-of-00003.bin",
401
+ "decoder.block.28.self_attn.qkv.weight": "pytorch_model-00002-of-00003.bin",
402
+ "decoder.block.29.input_layernorm.bias": "pytorch_model-00002-of-00003.bin",
403
+ "decoder.block.29.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
404
+ "decoder.block.29.inter_attn.kv.bias": "pytorch_model-00002-of-00003.bin",
405
+ "decoder.block.29.inter_attn.kv.weight": "pytorch_model-00002-of-00003.bin",
406
+ "decoder.block.29.inter_attn.o.bias": "pytorch_model-00003-of-00003.bin",
407
+ "decoder.block.29.inter_attn.o.weight": "pytorch_model-00003-of-00003.bin",
408
+ "decoder.block.29.inter_attn.q.bias": "pytorch_model-00002-of-00003.bin",
409
+ "decoder.block.29.inter_attn.q.weight": "pytorch_model-00002-of-00003.bin",
410
+ "decoder.block.29.mlp.fc_in.weight": "pytorch_model-00003-of-00003.bin",
411
+ "decoder.block.29.mlp.fc_out.weight": "pytorch_model-00003-of-00003.bin",
412
+ "decoder.block.29.post_attn_layernorm.bias": "pytorch_model-00002-of-00003.bin",
413
+ "decoder.block.29.post_attn_layernorm.weight": "pytorch_model-00002-of-00003.bin",
414
+ "decoder.block.29.post_inter_attn_layernorm.bias": "pytorch_model-00003-of-00003.bin",
415
+ "decoder.block.29.post_inter_attn_layernorm.weight": "pytorch_model-00003-of-00003.bin",
416
+ "decoder.block.29.self_attn.o.bias": "pytorch_model-00002-of-00003.bin",
417
+ "decoder.block.29.self_attn.o.weight": "pytorch_model-00002-of-00003.bin",
418
+ "decoder.block.29.self_attn.qkv.bias": "pytorch_model-00002-of-00003.bin",
419
+ "decoder.block.29.self_attn.qkv.weight": "pytorch_model-00002-of-00003.bin",
420
+ "decoder.block.3.input_layernorm.bias": "pytorch_model-00001-of-00003.bin",
421
+ "decoder.block.3.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
422
+ "decoder.block.3.inter_attn.kv.bias": "pytorch_model-00001-of-00003.bin",
423
+ "decoder.block.3.inter_attn.kv.weight": "pytorch_model-00001-of-00003.bin",
424
+ "decoder.block.3.inter_attn.o.bias": "pytorch_model-00001-of-00003.bin",
425
+ "decoder.block.3.inter_attn.o.weight": "pytorch_model-00001-of-00003.bin",
426
+ "decoder.block.3.inter_attn.q.bias": "pytorch_model-00001-of-00003.bin",
427
+ "decoder.block.3.inter_attn.q.weight": "pytorch_model-00001-of-00003.bin",
428
+ "decoder.block.3.mlp.fc_in.weight": "pytorch_model-00003-of-00003.bin",
429
+ "decoder.block.3.mlp.fc_out.weight": "pytorch_model-00001-of-00003.bin",
430
+ "decoder.block.3.post_attn_layernorm.bias": "pytorch_model-00001-of-00003.bin",
431
+ "decoder.block.3.post_attn_layernorm.weight": "pytorch_model-00001-of-00003.bin",
432
+ "decoder.block.3.post_inter_attn_layernorm.bias": "pytorch_model-00001-of-00003.bin",
433
+ "decoder.block.3.post_inter_attn_layernorm.weight": "pytorch_model-00001-of-00003.bin",
434
+ "decoder.block.3.self_attn.o.bias": "pytorch_model-00001-of-00003.bin",
435
+ "decoder.block.3.self_attn.o.weight": "pytorch_model-00001-of-00003.bin",
436
+ "decoder.block.3.self_attn.qkv.bias": "pytorch_model-00001-of-00003.bin",
437
+ "decoder.block.3.self_attn.qkv.weight": "pytorch_model-00001-of-00003.bin",
438
+ "decoder.block.30.input_layernorm.bias": "pytorch_model-00003-of-00003.bin",
439
+ "decoder.block.30.input_layernorm.weight": "pytorch_model-00003-of-00003.bin",
440
+ "decoder.block.30.inter_attn.kv.bias": "pytorch_model-00003-of-00003.bin",
441
+ "decoder.block.30.inter_attn.kv.weight": "pytorch_model-00003-of-00003.bin",
442
+ "decoder.block.30.inter_attn.o.bias": "pytorch_model-00003-of-00003.bin",
443
+ "decoder.block.30.inter_attn.o.weight": "pytorch_model-00003-of-00003.bin",
444
+ "decoder.block.30.inter_attn.q.bias": "pytorch_model-00003-of-00003.bin",
445
+ "decoder.block.30.inter_attn.q.weight": "pytorch_model-00003-of-00003.bin",
446
+ "decoder.block.30.mlp.fc_in.weight": "pytorch_model-00003-of-00003.bin",
447
+ "decoder.block.30.mlp.fc_out.weight": "pytorch_model-00003-of-00003.bin",
448
+ "decoder.block.30.post_attn_layernorm.bias": "pytorch_model-00003-of-00003.bin",
449
+ "decoder.block.30.post_attn_layernorm.weight": "pytorch_model-00003-of-00003.bin",
450
+ "decoder.block.30.post_inter_attn_layernorm.bias": "pytorch_model-00003-of-00003.bin",
451
+ "decoder.block.30.post_inter_attn_layernorm.weight": "pytorch_model-00003-of-00003.bin",
452
+ "decoder.block.30.self_attn.o.bias": "pytorch_model-00003-of-00003.bin",
453
+ "decoder.block.30.self_attn.o.weight": "pytorch_model-00003-of-00003.bin",
454
+ "decoder.block.30.self_attn.qkv.bias": "pytorch_model-00003-of-00003.bin",
455
+ "decoder.block.30.self_attn.qkv.weight": "pytorch_model-00003-of-00003.bin",
456
+ "decoder.block.31.input_layernorm.bias": "pytorch_model-00003-of-00003.bin",
457
+ "decoder.block.31.input_layernorm.weight": "pytorch_model-00003-of-00003.bin",
458
+ "decoder.block.31.inter_attn.kv.bias": "pytorch_model-00003-of-00003.bin",
459
+ "decoder.block.31.inter_attn.kv.weight": "pytorch_model-00003-of-00003.bin",
460
+ "decoder.block.31.inter_attn.o.bias": "pytorch_model-00003-of-00003.bin",
461
+ "decoder.block.31.inter_attn.o.weight": "pytorch_model-00003-of-00003.bin",
462
+ "decoder.block.31.inter_attn.q.bias": "pytorch_model-00003-of-00003.bin",
463
+ "decoder.block.31.inter_attn.q.weight": "pytorch_model-00003-of-00003.bin",
464
+ "decoder.block.31.mlp.fc_in.weight": "pytorch_model-00003-of-00003.bin",
465
+ "decoder.block.31.mlp.fc_out.weight": "pytorch_model-00003-of-00003.bin",
466
+ "decoder.block.31.post_attn_layernorm.bias": "pytorch_model-00003-of-00003.bin",
467
+ "decoder.block.31.post_attn_layernorm.weight": "pytorch_model-00003-of-00003.bin",
468
+ "decoder.block.31.post_inter_attn_layernorm.bias": "pytorch_model-00003-of-00003.bin",
469
+ "decoder.block.31.post_inter_attn_layernorm.weight": "pytorch_model-00003-of-00003.bin",
470
+ "decoder.block.31.self_attn.o.bias": "pytorch_model-00003-of-00003.bin",
471
+ "decoder.block.31.self_attn.o.weight": "pytorch_model-00003-of-00003.bin",
472
+ "decoder.block.31.self_attn.qkv.bias": "pytorch_model-00003-of-00003.bin",
473
+ "decoder.block.31.self_attn.qkv.weight": "pytorch_model-00003-of-00003.bin",
474
+ "decoder.block.32.input_layernorm.bias": "pytorch_model-00003-of-00003.bin",
475
+ "decoder.block.32.input_layernorm.weight": "pytorch_model-00003-of-00003.bin",
476
+ "decoder.block.32.inter_attn.kv.bias": "pytorch_model-00003-of-00003.bin",
477
+ "decoder.block.32.inter_attn.kv.weight": "pytorch_model-00003-of-00003.bin",
478
+ "decoder.block.32.inter_attn.o.bias": "pytorch_model-00003-of-00003.bin",
479
+ "decoder.block.32.inter_attn.o.weight": "pytorch_model-00003-of-00003.bin",
480
+ "decoder.block.32.inter_attn.q.bias": "pytorch_model-00003-of-00003.bin",
481
+ "decoder.block.32.inter_attn.q.weight": "pytorch_model-00003-of-00003.bin",
482
+ "decoder.block.32.mlp.fc_in.weight": "pytorch_model-00003-of-00003.bin",
483
+ "decoder.block.32.mlp.fc_out.weight": "pytorch_model-00003-of-00003.bin",
484
+ "decoder.block.32.post_attn_layernorm.bias": "pytorch_model-00003-of-00003.bin",
485
+ "decoder.block.32.post_attn_layernorm.weight": "pytorch_model-00003-of-00003.bin",
486
+ "decoder.block.32.post_inter_attn_layernorm.bias": "pytorch_model-00003-of-00003.bin",
487
+ "decoder.block.32.post_inter_attn_layernorm.weight": "pytorch_model-00003-of-00003.bin",
488
+ "decoder.block.32.self_attn.o.bias": "pytorch_model-00003-of-00003.bin",
489
+ "decoder.block.32.self_attn.o.weight": "pytorch_model-00003-of-00003.bin",
490
+ "decoder.block.32.self_attn.qkv.bias": "pytorch_model-00003-of-00003.bin",
491
+ "decoder.block.32.self_attn.qkv.weight": "pytorch_model-00003-of-00003.bin",
492
+ "decoder.block.33.input_layernorm.bias": "pytorch_model-00003-of-00003.bin",
493
+ "decoder.block.33.input_layernorm.weight": "pytorch_model-00003-of-00003.bin",
494
+ "decoder.block.33.inter_attn.kv.bias": "pytorch_model-00003-of-00003.bin",
495
+ "decoder.block.33.inter_attn.kv.weight": "pytorch_model-00003-of-00003.bin",
496
+ "decoder.block.33.inter_attn.o.bias": "pytorch_model-00003-of-00003.bin",
497
+ "decoder.block.33.inter_attn.o.weight": "pytorch_model-00003-of-00003.bin",
498
+ "decoder.block.33.inter_attn.q.bias": "pytorch_model-00003-of-00003.bin",
499
+ "decoder.block.33.inter_attn.q.weight": "pytorch_model-00003-of-00003.bin",
500
+ "decoder.block.33.mlp.fc_in.weight": "pytorch_model-00003-of-00003.bin",
501
+ "decoder.block.33.mlp.fc_out.weight": "pytorch_model-00003-of-00003.bin",
502
+ "decoder.block.33.post_attn_layernorm.bias": "pytorch_model-00003-of-00003.bin",
503
+ "decoder.block.33.post_attn_layernorm.weight": "pytorch_model-00003-of-00003.bin",
504
+ "decoder.block.33.post_inter_attn_layernorm.bias": "pytorch_model-00003-of-00003.bin",
505
+ "decoder.block.33.post_inter_attn_layernorm.weight": "pytorch_model-00003-of-00003.bin",
506
+ "decoder.block.33.self_attn.o.bias": "pytorch_model-00003-of-00003.bin",
507
+ "decoder.block.33.self_attn.o.weight": "pytorch_model-00003-of-00003.bin",
508
+ "decoder.block.33.self_attn.qkv.bias": "pytorch_model-00003-of-00003.bin",
509
+ "decoder.block.33.self_attn.qkv.weight": "pytorch_model-00003-of-00003.bin",
510
+ "decoder.block.34.input_layernorm.bias": "pytorch_model-00003-of-00003.bin",
511
+ "decoder.block.34.input_layernorm.weight": "pytorch_model-00003-of-00003.bin",
512
+ "decoder.block.34.inter_attn.kv.bias": "pytorch_model-00003-of-00003.bin",
513
+ "decoder.block.34.inter_attn.kv.weight": "pytorch_model-00003-of-00003.bin",
514
+ "decoder.block.34.inter_attn.o.bias": "pytorch_model-00003-of-00003.bin",
515
+ "decoder.block.34.inter_attn.o.weight": "pytorch_model-00003-of-00003.bin",
516
+ "decoder.block.34.inter_attn.q.bias": "pytorch_model-00003-of-00003.bin",
517
+ "decoder.block.34.inter_attn.q.weight": "pytorch_model-00003-of-00003.bin",
518
+ "decoder.block.34.mlp.fc_in.weight": "pytorch_model-00003-of-00003.bin",
519
+ "decoder.block.34.mlp.fc_out.weight": "pytorch_model-00003-of-00003.bin",
520
+ "decoder.block.34.post_attn_layernorm.bias": "pytorch_model-00003-of-00003.bin",
521
+ "decoder.block.34.post_attn_layernorm.weight": "pytorch_model-00003-of-00003.bin",
522
+ "decoder.block.34.post_inter_attn_layernorm.bias": "pytorch_model-00003-of-00003.bin",
523
+ "decoder.block.34.post_inter_attn_layernorm.weight": "pytorch_model-00003-of-00003.bin",
524
+ "decoder.block.34.self_attn.o.bias": "pytorch_model-00003-of-00003.bin",
525
+ "decoder.block.34.self_attn.o.weight": "pytorch_model-00003-of-00003.bin",
526
+ "decoder.block.34.self_attn.qkv.bias": "pytorch_model-00003-of-00003.bin",
527
+ "decoder.block.34.self_attn.qkv.weight": "pytorch_model-00003-of-00003.bin",
528
+ "decoder.block.35.input_layernorm.bias": "pytorch_model-00003-of-00003.bin",
529
+ "decoder.block.35.input_layernorm.weight": "pytorch_model-00003-of-00003.bin",
530
+ "decoder.block.35.inter_attn.kv.bias": "pytorch_model-00003-of-00003.bin",
531
+ "decoder.block.35.inter_attn.kv.weight": "pytorch_model-00003-of-00003.bin",
532
+ "decoder.block.35.inter_attn.o.bias": "pytorch_model-00003-of-00003.bin",
533
+ "decoder.block.35.inter_attn.o.weight": "pytorch_model-00003-of-00003.bin",
534
+ "decoder.block.35.inter_attn.q.bias": "pytorch_model-00003-of-00003.bin",
535
+ "decoder.block.35.inter_attn.q.weight": "pytorch_model-00003-of-00003.bin",
536
+ "decoder.block.35.mlp.fc_in.weight": "pytorch_model-00003-of-00003.bin",
537
+ "decoder.block.35.mlp.fc_out.weight": "pytorch_model-00003-of-00003.bin",
538
+ "decoder.block.35.post_attn_layernorm.bias": "pytorch_model-00003-of-00003.bin",
539
+ "decoder.block.35.post_attn_layernorm.weight": "pytorch_model-00003-of-00003.bin",
540
+ "decoder.block.35.post_inter_attn_layernorm.bias": "pytorch_model-00003-of-00003.bin",
541
+ "decoder.block.35.post_inter_attn_layernorm.weight": "pytorch_model-00003-of-00003.bin",
542
+ "decoder.block.35.self_attn.o.bias": "pytorch_model-00003-of-00003.bin",
543
+ "decoder.block.35.self_attn.o.weight": "pytorch_model-00003-of-00003.bin",
544
+ "decoder.block.35.self_attn.qkv.bias": "pytorch_model-00003-of-00003.bin",
545
+ "decoder.block.35.self_attn.qkv.weight": "pytorch_model-00003-of-00003.bin",
546
+ "decoder.block.4.input_layernorm.bias": "pytorch_model-00001-of-00003.bin",
547
+ "decoder.block.4.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
548
+ "decoder.block.4.inter_attn.kv.bias": "pytorch_model-00001-of-00003.bin",
549
+ "decoder.block.4.inter_attn.kv.weight": "pytorch_model-00001-of-00003.bin",
550
+ "decoder.block.4.inter_attn.o.bias": "pytorch_model-00001-of-00003.bin",
551
+ "decoder.block.4.inter_attn.o.weight": "pytorch_model-00001-of-00003.bin",
552
+ "decoder.block.4.inter_attn.q.bias": "pytorch_model-00001-of-00003.bin",
553
+ "decoder.block.4.inter_attn.q.weight": "pytorch_model-00001-of-00003.bin",
554
+ "decoder.block.4.mlp.fc_in.weight": "pytorch_model-00003-of-00003.bin",
555
+ "decoder.block.4.mlp.fc_out.weight": "pytorch_model-00001-of-00003.bin",
556
+ "decoder.block.4.post_attn_layernorm.bias": "pytorch_model-00001-of-00003.bin",
557
+ "decoder.block.4.post_attn_layernorm.weight": "pytorch_model-00001-of-00003.bin",
558
+ "decoder.block.4.post_inter_attn_layernorm.bias": "pytorch_model-00001-of-00003.bin",
559
+ "decoder.block.4.post_inter_attn_layernorm.weight": "pytorch_model-00001-of-00003.bin",
560
+ "decoder.block.4.self_attn.o.bias": "pytorch_model-00001-of-00003.bin",
561
+ "decoder.block.4.self_attn.o.weight": "pytorch_model-00001-of-00003.bin",
562
+ "decoder.block.4.self_attn.qkv.bias": "pytorch_model-00001-of-00003.bin",
563
+ "decoder.block.4.self_attn.qkv.weight": "pytorch_model-00001-of-00003.bin",
564
+ "decoder.block.5.input_layernorm.bias": "pytorch_model-00001-of-00003.bin",
565
+ "decoder.block.5.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
566
+ "decoder.block.5.inter_attn.kv.bias": "pytorch_model-00001-of-00003.bin",
567
+ "decoder.block.5.inter_attn.kv.weight": "pytorch_model-00001-of-00003.bin",
568
+ "decoder.block.5.inter_attn.o.bias": "pytorch_model-00001-of-00003.bin",
569
+ "decoder.block.5.inter_attn.o.weight": "pytorch_model-00001-of-00003.bin",
570
+ "decoder.block.5.inter_attn.q.bias": "pytorch_model-00001-of-00003.bin",
571
+ "decoder.block.5.inter_attn.q.weight": "pytorch_model-00001-of-00003.bin",
572
+ "decoder.block.5.mlp.fc_in.weight": "pytorch_model-00003-of-00003.bin",
573
+ "decoder.block.5.mlp.fc_out.weight": "pytorch_model-00001-of-00003.bin",
574
+ "decoder.block.5.post_attn_layernorm.bias": "pytorch_model-00001-of-00003.bin",
575
+ "decoder.block.5.post_attn_layernorm.weight": "pytorch_model-00001-of-00003.bin",
576
+ "decoder.block.5.post_inter_attn_layernorm.bias": "pytorch_model-00001-of-00003.bin",
577
+ "decoder.block.5.post_inter_attn_layernorm.weight": "pytorch_model-00001-of-00003.bin",
578
+ "decoder.block.5.self_attn.o.bias": "pytorch_model-00001-of-00003.bin",
579
+ "decoder.block.5.self_attn.o.weight": "pytorch_model-00001-of-00003.bin",
580
+ "decoder.block.5.self_attn.qkv.bias": "pytorch_model-00001-of-00003.bin",
581
+ "decoder.block.5.self_attn.qkv.weight": "pytorch_model-00001-of-00003.bin",
582
+ "decoder.block.6.input_layernorm.bias": "pytorch_model-00001-of-00003.bin",
583
+ "decoder.block.6.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
584
+ "decoder.block.6.inter_attn.kv.bias": "pytorch_model-00002-of-00003.bin",
585
+ "decoder.block.6.inter_attn.kv.weight": "pytorch_model-00002-of-00003.bin",
586
+ "decoder.block.6.inter_attn.o.bias": "pytorch_model-00002-of-00003.bin",
587
+ "decoder.block.6.inter_attn.o.weight": "pytorch_model-00002-of-00003.bin",
588
+ "decoder.block.6.inter_attn.q.bias": "pytorch_model-00002-of-00003.bin",
589
+ "decoder.block.6.inter_attn.q.weight": "pytorch_model-00002-of-00003.bin",
590
+ "decoder.block.6.mlp.fc_in.weight": "pytorch_model-00003-of-00003.bin",
591
+ "decoder.block.6.mlp.fc_out.weight": "pytorch_model-00002-of-00003.bin",
592
+ "decoder.block.6.post_attn_layernorm.bias": "pytorch_model-00002-of-00003.bin",
593
+ "decoder.block.6.post_attn_layernorm.weight": "pytorch_model-00002-of-00003.bin",
594
+ "decoder.block.6.post_inter_attn_layernorm.bias": "pytorch_model-00002-of-00003.bin",
595
+ "decoder.block.6.post_inter_attn_layernorm.weight": "pytorch_model-00002-of-00003.bin",
596
+ "decoder.block.6.self_attn.o.bias": "pytorch_model-00002-of-00003.bin",
597
+ "decoder.block.6.self_attn.o.weight": "pytorch_model-00002-of-00003.bin",
598
+ "decoder.block.6.self_attn.qkv.bias": "pytorch_model-00001-of-00003.bin",
599
+ "decoder.block.6.self_attn.qkv.weight": "pytorch_model-00001-of-00003.bin",
600
+ "decoder.block.7.input_layernorm.bias": "pytorch_model-00002-of-00003.bin",
601
+ "decoder.block.7.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
602
+ "decoder.block.7.inter_attn.kv.bias": "pytorch_model-00002-of-00003.bin",
603
+ "decoder.block.7.inter_attn.kv.weight": "pytorch_model-00002-of-00003.bin",
604
+ "decoder.block.7.inter_attn.o.bias": "pytorch_model-00002-of-00003.bin",
605
+ "decoder.block.7.inter_attn.o.weight": "pytorch_model-00002-of-00003.bin",
606
+ "decoder.block.7.inter_attn.q.bias": "pytorch_model-00002-of-00003.bin",
607
+ "decoder.block.7.inter_attn.q.weight": "pytorch_model-00002-of-00003.bin",
608
+ "decoder.block.7.mlp.fc_in.weight": "pytorch_model-00003-of-00003.bin",
609
+ "decoder.block.7.mlp.fc_out.weight": "pytorch_model-00002-of-00003.bin",
610
+ "decoder.block.7.post_attn_layernorm.bias": "pytorch_model-00002-of-00003.bin",
611
+ "decoder.block.7.post_attn_layernorm.weight": "pytorch_model-00002-of-00003.bin",
612
+ "decoder.block.7.post_inter_attn_layernorm.bias": "pytorch_model-00002-of-00003.bin",
613
+ "decoder.block.7.post_inter_attn_layernorm.weight": "pytorch_model-00002-of-00003.bin",
614
+ "decoder.block.7.self_attn.o.bias": "pytorch_model-00002-of-00003.bin",
615
+ "decoder.block.7.self_attn.o.weight": "pytorch_model-00002-of-00003.bin",
616
+ "decoder.block.7.self_attn.qkv.bias": "pytorch_model-00002-of-00003.bin",
617
+ "decoder.block.7.self_attn.qkv.weight": "pytorch_model-00002-of-00003.bin",
618
+ "decoder.block.8.input_layernorm.bias": "pytorch_model-00002-of-00003.bin",
619
+ "decoder.block.8.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
620
+ "decoder.block.8.inter_attn.kv.bias": "pytorch_model-00002-of-00003.bin",
621
+ "decoder.block.8.inter_attn.kv.weight": "pytorch_model-00002-of-00003.bin",
622
+ "decoder.block.8.inter_attn.o.bias": "pytorch_model-00002-of-00003.bin",
623
+ "decoder.block.8.inter_attn.o.weight": "pytorch_model-00002-of-00003.bin",
624
+ "decoder.block.8.inter_attn.q.bias": "pytorch_model-00002-of-00003.bin",
625
+ "decoder.block.8.inter_attn.q.weight": "pytorch_model-00002-of-00003.bin",
626
+ "decoder.block.8.mlp.fc_in.weight": "pytorch_model-00003-of-00003.bin",
627
+ "decoder.block.8.mlp.fc_out.weight": "pytorch_model-00002-of-00003.bin",
628
+ "decoder.block.8.post_attn_layernorm.bias": "pytorch_model-00002-of-00003.bin",
629
+ "decoder.block.8.post_attn_layernorm.weight": "pytorch_model-00002-of-00003.bin",
630
+ "decoder.block.8.post_inter_attn_layernorm.bias": "pytorch_model-00002-of-00003.bin",
631
+ "decoder.block.8.post_inter_attn_layernorm.weight": "pytorch_model-00002-of-00003.bin",
632
+ "decoder.block.8.self_attn.o.bias": "pytorch_model-00002-of-00003.bin",
633
+ "decoder.block.8.self_attn.o.weight": "pytorch_model-00002-of-00003.bin",
634
+ "decoder.block.8.self_attn.qkv.bias": "pytorch_model-00002-of-00003.bin",
635
+ "decoder.block.8.self_attn.qkv.weight": "pytorch_model-00002-of-00003.bin",
636
+ "decoder.block.9.input_layernorm.bias": "pytorch_model-00002-of-00003.bin",
637
+ "decoder.block.9.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
638
+ "decoder.block.9.inter_attn.kv.bias": "pytorch_model-00002-of-00003.bin",
639
+ "decoder.block.9.inter_attn.kv.weight": "pytorch_model-00002-of-00003.bin",
640
+ "decoder.block.9.inter_attn.o.bias": "pytorch_model-00002-of-00003.bin",
641
+ "decoder.block.9.inter_attn.o.weight": "pytorch_model-00002-of-00003.bin",
642
+ "decoder.block.9.inter_attn.q.bias": "pytorch_model-00002-of-00003.bin",
643
+ "decoder.block.9.inter_attn.q.weight": "pytorch_model-00002-of-00003.bin",
644
+ "decoder.block.9.mlp.fc_in.weight": "pytorch_model-00003-of-00003.bin",
645
+ "decoder.block.9.mlp.fc_out.weight": "pytorch_model-00002-of-00003.bin",
646
+ "decoder.block.9.post_attn_layernorm.bias": "pytorch_model-00002-of-00003.bin",
647
+ "decoder.block.9.post_attn_layernorm.weight": "pytorch_model-00002-of-00003.bin",
648
+ "decoder.block.9.post_inter_attn_layernorm.bias": "pytorch_model-00002-of-00003.bin",
649
+ "decoder.block.9.post_inter_attn_layernorm.weight": "pytorch_model-00002-of-00003.bin",
650
+ "decoder.block.9.self_attn.o.bias": "pytorch_model-00002-of-00003.bin",
651
+ "decoder.block.9.self_attn.o.weight": "pytorch_model-00002-of-00003.bin",
652
+ "decoder.block.9.self_attn.qkv.bias": "pytorch_model-00002-of-00003.bin",
653
+ "decoder.block.9.self_attn.qkv.weight": "pytorch_model-00002-of-00003.bin",
654
+ "decoder.final_layernorm.bias": "pytorch_model-00003-of-00003.bin",
655
+ "decoder.final_layernorm.weight": "pytorch_model-00003-of-00003.bin",
656
+ "encoder.block.0.input_layernorm.bias": "pytorch_model-00001-of-00003.bin",
657
+ "encoder.block.0.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
658
+ "encoder.block.0.mlp.fc_in.weight": "pytorch_model-00001-of-00003.bin",
659
+ "encoder.block.0.mlp.fc_out.weight": "pytorch_model-00001-of-00003.bin",
660
+ "encoder.block.0.post_attn_layernorm.bias": "pytorch_model-00001-of-00003.bin",
661
+ "encoder.block.0.post_attn_layernorm.weight": "pytorch_model-00001-of-00003.bin",
662
+ "encoder.block.0.self_attn.o.bias": "pytorch_model-00001-of-00003.bin",
663
+ "encoder.block.0.self_attn.o.weight": "pytorch_model-00001-of-00003.bin",
664
+ "encoder.block.0.self_attn.qkv.bias": "pytorch_model-00001-of-00003.bin",
665
+ "encoder.block.0.self_attn.qkv.weight": "pytorch_model-00001-of-00003.bin",
666
+ "encoder.block.1.input_layernorm.bias": "pytorch_model-00001-of-00003.bin",
667
+ "encoder.block.1.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
668
+ "encoder.block.1.mlp.fc_in.weight": "pytorch_model-00001-of-00003.bin",
669
+ "encoder.block.1.mlp.fc_out.weight": "pytorch_model-00001-of-00003.bin",
670
+ "encoder.block.1.post_attn_layernorm.bias": "pytorch_model-00001-of-00003.bin",
671
+ "encoder.block.1.post_attn_layernorm.weight": "pytorch_model-00001-of-00003.bin",
672
+ "encoder.block.1.self_attn.o.bias": "pytorch_model-00001-of-00003.bin",
673
+ "encoder.block.1.self_attn.o.weight": "pytorch_model-00001-of-00003.bin",
674
+ "encoder.block.1.self_attn.qkv.bias": "pytorch_model-00001-of-00003.bin",
675
+ "encoder.block.1.self_attn.qkv.weight": "pytorch_model-00001-of-00003.bin",
676
+ "encoder.block.10.input_layernorm.bias": "pytorch_model-00001-of-00003.bin",
677
+ "encoder.block.10.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
678
+ "encoder.block.10.mlp.fc_in.weight": "pytorch_model-00001-of-00003.bin",
679
+ "encoder.block.10.mlp.fc_out.weight": "pytorch_model-00001-of-00003.bin",
680
+ "encoder.block.10.post_attn_layernorm.bias": "pytorch_model-00001-of-00003.bin",
681
+ "encoder.block.10.post_attn_layernorm.weight": "pytorch_model-00001-of-00003.bin",
682
+ "encoder.block.10.self_attn.o.bias": "pytorch_model-00001-of-00003.bin",
683
+ "encoder.block.10.self_attn.o.weight": "pytorch_model-00001-of-00003.bin",
684
+ "encoder.block.10.self_attn.qkv.bias": "pytorch_model-00001-of-00003.bin",
685
+ "encoder.block.10.self_attn.qkv.weight": "pytorch_model-00001-of-00003.bin",
686
+ "encoder.block.11.input_layernorm.bias": "pytorch_model-00001-of-00003.bin",
687
+ "encoder.block.11.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
688
+ "encoder.block.11.mlp.fc_in.weight": "pytorch_model-00001-of-00003.bin",
689
+ "encoder.block.11.mlp.fc_out.weight": "pytorch_model-00001-of-00003.bin",
690
+ "encoder.block.11.post_attn_layernorm.bias": "pytorch_model-00001-of-00003.bin",
691
+ "encoder.block.11.post_attn_layernorm.weight": "pytorch_model-00001-of-00003.bin",
692
+ "encoder.block.11.self_attn.o.bias": "pytorch_model-00001-of-00003.bin",
693
+ "encoder.block.11.self_attn.o.weight": "pytorch_model-00001-of-00003.bin",
694
+ "encoder.block.11.self_attn.qkv.bias": "pytorch_model-00001-of-00003.bin",
695
+ "encoder.block.11.self_attn.qkv.weight": "pytorch_model-00001-of-00003.bin",
696
+ "encoder.block.2.input_layernorm.bias": "pytorch_model-00001-of-00003.bin",
697
+ "encoder.block.2.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
698
+ "encoder.block.2.mlp.fc_in.weight": "pytorch_model-00001-of-00003.bin",
699
+ "encoder.block.2.mlp.fc_out.weight": "pytorch_model-00001-of-00003.bin",
700
+ "encoder.block.2.post_attn_layernorm.bias": "pytorch_model-00001-of-00003.bin",
701
+ "encoder.block.2.post_attn_layernorm.weight": "pytorch_model-00001-of-00003.bin",
702
+ "encoder.block.2.self_attn.o.bias": "pytorch_model-00001-of-00003.bin",
703
+ "encoder.block.2.self_attn.o.weight": "pytorch_model-00001-of-00003.bin",
704
+ "encoder.block.2.self_attn.qkv.bias": "pytorch_model-00001-of-00003.bin",
705
+ "encoder.block.2.self_attn.qkv.weight": "pytorch_model-00001-of-00003.bin",
706
+ "encoder.block.3.input_layernorm.bias": "pytorch_model-00001-of-00003.bin",
707
+ "encoder.block.3.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
708
+ "encoder.block.3.mlp.fc_in.weight": "pytorch_model-00001-of-00003.bin",
709
+ "encoder.block.3.mlp.fc_out.weight": "pytorch_model-00001-of-00003.bin",
710
+ "encoder.block.3.post_attn_layernorm.bias": "pytorch_model-00001-of-00003.bin",
711
+ "encoder.block.3.post_attn_layernorm.weight": "pytorch_model-00001-of-00003.bin",
712
+ "encoder.block.3.self_attn.o.bias": "pytorch_model-00001-of-00003.bin",
713
+ "encoder.block.3.self_attn.o.weight": "pytorch_model-00001-of-00003.bin",
714
+ "encoder.block.3.self_attn.qkv.bias": "pytorch_model-00001-of-00003.bin",
715
+ "encoder.block.3.self_attn.qkv.weight": "pytorch_model-00001-of-00003.bin",
716
+ "encoder.block.4.input_layernorm.bias": "pytorch_model-00001-of-00003.bin",
717
+ "encoder.block.4.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
718
+ "encoder.block.4.mlp.fc_in.weight": "pytorch_model-00001-of-00003.bin",
719
+ "encoder.block.4.mlp.fc_out.weight": "pytorch_model-00001-of-00003.bin",
720
+ "encoder.block.4.post_attn_layernorm.bias": "pytorch_model-00001-of-00003.bin",
721
+ "encoder.block.4.post_attn_layernorm.weight": "pytorch_model-00001-of-00003.bin",
722
+ "encoder.block.4.self_attn.o.bias": "pytorch_model-00001-of-00003.bin",
723
+ "encoder.block.4.self_attn.o.weight": "pytorch_model-00001-of-00003.bin",
724
+ "encoder.block.4.self_attn.qkv.bias": "pytorch_model-00001-of-00003.bin",
725
+ "encoder.block.4.self_attn.qkv.weight": "pytorch_model-00001-of-00003.bin",
726
+ "encoder.block.5.input_layernorm.bias": "pytorch_model-00001-of-00003.bin",
727
+ "encoder.block.5.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
728
+ "encoder.block.5.mlp.fc_in.weight": "pytorch_model-00001-of-00003.bin",
729
+ "encoder.block.5.mlp.fc_out.weight": "pytorch_model-00001-of-00003.bin",
730
+ "encoder.block.5.post_attn_layernorm.bias": "pytorch_model-00001-of-00003.bin",
731
+ "encoder.block.5.post_attn_layernorm.weight": "pytorch_model-00001-of-00003.bin",
732
+ "encoder.block.5.self_attn.o.bias": "pytorch_model-00001-of-00003.bin",
733
+ "encoder.block.5.self_attn.o.weight": "pytorch_model-00001-of-00003.bin",
734
+ "encoder.block.5.self_attn.qkv.bias": "pytorch_model-00001-of-00003.bin",
735
+ "encoder.block.5.self_attn.qkv.weight": "pytorch_model-00001-of-00003.bin",
736
+ "encoder.block.6.input_layernorm.bias": "pytorch_model-00001-of-00003.bin",
737
+ "encoder.block.6.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
738
+ "encoder.block.6.mlp.fc_in.weight": "pytorch_model-00001-of-00003.bin",
739
+ "encoder.block.6.mlp.fc_out.weight": "pytorch_model-00001-of-00003.bin",
740
+ "encoder.block.6.post_attn_layernorm.bias": "pytorch_model-00001-of-00003.bin",
741
+ "encoder.block.6.post_attn_layernorm.weight": "pytorch_model-00001-of-00003.bin",
742
+ "encoder.block.6.self_attn.o.bias": "pytorch_model-00001-of-00003.bin",
743
+ "encoder.block.6.self_attn.o.weight": "pytorch_model-00001-of-00003.bin",
744
+ "encoder.block.6.self_attn.qkv.bias": "pytorch_model-00001-of-00003.bin",
745
+ "encoder.block.6.self_attn.qkv.weight": "pytorch_model-00001-of-00003.bin",
746
+ "encoder.block.7.input_layernorm.bias": "pytorch_model-00001-of-00003.bin",
747
+ "encoder.block.7.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
748
+ "encoder.block.7.mlp.fc_in.weight": "pytorch_model-00001-of-00003.bin",
749
+ "encoder.block.7.mlp.fc_out.weight": "pytorch_model-00001-of-00003.bin",
750
+ "encoder.block.7.post_attn_layernorm.bias": "pytorch_model-00001-of-00003.bin",
751
+ "encoder.block.7.post_attn_layernorm.weight": "pytorch_model-00001-of-00003.bin",
752
+ "encoder.block.7.self_attn.o.bias": "pytorch_model-00001-of-00003.bin",
753
+ "encoder.block.7.self_attn.o.weight": "pytorch_model-00001-of-00003.bin",
754
+ "encoder.block.7.self_attn.qkv.bias": "pytorch_model-00001-of-00003.bin",
755
+ "encoder.block.7.self_attn.qkv.weight": "pytorch_model-00001-of-00003.bin",
756
+ "encoder.block.8.input_layernorm.bias": "pytorch_model-00001-of-00003.bin",
757
+ "encoder.block.8.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
758
+ "encoder.block.8.mlp.fc_in.weight": "pytorch_model-00001-of-00003.bin",
759
+ "encoder.block.8.mlp.fc_out.weight": "pytorch_model-00001-of-00003.bin",
760
+ "encoder.block.8.post_attn_layernorm.bias": "pytorch_model-00001-of-00003.bin",
761
+ "encoder.block.8.post_attn_layernorm.weight": "pytorch_model-00001-of-00003.bin",
762
+ "encoder.block.8.self_attn.o.bias": "pytorch_model-00001-of-00003.bin",
763
+ "encoder.block.8.self_attn.o.weight": "pytorch_model-00001-of-00003.bin",
764
+ "encoder.block.8.self_attn.qkv.bias": "pytorch_model-00001-of-00003.bin",
765
+ "encoder.block.8.self_attn.qkv.weight": "pytorch_model-00001-of-00003.bin",
766
+ "encoder.block.9.input_layernorm.bias": "pytorch_model-00001-of-00003.bin",
767
+ "encoder.block.9.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
768
+ "encoder.block.9.mlp.fc_in.weight": "pytorch_model-00001-of-00003.bin",
769
+ "encoder.block.9.mlp.fc_out.weight": "pytorch_model-00001-of-00003.bin",
770
+ "encoder.block.9.post_attn_layernorm.bias": "pytorch_model-00001-of-00003.bin",
771
+ "encoder.block.9.post_attn_layernorm.weight": "pytorch_model-00001-of-00003.bin",
772
+ "encoder.block.9.self_attn.o.bias": "pytorch_model-00001-of-00003.bin",
773
+ "encoder.block.9.self_attn.o.weight": "pytorch_model-00001-of-00003.bin",
774
+ "encoder.block.9.self_attn.qkv.bias": "pytorch_model-00001-of-00003.bin",
775
+ "encoder.block.9.self_attn.qkv.weight": "pytorch_model-00001-of-00003.bin",
776
+ "encoder.final_layernorm.bias": "pytorch_model-00001-of-00003.bin",
777
+ "encoder.final_layernorm.weight": "pytorch_model-00001-of-00003.bin",
778
+ "lm_head.bias": "pytorch_model-00003-of-00003.bin",
779
+ "lm_head.weight": "pytorch_model-00001-of-00003.bin",
780
+ "shared_embedding.weight": "pytorch_model-00001-of-00003.bin"
781
+ }
782
+ }
special_tokens_map.json ADDED
@@ -0,0 +1,107 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<extra_id_0>",
4
+ "<extra_id_1>",
5
+ "<extra_id_2>",
6
+ "<extra_id_3>",
7
+ "<extra_id_4>",
8
+ "<extra_id_5>",
9
+ "<extra_id_6>",
10
+ "<extra_id_7>",
11
+ "<extra_id_8>",
12
+ "<extra_id_9>",
13
+ "<extra_id_10>",
14
+ "<extra_id_11>",
15
+ "<extra_id_12>",
16
+ "<extra_id_13>",
17
+ "<extra_id_14>",
18
+ "<extra_id_15>",
19
+ "<extra_id_16>",
20
+ "<extra_id_17>",
21
+ "<extra_id_18>",
22
+ "<extra_id_19>",
23
+ "<extra_id_20>",
24
+ "<extra_id_21>",
25
+ "<extra_id_22>",
26
+ "<extra_id_23>",
27
+ "<extra_id_24>",
28
+ "<extra_id_25>",
29
+ "<extra_id_26>",
30
+ "<extra_id_27>",
31
+ "<extra_id_28>",
32
+ "<extra_id_29>",
33
+ "<extra_id_30>",
34
+ "<extra_id_31>",
35
+ "<extra_id_32>",
36
+ "<extra_id_33>",
37
+ "<extra_id_34>",
38
+ "<extra_id_35>",
39
+ "<extra_id_36>",
40
+ "<extra_id_37>",
41
+ "<extra_id_38>",
42
+ "<extra_id_39>",
43
+ "<extra_id_40>",
44
+ "<extra_id_41>",
45
+ "<extra_id_42>",
46
+ "<extra_id_43>",
47
+ "<extra_id_44>",
48
+ "<extra_id_45>",
49
+ "<extra_id_46>",
50
+ "<extra_id_47>",
51
+ "<extra_id_48>",
52
+ "<extra_id_49>",
53
+ "<extra_id_50>",
54
+ "<extra_id_51>",
55
+ "<extra_id_52>",
56
+ "<extra_id_53>",
57
+ "<extra_id_54>",
58
+ "<extra_id_55>",
59
+ "<extra_id_56>",
60
+ "<extra_id_57>",
61
+ "<extra_id_58>",
62
+ "<extra_id_59>",
63
+ "<extra_id_60>",
64
+ "<extra_id_61>",
65
+ "<extra_id_62>",
66
+ "<extra_id_63>",
67
+ "<extra_id_64>",
68
+ "<extra_id_65>",
69
+ "<extra_id_66>",
70
+ "<extra_id_67>",
71
+ "<extra_id_68>",
72
+ "<extra_id_69>",
73
+ "<extra_id_70>",
74
+ "<extra_id_71>",
75
+ "<extra_id_72>",
76
+ "<extra_id_73>",
77
+ "<extra_id_74>",
78
+ "<extra_id_75>",
79
+ "<extra_id_76>",
80
+ "<extra_id_77>",
81
+ "<extra_id_78>",
82
+ "<extra_id_79>",
83
+ "<extra_id_80>",
84
+ "<extra_id_81>",
85
+ "<extra_id_82>",
86
+ "<extra_id_83>",
87
+ "<extra_id_84>",
88
+ "<extra_id_85>",
89
+ "<extra_id_86>",
90
+ "<extra_id_87>",
91
+ "<extra_id_88>",
92
+ "<extra_id_89>",
93
+ "<extra_id_90>",
94
+ "<extra_id_91>",
95
+ "<extra_id_92>",
96
+ "<extra_id_93>",
97
+ "<extra_id_94>",
98
+ "<extra_id_95>",
99
+ "<extra_id_96>",
100
+ "<extra_id_97>",
101
+ "<extra_id_98>",
102
+ "<extra_id_99>"
103
+ ],
104
+ "eos_token": "</s>",
105
+ "pad_token": "<pad>",
106
+ "unk_token": "<unk>"
107
+ }
spiece.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ef78f86560d809067d12bac6c09f19a462cb3af3f54d2b8acbba26e1433125d6
3
+ size 4309802
tokenization_openba.py ADDED
@@ -0,0 +1,263 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # coding=utf-8
2
+
3
+
4
+ import os
5
+ import re
6
+ import warnings
7
+ from shutil import copyfile
8
+ from typing import Any, Dict, List, Optional, Tuple
9
+
10
+ import sentencepiece as spm
11
+
12
+ from transformers.tokenization_utils import PreTrainedTokenizer
13
+ from transformers.utils import logging
14
+
15
+
16
+ logger = logging.get_logger(__name__)
17
+
18
+ VOCAB_FILES_NAMES = {"vocab_file": "spiece.model"}
19
+
20
+
21
+ class OpenBATokenizer(PreTrainedTokenizer):
22
+
23
+ vocab_files_names = VOCAB_FILES_NAMES
24
+ model_input_names = ["input_ids", "attention_mask"]
25
+
26
+ def __init__(
27
+ self,
28
+ vocab_file,
29
+ eos_token="</s>",
30
+ unk_token="<unk>",
31
+ pad_token="<pad>",
32
+ extra_ids=100,
33
+ additional_special_tokens=None,
34
+ sp_model_kwargs: Optional[Dict[str, Any]] = None,
35
+ **kwargs,
36
+ ) -> None:
37
+ # Add extra_ids to the special token list
38
+ if extra_ids > 0 and additional_special_tokens is None:
39
+ additional_special_tokens = [f"<extra_id_{i}>" for i in range(extra_ids)]
40
+ elif extra_ids > 0 and additional_special_tokens is not None:
41
+ # Check that we have the right number of extra_id special tokens
42
+ extra_tokens = len(set(filter(lambda x: bool("extra_id" in str(x)), additional_special_tokens)))
43
+ if extra_tokens != extra_ids:
44
+ raise ValueError(
45
+ f"Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are"
46
+ " provided to T5Tokenizer. In this case the additional_special_tokens must include the extra_ids"
47
+ " tokens"
48
+ )
49
+
50
+ self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs
51
+
52
+ super().__init__(
53
+ eos_token=eos_token,
54
+ unk_token=unk_token,
55
+ pad_token=pad_token,
56
+ extra_ids=extra_ids,
57
+ additional_special_tokens=additional_special_tokens,
58
+ sp_model_kwargs=self.sp_model_kwargs,
59
+ **kwargs,
60
+ )
61
+
62
+ self.vocab_file = vocab_file
63
+ self._extra_ids = extra_ids
64
+
65
+ self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
66
+ self.sp_model.Load(vocab_file)
67
+
68
+ @staticmethod
69
+ def _eventually_correct_t5_max_length(pretrained_model_name_or_path, max_model_length, init_max_model_length):
70
+ if pretrained_model_name_or_path in OpenBATokenizer.max_model_input_sizes:
71
+ deprecated_max_model_length = OpenBATokenizer.max_model_input_sizes[pretrained_model_name_or_path]
72
+ if init_max_model_length is not None and init_max_model_length != max_model_length:
73
+ return init_max_model_length
74
+ elif init_max_model_length is None:
75
+ warnings.warn(
76
+ "This tokenizer was incorrectly instantiated with a model max length of"
77
+ f" {deprecated_max_model_length} which will be corrected in Transformers v5.\nFor now, this"
78
+ " behavior is kept to avoid breaking backwards compatibility when padding/encoding with"
79
+ " `truncation is True`.\n- Be aware that you SHOULD NOT rely on"
80
+ f" {pretrained_model_name_or_path} automatically truncating your input to"
81
+ f" {deprecated_max_model_length} when padding/encoding.\n- If you want to encode/pad to sequences"
82
+ f" longer than {deprecated_max_model_length} you can either instantiate this tokenizer with"
83
+ " `model_max_length` or pass `max_length` when encoding/padding.\n- To avoid this warning, please"
84
+ " instantiate this tokenizer with `model_max_length` set to your preferred value.",
85
+ FutureWarning,
86
+ )
87
+
88
+ return max_model_length
89
+
90
+ @property
91
+ def vocab_size(self):
92
+ return self.sp_model.get_piece_size() + self._extra_ids
93
+
94
+ def get_vocab(self):
95
+ vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
96
+ vocab.update(self.added_tokens_encoder)
97
+ return vocab
98
+
99
+ def get_special_tokens_mask(
100
+ self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
101
+ ) -> List[int]:
102
+ """
103
+ Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
104
+ special tokens using the tokenizer `prepare_for_model` method.
105
+
106
+ Args:
107
+ token_ids_0 (`List[int]`):
108
+ List of IDs.
109
+ token_ids_1 (`List[int]`, *optional*):
110
+ Optional second list of IDs for sequence pairs.
111
+ already_has_special_tokens (`bool`, *optional*, defaults to `False`):
112
+ Whether or not the token list is already formatted with special tokens for the model.
113
+
114
+ Returns:
115
+ `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
116
+ """
117
+ if already_has_special_tokens:
118
+ return super().get_special_tokens_mask(
119
+ token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
120
+ )
121
+
122
+ # normal case: some special tokens
123
+ if token_ids_1 is None:
124
+ return ([0] * len(token_ids_0)) + [1]
125
+ return ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1]
126
+
127
+ def get_sentinel_tokens(self):
128
+ return list(
129
+ set(filter(lambda x: bool(re.search(r"<extra_id_\d+>", x)) is not None, self.additional_special_tokens))
130
+ )
131
+
132
+ def get_sentinel_token_ids(self):
133
+ return [self._convert_token_to_id(token) for token in self.get_sentinel_tokens()]
134
+
135
+ def _add_eos_if_not_present(self, token_ids: List[int]) -> List[int]:
136
+ """Do not add eos again if user already added it."""
137
+ if len(token_ids) > 0 and token_ids[-1] == self.eos_token_id:
138
+ warnings.warn(
139
+ f"This sequence already has {self.eos_token}. In future versions this behavior may lead to duplicated"
140
+ " eos tokens being added."
141
+ )
142
+ return token_ids
143
+ else:
144
+ return token_ids + [self.eos_token_id]
145
+
146
+ def create_token_type_ids_from_sequences(
147
+ self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
148
+ ) -> List[int]:
149
+ """
150
+ Create a mask from the two sequences passed to be used in a sequence-pair classification task. T5 does not make
151
+ use of token type ids, therefore a list of zeros is returned.
152
+
153
+ Args:
154
+ token_ids_0 (`List[int]`):
155
+ List of IDs.
156
+ token_ids_1 (`List[int]`, *optional*):
157
+ Optional second list of IDs for sequence pairs.
158
+
159
+ Returns:
160
+ `List[int]`: List of zeros.
161
+ """
162
+ eos = [self.eos_token_id]
163
+
164
+ if token_ids_1 is None:
165
+ return len(token_ids_0 + eos) * [0]
166
+ return len(token_ids_0 + eos + token_ids_1 + eos) * [0]
167
+
168
+ def build_inputs_with_special_tokens(
169
+ self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
170
+ ) -> List[int]:
171
+ """
172
+ Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
173
+ adding special tokens. A sequence has the following format:
174
+
175
+ - single sequence: `X </s>`
176
+ - pair of sequences: `A </s> B </s>`
177
+
178
+ Args:
179
+ token_ids_0 (`List[int]`):
180
+ List of IDs to which the special tokens will be added.
181
+ token_ids_1 (`List[int]`, *optional*):
182
+ Optional second list of IDs for sequence pairs.
183
+
184
+ Returns:
185
+ `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.
186
+ """
187
+ token_ids_0 = self._add_eos_if_not_present(token_ids_0)
188
+ if token_ids_1 is None:
189
+ return token_ids_0
190
+ else:
191
+ token_ids_1 = self._add_eos_if_not_present(token_ids_1)
192
+ return token_ids_0 + token_ids_1
193
+
194
+ def __getstate__(self):
195
+ state = self.__dict__.copy()
196
+ state["sp_model"] = None
197
+ return state
198
+
199
+ def __setstate__(self, d):
200
+ self.__dict__ = d
201
+
202
+ # for backward compatibility
203
+ if not hasattr(self, "sp_model_kwargs"):
204
+ self.sp_model_kwargs = {}
205
+
206
+ self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
207
+ self.sp_model.Load(self.vocab_file)
208
+
209
+ def _tokenize(self, text: str) -> List[str]:
210
+ """Take as input a string and return a list of strings (tokens) for words/sub-words"""
211
+ return self.sp_model.encode(text, out_type=str)
212
+
213
+ def _convert_token_to_id(self, token):
214
+ """Converts a token (str) in an id using the vocab."""
215
+ if token.startswith("<extra_id_"):
216
+ match = re.match(r"<extra_id_(\d+)>", token)
217
+ num = int(match.group(1))
218
+ return self.vocab_size - num - 1
219
+ return self.sp_model.piece_to_id(token)
220
+
221
+ def _convert_id_to_token(self, index):
222
+ """Converts an index (integer) in a token (str) using the vocab."""
223
+ if index < self.sp_model.get_piece_size():
224
+ token = self.sp_model.IdToPiece(index)
225
+ else:
226
+ token = f"<extra_id_{self.vocab_size - 1 - index}>"
227
+ return token
228
+
229
+ def convert_tokens_to_string(self, tokens):
230
+ """Converts a sequence of tokens (string) in a single string."""
231
+ current_sub_tokens = []
232
+ out_string = ""
233
+ prev_is_special = False
234
+ for token in tokens:
235
+ # make sure that special tokens are not decoded using sentencepiece model
236
+ if token in self.all_special_tokens:
237
+ if not prev_is_special:
238
+ out_string += " "
239
+ out_string += self.sp_model.decode(current_sub_tokens) + token
240
+ prev_is_special = True
241
+ current_sub_tokens = []
242
+ else:
243
+ current_sub_tokens.append(token)
244
+ prev_is_special = False
245
+ out_string += self.sp_model.decode(current_sub_tokens)
246
+ return out_string.strip()
247
+
248
+ def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
249
+ if not os.path.isdir(save_directory):
250
+ logger.error(f"Vocabulary path ({save_directory}) should be a directory")
251
+ return
252
+ out_vocab_file = os.path.join(
253
+ save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
254
+ )
255
+
256
+ if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file) and os.path.isfile(self.vocab_file):
257
+ copyfile(self.vocab_file, out_vocab_file)
258
+ elif not os.path.isfile(self.vocab_file):
259
+ with open(out_vocab_file, "wb") as fi:
260
+ content_spiece_model = self.sp_model.serialized_model_proto()
261
+ fi.write(content_spiece_model)
262
+
263
+ return (out_vocab_file,)
tokenizer_config.json ADDED
@@ -0,0 +1,119 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<extra_id_0>",
4
+ "<extra_id_1>",
5
+ "<extra_id_2>",
6
+ "<extra_id_3>",
7
+ "<extra_id_4>",
8
+ "<extra_id_5>",
9
+ "<extra_id_6>",
10
+ "<extra_id_7>",
11
+ "<extra_id_8>",
12
+ "<extra_id_9>",
13
+ "<extra_id_10>",
14
+ "<extra_id_11>",
15
+ "<extra_id_12>",
16
+ "<extra_id_13>",
17
+ "<extra_id_14>",
18
+ "<extra_id_15>",
19
+ "<extra_id_16>",
20
+ "<extra_id_17>",
21
+ "<extra_id_18>",
22
+ "<extra_id_19>",
23
+ "<extra_id_20>",
24
+ "<extra_id_21>",
25
+ "<extra_id_22>",
26
+ "<extra_id_23>",
27
+ "<extra_id_24>",
28
+ "<extra_id_25>",
29
+ "<extra_id_26>",
30
+ "<extra_id_27>",
31
+ "<extra_id_28>",
32
+ "<extra_id_29>",
33
+ "<extra_id_30>",
34
+ "<extra_id_31>",
35
+ "<extra_id_32>",
36
+ "<extra_id_33>",
37
+ "<extra_id_34>",
38
+ "<extra_id_35>",
39
+ "<extra_id_36>",
40
+ "<extra_id_37>",
41
+ "<extra_id_38>",
42
+ "<extra_id_39>",
43
+ "<extra_id_40>",
44
+ "<extra_id_41>",
45
+ "<extra_id_42>",
46
+ "<extra_id_43>",
47
+ "<extra_id_44>",
48
+ "<extra_id_45>",
49
+ "<extra_id_46>",
50
+ "<extra_id_47>",
51
+ "<extra_id_48>",
52
+ "<extra_id_49>",
53
+ "<extra_id_50>",
54
+ "<extra_id_51>",
55
+ "<extra_id_52>",
56
+ "<extra_id_53>",
57
+ "<extra_id_54>",
58
+ "<extra_id_55>",
59
+ "<extra_id_56>",
60
+ "<extra_id_57>",
61
+ "<extra_id_58>",
62
+ "<extra_id_59>",
63
+ "<extra_id_60>",
64
+ "<extra_id_61>",
65
+ "<extra_id_62>",
66
+ "<extra_id_63>",
67
+ "<extra_id_64>",
68
+ "<extra_id_65>",
69
+ "<extra_id_66>",
70
+ "<extra_id_67>",
71
+ "<extra_id_68>",
72
+ "<extra_id_69>",
73
+ "<extra_id_70>",
74
+ "<extra_id_71>",
75
+ "<extra_id_72>",
76
+ "<extra_id_73>",
77
+ "<extra_id_74>",
78
+ "<extra_id_75>",
79
+ "<extra_id_76>",
80
+ "<extra_id_77>",
81
+ "<extra_id_78>",
82
+ "<extra_id_79>",
83
+ "<extra_id_80>",
84
+ "<extra_id_81>",
85
+ "<extra_id_82>",
86
+ "<extra_id_83>",
87
+ "<extra_id_84>",
88
+ "<extra_id_85>",
89
+ "<extra_id_86>",
90
+ "<extra_id_87>",
91
+ "<extra_id_88>",
92
+ "<extra_id_89>",
93
+ "<extra_id_90>",
94
+ "<extra_id_91>",
95
+ "<extra_id_92>",
96
+ "<extra_id_93>",
97
+ "<extra_id_94>",
98
+ "<extra_id_95>",
99
+ "<extra_id_96>",
100
+ "<extra_id_97>",
101
+ "<extra_id_98>",
102
+ "<extra_id_99>"
103
+ ],
104
+ "auto_map": {
105
+ "AutoTokenizer": [
106
+ "tokenization_openba.OpenBATokenizer",
107
+ null
108
+ ]
109
+ },
110
+ "clean_up_tokenization_spaces": true,
111
+ "eos_token": "</s>",
112
+ "extra_ids": 100,
113
+ "model_max_length": 1000000000000000019884624838656,
114
+ "pad_token": "<pad>",
115
+ "sp_model_kwargs": {},
116
+ "tokenizer_class": "OpenBATokenizer",
117
+ "tokenizer_file": null,
118
+ "unk_token": "<unk>"
119
+ }