config.json

{
    "_name_or_path": "PagnoXL",
    "activation_function": "gelu",
    "architectures": [
        "PagnolXlForCausalLM"
    ],
    "auto_map": {
        "AutoConfig": "configuration_pagnolxl.PagnolXlConfig",
        "AutoModelForCausalLM": "modeling_pagnolxl.PagnolXlForCausalLM",
        "AutoModel": "modeling_pagnolxl.PagnolXlModel"
    },
    "bos_token_id": 1,
    "d_feedforward": 6400,
    "d_model": 1600,
    "dropout": 0.1,
    "eos_token_id": 1,
    "layer_norm_epsilon": 1e-06,
    "max_seq_len": 2048,
    "model_type": "pagnolxl",
    "n_heads": 25,
    "n_layers": 48,
    "sigma": 0.02,
    "torch_dtype": "float32",
    "transformers_version": "4.38.2",
    "use_cache": true,
    "vocab_size": 50262
}