UltraFastBERT-1x11-long / train_budget_hours_24.json
pbelcak's picture
Upload 11 files
8c43fc6
raw
history blame contribute delete
890 Bytes
{
"optim": {
"type": "AdamW",
"lr": 0.0005,
"betas": [
0.9,
0.98
],
"eps": 1e-12,
"weight_decay": 0.01,
"amsgrad": false,
"fused": null
},
"optim_mod": {
"name": "none"
},
"name": "bert-base",
"limited_decay_keys": [
"bias",
"LayerNorm.bias",
"LayerNorm.weight",
"norm"
],
"warmup_steps": 30000,
"cooldown_steps": 0,
"steps": 900000,
"scheduler": "budget-cosine-decay",
"batch_size": 1536,
"batch_size_ramp": 0,
"gradient_clipping": null,
"pretrain_in_train_mode": false,
"objective": {
"name": "masked-lm",
"mlm_probability": 0.15,
"use_80_20_rule": true,
"disable_mlm": false,
"token_drop": 0.0
},
"reverse_dataset_order": false,
"budget": 24
}