add additional 2-epoch checkpoint, better regularization

config.json

@@ -0,0 +1,43 @@
+{
+  "_name_or_path": "pszemraj/long-t5-tglobal-base-16384-booksum-V6-partial",
+  "architectures": [
+    "LongT5ForConditionalGeneration"
+  ],
+  "d_ff": 2048,
+  "d_kv": 64,
+  "d_model": 768,
+  "decoder_start_token_id": 0,
+  "dense_act_fn": "gelu_new",
+  "dropout_rate": 0.1,
+  "early_stopping": true,
+  "encoder_attention_type": "transient-global",
+  "encoder_no_repeat_ngram_size": 4,
+  "eos_token_id": 1,
+  "feed_forward_proj": "gated-gelu",
+  "global_block_size": 16,
+  "initializer_factor": 1.0,
+  "is_encoder_decoder": true,
+  "is_gated_act": true,
+  "layer_norm_epsilon": 1e-06,
+  "length_penalty": 0.8,
+  "local_radius": 127,
+  "max_length": 512,
+  "min_length": 8,
+  "model_type": "longt5",
+  "n_positions": 4096,
+  "no_repeat_ngram_size": 3,
+  "num_beams": 2,
+  "num_decoder_layers": 12,
+  "num_heads": 12,
+  "num_layers": 12,
+  "output_past": true,
+  "pad_token_id": 0,
+  "relative_attention_max_distance": 128,
+  "relative_attention_num_buckets": 32,
+  "repetition_penalty": 3.5,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.20.1",
+  "use_cache": false,
+  "vocab_size": 32128
+}

long-t5-tglobal-base-16384-booksum-V6-partial-ft3-booksum_training_metadata.json

@@ -0,0 +1 @@
+{"output_dir": "/content/drive/MyDrive/Programming/hf-trainer/long-t5-tglobal-base-16384-booksum-V6-partial-ft3-booksum", "overwrite_output_dir": true, "do_train": false, "do_eval": false, "do_predict": false, "evaluation_strategy": "no", "prediction_loss_only": false, "per_device_train_batch_size": 2, "per_device_eval_batch_size": 1, "per_gpu_train_batch_size": "None", "per_gpu_eval_batch_size": "None", "gradient_accumulation_steps": 64, "eval_accumulation_steps": "None", "eval_delay": 0, "learning_rate": 0.0004, "weight_decay": 0.05, "adam_beta1": 0.9, "adam_beta2": 0.999, "adam_epsilon": 1e-08, "max_grad_norm": 0.2, "num_train_epochs": 2, "max_steps": -1, "lr_scheduler_type": "cosine_with_restarts", "warmup_ratio": 0.02, "warmup_steps": 0, "log_level": -1, "log_level_replica": -1, "log_on_each_node": true, "logging_dir": "/content/drive/MyDrive/Programming/hf-trainer/long-t5-tglobal-base-16384-booksum-V6-partial-ft3-booksum/logs", "logging_strategy": "steps", "logging_first_step": false, "logging_steps": 3, "logging_nan_inf_filter": true, "save_strategy": "epoch", "save_steps": 500, "save_total_limit": 1, "save_on_each_node": false, "no_cuda": false, "seed": 42, "data_seed": "None", "jit_mode_eval": false, "use_ipex": false, "bf16": false, "fp16": false, "fp16_opt_level": "O1", "half_precision_backend": "auto", "bf16_full_eval": false, "fp16_full_eval": false, "tf32": "None", "local_rank": 0, "xpu_backend": "None", "tpu_num_cores": "None", "tpu_metrics_debug": false, "debug": "[]", "dataloader_drop_last": false, "eval_steps": "None", "dataloader_num_workers": 0, "past_index": -1, "run_name": "/content/drive/MyDrive/Programming/hf-trainer/long-t5-tglobal-base-16384-booksum-V6-partial-ft3-booksum", "disable_tqdm": false, "remove_unused_columns": true, "label_names": "None", "load_best_model_at_end": false, "metric_for_best_model": "None", "greater_is_better": "None", "ignore_data_skip": false, "sharded_ddp": "[]", "fsdp": "[]", "fsdp_min_num_params": 0, "deepspeed": "/content/ds_config_zero2_bf16.json", "label_smoothing_factor": 0.0, "optim": "adamw_hf", "adafactor": false, "group_by_length": false, "length_column_name": "length", "report_to": "['tensorboard']", "ddp_find_unused_parameters": "None", "ddp_bucket_cap_mb": "None", "dataloader_pin_memory": true, "skip_memory_metrics": true, "use_legacy_prediction_loop": false, "push_to_hub": true, "resume_from_checkpoint": "None", "hub_model_id": "long-t5-tglobal-base-16384-booksum-V6-partial-ft3-booksum", "hub_strategy": "end", "hub_token": "<HUB_TOKEN>", "hub_private_repo": true, "gradient_checkpointing": true, "include_inputs_for_metrics": false, "fp16_backend": "auto", "push_to_hub_model_id": "None", "push_to_hub_organization": "None", "push_to_hub_token": "<PUSH_TO_HUB_TOKEN>", "_n_gpu": 1, "mp_parameters": "", "auto_find_batch_size": false, "full_determinism": false, "torchdynamo": "None", "ray_scope": "last", "sortish_sampler": false, "predict_with_generate": false, "generation_max_length": "None", "generation_num_beams": "None", "train_batch_size": 2, "eval_batch_size": 1, "configs_src": "long-t5-tglobal-base-16384-booksum-V6-partial-ft3-booksum"}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4adb405b88e1acd599988a723bb03f622f2bc5b7b441d9a480ed5b5ed75fa190
+size 990388907

special_tokens_map.json

@@ -0,0 +1,107 @@
+{
+  "additional_special_tokens": [
+    "<extra_id_0>",
+    "<extra_id_1>",
+    "<extra_id_2>",
+    "<extra_id_3>",
+    "<extra_id_4>",
+    "<extra_id_5>",
+    "<extra_id_6>",
+    "<extra_id_7>",
+    "<extra_id_8>",
+    "<extra_id_9>",
+    "<extra_id_10>",
+    "<extra_id_11>",
+    "<extra_id_12>",
+    "<extra_id_13>",
+    "<extra_id_14>",
+    "<extra_id_15>",
+    "<extra_id_16>",
+    "<extra_id_17>",
+    "<extra_id_18>",
+    "<extra_id_19>",
+    "<extra_id_20>",
+    "<extra_id_21>",
+    "<extra_id_22>",
+    "<extra_id_23>",
+    "<extra_id_24>",
+    "<extra_id_25>",
+    "<extra_id_26>",
+    "<extra_id_27>",
+    "<extra_id_28>",
+    "<extra_id_29>",
+    "<extra_id_30>",
+    "<extra_id_31>",
+    "<extra_id_32>",
+    "<extra_id_33>",
+    "<extra_id_34>",
+    "<extra_id_35>",
+    "<extra_id_36>",
+    "<extra_id_37>",
+    "<extra_id_38>",
+    "<extra_id_39>",
+    "<extra_id_40>",
+    "<extra_id_41>",
+    "<extra_id_42>",
+    "<extra_id_43>",
+    "<extra_id_44>",
+    "<extra_id_45>",
+    "<extra_id_46>",
+    "<extra_id_47>",
+    "<extra_id_48>",
+    "<extra_id_49>",
+    "<extra_id_50>",
+    "<extra_id_51>",
+    "<extra_id_52>",
+    "<extra_id_53>",
+    "<extra_id_54>",
+    "<extra_id_55>",
+    "<extra_id_56>",
+    "<extra_id_57>",
+    "<extra_id_58>",
+    "<extra_id_59>",
+    "<extra_id_60>",
+    "<extra_id_61>",
+    "<extra_id_62>",
+    "<extra_id_63>",
+    "<extra_id_64>",
+    "<extra_id_65>",
+    "<extra_id_66>",
+    "<extra_id_67>",
+    "<extra_id_68>",
+    "<extra_id_69>",
+    "<extra_id_70>",
+    "<extra_id_71>",
+    "<extra_id_72>",
+    "<extra_id_73>",
+    "<extra_id_74>",
+    "<extra_id_75>",
+    "<extra_id_76>",
+    "<extra_id_77>",
+    "<extra_id_78>",
+    "<extra_id_79>",
+    "<extra_id_80>",
+    "<extra_id_81>",
+    "<extra_id_82>",
+    "<extra_id_83>",
+    "<extra_id_84>",
+    "<extra_id_85>",
+    "<extra_id_86>",
+    "<extra_id_87>",
+    "<extra_id_88>",
+    "<extra_id_89>",
+    "<extra_id_90>",
+    "<extra_id_91>",
+    "<extra_id_92>",
+    "<extra_id_93>",
+    "<extra_id_94>",
+    "<extra_id_95>",
+    "<extra_id_96>",
+    "<extra_id_97>",
+    "<extra_id_98>",
+    "<extra_id_99>"
+  ],
+  "eos_token": "</s>",
+  "pad_token": "<pad>",
+  "unk_token": "<unk>"
+}

spiece.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d60acb128cf7b7f2536e8f38a5b18a05535c9e14c7a355904270e15b0945ea86
+size 791656

tokenizer_config.json

@@ -0,0 +1,111 @@
+{
+  "additional_special_tokens": [
+    "<extra_id_0>",
+    "<extra_id_1>",
+    "<extra_id_2>",
+    "<extra_id_3>",
+    "<extra_id_4>",
+    "<extra_id_5>",
+    "<extra_id_6>",
+    "<extra_id_7>",
+    "<extra_id_8>",
+    "<extra_id_9>",
+    "<extra_id_10>",
+    "<extra_id_11>",
+    "<extra_id_12>",
+    "<extra_id_13>",
+    "<extra_id_14>",
+    "<extra_id_15>",
+    "<extra_id_16>",
+    "<extra_id_17>",
+    "<extra_id_18>",
+    "<extra_id_19>",
+    "<extra_id_20>",
+    "<extra_id_21>",
+    "<extra_id_22>",
+    "<extra_id_23>",
+    "<extra_id_24>",
+    "<extra_id_25>",
+    "<extra_id_26>",
+    "<extra_id_27>",
+    "<extra_id_28>",
+    "<extra_id_29>",
+    "<extra_id_30>",
+    "<extra_id_31>",
+    "<extra_id_32>",
+    "<extra_id_33>",
+    "<extra_id_34>",
+    "<extra_id_35>",
+    "<extra_id_36>",
+    "<extra_id_37>",
+    "<extra_id_38>",
+    "<extra_id_39>",
+    "<extra_id_40>",
+    "<extra_id_41>",
+    "<extra_id_42>",
+    "<extra_id_43>",
+    "<extra_id_44>",
+    "<extra_id_45>",
+    "<extra_id_46>",
+    "<extra_id_47>",
+    "<extra_id_48>",
+    "<extra_id_49>",
+    "<extra_id_50>",
+    "<extra_id_51>",
+    "<extra_id_52>",
+    "<extra_id_53>",
+    "<extra_id_54>",
+    "<extra_id_55>",
+    "<extra_id_56>",
+    "<extra_id_57>",
+    "<extra_id_58>",
+    "<extra_id_59>",
+    "<extra_id_60>",
+    "<extra_id_61>",
+    "<extra_id_62>",
+    "<extra_id_63>",
+    "<extra_id_64>",
+    "<extra_id_65>",
+    "<extra_id_66>",
+    "<extra_id_67>",
+    "<extra_id_68>",
+    "<extra_id_69>",
+    "<extra_id_70>",
+    "<extra_id_71>",
+    "<extra_id_72>",
+    "<extra_id_73>",
+    "<extra_id_74>",
+    "<extra_id_75>",
+    "<extra_id_76>",
+    "<extra_id_77>",
+    "<extra_id_78>",
+    "<extra_id_79>",
+    "<extra_id_80>",
+    "<extra_id_81>",
+    "<extra_id_82>",
+    "<extra_id_83>",
+    "<extra_id_84>",
+    "<extra_id_85>",
+    "<extra_id_86>",
+    "<extra_id_87>",
+    "<extra_id_88>",
+    "<extra_id_89>",
+    "<extra_id_90>",
+    "<extra_id_91>",
+    "<extra_id_92>",
+    "<extra_id_93>",
+    "<extra_id_94>",
+    "<extra_id_95>",
+    "<extra_id_96>",
+    "<extra_id_97>",
+    "<extra_id_98>",
+    "<extra_id_99>"
+  ],
+  "eos_token": "</s>",
+  "extra_ids": 100,
+  "name_or_path": "pszemraj/long-t5-tglobal-base-16384-booksum-V6-partial",
+  "pad_token": "<pad>",
+  "special_tokens_map_file": null,
+  "tokenizer_class": "T5Tokenizer",
+  "unk_token": "<unk>"
+}

trainer_state.json

@@ -0,0 +1,349 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.991123701605288,
+  "global_step": 164,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00030000000000000003,
+      "loss": 2.5934,
+      "step": 3
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0003998458072481446,
+      "loss": 2.5887,
+      "step": 6
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 0.0003990369453344394,
+      "loss": 2.5383,
+      "step": 9
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00039753766811902755,
+      "loss": 2.5382,
+      "step": 12
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 0.0003953531762641745,
+      "loss": 2.5684,
+      "step": 15
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 0.00039249104729072946,
+      "loss": 2.5694,
+      "step": 18
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00038896120929337566,
+      "loss": 2.5685,
+      "step": 21
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.0003847759065022574,
+      "loss": 2.5249,
+      "step": 24
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00037994965681044433,
+      "loss": 2.5835,
+      "step": 27
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00037449920141455944,
+      "loss": 2.5326,
+      "step": 30
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00036844344674325733,
+      "loss": 2.564,
+      "step": 33
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0003618033988749895,
+      "loss": 2.5717,
+      "step": 36
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.0003546020906725474,
+      "loss": 2.5149,
+      "step": 39
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0003468645018871371,
+      "loss": 2.5695,
+      "step": 42
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.0003386174725091272,
+      "loss": 2.5374,
+      "step": 45
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0003298896096660367,
+      "loss": 2.5413,
+      "step": 48
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0003207111883907143,
+      "loss": 2.509,
+      "step": 51
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00031111404660392046,
+      "loss": 2.5628,
+      "step": 54
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00030113147467559695,
+      "loss": 2.5069,
+      "step": 57
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 0.00029079809994790937,
+      "loss": 2.5331,
+      "step": 60
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 0.0002801497666206282,
+      "loss": 2.5527,
+      "step": 63
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 0.0002692234114154986,
+      "loss": 2.6179,
+      "step": 66
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 0.00025805693545089247,
+      "loss": 2.5411,
+      "step": 69
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 0.00024668907277118114,
+      "loss": 2.5583,
+      "step": 72
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 0.00023515925598687094,
+      "loss": 2.534,
+      "step": 75
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 0.00022350747949156756,
+      "loss": 2.5433,
+      "step": 78
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 0.0002117741607302378,
+      "loss": 2.5487,
+      "step": 81
+    },
+    {
+      "epoch": 1.02,
+      "learning_rate": 0.0002,
+      "loss": 3.1324,
+      "step": 84
+    },
+    {
+      "epoch": 1.06,
+      "learning_rate": 0.00018822583926976218,
+      "loss": 2.4641,
+      "step": 87
+    },
+    {
+      "epoch": 1.1,
+      "learning_rate": 0.00017649252050843252,
+      "loss": 2.4806,
+      "step": 90
+    },
+    {
+      "epoch": 1.13,
+      "learning_rate": 0.0001648407440131291,
+      "loss": 2.4693,
+      "step": 93
+    },
+    {
+      "epoch": 1.17,
+      "learning_rate": 0.000153310927228819,
+      "loss": 2.5011,
+      "step": 96
+    },
+    {
+      "epoch": 1.21,
+      "learning_rate": 0.00014194306454910757,
+      "loss": 2.4595,
+      "step": 99
+    },
+    {
+      "epoch": 1.24,
+      "learning_rate": 0.00013077658858450138,
+      "loss": 2.4893,
+      "step": 102
+    },
+    {
+      "epoch": 1.28,
+      "learning_rate": 0.00011985023337937184,
+      "loss": 2.4932,
+      "step": 105
+    },
+    {
+      "epoch": 1.31,
+      "learning_rate": 0.00010920190005209065,
+      "loss": 2.4871,
+      "step": 108
+    },
+    {
+      "epoch": 1.35,
+      "learning_rate": 9.886852532440312e-05,
+      "loss": 2.4672,
+      "step": 111
+    },
+    {
+      "epoch": 1.39,
+      "learning_rate": 8.888595339607961e-05,
+      "loss": 2.4597,
+      "step": 114
+    },
+    {
+      "epoch": 1.42,
+      "learning_rate": 7.928881160928572e-05,
+      "loss": 2.4531,
+      "step": 117
+    },
+    {
+      "epoch": 1.46,
+      "learning_rate": 7.011039033396329e-05,
+      "loss": 2.4749,
+      "step": 120
+    },
+    {
+      "epoch": 1.5,
+      "learning_rate": 6.138252749087286e-05,
+      "loss": 2.4551,
+      "step": 123
+    },
+    {
+      "epoch": 1.53,
+      "learning_rate": 5.313549811286293e-05,
+      "loss": 2.4796,
+      "step": 126
+    },
+    {
+      "epoch": 1.57,
+      "learning_rate": 4.53979093274526e-05,
+      "loss": 2.4705,
+      "step": 129
+    },
+    {
+      "epoch": 1.6,
+      "learning_rate": 3.819660112501053e-05,
+      "loss": 2.4915,
+      "step": 132
+    },
+    {
+      "epoch": 1.64,
+      "learning_rate": 3.1556553256742713e-05,
+      "loss": 2.4958,
+      "step": 135
+    },
+    {
+      "epoch": 1.68,
+      "learning_rate": 2.5500798585440567e-05,
+      "loss": 2.4814,
+      "step": 138
+    },
+    {
+      "epoch": 1.71,
+      "learning_rate": 2.0050343189555743e-05,
+      "loss": 2.5034,
+      "step": 141
+    },
+    {
+      "epoch": 1.75,
+      "learning_rate": 1.5224093497742653e-05,
+      "loss": 2.4671,
+      "step": 144
+    },
+    {
+      "epoch": 1.79,
+      "learning_rate": 1.1038790706624391e-05,
+      "loss": 2.4987,
+      "step": 147
+    },
+    {
+      "epoch": 1.82,
+      "learning_rate": 7.508952709270567e-06,
+      "loss": 2.4618,
+      "step": 150
+    },
+    {
+      "epoch": 1.86,
+      "learning_rate": 4.646823735825523e-06,
+      "loss": 2.4815,
+      "step": 153
+    },
+    {
+      "epoch": 1.89,
+      "learning_rate": 2.462331880972468e-06,
+      "loss": 2.4595,
+      "step": 156
+    },
+    {
+      "epoch": 1.93,
+      "learning_rate": 9.630546655606364e-07,
+      "loss": 2.4925,
+      "step": 159
+    },
+    {
+      "epoch": 1.97,
+      "learning_rate": 1.5419275185541982e-07,
+      "loss": 2.4333,
+      "step": 162
+    },
+    {
+      "epoch": 1.99,
+      "step": 164,
+      "total_flos": 4.620604962546647e+17,
+      "train_loss": 2.525294606278582,
+      "train_runtime": 28715.0205,
+      "train_samples_per_second": 0.738,
+      "train_steps_per_second": 0.006
+    }
+  ],
+  "max_steps": 164,
+  "num_train_epochs": 2,
+  "total_flos": 4.620604962546647e+17,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:673e02f4e38479f806c1a20434ea2efa4f9cf7a6aa07067907fde9dc0160405c
+size 4527