Model save

README.md

@@ -0,0 +1,69 @@
+---
+license: apache-2.0
+base_model: deepmind/language-perceiver
+tags:
+- generated_from_trainer
+metrics:
+- f1
+model-index:
+- name: language-perceiver-goodreads-bookgenres-Book_cls-8e
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# language-perceiver-goodreads-bookgenres-Book_cls-8e
+
+This model is a fine-tuned version of [deepmind/language-perceiver](https://huggingface.co/deepmind/language-perceiver) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.2832
+- F1: 0.5108
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 2e-05
+- train_batch_size: 32
+- eval_batch_size: 32
+- seed: 42
+- gradient_accumulation_steps: 4
+- total_train_batch_size: 128
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- num_epochs: 8.0
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | F1     |
+|:-------------:|:-----:|:----:|:---------------:|:------:|
+| 0.3059        | 1.0   | 62   | 0.2893          | 0.3263 |
+| 0.2879        | 2.0   | 124  | 0.2795          | 0.4290 |
+| 0.2729        | 3.0   | 186  | 0.2730          | 0.4356 |
+| 0.2606        | 4.0   | 248  | 0.2722          | 0.4590 |
+| 0.2433        | 5.0   | 310  | 0.2747          | 0.4775 |
+| 0.227         | 6.0   | 372  | 0.2777          | 0.4976 |
+| 0.207         | 7.0   | 434  | 0.2814          | 0.5088 |
+| 0.1969        | 8.0   | 496  | 0.2832          | 0.5108 |
+
+
+### Framework versions
+
+- Transformers 4.33.3
+- Pytorch 2.2.0.dev20231001+cu121
+- Datasets 2.14.5
+- Tokenizers 0.13.3

added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "[MASK]": 128000
+}

all_results.json

@@ -0,0 +1,14 @@
+{
+    "epoch": 4.97,
+    "eval_f1": 0.7095034823635139,
+    "eval_loss": 0.20497334003448486,
+    "eval_runtime": 16.7618,
+    "eval_samples": 989,
+    "eval_samples_per_second": 59.003,
+    "eval_steps_per_second": 3.699,
+    "train_loss": 0.20427038223762822,
+    "train_runtime": 2213.5462,
+    "train_samples": 7914,
+    "train_samples_per_second": 17.876,
+    "train_steps_per_second": 0.278
+}

config.json

@@ -0,0 +1,86 @@
+{
+  "_label_trainable_num_channels": 1024,
+  "_name_or_path": "deepmind/language-perceiver",
+  "architectures": [
+    "PerceiverForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "audio_samples_per_frame": 1920,
+  "cross_attention_shape_for_attention": "kv",
+  "cross_attention_widening_factor": 1,
+  "d_latents": 1280,
+  "d_model": 768,
+  "finetuning_task": "text-classification",
+  "hidden_act": "gelu",
+  "id2label": {
+    "0": "History & Politics",
+    "1": "Health & Medicine",
+    "2": "Mystery & Thriller",
+    "3": "Arts & Design",
+    "4": "Self-Help & Wellness",
+    "5": "Sports & Recreation",
+    "6": "Non-Fiction",
+    "7": "Science Fiction & Fantasy",
+    "8": "Countries & Geography",
+    "9": "Other",
+    "10": "Nature & Environment",
+    "11": "Business & Finance",
+    "12": "Romance",
+    "13": "Philosophy & Religion",
+    "14": "Literature & Fiction",
+    "15": "Science & Technology",
+    "16": "Children & Young Adult",
+    "17": "Food & Cooking"
+  },
+  "image_size": 56,
+  "initializer_range": 0.02,
+  "label2id": {
+    "Arts & Design": 3,
+    "Business & Finance": 11,
+    "Children & Young Adult": 16,
+    "Countries & Geography": 8,
+    "Food & Cooking": 17,
+    "Health & Medicine": 1,
+    "History & Politics": 0,
+    "Literature & Fiction": 14,
+    "Mystery & Thriller": 2,
+    "Nature & Environment": 10,
+    "Non-Fiction": 6,
+    "Other": 9,
+    "Philosophy & Religion": 13,
+    "Romance": 12,
+    "Science & Technology": 15,
+    "Science Fiction & Fantasy": 7,
+    "Self-Help & Wellness": 4,
+    "Sports & Recreation": 5
+  },
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 2048,
+  "model_type": "perceiver",
+  "num_blocks": 1,
+  "num_cross_attention_heads": 8,
+  "num_frames": 16,
+  "num_latents": 256,
+  "num_self_attends_per_block": 26,
+  "num_self_attention_heads": 8,
+  "output_num_channels": 512,
+  "output_shape": [
+    1,
+    16,
+    224,
+    224
+  ],
+  "problem_type": "multi_label_classification",
+  "qk_channels": 256,
+  "samples_per_patch": 16,
+  "self_attention_widening_factor": 1,
+  "torch_dtype": "float32",
+  "train_size": [
+    368,
+    496
+  ],
+  "transformers_version": "4.33.3",
+  "use_query_residual": true,
+  "v_channels": 1280,
+  "vocab_size": 262
+}

eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 4.97,
+    "eval_f1": 0.7095034823635139,
+    "eval_loss": 0.20497334003448486,
+    "eval_runtime": 16.7618,
+    "eval_samples": 989,
+    "eval_samples_per_second": 59.003,
+    "eval_steps_per_second": 3.699
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9312728a4bb10a39a19e2e83c6166b9dc3ec22deee8d403c62ad974aaa094294
+size 824536032

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ffc07d47a48c3dcf3716053c4708ce476a72b2c1f5acf526b395d55c4bdaafb1
+size 567671666

special_tokens_map.json

@@ -0,0 +1,44 @@
+{
+  "bos_token": {
+    "content": "[BOS]",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "content": "[CLS]",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "[EOS]",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "[MASK]",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "[SEP]",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

spiece.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fefb02b667a6c5c2fe27602d28e5fb3428f66ab89c7d6f388e7c8d44a02d0336
+size 760289

spm.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c679fbf93643d19aab7ee10c0b99e460bdbc02fedf34b92b05af343b4af586fd
+size 2464616

tokenizer_config.json

@@ -0,0 +1,53 @@
+{
+  "bos_token": {
+    "__type": "AddedToken",
+    "content": "[BOS]",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "clean_up_tokenization_spaces": true,
+  "cls_token": {
+    "__type": "AddedToken",
+    "content": "[CLS]",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "__type": "AddedToken",
+    "content": "[EOS]",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "__type": "AddedToken",
+    "content": "[MASK]",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "model_max_length": 2048,
+  "pad_token": {
+    "__type": "AddedToken",
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "__type": "AddedToken",
+    "content": "[SEP]",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "tokenizer_class": "PerceiverTokenizer"
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 4.97,
+    "train_loss": 0.20427038223762822,
+    "train_runtime": 2213.5462,
+    "train_samples": 7914,
+    "train_samples_per_second": 17.876,
+    "train_steps_per_second": 0.278
+}

trainer_state.json

@@ -0,0 +1,439 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 4.96969696969697,
+  "eval_steps": 500,
+  "global_step": 615,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.08,
+      "learning_rate": 1.9674796747967483e-05,
+      "loss": 0.5781,
+      "step": 10
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 1.934959349593496e-05,
+      "loss": 0.4087,
+      "step": 20
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 1.902439024390244e-05,
+      "loss": 0.3448,
+      "step": 30
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 1.869918699186992e-05,
+      "loss": 0.322,
+      "step": 40
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 1.83739837398374e-05,
+      "loss": 0.3095,
+      "step": 50
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 1.804878048780488e-05,
+      "loss": 0.3056,
+      "step": 60
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 1.772357723577236e-05,
+      "loss": 0.2888,
+      "step": 70
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.739837398373984e-05,
+      "loss": 0.2753,
+      "step": 80
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.7073170731707317e-05,
+      "loss": 0.2737,
+      "step": 90
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.6747967479674798e-05,
+      "loss": 0.2633,
+      "step": 100
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.642276422764228e-05,
+      "loss": 0.2583,
+      "step": 110
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.6097560975609757e-05,
+      "loss": 0.2518,
+      "step": 120
+    },
+    {
+      "epoch": 0.99,
+      "eval_f1": 0.5708365708365708,
+      "eval_loss": 0.23673325777053833,
+      "eval_runtime": 17.0069,
+      "eval_samples_per_second": 58.153,
+      "eval_steps_per_second": 3.646,
+      "step": 123
+    },
+    {
+      "epoch": 1.05,
+      "learning_rate": 1.5772357723577235e-05,
+      "loss": 0.2427,
+      "step": 130
+    },
+    {
+      "epoch": 1.13,
+      "learning_rate": 1.5447154471544717e-05,
+      "loss": 0.2416,
+      "step": 140
+    },
+    {
+      "epoch": 1.21,
+      "learning_rate": 1.5121951219512196e-05,
+      "loss": 0.2285,
+      "step": 150
+    },
+    {
+      "epoch": 1.29,
+      "learning_rate": 1.4796747967479676e-05,
+      "loss": 0.2263,
+      "step": 160
+    },
+    {
+      "epoch": 1.37,
+      "learning_rate": 1.4471544715447157e-05,
+      "loss": 0.2241,
+      "step": 170
+    },
+    {
+      "epoch": 1.45,
+      "learning_rate": 1.4146341463414635e-05,
+      "loss": 0.2254,
+      "step": 180
+    },
+    {
+      "epoch": 1.54,
+      "learning_rate": 1.3821138211382115e-05,
+      "loss": 0.2213,
+      "step": 190
+    },
+    {
+      "epoch": 1.62,
+      "learning_rate": 1.3495934959349594e-05,
+      "loss": 0.2155,
+      "step": 200
+    },
+    {
+      "epoch": 1.7,
+      "learning_rate": 1.3170731707317076e-05,
+      "loss": 0.2201,
+      "step": 210
+    },
+    {
+      "epoch": 1.78,
+      "learning_rate": 1.2845528455284555e-05,
+      "loss": 0.2167,
+      "step": 220
+    },
+    {
+      "epoch": 1.86,
+      "learning_rate": 1.2520325203252033e-05,
+      "loss": 0.215,
+      "step": 230
+    },
+    {
+      "epoch": 1.94,
+      "learning_rate": 1.2195121951219513e-05,
+      "loss": 0.2111,
+      "step": 240
+    },
+    {
+      "epoch": 2.0,
+      "eval_f1": 0.6588750913075239,
+      "eval_loss": 0.2085004448890686,
+      "eval_runtime": 17.0139,
+      "eval_samples_per_second": 58.129,
+      "eval_steps_per_second": 3.644,
+      "step": 247
+    },
+    {
+      "epoch": 2.02,
+      "learning_rate": 1.1869918699186992e-05,
+      "loss": 0.1933,
+      "step": 250
+    },
+    {
+      "epoch": 2.1,
+      "learning_rate": 1.1544715447154474e-05,
+      "loss": 0.1915,
+      "step": 260
+    },
+    {
+      "epoch": 2.18,
+      "learning_rate": 1.1219512195121953e-05,
+      "loss": 0.187,
+      "step": 270
+    },
+    {
+      "epoch": 2.26,
+      "learning_rate": 1.0894308943089431e-05,
+      "loss": 0.1847,
+      "step": 280
+    },
+    {
+      "epoch": 2.34,
+      "learning_rate": 1.0569105691056911e-05,
+      "loss": 0.1868,
+      "step": 290
+    },
+    {
+      "epoch": 2.42,
+      "learning_rate": 1.024390243902439e-05,
+      "loss": 0.1835,
+      "step": 300
+    },
+    {
+      "epoch": 2.51,
+      "learning_rate": 9.91869918699187e-06,
+      "loss": 0.1813,
+      "step": 310
+    },
+    {
+      "epoch": 2.59,
+      "learning_rate": 9.59349593495935e-06,
+      "loss": 0.1819,
+      "step": 320
+    },
+    {
+      "epoch": 2.67,
+      "learning_rate": 9.268292682926831e-06,
+      "loss": 0.1746,
+      "step": 330
+    },
+    {
+      "epoch": 2.75,
+      "learning_rate": 8.94308943089431e-06,
+      "loss": 0.181,
+      "step": 340
+    },
+    {
+      "epoch": 2.83,
+      "learning_rate": 8.617886178861789e-06,
+      "loss": 0.1844,
+      "step": 350
+    },
+    {
+      "epoch": 2.91,
+      "learning_rate": 8.292682926829268e-06,
+      "loss": 0.169,
+      "step": 360
+    },
+    {
+      "epoch": 2.99,
+      "learning_rate": 7.967479674796748e-06,
+      "loss": 0.1833,
+      "step": 370
+    },
+    {
+      "epoch": 3.0,
+      "eval_f1": 0.6790123456790124,
+      "eval_loss": 0.2064265012741089,
+      "eval_runtime": 16.9971,
+      "eval_samples_per_second": 58.186,
+      "eval_steps_per_second": 3.648,
+      "step": 371
+    },
+    {
+      "epoch": 3.07,
+      "learning_rate": 7.64227642276423e-06,
+      "loss": 0.1625,
+      "step": 380
+    },
+    {
+      "epoch": 3.15,
+      "learning_rate": 7.317073170731707e-06,
+      "loss": 0.1617,
+      "step": 390
+    },
+    {
+      "epoch": 3.23,
+      "learning_rate": 6.991869918699188e-06,
+      "loss": 0.1582,
+      "step": 400
+    },
+    {
+      "epoch": 3.31,
+      "learning_rate": 6.666666666666667e-06,
+      "loss": 0.1584,
+      "step": 410
+    },
+    {
+      "epoch": 3.39,
+      "learning_rate": 6.341463414634147e-06,
+      "loss": 0.1512,
+      "step": 420
+    },
+    {
+      "epoch": 3.47,
+      "learning_rate": 6.016260162601627e-06,
+      "loss": 0.1574,
+      "step": 430
+    },
+    {
+      "epoch": 3.56,
+      "learning_rate": 5.691056910569106e-06,
+      "loss": 0.1564,
+      "step": 440
+    },
+    {
+      "epoch": 3.64,
+      "learning_rate": 5.365853658536586e-06,
+      "loss": 0.1584,
+      "step": 450
+    },
+    {
+      "epoch": 3.72,
+      "learning_rate": 5.040650406504065e-06,
+      "loss": 0.1495,
+      "step": 460
+    },
+    {
+      "epoch": 3.8,
+      "learning_rate": 4.715447154471545e-06,
+      "loss": 0.1591,
+      "step": 470
+    },
+    {
+      "epoch": 3.88,
+      "learning_rate": 4.390243902439025e-06,
+      "loss": 0.1564,
+      "step": 480
+    },
+    {
+      "epoch": 3.96,
+      "learning_rate": 4.0650406504065046e-06,
+      "loss": 0.1485,
+      "step": 490
+    },
+    {
+      "epoch": 4.0,
+      "eval_f1": 0.7020743104627308,
+      "eval_loss": 0.20207864046096802,
+      "eval_runtime": 17.0075,
+      "eval_samples_per_second": 58.151,
+      "eval_steps_per_second": 3.645,
+      "step": 495
+    },
+    {
+      "epoch": 4.04,
+      "learning_rate": 3.7398373983739838e-06,
+      "loss": 0.1477,
+      "step": 500
+    },
+    {
+      "epoch": 4.12,
+      "learning_rate": 3.414634146341464e-06,
+      "loss": 0.1406,
+      "step": 510
+    },
+    {
+      "epoch": 4.2,
+      "learning_rate": 3.0894308943089435e-06,
+      "loss": 0.1383,
+      "step": 520
+    },
+    {
+      "epoch": 4.28,
+      "learning_rate": 2.764227642276423e-06,
+      "loss": 0.1374,
+      "step": 530
+    },
+    {
+      "epoch": 4.36,
+      "learning_rate": 2.4390243902439027e-06,
+      "loss": 0.1282,
+      "step": 540
+    },
+    {
+      "epoch": 4.44,
+      "learning_rate": 2.1138211382113824e-06,
+      "loss": 0.1338,
+      "step": 550
+    },
+    {
+      "epoch": 4.53,
+      "learning_rate": 1.788617886178862e-06,
+      "loss": 0.1397,
+      "step": 560
+    },
+    {
+      "epoch": 4.61,
+      "learning_rate": 1.4634146341463414e-06,
+      "loss": 0.143,
+      "step": 570
+    },
+    {
+      "epoch": 4.69,
+      "learning_rate": 1.1382113821138213e-06,
+      "loss": 0.1392,
+      "step": 580
+    },
+    {
+      "epoch": 4.77,
+      "learning_rate": 8.130081300813009e-07,
+      "loss": 0.1374,
+      "step": 590
+    },
+    {
+      "epoch": 4.85,
+      "learning_rate": 4.878048780487805e-07,
+      "loss": 0.1401,
+      "step": 600
+    },
+    {
+      "epoch": 4.93,
+      "learning_rate": 1.6260162601626018e-07,
+      "loss": 0.1382,
+      "step": 610
+    },
+    {
+      "epoch": 4.97,
+      "eval_f1": 0.7095034823635139,
+      "eval_loss": 0.20497334003448486,
+      "eval_runtime": 17.0074,
+      "eval_samples_per_second": 58.151,
+      "eval_steps_per_second": 3.645,
+      "step": 615
+    },
+    {
+      "epoch": 4.97,
+      "step": 615,
+      "total_flos": 3.666068136773222e+16,
+      "train_loss": 0.20427038223762822,
+      "train_runtime": 2213.5462,
+      "train_samples_per_second": 17.876,
+      "train_steps_per_second": 0.278
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 615,
+  "num_train_epochs": 5,
+  "save_steps": 500,
+  "total_flos": 3.666068136773222e+16,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05febb53989d3609718ff89cfe07adccc4fe13af016c2a02e54484aa6e188756
+size 4600