Further train for another 800K steps on CC-100, Twitter, Updated Wikipedia

.gitattributes

@@ -16,3 +16,4 @@
 *.pth filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 model.onnx filter=lfs diff=lfs merge=lfs -text
+tf_model.h5 filter=lfs diff=lfs merge=lfs -text

config.json

@@ -26,7 +26,9 @@
   "n_layer": 6,
   "n_positions": 1024,
   "pad_token_id": 50257,
+  "reorder_and_upcast_attn": false,
   "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
   "scale_attn_weights": true,
   "summary_activation": null,
   "summary_first_dropout": 0.1,
@@ -40,7 +42,7 @@
     }
   },
   "torch_dtype": "float32",
-  "transformers_version": "4.9.0.dev0",
+  "transformers_version": "4.22.0.dev0",
   "use_cache": true,
   "vocab_size": 50257
 }

distilgpt2-base-pretrained-he.mlpackage/Data/com.apple.CoreML/FeatureDescriptions.json

@@ -1,15 +1,15 @@
 {
   "Outputs" : {
     "output_logits" : {
-      "MLFeatureShortDescription" : ""
+      "MLFeatureShortDescription" : "--"
     }
   },
   "Inputs" : {
     "position_ids" : {
-      "MLFeatureShortDescription" : ""
+      "MLFeatureShortDescription" : "--"
     },
     "input_ids" : {
-      "MLFeatureShortDescription" : ""
+      "MLFeatureShortDescription" : "--"
     }
   },
   "TrainingInputs" : {

distilgpt2-base-pretrained-he.mlpackage/Data/com.apple.CoreML/Metadata.json

@@ -1,5 +1,5 @@
 {
-  "MLModelVersionStringKey" : "1.0",
+  "MLModelVersionStringKey" : "1.01",
   "MLModelDescriptionKey" : "hebrew-distilgpt2\n\nA tiny GPT2 based Hebrew text generation model trained on a TPUv3-8 via the TPU Research Cloud Program.",
   "MLModelCreatorDefinedKey" : {
     "model_card_url" : "https:\/\/huggingface.co\/Norod78\/distilgpt2-base-pretrained-he"

distilgpt2-base-pretrained-he.mlpackage/Data/com.apple.CoreML/model.mlmodel

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dacb6071938fc719d81c51db527ae9a756d4292c183cf05b8d3e646340a1544e
+oid sha256:3873d1a72c554711785b15a1b27af3824f1269c2efdc1de65181f1031b6565fa
 size 482254328

distilgpt2-base-pretrained-he.mlpackage/Manifest.json

@@ -1,24 +1,24 @@
 {
     "fileFormatVersion": "1.0.0",
     "itemInfoEntries": {
-        "06B636B9-5BE3-49A2-A06B-C09242D53296": {
+        "5CA8030A-3376-40B9-9F77-FE7151EBE0F7": {
             "author": "com.apple.CoreML",
             "description": "External FeatureDescription Overlay",
             "name": "FeatureDescriptions.json",
             "path": "com.apple.CoreML/FeatureDescriptions.json"
         },
-        "A8586C2A-1DD7-4EEC-BB42-6B6242D7B530": {
+        "5EFA4247-BC5C-47CE-9E64-F1747845A076": {
+            "author": "com.apple.CoreML",
+            "description": "CoreML Model Specification",
+            "name": "model.mlmodel",
+            "path": "com.apple.CoreML/model.mlmodel"
+        },
+        "63DB2BDF-B3FD-4CE4-9251-66F29CF34545": {
             "author": "com.apple.CoreML",
             "description": "External Metadata Overlay",
             "name": "Metadata.json",
             "path": "com.apple.CoreML/Metadata.json"
-        },
-        "FD401BB0-2CA8-4DB7-BAD6-23B171A68404": {
-            "author": "com.apple.CoreML",
-            "description": "CoreML Model Specification",
-            "name": "distilgpt2-base-pretrained-he-64-6.mlmodel",
-            "path": "com.apple.CoreML/model.mlmodel"
         }
     },
-    "rootModelIdentifier": "FD401BB0-2CA8-4DB7-BAD6-23B171A68404"
+    "rootModelIdentifier": "5EFA4247-BC5C-47CE-9E64-F1747845A076"
 }

flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e802bdbb1a2aad4bce6864d4c79016213f70e7b877756cad0684a7c9bf70b0eb
+oid sha256:fda64ecb202c9ee0b7b7575f8db1c84efd42967f979ff7d1b2f2b0d32fd8b801
 size 327652826

model.onnx

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8b9459cc41adc2c406570135eb583a5d351b2f6ea636875d831cd2c36cfcc674
+oid sha256:d5276bfcba7af74619c75c10e9115f4198e2e30352a7cf0e7155c5152f236360
 size 488438673

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f2b55dc856c55569caf9944826aa90c5342e3138e46525af27fb7729dba37ad2
-size 333973519
+oid sha256:bef275decdf17187b39d8e6e0a4cd1213c9d8970b2dc737c9139152d9180b295
+size 333969117

tf_model.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21b09d55184b71e392d4b6ef2d1e064b34668413849a99d1a86a7312f6cf151e
+size 327744824

tokenizer.json

@@ -53,17 +53,20 @@
   "pre_tokenizer": {
     "type": "ByteLevel",
     "add_prefix_space": false,
-    "trim_offsets": true
+    "trim_offsets": true,
+    "use_regex": true
   },
   "post_processor": {
     "type": "ByteLevel",
     "add_prefix_space": true,
-    "trim_offsets": false
+    "trim_offsets": false,
+    "use_regex": true
   },
   "decoder": {
     "type": "ByteLevel",
     "add_prefix_space": true,
-    "trim_offsets": true
+    "trim_offsets": true,
+    "use_regex": true
   },
   "model": {
     "type": "BPE",

events.out.tfevents.1626785892.t1v-n-d9fb8529-w-0.668251.3.v2 → training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5e056841ed96b9de2cb1af0f327c3d566444a58da894d4e00021fe053564ac26
-size 1619079
+oid sha256:d2e3602b258d62c2b8a7bd97787a1787754e6a8e44f598e6a5cd298cb5e3fe5a
+size 3439