{ "nbformat": 4, "nbformat_minor": 0, "metadata": { "colab": { "provenance": [] }, "kernelspec": { "name": "python3", "display_name": "Python 3" }, "language_info": { "name": "python" }, "accelerator": "GPU", "gpuClass": "standard", "widgets": { "application/vnd.jupyter.widget-state+json": { "79e7afed6f204ced8012f9853e8ce153": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_92e3b0a9ee344022b0bf16a6617239b3", "IPY_MODEL_09fa538b6a05498282c2438a6c854fd1", "IPY_MODEL_3b905f68919049608a3eb3c48da12032" ], "layout": "IPY_MODEL_eee35ecb15c84ee994f9ffa479feeaaf" } }, "92e3b0a9ee344022b0bf16a6617239b3": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_c17b6f3b3436490e923835add11f8d74", "placeholder": "​", "style": "IPY_MODEL_4d5a02eb385d470f8b3c68a8db60561c", "value": "Map: 100%" } }, "09fa538b6a05498282c2438a6c854fd1": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_7946964d5e8741ba86b2662397ccb891", "max": 18000, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_5624c10fc93943dda1e0c93c1fc2fbba", "value": 18000 } }, "3b905f68919049608a3eb3c48da12032": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_101f117dfb0a441e807757524b857350", "placeholder": "​", "style": "IPY_MODEL_77819fe20f514982b8547abff9361b2d", "value": " 18000/18000 [00:03<00:00, 5372.46 examples/s]" } }, "eee35ecb15c84ee994f9ffa479feeaaf": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": "hidden", "width": null } }, "c17b6f3b3436490e923835add11f8d74": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "4d5a02eb385d470f8b3c68a8db60561c": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "7946964d5e8741ba86b2662397ccb891": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "5624c10fc93943dda1e0c93c1fc2fbba": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "101f117dfb0a441e807757524b857350": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "77819fe20f514982b8547abff9361b2d": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "8f76ba596bdc427aa868685b151f51d8": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_a14ec2755a2a477aa654287af8a463a3", "IPY_MODEL_cb82b162aae342888f353ea2fe507015", "IPY_MODEL_ebb3365ce29e476ab28201e9b825e302" ], "layout": "IPY_MODEL_31e7fabdde2f4129ab38d49b5c5b1d50" } }, "a14ec2755a2a477aa654287af8a463a3": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_b2fb5695afd04756975281c5a04fbcaf", "placeholder": "​", "style": "IPY_MODEL_6e358ea9a32a486db0c81a5f54a112e9", "value": "Map: 100%" } }, "cb82b162aae342888f353ea2fe507015": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_647a42b85a8044229d71fd1e7318878b", "max": 2000, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_f439095945fb413e916059f01f67bc79", "value": 2000 } }, "ebb3365ce29e476ab28201e9b825e302": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_fece699d83854b5e876164da8a3b6f3a", "placeholder": "​", "style": "IPY_MODEL_814444ea270d4739aa8588937801800b", "value": " 2000/2000 [00:00<00:00, 5993.97 examples/s]" } }, "31e7fabdde2f4129ab38d49b5c5b1d50": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": "hidden", "width": null } }, "b2fb5695afd04756975281c5a04fbcaf": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "6e358ea9a32a486db0c81a5f54a112e9": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "647a42b85a8044229d71fd1e7318878b": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "f439095945fb413e916059f01f67bc79": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "fece699d83854b5e876164da8a3b6f3a": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "814444ea270d4739aa8588937801800b": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "00047affc56f494c82268e7a8bc2c53c": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_dd7a5a0c04fa4b66a0b5078b0a3e59e0", "IPY_MODEL_eeaf71899a6743a081f79552dfc1672b", "IPY_MODEL_23f2d28b36c84ea78422ef020d0d5e4b" ], "layout": "IPY_MODEL_02a6351920ba4a0689b19bb9531dc8df" } }, "dd7a5a0c04fa4b66a0b5078b0a3e59e0": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_2859d235432e4d12a78f55412ba8c569", "placeholder": "​", "style": "IPY_MODEL_7a8e9cb5072b47fa9a5cee174d29bc47", "value": "Downloading pytorch_model.bin: 100%" } }, "eeaf71899a6743a081f79552dfc1672b": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_b6f515df241e44f7ba04114423fc1f7e", "max": 265629392, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_7fad565b40de40c2b3d8d72d8c50111e", "value": 265629392 } }, "23f2d28b36c84ea78422ef020d0d5e4b": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_0c271f3f91314bd094d8db4834c90e41", "placeholder": "​", "style": "IPY_MODEL_243a2bc40dbd48d2910616fa0e1a01a6", "value": " 266M/266M [00:15<00:00, 18.8MB/s]" } }, "02a6351920ba4a0689b19bb9531dc8df": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "2859d235432e4d12a78f55412ba8c569": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "7a8e9cb5072b47fa9a5cee174d29bc47": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "b6f515df241e44f7ba04114423fc1f7e": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "7fad565b40de40c2b3d8d72d8c50111e": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "0c271f3f91314bd094d8db4834c90e41": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "243a2bc40dbd48d2910616fa0e1a01a6": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } } } } }, "cells": [ { "cell_type": "markdown", "source": [ "Задача - определить диалект португальского языка по фразе.\n", "\n", "Данные взяты из субтитров кофнеренции TED" ], "metadata": { "id": "qF-Rki_SBT7-" } }, { "cell_type": "markdown", "source": [ "### Подготовка данных" ], "metadata": { "id": "gjsF2U3a9_HR" } }, { "cell_type": "code", "source": [ "!wget https://object.pouta.csc.fi/OPUS-TED2020/v1/mono/pt.txt.gz\n", "!wget https://object.pouta.csc.fi/OPUS-TED2020/v1/mono/pt_br.txt.gz\n", "!gunzip pt.txt.gz\n", "!gunzip pt_br.txt.gz\n" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "XwRRwPFp9-JV", "outputId": "acfbaa31-8efd-45ed-cc42-8b124fac681e" }, "execution_count": 2, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "--2023-04-15 10:19:27-- https://object.pouta.csc.fi/OPUS-TED2020/v1/mono/pt.txt.gz\n", "Resolving object.pouta.csc.fi (object.pouta.csc.fi)... 86.50.254.19\n", "Connecting to object.pouta.csc.fi (object.pouta.csc.fi)|86.50.254.19|:443... connected.\n", "HTTP request sent, awaiting response... 200 OK\n", "Length: 11610902 (11M) [application/gzip]\n", "Saving to: ‘pt.txt.gz’\n", "\n", "pt.txt.gz 100%[===================>] 11.07M 4.32MB/s in 2.6s \n", "\n", "2023-04-15 10:19:31 (4.32 MB/s) - ‘pt.txt.gz’ saved [11610902/11610902]\n", "\n", "--2023-04-15 10:19:31-- https://object.pouta.csc.fi/OPUS-TED2020/v1/mono/pt_br.txt.gz\n", "Resolving object.pouta.csc.fi (object.pouta.csc.fi)... 86.50.254.19\n", "Connecting to object.pouta.csc.fi (object.pouta.csc.fi)|86.50.254.19|:443... connected.\n", "HTTP request sent, awaiting response... 200 OK\n", "Length: 14802314 (14M) [application/gzip]\n", "Saving to: ‘pt_br.txt.gz’\n", "\n", "pt_br.txt.gz 100%[===================>] 14.12M 4.94MB/s in 2.9s \n", "\n", "2023-04-15 10:19:35 (4.94 MB/s) - ‘pt_br.txt.gz’ saved [14802314/14802314]\n", "\n" ] } ] }, { "cell_type": "code", "source": [ "import pandas as pd\n", "\n", "pt = pd.read_csv('pt.txt', sep='\\t', names=['text'])\n", "pt['target'] = 'pt'\n", "pt_br = pd.read_csv('pt_br.txt', sep='\\t', names=['text'])\n", "pt_br['target'] = 'pt_br'\n", "data = pd.concat([pt[:10000], pt_br[:10000]], ignore_index=True)" ], "metadata": { "id": "DIZD8gBB-6mo" }, "execution_count": 35, "outputs": [] }, { "cell_type": "code", "source": [ "data" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 423 }, "id": "FQdiHJihD46w", "outputId": "5de3ab8d-032a-4ed9-dab5-8961ad3b19e3" }, "execution_count": 4, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ " text target\n", "0 Em meados do século XVI, os italianos eram cat... pt\n", "1 Porém, este dom tinha um alto preço. pt\n", "2 Para impedir a alteração da voz, esses cantore... pt\n", "3 Conhecidos por \"castrati\", estas vozes agudas ... pt\n", "4 Embora o crescimento vocal retardado possa pro... pt\n", "... ... ...\n", "49995 Como você acha que seria o mundo se você fosse... pt_br\n", "49996 Bem, a resposta é nada. pt_br\n", "49997 A razão por que você não consegue enxergar no ... pt_br\n", "49998 Para ver uma maçã, a luz tem que atingir a maç... pt_br\n", "49999 Então, as retinas em seus olhos recebem o refl... pt_br\n", "\n", "[50000 rows x 2 columns]" ], "text/html": [ "\n", "
\n", "
\n", "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
texttarget
0Em meados do século XVI, os italianos eram cat...pt
1Porém, este dom tinha um alto preço.pt
2Para impedir a alteração da voz, esses cantore...pt
3Conhecidos por \"castrati\", estas vozes agudas ...pt
4Embora o crescimento vocal retardado possa pro...pt
.........
49995Como você acha que seria o mundo se você fosse...pt_br
49996Bem, a resposta é nada.pt_br
49997A razão por que você não consegue enxergar no ...pt_br
49998Para ver uma maçã, a luz tem que atingir a maç...pt_br
49999Então, as retinas em seus olhos recebem o refl...pt_br
\n", "

50000 rows × 2 columns

\n", "
\n", " \n", " \n", " \n", "\n", " \n", "
\n", "
\n", " " ] }, "metadata": {}, "execution_count": 4 } ] }, { "cell_type": "markdown", "source": [ "### Файнтюн модели" ], "metadata": { "id": "u-03LX5EBtED" } }, { "cell_type": "code", "execution_count": 5, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "kgsVl04L9r0o", "outputId": "1933d7b0-0ea1-485c-a1b9-05066786f448" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n", "Collecting transformers\n", " Downloading transformers-4.28.1-py3-none-any.whl (7.0 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.0/7.0 MB\u001b[0m \u001b[31m98.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting tokenizers!=0.11.3,<0.14,>=0.11.1\n", " Downloading tokenizers-0.13.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m102.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting huggingface-hub<1.0,>=0.11.0\n", " Downloading huggingface_hub-0.13.4-py3-none-any.whl (200 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m200.1/200.1 kB\u001b[0m \u001b[31m12.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.9/dist-packages (from transformers) (3.11.0)\n", "Requirement already satisfied: requests in /usr/local/lib/python3.9/dist-packages (from transformers) (2.27.1)\n", "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.9/dist-packages (from transformers) (6.0)\n", "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.9/dist-packages (from transformers) (1.22.4)\n", "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.9/dist-packages (from transformers) (4.65.0)\n", "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.9/dist-packages (from transformers) (2022.10.31)\n", "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.9/dist-packages (from transformers) (23.0)\n", "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.9/dist-packages (from huggingface-hub<1.0,>=0.11.0->transformers) (4.5.0)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (2022.12.7)\n", "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (1.26.15)\n", "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (2.0.12)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (3.4)\n", "Installing collected packages: tokenizers, huggingface-hub, transformers\n", "Successfully installed huggingface-hub-0.13.4 tokenizers-0.13.3 transformers-4.28.1\n" ] } ], "source": [ "!pip install transformers\n", "import transformers" ] }, { "cell_type": "code", "source": [ "!pip install datasets" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "D8OtthEoJ-jJ", "outputId": "aea9ef62-08cd-40c5-8d26-4590fb87de8b" }, "execution_count": 6, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n", "Collecting datasets\n", " Downloading datasets-2.11.0-py3-none-any.whl (468 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m468.7/468.7 kB\u001b[0m \u001b[31m5.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting dill<0.3.7,>=0.3.0\n", " Downloading dill-0.3.6-py3-none-any.whl (110 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m110.5/110.5 kB\u001b[0m \u001b[31m14.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: fsspec[http]>=2021.11.1 in /usr/local/lib/python3.9/dist-packages (from datasets) (2023.4.0)\n", "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.9/dist-packages (from datasets) (6.0)\n", "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.9/dist-packages (from datasets) (1.22.4)\n", "Requirement already satisfied: packaging in /usr/local/lib/python3.9/dist-packages (from datasets) (23.0)\n", "Requirement already satisfied: huggingface-hub<1.0.0,>=0.11.0 in /usr/local/lib/python3.9/dist-packages (from datasets) (0.13.4)\n", "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.9/dist-packages (from datasets) (4.65.0)\n", "Collecting responses<0.19\n", " Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n", "Collecting xxhash\n", " Downloading xxhash-3.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (212 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m212.2/212.2 kB\u001b[0m \u001b[31m28.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting aiohttp\n", " Downloading aiohttp-3.8.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.0 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.0/1.0 MB\u001b[0m \u001b[31m51.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.9/dist-packages (from datasets) (9.0.0)\n", "Collecting multiprocess\n", " Downloading multiprocess-0.70.14-py39-none-any.whl (132 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m132.9/132.9 kB\u001b[0m \u001b[31m13.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.9/dist-packages (from datasets) (1.5.3)\n", "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.9/dist-packages (from datasets) (2.27.1)\n", "Collecting yarl<2.0,>=1.0\n", " Downloading yarl-1.8.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (264 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m264.6/264.6 kB\u001b[0m \u001b[31m23.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting aiosignal>=1.1.2\n", " Downloading aiosignal-1.3.1-py3-none-any.whl (7.6 kB)\n", "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (22.2.0)\n", "Collecting async-timeout<5.0,>=4.0.0a3\n", " Downloading async_timeout-4.0.2-py3-none-any.whl (5.8 kB)\n", "Collecting multidict<7.0,>=4.5\n", " Downloading multidict-6.0.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (114 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.2/114.2 kB\u001b[0m \u001b[31m15.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (2.0.12)\n", "Collecting frozenlist>=1.1.1\n", " Downloading frozenlist-1.3.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (158 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m158.8/158.8 kB\u001b[0m \u001b[31m7.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.9/dist-packages (from huggingface-hub<1.0.0,>=0.11.0->datasets) (3.11.0)\n", "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.9/dist-packages (from huggingface-hub<1.0.0,>=0.11.0->datasets) (4.5.0)\n", "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests>=2.19.0->datasets) (1.26.15)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.9/dist-packages (from requests>=2.19.0->datasets) (3.4)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.9/dist-packages (from requests>=2.19.0->datasets) (2022.12.7)\n", "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.9/dist-packages (from pandas->datasets) (2.8.2)\n", "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.9/dist-packages (from pandas->datasets) (2022.7.1)\n", "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.9/dist-packages (from python-dateutil>=2.8.1->pandas->datasets) (1.16.0)\n", "Installing collected packages: xxhash, multidict, frozenlist, dill, async-timeout, yarl, responses, multiprocess, aiosignal, aiohttp, datasets\n", "Successfully installed aiohttp-3.8.4 aiosignal-1.3.1 async-timeout-4.0.2 datasets-2.11.0 dill-0.3.6 frozenlist-1.3.3 multidict-6.0.4 multiprocess-0.70.14 responses-0.18.0 xxhash-3.2.0 yarl-1.8.2\n" ] } ] }, { "cell_type": "code", "source": [ "from sklearn.model_selection import train_test_split\n", "from transformers import AutoModelForSequenceClassification, AutoTokenizer, TrainingArguments, Trainer, IntervalStrategy\n", "from datasets import load_metric, Dataset, ClassLabel\n", "\n", "\n", "X_train, X_test, y_train, y_test = train_test_split(\n", " data,\n", " data['target'],\n", " test_size=0.1,\n", " random_state=42\n", ")\n", "\n", "tokenizer = AutoTokenizer.from_pretrained('adalbertojunior/distilbert-portuguese-cased', do_lower_case=False)\n", "\n", "class_label = ClassLabel(names=['pt','pt_br'])\n", "\n", "def tokenize(text):\n", " tokens = tokenizer(text['text'], padding=True, truncation=True, max_length=256, add_special_tokens = True)\n", " tokens['label'] = class_label.str2int(text['target'])\n", " return tokens\n", "\n", "train_ds = Dataset.from_pandas(X_train).map(tokenize, batched=True)\n", "test_ds = Dataset.from_pandas(X_test).map(tokenize, batched=True)\n", "\n", "train_ds.set_format('torch', columns=['input_ids', 'attention_mask', 'label'])\n", "test_ds.set_format('torch', columns=['input_ids', 'attention_mask', 'label'])" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 17, "referenced_widgets": [ "79e7afed6f204ced8012f9853e8ce153", "92e3b0a9ee344022b0bf16a6617239b3", "09fa538b6a05498282c2438a6c854fd1", "3b905f68919049608a3eb3c48da12032", "eee35ecb15c84ee994f9ffa479feeaaf", "c17b6f3b3436490e923835add11f8d74", "4d5a02eb385d470f8b3c68a8db60561c", "7946964d5e8741ba86b2662397ccb891", "5624c10fc93943dda1e0c93c1fc2fbba", "101f117dfb0a441e807757524b857350", "77819fe20f514982b8547abff9361b2d", "8f76ba596bdc427aa868685b151f51d8", "a14ec2755a2a477aa654287af8a463a3", "cb82b162aae342888f353ea2fe507015", "ebb3365ce29e476ab28201e9b825e302", "31e7fabdde2f4129ab38d49b5c5b1d50", "b2fb5695afd04756975281c5a04fbcaf", "6e358ea9a32a486db0c81a5f54a112e9", "647a42b85a8044229d71fd1e7318878b", "f439095945fb413e916059f01f67bc79", "fece699d83854b5e876164da8a3b6f3a", "814444ea270d4739aa8588937801800b" ] }, "id": "Nf7Cew2HLKi6", "outputId": "f107d518-4016-4d4a-d3b8-161f4724bce4" }, "execution_count": 46, "outputs": [ { "output_type": "display_data", "data": { "text/plain": [ "Map: 0%| | 0/18000 [00:0012} {}\".format(p[0], str(tuple(p[1].size())), p[1].requires_grad))\n", "\n", "print('\\n==== First Transformer ====\\n')\n", "\n", "for p in params[5:21]:\n", " print(\"{:<55} {:>12} {}\".format(p[0], str(tuple(p[1].size())), p[1].requires_grad))\n", "\n", "print('\\n==== Output Layer ====\\n')\n", "\n", "for p in params[-4:]:\n", " print(\"{:<55} {:>12} {}\".format(p[0], str(tuple(p[1].size())), p[1].requires_grad))" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "6KDOKbGCXHuQ", "outputId": "5d4cd088-3a0e-4be8-e7a4-427f3ba087c7" }, "execution_count": 33, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "The BERT model has 105 different named parameters.\n", "\n", "==== Embedding Layer ====\n", "\n", "bert.embeddings.word_embeddings.weight (29794, 768) False\n", "bert.embeddings.position_embeddings.weight (512, 768) False\n", "bert.embeddings.token_type_embeddings.weight (2, 768) False\n", "bert.embeddings.LayerNorm.weight (768,) False\n", "bert.embeddings.LayerNorm.bias (768,) False\n", "\n", "==== First Transformer ====\n", "\n", "bert.encoder.layer.0.attention.self.query.weight (768, 768) False\n", "bert.encoder.layer.0.attention.self.query.bias (768,) False\n", "bert.encoder.layer.0.attention.self.key.weight (768, 768) False\n", "bert.encoder.layer.0.attention.self.key.bias (768,) False\n", "bert.encoder.layer.0.attention.self.value.weight (768, 768) False\n", "bert.encoder.layer.0.attention.self.value.bias (768,) False\n", "bert.encoder.layer.0.attention.output.dense.weight (768, 768) False\n", "bert.encoder.layer.0.attention.output.dense.bias (768,) False\n", "bert.encoder.layer.0.attention.output.LayerNorm.weight (768,) False\n", "bert.encoder.layer.0.attention.output.LayerNorm.bias (768,) False\n", "bert.encoder.layer.0.intermediate.dense.weight (3072, 768) False\n", "bert.encoder.layer.0.intermediate.dense.bias (3072,) False\n", "bert.encoder.layer.0.output.dense.weight (768, 3072) False\n", "bert.encoder.layer.0.output.dense.bias (768,) False\n", "bert.encoder.layer.0.output.LayerNorm.weight (768,) False\n", "bert.encoder.layer.0.output.LayerNorm.bias (768,) False\n", "\n", "==== Output Layer ====\n", "\n", "bert.pooler.dense.weight (768, 768) False\n", "bert.pooler.dense.bias (768,) False\n", "classifier.weight (2, 768) True\n", "classifier.bias (2,) True\n" ] } ] }, { "cell_type": "code", "source": [ "for parameter in model.parameters():\n", " parameter.requires_grad = False\n", "for parameter in model.classifier.parameters():\n", " parameter.requires_grad = True" ], "metadata": { "id": "9DOy7eY5Y_px" }, "execution_count": 16, "outputs": [] }, { "cell_type": "code", "source": [ "import numpy as np\n", "\n", "training_args = TrainingArguments(\n", " output_dir=\"./trainer_out\",\n", " learning_rate=2e-4,\n", " per_device_train_batch_size=256,\n", " per_device_eval_batch_size=256,\n", " num_train_epochs=5,\n", " weight_decay=0.01,\n", " evaluation_strategy=IntervalStrategy.EPOCH,\n", ")\n", "\n", "metric = load_metric('accuracy')\n", "\n", "def compute_metrics(eval_pred):\n", " logits, labels = eval_pred\n", " predictions = np.argmax(logits, axis=-1)\n", " return metric.compute(predictions=predictions, references=labels)\n", "\n", "# # transfer learning\n", "# for module in list(model.modules())[:-3]:\n", "# for param in module.parameters():\n", "# param.requires_grad = False\n", "\n", "trainer = Trainer(\n", " model=model,\n", " args=training_args,\n", " train_dataset=train_ds,\n", " eval_dataset=test_ds,\n", " tokenizer=tokenizer,\n", " compute_metrics=compute_metrics,\n", ")\n", "\n", "trainer.train()" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 362 }, "id": "WiAQCmLZXHzy", "outputId": "a9f508ae-3eb8-4b7d-d333-193f2d236796" }, "execution_count": 47, "outputs": [ { "output_type": "stream", "name": "stderr", "text": [ "/usr/local/lib/python3.9/dist-packages/transformers/optimization.py:391: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", " warnings.warn(\n", "You're using a BertTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n" ] }, { "output_type": "display_data", "data": { "text/plain": [ "" ], "text/html": [ "\n", "
\n", " \n", " \n", " [355/355 12:02, Epoch 5/5]\n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
EpochTraining LossValidation LossAccuracy
1No log0.6349150.663500
2No log0.6289110.670500
3No log0.6256010.669000
4No log0.6239090.664000
5No log0.6232800.664500

" ] }, "metadata": {} }, { "output_type": "execute_result", "data": { "text/plain": [ "TrainOutput(global_step=355, training_loss=0.6338723196110255, metrics={'train_runtime': 724.7707, 'train_samples_per_second': 124.177, 'train_steps_per_second': 0.49, 'total_flos': 5961032939520000.0, 'train_loss': 0.6338723196110255, 'epoch': 5.0})" ] }, "metadata": {}, "execution_count": 47 } ] }, { "cell_type": "code", "source": [ "trainer.train()" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 287 }, "id": "ounT1oDOniUI", "outputId": "50d5bd72-eab2-4be8-9f1d-04120b97cce9" }, "execution_count": 53, "outputs": [ { "output_type": "display_data", "data": { "text/plain": [ "" ], "text/html": [ "\n", "

\n", " \n", " \n", " [355/355 12:01, Epoch 5/5]\n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
EpochTraining LossValidation LossAccuracy
1No log0.6232800.664500
2No log0.6232800.664500
3No log0.6232800.664500
4No log0.6232800.664500
5No log0.6232800.664500

" ] }, "metadata": {} }, { "output_type": "execute_result", "data": { "text/plain": [ "TrainOutput(global_step=355, training_loss=0.6281896456866197, metrics={'train_runtime': 723.41, 'train_samples_per_second': 124.411, 'train_steps_per_second': 0.491, 'total_flos': 5961032939520000.0, 'train_loss': 0.6281896456866197, 'epoch': 5.0})" ] }, "metadata": {}, "execution_count": 53 } ] }, { "cell_type": "code", "source": [ "trainer.save_model('./pt_br_model')" ], "metadata": { "id": "Bp7qkdjkXH2j" }, "execution_count": 54, "outputs": [] }, { "cell_type": "code", "source": [ "!tar jcpvf model.tar.bz2 pt_br_model" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "Wmu5JQSue0Hb", "outputId": "54dae7e4-0c29-46ad-e304-04ab5703f1aa" }, "execution_count": 55, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "pt_br_model/\n", "pt_br_model/tokenizer_config.json\n", "pt_br_model/training_args.bin\n", "pt_br_model/vocab.txt\n", "pt_br_model/pytorch_model.bin\n", "pt_br_model/tokenizer.json\n", "pt_br_model/config.json\n", "pt_br_model/special_tokens_map.json\n" ] } ] }, { "cell_type": "code", "source": [ "!ls -al ./pt_br_model/*" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "evbwLlLIi3QK", "outputId": "c1736d23-8d93-4597-ed76-3428d8d75e82" }, "execution_count": 42, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "-rw-r--r-- 1 root root 752 Apr 15 10:51 ./pt_br_model/config.json\n", "-rw-r--r-- 1 root root 265629621 Apr 15 10:51 ./pt_br_model/pytorch_model.bin\n", "-rw-r--r-- 1 root root 125 Apr 15 10:51 ./pt_br_model/special_tokens_map.json\n", "-rw-r--r-- 1 root root 395 Apr 15 10:51 ./pt_br_model/tokenizer_config.json\n", "-rw-r--r-- 1 root root 678043 Apr 15 10:51 ./pt_br_model/tokenizer.json\n", "-rw-r--r-- 1 root root 3579 Apr 15 10:51 ./pt_br_model/training_args.bin\n", "-rw-r--r-- 1 root root 209528 Apr 15 10:51 ./pt_br_model/vocab.txt\n" ] } ] }, { "cell_type": "code", "source": [ "!ls -al" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "y68jWdM5jJJD", "outputId": "f9910474-5a0b-44da-b88d-ecaed8fae13a" }, "execution_count": 44, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "total 317924\n", "drwxr-xr-x 1 root root 4096 Apr 15 10:51 .\n", "drwxr-xr-x 1 root root 4096 Apr 15 10:18 ..\n", "drwxr-xr-x 4 root root 4096 Apr 13 13:29 .config\n", "-rw-r--r-- 1 root root 252222673 Apr 15 10:52 model.tar.bz2\n", "drwxr-xr-x 2 root root 4096 Apr 15 10:51 pt_br_model\n", "-rw-r--r-- 1 root root 41144160 Dec 3 2020 pt_br.txt\n", "-rw-r--r-- 1 root root 32155453 Dec 3 2020 pt.txt\n", "drwxr-xr-x 1 root root 4096 Apr 13 13:30 sample_data\n", "drwxr-xr-x 3 root root 4096 Apr 15 10:25 trainer_out\n" ] } ] }, { "cell_type": "code", "source": [ "!cp model.tar.bz2 /content/drive/MyDrive" ], "metadata": { "id": "_4BB3YnvRHZw" }, "execution_count": 56, "outputs": [] }, { "cell_type": "code", "source": [ "print(transformers.__version__)" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "K-B-mwGryWOw", "outputId": "1742e546-04d3-47d7-e1bf-34da07ef35b8" }, "execution_count": 58, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "4.28.1\n" ] } ] } ] }