peyma.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "NLP_SBU.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyMAVU5BKSHQ2LU/u6kXO4b7"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "0b6cd01d328e4373b7ae17ba56f0d9df": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_7ec5dafc857b4cf49199c7a256aafa6f",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_8a8ef6b02f264fe99dc5e9625f109293",
              "IPY_MODEL_d5ef8d5263d94476a02d82f56cee0777"
            ]
          }
        },
        "7ec5dafc857b4cf49199c7a256aafa6f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8a8ef6b02f264fe99dc5e9625f109293": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a77e6f8775c642fcbd26966d9cfaa0d6",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 497,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 497,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a4e052b4ed8f4218aa65400f9fcc2f50"
          }
        },
        "d5ef8d5263d94476a02d82f56cee0777": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_9d4d529025df4af4aecc046b420352c6",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 497/497 [00:00&lt;00:00, 625.87it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_33c7a31ae143480ea86daa5fd67475cf"
          }
        },
        "a77e6f8775c642fcbd26966d9cfaa0d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a4e052b4ed8f4218aa65400f9fcc2f50": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9d4d529025df4af4aecc046b420352c6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "33c7a31ae143480ea86daa5fd67475cf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5f75b1f287d3448494b63644349df423": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_8c9df43e95464485a6288c24272c54ac",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_cb6b589b5006404d93123fd9f652b460",
              "IPY_MODEL_d8ad8bb93cc141558e553052d5a80296"
            ]
          }
        },
        "8c9df43e95464485a6288c24272c54ac": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "cb6b589b5006404d93123fd9f652b460": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_714085c25e154cc1ae7a6a84f5a38cb7",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 497,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 497,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4d0d54f1bd914fe6a203871a42f76a12"
          }
        },
        "d8ad8bb93cc141558e553052d5a80296": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_fc87f7ff3a304e3097957db89daac3b3",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 497/497 [00:04&lt;00:00, 122.88it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7e28a85d299f473eaaf8efa95c961d8a"
          }
        },
        "714085c25e154cc1ae7a6a84f5a38cb7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4d0d54f1bd914fe6a203871a42f76a12": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fc87f7ff3a304e3097957db89daac3b3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7e28a85d299f473eaaf8efa95c961d8a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0ad844e5452e4983beb889fad7807478": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_e2d92e17cdf344c496a74af401465da8",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_4ae4bd9836704b8d94fb8dad5a51f979",
              "IPY_MODEL_248d46046e224b8190b09ef0e9bfddda"
            ]
          }
        },
        "e2d92e17cdf344c496a74af401465da8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4ae4bd9836704b8d94fb8dad5a51f979": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_0ab3a13cd436423ea37c59643f6743e2",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 56,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 56,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_877429dd89a24637866113653a2a0d09"
          }
        },
        "248d46046e224b8190b09ef0e9bfddda": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_2891ce6b5345458a8da15aee75ba65b4",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 56/56 [00:00&lt;00:00, 624.86it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_445fdd80ff1948648807c2da96209dee"
          }
        },
        "0ab3a13cd436423ea37c59643f6743e2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "877429dd89a24637866113653a2a0d09": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2891ce6b5345458a8da15aee75ba65b4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "445fdd80ff1948648807c2da96209dee": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c4a16915516945828be6c7b23798580b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_00d6855c4130482baed58769f934296a",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_c506a7a8bdc2497d9cfbedf6ee99f45c",
              "IPY_MODEL_29b0ec8b9f6c414287fd599b1b3442fb"
            ]
          }
        },
        "00d6855c4130482baed58769f934296a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c506a7a8bdc2497d9cfbedf6ee99f45c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_b835c0a07fe040e78fbcb4eedb75752b",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 56,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 56,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_56ea82c8a7154fbb9f51ed7db519fd4e"
          }
        },
        "29b0ec8b9f6c414287fd599b1b3442fb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ad8f4112bcb3459fa81a89e791ae5a4e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 56/56 [00:00&lt;00:00, 84.78it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c057f059f1ff42ddadb4e58cacab6ad2"
          }
        },
        "b835c0a07fe040e78fbcb4eedb75752b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "56ea82c8a7154fbb9f51ed7db519fd4e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ad8f4112bcb3459fa81a89e791ae5a4e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c057f059f1ff42ddadb4e58cacab6ad2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "91650448048e45ddaf99506beec72357": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_00c8dadff884489ebb82723aa59532e0",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_3cb961f8510942a0a4ae55f3869b35bd",
              "IPY_MODEL_f97ead07d2da407cafb765bf91ef5464"
            ]
          }
        },
        "00c8dadff884489ebb82723aa59532e0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3cb961f8510942a0a4ae55f3869b35bd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_8947598d58494e029867f37e9641e6bf",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 62,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 62,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_684cad70d9e4422d8b99ca8764916946"
          }
        },
        "f97ead07d2da407cafb765bf91ef5464": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_aef55c1cc78149258fd208d3865dae4c",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 62/62 [00:00&lt;00:00, 840.40it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6629b7d7f6d44c71abe1cd535f617457"
          }
        },
        "8947598d58494e029867f37e9641e6bf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "684cad70d9e4422d8b99ca8764916946": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "aef55c1cc78149258fd208d3865dae4c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6629b7d7f6d44c71abe1cd535f617457": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d03852e7ba5f41fbaa33d2f2e80d64ff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_30b84c19f2334abcada5041f1796e466",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_a333d4f7fddf491ca1cc4b9d5d7f38b2",
              "IPY_MODEL_0280f8f014ba4977912a5cbe0558d205"
            ]
          }
        },
        "30b84c19f2334abcada5041f1796e466": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a333d4f7fddf491ca1cc4b9d5d7f38b2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_9c2ebd8074cf4b19adec8283adeb7194",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 62,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 62,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_eb9ff7f254d74755b21b10df306427db"
          }
        },
        "0280f8f014ba4977912a5cbe0558d205": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_0b79768d7f5e4bbd9aa38705c4e1777f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 62/62 [00:00&lt;00:00, 112.01it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2558587165ae48099effdf0db651e758"
          }
        },
        "9c2ebd8074cf4b19adec8283adeb7194": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "eb9ff7f254d74755b21b10df306427db": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0b79768d7f5e4bbd9aa38705c4e1777f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2558587165ae48099effdf0db651e758": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "60af1ad85eb5463789a275a7e95383ed": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_6390d441b2e74e96aeda5484c702d022",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_2794904657db416a99c0ce3eb6e2ed28",
              "IPY_MODEL_cac9b31c63444573bcf353f9922b9ccf"
            ]
          }
        },
        "6390d441b2e74e96aeda5484c702d022": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2794904657db416a99c0ce3eb6e2ed28": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_5aa00efdefeb4d3c88c95110a2dbd44a",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 62,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 62,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0d75528ae2af4a8484dfffff19a6e741"
          }
        },
        "cac9b31c63444573bcf353f9922b9ccf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_a65da29ee10a4972900a71a872e909fb",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 62/62 [00:00&lt;00:00, 634.20it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1c6cef2963ab40f783f0d88045085b51"
          }
        },
        "5aa00efdefeb4d3c88c95110a2dbd44a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0d75528ae2af4a8484dfffff19a6e741": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a65da29ee10a4972900a71a872e909fb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1c6cef2963ab40f783f0d88045085b51": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e21a583ec788494eb6f4d3316ccbe03a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_58f024ef154345e098bcfab6632a46d9",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_061b2b272dbd4dd4beb1dcb9f8a437d5",
              "IPY_MODEL_b4d86a8c5e2442f0aa076ba36033375f"
            ]
          }
        },
        "58f024ef154345e098bcfab6632a46d9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "061b2b272dbd4dd4beb1dcb9f8a437d5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_5bb88c46b266450c88a529e9556c7403",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 62,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 62,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e679535b1d46492482bef60afb520dfb"
          }
        },
        "b4d86a8c5e2442f0aa076ba36033375f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_e72b3e91e7fc462f86cae220ad208c51",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 62/62 [00:00&lt;00:00, 146.23it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c91466406e794fa09c58e47029b45d8f"
          }
        },
        "5bb88c46b266450c88a529e9556c7403": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e679535b1d46492482bef60afb520dfb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e72b3e91e7fc462f86cae220ad208c51": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c91466406e794fa09c58e47029b45d8f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "dnO7CM5PRJVX",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3d556b9e-f8ef-40c2-daec-3bc2be7a8f60"
      },
      "source": [
        "!nvidia-smi"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sun Jan 24 16:00:44 2021       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 460.32.03    Driver Version: 418.67       CUDA Version: 10.1     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla K80           Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   49C    P8    67W / 149W |      0MiB / 11441MiB |      0%      Default |\n",
            "|                               |                      |                 ERR! |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aQB4nl4VP2-V"
      },
      "source": [
        "# Install required packages\n",
        "\n",
        "!pip install -q transformers\n",
        "!pip install -q hazm\n",
        "!pip install -q clean-text[gpl]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k7osRSezQB5P"
      },
      "source": [
        "# Import required packages\n",
        "\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import classification_report\n",
        "from sklearn.metrics import f1_score\n",
        "from sklearn.utils import shuffle\n",
        "\n",
        "import hazm\n",
        "from cleantext import clean\n",
        "\n",
        "import plotly.express as px\n",
        "import plotly.graph_objects as go\n",
        "\n",
        "from tqdm.notebook import tqdm\n",
        "\n",
        "import os\n",
        "import re\n",
        "import json\n",
        "import copy\n",
        "import collections"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kWuZ5abh1EqI"
      },
      "source": [
        "PERCENT = 0.2 # define how much you want to use the dataset"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6q4lCf6-t4lf",
        "outputId": "f2960d8f-8734-4fef-ecee-262fa7782145"
      },
      "source": [
        "# Load the Drive helper and mount\n",
        "from google.colab import drive\n",
        "drive.flush_and_unmount()\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VLNfx1EutM2S"
      },
      "source": [
        "os.chdir('/content/drive/MyDrive/NLP/')\n",
        "f = open(\"peyma/train.txt\", encoding=\"utf8\")\n",
        "lines = f.readlines()\n",
        "c = 0 # Getting lines count\n",
        "for line in lines:\n",
        "    c += 1\n",
        "c\n",
        "f.close()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yIPdFE8atM2T",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 86
        },
        "outputId": "3e64a11a-a69c-4f61-867f-b307f22ba385"
      },
      "source": [
        "f = open(\"peyma/train.txt\", encoding=\"utf8\")\n",
        "training = ''\n",
        "x = 0\n",
        "while x < int(PERCENT * c): # using PERCENT of the dataset\n",
        "    training += f.readline()\n",
        "    x += 1\n",
        "training[0:1000]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'مدیران|O\\nچهارمین|O\\nجشنواره|O\\nفیلم|O\\nو|O\\nعکس|O\\nفناوری|O\\nو|O\\nصنعتی|O\\nمعرفی|O\\nشدند|O\\n\\nبه|O\\nگزارش|O\\nخبرگزاری|B_ORG\\nمهر|I_ORG\\n،|O\\nسعید|B_PER\\nپورعلی|I_PER\\nمعاون|O\\nفرهنگی|O\\nجهاد|B_ORG\\nدانشگاهی|I_ORG\\nکشور|I_ORG\\nبا|O\\nتاکید|O\\nبر|O\\nاینکه|O\\nسینما|O\\nیکی|O\\nاز|O\\nموثرترین|O\\nروش های|O\\nفرهنگ سازی|O\\nدر|O\\nحوزه|O\\nصنعت|O\\nو|O\\nتولید|O\\nاست|O\\n،|O\\nگفت|O\\n:|O\\nبا|O\\nتوجه|O\\nبه|O\\nگستردگی|O\\nروزافزون|O\\nسینما|O\\nو|O\\nصنعت|O\\nدر|O\\nکشور|O\\n،|O\\nاین|O\\nدو|O\\nنمی توانند|O\\nاز|O\\nیکدیگر|O\\nکناره گیری|O\\nکنند|O\\n.|O\\n\\nچه بسا|O\\nامروز|O\\nدر|O\\nکشورهای|O\\nپیشرفته|O\\n،|O\\nجایگاه|O\\nفرهنگ|O\\nبه|O\\nعنوان|O\\nیکی|O\\nاز|O\\nمحورهای|O\\nصنعت|O\\nو|O\\nاقتصاد|O\\nجای|O\\nخود|O\\nرا|O\\nباز|O\\nکرده|O\\nاست|O\\n.|O\\n\\nوی|O\\nادامه|O\\nداد|O\\n:|O\\nمتاسفانه|O\\nبا|O\\nتوجه|O\\nبه|O\\nساختار|O\\nدولتی|O\\nفرهنگ|O\\nو|O\\nهنر|O\\nدر|O\\nکشور|O\\nما|O\\n،|O\\nاینگونه|O\\nپذیرفته|O\\nشده|O\\nکه|O\\nفرهنگ|O\\nمقوله ای|O\\nهزینه بر|O\\nاست|O\\nکه|O\\nمنابع|O\\nدرآمدی|O\\nنامشخصی|O\\nدارد|O\\n.|O\\n\\nبه|O\\nنظر|O\\nمی رسد|O\\nکه|O\\nهمگامی|O\\nفرهنگ|O\\nو|O\\nاقتصاد|O\\nو|O\\nصنعت|O\\nبتواند|O\\nعلاوه|O\\nبر|O\\nتاثیرات|O\\nشگرفت|O\\nبر|O\\nحوزه|O\\nاقتصاد|O\\nو|O\\nصنعت|O\\n،|O\\nفرهنگ|O\\nرا|O\\n'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gqYfUTEhKXUw"
      },
      "source": [
        "### Tokenizing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VACZZmn7tM2U"
      },
      "source": [
        "data = training.split('.') # Separate by sentence\n",
        "training = []\n",
        "for i in range(len(data)):\n",
        "    data[i] = data[i].rstrip()\n",
        "    training.append(data[i].split('\\n'))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fp_4eBJMtM2V"
      },
      "source": [
        "token = [] \n",
        "label = []\n",
        "for i in range(len(training)):\n",
        "    for j in range(len(training[i])):\n",
        "        temp = training[i][j].split('|')\n",
        "        if len(temp) == 2:\n",
        "            token.append(temp[0])\n",
        "            label.append(temp[1])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2ReKKSBAtM2V",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "17a9f465-c766-4009-c464-03dde302237a"
      },
      "source": [
        "token[13],label[13]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "('خبرگزاری', 'B_ORG')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lrNO0AqRtM2W",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "44464110-daf7-4ec6-feed-4aefeb42c58d"
      },
      "source": [
        "len(token)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "48741"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "56ynCl4GtM2W",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2831eabe-383e-41ae-91f4-59c8dc0ddfc7"
      },
      "source": [
        "data = pd.DataFrame({'token':token, 'label':label})\n",
        "data.head()\n",
        "labels = list(sorted(data['label'].unique()))\n",
        "labels"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['B_DAT',\n",
              " 'B_LOC',\n",
              " 'B_MON',\n",
              " 'B_ORG',\n",
              " 'B_PCT',\n",
              " 'B_PER',\n",
              " 'B_TIM',\n",
              " 'I_DAT',\n",
              " 'I_LOC',\n",
              " 'I_MON',\n",
              " 'I_ORG',\n",
              " 'I_PCT',\n",
              " 'I_PER',\n",
              " 'I_TIM',\n",
              " 'O']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sqBN--hYtM2X",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9e4756a4-babe-4707-8ea7-31eda0993921"
      },
      "source": [
        "data.info()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 48741 entries, 0 to 48740\n",
            "Data columns (total 2 columns):\n",
            " #   Column  Non-Null Count  Dtype \n",
            "---  ------  --------------  ----- \n",
            " 0   token   48741 non-null  object\n",
            " 1   label   48741 non-null  object\n",
            "dtypes: object(2)\n",
            "memory usage: 761.7+ KB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bD0Eh4O9tM2X",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 669
        },
        "outputId": "0f3da504-0244-4683-dbaa-410ff217126e"
      },
      "source": [
        "data.head(20)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>token</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>مدیران</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>چهارمین</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>جشنواره</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>فیلم</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>و</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>عکس</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>فناوری</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>و</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>صنعتی</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>معرفی</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>شدند</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>به</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>گزارش</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>خبرگزاری</td>\n",
              "      <td>B_ORG</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>مهر</td>\n",
              "      <td>I_ORG</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>،</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>سعید</td>\n",
              "      <td>B_PER</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>پورعلی</td>\n",
              "      <td>I_PER</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>معاون</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>فرهنگی</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "       token  label\n",
              "0     مدیران      O\n",
              "1    چهارمین      O\n",
              "2    جشنواره      O\n",
              "3       فیلم      O\n",
              "4          و      O\n",
              "5        عکس      O\n",
              "6     فناوری      O\n",
              "7          و      O\n",
              "8      صنعتی      O\n",
              "9      معرفی      O\n",
              "10      شدند      O\n",
              "11        به      O\n",
              "12     گزارش      O\n",
              "13  خبرگزاری  B_ORG\n",
              "14       مهر  I_ORG\n",
              "15         ،      O\n",
              "16      سعید  B_PER\n",
              "17    پورعلی  I_PER\n",
              "18     معاون      O\n",
              "19    فرهنگی      O"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hebxxLZrtM2X",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 542
        },
        "outputId": "f8b12ac0-e773-4076-8b73-fd539af1971f"
      },
      "source": [
        "fig = go.Figure()\n",
        "\n",
        "groupby_label = data.groupby('label')['label'].count()\n",
        "\n",
        "fig.add_trace(go.Bar(\n",
        "    x=list(sorted(groupby_label.index)),\n",
        "    y=groupby_label.tolist(),\n",
        "    text=groupby_label.tolist(),\n",
        "    textposition='auto'\n",
        "))\n",
        "\n",
        "fig.update_layout(\n",
        "    title_text='Distribution of label within comments [DATA]',\n",
        "    xaxis_title_text='Label',\n",
        "    yaxis_title_text='Frequency',\n",
        "    bargap=0.2,\n",
        "    bargroupgap=0.2)\n",
        "\n",
        "fig.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>\n",
              "            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>\n",
              "                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script src=\"https://cdn.plot.ly/plotly-latest.min.js\"></script>    \n",
              "            <div id=\"54de3031-f375-4130-8085-e6405f958d88\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>\n",
              "            <script type=\"text/javascript\">\n",
              "                \n",
              "                    window.PLOTLYENV=window.PLOTLYENV || {};\n",
              "                    \n",
              "                if (document.getElementById(\"54de3031-f375-4130-8085-e6405f958d88\")) {\n",
              "                    Plotly.newPlot(\n",
              "                        '54de3031-f375-4130-8085-e6405f958d88',\n",
              "                        [{\"text\": [\"273\", \"923\", \"58\", \"1095\", \"46\", \"824\", \"41\", \"340\", \"319\", \"153\", \"1888\", \"56\", \"600\", \"57\", \"42068\"], \"textposition\": \"auto\", \"type\": \"bar\", \"x\": [\"B_DAT\", \"B_LOC\", \"B_MON\", \"B_ORG\", \"B_PCT\", \"B_PER\", \"B_TIM\", \"I_DAT\", \"I_LOC\", \"I_MON\", \"I_ORG\", \"I_PCT\", \"I_PER\", \"I_TIM\", \"O\"], \"y\": [273, 923, 58, 1095, 46, 824, 41, 340, 319, 153, 1888, 56, 600, 57, 42068]}],\n",
              "                        {\"bargap\": 0.2, \"bargroupgap\": 0.2, \"template\": {\"data\": {\"bar\": [{\"error_x\": {\"color\": \"#2a3f5f\"}, \"error_y\": {\"color\": \"#2a3f5f\"}, \"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"bar\"}], \"barpolar\": [{\"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"baxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"choropleth\"}], \"contour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"contour\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"contourcarpet\"}], \"heatmap\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmap\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmapgl\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"histogram\"}], \"histogram2d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2d\"}], \"histogram2dcontour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2dcontour\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"parcoords\"}], \"pie\": [{\"automargin\": true, \"type\": \"pie\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter\"}], \"scatter3d\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattermapbox\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolar\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolargl\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"#EBF0F8\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"#C8D4E3\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowcolor\": \"#2a3f5f\", \"arrowhead\": 0, \"arrowwidth\": 1}, \"coloraxis\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"colorscale\": {\"diverging\": [[0, \"#8e0152\"], [0.1, \"#c51b7d\"], [0.2, \"#de77ae\"], [0.3, \"#f1b6da\"], [0.4, \"#fde0ef\"], [0.5, \"#f7f7f7\"], [0.6, \"#e6f5d0\"], [0.7, \"#b8e186\"], [0.8, \"#7fbc41\"], [0.9, \"#4d9221\"], [1, \"#276419\"]], \"sequential\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"sequentialminus\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]]}, \"colorway\": [\"#636efa\", \"#EF553B\", \"#00cc96\", \"#ab63fa\", \"#FFA15A\", \"#19d3f3\", \"#FF6692\", \"#B6E880\", \"#FF97FF\", \"#FECB52\"], \"font\": {\"color\": \"#2a3f5f\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"#E5ECF6\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hoverlabel\": {\"align\": \"left\"}, \"hovermode\": \"closest\", \"mapbox\": {\"style\": \"light\"}, \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"#E5ECF6\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"line\": {\"color\": \"#2a3f5f\"}}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"title\": {\"x\": 0.05}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"title\": {\"standoff\": 15}, \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"title\": {\"standoff\": 15}, \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}}}, \"title\": {\"text\": \"Distribution of label within comments [DATA]\"}, \"xaxis\": {\"title\": {\"text\": \"Label\"}}, \"yaxis\": {\"title\": {\"text\": \"Frequency\"}}},\n",
              "                        {\"responsive\": true}\n",
              "                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('54de3031-f375-4130-8085-e6405f958d88');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })\n",
              "                };\n",
              "                \n",
              "            </script>\n",
              "        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CxKn2faBLdwH"
      },
      "source": [
        "### Equalizing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I4CU601wtM2Y",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9dae9106-fa2c-4368-ce68-55bd93caee57"
      },
      "source": [
        "size = len(data.groupby('label')['label'].count()) # How many labels?\n",
        "sep = []\n",
        "temp = ''\n",
        "for i in range(size):\n",
        "    temp = data.groupby('label')['label'].count().index[i] # Separate based on label\n",
        "    sep.append(data[data['label'] == temp])\n",
        "\n",
        "cutting_point = min(data.groupby('label')['label'].count().tolist())\n",
        "print(cutting_point)\n",
        "\n",
        "for i in range(size):\n",
        "    sep[i] = sep[i].sample(n=cutting_point).reset_index(drop=True)\n",
        "\n",
        "new_data = pd.concat(sep)\n",
        "new_data = new_data.sample(frac=1).reset_index(drop=True)\n",
        "new_data.info()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "41\n",
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 615 entries, 0 to 614\n",
            "Data columns (total 2 columns):\n",
            " #   Column  Non-Null Count  Dtype \n",
            "---  ------  --------------  ----- \n",
            " 0   token   615 non-null    object\n",
            " 1   label   615 non-null    object\n",
            "dtypes: object(2)\n",
            "memory usage: 9.7+ KB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F0ugfze2tM2Y",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 542
        },
        "outputId": "d84fbce8-c190-4609-bacc-1ec91dbae043"
      },
      "source": [
        "fig = go.Figure()\n",
        "\n",
        "groupby_label = new_data.groupby('label')['label'].count()\n",
        "\n",
        "fig.add_trace(go.Bar(\n",
        "    x=list(sorted(groupby_label.index)),\n",
        "    y=groupby_label.tolist(),\n",
        "    text=groupby_label.tolist(),\n",
        "    textposition='auto'\n",
        "))\n",
        "\n",
        "fig.update_layout(\n",
        "    title_text='Distribution of label within comments [NEW DATA]',\n",
        "    xaxis_title_text='Label',\n",
        "    yaxis_title_text='Frequency',\n",
        "    bargap=0.2,\n",
        "    bargroupgap=0.2)\n",
        "\n",
        "fig.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>\n",
              "            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>\n",
              "                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script src=\"https://cdn.plot.ly/plotly-latest.min.js\"></script>    \n",
              "            <div id=\"45fbb7bd-891e-497a-9c1e-d529cc132c7b\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>\n",
              "            <script type=\"text/javascript\">\n",
              "                \n",
              "                    window.PLOTLYENV=window.PLOTLYENV || {};\n",
              "                    \n",
              "                if (document.getElementById(\"45fbb7bd-891e-497a-9c1e-d529cc132c7b\")) {\n",
              "                    Plotly.newPlot(\n",
              "                        '45fbb7bd-891e-497a-9c1e-d529cc132c7b',\n",
              "                        [{\"text\": [\"41\", \"41\", \"41\", \"41\", \"41\", \"41\", \"41\", \"41\", \"41\", \"41\", \"41\", \"41\", \"41\", \"41\", \"41\"], \"textposition\": \"auto\", \"type\": \"bar\", \"x\": [\"B_DAT\", \"B_LOC\", \"B_MON\", \"B_ORG\", \"B_PCT\", \"B_PER\", \"B_TIM\", \"I_DAT\", \"I_LOC\", \"I_MON\", \"I_ORG\", \"I_PCT\", \"I_PER\", \"I_TIM\", \"O\"], \"y\": [41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41]}],\n",
              "                        {\"bargap\": 0.2, \"bargroupgap\": 0.2, \"template\": {\"data\": {\"bar\": [{\"error_x\": {\"color\": \"#2a3f5f\"}, \"error_y\": {\"color\": \"#2a3f5f\"}, \"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"bar\"}], \"barpolar\": [{\"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"baxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"choropleth\"}], \"contour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"contour\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"contourcarpet\"}], \"heatmap\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmap\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmapgl\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"histogram\"}], \"histogram2d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2d\"}], \"histogram2dcontour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2dcontour\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"parcoords\"}], \"pie\": [{\"automargin\": true, \"type\": \"pie\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter\"}], \"scatter3d\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattermapbox\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolar\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolargl\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"#EBF0F8\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"#C8D4E3\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowcolor\": \"#2a3f5f\", \"arrowhead\": 0, \"arrowwidth\": 1}, \"coloraxis\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"colorscale\": {\"diverging\": [[0, \"#8e0152\"], [0.1, \"#c51b7d\"], [0.2, \"#de77ae\"], [0.3, \"#f1b6da\"], [0.4, \"#fde0ef\"], [0.5, \"#f7f7f7\"], [0.6, \"#e6f5d0\"], [0.7, \"#b8e186\"], [0.8, \"#7fbc41\"], [0.9, \"#4d9221\"], [1, \"#276419\"]], \"sequential\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"sequentialminus\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]]}, \"colorway\": [\"#636efa\", \"#EF553B\", \"#00cc96\", \"#ab63fa\", \"#FFA15A\", \"#19d3f3\", \"#FF6692\", \"#B6E880\", \"#FF97FF\", \"#FECB52\"], \"font\": {\"color\": \"#2a3f5f\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"#E5ECF6\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hoverlabel\": {\"align\": \"left\"}, \"hovermode\": \"closest\", \"mapbox\": {\"style\": \"light\"}, \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"#E5ECF6\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"line\": {\"color\": \"#2a3f5f\"}}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"title\": {\"x\": 0.05}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"title\": {\"standoff\": 15}, \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"title\": {\"standoff\": 15}, \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}}}, \"title\": {\"text\": \"Distribution of label within comments [NEW DATA]\"}, \"xaxis\": {\"title\": {\"text\": \"Label\"}}, \"yaxis\": {\"title\": {\"text\": \"Frequency\"}}},\n",
              "                        {\"responsive\": true}\n",
              "                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('45fbb7bd-891e-497a-9c1e-d529cc132c7b');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })\n",
              "                };\n",
              "                \n",
              "            </script>\n",
              "        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DM4_zSSGtM2Z",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "1537b494-ef6f-45ef-e155-49bd14d4bc78"
      },
      "source": [
        "new_data.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>token</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>توسلی</td>\n",
              "      <td>I_PER</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>تا</td>\n",
              "      <td>I_DAT</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>اداره</td>\n",
              "      <td>B_ORG</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>میلیون</td>\n",
              "      <td>I_MON</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>جب</td>\n",
              "      <td>I_LOC</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    token  label\n",
              "0   توسلی  I_PER\n",
              "1      تا  I_DAT\n",
              "2   اداره  B_ORG\n",
              "3  میلیون  I_MON\n",
              "4      جب  I_LOC"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "De7iAF9uMa8_"
      },
      "source": [
        "### Train Test Validation Split"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aTCrt3PztM2Z",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8199b226-e2a7-40a8-aeaf-2b9d3ab5d570"
      },
      "source": [
        "new_data['label_id'] = new_data['label'].apply(lambda t: labels.index(t))\n",
        "\n",
        "train, test = train_test_split(new_data, test_size=0.1, random_state=1, stratify=new_data['label'])\n",
        "train, valid = train_test_split(train, test_size=0.1, random_state=1, stratify=train['label'])\n",
        "\n",
        "train = train.reset_index(drop=True)\n",
        "valid = valid.reset_index(drop=True)\n",
        "test = test.reset_index(drop=True)\n",
        "\n",
        "x_train, y_train = train['token'].values.tolist(), train['label_id'].values.tolist()\n",
        "x_valid, y_valid = valid['token'].values.tolist(), valid['label_id'].values.tolist()\n",
        "x_test, y_test = test['token'].values.tolist(), test['label_id'].values.tolist()\n",
        "\n",
        "print(train.shape)\n",
        "print(valid.shape)\n",
        "print(test.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(497, 3)\n",
            "(56, 3)\n",
            "(62, 3)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "G9eY5tJrwLUZ"
      },
      "source": [
        "## TensorFlow"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BuiOuFkxv_to"
      },
      "source": [
        "from transformers import BertConfig, BertTokenizer\n",
        "from transformers import TFBertModel, TFBertForSequenceClassification\n",
        "from transformers import glue_convert_examples_to_features\n",
        "\n",
        "import tensorflow as tf"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "X7Ro-8_pw_uw"
      },
      "source": [
        "### Configuration"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zx6k3XBxw_Nw"
      },
      "source": [
        "# general config\n",
        "MAX_LEN = 128\n",
        "TRAIN_BATCH_SIZE = 16\n",
        "VALID_BATCH_SIZE = 16\n",
        "TEST_BATCH_SIZE = 16\n",
        "\n",
        "EPOCHS = 3\n",
        "EEVERY_EPOCH = 1000\n",
        "LEARNING_RATE = 2e-5\n",
        "CLIP = 0.0\n",
        "\n",
        "MODEL_NAME_OR_PATH = 'HooshvareLab/bert-fa-base-uncased'\n",
        "OUTPUT_PATH = '/content/bert-fa-base-uncased-sentiment-taaghceh/pytorch_model.bin'\n",
        "\n",
        "os.makedirs(os.path.dirname(OUTPUT_PATH), exist_ok=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X8uZ7728RmCw",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8d254c93-c738-484e-9520-84894488087d"
      },
      "source": [
        "label2id = {label: i for i, label in enumerate(labels)}\n",
        "id2label = {v: k for k, v in label2id.items()}\n",
        "\n",
        "print(f'label2id: {label2id}')\n",
        "print(f'id2label: {id2label}')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "label2id: {'B_DAT': 0, 'B_LOC': 1, 'B_MON': 2, 'B_ORG': 3, 'B_PCT': 4, 'B_PER': 5, 'B_TIM': 6, 'I_DAT': 7, 'I_LOC': 8, 'I_MON': 9, 'I_ORG': 10, 'I_PCT': 11, 'I_PER': 12, 'I_TIM': 13, 'O': 14}\n",
            "id2label: {0: 'B_DAT', 1: 'B_LOC', 2: 'B_MON', 3: 'B_ORG', 4: 'B_PCT', 5: 'B_PER', 6: 'B_TIM', 7: 'I_DAT', 8: 'I_LOC', 9: 'I_MON', 10: 'I_ORG', 11: 'I_PCT', 12: 'I_PER', 13: 'I_TIM', 14: 'O'}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j4iFFI0lR5Nr",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c381af1a-e81d-48d2-cc25-0864cddb8941"
      },
      "source": [
        "tokenizer = BertTokenizer.from_pretrained(MODEL_NAME_OR_PATH)\n",
        "config = BertConfig.from_pretrained(\n",
        "    MODEL_NAME_OR_PATH, **{\n",
        "        'label2id': label2id,\n",
        "        'id2label': id2label,\n",
        "    })\n",
        "\n",
        "print(config.to_json_string())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{\n",
            "  \"architectures\": [\n",
            "    \"BertForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"gradient_checkpointing\": false,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"B_DAT\",\n",
            "    \"1\": \"B_LOC\",\n",
            "    \"2\": \"B_MON\",\n",
            "    \"3\": \"B_ORG\",\n",
            "    \"4\": \"B_PCT\",\n",
            "    \"5\": \"B_PER\",\n",
            "    \"6\": \"B_TIM\",\n",
            "    \"7\": \"I_DAT\",\n",
            "    \"8\": \"I_LOC\",\n",
            "    \"9\": \"I_MON\",\n",
            "    \"10\": \"I_ORG\",\n",
            "    \"11\": \"I_PCT\",\n",
            "    \"12\": \"I_PER\",\n",
            "    \"13\": \"I_TIM\",\n",
            "    \"14\": \"O\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"B_DAT\": 0,\n",
            "    \"B_LOC\": 1,\n",
            "    \"B_MON\": 2,\n",
            "    \"B_ORG\": 3,\n",
            "    \"B_PCT\": 4,\n",
            "    \"B_PER\": 5,\n",
            "    \"B_TIM\": 6,\n",
            "    \"I_DAT\": 7,\n",
            "    \"I_LOC\": 8,\n",
            "    \"I_MON\": 9,\n",
            "    \"I_ORG\": 10,\n",
            "    \"I_PCT\": 11,\n",
            "    \"I_PER\": 12,\n",
            "    \"I_TIM\": 13,\n",
            "    \"O\": 14\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-12,\n",
            "  \"max_position_embeddings\": 512,\n",
            "  \"model_type\": \"bert\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.2.2\",\n",
            "  \"type_vocab_size\": 2,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 100000\n",
            "}\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ppvfKLczSFAG"
      },
      "source": [
        "### Input Embeddings / Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5_YqB7S7R9AK"
      },
      "source": [
        "class InputExample:\n",
        "    \"\"\" A single example for simple sequence classification. \"\"\"\n",
        "\n",
        "    def __init__(self, guid, text_a, text_b=None, label=None):\n",
        "        \"\"\" Constructs a InputExample. \"\"\"\n",
        "        self.guid = guid\n",
        "        self.text_a = text_a\n",
        "        self.text_b = text_b\n",
        "        self.label = label\n",
        "\n",
        "\n",
        "def make_examples(tokenizer, x, y=None, maxlen=128, output_mode=\"classification\", is_tf_dataset=True):\n",
        "    examples = []\n",
        "    y = y if isinstance(y, list) or isinstance(y, np.ndarray) else [None] * len(x)\n",
        "\n",
        "    for i, (_x, _y) in tqdm(enumerate(zip(x, y)), position=0, total=len(x)):\n",
        "        guid = \"%s\" % i\n",
        "        label = int(_y)\n",
        "        \n",
        "        if isinstance(_x, str):\n",
        "            text_a = _x\n",
        "            text_b = None\n",
        "        else:\n",
        "            assert len(_x) == 2\n",
        "            text_a = _x[0]\n",
        "            text_b = _x[1]\n",
        "        \n",
        "        examples.append(InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label))\n",
        "    \n",
        "    features = glue_convert_examples_to_features(\n",
        "        examples, \n",
        "        tokenizer, \n",
        "        maxlen, \n",
        "        output_mode=output_mode, \n",
        "        label_list=list(np.unique(y)))\n",
        "\n",
        "    all_input_ids = []\n",
        "    all_attention_masks = []\n",
        "    all_token_type_ids = []\n",
        "    all_labels = []\n",
        "\n",
        "    for f in tqdm(features, position=0, total=len(examples)):\n",
        "        if is_tf_dataset:\n",
        "            all_input_ids.append(tf.constant(f.input_ids))\n",
        "            all_attention_masks.append(tf.constant(f.attention_mask))\n",
        "            all_token_type_ids.append(tf.constant(f.token_type_ids))\n",
        "            all_labels.append(tf.constant(f.label))\n",
        "        else:\n",
        "            all_input_ids.append(f.input_ids)\n",
        "            all_attention_masks.append(f.attention_mask)\n",
        "            all_token_type_ids.append(f.token_type_ids)\n",
        "            all_labels.append(f.label)\n",
        "\n",
        "    if is_tf_dataset:\n",
        "        dataset = tf.data.Dataset.from_tensor_slices(({\n",
        "            'input_ids': all_input_ids,\n",
        "            'attention_mask': all_attention_masks,\n",
        "            'token_type_ids': all_token_type_ids\n",
        "        }, all_labels))\n",
        "\n",
        "        return dataset, features\n",
        "    \n",
        "    xdata = [np.array(all_input_ids), np.array(all_attention_masks), np.array(all_token_type_ids)]\n",
        "    ydata = all_labels\n",
        "\n",
        "    return [xdata, ydata], features"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I7x5C4XPSLFz",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 497,
          "referenced_widgets": [
            "0b6cd01d328e4373b7ae17ba56f0d9df",
            "7ec5dafc857b4cf49199c7a256aafa6f",
            "8a8ef6b02f264fe99dc5e9625f109293",
            "d5ef8d5263d94476a02d82f56cee0777",
            "a77e6f8775c642fcbd26966d9cfaa0d6",
            "a4e052b4ed8f4218aa65400f9fcc2f50",
            "9d4d529025df4af4aecc046b420352c6",
            "33c7a31ae143480ea86daa5fd67475cf",
            "5f75b1f287d3448494b63644349df423",
            "8c9df43e95464485a6288c24272c54ac",
            "cb6b589b5006404d93123fd9f652b460",
            "d8ad8bb93cc141558e553052d5a80296",
            "714085c25e154cc1ae7a6a84f5a38cb7",
            "4d0d54f1bd914fe6a203871a42f76a12",
            "fc87f7ff3a304e3097957db89daac3b3",
            "7e28a85d299f473eaaf8efa95c961d8a",
            "0ad844e5452e4983beb889fad7807478",
            "e2d92e17cdf344c496a74af401465da8",
            "4ae4bd9836704b8d94fb8dad5a51f979",
            "248d46046e224b8190b09ef0e9bfddda",
            "0ab3a13cd436423ea37c59643f6743e2",
            "877429dd89a24637866113653a2a0d09",
            "2891ce6b5345458a8da15aee75ba65b4",
            "445fdd80ff1948648807c2da96209dee",
            "c4a16915516945828be6c7b23798580b",
            "00d6855c4130482baed58769f934296a",
            "c506a7a8bdc2497d9cfbedf6ee99f45c",
            "29b0ec8b9f6c414287fd599b1b3442fb",
            "b835c0a07fe040e78fbcb4eedb75752b",
            "56ea82c8a7154fbb9f51ed7db519fd4e",
            "ad8f4112bcb3459fa81a89e791ae5a4e",
            "c057f059f1ff42ddadb4e58cacab6ad2",
            "91650448048e45ddaf99506beec72357",
            "00c8dadff884489ebb82723aa59532e0",
            "3cb961f8510942a0a4ae55f3869b35bd",
            "f97ead07d2da407cafb765bf91ef5464",
            "8947598d58494e029867f37e9641e6bf",
            "684cad70d9e4422d8b99ca8764916946",
            "aef55c1cc78149258fd208d3865dae4c",
            "6629b7d7f6d44c71abe1cd535f617457",
            "d03852e7ba5f41fbaa33d2f2e80d64ff",
            "30b84c19f2334abcada5041f1796e466",
            "a333d4f7fddf491ca1cc4b9d5d7f38b2",
            "0280f8f014ba4977912a5cbe0558d205",
            "9c2ebd8074cf4b19adec8283adeb7194",
            "eb9ff7f254d74755b21b10df306427db",
            "0b79768d7f5e4bbd9aa38705c4e1777f",
            "2558587165ae48099effdf0db651e758",
            "60af1ad85eb5463789a275a7e95383ed",
            "6390d441b2e74e96aeda5484c702d022",
            "2794904657db416a99c0ce3eb6e2ed28",
            "cac9b31c63444573bcf353f9922b9ccf",
            "5aa00efdefeb4d3c88c95110a2dbd44a",
            "0d75528ae2af4a8484dfffff19a6e741",
            "a65da29ee10a4972900a71a872e909fb",
            "1c6cef2963ab40f783f0d88045085b51",
            "e21a583ec788494eb6f4d3316ccbe03a",
            "58f024ef154345e098bcfab6632a46d9",
            "061b2b272dbd4dd4beb1dcb9f8a437d5",
            "b4d86a8c5e2442f0aa076ba36033375f",
            "5bb88c46b266450c88a529e9556c7403",
            "e679535b1d46492482bef60afb520dfb",
            "e72b3e91e7fc462f86cae220ad208c51",
            "c91466406e794fa09c58e47029b45d8f"
          ]
        },
        "outputId": "8ed36d0d-d571-485f-e018-7ac1ed24a4f5"
      },
      "source": [
        "train_dataset_base, train_examples = make_examples(tokenizer, x_train, y_train, maxlen=128)\n",
        "valid_dataset_base, valid_examples = make_examples(tokenizer, x_valid, y_valid, maxlen=128)\n",
        "\n",
        "test_dataset_base, test_examples = make_examples(tokenizer, x_test, y_test, maxlen=128)\n",
        "[xtest, ytest], test_examples = make_examples(tokenizer, x_test, y_test, maxlen=128, is_tf_dataset=False)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0b6cd01d328e4373b7ae17ba56f0d9df",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=497.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/transformers/data/processors/glue.py:67: FutureWarning:\n",
            "\n",
            "This function will be removed from the library soon, preprocessing should be handled with the 🤗 Datasets library. You can have a look at this example script for pointers: https://github.com/huggingface/transformers/blob/master/examples/text-classification/run_glue.py\n",
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "5f75b1f287d3448494b63644349df423",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=497.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0ad844e5452e4983beb889fad7807478",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=56.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c4a16915516945828be6c7b23798580b",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=56.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "91650448048e45ddaf99506beec72357",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=62.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d03852e7ba5f41fbaa33d2f2e80d64ff",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=62.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "60af1ad85eb5463789a275a7e95383ed",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=62.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e21a583ec788494eb6f4d3316ccbe03a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=62.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eOkiA4c6SeKP",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ff53db01-a332-4aef-e690-4c4d37606789"
      },
      "source": [
        "for value in train_dataset_base.take(1):\n",
        "    print(f'     input_ids: {value[0][\"input_ids\"]}')\n",
        "    print(f'attention_mask: {value[0][\"attention_mask\"]}')\n",
        "    print(f'token_type_ids: {value[0][\"token_type_ids\"]}')\n",
        "    print(f'        target: {value[1]}')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "     input_ids: [   2 5252    4    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0]\n",
            "attention_mask: [1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]\n",
            "token_type_ids: [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]\n",
            "        target: 9\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sGPW0b6PSsMN"
      },
      "source": [
        "def get_training_dataset(dataset, batch_size):\n",
        "    dataset = dataset.repeat()\n",
        "    dataset = dataset.shuffle(2048)\n",
        "    dataset = dataset.batch(batch_size)\n",
        "\n",
        "    return dataset\n",
        "\n",
        "def get_validation_dataset(dataset, batch_size):\n",
        "    dataset = dataset.batch(batch_size)\n",
        "\n",
        "    return dataset"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3JC5rd6YVEHc",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "db1a5189-6791-4b5a-b19f-9d7e5be647d0"
      },
      "source": [
        "train_dataset = get_training_dataset(train_dataset_base, TRAIN_BATCH_SIZE)\n",
        "valid_dataset = get_training_dataset(valid_dataset_base, VALID_BATCH_SIZE)\n",
        "\n",
        "train_steps = len(train_examples) // TRAIN_BATCH_SIZE\n",
        "valid_steps = len(valid_examples) // VALID_BATCH_SIZE\n",
        "\n",
        "train_steps, valid_steps"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(31, 3)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WwsARx2QVLjK"
      },
      "source": [
        "def build_model(model_name, config, learning_rate=3e-5):\n",
        "    model = TFBertForSequenceClassification.from_pretrained(model_name, config=config)\n",
        "\n",
        "    optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)\n",
        "    loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
        "    metric = tf.keras.metrics.SparseCategoricalAccuracy('accuracy')\n",
        "    model.compile(optimizer=optimizer, loss=loss, metrics=[metric])\n",
        "\n",
        "    return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8aFLw-MQVT_V",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b10864fb-a0bd-4f67-d90d-caa7faa1f3c6"
      },
      "source": [
        "model = build_model(MODEL_NAME_OR_PATH, config, learning_rate=LEARNING_RATE)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "All model checkpoint layers were used when initializing TFBertForSequenceClassification.\n",
            "\n",
            "Some layers of TFBertForSequenceClassification were not initialized from the model checkpoint at HooshvareLab/bert-fa-base-uncased and are newly initialized: ['classifier']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i3qOIz1WVZLw",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "26e87e00-8f93-4a5a-cb8a-b3b302d10f50"
      },
      "source": [
        "%%time\n",
        "\n",
        "r = model.fit(\n",
        "    train_dataset,\n",
        "    validation_data=valid_dataset,\n",
        "    steps_per_epoch=train_steps,\n",
        "    validation_steps=valid_steps,\n",
        "    epochs=EPOCHS,\n",
        "    verbose=1)\n",
        "\n",
        "final_accuracy = r.history['val_accuracy']\n",
        "print('FINAL ACCURACY MEAN: ', np.mean(final_accuracy))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/3\n",
            "WARNING:tensorflow:AutoGraph could not transform <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fe95d452590>> and will run it as-is.\n",
            "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
            "Cause: <cyfunction Socket.send at 0x7fe97ae00d90> is not a module, class, method, function, traceback, frame, or code object\n",
            "To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n",
            "WARNING: AutoGraph could not transform <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fe95d452590>> and will run it as-is.\n",
            "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
            "Cause: <cyfunction Socket.send at 0x7fe97ae00d90> is not a module, class, method, function, traceback, frame, or code object\n",
            "To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:AutoGraph could not transform <function wrap at 0x7fe97878c9d8> and will run it as-is.\n",
            "Cause: while/else statement not yet supported\n",
            "To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING: AutoGraph could not transform <function wrap at 0x7fe97878c9d8> and will run it as-is.\n",
            "Cause: while/else statement not yet supported\n",
            "To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n",
            "The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "31/31 [==============================] - ETA: 0s - loss: 2.6470 - accuracy: 0.1489"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n",
            "The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r31/31 [==============================] - 65s 861ms/step - loss: 2.6418 - accuracy: 0.1527 - val_loss: 2.1224 - val_accuracy: 0.5000\n",
            "Epoch 2/3\n",
            "31/31 [==============================] - 25s 804ms/step - loss: 2.0507 - accuracy: 0.4902 - val_loss: 1.7088 - val_accuracy: 0.5000\n",
            "Epoch 3/3\n",
            "31/31 [==============================] - 25s 809ms/step - loss: 1.6152 - accuracy: 0.5213 - val_loss: 1.5626 - val_accuracy: 0.4167\n",
            "FINAL ACCURACY MEAN:  0.4722222189108531\n",
            "CPU times: user 39.1 s, sys: 7.78 s, total: 46.9 s\n",
            "Wall time: 1min 54s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JwIpJ1PIiOf6"
      },
      "source": [
        "# save the model\n",
        "\n",
        "model.save_pretrained(os.path.dirname(OUTPUT_PATH))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VsgURdVH9V5w",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7f1faa9d-1c03-4415-a1e7-492be7a471ed"
      },
      "source": [
        "ev = model.evaluate(test_dataset_base.batch(TEST_BATCH_SIZE))\n",
        "print()\n",
        "print(f'Evaluation: {ev}')\n",
        "print()\n",
        "\n",
        "predictions = model.predict(xtest)\n",
        "ypred = predictions[0].argmax(axis=-1).tolist()\n",
        "\n",
        "print()\n",
        "print(classification_report(ytest, ypred, target_names=labels))\n",
        "print()\n",
        "\n",
        "print(f'F1: {f1_score(ytest, ypred, average=\"weighted\")}')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "4/4 [==============================] - 1s 244ms/step - loss: 1.4182 - accuracy: 0.6129\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n",
            "The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Evaluation: [1.4181872606277466, 0.6129032373428345]\n",
            "\n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "       B_DAT       0.50      0.25      0.33         4\n",
            "       B_LOC       0.67      0.50      0.57         4\n",
            "       B_MON       1.00      0.25      0.40         4\n",
            "       B_ORG       0.80      1.00      0.89         4\n",
            "       B_PCT       0.36      1.00      0.53         4\n",
            "       B_PER       0.42      1.00      0.59         5\n",
            "       B_TIM       0.80      1.00      0.89         4\n",
            "       I_DAT       0.50      0.25      0.33         4\n",
            "       I_LOC       0.50      0.25      0.33         4\n",
            "       I_MON       1.00      0.25      0.40         4\n",
            "       I_ORG       1.00      0.25      0.40         4\n",
            "       I_PCT       1.00      1.00      1.00         5\n",
            "       I_PER       1.00      0.25      0.40         4\n",
            "       I_TIM       0.60      0.75      0.67         4\n",
            "           O       0.67      1.00      0.80         4\n",
            "\n",
            "    accuracy                           0.61        62\n",
            "   macro avg       0.72      0.60      0.57        62\n",
            "weighted avg       0.72      0.61      0.58        62\n",
            "\n",
            "\n",
            "F1: 0.5764193849582845\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dH41uoY80z8s"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}