classifications_pytorch_kobert.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.6-final"
    },
    "colab": {
      "name": "classifications_pytorch_kobert.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "37273726c2c540adbc69d94681e52716": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_6b8e6ac7069e4e3680818d6a6fca369a",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_fd6322019c9446d0bc5cad83a13485f8",
              "IPY_MODEL_b5cfb318ed5b46c8bfcc6ef6ba2f6d03"
            ]
          }
        },
        "6b8e6ac7069e4e3680818d6a6fca369a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fd6322019c9446d0bc5cad83a13485f8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_d6457456d147438bb99e2a2d0a33c9dc",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 997,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 997,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c5c981cfa3c846b39a6933713e0d92c1"
          }
        },
        "b5cfb318ed5b46c8bfcc6ef6ba2f6d03": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_3731f2b21fbf40cbb8353310875d0e25",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 997/997 [37:36&lt;00:00,  2.26s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_26d02244923e4f5dbac60f9aa0534b55"
          }
        },
        "d6457456d147438bb99e2a2d0a33c9dc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c5c981cfa3c846b39a6933713e0d92c1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3731f2b21fbf40cbb8353310875d0e25": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "26d02244923e4f5dbac60f9aa0534b55": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "91c365688b534e308fba924044004c3a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_23ab2e3adc404895bb8d9486acd1c601",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_541b2d68e0164851b191296e7342547f",
              "IPY_MODEL_e8e91c4725f04d6d887b0a48562fe9b9"
            ]
          }
        },
        "23ab2e3adc404895bb8d9486acd1c601": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "541b2d68e0164851b191296e7342547f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_c4132a2ade684e2caf8c9835b5527e50",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 111,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 111,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6613d724aec04af5bf8d4f3a8ab22580"
          }
        },
        "e8e91c4725f04d6d887b0a48562fe9b9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_98e7e070241e4da690217274753a7f05",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 111/111 [02:54&lt;00:00,  1.57s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_89404a5b5b424b4ca1240fbc924a6374"
          }
        },
        "c4132a2ade684e2caf8c9835b5527e50": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6613d724aec04af5bf8d4f3a8ab22580": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "98e7e070241e4da690217274753a7f05": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "89404a5b5b424b4ca1240fbc924a6374": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2e9a0c282d004eb2a69cc56a1fb8317d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_547ad924bc174ca9a36818429cf3db70",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_c67b6b2eefd44efba6bae758cb304a7e",
              "IPY_MODEL_c96c743f9eb74677aadc05d3fbca86d6"
            ]
          }
        },
        "547ad924bc174ca9a36818429cf3db70": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c67b6b2eefd44efba6bae758cb304a7e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_72e30c0ecc9245c8a4e033a2f716551b",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 997,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 997,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_cd812267ca7a49b885faef329aa487b3"
          }
        },
        "c96c743f9eb74677aadc05d3fbca86d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ee1214fcedc846b99b53044eb6298184",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 997/997 [30:09&lt;00:00,  1.81s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b39e02f3c9c04c8fa1514644bf37712b"
          }
        },
        "72e30c0ecc9245c8a4e033a2f716551b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "cd812267ca7a49b885faef329aa487b3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ee1214fcedc846b99b53044eb6298184": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b39e02f3c9c04c8fa1514644bf37712b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8643c9223aa74ccbaa079c3f4bfbb465": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_1ec887fb0bbe4f6db9afa5dcee0e22b2",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_ea7a9693bf15499595ce9d668f681c0e",
              "IPY_MODEL_cd3fa77709a94a6a8dcda4a06e8c899d"
            ]
          }
        },
        "1ec887fb0bbe4f6db9afa5dcee0e22b2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ea7a9693bf15499595ce9d668f681c0e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_738653bdc4c4402098b658d32d953fb2",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 111,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 111,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_429a05d2817148859e60529e212dc5b5"
          }
        },
        "cd3fa77709a94a6a8dcda4a06e8c899d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_5e4bd9d92fbc4fcc821f797167f5c9ef",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 111/111 [00:16&lt;00:00,  6.72it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4f45ac8485a4443ebf863c4c3108fcc9"
          }
        },
        "738653bdc4c4402098b658d32d953fb2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "429a05d2817148859e60529e212dc5b5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5e4bd9d92fbc4fcc821f797167f5c9ef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4f45ac8485a4443ebf863c4c3108fcc9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5a814af62c7b4db7bb89248c127e6373": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_036f94737db24cbdacaa04ec5d04d273",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_30088310d7684025bb95c8908ee4ae0a",
              "IPY_MODEL_28f606c42a2f4b769e42cb9bc48a5483"
            ]
          }
        },
        "036f94737db24cbdacaa04ec5d04d273": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "30088310d7684025bb95c8908ee4ae0a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_7f8915e11f0e43908afef4d5a5ba0bcf",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 997,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 997,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_82cfdc8faa684e99a8a1186739da1b76"
          }
        },
        "28f606c42a2f4b769e42cb9bc48a5483": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_8deb4a78f43f4117a6eb468a01fa5b25",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 997/997 [22:42&lt;00:00,  1.37s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c98a3bf72bde496b85e416af2f4500e8"
          }
        },
        "7f8915e11f0e43908afef4d5a5ba0bcf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "82cfdc8faa684e99a8a1186739da1b76": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8deb4a78f43f4117a6eb468a01fa5b25": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c98a3bf72bde496b85e416af2f4500e8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e771282a21bd4fdba6b4f2fd6d0c96fb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f8f80242160c438faddc258c1ff7e16d",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_74b6e567008646f3b14642fbc4c03839",
              "IPY_MODEL_61d8e7669ca742deb84ac77a2d1e37d4"
            ]
          }
        },
        "f8f80242160c438faddc258c1ff7e16d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "74b6e567008646f3b14642fbc4c03839": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_cab30027f24f46b58d12dd5ce7334ad7",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 111,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 111,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5fd6b876a4f146809f3559ea4988c29f"
          }
        },
        "61d8e7669ca742deb84ac77a2d1e37d4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c6b51920376b455aa4346d7d9dae04ea",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 111/111 [00:16&lt;00:00,  6.71it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_33c4d1e736724f0c91b54f049378ae4c"
          }
        },
        "cab30027f24f46b58d12dd5ce7334ad7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5fd6b876a4f146809f3559ea4988c29f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c6b51920376b455aa4346d7d9dae04ea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "33c4d1e736724f0c91b54f049378ae4c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "871e69fe7e914871b7aac956bbe4e3f2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_e557870d2d1740acb7c76fd1512c0a30",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_12943cf412714fd5a88625f41e5351ea",
              "IPY_MODEL_5810235bfbca4614a22710719e1e6605"
            ]
          }
        },
        "e557870d2d1740acb7c76fd1512c0a30": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "12943cf412714fd5a88625f41e5351ea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_7a8fb506bbe54025ab300de587d2a14a",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 997,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 997,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f00bb34b1dc24766b143104999fa481d"
          }
        },
        "5810235bfbca4614a22710719e1e6605": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_5f235371046e4425850a01b75ea514ac",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 997/997 [15:16&lt;00:00,  1.09it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d9ef3277c33d461ca24fd27756629991"
          }
        },
        "7a8fb506bbe54025ab300de587d2a14a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f00bb34b1dc24766b143104999fa481d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5f235371046e4425850a01b75ea514ac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d9ef3277c33d461ca24fd27756629991": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "869419228f6c49d88f42e1210e90714b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_c91a55201ab54c5998419b0879059863",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_8848ca57f5534430b8c450a3f360fa57",
              "IPY_MODEL_698446c69aa543d0828318cccd8319e9"
            ]
          }
        },
        "c91a55201ab54c5998419b0879059863": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8848ca57f5534430b8c450a3f360fa57": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_d5f62bd3712246fb8b62c23e98b06dc3",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 111,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 111,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f69347cb38644a4980864301d4903bdf"
          }
        },
        "698446c69aa543d0828318cccd8319e9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_70af0b2612a74ac1ac66d9007dd9bfb0",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 111/111 [01:03&lt;00:00,  1.75it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e17dbd91820e4b798db91a8a93732c78"
          }
        },
        "d5f62bd3712246fb8b62c23e98b06dc3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f69347cb38644a4980864301d4903bdf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "70af0b2612a74ac1ac66d9007dd9bfb0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e17dbd91820e4b798db91a8a93732c78": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d8e72056e50f4a148474e552dd164e9c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_4b3483ddbff74cee84a6c1f5d3a0dc19",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_91d0286035874549b19e0000fa67f54c",
              "IPY_MODEL_27dd53bd9bad4dbfaf97ce2cc4fbf48f"
            ]
          }
        },
        "4b3483ddbff74cee84a6c1f5d3a0dc19": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "91d0286035874549b19e0000fa67f54c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_c22c7287d2304520bde41bdada857547",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 997,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 997,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_12efff84808b453084eed45a356ec18b"
          }
        },
        "27dd53bd9bad4dbfaf97ce2cc4fbf48f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_736893cb0fca48eeb084012da5f45abb",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 997/997 [07:51&lt;00:00,  2.12it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d62b3a51e9b4413a91e49329106d40b3"
          }
        },
        "c22c7287d2304520bde41bdada857547": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "12efff84808b453084eed45a356ec18b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "736893cb0fca48eeb084012da5f45abb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d62b3a51e9b4413a91e49329106d40b3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a3bea9954fad4153a8964375e1222aaf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b0404c750ebe4c15b3bd5a64a2442bc7",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_f73f2bbb37cd454fa55aff8e0f0abbcc",
              "IPY_MODEL_53e0927e092a45428645c5176df2c43b"
            ]
          }
        },
        "b0404c750ebe4c15b3bd5a64a2442bc7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f73f2bbb37cd454fa55aff8e0f0abbcc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_03993a8bd6b644e494b4da70dda3703a",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 111,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 111,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_88cba362d20047cea8467c6e6c785918"
          }
        },
        "53e0927e092a45428645c5176df2c43b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_db6027345f2b49239d8705faa8b42941",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 111/111 [00:24&lt;00:00,  4.55it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6a34d4b8d53f43f1b66c36095412a4c7"
          }
        },
        "03993a8bd6b644e494b4da70dda3703a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "88cba362d20047cea8467c6e6c785918": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "db6027345f2b49239d8705faa8b42941": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6a34d4b8d53f43f1b66c36095412a4c7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6325cce63d2e42c19f01bd9499283c31": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_1eb3e744b0cc4496b0a0aabd9b5ff690",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_9ab2f78e4ad64342b290c1f51efa0fec",
              "IPY_MODEL_762ddee4a7b7499685768528082a750e"
            ]
          }
        },
        "1eb3e744b0cc4496b0a0aabd9b5ff690": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9ab2f78e4ad64342b290c1f51efa0fec": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_8c5e8c211e5d48f9a3398803b321ae67",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 2228,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 2228,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_90852e0f4d4b491eabc76bea9ba28483"
          }
        },
        "762ddee4a7b7499685768528082a750e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_51112e88eb664bfaa499b21095b58c0f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 2228/2228 [05:19&lt;00:00,  6.98it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c77f42aef0734e1cbfb81f49c203f6bb"
          }
        },
        "8c5e8c211e5d48f9a3398803b321ae67": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "90852e0f4d4b491eabc76bea9ba28483": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "51112e88eb664bfaa499b21095b58c0f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c77f42aef0734e1cbfb81f49c203f6bb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/skaurl/NH-bigdata/blob/main/classifications_pytorch_kobert.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FyJmIgM7f9tU"
      },
      "source": [
        "# Setting"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Bh0ACNrluvjX"
      },
      "source": [
        "구글 드라이브 연걸"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1QXUWdZGfYsW",
        "outputId": "6a37e503-88dd-4a78-c965-9e32c698ae96"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/gdrive')"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6cZW0-nBuyY7"
      },
      "source": [
        "라이브러리 다운로드"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JfTJcmOV9GWY",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1e0eecbe-e569-45c6-9898-02c2b99210f3"
      },
      "source": [
        "!pip install mxnet\n",
        "!pip install gluonnlp pandas tqdm\n",
        "!pip install sentencepiece\n",
        "!pip install transformers==3\n",
        "!pip install torch\n",
        "!pip install git+https://git@github.com/SKTBrain/KoBERT.git@master"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting mxnet\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/29/bb/54cbabe428351c06d10903c658878d29ee7026efbe45133fd133598d6eb6/mxnet-1.7.0.post1-py2.py3-none-manylinux2014_x86_64.whl (55.0MB)\n",
            "\u001b[K     |████████████████████████████████| 55.0MB 103kB/s \n",
            "\u001b[?25hRequirement already satisfied: requests<3,>=2.20.0 in /usr/local/lib/python3.6/dist-packages (from mxnet) (2.23.0)\n",
            "Requirement already satisfied: numpy<2.0.0,>1.16.0 in /usr/local/lib/python3.6/dist-packages (from mxnet) (1.19.4)\n",
            "Collecting graphviz<0.9.0,>=0.8.1\n",
            "  Downloading https://files.pythonhosted.org/packages/53/39/4ab213673844e0c004bed8a0781a0721a3f6bb23eb8854ee75c236428892/graphviz-0.8.4-py2.py3-none-any.whl\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.20.0->mxnet) (2020.12.5)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.20.0->mxnet) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.20.0->mxnet) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.20.0->mxnet) (2.10)\n",
            "Installing collected packages: graphviz, mxnet\n",
            "  Found existing installation: graphviz 0.10.1\n",
            "    Uninstalling graphviz-0.10.1:\n",
            "      Successfully uninstalled graphviz-0.10.1\n",
            "Successfully installed graphviz-0.8.4 mxnet-1.7.0.post1\n",
            "Collecting gluonnlp\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/9c/81/a238e47ccba0d7a61dcef4e0b4a7fd4473cb86bed3d84dd4fe28d45a0905/gluonnlp-0.10.0.tar.gz (344kB)\n",
            "\u001b[K     |████████████████████████████████| 348kB 12.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.6/dist-packages (1.1.5)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (4.41.1)\n",
            "Requirement already satisfied: numpy>=1.16.0 in /usr/local/lib/python3.6/dist-packages (from gluonnlp) (1.19.4)\n",
            "Requirement already satisfied: cython in /usr/local/lib/python3.6/dist-packages (from gluonnlp) (0.29.21)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from gluonnlp) (20.8)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas) (2018.9)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.6/dist-packages (from pandas) (2.8.1)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->gluonnlp) (2.4.7)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.6/dist-packages (from python-dateutil>=2.7.3->pandas) (1.15.0)\n",
            "Building wheels for collected packages: gluonnlp\n",
            "  Building wheel for gluonnlp (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for gluonnlp: filename=gluonnlp-0.10.0-cp36-cp36m-linux_x86_64.whl size=588515 sha256=27b42950c388e2941fd5d5bdd8be5a0332ed6410e178bae30d7fc0af91887338\n",
            "  Stored in directory: /root/.cache/pip/wheels/37/65/52/63032864a0f31a08b9a88569f803b5bafac8abd207fd7f7534\n",
            "Successfully built gluonnlp\n",
            "Installing collected packages: gluonnlp\n",
            "Successfully installed gluonnlp-0.10.0\n",
            "Collecting sentencepiece\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/e5/2d/6d4ca4bef9a67070fa1cac508606328329152b1df10bdf31fb6e4e727894/sentencepiece-0.1.94-cp36-cp36m-manylinux2014_x86_64.whl (1.1MB)\n",
            "\u001b[K     |████████████████████████████████| 1.1MB 14.0MB/s \n",
            "\u001b[?25hInstalling collected packages: sentencepiece\n",
            "Successfully installed sentencepiece-0.1.94\n",
            "Collecting transformers==3\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/9c/35/1c3f6e62d81f5f0daff1384e6d5e6c5758682a8357ebc765ece2b9def62b/transformers-3.0.0-py3-none-any.whl (754kB)\n",
            "\u001b[K     |████████████████████████████████| 757kB 13.9MB/s \n",
            "\u001b[?25hRequirement already satisfied: sentencepiece in /usr/local/lib/python3.6/dist-packages (from transformers==3) (0.1.94)\n",
            "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers==3) (0.8)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers==3) (2.23.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers==3) (20.8)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers==3) (1.19.4)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers==3) (4.41.1)\n",
            "Collecting tokenizers==0.8.0-rc4\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/e8/bd/e5abec46af977c8a1375c1dca7cb1e5b3ec392ef279067af7f6bc50491a0/tokenizers-0.8.0rc4-cp36-cp36m-manylinux1_x86_64.whl (3.0MB)\n",
            "\u001b[K     |████████████████████████████████| 3.0MB 31.9MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers==3) (3.0.12)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers==3) (2019.12.20)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
            "\u001b[K     |████████████████████████████████| 890kB 52.7MB/s \n",
            "\u001b[?25hRequirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==3) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==3) (2020.12.5)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==3) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==3) (3.0.4)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers==3) (2.4.7)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==3) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==3) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==3) (1.0.0)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893261 sha256=cae33e268bc1a6999260e8cdc306f11ee97dd0e603e53304f84365cb2ea73c06\n",
            "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: tokenizers, sacremoses, transformers\n",
            "Successfully installed sacremoses-0.0.43 tokenizers-0.8.0rc4 transformers-3.0.0\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.6/dist-packages (1.7.0+cu101)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from torch) (1.19.4)\n",
            "Requirement already satisfied: dataclasses in /usr/local/lib/python3.6/dist-packages (from torch) (0.8)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from torch) (0.16.0)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.6/dist-packages (from torch) (3.7.4.3)\n",
            "Collecting git+https://****@github.com/SKTBrain/KoBERT.git@master\n",
            "  Cloning https://****@github.com/SKTBrain/KoBERT.git (to revision master) to /tmp/pip-req-build-nrdunj79\n",
            "  Running command git clone -q 'https://****@github.com/SKTBrain/KoBERT.git' /tmp/pip-req-build-nrdunj79\n",
            "Building wheels for collected packages: kobert\n",
            "  Building wheel for kobert (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for kobert: filename=kobert-0.1.2-cp36-none-any.whl size=12734 sha256=79a02e6bf0cd4e53ba9dee5fadab3178313d5bbcee281ef35add7adeba17bf5d\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-8p9no5my/wheels/a2/b0/41/435ee4e918f91918be41529283c5ff86cd010f02e7525aecf3\n",
            "Successfully built kobert\n",
            "Installing collected packages: kobert\n",
            "Successfully installed kobert-0.1.2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lPfyLLNvu57S"
      },
      "source": [
        "라이브러리 불러오기"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r7B-9uae9GWf"
      },
      "source": [
        "import torch\n",
        "from torch import nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "from torch.utils.data import Dataset, DataLoader\n",
        "import gluonnlp as nlp\n",
        "import numpy as np\n",
        "from tqdm import tqdm, tqdm_notebook\n",
        "from kobert.utils import get_tokenizer\n",
        "from kobert.pytorch_kobert import get_pytorch_kobert_model\n",
        "from transformers import AdamW\n",
        "from transformers.optimization import get_cosine_schedule_with_warmup"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n4B0Qu5Su7xr"
      },
      "source": [
        "GPU 설정"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HtL-6mKP9GWf"
      },
      "source": [
        "device = torch.device(\"cuda:0\")"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XHR2pofhu_oy"
      },
      "source": [
        "kobert 불러오기"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RCfTfCDn9GWg",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ff52ee70-8c61-4689-da0b-62a31e264733"
      },
      "source": [
        "bertmodel, vocab = get_pytorch_kobert_model()"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[██████████████████████████████████████████████████]\n",
            "[██████████████████████████████████████████████████]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A205wkO7gHx_"
      },
      "source": [
        "# Preprocessing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0AbNw4eWgvjN"
      },
      "source": [
        "import pandas as pd\n",
        "import re\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "news_dataset_path = r\"/gdrive/My Drive/한양대학교/NH투자증권 빅데이터 경진대회/코난/1.Data/open/news_train.csv\"\n",
        "\n",
        "news_dataset = pd.read_csv(news_dataset_path)\n",
        "\n",
        "#카테고리 오류 데이터 제거\n",
        "news_dataset = news_dataset[news_dataset[\"content\"]!=\"데이터에서 찾은 AI 속보\"]\n",
        "news_dataset = news_dataset[news_dataset[\"content\"]!=\"씨에스윈드, 193억 원 규모 윈드타워 공급계약 체결\"]\n",
        "news_dataset = news_dataset[news_dataset[\"content\"]!=\"씨에스윈드, 1분기 영업이익 158억 원전년비 16.1%\"]\n",
        "\n",
        "#title + content로 데이터 구축\n",
        "news_dataset[\"content\"] = news_dataset[\"title\"] + \" \" + news_dataset[\"content\"]\n",
        "\n",
        "del news_dataset[\"n_id\"]\n",
        "del news_dataset[\"date\"]\n",
        "del news_dataset[\"title\"]\n",
        "del news_dataset[\"ord\"]\n",
        "\n",
        "#중복 데이터 제거\n",
        "news_dataset = news_dataset.drop_duplicates()\n",
        "\n",
        "news_dataset[\"content\"] = news_dataset[\"content\"].apply(lambda x : ' '.join(x.strip().split()))\n",
        "\n",
        "#Train : Test = 9 : 1\n",
        "train, test = train_test_split(news_dataset, test_size=0.1, random_state=42)\n",
        "\n",
        "train.to_csv(\"/gdrive/My Drive/한양대학교/NH투자증권 빅데이터 경진대회/코난/1.Data/news_train_train.txt\", sep='\\t', index=False)\n",
        "test.to_csv(\"/gdrive/My Drive/한양대학교/NH투자증권 빅데이터 경진대회/코난/1.Data/news_train_test.txt\", sep='\\t', index=False)"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ARBEnPXh9GWg"
      },
      "source": [
        "dataset_train = nlp.data.TSVDataset(\"/gdrive/My Drive/한양대학교/NH투자증권 빅데이터 경진대회/코난/1.Data/news_train_train.txt\", field_indices=[0,1], num_discard_samples=1)\n",
        "dataset_test = nlp.data.TSVDataset(\"/gdrive/My Drive/한양대학교/NH투자증권 빅데이터 경진대회/코난/1.Data/news_train_test.txt\", field_indices=[0,1], num_discard_samples=1)"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nmPDgEq79GWh",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "cefee4c8-4d85-495b-b170-399307618727"
      },
      "source": [
        "tokenizer = get_tokenizer()\n",
        "tok = nlp.data.BERTSPTokenizer(tokenizer, vocab, lower=False)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "using cached model\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UqoUHWPx9GWh"
      },
      "source": [
        "class BERTDataset(Dataset):\n",
        "    def __init__(self, dataset, sent_idx, label_idx, bert_tokenizer, max_len, pad, pair):\n",
        "        transform = nlp.data.BERTSentenceTransform(bert_tokenizer, max_seq_length=max_len, pad=pad, pair=pair)\n",
        "        self.sentences = [transform([i[sent_idx]]) for i in dataset]\n",
        "        self.labels = [np.int32(i[label_idx]) for i in dataset]\n",
        "\n",
        "    def __getitem__(self, i):\n",
        "        return (self.sentences[i] + (self.labels[i], ))\n",
        "\n",
        "    def __len__(self):\n",
        "        return (len(self.labels))"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qv6dJaCQ9GWh"
      },
      "source": [
        "max_len = 128\n",
        "batch_size = 64\n",
        "warmup_ratio = 0.01\n",
        "num_epochs = 5\n",
        "max_grad_norm = 1\n",
        "log_interval = 200\n",
        "learning_rate =  5e-5"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n16h0bMJ9GWh"
      },
      "source": [
        "data_train = BERTDataset(dataset_train, 0, 1, tok, max_len, True, False)\n",
        "data_test = BERTDataset(dataset_test, 0, 1, tok, max_len, True, False)"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "duIuijhx9GWi"
      },
      "source": [
        "train_dataloader = torch.utils.data.DataLoader(data_train, batch_size=batch_size, num_workers=5)\n",
        "test_dataloader = torch.utils.data.DataLoader(data_test, batch_size=batch_size, num_workers=5)"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tyN1iC9bgSsp"
      },
      "source": [
        "# Classification"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4L5Vd-QU9GWi"
      },
      "source": [
        "class BERTClassifier(nn.Module):\n",
        "    def __init__(self,\n",
        "                 bert,\n",
        "                 hidden_size = 768,\n",
        "                 num_classes=2,\n",
        "                 dr_rate=None,\n",
        "                 params=None):\n",
        "        super(BERTClassifier, self).__init__()\n",
        "        self.bert = bert\n",
        "        self.dr_rate = dr_rate \n",
        "        self.classifier = nn.Linear(hidden_size , num_classes)\n",
        "        if dr_rate:\n",
        "            self.dropout = nn.Dropout(p=dr_rate)\n",
        "    \n",
        "    def gen_attention_mask(self, token_ids, valid_length):\n",
        "        attention_mask = torch.zeros_like(token_ids)\n",
        "        for i, v in enumerate(valid_length):\n",
        "            attention_mask[i][:v] = 1\n",
        "        return attention_mask.float()\n",
        "\n",
        "    def forward(self, token_ids, valid_length, segment_ids):\n",
        "        attention_mask = self.gen_attention_mask(token_ids, valid_length)\n",
        "        _, pooler = self.bert(input_ids = token_ids, token_type_ids = segment_ids.long(), attention_mask = attention_mask.float().to(token_ids.device))\n",
        "        if self.dr_rate:\n",
        "            out = self.dropout(pooler)\n",
        "        return self.classifier(out)"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c8wwjQlj9GWi"
      },
      "source": [
        "model = BERTClassifier(bertmodel, dr_rate=0.5).to(device)"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3dunfIkC9GWj"
      },
      "source": [
        "no_decay = ['bias', 'LayerNorm.weight']\n",
        "optimizer_grouped_parameters = [\n",
        "    {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},\n",
        "    {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}\n",
        "]"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-hJK1Y7g9GWj"
      },
      "source": [
        "optimizer = AdamW(optimizer_grouped_parameters, lr=learning_rate)\n",
        "loss_fn = nn.CrossEntropyLoss()"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Jgz5gjIH9GWk"
      },
      "source": [
        "t_total = len(train_dataloader) * num_epochs\n",
        "warmup_step = int(t_total * warmup_ratio)"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VznGre-F9GWk"
      },
      "source": [
        "scheduler = get_cosine_schedule_with_warmup(optimizer, num_warmup_steps=warmup_step, num_training_steps=t_total)"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bQuF_O_K9GWk"
      },
      "source": [
        "def calc_accuracy(X,Y):\n",
        "    max_vals, max_indices = torch.max(X, 1)\n",
        "    train_acc = (max_indices == Y).sum().data.cpu().numpy()/max_indices.size()[0]\n",
        "    return train_acc"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aN5JhOKb9GWk",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "37273726c2c540adbc69d94681e52716",
            "6b8e6ac7069e4e3680818d6a6fca369a",
            "fd6322019c9446d0bc5cad83a13485f8",
            "b5cfb318ed5b46c8bfcc6ef6ba2f6d03",
            "d6457456d147438bb99e2a2d0a33c9dc",
            "c5c981cfa3c846b39a6933713e0d92c1",
            "3731f2b21fbf40cbb8353310875d0e25",
            "26d02244923e4f5dbac60f9aa0534b55",
            "91c365688b534e308fba924044004c3a",
            "23ab2e3adc404895bb8d9486acd1c601",
            "541b2d68e0164851b191296e7342547f",
            "e8e91c4725f04d6d887b0a48562fe9b9",
            "c4132a2ade684e2caf8c9835b5527e50",
            "6613d724aec04af5bf8d4f3a8ab22580",
            "98e7e070241e4da690217274753a7f05",
            "89404a5b5b424b4ca1240fbc924a6374",
            "2e9a0c282d004eb2a69cc56a1fb8317d",
            "547ad924bc174ca9a36818429cf3db70",
            "c67b6b2eefd44efba6bae758cb304a7e",
            "c96c743f9eb74677aadc05d3fbca86d6",
            "72e30c0ecc9245c8a4e033a2f716551b",
            "cd812267ca7a49b885faef329aa487b3",
            "ee1214fcedc846b99b53044eb6298184",
            "b39e02f3c9c04c8fa1514644bf37712b",
            "8643c9223aa74ccbaa079c3f4bfbb465",
            "1ec887fb0bbe4f6db9afa5dcee0e22b2",
            "ea7a9693bf15499595ce9d668f681c0e",
            "cd3fa77709a94a6a8dcda4a06e8c899d",
            "738653bdc4c4402098b658d32d953fb2",
            "429a05d2817148859e60529e212dc5b5",
            "5e4bd9d92fbc4fcc821f797167f5c9ef",
            "4f45ac8485a4443ebf863c4c3108fcc9",
            "5a814af62c7b4db7bb89248c127e6373",
            "036f94737db24cbdacaa04ec5d04d273",
            "30088310d7684025bb95c8908ee4ae0a",
            "28f606c42a2f4b769e42cb9bc48a5483",
            "7f8915e11f0e43908afef4d5a5ba0bcf",
            "82cfdc8faa684e99a8a1186739da1b76",
            "8deb4a78f43f4117a6eb468a01fa5b25",
            "c98a3bf72bde496b85e416af2f4500e8",
            "e771282a21bd4fdba6b4f2fd6d0c96fb",
            "f8f80242160c438faddc258c1ff7e16d",
            "74b6e567008646f3b14642fbc4c03839",
            "61d8e7669ca742deb84ac77a2d1e37d4",
            "cab30027f24f46b58d12dd5ce7334ad7",
            "5fd6b876a4f146809f3559ea4988c29f",
            "c6b51920376b455aa4346d7d9dae04ea",
            "33c4d1e736724f0c91b54f049378ae4c",
            "871e69fe7e914871b7aac956bbe4e3f2",
            "e557870d2d1740acb7c76fd1512c0a30",
            "12943cf412714fd5a88625f41e5351ea",
            "5810235bfbca4614a22710719e1e6605",
            "7a8fb506bbe54025ab300de587d2a14a",
            "f00bb34b1dc24766b143104999fa481d",
            "5f235371046e4425850a01b75ea514ac",
            "d9ef3277c33d461ca24fd27756629991",
            "869419228f6c49d88f42e1210e90714b",
            "c91a55201ab54c5998419b0879059863",
            "8848ca57f5534430b8c450a3f360fa57",
            "698446c69aa543d0828318cccd8319e9",
            "d5f62bd3712246fb8b62c23e98b06dc3",
            "f69347cb38644a4980864301d4903bdf",
            "70af0b2612a74ac1ac66d9007dd9bfb0",
            "e17dbd91820e4b798db91a8a93732c78",
            "d8e72056e50f4a148474e552dd164e9c",
            "4b3483ddbff74cee84a6c1f5d3a0dc19",
            "91d0286035874549b19e0000fa67f54c",
            "27dd53bd9bad4dbfaf97ce2cc4fbf48f",
            "c22c7287d2304520bde41bdada857547",
            "12efff84808b453084eed45a356ec18b",
            "736893cb0fca48eeb084012da5f45abb",
            "d62b3a51e9b4413a91e49329106d40b3",
            "a3bea9954fad4153a8964375e1222aaf",
            "b0404c750ebe4c15b3bd5a64a2442bc7",
            "f73f2bbb37cd454fa55aff8e0f0abbcc",
            "53e0927e092a45428645c5176df2c43b",
            "03993a8bd6b644e494b4da70dda3703a",
            "88cba362d20047cea8467c6e6c785918",
            "db6027345f2b49239d8705faa8b42941",
            "6a34d4b8d53f43f1b66c36095412a4c7"
          ]
        },
        "outputId": "12593108-019e-4a4e-8741-5b9e4177012f"
      },
      "source": [
        "for e in range(num_epochs):\n",
        "    train_acc = 0.0\n",
        "    test_acc = 0.0\n",
        "    model.train()\n",
        "    for batch_id, (token_ids, valid_length, segment_ids, label) in enumerate(tqdm_notebook(train_dataloader)):\n",
        "        optimizer.zero_grad()\n",
        "        token_ids = token_ids.long().to(device)\n",
        "        segment_ids = segment_ids.long().to(device)\n",
        "        valid_length= valid_length\n",
        "        label = label.long().to(device)\n",
        "        out = model(token_ids, valid_length, segment_ids)\n",
        "        loss = loss_fn(out, label)\n",
        "        loss.backward()\n",
        "        torch.nn.utils.clip_grad_norm_(model.parameters(), max_grad_norm)\n",
        "        optimizer.step()\n",
        "        scheduler.step()\n",
        "        train_acc += calc_accuracy(out, label)\n",
        "        if batch_id % log_interval == 0:\n",
        "            print(\"epoch {} batch id {} loss {} train acc {}\".format(e+1, batch_id+1, loss.data.cpu().numpy(), train_acc / (batch_id+1)))\n",
        "    print(\"epoch {} train acc {}\".format(e+1, train_acc / (batch_id+1)))\n",
        "    model.eval()\n",
        "    for batch_id, (token_ids, valid_length, segment_ids, label) in enumerate(tqdm_notebook(test_dataloader)):\n",
        "        token_ids = token_ids.long().to(device)\n",
        "        segment_ids = segment_ids.long().to(device)\n",
        "        valid_length = valid_length\n",
        "        label = label.long().to(device)\n",
        "        out = model(token_ids, valid_length, segment_ids)\n",
        "        test_acc += calc_accuracy(out, label)\n",
        "    print(\"epoch {} test acc {}\".format(e+1, test_acc / (batch_id+1)))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:5: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
            "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n",
            "  \"\"\"\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "37273726c2c540adbc69d94681e52716",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=997.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "epoch 1 batch id 1 loss 0.7145305871963501 train acc 0.4375\n",
            "epoch 1 batch id 201 loss 0.10184042155742645 train acc 0.921875\n",
            "epoch 1 batch id 401 loss 0.03117986023426056 train acc 0.9464230049875312\n",
            "epoch 1 batch id 601 loss 0.07595422118902206 train acc 0.9580906821963394\n",
            "epoch 1 batch id 801 loss 0.1335894614458084 train acc 0.9649071473158551\n",
            "\n",
            "epoch 1 train acc 0.969075859654435\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:22: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
            "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "91c365688b534e308fba924044004c3a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=111.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "epoch 1 test acc 0.9903417448060304\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "2e9a0c282d004eb2a69cc56a1fb8317d",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=997.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "epoch 2 batch id 1 loss 0.01774946041405201 train acc 1.0\n",
            "epoch 2 batch id 201 loss 0.037871070206165314 train acc 0.9896610696517413\n",
            "epoch 2 batch id 401 loss 0.008383787237107754 train acc 0.9909211346633416\n",
            "epoch 2 batch id 601 loss 0.0782151073217392 train acc 0.9920445091514143\n",
            "epoch 2 batch id 801 loss 0.0650489404797554 train acc 0.99270443196005\n",
            "\n",
            "epoch 2 train acc 0.9933550651955868\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8643c9223aa74ccbaa079c3f4bfbb465",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=111.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "epoch 2 test acc 0.9926801801801802\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "5a814af62c7b4db7bb89248c127e6373",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=997.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "epoch 3 batch id 1 loss 0.041974276304244995 train acc 0.984375\n",
            "epoch 3 batch id 201 loss 0.00035623228177428246 train acc 0.9963463930348259\n",
            "epoch 3 batch id 401 loss 0.0010382692562416196 train acc 0.9968048628428927\n",
            "epoch 3 batch id 601 loss 0.03541859611868858 train acc 0.9973481697171381\n",
            "epoch 3 batch id 801 loss 0.0004100111254956573 train acc 0.9974250936329588\n",
            "\n",
            "epoch 3 train acc 0.9975394934804414\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e771282a21bd4fdba6b4f2fd6d0c96fb",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=111.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "epoch 3 test acc 0.993384009009009\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "871e69fe7e914871b7aac956bbe4e3f2",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=997.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "epoch 4 batch id 1 loss 0.0004114733310416341 train acc 1.0\n",
            "epoch 4 batch id 201 loss 0.00020492012845352292 train acc 0.9992226368159204\n",
            "epoch 4 batch id 401 loss 0.00018793960043694824 train acc 0.9991038029925187\n",
            "epoch 4 batch id 601 loss 0.0004016011953353882 train acc 0.9992720465890182\n",
            "epoch 4 batch id 801 loss 0.00012136494478909299 train acc 0.9993172596754057\n",
            "\n",
            "epoch 4 train acc 0.9993261033099298\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "869419228f6c49d88f42e1210e90714b",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=111.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "epoch 4 test acc 0.9943262778084208\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d8e72056e50f4a148474e552dd164e9c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=997.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "epoch 5 batch id 1 loss 0.000324369320878759 train acc 1.0\n",
            "epoch 5 batch id 201 loss 0.00014836004993412644 train acc 0.9999222636815921\n",
            "epoch 5 batch id 401 loss 0.00010512120206840336 train acc 0.9998441396508728\n",
            "epoch 5 batch id 601 loss 0.0001154237543232739 train acc 0.9998700083194676\n",
            "epoch 5 batch id 801 loss 8.862611139193177e-05 train acc 0.9999024656679151\n",
            "\n",
            "epoch 5 train acc 0.9998746238716149\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a3bea9954fad4153a8964375e1222aaf",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=111.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "epoch 5 test acc 0.9943693693693694\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ocf97qP7nvi8"
      },
      "source": [
        "torch.save(model.state_dict(), \"/gdrive/My Drive/한양대학교/NH투자증권 빅데이터 경진대회/코난/5.Model/model_state_dict.pt\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7mt3iI52in0v"
      },
      "source": [
        "# Scoring"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KMTOxx7HkaQT"
      },
      "source": [
        "데이터 불러오기"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fd6iSL5Zj87w"
      },
      "source": [
        "import pandas as pd\n",
        "\n",
        "news_dataset_path = r\"/gdrive/My Drive/한양대학교/NH투자증권 빅데이터 경진대회/코난/1.Data/open/news_test.csv\"\n",
        "\n",
        "news_dataset = pd.read_csv(news_dataset_path)"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lKj7fJqvkfDE"
      },
      "source": [
        "시간 측정 시작"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iadiQdpDkYmv"
      },
      "source": [
        "import time\n",
        "start = time.time()"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7Z94-nf6khAw"
      },
      "source": [
        "라이브러리 불러오기"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WI2g4IE-kjCR"
      },
      "source": [
        "import torch\n",
        "from torch import nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "from torch.utils.data import Dataset, DataLoader\n",
        "import gluonnlp as nlp\n",
        "import numpy as np\n",
        "from tqdm import tqdm, tqdm_notebook\n",
        "from kobert.utils import get_tokenizer\n",
        "from kobert.pytorch_kobert import get_pytorch_kobert_model\n",
        "from transformers import AdamW\n",
        "from transformers.optimization import get_cosine_schedule_with_warmup"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mZ6YnCcokjb-"
      },
      "source": [
        "Model 불러오기"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hCBnBi9fkrAD",
        "outputId": "8472e131-7f6b-4393-f010-dc89500729c8"
      },
      "source": [
        "device = torch.device(\"cuda:0\")\n",
        "\n",
        "bertmodel, vocab = get_pytorch_kobert_model()\n",
        "\n",
        "class BERTClassifier(nn.Module):\n",
        "    def __init__(self,\n",
        "                 bert,\n",
        "                 hidden_size = 768,\n",
        "                 num_classes=2,\n",
        "                 dr_rate=None,\n",
        "                 params=None):\n",
        "        super(BERTClassifier, self).__init__()\n",
        "        self.bert = bert\n",
        "        self.dr_rate = dr_rate \n",
        "        self.classifier = nn.Linear(hidden_size , num_classes)\n",
        "        if dr_rate:\n",
        "            self.dropout = nn.Dropout(p=dr_rate)\n",
        "    \n",
        "    def gen_attention_mask(self, token_ids, valid_length):\n",
        "        attention_mask = torch.zeros_like(token_ids)\n",
        "        for i, v in enumerate(valid_length):\n",
        "            attention_mask[i][:v] = 1\n",
        "        return attention_mask.float()\n",
        "\n",
        "    def forward(self, token_ids, valid_length, segment_ids):\n",
        "        attention_mask = self.gen_attention_mask(token_ids, valid_length)\n",
        "        _, pooler = self.bert(input_ids = token_ids, token_type_ids = segment_ids.long(), attention_mask = attention_mask.float().to(token_ids.device))\n",
        "        if self.dr_rate:\n",
        "            out = self.dropout(pooler)\n",
        "        return self.classifier(out)\n",
        "\n",
        "model = BERTClassifier(bertmodel, dr_rate=0.5).to(device)\n",
        "\n",
        "model.load_state_dict(torch.load(\"/gdrive/My Drive/한양대학교/NH투자증권 빅데이터 경진대회/코난/5.Model/model_state_dict.pt\"))"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "using cached model\n",
            "using cached model\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<All keys matched successfully>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "36Rb54e5krYX"
      },
      "source": [
        "형태소 분석 + 전처리"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xPIzgCPHkBNm",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "240ac75a-64b8-4073-e52e-0f08d4b285f7"
      },
      "source": [
        "tokenizer = get_tokenizer()\n",
        "tok = nlp.data.BERTSPTokenizer(tokenizer, vocab, lower=False)\n",
        "\n",
        "class BERTDataset(Dataset):\n",
        "    def __init__(self, dataset, sent_idx, label_idx, bert_tokenizer, max_len, pad, pair):\n",
        "        transform = nlp.data.BERTSentenceTransform(bert_tokenizer, max_seq_length=max_len, pad=pad, pair=pair)\n",
        "        self.sentences = [transform([i[sent_idx]]) for i in dataset]\n",
        "        self.labels = [np.int32(i[label_idx]) for i in dataset]\n",
        "\n",
        "    def __getitem__(self, i):\n",
        "        return (self.sentences[i] + (self.labels[i], ))\n",
        "\n",
        "    def __len__(self):\n",
        "        return (len(self.labels))\n",
        "\n",
        "news_dataset[\"content\"] = news_dataset[\"title\"] + \" \" + news_dataset[\"content\"]\n",
        "\n",
        "del news_dataset[\"n_id\"]\n",
        "del news_dataset[\"date\"]\n",
        "del news_dataset[\"title\"]\n",
        "del news_dataset[\"ord\"]\n",
        "\n",
        "news_dataset[\"content\"] = news_dataset[\"content\"].apply(lambda x : ' '.join(x.strip().split()))\n",
        "\n",
        "news_dataset[\"info\"] = 0\n",
        "\n",
        "news_dataset[[\"content\",\"info\"]].to_csv(\"/gdrive/My Drive/한양대학교/NH투자증권 빅데이터 경진대회/코난/1.Data/news_test.txt\", sep='\\t', index=False)\n",
        "\n",
        "dataset_test = nlp.data.TSVDataset(\"/gdrive/My Drive/한양대학교/NH투자증권 빅데이터 경진대회/코난/1.Data/news_test.txt\", field_indices=[0,1], num_discard_samples=1)\n",
        "\n",
        "data_test = BERTDataset(dataset_test, 0, 1, tok, 128, True, False)\n",
        "\n",
        "test_dataloader = torch.utils.data.DataLoader(data_test, batch_size=64, num_workers=5)"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "using cached model\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mRUAxStTkvpv"
      },
      "source": [
        "예측"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 137,
          "referenced_widgets": [
            "6325cce63d2e42c19f01bd9499283c31",
            "1eb3e744b0cc4496b0a0aabd9b5ff690",
            "9ab2f78e4ad64342b290c1f51efa0fec",
            "762ddee4a7b7499685768528082a750e",
            "8c5e8c211e5d48f9a3398803b321ae67",
            "90852e0f4d4b491eabc76bea9ba28483",
            "51112e88eb664bfaa499b21095b58c0f",
            "c77f42aef0734e1cbfb81f49c203f6bb"
          ]
        },
        "id": "4fmhhXdQJEPi",
        "outputId": "9cd45dee-f061-4a5d-9c6c-c8c62b715508"
      },
      "source": [
        "model.eval()\n",
        "\n",
        "Predict = []\n",
        "\n",
        "for batch_id, (token_ids, valid_length, segment_ids, label) in enumerate(tqdm_notebook(test_dataloader)):\n",
        "    token_ids = token_ids.long().to(device)\n",
        "    segment_ids = segment_ids.long().to(device)\n",
        "    valid_length = valid_length\n",
        "    label = label.long().to(device)\n",
        "    out = model(token_ids, valid_length, segment_ids)\n",
        "    _, predict = torch.max(out,1)\n",
        "    Predict.extend(predict.tolist())"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:5: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
            "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n",
            "  \"\"\"\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "6325cce63d2e42c19f01bd9499283c31",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=2228.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZUQh2K-kVp13",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "cd17c619-5531-4e1f-ebc7-4d46ec39fc5a"
      },
      "source": [
        "news_dataset[\"info\"] = Predict\n",
        "\n",
        "print(news_dataset.groupby(\"info\").size())\n",
        "\n",
        "news_dataset = news_dataset[[\"id\",\"info\"]]"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "info\n",
            "0    88161\n",
            "1    54404\n",
            "dtype: int64\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ssc-Hup0uo9R"
      },
      "source": [
        "시간 측정 종료"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nEUsGyYek02w",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ac5cd082-e201-4820-ac15-d54763425f7a"
      },
      "source": [
        "print(time.time() - start)"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "408.93971490859985\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_EfBBdFik4gW"
      },
      "source": [
        "제출"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z_2Qp1pJk4_T"
      },
      "source": [
        "news_dataset.to_csv(\"/gdrive/My Drive/한양대학교/NH투자증권 빅데이터 경진대회/result/sample_submission.csv\", index=False)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}