notebooks/BertModels4ArticleClassification.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "BertModels4ArticleClassification.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "TtvjJLWWu11l"
      ],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "1c33b2b3828644f1a767befef3553d6f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b2fc986d66384b9fabfd5ca52e3602e6",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_3dcc0591fbf84d31a60097cc77dad4ff",
              "IPY_MODEL_e569f1affb9440a78540ccfde9cdf2be"
            ]
          }
        },
        "b2fc986d66384b9fabfd5ca52e3602e6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3dcc0591fbf84d31a60097cc77dad4ff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_9c9a538260f6433abec59016a4a9d3be",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 433,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 433,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ca8ddd3508d94f6ab73821add5795ec4"
          }
        },
        "e569f1affb9440a78540ccfde9cdf2be": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_e5eb9572998e477eb03251cbb4722a77",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 433/433 [00:00&lt;00:00, 1.99kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_21e79db632bc4c87b73deee63baaa85f"
          }
        },
        "9c9a538260f6433abec59016a4a9d3be": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ca8ddd3508d94f6ab73821add5795ec4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e5eb9572998e477eb03251cbb4722a77": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "21e79db632bc4c87b73deee63baaa85f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ae3ea810fe0c422baf180b8c25f9680c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_8b8da1d6612741a5b9deba02dd119db5",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_9b1aaaa4e29042a3aaadc61fbf6c746a",
              "IPY_MODEL_b34c706d038745428325a8691b9e20b0"
            ]
          }
        },
        "8b8da1d6612741a5b9deba02dd119db5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9b1aaaa4e29042a3aaadc61fbf6c746a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_4823a184c7c3472098b18fd9ca0b8067",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 435780013,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 435780013,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_21550b3284d748ea8c7af688b63652c5"
          }
        },
        "b34c706d038745428325a8691b9e20b0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_2639d0a34be446ac80c947397ab32791",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 436M/436M [00:13&lt;00:00, 33.0MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_637ec7b78579436b8d2392989047510a"
          }
        },
        "4823a184c7c3472098b18fd9ca0b8067": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "21550b3284d748ea8c7af688b63652c5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2639d0a34be446ac80c947397ab32791": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "637ec7b78579436b8d2392989047510a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "978819fce8284cffb07d06a0daebd57c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_01cfbf1b7c6f4875bc87f9302fc1964c",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_aad0cbdd3e2647d6a501a918ef73f050",
              "IPY_MODEL_4d2a64d2b64648cb872ec2f293f284bd"
            ]
          }
        },
        "01cfbf1b7c6f4875bc87f9302fc1964c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "aad0cbdd3e2647d6a501a918ef73f050": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_83b288db8dc241d89f14499b0a41cbac",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 213450,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 213450,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_fec08a2410ad46d0a16e3e116edd8a56"
          }
        },
        "4d2a64d2b64648cb872ec2f293f284bd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_84a68689456642449cda6ff830bf4154",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 213k/213k [00:00&lt;00:00, 797kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0690b1b905cb4110a252be5af1ba0d0b"
          }
        },
        "83b288db8dc241d89f14499b0a41cbac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "fec08a2410ad46d0a16e3e116edd8a56": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "84a68689456642449cda6ff830bf4154": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0690b1b905cb4110a252be5af1ba0d0b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7d626ee41ed146829993e4df2c14363f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_92366cb7c71748d1bcb9cad3fb56fcfe",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_93a7992a06b04f3faf1d407c4c472928",
              "IPY_MODEL_620602440c214aa7af4b18768a589c7d"
            ]
          }
        },
        "92366cb7c71748d1bcb9cad3fb56fcfe": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "93a7992a06b04f3faf1d407c4c472928": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_78f87e645f6549dd9abba1e22580925f",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 112,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 112,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7f372ec8a2cb48758dbc2f0fb429b852"
          }
        },
        "620602440c214aa7af4b18768a589c7d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_5e91a50ad1a14941973e573ab98a7aa5",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 112/112 [00:00&lt;00:00, 973B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4cb1f514acd845cab2c5bc3461b76f09"
          }
        },
        "78f87e645f6549dd9abba1e22580925f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7f372ec8a2cb48758dbc2f0fb429b852": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5e91a50ad1a14941973e573ab98a7aa5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4cb1f514acd845cab2c5bc3461b76f09": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "bce2ee895b4b42549ec62de6551fc21a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_370ce62bb6d84ebb9d03ceab6d4865bb",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_298e069e016b4c559246a96ff4d9a513",
              "IPY_MODEL_33bf0d4bf3254bb1911bec9021b15546"
            ]
          }
        },
        "370ce62bb6d84ebb9d03ceab6d4865bb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "298e069e016b4c559246a96ff4d9a513": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_1eb4f64e14564150a561f1c32ee0a388",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 51,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 51,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3aa422132ffc4f099ea40611763c4052"
          }
        },
        "33bf0d4bf3254bb1911bec9021b15546": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d92a3d09e3a043979debd174fd6b76c2",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 51.0/51.0 [00:00&lt;00:00, 635B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_94149a4578844cbf88c18ba4b8bd27c2"
          }
        },
        "1eb4f64e14564150a561f1c32ee0a388": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3aa422132ffc4f099ea40611763c4052": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d92a3d09e3a043979debd174fd6b76c2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "94149a4578844cbf88c18ba4b8bd27c2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GqTlG1bK_omK"
      },
      "source": [
        "## Fine-tuning BERT/BioBERT and SciBERT for the classification of articles in research categories\n",
        "\n",
        "Important Note: The library used to load and train the transformers HugginFace have deprecated the transform type used by the pre-trained BioBERT (AutoModelWithLMHead) so it is not possible to train BioBERT with this version of the library. "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dWlzS0Y-NeFO"
      },
      "source": [
        "## Environment setup: dataset & libraries. "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6_zGQHmbISLC"
      },
      "source": [
        "### Download dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c_4Ddloyk8DO",
        "outputId": "871e382e-09e0-48ef-f74f-31ba534c09fe",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 87
        }
      },
      "source": [
        "!gdown --id 1XOtSCqfzMC3_XWY8Ylw_josDAJOjMwOF"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading...\n",
            "From: https://drive.google.com/uc?id=1XOtSCqfzMC3_XWY8Ylw_josDAJOjMwOF\n",
            "To: /content/articles_scigraph_2011.json\n",
            "274MB [00:05, 50.5MB/s]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZzVaPSqDM8EP"
      },
      "source": [
        "Download library to fine-tune the LM on mutilabel classification"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qIcu-vgxly0h",
        "outputId": "fee9fefa-5ba0-4d4f-e6f5-f687a23fbc32",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 87
        }
      },
      "source": [
        "!gdown --id 1LpufGkbVYTGxgAHr2TyqQimqVRte420U"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading...\n",
            "From: https://drive.google.com/uc?id=1LpufGkbVYTGxgAHr2TyqQimqVRte420U\n",
            "To: /content/BertModeling.py\n",
            "\r  0% 0.00/14.4k [00:00<?, ?B/s]\r100% 14.4k/14.4k [00:00<00:00, 21.8MB/s]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HLOsQ5F5l-m3",
        "outputId": "ab3841ad-b337-4084-87e3-f0801349f63f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "!ls"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "articles_scigraph_2011.json  BertModeling.py  sample_data\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3FdMlp30eGPz"
      },
      "source": [
        "### Install and import required libraries"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pDAwPZI5hZ2e",
        "outputId": "9ab5ee56-aa26-4318-8e75-0a6e88de8ba5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 816
        }
      },
      "source": [
        "!pip install 'transformers==2.8.0'"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers==2.8.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/a3/78/92cedda05552398352ed9784908b834ee32a0bd071a9b32de287327370b7/transformers-2.8.0-py3-none-any.whl (563kB)\n",
            "\r\u001b[K     |▋                               | 10kB 24.9MB/s eta 0:00:01\r\u001b[K     |█▏                              | 20kB 24.3MB/s eta 0:00:01\r\u001b[K     |█▊                              | 30kB 17.5MB/s eta 0:00:01\r\u001b[K     |██▎                             | 40kB 12.6MB/s eta 0:00:01\r\u001b[K     |███                             | 51kB 11.1MB/s eta 0:00:01\r\u001b[K     |███▌                            | 61kB 11.5MB/s eta 0:00:01\r\u001b[K     |████                            | 71kB 10.8MB/s eta 0:00:01\r\u001b[K     |████▋                           | 81kB 9.8MB/s eta 0:00:01\r\u001b[K     |█████▎                          | 92kB 9.8MB/s eta 0:00:01\r\u001b[K     |█████▉                          | 102kB 10.6MB/s eta 0:00:01\r\u001b[K     |██████▍                         | 112kB 10.6MB/s eta 0:00:01\r\u001b[K     |███████                         | 122kB 10.6MB/s eta 0:00:01\r\u001b[K     |███████▋                        | 133kB 10.6MB/s eta 0:00:01\r\u001b[K     |████████▏                       | 143kB 10.6MB/s eta 0:00:01\r\u001b[K     |████████▊                       | 153kB 10.6MB/s eta 0:00:01\r\u001b[K     |█████████▎                      | 163kB 10.6MB/s eta 0:00:01\r\u001b[K     |█████████▉                      | 174kB 10.6MB/s eta 0:00:01\r\u001b[K     |██████████▌                     | 184kB 10.6MB/s eta 0:00:01\r\u001b[K     |███████████                     | 194kB 10.6MB/s eta 0:00:01\r\u001b[K     |███████████▋                    | 204kB 10.6MB/s eta 0:00:01\r\u001b[K     |████████████▏                   | 215kB 10.6MB/s eta 0:00:01\r\u001b[K     |████████████▉                   | 225kB 10.6MB/s eta 0:00:01\r\u001b[K     |█████████████▍                  | 235kB 10.6MB/s eta 0:00:01\r\u001b[K     |██████████████                  | 245kB 10.6MB/s eta 0:00:01\r\u001b[K     |██████████████▌                 | 256kB 10.6MB/s eta 0:00:01\r\u001b[K     |███████████████▏                | 266kB 10.6MB/s eta 0:00:01\r\u001b[K     |███████████████▊                | 276kB 10.6MB/s eta 0:00:01\r\u001b[K     |████████████████▎               | 286kB 10.6MB/s eta 0:00:01\r\u001b[K     |████████████████▉               | 296kB 10.6MB/s eta 0:00:01\r\u001b[K     |█████████████████▍              | 307kB 10.6MB/s eta 0:00:01\r\u001b[K     |██████████████████              | 317kB 10.6MB/s eta 0:00:01\r\u001b[K     |██████████████████▋             | 327kB 10.6MB/s eta 0:00:01\r\u001b[K     |███████████████████▏            | 337kB 10.6MB/s eta 0:00:01\r\u001b[K     |███████████████████▊            | 348kB 10.6MB/s eta 0:00:01\r\u001b[K     |████████████████████▍           | 358kB 10.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████           | 368kB 10.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████▌          | 378kB 10.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████          | 389kB 10.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████▊         | 399kB 10.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████▎        | 409kB 10.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████▉        | 419kB 10.6MB/s eta 0:00:01\r\u001b[K     |████████████████████████▍       | 430kB 10.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████████       | 440kB 10.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▋      | 450kB 10.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▏     | 460kB 10.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▊     | 471kB 10.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▎    | 481kB 10.6MB/s eta 0:00:01\r\u001b[K     |████████████████████████████    | 491kB 10.6MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▌   | 501kB 10.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████   | 512kB 10.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▋  | 522kB 10.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▎ | 532kB 10.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▉ | 542kB 10.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▍| 552kB 10.6MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 563kB 10.6MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 573kB 10.6MB/s \n",
            "\u001b[?25hCollecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
            "\u001b[K     |████████████████████████████████| 890kB 12.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers==2.8.0) (2019.12.20)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers==2.8.0) (4.41.1)\n",
            "Collecting tokenizers==0.5.2\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d1/3f/73c881ea4723e43c1e9acf317cf407fab3a278daab3a69c98dcac511c04f/tokenizers-0.5.2-cp36-cp36m-manylinux1_x86_64.whl (3.7MB)\n",
            "\u001b[K     |████████████████████████████████| 3.7MB 39.1MB/s \n",
            "\u001b[?25hCollecting sentencepiece\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/e5/2d/6d4ca4bef9a67070fa1cac508606328329152b1df10bdf31fb6e4e727894/sentencepiece-0.1.94-cp36-cp36m-manylinux2014_x86_64.whl (1.1MB)\n",
            "\u001b[K     |████████████████████████████████| 1.1MB 49.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers==2.8.0) (2.23.0)\n",
            "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers==2.8.0) (0.7)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers==2.8.0) (3.0.12)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers==2.8.0) (1.18.5)\n",
            "Collecting boto3\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/28/76/1f853a1ff319c173c638f38c34ebb389389253bf828e18fc4de52a2f4288/boto3-1.16.7-py2.py3-none-any.whl (129kB)\n",
            "\u001b[K     |████████████████████████████████| 133kB 56.3MB/s \n",
            "\u001b[?25hRequirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==2.8.0) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==2.8.0) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==2.8.0) (0.17.0)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==2.8.0) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==2.8.0) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==2.8.0) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==2.8.0) (2020.6.20)\n",
            "Collecting jmespath<1.0.0,>=0.7.1\n",
            "  Downloading https://files.pythonhosted.org/packages/07/cb/5f001272b6faeb23c1c9e0acc04d48eaaf5c862c17709d20e3469c6e0139/jmespath-0.10.0-py2.py3-none-any.whl\n",
            "Collecting s3transfer<0.4.0,>=0.3.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/69/79/e6afb3d8b0b4e96cefbdc690f741d7dd24547ff1f94240c997a26fa908d3/s3transfer-0.3.3-py2.py3-none-any.whl (69kB)\n",
            "\u001b[K     |████████████████████████████████| 71kB 11.7MB/s \n",
            "\u001b[?25hCollecting botocore<1.20.0,>=1.19.7\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d1/58/64883046f9c98d9f94cc81174d0b83e0be35b8f6c252e255b709b9024ef1/botocore-1.19.7-py2.py3-none-any.whl (6.7MB)\n",
            "\u001b[K     |████████████████████████████████| 6.7MB 50.7MB/s \n",
            "\u001b[?25hRequirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.6/dist-packages (from botocore<1.20.0,>=1.19.7->boto3->transformers==2.8.0) (2.8.1)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893257 sha256=2668d9add5fc4626276655301e1df0c57badc4570e372ecbed7905a1f0698204\n",
            "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
            "Successfully built sacremoses\n",
            "\u001b[31mERROR: botocore 1.19.7 has requirement urllib3<1.26,>=1.25.4; python_version != \"3.4\", but you'll have urllib3 1.24.3 which is incompatible.\u001b[0m\n",
            "Installing collected packages: sacremoses, tokenizers, sentencepiece, jmespath, botocore, s3transfer, boto3, transformers\n",
            "Successfully installed boto3-1.16.7 botocore-1.19.7 jmespath-0.10.0 s3transfer-0.3.3 sacremoses-0.0.43 sentencepiece-0.1.94 tokenizers-0.5.2 transformers-2.8.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nWPUNEClNKWG"
      },
      "source": [
        "import glob\n",
        "import pprint\n",
        "import logging\n",
        "import os\n",
        "import random\n",
        "import json\n",
        "import time\n",
        "\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import torch\n",
        "from torch.utils.data import (DataLoader, RandomSampler, SequentialSampler,\n",
        "                              TensorDataset)\n",
        "import random\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from torch.utils.data.distributed import DistributedSampler\n",
        "from tqdm import tqdm_notebook, trange\n",
        "\n",
        "from transformers import (WEIGHTS_NAME, BertConfig, BertForSequenceClassification, \n",
        "                          BertTokenizer, AutoTokenizer, AutoModelWithLMHead,AutoConfig)\n",
        "from BertModeling import BertForMultiLabelSequenceClassification\n",
        "from BertModeling import BioBertForMultiLabelSequenceClassification2\n",
        "\n",
        "from transformers import AdamW#, WarmupLinearSchedule\n",
        "\n",
        "logging.basicConfig(level=logging.INFO)\n",
        "logger = logging.getLogger(__name__)"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A5xWrmjqhOu4",
        "outputId": "d6365cd7-c9c7-4bcc-f196-316963f92b91",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "n_gpu = torch.cuda.device_count()\n",
        "torch.cuda.get_device_name(0)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'Tesla T4'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gtWL2vhMoNBS"
      },
      "source": [
        "## Get train and test data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rZ-vKe1dlEol"
      },
      "source": [
        "Set the number of articles to train and validate"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0nNHYWaZlDJR"
      },
      "source": [
        "max_number_articles = 500 # set -1 to process all articles"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t7mBTK0fjtnp",
        "outputId": "fc666850-99c2-4661-e99d-a167d7b15112",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 674
        }
      },
      "source": [
        "with open('./articles_scigraph_2011.json') as json_file:\n",
        "  data = json.load(json_file)\n",
        "\n",
        "if max_number_articles != -1:\n",
        "  data = data[:max_number_articles]\n",
        "pprint.pprint(data[2])\n"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'abstract': 'Norflurazon and simazine are pre-emergent herbicides detected '\n",
            "             'frequently in surface water associated with South Florida '\n",
            "             'agricultural canals and drainage water. This study investigated '\n",
            "             'the potential use of a 1.34\\xa0ha constructed wetland for '\n",
            "             'removing these herbicides from surface water. The total length '\n",
            "             'of the wetland was 400\\xa0m and width was 35\\xa0m. A surface '\n",
            "             'water flow rate of 740\\xa0L/min was maintained in the system '\n",
            "             'using a pump. The plant community within the system consisted '\n",
            "             'primarily of Panicum repens, Alternanthera philoxeroides, and '\n",
            "             'Bacopa caroliniana. Norflurazon and simazine, derived from '\n",
            "             'commercial formulations, were injected (51.1\\xa0g active '\n",
            "             'ingredient each) directly into the water pumped into the wetland '\n",
            "             'over a 2\\xa0h period. Water samples were collected from the '\n",
            "             'wetland upstream of the dosing system at 3\\xa0h intervals from '\n",
            "             'the beginning through 360\\xa0h and at the exit point at 1, 2, '\n",
            "             'and 3\\xa0h intervals for the periods of 0–24, 25–48 and '\n",
            "             '49–360\\xa0h after dosing, respectively. The herbicides were '\n",
            "             'extracted using C-18 cartridges and were analyzed by GC-TSD. The '\n",
            "             'total mass of each herbicide discharged from the system was '\n",
            "             'estimated by multiplying the concentration by the total volume '\n",
            "             'discharged during the sampled period. Neither herbicide was '\n",
            "             'detected in the inflow water during the entire study. '\n",
            "             'Norflurazon was first detected at the exit 19\\xa0h after dosing '\n",
            "             'and simazine after 23\\xa0h. Discharge patterns of the two '\n",
            "             'herbicides differed dramatically. Norflurazon tended to bleed '\n",
            "             'off from the wetland with no distinct peak concentration. '\n",
            "             'However, the mobile fraction of simazine was discharged over a '\n",
            "             '58\\xa0h period. Mean/maximum/median detectable concentrations of '\n",
            "             'the herbicides were 3.9\\xa0±\\xa01.7/8.1/3.4\\xa0μg\\xa0L−1 for '\n",
            "             'norflurazon, and 11.9\\xa0±\\xa06.8/23.6/12.0\\xa0μg\\xa0L−1 for '\n",
            "             'simazine, respectively. The total masses of norflurazon and '\n",
            "             'simazine discharged from the exit during the 15\\xa0day study '\n",
            "             'were 51.7 and 26.9\\xa0g, indicating 0% and 47.4% removal from '\n",
            "             'the surface water by the system.',\n",
            " 'fieldcodes': ['09', '0907'],\n",
            " 'title': 'Norflurazon and Simazine Removal from Surface Water Using a '\n",
            "          'Constructed Wetland'}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-_v-k40IkA8s"
      },
      "source": [
        "train = pd.DataFrame({\n",
        "    'id': range(len(data)),\n",
        "    'label': [d['fieldcodes']for d in data],\n",
        "    'mark': ['a'] * len(data),\n",
        "    'text': [d['title'] + ' ' + d['abstract'] for d in data]\n",
        "    \n",
        "})\n",
        "\n",
        "for t in train['label']:\n",
        "    i = 0\n",
        "    while i < (len(t)):\n",
        "        if len(t[i]) > 2:\n",
        "            t.pop(i)\n",
        "            i = i - 1\n",
        "        i = i + 1\n",
        "test = train[int(0.75*len(train)):]\n",
        "train = train[:int(0.75*len(train))]\n",
        "\n",
        "# train.to_csv(\"./Data/classifier/train.tsv\", sep = '\\t', index = False, header = 'False')\n",
        "# test.to_csv(\"./Data/classifier/test.tsv\", sep = '\\t', index = False, header = 'False')"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "r4HspAI9CPml"
      },
      "source": [
        "## Set the model to fine-tune and its tokenizer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b3tzfEHFCU8s",
        "outputId": "ffbcbf82-8775-46a2-d929-85fd0b126b73",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "1c33b2b3828644f1a767befef3553d6f",
            "b2fc986d66384b9fabfd5ca52e3602e6",
            "3dcc0591fbf84d31a60097cc77dad4ff",
            "e569f1affb9440a78540ccfde9cdf2be",
            "9c9a538260f6433abec59016a4a9d3be",
            "ca8ddd3508d94f6ab73821add5795ec4",
            "e5eb9572998e477eb03251cbb4722a77",
            "21e79db632bc4c87b73deee63baaa85f",
            "ae3ea810fe0c422baf180b8c25f9680c",
            "8b8da1d6612741a5b9deba02dd119db5",
            "9b1aaaa4e29042a3aaadc61fbf6c746a",
            "b34c706d038745428325a8691b9e20b0",
            "4823a184c7c3472098b18fd9ca0b8067",
            "21550b3284d748ea8c7af688b63652c5",
            "2639d0a34be446ac80c947397ab32791",
            "637ec7b78579436b8d2392989047510a",
            "978819fce8284cffb07d06a0daebd57c",
            "01cfbf1b7c6f4875bc87f9302fc1964c",
            "aad0cbdd3e2647d6a501a918ef73f050",
            "4d2a64d2b64648cb872ec2f293f284bd",
            "83b288db8dc241d89f14499b0a41cbac",
            "fec08a2410ad46d0a16e3e116edd8a56",
            "84a68689456642449cda6ff830bf4154",
            "0690b1b905cb4110a252be5af1ba0d0b",
            "7d626ee41ed146829993e4df2c14363f",
            "92366cb7c71748d1bcb9cad3fb56fcfe",
            "93a7992a06b04f3faf1d407c4c472928",
            "620602440c214aa7af4b18768a589c7d",
            "78f87e645f6549dd9abba1e22580925f",
            "7f372ec8a2cb48758dbc2f0fb429b852",
            "5e91a50ad1a14941973e573ab98a7aa5",
            "4cb1f514acd845cab2c5bc3461b76f09",
            "bce2ee895b4b42549ec62de6551fc21a",
            "370ce62bb6d84ebb9d03ceab6d4865bb",
            "298e069e016b4c559246a96ff4d9a513",
            "33bf0d4bf3254bb1911bec9021b15546",
            "1eb4f64e14564150a561f1c32ee0a388",
            "3aa422132ffc4f099ea40611763c4052",
            "d92a3d09e3a043979debd174fd6b76c2",
            "94149a4578844cbf88c18ba4b8bd27c2"
          ]
        }
      },
      "source": [
        "max_length = 512 #max tokens in sequence\n",
        "\n",
        "# For BERT\n",
        "# LM='Bert'\n",
        "# modelpath = 'bert-base-uncased'\n",
        "# model = BertForMultiLabelSequenceClassification.from_pretrained(modelpath, num_labels=22, output_attentions=True)\n",
        "# tokenizer = BertTokenizer.from_pretrained(modelpath, do_lower_case=True, return_token_type_ids=True, max_len = max_length)\n",
        "\n",
        "# For SciBERT comment the lines above and uncomment the following\n",
        "# LM='sciBert'\n",
        "# modelpath = 'allenai/scibert_scivocab_uncased'\n",
        "# model = BertForMultiLabelSequenceClassification.from_pretrained(modelpath, num_labels=22, output_attentions=True)\n",
        "# tokenizer = BertTokenizer.from_pretrained(modelpath, do_lower_case=True, return_token_type_ids=True, max_len = max_length)\n",
        "\n",
        "# NOTE BioBERT model is being deprecated and does not work with the transformers current version. D\n",
        "# For BioBERT comment the lines above and uncomment the following\n",
        "# LM='bioBert'\n",
        "modelpath = 'monologg/biobert_v1.1_pubmed' \n",
        "config = AutoConfig.from_pretrained(modelpath, output_hidden_states=True, num_labels=22)\n",
        "model = BioBertForMultiLabelSequenceClassification2.from_pretrained(modelpath, config=config)\n",
        "tokenizer = BertTokenizer.from_pretrained(modelpath, do_lower_case=True, return_token_type_ids=True, max_len = max_length)\n",
        "\n",
        "\n",
        "model.cuda()"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:filelock:Lock 140338497360336 acquired on /root/.cache/torch/transformers/e20f0be854c50b92cb2a92e6bc23221658ab6365f5cc2829d762520e4580cded.9da767be51e1327499df13488672789394e2ca38b877837e52618a67d7002391.lock\n",
            "INFO:transformers.file_utils:https://s3.amazonaws.com/models.huggingface.co/bert/monologg/biobert_v1.1_pubmed/config.json not found in cache or force_download set to True, downloading to /root/.cache/torch/transformers/tmpg3e3zfl6\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "1c33b2b3828644f1a767befef3553d6f",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=433.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:transformers.file_utils:storing https://s3.amazonaws.com/models.huggingface.co/bert/monologg/biobert_v1.1_pubmed/config.json in cache at /root/.cache/torch/transformers/e20f0be854c50b92cb2a92e6bc23221658ab6365f5cc2829d762520e4580cded.9da767be51e1327499df13488672789394e2ca38b877837e52618a67d7002391\n",
            "INFO:transformers.file_utils:creating metadata file for /root/.cache/torch/transformers/e20f0be854c50b92cb2a92e6bc23221658ab6365f5cc2829d762520e4580cded.9da767be51e1327499df13488672789394e2ca38b877837e52618a67d7002391\n",
            "INFO:filelock:Lock 140338497360336 released on /root/.cache/torch/transformers/e20f0be854c50b92cb2a92e6bc23221658ab6365f5cc2829d762520e4580cded.9da767be51e1327499df13488672789394e2ca38b877837e52618a67d7002391.lock\n",
            "INFO:transformers.configuration_utils:loading configuration file https://s3.amazonaws.com/models.huggingface.co/bert/monologg/biobert_v1.1_pubmed/config.json from cache at /root/.cache/torch/transformers/e20f0be854c50b92cb2a92e6bc23221658ab6365f5cc2829d762520e4580cded.9da767be51e1327499df13488672789394e2ca38b877837e52618a67d7002391\n",
            "INFO:transformers.configuration_utils:Model config BertConfig {\n",
            "  \"_num_labels\": 22,\n",
            "  \"architectures\": [\n",
            "    \"BertForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bad_words_ids\": null,\n",
            "  \"bos_token_id\": null,\n",
            "  \"decoder_start_token_id\": null,\n",
            "  \"do_sample\": false,\n",
            "  \"early_stopping\": false,\n",
            "  \"eos_token_id\": null,\n",
            "  \"finetuning_task\": null,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"is_decoder\": false,\n",
            "  \"is_encoder_decoder\": false,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-12,\n",
            "  \"length_penalty\": 1.0,\n",
            "  \"max_length\": 20,\n",
            "  \"max_position_embeddings\": 512,\n",
            "  \"min_length\": 0,\n",
            "  \"model_type\": \"bert\",\n",
            "  \"no_repeat_ngram_size\": 0,\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_beams\": 1,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"num_return_sequences\": 1,\n",
            "  \"output_attentions\": false,\n",
            "  \"output_hidden_states\": true,\n",
            "  \"output_past\": true,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"prefix\": null,\n",
            "  \"pruned_heads\": {},\n",
            "  \"repetition_penalty\": 1.0,\n",
            "  \"task_specific_params\": null,\n",
            "  \"temperature\": 1.0,\n",
            "  \"top_k\": 50,\n",
            "  \"top_p\": 1.0,\n",
            "  \"torchscript\": false,\n",
            "  \"type_vocab_size\": 2,\n",
            "  \"use_bfloat16\": false,\n",
            "  \"vocab_size\": 28996\n",
            "}\n",
            "\n",
            "INFO:filelock:Lock 140338499727032 acquired on /root/.cache/torch/transformers/d2f56c891fb722c00df8909671f856f1a531a799d3daec453594eeb85d513c45.047c1b2094b97ef71fe2535ec6bb21d3238c6c3e00157d5b1b07731a9ee8cfe5.lock\n",
            "INFO:transformers.file_utils:https://s3.amazonaws.com/models.huggingface.co/bert/monologg/biobert_v1.1_pubmed/pytorch_model.bin not found in cache or force_download set to True, downloading to /root/.cache/torch/transformers/tmpwsqdaz5s\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "ae3ea810fe0c422baf180b8c25f9680c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=435780013.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:transformers.file_utils:storing https://s3.amazonaws.com/models.huggingface.co/bert/monologg/biobert_v1.1_pubmed/pytorch_model.bin in cache at /root/.cache/torch/transformers/d2f56c891fb722c00df8909671f856f1a531a799d3daec453594eeb85d513c45.047c1b2094b97ef71fe2535ec6bb21d3238c6c3e00157d5b1b07731a9ee8cfe5\n",
            "INFO:transformers.file_utils:creating metadata file for /root/.cache/torch/transformers/d2f56c891fb722c00df8909671f856f1a531a799d3daec453594eeb85d513c45.047c1b2094b97ef71fe2535ec6bb21d3238c6c3e00157d5b1b07731a9ee8cfe5\n",
            "INFO:filelock:Lock 140338499727032 released on /root/.cache/torch/transformers/d2f56c891fb722c00df8909671f856f1a531a799d3daec453594eeb85d513c45.047c1b2094b97ef71fe2535ec6bb21d3238c6c3e00157d5b1b07731a9ee8cfe5.lock\n",
            "INFO:transformers.modeling_utils:loading weights file https://s3.amazonaws.com/models.huggingface.co/bert/monologg/biobert_v1.1_pubmed/pytorch_model.bin from cache at /root/.cache/torch/transformers/d2f56c891fb722c00df8909671f856f1a531a799d3daec453594eeb85d513c45.047c1b2094b97ef71fe2535ec6bb21d3238c6c3e00157d5b1b07731a9ee8cfe5\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:transformers.modeling_utils:Weights of BioBertForMultiLabelSequenceClassification2 not initialized from pretrained model: ['classifier.weight', 'classifier.bias']\n",
            "INFO:transformers.modeling_utils:Weights from pretrained model not used in BioBertForMultiLabelSequenceClassification2: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias']\n",
            "INFO:transformers.tokenization_utils:Model name 'monologg/biobert_v1.1_pubmed' not found in model shortcut name list (bert-base-uncased, bert-large-uncased, bert-base-cased, bert-large-cased, bert-base-multilingual-uncased, bert-base-multilingual-cased, bert-base-chinese, bert-base-german-cased, bert-large-uncased-whole-word-masking, bert-large-cased-whole-word-masking, bert-large-uncased-whole-word-masking-finetuned-squad, bert-large-cased-whole-word-masking-finetuned-squad, bert-base-cased-finetuned-mrpc, bert-base-german-dbmdz-cased, bert-base-german-dbmdz-uncased, bert-base-finnish-cased-v1, bert-base-finnish-uncased-v1, bert-base-dutch-cased). Assuming 'monologg/biobert_v1.1_pubmed' is a path, a model identifier, or url to a directory containing tokenizer files.\n",
            "INFO:filelock:Lock 140338499726864 acquired on /root/.cache/torch/transformers/6099cdfc2bf60a1d2ce4ca092142f63db59cb900144c9f32017448cdae1c4055.e13dbb970cb325137104fb2e5f36fe865f27746c6b526f6352861b1980eb80b1.lock\n",
            "INFO:transformers.file_utils:https://s3.amazonaws.com/models.huggingface.co/bert/monologg/biobert_v1.1_pubmed/vocab.txt not found in cache or force_download set to True, downloading to /root/.cache/torch/transformers/tmpc4zjwx7n\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "978819fce8284cffb07d06a0daebd57c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=213450.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:transformers.file_utils:storing https://s3.amazonaws.com/models.huggingface.co/bert/monologg/biobert_v1.1_pubmed/vocab.txt in cache at /root/.cache/torch/transformers/6099cdfc2bf60a1d2ce4ca092142f63db59cb900144c9f32017448cdae1c4055.e13dbb970cb325137104fb2e5f36fe865f27746c6b526f6352861b1980eb80b1\n",
            "INFO:transformers.file_utils:creating metadata file for /root/.cache/torch/transformers/6099cdfc2bf60a1d2ce4ca092142f63db59cb900144c9f32017448cdae1c4055.e13dbb970cb325137104fb2e5f36fe865f27746c6b526f6352861b1980eb80b1\n",
            "INFO:filelock:Lock 140338499726864 released on /root/.cache/torch/transformers/6099cdfc2bf60a1d2ce4ca092142f63db59cb900144c9f32017448cdae1c4055.e13dbb970cb325137104fb2e5f36fe865f27746c6b526f6352861b1980eb80b1.lock\n",
            "INFO:filelock:Lock 140338499726864 acquired on /root/.cache/torch/transformers/a7d51998f3e1033c9f4d16d0c96f2de1c78e1e234ec025a216e45e7b81e72a8e.275045728fbf41c11d3dae08b8742c054377e18d92cc7b72b6351152a99b64e4.lock\n",
            "INFO:transformers.file_utils:https://s3.amazonaws.com/models.huggingface.co/bert/monologg/biobert_v1.1_pubmed/special_tokens_map.json not found in cache or force_download set to True, downloading to /root/.cache/torch/transformers/tmphydtmt48\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "7d626ee41ed146829993e4df2c14363f",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=112.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:transformers.file_utils:storing https://s3.amazonaws.com/models.huggingface.co/bert/monologg/biobert_v1.1_pubmed/special_tokens_map.json in cache at /root/.cache/torch/transformers/a7d51998f3e1033c9f4d16d0c96f2de1c78e1e234ec025a216e45e7b81e72a8e.275045728fbf41c11d3dae08b8742c054377e18d92cc7b72b6351152a99b64e4\n",
            "INFO:transformers.file_utils:creating metadata file for /root/.cache/torch/transformers/a7d51998f3e1033c9f4d16d0c96f2de1c78e1e234ec025a216e45e7b81e72a8e.275045728fbf41c11d3dae08b8742c054377e18d92cc7b72b6351152a99b64e4\n",
            "INFO:filelock:Lock 140338499726864 released on /root/.cache/torch/transformers/a7d51998f3e1033c9f4d16d0c96f2de1c78e1e234ec025a216e45e7b81e72a8e.275045728fbf41c11d3dae08b8742c054377e18d92cc7b72b6351152a99b64e4.lock\n",
            "INFO:filelock:Lock 140338499726864 acquired on /root/.cache/torch/transformers/411e4b56adae7178368f3bdd9a9040dbc43685308ce8d88fae3e0c21dfcb9255.f823277c1796df7b9584d6424272b3cfa2a493c007b227382c479e47ef12b985.lock\n",
            "INFO:transformers.file_utils:https://s3.amazonaws.com/models.huggingface.co/bert/monologg/biobert_v1.1_pubmed/tokenizer_config.json not found in cache or force_download set to True, downloading to /root/.cache/torch/transformers/tmpqymn1a7e\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "bce2ee895b4b42549ec62de6551fc21a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=51.0, style=ProgressStyle(description_w…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:transformers.file_utils:storing https://s3.amazonaws.com/models.huggingface.co/bert/monologg/biobert_v1.1_pubmed/tokenizer_config.json in cache at /root/.cache/torch/transformers/411e4b56adae7178368f3bdd9a9040dbc43685308ce8d88fae3e0c21dfcb9255.f823277c1796df7b9584d6424272b3cfa2a493c007b227382c479e47ef12b985\n",
            "INFO:transformers.file_utils:creating metadata file for /root/.cache/torch/transformers/411e4b56adae7178368f3bdd9a9040dbc43685308ce8d88fae3e0c21dfcb9255.f823277c1796df7b9584d6424272b3cfa2a493c007b227382c479e47ef12b985\n",
            "INFO:filelock:Lock 140338499726864 released on /root/.cache/torch/transformers/411e4b56adae7178368f3bdd9a9040dbc43685308ce8d88fae3e0c21dfcb9255.f823277c1796df7b9584d6424272b3cfa2a493c007b227382c479e47ef12b985.lock\n",
            "INFO:transformers.tokenization_utils:loading file https://s3.amazonaws.com/models.huggingface.co/bert/monologg/biobert_v1.1_pubmed/vocab.txt from cache at /root/.cache/torch/transformers/6099cdfc2bf60a1d2ce4ca092142f63db59cb900144c9f32017448cdae1c4055.e13dbb970cb325137104fb2e5f36fe865f27746c6b526f6352861b1980eb80b1\n",
            "INFO:transformers.tokenization_utils:loading file https://s3.amazonaws.com/models.huggingface.co/bert/monologg/biobert_v1.1_pubmed/added_tokens.json from cache at None\n",
            "INFO:transformers.tokenization_utils:loading file https://s3.amazonaws.com/models.huggingface.co/bert/monologg/biobert_v1.1_pubmed/special_tokens_map.json from cache at /root/.cache/torch/transformers/a7d51998f3e1033c9f4d16d0c96f2de1c78e1e234ec025a216e45e7b81e72a8e.275045728fbf41c11d3dae08b8742c054377e18d92cc7b72b6351152a99b64e4\n",
            "INFO:transformers.tokenization_utils:loading file https://s3.amazonaws.com/models.huggingface.co/bert/monologg/biobert_v1.1_pubmed/tokenizer_config.json from cache at /root/.cache/torch/transformers/411e4b56adae7178368f3bdd9a9040dbc43685308ce8d88fae3e0c21dfcb9255.f823277c1796df7b9584d6424272b3cfa2a493c007b227382c479e47ef12b985\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "BioBertForMultiLabelSequenceClassification2(\n",
              "  (bert): BertModel(\n",
              "    (embeddings): BertEmbeddings(\n",
              "      (word_embeddings): Embedding(28996, 768, padding_idx=0)\n",
              "      (position_embeddings): Embedding(512, 768)\n",
              "      (token_type_embeddings): Embedding(2, 768)\n",
              "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (encoder): BertEncoder(\n",
              "      (layer): ModuleList(\n",
              "        (0): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (1): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (2): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (3): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (4): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (5): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (6): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (7): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (8): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (9): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (10): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (11): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (pooler): BertPooler(\n",
              "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "      (activation): Tanh()\n",
              "    )\n",
              "  )\n",
              "  (dropout): Dropout(p=0.1, inplace=False)\n",
              "  (classifier): Linear(in_features=768, out_features=22, bias=True)\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TtvjJLWWu11l"
      },
      "source": [
        "## Preprocess data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jlMopHQNsvx9"
      },
      "source": [
        "### Tokenize"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sRtNqjyDoRi-"
      },
      "source": [
        "train_sentences = train.text.values\n",
        "train_sentences = [\"[CLS] \" + sentence + \" [SEP]\" for sentence in train_sentences]\n",
        "train_labels = train.label.values\n",
        "\n",
        "test_sentences = test.text.values\n",
        "test_sentences = [\"[CLS] \" + sentence + \" [SEP]\" for sentence in test_sentences]\n",
        "test_labels = test.label.values"
      ],
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zs_UJyOipGtY",
        "outputId": "08b868c1-ddd2-4fc2-f57d-e59accc682b2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        }
      },
      "source": [
        "train_tokenized_text = [tokenizer.tokenize(sent) for sent in train_sentences]\n",
        "test_tokenized_text = [tokenizer.tokenize(sent) for sent in test_sentences]\n",
        "\n",
        "print (\"Tokenize the first sentence:\")\n",
        "print (train_tokenized_text[0])"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Tokenize the first sentence:\n",
            "['[CLS]', 'physicians', '’', 'and', 'p', '##har', '##ma', '##cies', '’', 'overview', 'of', 'patients', '’', 'medication', '.', 'an', 'analysis', 'of', 'fi', '##delity', 'coefficients', 'background', '##it', 'is', 'essential', 'that', 'p', '##har', '##ma', '##cies', 'and', 'pre', '##s', '##cribe', '##rs', 'have', 'an', 'overview', 'of', 'each', 'patient', '’', 's', 'medication', 'in', 'order', 'to', 'prevent', 'drug', 'interactions', ',', 'un', '##int', '##ent', '##ional', 'co', '-', 'pre', '##s', '##cribing', ',', 'unnecessary', 'p', '##oly', '##pha', '##rma', '##cy', 'and', 'under', '##p', '##res', '##cribing', '.', 'we', 'have', 'assessed', 'this', 'overview', 'by', 'measuring', 'the', '‘', 'fi', '##delity', 'coefficient', '’', ',', 'a', 'measure', 'of', 'the', 'extent', 'to', 'which', 'a', 'drug', 'user', 'has', 'a', 'preference', 'for', 'one', 'pre', '##s', '##cribe', '##r', 'or', 'one', 'pharmacy', '.', 'methods', 'and', 'setting', '##data', 'for', 'all', 'prescription', '##s', 'issued', 'for', 'the', 'population', 'in', 'southern', 'den', '##mark', '(', 'population', '1', '.', '2', 'million', ')', 'in', '2009', 'was', 'extracted', 'from', 'the', 'o', '##den', '##se', 'university', 'p', '##har', '##ma', '##coe', '##pid', '##em', '##iol', '##ogical', 'database', '(', 'op', '##ed', ')', '.', 'analysis', 'of', 'the', 'extracted', 'data', 'was', 'then', 'limited', 'to', 'persons', 'with', 'at', 'least', 'ten', 'prescription', '##s', 'within', 'the', 'year', ',', 'resulting', 'in', '8', ',', '246', ',', '06', '##4', 'prescription', '##s', 'issued', 'to', '28', '##3', ',', '38', '##8', 'individuals', '.', 'for', 'each', 'individual', ',', 'we', 'identified', 'the', 'most', 'used', 'pre', '##s', '##cribe', '##r', 'and', 'calculated', 'the', 'proportion', 'of', 'all', 'prescription', '##s', 'accounted', 'for', 'by', 'that', 'pre', '##s', '##cribe', '##r', '(', 'f', '##c', '##p', '##res', '##c', ')', '.', 'the', 'individual', 'user', '’', 's', 'most', 'frequented', 'pharmacy', 'was', 'also', 'identified', 'and', 'the', 'f', '##c', '##pha', '##rm', 'calculated', 'in', 'a', 'similar', 'fashion', '.', 'results', '##the', 'average', 'f', '##c', '##p', '##res', '##c', 'and', 'average', 'f', '##c', '##pha', '##rm', 'were', '0', '.', '88', '##3', '(', 'standard', 'de', '##viation', '0', '.', '158', ')', 'and', '0', '.', '92', '##7', '(', '0', '.', '139', ')', ',', 'respectively', '.', 'the', 'estimated', 'difference', 'was', '0', '.', '04', '##46', '(', '95', '%', 'confidence', 'interval', '0', '.', '04', '##39', '–', '0', '.', '04', '##53', ')', '.', 'among', 'the', 'factors', 'associated', 'with', 'a', 'high', 'f', '##c', '##p', '##res', '##c', 'and', 'high', 'f', '##c', '##pha', '##rm', 'were', 'older', 'age', ',', 'male', 'gender', 'and', 'a', 'high', 'volume', 'of', 'prescription', '##s', '.', 'the', 'major', 'drug', 'classes', 'that', 'were', 'most', 'often', 'prescribed', 'by', 'a', 'non', '-', 'main', 'pre', '##s', '##cribe', '##r', 'were', 'beta', '-', 'la', '##ct', '##ams', ',', 'anti', '##de', '##press', '##ants', 'and', 'op', '##io', '##ids', '.', 'similarly', ',', 'the', 'major', 'drug', 'classes', 'associated', 'with', 'use', 'of', 'non', '-', 'main', 'pharmacy', 'were', 'beta', '-', 'la', '##ct', '##ams', ',', 'anti', '##de', '##press', '##ants', 'and', 'inhaled', 'beta', '-', 'ago', '##nist', '##s', '.', 'conclusion', '##base', '##d', 'on', 'this', 'analysis', ',', 'both', 'pre', '##s', '##cribe', '##rs', 'and', 'p', '##har', '##ma', '##cies', 'generally', 'have', 'an', 'equal', 'potential', 'for', 'maintaining', 'an', 'excellent', 'overview', 'of', 'their', 'patients', '’', 'medication', ',', 'but', 'the', 'p', '##har', '##ma', '##cies', 'account', 'for', 'a', 'slightly', 'higher', 'proportion', 'of', 'patients', '.', '[SEP]']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SQEDof7_tlkN"
      },
      "source": [
        "### Pad sequences"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zED2yK7rs9JM"
      },
      "source": [
        "train_input = pad_sequences(train_tokenized_text, maxlen=max_length, dtype=\"object\", truncating=\"post\", padding=\"post\")\n",
        "test_input = pad_sequences(test_tokenized_text, maxlen=max_length, dtype=\"object\", truncating=\"post\", padding=\"post\")\n",
        "\n",
        "for t in train_input:\n",
        "    if t[-1] == 0.0 or t[-1] == ['SEP']:\n",
        "        continue\n",
        "    else:\n",
        "        t[-1] = '[SEP]'\n",
        "for t in test_input:\n",
        "    if t[-1] == 0.0 or t[-1] == ['SEP']:\n",
        "        continue\n",
        "    else:\n",
        "        t[-1] = '[SEP]'\n",
        "\n",
        "train_input_ids = [tokenizer.convert_tokens_to_ids(x) for x in train_input]\n",
        "test_input_ids = [tokenizer.convert_tokens_to_ids(x) for x in test_input]"
      ],
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yARiy7AFtzkM"
      },
      "source": [
        "### Masks"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7ivzfMmxtnmI"
      },
      "source": [
        "train_attention_masks = []\n",
        "test_attention_masks = []\n",
        "\n",
        "for seq in train_input_ids:\n",
        "    seq_mask = [float(i>0) for i in seq]\n",
        "    train_attention_masks.append(seq_mask)\n",
        "    \n",
        "for seq in test_input_ids:\n",
        "    seq_mask = [float(i>0) for i in seq]\n",
        "    test_attention_masks.append(seq_mask)"
      ],
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OVZXBuaFuVjE"
      },
      "source": [
        "### Transform labels"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VTVkssE_t1uW",
        "outputId": "b1668f65-dc18-4345-ae1d-c5d5e5012732",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        }
      },
      "source": [
        "tr_labels = np.zeros((len(train_labels),22))\n",
        "tst_labels = np.zeros((len(test_labels), 22))\n",
        "\n",
        "for i,tr in enumerate(train_labels):\n",
        "    for t in tr:\n",
        "        if int(t) > 0 and int(t) < 23:\n",
        "            tr_labels[i, int(t) - 1] = 1\n",
        "            \n",
        "for i,tr in enumerate(test_labels):\n",
        "    for t in tr:\n",
        "        if int(t) > 0 and int(t) < 23:\n",
        "            tst_labels[i, int(t) - 1] = 1\n",
        "            \n",
        "print(train_labels[:3])\n",
        "print(tr_labels[:3][:])"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[list(['11']) list(['11']) list(['09'])]\n",
            "[[0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
            " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
            " [0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mKeBgdB9utAE"
      },
      "source": [
        "### Create Tensors out of data and masks"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6xY2F_hLuvpo",
        "outputId": "af5f1cd6-0d81-4db8-9bf7-d8c3411eda5b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        }
      },
      "source": [
        "train_inputs = torch.tensor(train_input_ids)\n",
        "train_masks = torch.tensor(train_attention_masks).double()\n",
        "train_labels = torch.tensor(tr_labels).double()\n",
        "\n",
        "test_inputs = torch.tensor(test_input_ids)\n",
        "test_masks = torch.tensor(test_attention_masks).double()\n",
        "test_labels = torch.tensor(tst_labels).double()\n",
        "\n",
        "print(train_masks.dtype)"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.float64\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3vEW3m7PvMPw"
      },
      "source": [
        "batch_size = 4 #8\n",
        "train_data = TensorDataset(train_inputs, train_masks, train_labels)\n",
        "train_sampler = RandomSampler(train_data)\n",
        "train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)\n",
        "\n",
        "test_data = TensorDataset(test_inputs, test_masks, test_labels)\n",
        "test_sampler = RandomSampler(test_data)\n",
        "test_dataloader = DataLoader(test_data, sampler=test_sampler, batch_size=batch_size)"
      ],
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xWC47daawPbZ"
      },
      "source": [
        "## Fine-tune the language model for multilabel classification"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iRtf_O49xCBr"
      },
      "source": [
        "param_optimizer = list(model.named_parameters())\n",
        "no_decay = ['bias', 'gamma', 'beta']\n",
        "optimizer_grouped_parameters = [\n",
        "    {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],\n",
        "     'weight_decay_rate': 0.01},\n",
        "    {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)],\n",
        "     'weight_decay_rate': 0.0}\n",
        "]\n",
        "\n",
        "optimizer = AdamW(optimizer_grouped_parameters, lr=2e-5,)"
      ],
      "execution_count": 36,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "c8JxeJx27NcZ"
      },
      "source": [
        "### Train the model. \n",
        "In colab it takes around 30 minuttes to train and evaluate the model for 5000 articles"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FBRwvN7jxPma",
        "outputId": "d1fd5425-3a73-4ada-e8b2-f22651815509",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 158
        }
      },
      "source": [
        "# Store our loss and accuracy for plotting\n",
        "train_loss_set = []\n",
        "\n",
        "torch.cuda.empty_cache()\n",
        "\n",
        "# Number of training epochs (authors recommend between 2 and 4)\n",
        "epochs = 2\n",
        "start = time.time()\n",
        "# trange is a tqdm wrapper around the normal python range\n",
        "for _ in trange(epochs, desc=\"Epoch\"):\n",
        "    \n",
        "  # Training\n",
        "  \n",
        "  # Set our model to training mode (as opposed to evaluation mode)\n",
        "  model.train()\n",
        "  \n",
        "  # Tracking variables\n",
        "  tr_loss = 0\n",
        "  nb_tr_examples, nb_tr_steps = 0, 0\n",
        "  \n",
        "  # Train the data for one epoch\n",
        "  for step, batch in enumerate(train_dataloader):\n",
        "    # Add batch to GPU\n",
        "    batch = tuple(t.to(device) for t in batch)\n",
        "    # Unpack the inputs from our dataloader\n",
        "    b_input_ids, b_input_mask, b_labels = batch\n",
        "    # Clear out the gradients (by default they accumulate)\n",
        "    optimizer.zero_grad()\n",
        "    # Forward pass\n",
        "    loss = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask, labels=b_labels)\n",
        "    \n",
        "    train_loss_set.append(loss[0].item())    \n",
        "    # Backward pass\n",
        "    loss[0].backward()\n",
        "    # Update parameters and take a step using the computed gradient\n",
        "    optimizer.step()\n",
        "        \n",
        "    # Update tracking variables\n",
        "    tr_loss += loss[0].item()\n",
        "    nb_tr_examples += b_input_ids.size(0)\n",
        "    nb_tr_steps += 1\n",
        "    if step % 10000 == 0:\n",
        "        t = time.time()\n",
        "        print(\"Train loss: {}\".format(tr_loss/nb_tr_steps))\n",
        "        print(\"Time: {}\".format(t - start))\n",
        "\n",
        "  print(\"Train loss: {}\".format(tr_loss/nb_tr_steps))      \n",
        "  # Validation\n",
        "\n",
        "  # Put model in evaluation mode to evaluate loss on the validation set\n",
        "  model.eval()\n",
        "\n",
        "  # Tracking variables \n",
        "  eval_loss, eval_accuracy = 0, 0\n",
        "  nb_eval_steps, nb_eval_examples = 0, 0\n",
        "\n",
        "  # Evaluate data for one epoch\n",
        "  for batch in test_dataloader:\n",
        "    # Add batch to GPU\n",
        "    batch = tuple(t.to(device) for t in batch)\n",
        "    # Unpack the inputs from our dataloader\n",
        "    b_input_ids, b_input_mask, b_labels = batch\n",
        "    # Telling the model not to compute or store gradients, saving memory and speeding up validation\n",
        "    with torch.no_grad():\n",
        "      # Forward pass, calculate logit predictions\n",
        "      logits = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask)\n",
        "    \n",
        "    # Move logits and labels to CPU\n",
        "    logits = logits[0].detach().cpu().numpy()\n",
        "    label_ids = b_labels.to('cpu').numpy()\n",
        "\n",
        "end = time.time()\n",
        "t = end - start\n",
        "print(\"Elapsed time: \", t, \"s\")\n"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\rEpoch:   0%|          | 0/2 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Train loss: 0.7244307033060945\n",
            "Time: 0.48993921279907227\n",
            "Train loss: 0.555086093227894\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\rEpoch:  50%|█████     | 1/2 [00:09<00:09,  9.53s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Train loss: 0.4420627031755309\n",
            "Time: 9.98888611793518\n",
            "Train loss: 0.3852050602316251\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 100%|██████████| 2/2 [00:19<00:00,  9.62s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Elapsed time:  19.238308668136597 s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Qu9D8sDl8Ihf"
      },
      "source": [
        "### [Optional] Save the fine-tuned model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qup4LSrv8Hx-"
      },
      "source": [
        "torch.save(model, \"./Models/BERT_scigraph.pt\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tn30TAZm-h9z"
      },
      "source": [
        "### [Optional] Load the fine-tuned model \n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zeRZcPlO-m_Z"
      },
      "source": [
        "torch.cuda.empty_cache()\n",
        "model = torch.load(\"./Models/BERT_scigraph_test.pt\")\n",
        "model.eval()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gbe-ymi09Fn0"
      },
      "source": [
        "### Eval the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rLNUlFEy9H_-",
        "outputId": "4bb0a098-48c5-488c-c391-8b3cb3ced166",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "\n",
        "preds = []\n",
        "start = time.time()\n",
        "for i in range(int(len(test_inputs)/10)):\n",
        "    batch = (test_inputs[i*10: (i+1)*10].to(device), test_masks[i*10: (i+1)*10].to(device))\n",
        "    with torch.no_grad():\n",
        "        logits = model(batch[0], token_type_ids=None, attention_mask = batch[1])[0]\n",
        "    \n",
        "    logits = logits.detach().cpu().numpy()\n",
        "    for l in logits:\n",
        "        preds.append(l)\n",
        "    if i%1000 == 0:\n",
        "        print(\"Processing: \", i*10/1000, \"%\")\n",
        "end = time.time()\n",
        "print(\"Time: {}\".format(end - start))"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Processing:  0.0 %\n",
            "Time: 0.7535624504089355\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ra65INdq9SoF"
      },
      "source": [
        "sigmoid = torch.nn.Sigmoid()\n",
        "preds = sigmoid(torch.tensor(preds))\n",
        "preds = np.asarray(preds)\n",
        "test_labels = np.asarray(test_labels)"
      ],
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aYL8YJE19vRy",
        "outputId": "df6518fe-4a0c-4884-d1d2-b1b2a95a54f4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 872
        }
      },
      "source": [
        "from sklearn.metrics import classification_report\n",
        "from sklearn.metrics import precision_score\n",
        "from sklearn.metrics import recall_score\n",
        "from sklearn.metrics import f1_score\n",
        "from sklearn.metrics import accuracy_score\n",
        "\n",
        "predictions = np.zeros(preds.shape)\n",
        "predictions[preds >= 0.5] = 1\n",
        "\n",
        "print(classification_report(test_labels[:len(predictions)], predictions, digits=4))\n",
        "precision = precision_score(test_labels[:len(predictions)], predictions, average=\"weighted\")\n",
        "recall = recall_score(test_labels[:len(predictions)], predictions, average=\"weighted\")\n",
        "f1s = f1_score(test_labels[:len(predictions)], predictions, average=\"weighted\")\n",
        "print(\"Precision: %.4f\" % precision)\n",
        "print(\"Recall: %.4f\"% recall)\n",
        "print(\"F1 Score: %.4f\"%f1s)"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0     0.0000    0.0000    0.0000         6\n",
            "           1     0.0000    0.0000    0.0000         0\n",
            "           2     0.0000    0.0000    0.0000         3\n",
            "           3     0.0000    0.0000    0.0000         0\n",
            "           4     0.0000    0.0000    0.0000         1\n",
            "           5     0.0000    0.0000    0.0000         5\n",
            "           6     0.0000    0.0000    0.0000         0\n",
            "           7     0.0000    0.0000    0.0000         1\n",
            "           8     0.0000    0.0000    0.0000         1\n",
            "           9     0.0000    0.0000    0.0000         0\n",
            "          10     0.0000    0.0000    0.0000         4\n",
            "          11     0.0000    0.0000    0.0000         0\n",
            "          12     0.0000    0.0000    0.0000         0\n",
            "          13     0.0000    0.0000    0.0000         1\n",
            "          14     0.0000    0.0000    0.0000         0\n",
            "          15     0.0000    0.0000    0.0000         0\n",
            "          16     0.0000    0.0000    0.0000         0\n",
            "          17     0.0000    0.0000    0.0000         0\n",
            "          18     0.0000    0.0000    0.0000         0\n",
            "          19     0.0000    0.0000    0.0000         0\n",
            "          20     0.0000    0.0000    0.0000         0\n",
            "          21     0.0000    0.0000    0.0000         0\n",
            "\n",
            "   micro avg     0.0000    0.0000    0.0000        22\n",
            "   macro avg     0.0000    0.0000    0.0000        22\n",
            "weighted avg     0.0000    0.0000    0.0000        22\n",
            " samples avg     0.0000    0.0000    0.0000        22\n",
            "\n",
            "Precision: 0.0000\n",
            "Recall: 0.0000\n",
            "F1 Score: 0.0000\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in samples with no predicted labels. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1515: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true nor predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "  average, \"true nor predicted\", 'F-score is', len(true_sum)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CkPJUBlW9zES"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}