SimCLR_Blogpost_zablo_net.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.7"
    },
    "colab": {
      "name": "SimCLR-Blogpost-zablo.net.ipynb",
      "provenance": []
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "3a2b9cb4f7a74d27b8e7ee513cea5bab": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_63a1def182a5493cb8b6bfdb7a615b21",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_672931e8cf7c40cb8218329823311c58",
              "IPY_MODEL_9ba43c03f1964527b1d8e3487abd2fad"
            ]
          }
        },
        "63a1def182a5493cb8b6bfdb7a615b21": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "672931e8cf7c40cb8218329823311c58": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e57c9c510ae24555a615ef43e7bd4ebc",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_923134504bcb4ca1a2e3751857285be2"
          }
        },
        "9ba43c03f1964527b1d8e3487abd2fad": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_1a5ed5e067344488a536d7895d089b6f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 2640404480/? [02:20&lt;00:00, 20702277.44it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_26df9334360a4e9e96a77e8cd4942809"
          }
        },
        "e57c9c510ae24555a615ef43e7bd4ebc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "923134504bcb4ca1a2e3751857285be2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1a5ed5e067344488a536d7895d089b6f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "26df9334360a4e9e96a77e8cd4942809": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "66d6c350dc6b4a57927aef2f4bf3065c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_4b718bdfdedd440a9308a585af05eaaf",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_712bf434e0a541a382cc040a6d2b452b",
              "IPY_MODEL_c1c34ea327024c5cbfcfb147c3b2abe4"
            ]
          }
        },
        "4b718bdfdedd440a9308a585af05eaaf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "712bf434e0a541a382cc040a6d2b452b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_2020dfc1254e42f8867d99877903bea5",
            "_dom_classes": [],
            "description": "Finding best initial lr: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 100,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 100,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_12d98e263dbd4950b82e76d7c28995bd"
          }
        },
        "c1c34ea327024c5cbfcfb147c3b2abe4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f5db69016961498ca526b0f389677491",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 100/100 [02:00&lt;00:00,  1.14s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_bcd3bff210174fbbaaf6459ff704eefe"
          }
        },
        "2020dfc1254e42f8867d99877903bea5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "12d98e263dbd4950b82e76d7c28995bd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f5db69016961498ca526b0f389677491": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "bcd3bff210174fbbaaf6459ff704eefe": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fd93fcfb47284680a4ae25178dd951fa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_8171c55f8f1c49f384d9f38784f22c13",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_3733cce25e7c40cb83c7424036e4f372",
              "IPY_MODEL_c0c72566cfd7404f9e81e6efc397dc8e"
            ]
          }
        },
        "8171c55f8f1c49f384d9f38784f22c13": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": "row wrap",
            "width": "100%",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": "inline-flex",
            "left": null
          }
        },
        "3733cce25e7c40cb83c7424036e4f372": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_eee8913da3b04fcaa27093426b25fd38",
            "_dom_classes": [],
            "description": "Validation sanity check: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_27cfa39fbd6441548cf77d3e8735485d"
          }
        },
        "c0c72566cfd7404f9e81e6efc397dc8e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ba588cacf6c94025892f5257e83322cd",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 2/2 [00:07&lt;00:00,  4.95s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_53d6d9ee2c2f493a8bfe5269490736d5"
          }
        },
        "eee8913da3b04fcaa27093426b25fd38": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "27cfa39fbd6441548cf77d3e8735485d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": "2",
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ba588cacf6c94025892f5257e83322cd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "53d6d9ee2c2f493a8bfe5269490736d5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3974604f19b840b2a09ebd284495ed53": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_117c99eccbe94c78b630849a9db450c2",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_e3917ebbc51e487481f66c8df68316ce",
              "IPY_MODEL_33ba2cf3460542f4a632e05696351dee"
            ]
          }
        },
        "117c99eccbe94c78b630849a9db450c2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": "row wrap",
            "width": "100%",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": "inline-flex",
            "left": null
          }
        },
        "e3917ebbc51e487481f66c8df68316ce": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_01c7832e21554cfda7c0b759a59f4063",
            "_dom_classes": [],
            "description": "Epoch 3:  76%",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 131,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 99,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e647d0aeeba648d5860ba2c68ad37ba1"
          }
        },
        "33ba2cf3460542f4a632e05696351dee": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_fe9475251c4042aca5c598b33549aa40",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 99/131 [01:54&lt;00:37,  1.16s/it, loss=4.235, v_num=0, train_loss=4.23]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_85d7b051ac3446d9b72a98c4c331599d"
          }
        },
        "01c7832e21554cfda7c0b759a59f4063": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e647d0aeeba648d5860ba2c68ad37ba1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": "2",
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fe9475251c4042aca5c598b33549aa40": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "85d7b051ac3446d9b72a98c4c331599d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e0bd1cbc38b34d2cb633966565325b7d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_6b37cdaa12c74de6b86fce96c838f381",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_e3c18fc7317d44d2944876185d52127a",
              "IPY_MODEL_964d4acb696540afb93260770ae6ce2c"
            ]
          }
        },
        "6b37cdaa12c74de6b86fce96c838f381": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": "row wrap",
            "width": "100%",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": "inline-flex",
            "left": null
          }
        },
        "e3c18fc7317d44d2944876185d52127a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_11759fc12080455094624b153eb35dce",
            "_dom_classes": [],
            "description": "Validating: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_678f7c3bb42549d884e3b8995dd47e81"
          }
        },
        "964d4acb696540afb93260770ae6ce2c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_1c1c1eba0d5e436ba66de315449294e3",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 6/6 [00:04&lt;00:00,  1.62it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b2db4f9b5ecd437586c1153c3e21e661"
          }
        },
        "11759fc12080455094624b153eb35dce": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "678f7c3bb42549d884e3b8995dd47e81": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": "2",
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1c1c1eba0d5e436ba66de315449294e3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b2db4f9b5ecd437586c1153c3e21e661": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "313feecc97084ace93710d8e11237d06": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_e611467f4c544611aade87ee71d60914",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_2d139b573db54e6f90b5e545de34f28d",
              "IPY_MODEL_a8a0c1847cec4eef98264aea6b42308c"
            ]
          }
        },
        "e611467f4c544611aade87ee71d60914": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": "row wrap",
            "width": "100%",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": "inline-flex",
            "left": null
          }
        },
        "2d139b573db54e6f90b5e545de34f28d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_060fa39300cc4a488d846900165600ad",
            "_dom_classes": [],
            "description": "Validating: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_115e638d13d64a2ba4cd899451a1c005"
          }
        },
        "a8a0c1847cec4eef98264aea6b42308c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_b88870fb6ffe483fa6fc02d5e729949e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 6/6 [00:04&lt;00:00,  1.62it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_69ba8210c0f747fe9c0b3c8df8f5ca26"
          }
        },
        "060fa39300cc4a488d846900165600ad": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "115e638d13d64a2ba4cd899451a1c005": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": "2",
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b88870fb6ffe483fa6fc02d5e729949e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "69ba8210c0f747fe9c0b3c8df8f5ca26": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-Z-QxFAEGdhQ",
        "colab_type": "text"
      },
      "source": [
        "# SimCLR\n",
        "THIS IS A COLAB VERSION OF MY BLOGPOST\n",
        "\n",
        "https://zablo.net/blog/post/understanding-implementing-simclr-guide-eli5-pytorch/\n",
        "\n",
        "---\n",
        "---\n",
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NgIgle3ZGY7u",
        "colab_type": "text"
      },
      "source": [
        "Transfer learning and pre-training schemas for both NLP and Computer Vision have gained a lot of attention in the last months. Research showed that carefully designed unsupervised/self-supervised training can produce high quality base models and embeddings that greatly decrease the amount of data needed to obtain good classfication models downstream. This approach becomes more and more important as the companies collect a lot of data from which only fraction can be labelled by human - either due to large cost of labelling process or due to some time constraints.\n",
        "\n",
        "Here I explore SimCLR pre-training framework proposed by Google in <a href=\"/links/simclr-arxiv-2002.05709/?utm_source=blog&utm_medium=link&utm_campaign=external_links&utm_content=SimCLR-arxiv\" target=\"blank\">this arxiv paper</a>. I will explain the SimCLR and its contrastive loss function step by step, starting from naive implementation followed by faster, vectorized one. Then I will show how to use SimCLR's pretraining routine to first build image embeddings using EfficientNet network architecture and finally I will show how to build classifier on top of it.\n",
        "\n",
        "## TL;DR\n",
        "This post covers:\n",
        "* understanding the SimCLR framework\n",
        "* from scratch explanation & implementation of SimCLR's loss function (NT-Xent) in PyTorch\n",
        "* pre-training image embeddings using EfficientNet architecture\n",
        "* training classifier by using transfer learning from the pre-trained embeddings"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "weeaWOYnGY7v",
        "colab_type": "text"
      },
      "source": [
        "## Understanding SimCLR framework\n",
        "In general, SimCLR is <i>a simple framework for contrastive learning of visual representations</i>. It's not any new framework for deep learning, it's a set of fixed steps that one should to follow in order to train image embeddings of good quality.\n",
        "I drew a schema which explains the flow and the whole representation learning process."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n7zDz74LGY7w",
        "colab_type": "text"
      },
      "source": [
        "<img src=\"simclr-schema.png\"/>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "imA2CU2MGY7w",
        "colab_type": "text"
      },
      "source": [
        "The flow is as follows (from left to right):\n",
        "1. Take an input image\n",
        "2. Prepare 2 random augumentations on the image, including: rotations, hue/saturation/brightness changes, zooming, cropping, etc. The range of augumentations as well as analysis which ones works best are discussed in details in the parper.\n",
        "1. Run deep neural network (preferably convolutional one, like ResNet50) to obtain image representations (embeddings) for those augumented images.\n",
        "1. Run small fully connected linear neural network to project embeddings into another vector space.\n",
        "1. Calculate contrastive loss and run backpropagation throught both networks. Contrastive loss decreases when projections coming from the same image are similar. The similarity between projections can be arbitrary, here I will use cosine similarity, same as in the paper."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "B4H_1kbWGY7x",
        "colab_type": "text"
      },
      "source": [
        "## Contrastive loss function\n",
        "### Theory behind constrastive loss function\n",
        "One can reason about contrastive loss function form two angles:\n",
        "1. Contrastive loss decreases when projections of augumented images coming from the same input image are similar.\n",
        "1. For two augumented images: (i), (j) (coming from the same input image - I will call them \"positive\" pair later on), the constrative loss for (i) tries to identify (j) among other images (\"negative\" examples) that are in the same batch .\n",
        "\n",
        "Formal definition of the loss for pair of positive exmaples (i) and (j) defined as:\n",
        "<img src=\"contrastive-loss-lij.png\"/>\n",
        "\n",
        "The final loss is an arithmetic mean of the losses for all positive pairs in the batch:\n",
        "<img src=\"contrastive-loss-sum.png\"/>\n",
        "(keep in mind that the indexing in <i>l(2k-1, 2k) + l(2k, 2k-1)</i> is purely dependand on how you implement the loss - I find it easier to understand when I reason about them as <i>l(i,j) + l(j,i)</i>)."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rUbq1ShUGY7x",
        "colab_type": "text"
      },
      "source": [
        "### Contrastive loss function - implementation in PyTorch, ELI5 version\n",
        "It's much easier to implement the loss function without vectorization first and then follow up with the vectorization phase."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mWSdMw3MRHXH",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "eb2f750e-5fd4-4d9a-90e6-7a4e4d4ad1d4"
      },
      "source": [
        "!pip install efficientnet_pytorch pytorch-lightning"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting efficientnet_pytorch\n",
            "  Downloading https://files.pythonhosted.org/packages/b8/cb/0309a6e3d404862ae4bc017f89645cf150ac94c14c88ef81d215c8e52925/efficientnet_pytorch-0.6.3.tar.gz\n",
            "Collecting pytorch-lightning\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/32/64/65a5bd6b0c286217f2e53bb067c4099c0584a8eff1d229046b9a35ae3e26/pytorch_lightning-0.8.5-py3-none-any.whl (313kB)\n",
            "\u001b[K     |████████████████████████████████| 317kB 9.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: torch in /usr/local/lib/python3.6/dist-packages (from efficientnet_pytorch) (1.5.1+cu101)\n",
            "Requirement already satisfied: tqdm>=4.41.0 in /usr/local/lib/python3.6/dist-packages (from pytorch-lightning) (4.41.1)\n",
            "Collecting PyYAML>=5.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/64/c2/b80047c7ac2478f9501676c988a5411ed5572f35d1beff9cae07d321512c/PyYAML-5.3.1.tar.gz (269kB)\n",
            "\u001b[K     |████████████████████████████████| 276kB 29.4MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.16.4 in /usr/local/lib/python3.6/dist-packages (from pytorch-lightning) (1.18.5)\n",
            "Collecting future>=0.17.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/45/0b/38b06fd9b92dc2b68d58b75f900e97884c45bedd2ff83203d933cf5851c9/future-0.18.2.tar.gz (829kB)\n",
            "\u001b[K     |████████████████████████████████| 829kB 40.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: tensorboard>=1.14 in /usr/local/lib/python3.6/dist-packages (from pytorch-lightning) (2.2.2)\n",
            "Requirement already satisfied: absl-py>=0.4 in /usr/local/lib/python3.6/dist-packages (from tensorboard>=1.14->pytorch-lightning) (0.9.0)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tensorboard>=1.14->pytorch-lightning) (1.0.1)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tensorboard>=1.14->pytorch-lightning) (3.2.2)\n",
            "Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorboard>=1.14->pytorch-lightning) (1.15.0)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.6/dist-packages (from tensorboard>=1.14->pytorch-lightning) (0.4.1)\n",
            "Requirement already satisfied: google-auth<2,>=1.6.3 in /usr/local/lib/python3.6/dist-packages (from tensorboard>=1.14->pytorch-lightning) (1.17.2)\n",
            "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.6/dist-packages (from tensorboard>=1.14->pytorch-lightning) (2.23.0)\n",
            "Requirement already satisfied: protobuf>=3.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorboard>=1.14->pytorch-lightning) (3.12.2)\n",
            "Requirement already satisfied: grpcio>=1.24.3 in /usr/local/lib/python3.6/dist-packages (from tensorboard>=1.14->pytorch-lightning) (1.30.0)\n",
            "Requirement already satisfied: wheel>=0.26; python_version >= \"3\" in /usr/local/lib/python3.6/dist-packages (from tensorboard>=1.14->pytorch-lightning) (0.34.2)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorboard>=1.14->pytorch-lightning) (1.7.0)\n",
            "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.6/dist-packages (from tensorboard>=1.14->pytorch-lightning) (49.1.0)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from markdown>=2.6.8->tensorboard>=1.14->pytorch-lightning) (1.7.0)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard>=1.14->pytorch-lightning) (1.3.0)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tensorboard>=1.14->pytorch-lightning) (4.1.1)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tensorboard>=1.14->pytorch-lightning) (0.2.8)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4; python_version >= \"3\" in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tensorboard>=1.14->pytorch-lightning) (4.6)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tensorboard>=1.14->pytorch-lightning) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tensorboard>=1.14->pytorch-lightning) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tensorboard>=1.14->pytorch-lightning) (2020.6.20)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tensorboard>=1.14->pytorch-lightning) (1.24.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.6/dist-packages (from importlib-metadata; python_version < \"3.8\"->markdown>=2.6.8->tensorboard>=1.14->pytorch-lightning) (3.1.0)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.6/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard>=1.14->pytorch-lightning) (3.1.0)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.6/dist-packages (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.3->tensorboard>=1.14->pytorch-lightning) (0.4.8)\n",
            "Building wheels for collected packages: efficientnet-pytorch, PyYAML, future\n",
            "  Building wheel for efficientnet-pytorch (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for efficientnet-pytorch: filename=efficientnet_pytorch-0.6.3-cp36-none-any.whl size=12422 sha256=7ee22664ff29e784e2d7f77058382b98b781a851fb65db5d177e6f843c88a8be\n",
            "  Stored in directory: /root/.cache/pip/wheels/42/1e/a9/2a578ba9ad04e776e80bf0f70d8a7f4c29ec0718b92d8f6ccd\n",
            "  Building wheel for PyYAML (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for PyYAML: filename=PyYAML-5.3.1-cp36-cp36m-linux_x86_64.whl size=44621 sha256=346524f8b67b62007b66e82fca6049a1662549dcb39d88b5ffbec99dad0e7aa8\n",
            "  Stored in directory: /root/.cache/pip/wheels/a7/c1/ea/cf5bd31012e735dc1dfea3131a2d5eae7978b251083d6247bd\n",
            "  Building wheel for future (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for future: filename=future-0.18.2-cp36-none-any.whl size=491057 sha256=64e3348d68827f6fde333ddb3c0461eb5afc9e099620ed5915ec76d84e4c1211\n",
            "  Stored in directory: /root/.cache/pip/wheels/8b/99/a0/81daf51dcd359a9377b110a8a886b3895921802d2fc1b2397e\n",
            "Successfully built efficientnet-pytorch PyYAML future\n",
            "Installing collected packages: efficientnet-pytorch, PyYAML, future, pytorch-lightning\n",
            "  Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "  Found existing installation: future 0.16.0\n",
            "    Uninstalling future-0.16.0:\n",
            "      Successfully uninstalled future-0.16.0\n",
            "Successfully installed PyYAML-5.3.1 efficientnet-pytorch-0.6.3 future-0.18.2 pytorch-lightning-0.8.5\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H2wMlyz0GY7y",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch\n",
        "from torch import nn\n",
        "import torch.nn.functional as F"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E4excQa6GY72",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class ContrastiveLossELI5(nn.Module):\n",
        "    def __init__(self, batch_size, temperature=0.5, verbose=True):\n",
        "        super().__init__()\n",
        "        self.batch_size = batch_size\n",
        "        self.register_buffer(\"temperature\", torch.tensor(temperature))\n",
        "        self.verbose = verbose\n",
        "            \n",
        "    def forward(self, emb_i, emb_j):\n",
        "        \"\"\"\n",
        "        emb_i and emb_j are batches of embeddings, where corresponding indices are pairs\n",
        "        z_i, z_j as per SimCLR paper\n",
        "        \"\"\"\n",
        "        z_i = F.normalize(emb_i, dim=1)\n",
        "        z_j = F.normalize(emb_j, dim=1)\n",
        "\n",
        "        representations = torch.cat([z_i, z_j], dim=0)\n",
        "        similarity_matrix = F.cosine_similarity(representations.unsqueeze(1), representations.unsqueeze(0), dim=2)\n",
        "        if self.verbose: print(\"Similarity matrix\\n\", similarity_matrix, \"\\n\")\n",
        "            \n",
        "        def l_ij(i, j):\n",
        "            z_i_, z_j_ = representations[i], representations[j]\n",
        "            sim_i_j = similarity_matrix[i, j]\n",
        "            if self.verbose: print(f\"sim({i}, {j})={sim_i_j}\")\n",
        "                \n",
        "            numerator = torch.exp(sim_i_j / self.temperature)\n",
        "            one_for_not_i = torch.ones((2 * self.batch_size, )).to(emb_i.device).scatter_(0, torch.tensor([i]), 0.0)\n",
        "            if self.verbose: print(f\"1{{k!={i}}}\",one_for_not_i)\n",
        "            \n",
        "            denominator = torch.sum(\n",
        "                one_for_not_i * torch.exp(similarity_matrix[i, :] / self.temperature)\n",
        "            )    \n",
        "            if self.verbose: print(\"Denominator\", denominator)\n",
        "                \n",
        "            loss_ij = -torch.log(numerator / denominator)\n",
        "            if self.verbose: print(f\"loss({i},{j})={loss_ij}\\n\")\n",
        "                \n",
        "            return loss_ij.squeeze(0)\n",
        "\n",
        "        N = self.batch_size\n",
        "        loss = 0.0\n",
        "        for k in range(0, N):\n",
        "            loss += l_ij(k, k + N) + l_ij(k + N, k)\n",
        "        return 1.0 / (2*N) * loss"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CIjJ0gTUGY74",
        "colab_type": "text"
      },
      "source": [
        "#### Explanation\n",
        "Contrastive loss needs to know the batch size and `temperature` (scaling) parameter. You can find details about setting the optimal `temperature` parameter in the paper.\n",
        "\n",
        "My implementation of the `forward` of the contrastive loss takes two parameters. First one will be a batch projections of images after first augumentation, the second will be a batch projections of images after second augumentaion.\n",
        "\n",
        "Projections needs to be normalized first, hence:\n",
        "```python\n",
        "z_i = F.normalize(emb_i, dim=1)\n",
        "z_j = F.normalize(emb_j, dim=1)\n",
        "```\n",
        "\n",
        "All representations are concatenated together in order to efficiently calculate cosine similarities between each image pair.\n",
        "\n",
        "```python\n",
        "representations = torch.cat([z_i, z_j], dim=0)\n",
        "similarity_matrix = F.cosine_similarity(representations.unsqueeze(1), representations.unsqueeze(0), dim=2)\n",
        "```\n",
        "\n",
        "Next is the naive implementation of <i>l(i,j)</i> for clarity and easiness of understanding. The code bellow almost directly implements the equation:\n",
        "<img src=\"contrastive-loss-lij.png\"/>\n",
        "```python\n",
        "def l_ij(i, j):\n",
        "    z_i_, z_j_ = representations[i], representations[j]\n",
        "    sim_i_j = similarity_matrix[i, j]\n",
        "    numerator = torch.exp(sim_i_j / self.temperature)\n",
        "    one_for_not_i = torch.ones((2 * self.batch_size, )).to(emb_i.device).scatter_(0, torch.tensor([i]), 0.0)\n",
        "    denominator = torch.sum(\n",
        "        one_for_not_i * torch.exp(similarity_matrix[i, :] / self.temperature)\n",
        "    )\n",
        "\n",
        "    loss_ij = -torch.log(numerator / denominator)\n",
        "    return loss_ij.squeeze(0)\n",
        "```\n",
        "\n",
        "Then, the final loss for the batch is computed arithmetic mean of all combinations of positive examples:\n",
        "```python\n",
        "N = self.batch_size\n",
        "loss = 0.0\n",
        "for k in range(0, N):\n",
        "    loss += l_ij(k, k + N) + l_ij(k + N, k)\n",
        "return 1.0 / (2*N) * loss\n",
        "```"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "detEu6B0GY75",
        "colab_type": "text"
      },
      "source": [
        "Now, let's run it with `verbose` mode to see what's inside."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CZPX-QDFGY75",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 680
        },
        "outputId": "c4a00eaa-c0a8-400e-8f6f-a288301ad416"
      },
      "source": [
        "I = torch.tensor([[1.0, 2.0], [3.0, -2.0], [1.0, 5.0]])\n",
        "J = torch.tensor([[1.0, 0.75], [2.8, -1.75], [1.0, 4.7]])\n",
        "loss_eli5 = ContrastiveLossELI5(batch_size=3, temperature=1.0, verbose=True)\n",
        "loss_eli5(I, J)"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Similarity matrix\n",
            " tensor([[ 1.0000, -0.1240,  0.9648,  0.8944, -0.0948,  0.9679],\n",
            "        [-0.1240,  1.0000, -0.3807,  0.3328,  0.9996, -0.3694],\n",
            "        [ 0.9648, -0.3807,  1.0000,  0.7452, -0.3534,  0.9999],\n",
            "        [ 0.8944,  0.3328,  0.7452,  1.0000,  0.3604,  0.7533],\n",
            "        [-0.0948,  0.9996, -0.3534,  0.3604,  1.0000, -0.3419],\n",
            "        [ 0.9679, -0.3694,  0.9999,  0.7533, -0.3419,  1.0000]]) \n",
            "\n",
            "sim(0, 3)=0.8944272398948669\n",
            "1{k!=0} tensor([0., 1., 1., 1., 1., 1.])\n",
            "Denominator tensor(9.4954)\n",
            "loss(0,3)=1.3563847541809082\n",
            "\n",
            "sim(3, 0)=0.8944272398948669\n",
            "1{k!=3} tensor([1., 1., 1., 0., 1., 1.])\n",
            "Denominator tensor(9.5058)\n",
            "loss(3,0)=1.357473373413086\n",
            "\n",
            "sim(1, 4)=0.9995677471160889\n",
            "1{k!=1} tensor([1., 0., 1., 1., 1., 1.])\n",
            "Denominator tensor(6.3699)\n",
            "loss(1,4)=0.8520082831382751\n",
            "\n",
            "sim(4, 1)=0.9995677471160889\n",
            "1{k!=4} tensor([1., 1., 1., 1., 0., 1.])\n",
            "Denominator tensor(6.4733)\n",
            "loss(4,1)=0.8681114912033081\n",
            "\n",
            "sim(2, 5)=0.9999250769615173\n",
            "1{k!=2} tensor([1., 1., 0., 1., 1., 1.])\n",
            "Denominator tensor(8.8348)\n",
            "loss(2,5)=1.1787779331207275\n",
            "\n",
            "sim(5, 2)=0.9999250769615173\n",
            "1{k!=5} tensor([1., 1., 1., 1., 1., 0.])\n",
            "Denominator tensor(8.8762)\n",
            "loss(5,2)=1.1834464073181152\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor(1.1327)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ocSFdks5GY79",
        "colab_type": "text"
      },
      "source": [
        "A few things happened there, but by going back and forward between the verbose logs and the equation, everything should become clear.\n",
        "The indexing jumps by batch size (first `l(0,3), l(3,0)` then `l(1,4), l(4,1)` because the way the similarity matrix was constructed. First row of the `similarity_matrix` is:\n",
        "```python\n",
        "[ 1.0000, -0.1240,  0.9648,  0.8944, -0.0948,  0.9679]\n",
        "```\n",
        "\n",
        "Remember the input:\n",
        "```python\n",
        "I = torch.tensor([[1.0, 2.0], [3.0, -2.0], [1.0, 5.0]])\n",
        "J = torch.tensor([[1.0, 0.75], [2.8, -1.75], [1.0, 4.7]])\n",
        "```\n",
        "\n",
        "Now:\n",
        "\n",
        "`1.0000` is the cosine similarity between `I[0] and I[0]` (`[1.0, 2.0] and [1.0, 2.0]`)\n",
        "\n",
        "`-0.1240` is the cosine similarity between `I[0] and I[1]` (`[1.0, 2.0] and [3.0, -2.0]`)\n",
        "\n",
        "`-0.0948` is the cosine similarity between `I[0] and J[2]` (`[1.0, 2.0] and [2.8, -1.75]`)\n",
        "\n",
        "... and so on."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "z5RDz3D-GY7-",
        "colab_type": "text"
      },
      "source": [
        "Let's see if the loss decreases if the similarity between first image projection increases:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cuIyNvkvGY7-",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "0ee4fccb-0283-402d-cd95-28ae7523939f"
      },
      "source": [
        "I = torch.tensor([[1.0, 2.0], [3.0, -2.0], [1.0, 5.0]])\n",
        "J = torch.tensor([[1.0, 0.75], [2.8, -1.75], [1.0, 4.7]])\n",
        "J = torch.tensor([[1.0, 1.75], [2.8, -1.75], [1.0, 4.7]]) # note the change\n",
        "ContrastiveLossELI5(3, 1.0, verbose=False)(I, J)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor(1.0996)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "79ahoOxjGY8B",
        "colab_type": "text"
      },
      "source": [
        "Indeed the loss decreases! Now I will follow up with the vectorized implementation.\n",
        "\n",
        "### Contrastive loss function - implementation in PyTorch, vectorized version\n",
        "Performance of naive implementation is really poor (mostly due to manual loop), see the results:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2BLy2u6GGY8C",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "contrastive_loss_eli5 = ContrastiveLossELI5(3, 1.0, verbose=False)\n",
        "I = torch.tensor([[1.0, 2.0], [3.0, -2.0], [1.0, 5.0]], requires_grad=True)\n",
        "J = torch.tensor([[1.0, 0.75], [2.8, -1.75], [1.0, 4.7]], requires_grad=True)"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d6aJqJziGY8G",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "9d87f501-f1a9-4887-b60c-f2b9a82864be"
      },
      "source": [
        "%%timeit\n",
        "contrastive_loss_eli5(I, J)"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The slowest run took 8.30 times longer than the fastest. This could mean that an intermediate result is being cached.\n",
            "1000 loops, best of 3: 1.09 ms per loop\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tZNKfUKKGY8J",
        "colab_type": "text"
      },
      "source": [
        "Once I understood the internals of the loss, it's easy to vectorize it and remove the manual loop:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1160mtuhGY8J",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class ContrastiveLoss(nn.Module):\n",
        "    def __init__(self, batch_size, temperature=0.5):\n",
        "        super().__init__()\n",
        "        self.batch_size = batch_size\n",
        "        self.register_buffer(\"temperature\", torch.tensor(temperature))\n",
        "        self.register_buffer(\"negatives_mask\", (~torch.eye(batch_size * 2, batch_size * 2, dtype=bool)).float())\n",
        "            \n",
        "    def forward(self, emb_i, emb_j):\n",
        "        \"\"\"\n",
        "        emb_i and emb_j are batches of embeddings, where corresponding indices are pairs\n",
        "        z_i, z_j as per SimCLR paper\n",
        "        \"\"\"\n",
        "        z_i = F.normalize(emb_i, dim=1)\n",
        "        z_j = F.normalize(emb_j, dim=1)\n",
        "\n",
        "        representations = torch.cat([z_i, z_j], dim=0)\n",
        "        similarity_matrix = F.cosine_similarity(representations.unsqueeze(1), representations.unsqueeze(0), dim=2)\n",
        "        \n",
        "        sim_ij = torch.diag(similarity_matrix, self.batch_size)\n",
        "        sim_ji = torch.diag(similarity_matrix, -self.batch_size)\n",
        "        positives = torch.cat([sim_ij, sim_ji], dim=0)\n",
        "        \n",
        "        nominator = torch.exp(positives / self.temperature)\n",
        "        denominator = self.negatives_mask * torch.exp(similarity_matrix / self.temperature)\n",
        "    \n",
        "        loss_partial = -torch.log(nominator / torch.sum(denominator, dim=1))\n",
        "        loss = torch.sum(loss_partial) / (2 * self.batch_size)\n",
        "        return loss"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AZ0TKBpCGY8M",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "c6f09e93-c71e-46a4-8d3e-b8df53a68608"
      },
      "source": [
        "contrastive_loss = ContrastiveLoss(3, 1.0)\n",
        "contrastive_loss(I, J).item() - contrastive_loss_eli5(I, J).item()"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "-1.1920928955078125e-07"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "t83pYD0rGY8O",
        "colab_type": "text"
      },
      "source": [
        "The difference should be zero or close to zero (< 1e-6 due to fp arithmetics). Performance comparison:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wtzpmLSNGY8P",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "I = torch.tensor([[1.0, 2.0], [3.0, -2.0], [1.0, 5.0]], requires_grad=True)\n",
        "J = torch.tensor([[1.0, 0.75], [2.8, -1.75], [1.0, 4.7]], requires_grad=True)"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fis8WxSCGY8R",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "c16868d2-1da6-4f3f-a64f-98f013b48cc0"
      },
      "source": [
        "%%timeit\n",
        "contrastive_loss_eli5(I, J)"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1000 loops, best of 3: 1.09 ms per loop\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fcwDRVckGY8U",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "194acfa4-61e6-4a3a-f486-95d862392190"
      },
      "source": [
        "%%timeit\n",
        "contrastive_loss(I, J)"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The slowest run took 5.86 times longer than the fastest. This could mean that an intermediate result is being cached.\n",
            "1000 loops, best of 3: 325 µs per loop\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WYmbLxIyGY8W",
        "colab_type": "text"
      },
      "source": [
        "Almost 4x improvement, it works."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Hp29gFYIGY8W",
        "colab_type": "text"
      },
      "source": [
        "## Pre-training image embeddings using SimCLR with EfficientNet\n",
        "Once the loss function is established and understood it's time to make a good use of it. I will pre-train image embeddings using EfficientNet architecture, following the SimCLR framework. For convenenience, I've implemented a few utility functions and classes that I will explain briefly below. The training code is structured using PyTorch-Lightning.\n",
        "\n",
        "I will use a great EfficientNet [<a href=\"https://arxiv.org/abs/1905.11946\" target=\"blank\">https://arxiv.org/abs/1905.11946</a>] implementation by <a href=\"https://github.com/lukemelas/EfficientNet-PyTorch.git\">Luke Melas-Kyriazi from GitHub</a>, already pre-trained on ImageNet (transfer learning inception!). The dataset I choose is <a href=\"https://ai.stanford.edu/~acoates/stl10/\">STL10</a> (from `torchvision`) as it contains both training and unlabeled splits for unsupervised / self-supervised learning tasks.\n",
        "\n",
        "> My goal here is to demonstrate the whole SimCLR flow from start to the end. I had no intent to reach new SOTA with the presented configuration."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9vAYNZVCGY8W",
        "colab_type": "text"
      },
      "source": [
        "### Utility functions for image augumentations\n",
        "Training with SimCLR produces good image embeddings that are not affected by image transformations - this is because during training, various data augumentations are done to force the network to understand the contents of the images regardless of i.e color of the image or the position where the object in the image is placed.\n",
        "SimCLR's authors say that *composition of data augmentations plays a critical role in defining effective predictive tasks* and also *contrastive learning needs stronger data augmentation than supervised learning*. To sum this up: when pre-training the image embeddings it's good to make this task difficult for the network to learn by strongly augumenting the images in order to generalize better afterwards.\n",
        "\n",
        "> I strongly advise to read the SimCLR's paper and it's appendix as they did ablation studies on which data augumentations bring the best effects on the embeddings.\n",
        "\n",
        "To keep this blog-post simple to go through, I will mostly use built-in Torchvision's data augumentations , with one additional one - random resized rotation."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C4eG-aIqGY8X",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def random_rotate(image):\n",
        "    if random.random() > 0.5:\n",
        "        return tvf.rotate(image, angle=random.choice((0, 90, 180, 270)))\n",
        "    return image"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pH9uNzICGY8Z",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class ResizedRotation():\n",
        "    def __init__(self, angle, output_size=(96, 96)):\n",
        "        self.angle = angle\n",
        "        self.output_size = output_size\n",
        "        \n",
        "    def angle_to_rad(self, ang): return np.pi * ang / 180.0\n",
        "        \n",
        "    def __call__(self, image):\n",
        "        w, h = image.size\n",
        "        new_h = int(np.abs(w * np.sin(self.angle_to_rad(90 - self.angle))) + np.abs(h * np.sin(self.angle_to_rad(self.angle))))\n",
        "        new_w = int(np.abs(h * np.sin(self.angle_to_rad(90 - self.angle))) + np.abs(w * np.sin(self.angle_to_rad(self.angle))))\n",
        "        img = tvf.resize(image, (new_w, new_h))\n",
        "        img = tvf.rotate(img, self.angle)\n",
        "        img = tvf.center_crop(img, self.output_size)\n",
        "        return img"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zKu4pn5uGY8b",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class WrapWithRandomParams():\n",
        "    def __init__(self, constructor, ranges):\n",
        "        self.constructor = constructor\n",
        "        self.ranges = ranges\n",
        "    \n",
        "    def __call__(self, image):\n",
        "        randoms = [float(np.random.uniform(low, high)) for _, (low, high) in zip(range(len(self.ranges)), self.ranges)]\n",
        "        return self.constructor(*randoms)(image)\n",
        "            "
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bJFKQQbXGY8d",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from torchvision.datasets import STL10\n",
        "import torchvision.transforms.functional as tvf\n",
        "from torchvision import transforms\n",
        "import numpy as np"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yNJ29JjBGY8g",
        "colab_type": "text"
      },
      "source": [
        "A brief look on the transformation results:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uFYBs12XGY8g",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 83,
          "referenced_widgets": [
            "3a2b9cb4f7a74d27b8e7ee513cea5bab",
            "63a1def182a5493cb8b6bfdb7a615b21",
            "672931e8cf7c40cb8218329823311c58",
            "9ba43c03f1964527b1d8e3487abd2fad",
            "e57c9c510ae24555a615ef43e7bd4ebc",
            "923134504bcb4ca1a2e3751857285be2",
            "1a5ed5e067344488a536d7895d089b6f",
            "26df9334360a4e9e96a77e8cd4942809"
          ]
        },
        "outputId": "dd0b5680-e02b-494e-a662-4dc69735d289"
      },
      "source": [
        "stl10_unlabeled = STL10(\".\", split=\"unlabeled\", download=True)"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading http://ai.stanford.edu/~acoates/stl10/stl10_binary.tar.gz to ./stl10_binary.tar.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3a2b9cb4f7a74d27b8e7ee513cea5bab",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Extracting ./stl10_binary.tar.gz to .\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SRTD7rt8GY8i",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "idx = 123"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Uc0VHCyvGY8k",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "random_resized_rotation = WrapWithRandomParams(lambda angle: ResizedRotation(angle), [(0.0, 360.0)])"
      ],
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Oo2VeboVGY8l",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 113
        },
        "outputId": "313df649-14ad-4b4f-be1c-caa705e41df0"
      },
      "source": [
        "random_resized_rotation(tvf.resize(stl10_unlabeled[idx][0], (96, 96)))"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAGAAAABgCAIAAABt+uBvAABAL0lEQVR4nE39Wa9ly5Yeho0uIuZc3W6yO93t7z1UlapIgWSJtCEbfvGDZejJgAX/JL/JT/orAgzDFmjIIgk2pqhi1S1W3ao697SZuZu11mwiYjR6mPuUvTKRSOwF7LVmzIjxjfGN7xsTAeC//W//mwBAiO3l7qZq2lVVXcPDAxGDiISZiBAAEczNzNalPZ+Xh+f6/uP6tz/ENw84ze46oz0JPo9F93spA+cinLKMZb+nt+Xyq/Hp18Pls1J3LOGnpb157p+c++00e0yM18WXp6WdP/j0XbQPYQuBuaPwMI7H0xEiiDgPAyISgKsKS86ZmKyvZl2S3Lz+7NW7L25fvX315t3pdJeHkYg8QNXmeblel/PUAOD7b99/9/X7r755/Nvff3z/w8M6rRHQzv8d/PgSAAhwhMAAgAiACHf38EAARAIMQQAiZhFmJEQAAAgAjwDAAIoAdexO6uAQiIFgYD20mar2RBSAhCYAJAyFYRCQRELkykiCyIEpE3sgkxJyEAsJgVMQY4CbJB5KYWZm3u4WsxBEByREBCAIC8VwQUpMDJBEEgsTYUCYu5u1quvVW/VuP3z3/bffPnz19eWrrx8+fnhqaycWxMCb/zKAAALQBX5cFgOMiHAzUze1MAdHIKAgACBmJmJCgEAAAApmd0AMoHB0R3P0QIQAREQA8HBzA3dyE4ugCAgDcMYgQkIMAPdQg2pQ1SGGCMtm6BroitiAHIAZ0TAlTkmEJaXETISYElEoOGIAgYc7ASSWLEwB0VcMB/AIN22mvbXa1nmZrrX26BS6oFfGzmiJg0sIRwCcJyIkYSBAAQAijgiMAAD3CPOI8IAIBDCAcKBEREgQEQgEgACGEAAR4O7m0R3MMWK7akAAhG0xAyECI7bdGQHhAQah4eaI6t7dqhoAtKbUm/XFrS7YZ/RGGCTC5IQiIsSCmBgzIzNl7mwViAmcIMCVCDOjiCdbYH5o090yFHADCK1rXy5tmdd1ra15ugFbBOpelrux5hsjiCQQiH/rwIhJFBHkZVEi3D3C1Mw8zLdrQkQgQOLEIkxAEAEYEAEB4e6uHmZuGl2pGwEIoCMSIiICIsH2ewAREODlVmMEeIRHQISHW6j72rvW4FZZ1+5tIV0xnDkJmXtKOYskpkKxwzoIJ6JCLcGKJhRK3iOCiQQih2F3Dm6Pw6Vfl5TBzfuq69n72rvX3uP4ha9rjuVUFrxT3SMTCyMiQihhpAQWLv/N/+3/auYe7trNTFVf4g8iEYJv0YeZX2IPATjEttvAIyzcoFt0I7XkQYAUSIEMyAiAEAjg29oEICIBESJuZwwIkDzQ3bupmplWsl5DG3ow5sRD4tU8EQ1ZDil2qd9mGwQQIaNNjrc0uXZwjQgIQPdEHeNKHfyR1nMKcHAN62ALhjsSqJshdx2sibTDwSAAAZiCGQKEEXLCpiGmGhzhrqpmZmbhEYBECMEikpLwy8XEtjII4arLUp/O68Pj8sPH5W/e89cf+LqQe8O4wI8bBgACEOHv/h8IwIQMyI7kRIQIhEiAaG7qRGHNtbs6RhI6DLQb8NpoEDjkuMmtiB8TCHogRGgWcmjVe3dVMw9HRGKjYHfHAPWXwBfuAI6ExCKBMX9bwlM4cjghIAEAIRCG3+6ZIAtWBQl3C3dzNTVzd9+2BwARISFFRIRh4BbNt1CywZx7mId6dIPuaIiAgBFsH9k/iiypYB4Sl4FLTkO5P+LPj89/OJ5/OUz3xRKV1m6e65sP9dVD3TmqIQSEIihCROwYJMeh6E2CjL0w7hgKe0YMcAfsvQkjhDKouaIrBBIzhIUZoOj6BBCI+IK2CGjbTaPkl0QctJ0FDCAIgPAAvMmEiCJcaItB284xd3d3AMCXM0Do4fgj4m/LFhHgsd0s84gIDEAMYhAOAIOwQAeMDXcDAiAwggAxth84QKADvOAAESIDtwvIdcnrs8Kl731/wzcn2I99EJjXFZBYWBjcrWv08FqrtjaWbGbqoR7qDsiSGJA8wKy+QBAgICIyIkSYuzETuiJiEAYgAUWAubkbABLOgARAFCSEaAEeHhEBRIwIQETMREQAvl0QeGy5z5ZOmvt2IiGCBXcFjqMz4fZNuznilg0hBm5HKzAAf/zz419CRKRAIiYGwA0PmGVnuwPenWQYOBUeruCAwQLMtXfoSmrMCInmdXUkM2gWxGkcB2E2IDVDBGYCREAi4oANnCkQw40AAx2Qt5QkABwgtr1m6oFm4MBC8QLJRECAiIRIRCRCiOi+7U2MAIQfYTogACwCEFl4yHJ7CHOfCrUeusK1AfiPsefHCLTF5ZcXvGDa9vZLxGZEgSRw2GfaxXhDh7t93o+cKCd2pGakgcFthYZoiRMbh7sFWDghUYR2J2QnDBZCBH750AhCCgIKNQR0BGNBxA1F48evEYCmps4BAEruJv7je7zlLwhIKFuOioAK28l9WRvczgUgMBJLSqngaBioTLa0WCvMArqgA+F29l9+KW6fgYhA9IL7gD9ufhQOzsQWO47THeVDTodUjvt02BEjIQQgVIvawxkRkUgIPdBLBg2hDQh7NwMl5AAkZEFCZCQkBPAIQjRCjlAEBweAcHAgIN5u33aKkMjNty8vyAxmiAIIiEFIJCwijAgIFgpb6hy2BZPtiohBhEsmAGbilNpQrHZfF7mwLFdswG5CFFtOBAjEhETERITEBBQQLycNAYm9lBiTHgfc36ZyKDRmHgcSJvQ0FGs9MXYwikbeccMLVQIgDIKAcHNzcBR2CxZA3GoS2goRMINwBPdwZgbriGJbSeER6A7gAE5MgcQEGO4sWxWBsW0j5K3iIkYERMeILZjCj+cLMBADEJIwgDOTCOfMo4aaLwUT1fOZpxDVDLQdf0Yi3CD9x+zn/+8EAqIR9d2It8lOx7S7K2mXYjv9pu6diM0bmEJfoy1ghu5mZlrDITzAHcIJTJBKIkDcSqAte8NwwEA0Nw8PCmBAAHRXQgYCB9ySQYQALOrwsp0iJCKQiBCJOaUkW2rycjQIIyy2BWIK99hqntg+i5iTQ05hntTCzJIERLs9DozaVDzCAUA4iPEll34Jz0BbGRMYnkh3GCx4POLulMvtyCWHm661t9V7lZS1rm1a6jS59nBER7AObuHB7hHBiJLJnAoHEVk4YgjaloUiBnqEKZrHS/Ya4AroCI7gYACuCEFE3WnDV/eQAEegDWwJA9AD4+XeBiIzBwT4BskEEQDkgYhm4QHI6OKqxmSL9v0o1vPNaSeMtWvT3tyC2bc4gC8h/oVWeSkvrXAvWUFkd1PKzlkCwMJatNmWqa3Vc67X+fJ0Pk9LSoxMW+lO6ICAHGCBhMzSDDE8HCKCUUEtPMzdAwJQu7q7ezBFeJhqIAFJoIM7mEaAJyLHLfcHdzFzQhBhwoC/Kyn/LjbAS+oMCH+XSQeGAQyZ/aW+RSYQAze8XGbXzsx5yAZuKIWKExrFaW8/2z9+Wc4/ydMJLdY81WONWy/DbvdS9so4SPHQZq1arTpN/Totc9Ui03l6fLxeln48lWFIAEEYQIjoAECx0SwQELWbhXnAWCDUVbU1d8Ag6d0ivHcjCAtX9UBCMgB09wjfsAQcmSAQ0ENUjShgAxfwF/CFAHARDgh3AAj6MQkCAIRghgh4eZsiATCiJ+6ZtfN+zABuyuqAkiRx8+agEeDo7m4R0ay2bsnyiHmf8y4Th7DavLZata5tnufrPC91Wap6PJzX2v32/iCJI8LM1V6yh3CIwK7aLaqBWgQJOCCa9t5ar90tGAW6OkKszRDQ3dzDAQENEV6wmalgiwBidgxBFlUnggDzAGbC/x80b1wNQCBh2BafY+OCnJn+DvfhhegJZhAGAEiZk7KkJI66JZy+cR8IDm6ggKrcQbhIOo5pFM7CsVqtfV3rPLdlXed1muZlrfPSWvd57oCoTRf3JGjmtXszz5nDoqobojk0dU6j9XCH2mpvvRkoMjHbYoTU1AH4PFu4IZFt9WE4RBATU4R3JKAEDpESi7YOWwAiQmZiJhbmJCKtzVtZhhhbtN2IHoiI4C2QYIS6Rxi4m3ULZQYiICYSIiXaKjfYikFwRmDxAKWskZAYCFGYbPFAm5f1Oi/TUtd1Xup0XeelzqsBRKs9gKfrwszI6BZd3RFV3TzUMYgtvDsx2Gph6lmwK5mDIUUgmDMFIFnXCkGSPcgB3CwMkJgBQSNxIEivWobSzKUti6luOSKxSBnL7jBwZhndItzcergFGAIiIRMhUzeNeAHqQNoK3AAmYrc53LYSkTDQw82ZAAOJgghfFpoFaDDkMI0gN3PV3tpynS7na+vWui61z6s1RQiDQCRs1T0UkcxRI1AQidSgWiCDAbdu3RYL6upDEVN3D4dwQNcujBro5rtSwhCJLMId3SHUmIkRqolpIIahIYK0aarL3NUciMs4HonTDgdGSjndupv2tbfFrSKCcJKcckq1ToT8YxJIG43tqpTG63UmdpZI4pYouoV5DmDDcFDFSiyBDkIp5yGDh02zDNLOl+t5nqb1Mnf3cDW3QCYEbJVEKBAcwALUPDZE7MESajGtFgRAsda2rJVZqkFTN1V3twB3gHBArBoAYA4WiEi+BVa3CNsKdGZSi5RIrQGg2PWhXq9L1RaUDjeUx71tmAgsAyMQSQQBIBHkUna7cTeMy5qZExEhIBC9oICZqs7zKqWWWoehrWtda++qah2wu/JKaSJ0J6I8JBQK1zpfJ9vl6+PzdVqnuTWLsFCNZqGOAMAlMVFbKieqCuoQEebmAYF9VbiuaqDqobrlNchA89Iwwty6OwYgogZ0DUB6uKwbZhOSMCJ4QAgxC17mZhYpoXUNAJm+/stz8zVETq9vTrd3r97cvnoNADknRCAklGS5AIII7HbDfr/fjeMwDCwJiH6MSxzuEeGutbVaa6+1tbrWWmtbl2Wept5n0GRUqiQwEOfkEdbq2p4fn8c5PT1cVXVeWvOwwN6jKjhSEkopuaMbIDowbTw8AIXHsurazTQ6UrcoQsw5MafEvigzEcq2v91BA6MAODwvbUjiiCxc0lY2UCKy8No6IOuqTR0RJS7folJAMeG+3LtWwri5uy2lmGGEteYiNI6H/Tjsd+MwDLmMDkFEW+AB2Ei1AAAPU1Ptqr1Za72tdVmvl6eLcF3weh7ZFU3dzNBU2zJp6/p4nucpzlOFwGXtBmQkCtIJUXJOSCm5YZKkrbOg14oQFG5LVQ8HSEyMLAwiLys0ZHJAJEzCxGSqrWk3Z0m9W2DkJN2BmUtKWywlCOieSkaU3ntKUEoWBpAwQUVvqLPrEq5biS2CbsiEGJhZskiSDecop0zC4C/J4/aPR7jZ8XDrqqZN67JeEaZLa+sR+97Pd7zsvGOrSChD1q5LJQDomNbLeTUK4ho5kDENSATqIAUSvIBiNOiA4NTadsLUnoWd3QkAiLrBMA7AnBP31vaHwiwpCTKpGs81uxGxijIjepA6ojNYbIUXRC60JfhlzCWTIAqiEmAiZKEECvWs8w9tcPKSxneu3bS6VSN1I7e0dSEIkfCly/F3JP5GFA3Dzs1MUwP3Ohdyy1yUGrkCautg3YXQMqQhcl5bb4kqe2TpXWM3chmDuHfjIvvjwftKKUOvKHvSFu5YBgBwB/PIbuCGWxLtlnJGYRbpiUgSp4zEEUHsEcQUKAlMr+druGNzRBCm7tC6QURmpiTukISHLO4h4E6Bgs5oAkZWYTn3S0DL4aqtu/YINy+NgxkCw9w4pdhIcIuNqjU1NzezcRh96wV1dW1Qr6A1CRqEhxoAcMKc5XiTD7fqoa1RrNhV8tDXNQJAUjhEOAT03kUSS46InIcgcA90DGQkwq1n5I6ubn0jOJAYCdM4IJEbuAcSZfH9zQ4AQvLl6el0c2i1cQ7CcAdflQIw3A1FiBGYIhADUcADXMMYdaU+Qd9ZzbYAqzS3raIkAG9T1Tl00X5MZWQWJPIIMzB3cFPt9rLd7rW3Xq/t8mDXp+XpB2LYCaddeq6cyzENwzCOadwFp/U6tdZ6XZ0TINemAWABbt66BmB3PxxGhw4BERYIjgAswgmFiF5BhLZV1xmU3QORwt21l8zhCoFCnBMDJCRykgDan45t7WMqI0StbZ4qAiTBwJRzDg9k4JQ3Hki+nkIgOHrqC7YzLBLJIq6RcoBuNYS7OwLqGFBDn5V5YkZAQgCr0Svo7H3BvmKdL6cv+3zR67PN56TXkRdkHHe7/f5OCuT9XnKCiN70cpmeH58uT89rbcYFqa3zbBYAGPTSaZCIlFNYFaHYKp8IJENiDMSU3dSQepAFqylZIIRbgDfC2MgtwgjkQNjatACcMhBhhOe1sizLWltrXT0CbCN12DeOUf79AwxM+xL3UZ0viWKM2epgpTBomFmAS+Eykledqp49wk9IBMEI7fJhPT9EW7V3C/Og4ycN1bgr2Rq6SpZcChAH4fHtG2KxXtd5mi7L8+Pz9fHh8vA4Kw9Has16j941ANNYUMQ8AnmtzdVSIvWIrfWIbhHkQiRaa63dAruFBzMhkxBJ77NQgHlHd3ApSYgksZRREkgSQHK1nBulTGmR3tva6zR3NWDC3sUjmOTZ09mDm10cY0jllAaXBGIKdH4IKTQcuQwAsVwvbZ7bUsEDwhAACLU1Xdfp6XF5vpgF5WLBsJHAgBCg7s4JDMJ9t0tW5zbP6zzP07qcL08Pz08Xi5J8XjcKDiSJSBoGkpSRJGfV5oyrWkJgBAR3Au8qAU6m2lS1aYtATtmtITEAkaVu3QODEEkkF2GWJLkMAwszRWDrPSCSewFic8mqrcdaGUnVTZsxye3pNA7l7aubz9/dvbndAUAaJAQ8/NqunMaEiarWZa7z0pZVawsgxO9p45UCkbA3vLy/tq58OqbdlMfMDFtp05sFOnN26xaj926117lOT88PH54+PrUoQ8qFcqKIIBYWYmZhznkYd6kM1+ezuQG4bDxfOBGwJGAG7Wbm4YDsYGF9U6AYIHIOYAMnCzAP943sMQQRAUJXe+GQiSQXtOCMdV2DmIjWedauCi7/+d//9du397eHYUxJRAgwTNHqus483Guvy/mh12YW4WG9u0eAn88rQJjpppip7z8CQJ9XQp5vOzBnAevVVcNaW1MuY7ilIERy8/l6/fj+4eG8KAkRulsqB0QETpQyJyHmlPO420mSCISwQIxeASDCkYhYEKEFUCbG7q11g0QAALWrqg45m5qpGaGkaOrYDQGtKZ1kIygsAIBZEAXMIJAOd3fH29veGjG3Wt1c/g//xR+i8If3j+fzZJyHnNEBFAoXXWudl+U6IzKyuJl201bTOM6X2cPdFMHdzRDW8xVTub055iGhr/U6q7YwJWZqaX4+G9AtpnY5f/zu/YeH67WHYgKG3lVN97vdeDxwLsjCOeVSUkqShIn3+xHR3c2Vts4u4ka6g7kDB1ESEk7mZubWWzX1xRYA4JQo505siwrFuq5lf0zzzCkRMSBJyeTePDAjAnjscsmtaSrjMk+mKvd3++++e1yura+9e+9cBQC8DyVr7d4UzBwhem/r0ubFanO13ruaBSBlAnBtulgwW+vVtTWddL1GOCBAp4jV1A0E8H2blutcGxIKontXi7BhGHenY97tkDmXkoeBGAmJKQV4SuEezAxJth7ej/1dj9iZuWftjagbAK9t9gAAsjqTCEkOZkUSgGqOmLBHUwXrzFlSCQDzACJCdNNhyI7MwuMx5XEPAGIuagyArtqbOrNDeFuhl3q9mnYw6+u0XKc6TbpW6zp2be5mUcMNESis93DfCQNAWya0JXpHAE7ZA8k0OFvIfF1q06lHNUCmUGeR3sPNtbU8DsJp09lgABIiAQASMLMQ0Y+iEdySQ3dMCQn73C0AJefelFAkORKAJxYOQLVI5LpV1Q5N7XqZILzsD5kEiRwQAtw9gkQk1FJiVcx5JCLZnV7fVGyt9lYRCXrvtbqu0KvVaq32eVnXtrZm2s29u+mHB0U05MV96aruFpAzB8K6zIx9LDLmUQiCEyNYl46pNzXgtffLqmuzVCAcUJKjW0Bdl+GwQywvRcPGqkEwI3DCrc8VgEQQ7o4QgWDCGE5ECLxJdLIIcwU1clVz997drAIIY8ol3G1Z9iWnkrR3wBVZgoiImAXAQ7u7B0BK2T3Wtcnu7pNjtzo9t6WCraoGiETc5onMrHc1c7dwM/Nu1gwitHXT4A6wAgACCoWwIk/XRYQPhzIeymHMGuCtX2yuTZHF1btG7742r96ZiFDDsXZXs01XsqlIAdAtEIFeIDnCYxPY+os8wDcBAoRhgAgTIaIhQoSQYk9Z1dy7ae+mFSJ1B3CKmN24NVnXPJQ8jJwLprxpKzdYQwizbg5bk7kcb1+1ZWpVMXAObQZghtudQiAIAiAkYFby6gGOHbh6LOYdUYQScyp5f9xTwsPxcHNzczqNhzHP0zybbVpO7WHEjoSckHFpLQJSVxZOTN0gHJAYANwNATZYDubgcLNNrbQx5IToL+S4QzhzsIC7YxhCCFN4iKSXbrCZgbVqiCaCKKJu2rX3Hu5IQlI2nWCYWYT2jghIQsREKLrM4/H+fhO/MIvwhD6pIq2BAeGEIYIW5O7KWAXnFZtFN2gOioFIKfCw37375HVm/Pyn7+5u91bXZVlbXdUxkDZSs9Xq1lOC7Lh06OYOGGoUMXbrPyYm4Y4RyIwvRH9ABIsgAUQQsr807titY1hiQApTjTB8EXvbJppiYcKh1dDuzIREql5yst6baqyNUpPBGUDVtK4BYapEjIyQkICEU0q5pDIOu/2nP9tfHj8+7zKLPLYVzcgIUkZ297WDQ0QSCjAF6FvzDGks8pN3Nz/56bv7V0fOaT9ma121Pz2etVUnNsC11gBalqrqhECEkngrhbtBc9ZNYkHELFskFkllKBEGAJKEiNw3eaghOBBCECMZAIS7BYU7uHdzd3ALCMBgYmDuvRELEqt5XTtAmJk3MzXkmXIGBCJqdUUANyOWLXQ7hDx9/dubT3+1O96VL4br04e+rjOfx93uGTEQnZgkelUjjCQQkICTc8o5wJ8v06e3JwD46U/effHZHaM5U6/r2lbtdZ2vbhEI3aLXqgG1drNwpEBMIgG4VAvrRCiSchlyznkoIolFhAkR3NndNyZiS4bBO7hi+EtLJSDMIyjMI1ytIRJAUJgQEpOpM1EqBSBcu7mfr+vW5WB0c3cHVx2GIdzUPboiKYuGS3iW5fE7X6bju58f3nxx+/aLNOzK7rBcztPTc71e+rp6b4KEJYu71OaBPDoK39wf5vN12O+//OWndzdZ17muq7Z109Zf5wXc3cGApuu8rL07qqE6VDP1TXQRqgpI7pZLGXa7PAwp5xfMglC1Lc8mJLdORK7dTMM1zBHRe3Prm4YZwNGdXrrnQIRMTCRMgQSw9l7niBfxoJtjBLrXuvbm25chJosAdwzbMokIELRY33/dnz62pw/7z36R96e7nMb94dNf/nJ6elynqV6v6zypKhAeAdR0nCtQHMb02ZtPjzenw8DsHcipSJ+q9WbahSCAJMm86nb4Xd3C+6bhCaoKj5cFEd+9uvnFL3760599vjsMm+KLkMChtuqmyJSQLJprdzMAwHB3c3PakC4A3P9OtkSctHcMFBYSASIy96CcBGNoLdyr+xbisQeqefPqCAAx7EaShMzhDoDuEb0L0kgx9Y/fPbz/Zv7hq8NPv8w3d9Hrp7/6cr2cl8t5fnqcr+e2rgBBTKbt44ePwjFmevfF2/2utOtZl1UIeciqPochWGI2IEdiDRJhCTTt6kv3gE1HEiKcUjrd3Y67HRFpbYjAKb0kyu7E4mHq2nvzuiKEh29hHNwcxEzVdCs4N4R2swgnEhRGIg905JSZGVgycjqfJ4+gH1UZRGQR17kxTsC8z3mTfUMYolGEBGagLCQ+f6h/9a/7+7/e/fRLz7v09lfleEsp53HczyddZtOGaNqXtl6F7O7m8Oru6HUxVyBIObGwjjnQe6OU8lLbvHaSDGwRGoFdw4NKKbmUEjjsx5TS/nDgxL03MxAWiG4vckZAJAJvdTU1Nwvr7k4v/bgwa2ZuZkQMAUEMDkQcpgEOZlsimkuCQO0dAkFxtx9b7701c3N3VWfCnNOwHw+nG+vaevfe1QwATLs4EiMjs+REMfn5q+XPvqPT57NKyEDCiMySUFgAEEHcDyN71ePNIZGvbRawVLKUghhF2T0TUhr26uFzd4CusXbtFhE0Dvl0OubdCJzMlYlKSdZtul4kpSTygnE5i5BqbIDjZh6h5gEhWx7kHoAe8aJdI9qyAUA3tzALNE5ZUDadKwQSJUn5uCzLiqt7NVeAgHDA7jBd1/1+NdhSS1vX2moDD/FwQiAmziIkpJIh+uVvYPlZHF5TOhGCWusUAQZWfb1m0Mh4OAygK7ixECeRIgigXQELp+QoLAmI12mdq87NzYOEh90wHvbDbp9KiXARcgetbbGe1FQSBDDjEBAhwrHB/I9aZUT6u7KxIWIEIPGPC9Q3K0JEqCoxYwi+yKI8zFkEII63r9KyCl/xOqnPZlvI89bh+fFp09bUql1VHdxDDIGEOAuhbDUhuRPi8hf/DI6f7n7yh7tPfpbvP/d2sz59t3z8pj8s3ubXn7wlnb33MghyIRGShEjWFYHQ4XLtTcMAprkutau5AXFKKb0cEIoYhnEYk7k/m7mZqkUwBKoDpUACRAIwZEEkdJdNBIpgG0tmnViYtuokXup83EArIMJVFQCtIWwq3kAAociJraTeU+mZAImAiWKLXwQeaIAO7GHmIcRVcI4hIybExEBe1ayPHOv590///v3DX5wO7z67/fTTIWNDxbAh55vbfdSJIrgIjSNJIilB7B7koddFe73Oa62td3U1IrKgXHLJDFbXSdtSESnnlJLsd7veWU0JaSslYNNsIwIKE6FAJnYAQgzXCGcPnS6+adWJAQJZXvT8RBawnUoOYyJmIaQwJyQRhoAYBjcLcxUiIknJAm29Wu2AaA4euDl4BMuAUYblUWIWvfoy92WtFkuNXu6VxtB2/favzr//M3XY39y/fnsnQ7b5OY8jDyMAoDCPA6YhgCwCWse5hlnX3roxIzIxCQWcDuN+TFrX6zxXxe6BRPux7HbFXFrvbmpqgMgEABDgjMzCRCRlQCSIsN6IhJOamgNiysSCm5g7AlRZWsRsZkQEAAQYtCU6QQREmLIQMSIjiWlnppRHc3j8oZkrhFMEbWIfEHEuIUOkEpoDOcJBFQwhSAAIo7Z1Xi6t1rS/Gd8VAIi2JEahkN0OU+I8ABe16HVt8/L88fz9dx/fn6fzZW7OnUtIvDqM4y6PJa/T9MPjFQB+983zMJzN4f72+Prt/TDsyujaW12rOdBm/wQKCGJGYsnDllkbC4toa366BXgRXiP6pjvi3tx0vs7woulnQP7RPRqb4B6IiRgpsYipEiFzNvf1cOi9qaq3dcvamXlboBKSg5NjelHgaRc54DAqJnNlzsPxePPuk/3NiWJF11wyD5nQiNLmTkROREbEFC6EpeTBBZx0reb1eHNzyFSyWKu167/8s28j4OefD+fz+dX97fGwB4TWGxHlYZSUc0rgbu6bGzilASEAwNGREBA9ogwDkhAhE4R7bFiFBAbDblU1JJRNR/6igQcEJGQgCaCEREieLMK3Cm9/PNZ1XZelaUd3JhAGASnuY8gAlIIESJAI3AAgOAEJp1yK8bDbH3eSCDqQB3kTLCQCKTsl9whvAIDEuaRhSPsgY5atrS274+3r+2Mm18tlJkRJaV3X//g336raz376ecoDMXmYuyKz5ExC3h2DAH7Ur4S7d9duqmrRdTOIzszcAYmJmGhjqoXzMOaX5UAAcOthBszIhJuFBjCQENlMw40omNljk6y4W7dwZiJGQc4gxTk7Z0RBICAkIpEUOQEJhAOYe1+eHrDl4/2BidEbgrMIpBKQEIIAEFseR1fbN4uCpHlV393cN43wKIfT9PT4/DxdrysCHLIsrb9/eK5rzTntjgckMHNVC1XkxJKBIoDMIxAQCJBjk3FjBMR1usyXZyFChFyklDyMI6dMTGrmau4OSN4aBhoxA0YgMm9K7xf17rYDNzdd75skjBF5GDGMiASJg5JRRkrABTgBEwojIycmyd6bu/fWrCv5PrEPCVA81JhzYGZMSOyAZsaS0jjsb24F9yPvu7qh1HV+/PDYWq+tLbU/TuvjtI6JhsRL7T98eIKAnAtzVo3W+tboSWWQXPBFgUSIiA7BmzfGqdD330x/8e//YrpOu0FuT7s3716dXt3vj4c8lJyku+PmmM4pItmyIAIRxUsltkUuRqSAIGKIsFQQZduOus7u4dbFAyyIQJAK8gAykGQSD8RwJOLwaK13tZTEgJfLwjvmQ8FUMB8x7T0AARlFhgNZD5SB96Xcd8pmoQ7LJHVZXVtvNtc4z8qcODHoummdn8/PqaTWajhGbAodJJFhHLvq5kYx7e6+3fPNOXz/5s3959f3f/6X3333fH/t6tR7zNd5d9zdvbo3JPOKxMLc1VgEA2yDp3Ai3tIreLFkWERISkEkppITQNGGqiYR4EjGGWXAtItyI6akHpz7JiAjNLPauqNIXfNYLBg48+6EkoMz9AreUUjK3nsNDyiZdm86kHtobwx9vTv2Zbk+n4GQRQS7qk6rZsnP58vTw0Niahbmziwl5zIOh8NRzXf7XdfuZgixyd8xgjjM43g6/L0//PXp9ubPf/s7N32qPv3+Y0I7nXb7f3xKQwGKXmtX6+7utsG2AyAxECGgb16xCIQgJghIRDDsAKktQnglMAFvjmyUQfaQKmdFwNyrQ/Lw6N3cPKzX2rppXYfyRp080NVtvUZ2Mw8PDMQsG43CQlJGkWJm3maBfe+1l7RM1/tXx0/Pt2Z2nWslGhOH9WVZauskadiPIpJzHoayPx0iMMKxQvcAkc2Av8E5uQfgMUnJ5e3b+6++/v7r33/3w7ffC9hlafd/8/Xt/e3+OGIurXXcDP3ERAk2sxkREgvSZsgAcES0zZnpsf0wtIF1qc3RGveV6+q1iRkFt3yz+uCSrK3zdL0sbXUQUJ2XsOOwf8VDqs8Ppo12N1COQSlUEVfX6tYjVgxHRGCSYWSCXtdKeDiMr1/drN3HnD98eFpb20zQgMQpjbt9GgYpeeut5pIBoK0LIkiScN7MkBunYesqKXmESLq5OY7jMI6jeTx8+Pi41N/+9qubm8d3b29fvb5NOYFQgAPwlgQhbp4siA2sNy8KOhJDhBBCSp6HsBaJZJondGMLUiqRUwwGMOLaqjKxpJSHsey6cwv3bn55nvf7XYDLfu/VERWjA6nOEyfRrpD3+dUnmAYUFpRQBWrg+vzhw/vvPh72+3dvabfbnY6Hvta1VVUjRCYadsPucEg5E28ZzWbrMETKw6azxBd9EJKkFw8yCQnR/d1Nymkc8u9+9/u//PPffris06rP5+nd0/mzT+73h/0o2ZDMgZGQCBC2zAFezOoRDow/zi/xYOaUhzCS3g0B1JlgcEYPAWCmVYqZerPmwGXYATFAXC1UkjurMjYADmgV3a319ekpHw8AIJJs+QiSmW6ojMgCgLvD6fa+teaP5+secqAkyaZNzY6H8ZPPPrl78+p4c0pMEGYWXPbBXM9nKuNha+AQbbR0uBubWQcFBCBEIgzCw2H8+c+/ON0cv/3mu2WarrWtXdfalnn54t3N4XSQ3YmEzR1fvP8vrppteAYRbWK9zSfNAAAFIglSwgigcKAOYpIdmPxMEQAa1rU3NxXClJMgzHPvNOxvX2Nf+vmi+gSpBHOrrV2+5VzSUnl38fJeDp/k23fpcMKAtLu7eZtlOJzO52Wp87pO09Jbbes6r5UJGZ28QiDnYTzdUBoskF7OAhGzu5ttHWdbfdl6XhhhHq7N3IkkFdntd8MwnJ/OtEkbEL/64Xld17vTZThMn/7il4Bhbi9GkxcDbmxxCTaKkRMlJFVCQnApqWw660B34ABuBIJjJhgSYWhbV2trhDNkwSCAFai8+yKXsT58N3/4bnp8mK9LBOb9GH3KvfPT07qua/+3MdyV+3eQhp/8gz9xp1QOp7tcxvkUROTapr6c/+pvPgwZsS9oaTjdjzevx9OdB9Z1TbmYVjclEmIx1djoaPc6zx5qHuuyJobuBmhEtBvKH/zhb8Dj44cPkgQQ1eJ5tqnOw8WPr+dxtydm1c3e8zIOiSVFOHFCTls9QMiMDOAy7A7beBIzc7NNh2pc3Na+1HVea1N1YAQORTdGr48P9Xze/fxdvn09fv6rw3St82RdP/7P/0ZykSyIlFNhp+vlcvn9b+fu8zQ/nS8GfHr1atwPbVlu7k+701hCPrsfelt0ajpkYEnjPpXRVBWJcjYCN9p6mrypWt1My5xzV+ute8S8VI/YSB1C+tWvfkqA/9O/s/P5PAxj772pTUvrjl9/9c2r1/e3d7c5ZQ+UJEQSAELioUi8zR+ICESUJIQkh8Otqbqpq3pv2qqbO+81rkyCIltPSgRJiNQGBmrX5Zu/xdB8d8cpAUTajUUy/dE/YpE8FA8DzA742qPN0zpd/7//8t88XWv3ODx8vLs9htv1w3fHfdK60jD05uXmZtgfh/1J8iiSCLGJqFYEFM61rxBGyC8DVhBLzm2ZrdXWWu/q4cLCgI6RCr15c/eTLz79iz+bMnMmtNqsdwF8/+H5ep2XaX799m3eH7eCPoAQiIBeEG3z1b4MmnAZx5225r0bgKuymdaqPDqkzSbv7pLK7jhmMu4o6ON+hDbV779aP35t29QYTJDKF//wf815J3kkZqRk9mJ11N5/fr4+PDzN08IiA9p4GhwoAMf96fnhA5dhdzzev/10d/eacyFCplTGIWbvvkI4uDftAMjMABCuBJEZYxvfBNBqAwkQ2UwuEP7Jm/v18ZM2zRRKOb8b92OSD9bqql/9zTfT5frZz34mr95uqtBAiGDc0tDNJeO2jRwQjnDtsc60TDAtcr3A5dkHYcc2r8vleZqmcrgZDjeFOjZxXA/HnYwZGOuy6LrU2nr35gh5GI53p9dvdvefyLALpHADYi7jz/7Bn7w5P9dlDvdNbRqIyCnnMvzuPwynm8ObT3aH0ziOyBmJCcFzdnMktN4QjSjczcwx3Fq13sLC1dqyqlm4T/NFhJEppWy9pSRv3745//DUL+dBVRyo29tdrujnxZ+++5gID8ddSjeI6B6ADODhsSzzuiymBhGmKu3rr+J6xeuVrhNcz3F+9rtDfn6q8/fn6w/P0zUfbj558/qwS+ROSHl/t3v9FgB4KHmtqfZcW50XmuvHv/6PZdwvH25Pb74vx1ve7V8GxLHsXv207G/Df/RkiUSEiLDklDgPQ9nfDPuTSEKRCATEnLK5E6GRuAOrEBEQaqvrPLVlbrUx4zxdTN26djNAyDnHGOBe6/p4uXz9/XvsKu6MmAE/j3GP+AbHRtIelutX3+6/3Dl4q405bwgwXefL+aJ1RdcwE/zuWzif8fmMl4ufn2I6y/raYsZkkkv2juBtWe2QXyYshUvJcjxZMOXgdWG6soIYpG4MGuu8PHxfL0/u4IAouWP6LB+3IoGYc9nlYc+SmJlTpp9+ycIimSUDCyBHBBAAcE4ZAQnJ3I0a0eYY5dyOtX398f3Hh4eH8/NZiKLbNpzHtBORqS3zutR6bTUDxTaqguiHaS3EO5YdpdsK8tdPdf6P5SdvDp+8fr4sERCBh+MNAF0eulonZoHf/RWdz/j8jNfnuDwqNtDFhuh3CQ+729tjER6S9HmOqIfTQGGgFYlJDsyRgrZ5VZzE1DgnQNB50vbUNTRwdXxedO2gDrmU29vbu7efMFEqAw97hNjf3DExIsPWnzAPJg9EsM0iwyIs4mYBgQQ551zG8Xjbvvnwu7/6+vHpacipCA+lDLsBCefLOTxqVe+6CfgQiYOYqBsCcSAZYEcZjPmHlR6+pq8/3vyTPw5ndUeW/c39m0+/6OuyXs/i/+FP4fyE57Ov83xMH3/92bhe5tOtnk7j/d3dq9tDAr+ebZmGQQCccw4MHg6SbzZhLQpTHvr69eHuFjltqbNaUHdb2vx8va79f/63/04dbk7Ht6/vpufHw+29pLLb7Q63r1797NeEgtsAQkSj8MDE3LGrKgis62K9uxsThtrz4+Nf/vmff/XV1//yn//r9x8/QgS5D4lvTruj7hKTSlrmCgBqnsooEYl4IC4krdUA7D9K1IIQA8PIv1+P8+qHmwTUau1t8a6t1mW6Sv3zP3VVPQz1732+/sEv4zQ+fHyMwzGNe97vkUtAJyJKKQ05JU0Zadi7qgxJyhBEaCMPXdd5mwknp9eyf4XD3t3Xabm5XP7Vf///WJuaQ1VduuHzVZuh9+ew/enmcP+WTrdJtv4fB7h3lZwIk/aOhpEKehhRhGqt9XJ5/PD49HQxxGG3W67T9Tpd3KfL9XAYT4fdaT/OU70qffe8igOXIiRFtjmUgeAUzoBA7O6KGB4iHpw9AsDctE7X+fK8zlfrTa4Y/qvP4o9+U379EzmcKOJZSh4PJecswgSuCgqh1i2SAOYxwlw7i5AMHh5mwGm4feXmgJJefZEO72Q8MMvGBJ6fny5P57rWVMr+dDuO493dXRHUy8f69PDwt//x9c+/TPkNEwOibVMDzHMuWpTVmXhIKax7r5fW27Iu1/np8VncdoTNTc2X2mvttXbtlojW1prJPE8U4dY99YVTYXm3OwIEmMZWhsE2qdBL9+GHj/H29VzbdH5cz891nswaEYn/0z+W/+RnfvdK7m6ZS06JiNEBkVJKSSjhSOMQl4/Rri7bFBRi6NHOLDkPB5Zhm+9l5liO5fQu7+5ZZBvPGOH/2f/u/1jny3J5nufFHPOwz2W3Gwvr9Zt/9d8/ffM348394e41S/YAIpAExIklDWVUtk4dI6FnZalpJWR0b5crr8s6z8u8XFtfuh0ShXurfVnbZoGkCI4g9zA1iNX6EzMjQjhY0GZVdmse2fx4WeBVbNbkH/kzAQw5/m/+8/T6LqhwSmGQpcANUBBGIKAkESHss1sH03TaIwMOd1BG4AKUiBOwoFmykwRgGnLZ51J4Y6NeGHbH3Y079ObRbHm+zu2hH/f7m+P9L35jXQPIDImHJC+C+QgACGEBRFUFCyLJhcdxPeSyJ5Qwsd5q7arV3CEYMQkDwtpUUo6gQy4FaZ/KKCkzQ8C1Lrzx85tH2IPc3Kz17lojPNzBDAGYGBgwVPY//QkgAzCYe1ifVh7GrUPv3QAzEJs3g5GHW7NGw4nSGEBuRroCBKIQ51xGN4PhmMqOU4HNLQAAEaXsmHsEuHP94eP3X/31dz/88PNXu88/f3v/mz8SEWap61q6cd6lbeiWmVlnooToKXezTWe2CQ3Q9NVxB9Pw/jwFYCFOiFmEWBRwUd9l2g+7I4/SrRAzk/zo0N7mApmqu7krmifVkdx7t96tt1BFd0bkJIwk4YxAXput63qdAmkvr7StulYIAOt2uUBb9/eH/au7YecGQKao7pScEKwHD4BCUtyXMNfeQTrxJj0AdxMWDrg8fff/+Wf/wz//5/+jPH/z+PHx21//5v/0f/k/C8T+eGva23y+PjGJpDIiJUTeaiFGGEiYQFvTrmqK4ABwe3Nq1yl/fCLEIsKExFgjBFDdxmH4xac/G1Z4+PY76J1hcyZDSEYkD9cIcw6wDHEPeHM6GUabl15Xbc3VWCilXJJInZbo5t28LdP5GQyI2Vtrl0ufrnq9+vUqnKj8ohxG8XU47sKDyg5ZzDeLpAJGrZMHcy69ruu6unkE5JKZ8LJMAPCX/+Zf/Iv/4f/9H/7620/oysvF3NP+kHdHLjseyN0CaLle3DwNIxETp82oSylnYpbGrU7p6sC7IbdW3r55/c233z1IMzcESO6ZWQH2w/j6/v4Xv/7Jp6d3f/s//rvzt99FhCO6qYG7Kqgm0xxRiA8OrweSz96sOa11st48FAlzyiXnYUiyXi59WlwdvLdpibUictR1eX5oj49xvkDtPIzpdLTrk96QNx1fvwFnNOdQgBeqiYjadKnL0lr0QPMIs+P93f7mtl/P/+qf/T//u//Xv+q0/09GzZj/V//Vf/1f/Jf/1e0nPwEeVg0PI6KcUpAYUnRDhiERAIS/DF7jwpTy/lZPb96RJATy2nc5CzOASZJPb2/I2tltN47CsvR1+Pz4B//7fxJfP9vz1aa1P13+9V//+WbnYqIR+JhxTCRvbp++eFVVgTCXLEyMOIzjUIZSknhvvVVdFustmvq6uqOv1/r84OczLSsYOOLlh+8vurb73fzwdE/7/f0R0RDwxSfQZwgGUKvr8uGhKeo29alP0Kdxf7y/PXxxO1SVn/7yj7/41S/+4B/+g9Ort8hZzXs3CyARBzZSxcZkLJlJiAiBgjbdBvZu5+frD19/9+HDZXl8fv/h8Yepnde6dn292xfwz3/z6w4EAJ9+8tnN7V0A6SnTzTu013Bd4f/+P/3R68+0LYhRcpLMtpfzSA97TkNJrSfCCJQdl1LG3YvKS1qtbVn7NIN26M1607r6cvXrNVrzCCAC8HW+Wl2jXubzoLJ/4z+D2xF2CXNhJI9m9YKBkiQl1rZG0/CYPn7o63L/yae78fDlb37dYvhP/+Qf/fzLLw93d5xHDQrvARQQZuHet/ViZsCF6QRI28TFbdYOS7l99+mv/uE//tN/+297bbdv6P46Pzw/7Yb07s39f/rHf/+Xf/TH5+fH21dvdsd9LhKqra7h3dWXNh///s9gWXqdLn2uVlEwH/ZSysHMIdIwaOsRUIZhfzztbu7Tbk+EMp8v9fmC2qF3dA2t3jTWxc2A2IsgC+YMBCzc++pLx+++55S8v9q/ln1hKgNExLpAaJJxPB0oJ5nrOtd1XtdVUYry7vM/+M92N68+/c3fK6dTiDRHVY8gB/9xPBapau8GCNoqYmjvqqq9E0LOuZQhpXTz7pM//Cf/9Kvf/na6XCGVV59/+pPffPmzX395/+6zvNtdn56YSdertSYiSDxdrnVdvbdZrPFccemxiuDx9rg73UVAW9fW11JGGvfMMhxPu9t7GQ+bzUPmeQbvYZ2tI0QAuXZ3gJwp5U03jEgQzrnYDLg/OPP14RGtmwXkkcrIaUd5RjMiUYrx5obTomoxr8t1on1Lx1fD6X68vetq58s1l4FZthnWrrr5vYE4ADcjZ10X097baqqq3bVDeEppGHdl3P+9f/Qnn/ziV8s01XkOsJvXb/O4X2u9Pj2O+32dnvsyuXt1W5dlPp/rPCOGOazTJXoVhN1uHPfHLFLXygje1LGVw344HIbjjQzj5sbS1iQQgNGMPKXEJMRhCsPIeSAR2EYeehBR3o31PB7fvh5LpnUNc2trOz/OTMNhL+XgZqb1ZVa5d3RNghh2+fiYlWr33mpo3Z1OnIokFM51noUQJAVAAAagth5ghDFdnt0MwiNCW+3LeTa9SuE8/OTXf3B6/er0+hUCuHtrfb5O8/lpOT8EkNalt9pb7V3n8/NyvSAAISCLMGJQSTIMWQjddBt5PeQkDONuNxxOXAoCuWlvzWqVMhTI5Fk9ouQixESIzMRCSLANOmApw37/6lWdrvvTieqi7YOUnE83mLCdv/VZbj77NUsGIC6jhXuviJGHMhq+/3C+LM3i25Tzz3/9y3H3ByJJyoBAZUREAgR/8Vg6IShGuGnvnHJsvTBiI/Ztfm/E91/9bRp3qSQm7q21Vpfrdble6nxlFq21r3U6P/W61nk2sxcZFeqYWIbDbrcv4y7cnXA47IFTnS55HIebWx72Edjruo2uafMkX/zmS6uzqTECUn765vf57Sey6ZUBUil5d8y7Yzkecsnn776RlNq35+XpYeB7Go7jqzcU1S4f6vWRxhPnwjro5RyOXMZ1qt1Biqy1euBQ0nqd1mXJyxJIADgMZTOwMQLiZstAZgLC3rsHmBuEAQKSIANyIsldm61gJgho2ltt63RZr8/r9UrMbV2stXWe+nztXbfyiyFCvZS8Ox7Gw23en8w0IMq4k7Kfzg95t+cyOkCf5nq91HnqdbG6yun1T72tGJ6GkUU4ZZJMYQQhecyHmzTuOSUi0unp1Rdf1O/+ynl58/d/c/rZl/s3n4lQrOdqtV4eRXWZz/VybS7L1K6XeVqaUnIkYcxlPByO4VHnqa6LmSKyu23TC4XlxcbyMqw5CbP2bdZUAPj2UAZkQeQfRzxC683N6rK0da3T1KbJwut0ZSaMYERkwugD45hS67ob8vHmRnYHGfeUJMyIE6dhpNebotHN2nK9fvy+np/7eiV0mWvSqfZa3VfX9pNf/QKJMNTrqrXZ+dw+fF+1Tc2//JN/ytA5/2b49CceEPkwdQRFxoMcGepS57NOF1d1ra4WWpkgJeow7PZJJOXEN599vn/zWUqivUO0SRshsEiUAluR1HtE9NYlZzNjJnfwbUIYECJvw19dtbtPz8+mta21Xi9tmly7dm3rAm4sQuC7UYR4GMr+eLOcz8f7+/HuHnPBrROJEK06BAO1+dpb9fA2XerleX1+733JArI4LRrXa7s+Py3ns88zmDI69xnXK9bJ2nXW9my88NCIDYklRQDC1bq2WhPB//Yf/0rpcH76dgRF71jnMY2wz1zdEZloe6yL7HeUkrfVc7LeIyITIXNAqHaAIGJE1NYgQlW3p8RsNaaZulZENDNEjoB1us6PHwJCW+3zBXuVcKeC2toyQcoD+2k47E+HMpR8vB3KMNzcpv3eAdw0eo3a6vlZSlaQ9fK8Xi+bo1dQM0dKJQvJWsrqx+oxu00ef/rb3/mySOgO2wHbjirH2kMDyl/+6b+fysnLnoYRJEOYLnOdZ3Q7DdjOD8/vH97sIUcvvuz3HZpCDRhfY6hbBJN11dZ6W2gRNbXemDnCiRkAq9bNfbyNFmVhZtkemQMOfZnr9clqoyTAqU2Tqa7nR+yVJZJVijYWebzOI8xDsnHQ1+/ejje3LIBEWi+7m1seJPo11sXW2R3mh48UvSPacKfnx1gXDCCifYrjq9tSMpdRPrS+OK4pX8dDdVwv505MvS6UlTWwDi7Yl6Th67XT4GkABGPuQZ1zF9XW/8WffWV1aUu9nyWD7RDetbaL1dY1Q1YQ4IyE5KlfHhgDtgxwuTKLlMLMqLrN4SEm0K7ao+wiHLemJ4VQJNDoF1jW2N3Hem5PDxmh1TMZ5ESIminuj9IQGWV3Go+vjpJR6wpLxdZ6r3YNSYKm9vxEuRxHadcKhP3xqxQxjONmQU+Z0+GGS/Zg+d1lWiGaW+XsOzzd3rdhpFYjjEiBuvpa2kXXJVuXUIWwiA6wErdUukcn+XqaPLIluRhweIr8wPA65QH55ul9cAEZqFXsU0wP/vRtlGNXO7z7Ynr/bRp3kjMxRwQSIASFuarWBRBJBMOjzbA+0/pEywPaDOysU6+PEXHYZUYoRUiSm96UHKc7cMMAtIWcyHtvU18rYbUwEEYIRicDTFD22WodCgcQl4KczTozMSEjMZH89XStAB6AqVDKdjiZJOkNwhKGMCBZt+rrCh9+j+7WW1/X7tAkr4Cd2AR9cNseYINIYRixMD0J3mSF579yg9CVqnZdd5kgoMN7A9Rl4pTybi+lUBpQElEEC3rfHe+jLW5NwzHcl0t/ft8uD+v1woLZDK0XWKs5anDizZCxdjdwSYyAaHp++GHcD8il9X5dll02jDCFCAxCigbTQrz11nfBiGjRV+jNEKLXCGAReVJzIhYpjMB8GQYglJ4JPGMMwsqMEN5brdUhOZD2roHdsbEoshFb3p6rgE5IEIDQiVbms3AOCzWylgAWSSMBE65VgzAefshD0XYBZJcCeccUjZIjfm6diNVa79168/mil4d6fpzn2Tm9vVV7GYxDrS5qUk1ZuNa+tp6KLGsfxM+PVzrPw/64ztd5Xu/2kRgBsBkG0m4opktCRiRbLgGMJBGgvXk4pVLXRsz/C6koUOnfeNAPAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<PIL.Image.Image image mode=RGB size=96x96 at 0x7FD86DFE5F60>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LMMfK7cpGY8n",
        "colab_type": "text"
      },
      "source": [
        "###  Automatic data augumentation wrapper\n",
        "Here I've also implemented utility dataset wrapper that automatically applies random data augumentations every time when the image is retrieved. It can be easily used with any image dataset as long as it follows the simple interface of returning `tuple` with `(PIL Image, anything)`. This wrapper can be set to return deterministic transformation with `debug` flag set to `True`. Note that there is a `preprocess` step that applies `ImageNet`-originated data standarization as I'm using already pre-trained EfficientNet."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_aPHuXolGY8n",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from torch.utils.data import Dataset, DataLoader, SubsetRandomSampler, SequentialSampler\n",
        "import random"
      ],
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LzjnG1wtGY8p",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class PretrainingDatasetWrapper(Dataset):\n",
        "    def __init__(self, ds: Dataset, target_size=(96, 96), debug=False):\n",
        "        super().__init__()\n",
        "        self.ds = ds\n",
        "        self.debug = debug\n",
        "        self.target_size = target_size\n",
        "        if debug:\n",
        "            print(\"DATASET IN DEBUG MODE\")\n",
        "        \n",
        "        # I will be using network pre-trained on ImageNet first, which uses this normalization.\n",
        "        # Remove this, if you're training from scratch or apply different transformations accordingly\n",
        "        self.preprocess = transforms.Compose([\n",
        "                transforms.ToTensor(),\n",
        "                transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n",
        "        ])\n",
        "        \n",
        "        random_resized_rotation = WrapWithRandomParams(lambda angle: ResizedRotation(angle, target_size), [(0.0, 360.0)])\n",
        "        self.randomize = transforms.Compose([\n",
        "            transforms.RandomResizedCrop(target_size, scale=(1/3, 1.0), ratio=(0.3, 2.0)),\n",
        "            transforms.RandomChoice([\n",
        "                transforms.RandomHorizontalFlip(p=0.5),\n",
        "                transforms.Lambda(random_rotate)\n",
        "            ]),\n",
        "            transforms.RandomApply([\n",
        "                random_resized_rotation\n",
        "            ], p=0.33),\n",
        "            transforms.RandomApply([\n",
        "                transforms.ColorJitter(brightness=0.5, contrast=0.5, saturation=0.5, hue=0.2)\n",
        "            ], p=0.8),\n",
        "            transforms.RandomGrayscale(p=0.2)\n",
        "        ])\n",
        "    \n",
        "    def __len__(self): return len(self.ds)\n",
        "    \n",
        "    def __getitem_internal__(self, idx, preprocess=True):\n",
        "        this_image_raw, _ = self.ds[idx]\n",
        "        \n",
        "        if self.debug:\n",
        "            random.seed(idx)\n",
        "            t1 = self.randomize(this_image_raw)\n",
        "            random.seed(idx + 1)\n",
        "            t2 = self.randomize(this_image_raw)\n",
        "        else:\n",
        "            t1 = self.randomize(this_image_raw)\n",
        "            t2 = self.randomize(this_image_raw)\n",
        "        \n",
        "        if preprocess:\n",
        "            t1 = self.preprocess(t1)\n",
        "            t2 = self.preprocess(t2)\n",
        "        else:\n",
        "            t1 = transforms.ToTensor()(t1)\n",
        "            t2 = transforms.ToTensor()(t2)\n",
        "\n",
        "        return (t1, t2), torch.tensor(0)\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        return self.__getitem_internal__(idx, True)\n",
        "    \n",
        "    def raw(self, idx):\n",
        "        return self.__getitem_internal__(idx, False)"
      ],
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "erq5KssbGY8r",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ds = PretrainingDatasetWrapper(stl10_unlabeled, debug=False)"
      ],
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0hlmfYcqGY8t",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 113
        },
        "outputId": "8496285b-913e-4d16-cd8b-a4a15cbb5c0b"
      },
      "source": [
        "tvf.to_pil_image(ds[idx][0][0])"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAGAAAABgCAIAAABt+uBvAAA3D0lEQVR4nHW9bahlaXYe9ri0VrRX6HfD2aT2Zc7BdRrdwl3KVOPuZLrj6UQzSIM+MhaWQDMoAscQSAJOQv4kP2JMAv4T8J/8SQzBEI/yJyMiKzjEEraMJKQxPcr0hG7oHul2qDJzG84xdxe8W9mr4dnDWoL8ePe51aMop5uu6jr3fOx3r8/nedaqv/BbH/XXx2F/MEvKQlORAHIFQgOiKgAiwGy/FVFVETMRJIAkIhMAAIUADgQFUfq+39v4yIZBJLwey/D07dd3+9EzzqfzdHt7++zZ9959r/oyHA6yG+e5kiuA8+nk7gh2JqUMUFGxs9eJSTIzRWQ4Djsr4+H4Ez/7la9+/WtvfOXLx8PDz42H3X43HI67YVgX/+ff+PVv/PJ/4M9u/5v/87ee/sIvRvLj3/2dP/wfvvHtb/6m9vbL//3f/Xf+i//8+u23SxmsdNIPa+S3vvnNP/zGNz/6nW/l3TQe9o9eHaUYRANEOo1pRJftwlMATRiASABgiogIFAApAgCJBKI9r6FAZEICUAhDSkCIEECh2o4xkbGdtAQAUWDMzPYuGWw/FhCBrQwB1qgBjfQEAgG1aB8MJJCZmWRGckGW5AoAaj9q9j91PQByBf49QHC5kRGRkZDtK/35j0xEilY3Uzv5uOQxYycqQK4LAAsIoIDky1eJq6qIaPs4rgSwPd9tb1ulIGmB4qsKmADd3hzEJFXataUiAKhoCiEgE2jWyGBErKRATVMUDJy9ziSRqqKdJMTJR3gSgC9eT1OJf/kpHn1uqv2TMSNFoIOVwzC9nz/43+7+raSaoV2QiJN1rpntZoSgKHJt/xNAREBITtVlBwx1GQ375D44JDoogkBCoYEOogQ0AQgEsgAwNQEykggYMptRGIA10ZUhbbVElwuE9GpqomHFAGRGAhBL0RSEmYg6wMyEeiRXJuDI4FK1V3cGzu4eEYqiEkhGIoLIdaX7Mp9OvWDXv/DyaFc9d+z6shtGuxpJLrNnQMtmQao612munvmZO//SblbgPQAkc6qyq8tDxbXhAN9H7hQmANckRQVQYTZDFNtsRURM21sHM1cHIgNQYyQ8BIYgVCGZdM8ImJVSRCwjc1mRISoCiCogTECgJlzcvXpMnghI+zwCayYRUJhZKf3Q73al3x/2b7391v76EJkkASQZEe7VjkPzrAAc4ouTFBQppr2EybnW6b8+BfnyYAKRLKU8evzo49tvvchY6sSADL6O4Ch4pNgDBWImIBGBhDBVNhfTaOcvEIgS0b4SAdABEQySLZYTLKKhTK6LB7CTQYdBVUjSF0SKqKqpCAIZhFiYrdUr3UmghTpkhJOehCjMRMWs25kdr45vvP329ZtPzSwRJCNB4AcrP+fOZYFIi2ozfXU2b1IzMQNkck5zZUS7BwnJZC5ehn5/fRi/f5xP08pkVhk9XotlrzoKxsEGSQkCYKauKcXgsIJwqF2CLJqfAYCpkLkFoZrthiuQMGSAGb6kwOy6WA9k0oMM5hr0uxNvcmUsyaSmmi81s2b7lBBmAk5IQkWlUxl2ZRyu9n3/aH91vd/vx5GZGVxJkrlwsT/x7nPiFVCSznBkXdzdy5OEIIE51omc6uxzza/FS+diiErZDa/8dvdH5bti5jUe7MiBHJZlTBZfSqIoTFASnYgFCiAOA8Qhji6kS5iKqVgvQhhggIoCkK0UKC1TZVABA6y3QNCXeaq5cPXKaTo/n9xZnS+qfzLV8+mukmjJRo2ZzuW81PMyBQKi1uluGPZXu8N+f7za7a+uxmKDiGXGVFevRMzfP823/zLdE/ED9+luqsD5/H1OFYBYjzJMpDNf1Oru5OYKAonVk1n6UoqVYYfSAZDC3DF3kgUoZl0IAgIREaQiIZ0oFBkC5Gb6AkJFAimhuSZEAkBftjJJ9xADEwmRFjkGbUFhcXen+3Q3nX7ntiImcl59YoRoUSgUKlvCjPRmp6UvpeyGMvT9delL3++OQzEJJ+m1Lobl9Me3OyszKYAfJpmH5d3Z/W/7GrW6T1MyoB1Uplo9vU7TD6r/u1yZtAwA82ny00kizQaY2tVVTLMMsAIpKmKmMFVDK2wy0VmzCJghYitkRFoiuBgmQzsG2Bus11KkP6pY2YJIgCv60pkBIMNJd5/upk9uvj/5MsNncnIS0KIWSEQ279gKIim9jeNYhsEUEha+BG0++Cffv12I2atnAri9m9Zkl2D189WLR8MbnllrnWt9Uf3TOv+b/CURZfCFOz+EL1xqda/GHekI3H7nO/PHz3fF/PYuMmzYDcejlIWDqolY33elWCmRGRJQERFVhQhsALZCudUu0CPCkUgZE6TpaoJhxG4UQ4GKyuZkgBYrvQngpK8kOZ/rzc3N2efVyq1zWkjFUHrBFhEc0epNUZQy7K+GMu5BNyYC67LMdxMOcp7qtDigDhSDvZ4ikoHjk5NfLyv5wutU59M01cps+QRYGDM51zpPL3z5mzuSzgQ+/ta7n/7O937s9c97/XSNUzkeSqSIQsxwNWaxHMfoCwBwRcQKXa3r1CAj1CAKMSCymU9LASuzTw2kBcqAwfphUC4FVutMhJqNx9HeLIxcFufC6W463Zwm9xUWyFLMI5nhXiHNPlvOJBRi1iKCmQUy6LM7Mpk5PePz57fT+8uUyYjDVf/Tt2UcyjidplpX97PIbn/YjTuIXGpCAOhL2fXW9YURy3marHf3b//Bu//jf/ir/92H/+B/fv1X98f38Oq1lFeGw6eiIrAe3YBxxOFxmkG61edouUMEKmIGsVS9NBovaweQcBfQxMKAXrpx3GGgL6HzCkWx3eFxuRpXcp7rXOs883yazpyWAhMVqJiAdGaIqDbnDKiompnterPOSukp6tXXhSRlUc3p+en00VQ/rlPFt998fPu9738xf/zxuHCazu6zaI6Ho41X+EwHo6ql74ebnZmt5DxVfVJnX/6PX/+t//Lv/bXnH978wptvfvXrX/vCq9e7wyHzFYH2LB2vxjg8znGPvqd0QV/JgMJEoZ11qSqjQO2+oUGmMjBQqALAkAoMZiaSqc0OOpO+iMkK1MWnu+l8Op+f357nyoiEJaRlOkBIJ1BMzLLVXgExMSuD2BPrCxbMikqfJ3dSgA+m25u76ROvKUXl2fPnt1+sV/PefaEvDsjuqoxXOzu24KkQlL9fHo7jw/Fqtyui6nSrPnn9jV/+5m/8a+9a5E/ePHtjmlGsDEMmhIYswzq8znHPw1EOVyniXv3VzAAGiIqZiVwidItHCPU5LSRoDikptnVsxYy+NvMSNZjlU1kWn+Y6T3W6qafT3XmZaAogMjMRl6jPSCDFrL18+wFATLpSQokbmejfuz09vzt1fe/L8nyqL+jFAic+P021vuNv0zNYFxv6cRz245UiRQ0KUdsNw2EY52F3+NKrIgKoc/m/n5/+9f/lm//2N6efefPNWXUibXi4e/jl+QWE1nEYYtytV0fuDzIcVsDL4LlFNYjQegVgBQogRSxz6dSMtXjKaKWYmWZGRGoGI7j4GmgxCMDqy3w6TdM0nW7PtdZkS5cIMC61D5AZhFoAqlt3LwEV680GXUXEZIXc1OmD589tGBH8pFYCUMWUt6fTXJdlcSd9XZVqpuOw69CCm9jQl2Hcj2Oth4dXo6qtGd2qt//kX/zu8we/ffpIvlwiTMpuNw5lGLsdHviu59XBhwOHA4fDOoy82ntfvJRaymRW+37WMlmpsIpC2VFLSL9KQlQlWq4zs1LUNBBM51Trebqba3UuXFd392Wtp2lyroCVQQTkQlLMivUQEaSYtXbkRa2T++Sz0xnpCwWxG65246tlGM12a4BLFYgAxUxEnP7JNE11np3uzmAkWIlEKb0KTLSU8rnr6/3xMIyDltINDyH64nS6ff99m+ajin5/enI8fvHnvvJjb72+G4dSyoO82vtux/HAcVz7/aJDpc5hlTbTati04ExU1znFqXNijnAVblE6zaSYiaEMxcwE8FqnqZ5Ot2d3RjrCF5+muxd3d1OtC1J6FdXMzAw1QxEzVSsKBJK5LsFpqdNSHXGfgEpv47jbDYMJCLg7ci2qRdXUmDFN0/l0N1Wvta6MNYLOiBz6UtTEpAzjeNiPrx52V3sztWJQzNN0vnlm07IPxPPb1676L37pjadvvTVc7UpvD7xc5fDqaiVhEbpSnEa3dVY61hnuYIXX8JokECpICxbQkJ21hjwNgBOR9NmXdXV3ZkQ6fa61+sKGBpqZmXUdEAIBQHquDUu5bx233k4g1jIpYGImUvp+fzhcP35yfTUCWEkBLMDFGelBd09uj061LwIkRAPZ6oey68fj3sz2wzgMQyfFqz+/fT6dbqW6Bl97842/9OZbw9XQWW+dPeBQeLWDDaQljYv5gnDhArpykbWCnk645+oRDACWLICBJihmxQwZQUorgX1Zqq+JlXRGXXyqszdsTQwGVSiQwQzSF0Zkhpi1loURG1KjsL7Ylp5RivXFDof9a48Prx0PgDBaVGR6nbkwEZnksgZImsluGPRy7sgUQRl24+FoZuM4vjIMorbU+Xy6q1PVaQLw5OmTN954/eFuVIGoCCFOkNhBCEmkO7NmLpGMRBIhKhCkZSeChCUL6yBrsdbTbt18Lu5z9VqnOs1cyEQZJBlMr8EMiogBxdTMsZTFmRAgc0PzRNVyTVWohllfyn43WD9svZlKse7h0B/3h1877MdhJD2QgERIp0BDuRI+z8kVkcMwFBvQgrSImGnpf3Tc/a+RnVnpjAkVmOowXu3L7gtvv/VPuj8c/+jbZq3AGB4QSiZd0hWuXimUWAIOEMGAZy4JD3GA1YS7xXeRO8WubzkKiAwySb/z+W7y6h4bosrEypi9rgGoqWkZhtKXYlaGoZiUUgToAgBMZCi7Il2R7lB2jw6HR68edrvhvjjtzPpSHg7jcTdeHwezkiFrZCqGcmUiTgYdCCaRUYZSbGseWxtsvZVimVBEJ1ZKGYfd9Ti+8c4Xn375nV/493/xL7/zxm4YzEwFZRzFk8nAAIh4MBgKKDU9wQvMCsIMTB3FWC0nQx36oZQChEhmZZL0rLXW53V2+jpnVzZoiHSuzATQFTMbzDpEvwMdU2YAAhFzQtWawwLW9/vDeDw+KVagxmMCUNFiMg5l/ztXT+cPE7enqc7LXGA7KzsDIjL4M2QuZKwFDQIKEUHpFEgRHXpBSMIUafqwL59//Phnfv5nf3L8+V/6W1//1he+Z7/SQ6WzAlByyacdcmEYzaykriScYDAJugpEOyHKgN0oZfUiKKYCChWiSWetdK/T4u6+NFi2A5ACE9PexLuoXJGtmjTr5Uq73kfb5+KVFIioroLORK0Us904jOPT/TAkKcggM7KVrGUY3nj9zU+GL732ZHp+e3t7Oq3MDtlLNw491CLS1yXnuzUA0qx0Zj+pJqU3YFeK/LLybub51F8/2e8P333rreHvjOW/Pfz0z/3kz/zGL1q/A6AqgAnJ8DU101eRVAWWoLuQShoCAUlYyQIUZpfVRikmEikKBOiRK1fnusx+XpyMaHxIiomVIqXIc70U15uzDLuCXTHpZthgEslZs8Wd3dU47t/+4tVYhoFD9X5ZkQlpzIBZGYZ47cn1V3PvxHsffunjm9u5Tsjstnof7mR1v3NkOlMU0r+i9jdEBKUfylDM1nqX1QW56+WvvP3G+PTJeH34mV/9xUdPX5dibEgWUtJJYzATS1oXxLp4ZGV6gMwFSGtwSLEBHAzFshQzMyDAzMV98VprrYsvzkxf3QNezOjuzpPPt5VkIMRkI0fMTDUqoNhBicePSIiU4fH1OI5X43gcIVqD8zQBoDvZ74DO7Lo/PDpobx25jl/uf/KPn704vxkRxQyZw3i8HgqAWqfpXJEs1pvoht4AZRwPx2PUOt/eDnw6PjqM87B/4017UspQYJa53Q8RE5JJohV06EEg3Rd3VOdCrKkAOFrpfCkFRVDM7GpnyFh89SVXkvDF53OdK52Lc3JmIpkIrx5Zp5poiFs2gEVUpZRhQEEPYAW4LNL3ZSjjk8MwjJ3ZStIEAvoiRUhm5DBYKeXheDUMOwCPjvuHX96db0dG9v0QywLFMI4CzLdTvT0hwjrZ8qAIAuXq8PDx8ZPfP823t395/trDN94avjSUHx+tmEDFSkAyEJEiEJJkdb1LMfIOqTWmGXee7soAFUjRIDXMLAWdSm+diWYszsVJui/Vl6nO1WtCiVO2inJrN2OlEwJTg8BMTGCdqmgpolCRHuChyCjDeLW7Godha4lF1CAURTbyNVo/bH3ZXV2ZCICVbqLMGMcr95qB0lsRJblGoO93Vwcbhtb9iUkZuh97843/6+YfZmRG9ENfDuNuf7j0zsjIZjfSD7KS7u7mM2vjUSuWSWfHksjAoqoCWVOR2SjlztBgmzWRgC9Jut/SF3qAIKSKSEO2YZ0QYpbOiPZCUVUt2nJX19JwsbImipVSSjERSTJJBaQ3swQQkRmtOme3OI6iQzHvOytlGAbtduNQvGTETtUikpS+F8AGs2KdqShUDGW4fuvN/d9/XQViJqWUXa9mTjT+PBHMSCggUkEPvuaTQYEQxYR5iuqYWjS1WDsRMBHFYKJFVbZYK9gIl4Xr4iS5MjuDFmAVG4AiqgY0+kvVIGJmYp0oBGomGHoJiIqawLT0xcxEJZmJhEoppWFTG7eDIOlcEgRKe0OzrpR+P45uBtEBiOluxmzFumK7/b6Uoj/fqZhAbBiu33zz0duf91pf675sX4FYB0Uk1gwJY+S6eKypspeZdbLHL6JYCpMCmVBfwENDkQUiooPyoejORPqCTiIyyVgjfMnFk84zmQk1BTPRTl+KaV+kKx41EyoCRCM9YGpWdrt+O2bdfFGhIiraAYBKpxbWftvQN8tMkgKV2c/Pbqe+evWp1kxABb3tr64UkKU6mb2X3bB/9fjw8fGt4WfFVFRUTIHhMH7+p97xxcvxGJCVq/Z9BshVbJNFaGdQiOc6Z61ZxkART5Ep50koCBMd1faCXWI0MYMdTE1ThStz8fXOSUSLxq3SVeO6JJAiYuiKqSnXRuxLKcNOBiBMxMbeSsGlLw1nIg1QFRWNLaNbRqhqt+kOMpNJT1Vf4vY5tDNmuDsgO7NS+vGwN0FOAidY9q8e3/jSO8OT67IbDda6GTEbr6+lMzpXBuk+abkaM4MRKnLR8wQAWYIePMdwJor2TE4RlERTKEgYooiaYTcMpUjXFwC5MKaFcw2vXu+c7stCYs1kkPTQ1kcj2S4WnWqxBuupmHXSSVN+CBIbRYQGjLazRmvBE03poSaCiHCPzgKIlfTFE2CiQS5WSjHTyChlOIyllOOTx4/efOPR8bE9GaUTlQ0y7vp+hCZicc7TlJEIwFQ3uUfkSu0MgKy5TGmy1tSrEhmAiyOlqEFo1peQEVpM7Emxq8FKD4C+cPFkrov72eeF7rkynKtvXsbMYCbSSQfCSq86tJYdgJqp6AZ54KXMQrQDQkXDoBEgkE0b0bARkOva1A7YgOdQtSI2FOvNhoK7SczK8QCzv3j92htf/KnxemgFTnNXAIooV4NaV5gA5joBkICIUZFERFoHBIRIBmcRRI2MVGRCTCBhYgVouI/JriE56AWVJNN9WRb3pWknvNESGRmxIJFF0ARSraVDQsREOzPV8rTvTAUIvYDPl4eKiiATKlgVTYekoqIq8vKHYBrt4xCCBHYmamYmQsDMSl+klPH6SXnnUIbSlf5eLbXpSqwr45UEyFzpAcAEKfdODyA0BQABIgUEBg2ESaeiagXyWtBgoiIKEZWiAqQgSfdldq+n9IT74vSliaCSGYl+IIkkRBJBQJKMq8GkDDuzrkWfzYh8syBJaZhEZKxs7iY29gUI0VZdWClNGQKEpiI2TUxjOtsZdmav7PdlfxiPR+0VpYcVWIGpQGFbXhCRUsouGxYRFkJJTSSg1kORgQcAEkEEMzy4IFShop1oUQyJkg23gKpIu7DM5BokFzrpy+K+LvQ5m0pMW+SAbN7ULn4NQEX63obezFQFGXJRQW4PU9ui9MXhinXWlWK9ipl2nYm2xF6aEkdVrN04gfUF29XtdvvDeDyWYYAVNVPVJrhTEVHbPNvMehuO4+64h6go1ExEW7HW7KgVdNCN8ArALgYmwpBEBxSBqCI8a28m60I0SWCS9Mj0CE8klEgVlGG0/SiH0cYhSOcqZhAd7WjWdV3pzCQA69UEkTBpDtiXAmnCtYwkkAKY6GoasR2lmfUqXTFBBAJMMROzzqyzzu9emOGV/Xg8HndXYymDdQUqEEFGUKi02PQRJLVX074Mp+IEV+cCIqOJKJtoRa3Blk3/o6ot+fSitoaRCuqulGa9GfBA0zyReZuZl/otEIrIUOvLrtirx3K4slIWh3mnG7gsYlr67YbCLmFFVCRNresNQIsKYCYCqpLSQRpZJopOVEoxk4zelpWmLTiqqIrO7jaOw/4wvnMs/WBd+axQMzfVKTIyM41EhPRqQyl1SFRDOPxlpANEJFTKZtBIQSv/B1FoqoAiGySHi6AsqpNcnU7Wyql6o9Ay1MxgzYbXS7kNUe2swYkmYpd0dflV2v00VbFuezbunzJkqBa7KCegKpKZTJhY3ymzlDKMQ+kLADHr+h3snctFQprY7+LIIhaMTIoYSGQiRLSX4dTp0NUUk4AqglxF9IFZL3qpZhUKmnYqspPSyeXUFRCBCiLZdEdLLu7TNL3wxRkeYGYgoNq129k8NhOAdWJ9b8VEGgolG8PQLFlFSm9m3TDc47eb/6t0tuEjoibaKBQkEKQkTKyM4+4w7sZxN44ArBTdFdnyJyRStekLdJPstnJz4epOaICtCitWVNCXoqotAGREZjxo/tVZEZPerFgRQVE1bXoOXBiBrcddudIXd58XOlmXudK5yVw1EGjnKtvNBiDabPAi4Wuq6lYW5sXD+qIAzJCRomBrHLcaQVSbIWxpqrWtAhTrzPrSl6vBmkdYJ621TbA6V4JERma7WQ3QbN9Q8v5eCNRUVKHalQJFZkZm0kUEjfgbSl/6sYhI6YuVAhaga42jFR6wZJhoZqyLT9PdNE2T++TVGRevFbWSmRQrWtRETNRK3p0yIoWJQl5qVYb21i66SbahAjKapEZUMkKx+pKMVJiWzaxEMyFAZJpZGYcyDqUUiLApc1UBeK0pBsEW50TFSrtvRQxDL7hIVZiiImpik0RsDE1GRpqUB8hNnlCGXdkNZRisLyYYQgeoid7HtmZBjHTSF051OS/0gGeSiZYKL8aTAmlgggDQTL00OHmJMD90M1uMCJW1pRAAohkZTH5GrHsfwi6Vt7Sa0LastKLTZmFc2ICqhWRkMpqRt3bYIDYURQTZ3quZqOil6ohEBt3FujIMw27oS+m7TWdbulhFG08W0m1aC2aCTOfiOfu88aGt8FABNCJBQlBYwetkwjwTghAJhLaOIjODlFZ9bl8mNVJKafLg5s9rZDgZRGbzL1UTAbS7j71WTE0gwkgw1/CyhnadHkwVmcxQMLNIKhAZkkmuotJbkR52ERs39kW3eM6F7hQEJJvt9GW3k76YqFlngocQQ0pyMwlRItQDobEs7ksl52XxSxOPS3ThAggLkPTIzVxVtZjN7rgYByIhugkgM5tndRkQlQyBRCC8VQ/RgpkCnUlzFYhCpWu1T+lFtaFrzMx+/VHR8SKq3yw2CVggJYNkZ1a0E8XWnb20T21JYE1GEIkwfdBdxD+CNOvMeoH2igKxbV4lmUSA0CDpnpGIKH1fTErfJPSbAHizswY9NsxDpDPbZGq62X/TWr88LLPWu7Y/WLkmmRF571wN3muHaib3iJ0iMxpo3DxRSskMkBuU4GSyoUi5tW7IS2gGqZ9RzN1/ISRIRuRKf6AmooamDrC+U5hGgRRkUWuIspNOj2WhL+tSkzRVE4xDPw7Drm8Mx8ZWSIsFKlqK9aUzayCYWGciWAhnClQv4bNl2c5gptigpeaGCbTRHdlIGIiIFJNicp8TyXVphdhWPPxpZjIj2r/MhcHNmtvQAr0N4AQypZUaABTWSllVRKaTXiMgpgYRUYMKBpiIiHZBSyCoKoyUyGS4QrmyIjJFpFgRhEeawFUjiUReyrFNSNyGhVTlrpmHfiZIf8a0O7u39syMiGj6BWc0I47IDCtFzAyaDT0yhQiYtEtd2VurA5jEKhfOgBlrrKS1TlsSkcg/B0lo3y+i0SfNth60KAjRTYTVhJMmXYReZrgaFOz0ysUzV0ECImH3cQSREb7JH9opSDBi8WjOkvkSxwaQATNkIhPWiUpT5jfUJ0lWz+oRrS/m5j5mZr30nUDUzLRT1VAkiTVEYGaiisxkILabde9cyWCrXQPJ5LrgM3JdRIg0VSqTjJXtrCUyIdLE9GjZLu7bAWypMWWlR1CZwAbylIYQJ0W1zTIlWjCMDgVbysutzGlV+p/3ELmI0skExDrUinaJzPQIUxGJSEBkKI3ubDlZAolAIJCdilmHJtgkUfoLgZNc2Anb56uCyUBIyEo22LcJLZrX5gb0bcJ56a2IwKx0pRgC21hcGwELNKgoG6SDuK+D1RipaGiWIDbcp4VN99pzydjB9Ic8KnlBC7az2TJ9qxIbGsz1UidlZrB93E43iT8j9N4OsUVxRaZsXxdA1zWHTTIioeLuYlYABERRxDLSQRAdCbPPAC6IaPgrkqSEiJkNfRsTEqhAW18o0oae5iRSAplbTG2+IIpIxGKAqKr1kRHIjDCRXIKBgGYbgAU6K+5Vqc2r7mchRWWbNWu5jLk2sxUFgnTGff0JBRSJkGRmQoo21ytpF5AmVV77UdFSjMlciIxtPrMpRpDmEpItTsuAwgSZxghRM1VZyW2OLJCRD1pYEzFBQ6FCkMo1k9mAjGArggUqGdHKChIQAgQy4j5v4tKgIj2Tn+3gtYU5uUBRzQQitwFENMXZ5ctdUrEq5CUvhMiMJra6jNHe+2nz1Pbpqmpy0RIzgCC50unkwsxo5fS60L22lHY/fbKxq5sRh0jpNRsIDFEYoHUpLcYy1iUis8lBtgknrlyoGYgUlWzSw4BKm9pEBouObdCXgKmoJmqzl4YbmKS0SrrhPiiAKJgZmcHmXS0RiKoUU+taG9/MN0mobTELgG1YWt473QXtyk9du3aaEZTIUEhuGTYB0FlqDbOLmSIXthmplIjMBy0wXWxYwYTPyNDEyjUy12DSG3qXzJZxg45sc3db3m7/Ni1dNiEjWlY1KUVFxYqImBkuYjmsBNuYVSIvhpkRsb1a1ETEVCUaE0Mkkww09VL+f+N++3r3ZvWnW/wVbEhxMoiVADqzprbKNV8WiZvhJLgGPemtRTMVQyCipjNIIcAECS4RAebqRHJdnO4ky5bLkLlGuyOJjEjI/YcFskVhC2RvsYSaSZNvSubGgESb7SYcgCSS7TZANkYel2tjewGA1V3MWvba3FFQRJIM5J+u+cpDL2YC/AiAde1e29w8k3BGBCI281SNiHS3HhlZzES/AdKdlmg6J6gKSlzYjgBDot1JAooWF9zW5spcmsm10oWIBJiRQOZKhpUSZMLaZLyKQhQZqhRTDNtocDBDIknRC0KfeckCW1mg1mjuiHACu36IXXOkSG4OiM9Y0YazBcOXrXbsuh+s618SaYMAGy2ejOrdwUQEEZkZwcthwKxD5Dqf1AwiD8z65q+Xu54Z6WsGVwBrbPVWIGLbHwDbhH9Y2aw8VVSQTXbH9PVyhQBCGyi2TWBsSwU+a89NncRsIF7cN7Qt4WaLr0xgDa7ujcPapL3xZ30DaCG6140gxI9cqORGhogil9XJTEB14zhbGbfVGgrAF2et4S6X6GEIZLLpbTLgEWvThGlD3nokMlOhCKoiAEZ6rYkofYklBDqLRsDUmstt564Cpm2Xk0BIChRJphNopWrCLJdNnwMwGKogmYuvKsISDNKxvQAJogAwgWhu+aslm04EZgquZiLMIwApvfmSQDKD7lzY0p6TXTLQWpHGHTHn6oCUIqUMDTXEdk8StyBdycjGAUhus8zt9iYy2TjiCwbV7lsypmQCFl2wYqvtQDIARGSbX2kt28veOS5bHCLbJhDmpbRo1+Jy6UsyIwKqeqkMNsMJoLWdTWa/0rdSl8x15cPqdsGzg9Pplkuuic5aYxSLZ4qN4qJWGul24SDFzDrdeioRBWdJrlyQuYYLuoykpaBFmZSIzTT+DE4gxgvKsMuGWW7ZIVsCYiCtldKJBCyzpbCNoU9nLIy262LDPdOrsy5C5JPICGye0RaKJAREmGANajMIrr48nD/VV7pQINZ15bLzzt1EBJmcfHpx173iSZpZpCSy3FUVk6EMV2GllDK0rlhKkU7FzBpSpd7EeYxMBKMpWwBpYTMyEG3FRQPgMte8MNcrycQacclhgWgWoeFEwN2FKgdrpeF9NZYRculguHrGZhctJUVmuyuXarK1loSZXAwpFMakhRFOqruYfeqhQCz0qZ7P+XDfUFFOz2/nmym/e/voWDuR9j2slPyrvQxFzcVubRjMBmkCr7aMQC/m2sgJROv514SkZKMRkJHhKh22pqy1OgGIMyZ6jYikbBPAlrkiMhnrUr3luZb2IhARIqKazFbnbk3x9t/IyLg4sJilantSRBD0hazVjkfcEySCzAzRJN0/7exh47zoPt/NMsmL83U/ZD1Nz997/q3b8xrxz4ZRUsHFzHb7R5//h4mvoRxGFen6IgZtckoRVRHJTHC750CmAjUZBLPs7uuRi0sJMrm2alA96PQp4dlEGVCoiiGC7rBkItZgRHLpaCULmGlJ5vZxftGzkLrxYveEjIjZ2poZVVWlN83oOh4O3aW1jgaJi3idAbjMaR18dXq9vXWEDtNcl/OzTz56771PTtNCjiqormIP+6Eff++fT2/+9B/89c//3JfGcSyltFUxPyLSNOwhzKZFETFgutTB94RkktTFI9OQYp2prMRMosECZq3dh9h9I4rUYKwLkumc259kRA7uxbI2hCiR0QyhUcIBrLHRJADULElIY5AEwBqcpumDD5/nO/jCm2/YUFoRrrAEQ0ESta42C5suJ86n86ofiNjzm/c/eP+D22ffXz2vd2Mx25lFhkdQ5J/95j/+W9/4WfvyO1bsR18ZfrBSRIV0BZILAIOamUKjFa/aULQVIQlmCkiY2sZQSyDCegmIxCeKjCCwUwNCEohl9cJLIkPb/UISMHJVSmCjnDWCmdzSUFz6RRFplBQUet2q4QjndDp/+/ff/YkPv/La16/HlnsVBoC50iFIrmG0BJlJns+nyZc18ub29oP3bz65PcXkvt9fv3rErs9MX/LMrPnbH737a9f/xuuqalZ+RMKKCSDBmoGidqlSL63QNsW+ZaJIKjJg2XJTriqKbH0jomGu27RyU5hlkBQkDJDMzEhm7sigNbZ7a1hCXy5xEmRsSVZUGjtmZo0P3NA1xvkfnD64ufmbpy/uDy6m3ctePECuSZ9qh9ZLczqdzgsn93Otz589n57VrPU1snsGOR4iRSWCAZM/+u57/9nf/XmxznrLtEYDIQPIVdRMpQM02KBPBdAbIIRG1JRo0ubYom2TxxnQ+JM5t+uVTbxCJg8QeE7t6pnoAo4UhHEjo6Vr24YkBZfa8uJfolBJ2EVAAUlI2Srdjz66ub39T157/XHBS+kAAiSDmb5pdABMd/Xb7390rrWufnvreVpyqhMP12JD7UXbchvX3j76zvvzNHXjTooJbUsKW8RAihhUYL36HeCtsYRZkJv0p7uwLWglYwS0tWO3l8ovkWpFW2keTEgERGVFSnBt2H4im+rmAuYltwD0Q/25XuzIGt4Itc3USN4++/7Hp9svfr3q4aAJ0U1EEMQ81SSDTCYQ0zR9/NFH372roTBETnNUlzw9x+G153YYlw6mw6B35fnN86X+7+UwljJ4XaAQqEqyiTIU0peCmy5k0wWKGKRsfbUsRa00JIaa6bGJyZIMTwYa0SBF0OCYSEILEDATtmZHsuFbW4TRzIAit14s24lk5IUO2G6ItnjehCaiRe3jOn300c3frhRI2+AARd22GW2zBEt19+pep7u72/PZimnRyDp7ddaZNx/w8Xg1jmXYV5TXR6/rmtyAFoWKiQEp2qmZSQeT6KX0YkPyJCqXzxVBqloxKwHAIGy9G2AJYxKhQBM0mvXDwy2XCckUFCseTXn1GaI9AUOuGU2AEYkGtkr7588Be+4lIkWkTnffe3Y7Tf/Vyzc0Y9NJNk0X19nr+fZ2rvP0j+bp7lRiLFi4TNNSAzm51+XDD87jfhyemo3L47neNRi3U7GuByBlP5qKmBaUntxcXbcAcYGSFLJ1R9Ato6ep5ABEkCu5JjdgvQkAL5KobfEQfasXAHJZx6JaUgBfEJdiuuX7bcXDRXx/saCXD4Mpyt+zfekFuJ2m892plCKQDJqo+7I6G3vY4DKzYr/e1A1092k6T/MCURdhTEw6zh/dXD99v3vtP33KZb18UwCQ8dWD2a6IIilMpYc069a2aC2h0NTUSDbQNAINhFMRS0nkbauN7xV8ampdJ5ZqAqhaMICEWZtudffODNAMZjaZ12VpUYZcsJGkZ+Nr2yOAodVDMpZ+HAYkPznd3n79/OjVIwhkmol7a4+3bjDa8oFSTM3dJ+e5LpUOgap50pMTmbj5CK//R7e3/ukCQK0AE5Oyv37SunldGNMEMdFNppobib4RbC3lfQaB6QKtIdjARVXrREzsXo7b5OStF2+6psiEU8k8RItkDeZiMtEWnjQdFF4eysu0tt3UYrY/jL9XbwA9ne5+6dnt0+sndqUpCG4kwMqNSkgSMFONeNcXj0SlT3QAxQLWJ32NRtY+8/p38K8gg3GRXcn+1aO1XqxwIR0hYkyQnpFQl1JU2jy3tnJsTTYBExLrFkWlTSWqmqkIqDq0E9qO00yky1xXMiFdbBEnInBPdTKlQU4iuW3l3GLP1nM1WMXU+jKO4/XV/hb5ye3tx7c/e57uxqsR+CzFkisZjQYku++IwugTMyb3F1y6Jk0XmhjXxddlW2X54C8Amhfg4MHheNwfr/fH4+5q7Pu+HVYG78mY7QRM2gzmZ5YzCiCEEJlIsdL2IZmgSNvGF00qt4mOFYiIaB7mjVBCJgSNjm9ig2gis0aVbDXnBj7HZQq7K/3+MO7HXoHz6fzRzc3t6cxgy/SfBZvWaIVINn1FknVxJyt9zU1Y0+jJU52nejeMg1lDltckc/EH+8ePjsf9/vhoN46llM8uUBLdqM8t5op0qi9r3Fa8JpHCwBoEwgSdAgqTDY3IjA3daUZzkfxk5lo3cCNw4aJUs42nXnb/NQ4jE21hZS4uijKUYeh3w9Ah61Q//vD5Bzc37lQxgbZhmaY55pqZ0diUYgaVzFwRiK0BBGBivRUAu+HqJ37uZ8fjcc1wr6R/6nzw6Ph4f7zeHw7DuOv60lSom89bUbOuYSK6LYRSUFr1LAJRBIK+gkTbchOGbHJWAEHec3u5kd4bMNhWv1Av+j6B9G0RHBu32ZRzDQ9KNKrJGyPal1LGcRxHBOY7/+Dm5ns3z3xxgVzmJmBqohp00hMws04hUAJt4mw7HWgHtEn9Ydj9xC9+dXc4cPEMZGCZq4z70ayAidvgeHgxDJ6JoKjrvZqlSaHaYkAsCfrihDDBbauEAoQYM2EDVCESrdeHxlKxSbEb6BnJq2jtzEKaZCAysNyz65uhtbUTqDRjDGzAjJnBdDcOZmJAUTDi5tnzD25unr7+RD4jGlOBmalIIjuzcRzH7lmFjNbDLitHgMH6R4djEfviW2//q8cfT8DdfZrmF9XrJOP+WFrJs4/zbw5iwmhbWsq9J+vmbmqQVG37Uj11BteQE8CthNtQCzUDSmZqJkSTIXkFwUZAClrznmpEZiPgEmtSAo2oAsCVjG2lx2qWd2EHM7EcomvCTVGBFOuYeP789qOb/3j7rtnoZoiadY17yrLYMJRipsBDKwrsypAaiBhLeXK8Hob6V9555//5i0P8SnitL86naZrS+WAYxm4YumEofW/WZaz3yWM7nUvcAcDw3FazaMvJkbleiBrVFrUuYH0DABvZHS3otm5c9DLosaItLr2o7VoOVN3ajtxmHpZavS2+STKZbcKnt1KkmCHWudaPb26eT6dWhtzj8y9VbEAx2w1t3l92VkwbyNRgGu7L1U/93F99ZRjcWadKknWZTucHr1wNpRSzTu43aEeKbBtd7rU7m+UDAWtXl5mIACSYElvvCaDY0NZKbtBiXhh7ejuy9mxk+MJl4faIBGCq7YZgq7DaLy1ht/UpEcxMqshuHA9Xg5mBZJ0+urn56MMbXNbsbN84sgxDq45M+tIPxUxF77eyqUhEzqezCX7yF3++K+Ze51rp7vWFn04POjPr2wyQNEUk1m2TbduXKNi0kw3TWdO5zd1HJNv6vZaSuzbRLHEvWAmE3SubG0DbBlniQvy3NqUlFFEpbdRDG+R0LxOOzAZ9tWjd8n0ZhofjWNqG8MAnz26//e57c3Vsr2ybvWXbHxQQxVBsV0oxs85UQkU701adXP/49Vtf/kKqzHfTp9O8Olm91vnBBgkhMtYLdaB5mTfTi1xFRBt+3OSDS8aqWC+UbEDJZVPrqjG21fOl76FNvgvJANg1nbHqhVfPjXFu2wYUXVPJqJoVsyKmuBQUkKZNSF+WyCxDOYxXpS/Wj6LddLr77d959+Obm5VEBrmuXJBrK0WbP5Qy7MeyGwZRDagoSinjYXz6ztOv/srX99ePJbJO03S+5bKsa5B48BLFYYCtQEALB6KbcNcaw779ZKNt9LOBag0GUiFm/dbHNXuRbardoIjsrNlPCxB5sTIIxCCmfVvHAN1eKN3LIZ/7R0Q2CU5fSjnsSymGEOA8zb/1T//go2e3bMpGLs3QoNroK4ncqRwPr45jkSIrQgBTGcfD599556t/42vj8biu8SfnyWv1+ie+MJMP2qeSK736fE73uASL++/Uguh2SSLo2sY7g2rYZf9UIIJx2VDbsOSu9R+igmgHda+gSufq3mgMfHZxItAkuWbWKCmRtm+y22jIlZHh7pE59rYfh93Qt7g51+n3fv8PvvXe+9OdI6AiDawCjMzZF69OX4pZb7br7XDYH4+vPnr85PrV63F/EDEuc3h7/6VOt16XBykaEX9K+jyzLnE/pA9cBpI/w3s3MZuZ2GXACIA16kXATbWQSBHsmuhUWwjTnVpRg0lrR4P0ZclsqwERqtn6OxEU6ayJoVVaG7fJyDWYrcFKX7gsu3HYXw0GSNuSXO9+95/+wXvfeX+a7rbdeSJN7eKLt0wIaFErJkMZ9uPVo+P18Xg8Xj/eXX2uM/n0xZ0vniSd8908n24fRBv7IZOBhekucl+ubdQCVDZlTiA2cedL+3rJkULWaDUwAYW23dNWSilmpRSVtpx0++Ft2IDkwowAA8xUiDZJ2b1gr03LNuTkwuJHrlxF5LAb9sOoqmZarD/X+t333nvv/ffPE905TdNc3UnP8OTaVEJmsH7/+MkX3nzrjdffvH78eH889kP/qdd6N69z5fJpRsa6xMZPAOEL3Vef/8xV3wfp1mm2c1w5M4MJiKxc7psJbjJ2JrlTQYha10lasTTlBbSQDJHWEa4qS9ghRHKlYfBEiTaqaciEJk7txKSRCCKqphAJJ1eKym4o4z8dir3LwK4Yffrg/Q+/++6bX3xHx/FwuqsvqvvMKTSlhHlmaSDaa0+efvHLX3mjlOE4jFdDseLns59OqzurB2ll8LtnsjI/rfW0eCzksiQXVU2RJCF2byqNq0qAjQRp5pRyMaj20BZAxRftr1pAUbPS9xlt0CYSaWJtLjQhkelLhWqxwl0amYNZ+8sAyDTZUqkoVF8i/EgV5EKIDcNwHIZfs1LDoV1ine6mj5+dzqev4GD/4lxrrSeumeHIy/g5At1+OP71p093V6MJ9oe9dErS60ynzzUzBQlVIZelrff7A09AbJDWVOAiq74XaWzGFkCHXJroXC86j1RkE1wFRuT9CNgmIlEFcyuPkWh6+NWpcFBNi4iT0NLhUoknwGy76wKiImCmMShWDCZM96So7ffj9Xj1yd0UWkoZ2/CDu+/uppuPzlM9ATCzs9fpbgJE1PYDNHW8evL5p6MN/X5/JU0Plrm6T3fneZroi4rIp+51Xtx95Rr4R00Q1EYDsjWWDd9rGEUw05EUqAlEZRLBpRJLi0hJspNS7ttlM7PSymjNtoSmiV0RERLwcEPQ+rJGd1EByL0/igF5UdshGAKimEARIKKIHg/74z8uAjjXYbwqZiay1OVM/PaHz85+2xX0fb2tp/NpQsauDHsbsMjeDm+9/QUpqf0ItMUcQnKuy1wXSUDxoFb3xdnmUb2teU6o3o/TNAd7+ftWT0OccTFX8NK+tdyvNsQ2SthkRyilWN+yVwO90BaNA5uAAkByXZ0rV9LXTG4UUJt/UqDJQhIbAhLNIBuku78ar4dBWpRs2hqCyyJAB1OYQCwalKMKmBSv08ffuTnfzkBpvrKJjiK3vX5LxcIH7stKrg2d2WQ7IrhUN4CImVrTx7cNfIxgNIdhbgC2MtngdRGBJlTysoJa2zINK9g4WxHpSt8DkJazFAhwWaJt0AOi/dU0TdS1Ab4Xj2/PRguGkYCZHsfx+mpsGpgmKlyXOWsq0gSGBgQ2PSk6mKku1W/e/XD647PmBlaKGTOQiJXhnr7S1wfZMnciGyAaL3mr/79HNnHYn/vcfXl56aOagv3S5n/2J+3S4l1etWFpiUaB5OXlqj/cMgMbII4L5C8iDdwR/BBS3EajWwb8IRpSILnCq3O+5KGkirVbmMGMDXX5fwH+Mnk7Y86vkgAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<PIL.Image.Image image mode=RGB size=96x96 at 0x7FD80CA1B2E8>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rt2dA_GlGY8v",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 113
        },
        "outputId": "7043a95f-dfc6-4689-bac2-7b86eb02186d"
      },
      "source": [
        "tvf.to_pil_image(ds[idx][0][1])"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAGAAAABgCAIAAABt+uBvAABSwklEQVR4nK28f5gcVZn3/TWeWuesnJY6y9RI9ToV063TkJ5lGtIjMzITkpEEkywEkxiigBghwCLy0xhAZFkUYuSnGH5EDJGAAZPIgCEwYWdCZnxm4sxAD04HerALp8at0jnjc0r7qKexju77B4+77u677/Vc17vfP+qqX1edqs913/d1133uqnctHVzctTgcf2/n+z9Al1y2w8qecfELH7gSv/67j+0x/p62YiWTyrn2Bj5/ySUXPuClrmS4WiDjY/dErffRPf3vumxjjFVr8AkHTUQfDcSWgy+WDvYPCl6Krrnk2dylD6H3Sdz9KM44iK8ySAcagAtEQIzO01F0gGM443v4wZ04NoPomtF/+ruPvPtg/93UGm1IjDEkn1v36fav5brPn5Bi295n5cPrLy5sWxZVViS3fSF7+BvZFwCoN8B9CIkb+j/9nN5ht3qZokdTxtVeFIogrMZShVLMqEkKtLBWRplRJAyDw+EwTRkyS+NAVeQ0ULd4zU5TcJbxWjOOQ1paPZqkxU/k9//U27bhPWvWXbEIoxqPFtvbAtwdW2OGiBil5lo8kt0Td2/vatrlYXWMTp66//nguOtWf++u9u3A24Bl6Gv1cGdsJrU3ip7dU2f/aRz9uzAQ4XaD4z0wgnAR4AIAxgFg2AI/HYUPoel0lLvx0U/h+Id23vSpyzuAI1JHDIRSLuTgxBs7qTccaL3s1AsqcpNlzXI3RqCF8C0GnQIFhJwdD/1tOKMG5QJaKso5gAQALACUMmo4gUJTjVM3oVRDtyEXq3pApzXVcSqgMi2ESgyhMBENI4C0MG/OKh17Yezosd3fuOu67zy5HY35HHLLsscfGuVChj5CGtt1PqjUhU6dU6ABAkAC6tHibfc/t+tJ7xB2Mvy1jaNz/MsjhcFo3e6g8zcBtvoYIjjAoM5Adwzvt+g/HtlG9MfgGVRcyIWgt2L8XQhSsH0EAv/0aO/jxtwsMKghOZoCAa0zU6m8Gn5KEdqy6gttqadGgi8krgaMrFUHSv0ZXtQiPjo5/Gw4OCg+bDg0UTZ4Am0AAMpIDWEhAUkoJaA0IdSkwGDZSDNtWgI2RUqMWBqKaubLOE0YI16kQWJLalKZEX3fGr5Jy9c13QF800VnA3ulmTuBDCI56lg9SqY98MYGh6FVgzFwB3RN7tZb7lr5UP+fnui5rQu/PoYrtN0xQH8z4X3cwExgoIwwD8GxWmIbBf885AX4FcErQPJp7HGAc4Hv4rjb8b4Z7Pk2bnlg9LkTPtjHUCEm+lVQek+NGcVS3ljMt0XhPgPH6amszN4Qzq5U2atTFqZCMeNUfcXemo0HDvRPxWOR3mNxxGBeygGhNaMMpNBS6xq0RajRALVhoKCoSsmUBhgBHFsze9oxUhFQCkipCJWO4QS0LMmgCPcMTO5zzYfNCePAMJDLpP8AxiLBgUro7PPjLeuT/nNpRWEyQTdDAmBRe9F1Vhzqb7v3jgfHseMQmtG0NMLaAH0xlk+hn4LGyLnwXBQ0cjYUEDpwNITCOgeEoqkdBwwCAD5uerb/qx8/692BnpyRJR3KuYpUEnw4+uTZVdcuwtLlsf2V2liObzQWEpCWdMHNvpSDV0kb9EKbhIIqSBgCwEDXazpSAjAxJAzV2lBKtAJjCWMkiR0fglOLM+I4qWNwjIEODQAKwmCUlMQQ4Xlllh5U9dN57tjbjfKXP0s0RCbjOdw7dOC6CWExDJdrWw+Wcps33v/p9nGGdQKawE17zrnnbXhen78l57Slb/3T8L41G694HhsotgvAQr+DQgQPKBEEBIUYAQUDKnWUgUmNbBteb8f6p/FeBkvgwJO9L8Zyxc/CSlgdC6oSFeVLyaz0UP++KzYcS6AnSqWhsQev2PQ1BMxh4xdkN67JXr8IqQ6vaf+GQPW1TkwOKkgbKQAANGosAQUYEEPhHZczAKCUFekIVL8ToQwo49T1nDoX9ZBBwgACknQVWOZT4ekbqpn9w82b3Nd/P/zsw8/vxxLW9GGesnlTztE5LfvR1FsOrx3of2XNja0W3kyj4RjmC2DFxsIQvnbmI9+76pGr3Epw472PduR7JsqTXT0FAwhAoyTRm0rGtOUAoFjnY9RgbDy4u8UbW4qBQ+h/Ht/PYbVOBieGB8tBtTz6vHh+2GjLfzVpgDRcvzK88/Ph2x3O0ljKw/07Lk5NLGu/5dzk6TXZ1zJIAchksZSt3dXdOXRgcySCGMIBpQYNcJQCajWqIKS2KEASpbUhylZUQ1spy9RsUAGqE22xJlgp1kI8n0rCKQUhn/yGypwbrplSZlO89BvVH768W/TscKED/EwjNjAOLSp1hedWGtzHxoODP8I1wCdCVCqQFYxMISkjY2dRPlA/I33RxSJ3275bu7g4NLrHaVre5QVDo/tkuStK7upqXZfv9BReZJjv1jLbhg/YXkHh05fB49gD9Kpg+JQPIq7Rg31hvFcmDP4Y2jwYyHC6v1z5ldPqKfjRkSgOFXrcNbj+ZdyCP2tRk9W2OOekXSgNogHImnaUVlIHgYKCkkYgdBzqUosDJNEWEpXIRMNoUEqoZpqohFq0iTmUUKQSXSNXXW9n/onyPPQNR1pe+Fd15ViMW1fgXT6+AdQJeinSEWGuhQyrIXXg9trZKrVnCqUZVCewfRyhQry+59Uf3Bv9+LKTRjbs4nvZMVOBFG0meHQUL/X3yeHJsza0t21c6kLMwGc42p36lVtbNTJae6K9axe8Zai0oQokHy90qFAceryPjSKOkIlAu2A7cLQ2ojQyvTJTaI0kZOIDdAXWZ/AftCRXPC+dRoULzGbg0lTKTCutpVFQ0/U4kJHQMREznYo1hQRMMWWBN/A6BQKhlEwMTUChdcpushzCopCQqzbNdz7lHctVgm2h6Pn4ePvonXj9JsjLcLXj/LDF+Xqs01L3yCDWru2kIhHvT1Ktz6JyGJceQjPgKYg6pH3ZugdQ1ehtW7edImgI0jmv26/4J6d732xPd/TkaftqC60CfRrRIDo3X3aBwm6BRKBtAgdGEKQh/ap7eN7PWEjVFNc7ZRmgA8BSuICoTTYbkDjftlj5v3zKwoe8/5NR/bsy4F46jUe0liSQsQ0rpTUUjEQUzOoKmxmeDEV5yssechpo2oxTOsFYgwcrZaJaIhDzdEODhkpqjCgnnQsUJx1n2zqvDxYHzXnPTPUMjFcmp3KPAee24IY1+eMoJQG0w7NtLl126vUXrN1Y8a4+igNHccbz+PIcCj7g436K3Q3ojNFXwc4UhhmijLdSJJUn7trxyoHHp7KB7XUD2TImZyBq8EcQAbNt2JbD6NjoVx/1Z68tdly20xq7dvThW3fSyae+Wb19fC+8LDwP7gTGl+HNcnAyLXckkcMyKUBCvR//L3KZR+nF0JpoGJpoCSMpqaEh4COPj5UfL4uwOmX62bWcpQnnRnuO7bBUyhIIBQ0czhlndgYB14eo5yuPECgDP069UEM/Hb61PNo9Urjjby+DhZV200/d7Bvq1btnouHxgF3g3FO27pjA157A+ieAGXw1wD0CyQyKGhWFXhd7HFyXRhMQVEb7MrI1f1P6xjuuYd4oyfbM4eoE1hgcYMwgYNAGJsI63r64q3374Ud2BNJR7ZbidOyTElbPyODeDmBiDCwFItD2hv6FK86ydKwgIoeCEVj/FRBnHiHEaTKUJloLCscG9SV768VJsS8M9lanhgIVaAC2R3kjiGfRpj7mIkYJXEV3SqTAHIs6LPZ8P5ZkPNwz0bfwcPs8cXvBDh849knn4OWdm/4VDK3AhvUrF3Ar30h1mU6OJXfnkT2Ep0eQjnHDFHorKAgkBmMaaQsVIGthA0FIcPuaXE9H6iH/G9FUT2UK1QhbCLwpsCkkwCCAFhTLqBhoH7yIq7su8zpu6Du0Z0zfpc6/7er9v7winsVIEzqA5iyiAWS2xO5ilOVoplZgCX0WV30Uky3I/idADRax0RRMB8zWABIDoxMdm7AS+Ucqc0+E8Re5uN9Clcm/SfycMq2SZEOnYGImdZMmrkm5CXEscMaylkFCDg/u9OmlY985xyp8Owq6xGnMv3Abrq8yUIbTVOrSJd2/v6P3RCstrJOIRDQOfgy0BV8OsHUAXYC0oYB9DiwHnTEe1+jrwgNtqc9n8JLAxwX+IUBeYC3H9QlKEb5kkHNBZ5BrBgLQDKoC3EfGzeKSW+T6Lec/3HRg5N7+g5+BqmIki2UpuDWIWSaY81tZWX8DI1Ifu3Hts9i6GVf/J0AU1pyRkVSLrEAJpmehFORkeDCoiTERPlKvfTxLDmbMyTSV0w0fVFN/UzF/K6O/USZN/tjuJ7ZuXKPhmZmmAJ5QVJKnvjfAs1o5e83wg830CbuqTHE5sP0GfNrDCkB0nb0m08oyOdHc83cKxxqwUqB3CbJrcMc4VmfgUGQVYoHnGcYM1lKMGdxyAV5wcezjoDE6I2iNczfj1y6cRZiYw6HjUZuDSGA5mBI44GA2xmMWujm6Nn/3HvuSfsdxhnqkrgDAzDSax+BrKCqiWU3I0eeGv2lX9/x1thc4/J8A6STRUkQIVAKjrakgoGFaVY3aquYGkvgjbG7AWz+ZawNXkEBhAtUhBDMQ4IlsN5rhFx+USVbNeVJkRNQUkKF9AS/qGmvNzF8FZxXT1UarHcA2/OkL+L2LW+cgFi3f0Ja9msJagm8/jfRCrGrD2gyyu7B7DVYzvEVAPoHrA5QFAgoJfPUn+P0f8JsteKIRS10Yiug6/Hw91r6E6+5A3zKUF2A7hyexABiSqFL0OihFeKKt01p/75U/PHHzVa9vfri9DzkIAzWNKS1mRInSypR/xWs7k8vMhtfxU+CD/wnQXFCJxIljtBpVGASiUkJKarw8OVIaqywYfe1AWn8suAQekMnASpBQ0Bbwz8Mck4EEj5EIkDlwAS7SEKfOkhd2J+xZTjPyFHeVk7uQzh/O2VRgFpjTqGn0SxzLZKWFaArbEjRY6CTo/l/YsBXdL8P/X1i7H71PYV8X2oG7fIz6GMjgyUZ8/0H8lqJCsbwNnz0XR9dgNQPNIQsspxiM0ePAK2EwjXNTeEyBSgSydjN0902zq8c35l9814r3X3npRKWqpqWysEwNl4+g+XM4uvcsXdwyPnvbr3Ov/y0u+U+A/KDy/i+ePq7G2Gx3HGjZr+ujqYntRw4e2PPD95R2ncohnV24/yzk02Aa9QQJcMCCFhA2LANpg3wS6QgwofcaBBk6wFktlzlp1VsDS0/O5oprA2kFM4iuwJ8o2gM85KML6IwxI9AENNjgHtyl6L4W14zjwiGUyqgcxAGBsBn3G1QoXlyA5sP4/BQ2MjgU+Qw2noNvP4W9m+ENoOMHmGVgXYgWIi9xs8EejSxBkUJKunssHiM20JP7zOWP3nfskq4Lbp4ZHiZZuexLfKJ/cNliR9v/0GwV2mqbXsa9q7D3L+kEs5iYLP+mb3CsMiC9XhNQPSzHt4vD93/1rPv7v/nJqWNXWQq0AvJpNOfgaSgGYhAHCDhsBmSQtpE7BYGLVgZ+Chwy8UwG2UxU4KLVe2VV9Rw5yLNk0QVn/gr4PM68AacrPL0Q16/Bc5dgeAgjNuyz0NOFToPER7AN9/voOw+3Ctx/DLttvDsDlYG3E+MnYDFDTPD1LmxSyOWQU1DvZHcC5fVYK8DzWApIgQUKRyWmmbVBNQzvf35P/CpZoq65d+9Dmw+u6Lg5yXcFXTdsckK6MN/ZmH272LNx4Y+6xY27R/CTDrT/u/nEwcn9f/rV47883Hugvqeb9EOMypf3TOy6/7FHb9pz6I9fgmBTEDYSFys9cAdMQSUIgY46kkY4ebR2YMN3sWQp0h+B24xuUt63mPcHenFNopRp9ycqO3ftquFoCYCL9CHsvhgP/BgffBmbgB0OuIA8F6vasLKM8gR0gIjAApTC7gQ7cugCsBJfbsTwNZiewS8JlpVxjGPjV7DlJnQexmANyoED4BgGCSoxpMEvNHbWoSmQb0r7NT8YM/kr821Ys2vxg59+bsPUqudzlznXbHz0yWueuGAjtxd7ShkvmZsInr8i+++A3pqefKn+2nnvv6BeYs0kkLH87RvxxMDszFZxsG/w0ON9vgxnIBwQG+wWgEIB0DAaEgCF04WNS+GsgIgBH14eqXmi5jRIl9WMlqI8OTkXjMp65Z3xNuIqCXEV7rFgPY19Qxh2kb4CG2PIBlgGSRkVg+Qgbp/B3cAhhgEEXlJJr8DyGYwZrHOwN8alM7j1EPYcxuBB9GnoYygJBIdwicDjMXYSUBsn5sEaQXUQPrFz8EcHhjWZHajsg4e7fvzgo6/cseyfLiRZetTZiVU+evico+rpGrQcMpv/0sUmSn2V0lh5uCQGzWuD4kcHRp472Htw8ODEG9OylgjUIifU7bNBa1hurRymYixtRlpFOY2JNKaaWHRSWnVycTYNzlZPpEuPQE2BEUY9ZSkqQQJphADi8+/pHlw6+nlc+zYezuO2ecjuR+9D6D2I58uonA9vP3ofRWEAQkAEGCsjAC41qFKMWg18kbvh/dg4ggsYPhihRNAp8DxwYACDHOVmuNdgTwbVAF+lCJoBAwa8pVBCbfdE/+Tro/5Yf+m3fPkSXXL5XGSxpVv2rN/SPb6097ngru82bflMa4PnLs7QLMFTQ5MHp3KntGD5O4BK5fK/jr339Y9dcfjZYT2LOIrE4/5UpXpouDzeCwucUMfxHGYhqKkFHrMoABvasmAnxqLc4QUGAoE4k1saLi4/d4QThztMesKfNFLJ80QUbW0oDD6E752HrY8ifSfuuRQbRzA6grsr6FuBf7gNWymCc5DOIyfgtKF7F1QCbSMNcNYEC73rse4j6JjBrwDBsJrBuGhSEBl4ExjMoFFhD/B1J6maiCY2pXTQhlfXK+s59h6UXxBqaHT4oGy/ZOeO9RetvfiW1WfdvvOfdvii5E/Njo1Xr17UuYpxNhH3i9b7x2onn586SGBFGB2ffM8Jf3XKyNDYxN4ZRzeHIZsTaT0LKRzCOEtnHRQsGAJkOCghhhqtQUAYDAFhaU9nKfUINKnPb/NpxWm1CaNpw8YACxJKKqWDob4Dr5x/8lE8qIFzMS9CMoPAg5tBngIH8dXv49gVuAWAwa4XcfAjEN/G6hkwgTRFoQvfz8CjoBoMYDbWLMPbF+OE8zH0JpoexiPfRssMbj7UP7j/xdILg8ECh7tN3rLlS1k6FYUSiqJmj/hlgsln6bqWkHW5y0kqm0o8l0lhfK3Lc7Ugn21xvdxQsuOOseee6DycQc9Y/+wZc184+f3zmv/kBxWjCYTiwIlSTSnFo9ghJO+lvYQkADxASCtB0sDAOCKpcmnGmlLE8UiGpy1GJOZIZVxLQpEzlBkCDVCq7Zyyc+IQbguw2cFFE5g8hP7D2GkjzWER3NwIthnXMgAAgXUJLnode9+HvkexiMMjuP67+APwJAVleI9G5GHjy7j8FWxainkC/3gNNl6MrqeHu795p/udzf0HS4OPEpYv5D75CBFNQs1O/nySCHhUV5wU5kS4vr8St2pzv7Yth8FTqM1BfyXF3GJOoTCFwebwqtuHL96FSx7eed0Dp951yZlbP3X86pEqSSWqgZJYmiAEiNPgFAwkhSKJBtBooZZCAwDAYZSmdaNFHYdRz7M5Z6mYwObVYELHxPXSAkw1gTZZ7pX6tHPx86cs5xvPK3gMe27BczH2BRj04DWiRHFrDrf9+s8V33e0HuuOxwfORMUGWvD1D2HVM9iQgfd+/GoKk+fgpY+h4RksBuCgCUAHlspnXp/bnJWPs5HStNZ1v2nwzmKOLYbDEPDESNiazgTGIWSsvxo9U0oq2uPskFKMZJiTa8kUMk0FC562PMuzTi8f8HT6oQPBZ+Z2LgzGGvKNKfAw5IwmdQ1+UsbZgAxVgImlVHGspkPohFMLGiTF8E6ROoFlpZy0HSUAk0VrPHBC40higVlpNJDEpQl1JM3N4uxDomd9HtRDKcZdNbR6KLpgDdicw3AznF/jP2sRCo14TSFguMzFJAAb3MFRjuxd6CN4DoBCTSAGQGHmKkYOmvHhuio1jQz3PhuOpj/5+PpbOhe1d85Yg2J6DCkzNznbUCi97FNNRD7tIU2BqlbuGdniFTxDk5UaVKOwoDA9Xr77qEzO/jye/1Dfh/WzMxuAfJpRRiUAQynpovbSXxtovaaekj8L1P/WSFj8e22z+ex3DQAXSqb+4NnzPO9ntpgMxh75RX2wlFiD8V5DbO6EOsWz232q7KzKnFGZeu8RiqMevtCGToZBg0mNHhvFNBYRCA4o1Nh/NKJxlE6BpmDvZBbvSCY3n28994945J2TRzA2hOFDGETS16A6f/CdF7f1Dh87Ar8MGyaoRPvV8NTju8FLulQNHvfNdj22ubtr1fI05U6Bn9Kz+OHprSIVxUqIoBxgLUcOEA8EFx4cHsyEmZfm859vyD/Atw7o3l3esPHqTsyIrYgFi/6vNvI7J0VlTTgzC2M01urSJMepX/yOBO+dE78sh9JJ/sW9JU0urdWHsXBoa3G06G81xKsSJrIW96JZvuQyfvnX2U0PvTWC4QuwowO6BQcSrGuBn0HZwVaKwEVWIvgvBoQEt1b2fjizrsNBwsEXAXfgqs/5z3Xk+t6ho1DbhT2P4m6CS6Z6yx963xWLPtwy9WLVDtAYFQ6W9lJpaLbS9QUn085kYOJZ3dzj+mHgcT+fzU4ckUdPLcQoxvEkZvOWyBuWB4RCBtnOQ/ZZNNbuhvaTX5Z3/iRYhgM3LXswygmKWZcDQJ412QVcfHHDFWuxbLZ+bUqrWg0AQ5PvY2paK1lT4Auti6cmE/whOz5+xtgDxcyWC5BmJHKQUNbSVFjzXW/m78r40uNe1riIO9DvQBTBp1Ag2NOCyhz4BJ74Av7qJ//RfACoZOstHzvnx/gyxUkt+AaABPsGbp/X+KT1OwDAHbj7WZSasc/Au6N3hxp1/vqEtNfyoS8+vOTe6x9btvXCocEym5UTX5eZomzo4HHMyF5EIkC55GysnNO9cmay4uSYkw7yTVaL48UYA0LgBobPuvzHjwx+KdO69cwri5dHeMXFY7+cjNb9n3uLAIHZiXc2PLgeyJ9vW2OWtWPRje+cJgG5Aijalds7Dow9TFteuencvjvedav/XeL0x007Dv7xijXLfqZ+fGTzuifW4JA7q0cGqkuLG2jWH9jz4LLMF+z2247hhjdx0jux9i91802PNb9v9Ae/efqRO16aj9N8BH8AP3Hu8Zde/usPrmtLkJyBZQJ3Aaw8fPdz922Flfv0hWsFdv/9+qXXfn7P1I7qszf3HhzuiwO/PDos0iheiK5MIfFkZn6Wg5Fpzk6SVg8/fWOydJP3je+97jpepuA0ZK93wJ58kd249cNuEUtOzZtaqy4E17ZvHcAs/txGMg5EAPDvVX73z+zcv1j+OZ7iCsDpR/mDUBP3EkXhIN2R9ybeXgETeCazCK0rMcOaiGv3DO08oqlpuf2CFVc/aXD9FPofwNADuOIqPHT7n+ekDJKjex8T7333/pceshs9APfivg/tfZccmTz86gnll88ZwOC9+OKWm06sC3zwqd/Tj5yv5uPMQu3hTV/1++z0fD31eJA5P7u5ylVc2bZ567OlQVW2yxOCZ52WRhgNfyAQcpSfzck3PXVV6VqsuuGLD6ie1cuyfcDqJZ2rFxbfb2crxULB5Z7y9C342muYBCpT6HvnyRX4OzOoGjUA5s9dOO8s/43Xv+kTPdCX4+WZXkIIWIpilbL+qhr/0m/RV3QkS1zrhmZ0qvnWYVk5Viota125LLUfeHgAP7sSV7176NeTubcONj6yApcB+DZ2Pyh6zvxY0X3ltt/hD38ESii/PfpX+T+e3pQ+VUAMYfMDl2/eeObXv//IHjx3/NiVP+l4i7Plvz/t7xZsft/WgTNlNCnbvHUdncungvKidcufFKGsmgZJSMDKu3UczSa9EFnLbkjHq/nhQXls9Vfvu/MfP33gAaen76yNm7zUkSvWXTjmXZJpZ0A3wWUXgH0Lwz76JdbNoUpQ0HAAi4LaQISahlIQv0DSgOgtSA1JwRVYAJVGVaIPQFcPmucdIdygxdPO8lpm3in6yVI6LHq31N0kz1Ggtj6nZ4P9yKxbRQb3MPzjNuT7nvqrr7z31U/IVy+9/Brsx9PYO4DSJRu+dNntm27f9g/PfOMnr+OVOjSd//aNn7ztrZ989IKeeQYf33/7e358X/mjJ1582vg5H5mB82V86DX67nvXHy+ii67Mb95ued3eJ3su/G7/QKa5SL5wMxMJoKIAxyqTtTcUb2LnrsoSQvyqSgQPpu07rtn66J3XnHHfFty4b83a81fkNqlyFzBdAYBMEZ028g4eyEC7GOuAtwJFD+lGjFuw/havADqCzEP9FYIsJIAaOAUzUAKj43iU4KpjVa1PA3EpFAs/3q7uepvHBX56tvDChibXWsWQZ03lNctz+NzzFLl/3uAAyOPXa6wtK4pjT23pG/7l7RHOB7wiihddtOX2ZxZd9Y2vvY57lmJxA6hb/NtlG8QPfvNXAs+O9I99YOLEJ9bfd81LzZ+qw8vi0AFcoPX+qb6ZAA2foZkPd5/bftvD7S38zv2Iuf8RKZRyWuFyWXk1Zobw+Wh0uPXbJNWo4znYC7ODd5YavnHXd7Zddc19tzbfdghcopIZqZXZ2b0Gl0kUJ7DYRYZBUYw2Q52L7iWVZdZktoVxHSSAEhCxTEb8gNnKTWUO5FJd6/jb0D9LDn9r8OAtwdPXfPDmo8OTBEgIlSZRLLEWfc5r+1d2PhZzrLNxR4SVPHX1FRumLJz6zwCAi/HoDWsuOnz8Pee/sWdzce8Nd/7BvjFYg9u6OlevOW9tW+fQUqx/x4c72ruHqIpRd5Aa3zt8Qt9Aw93Hn7hv/XnXwr0IThO8G3DwcXSswnVd17/y3jc/tvEC7qWkknVl/LKZGp50c01TntQgmRynOWqUm26wylG/mTWLmnJecuHMzt6Rx/f+8+f33Lj7gZZcbmwy+E6w56xr0s0bdo7jHoWCD9IGO8EUUJ4A7NDyRtO13sC/qzpXFVJG8dsIU4q+nWTez4ue2/y+YteF3sxJa6eOfHtZyfJu2iXedRUpi6BNyT/S6ec+kGSWQJ1ttWDd6Rii+GuFP2Xwpwz+BDz5b9FrBT6XvXFw/4eOd3bgtue/+it8nsACNhWLv/k0tj+O9QBSYCpwnVbo7IYpXDjQ+7/f/OvmD/3uPWf5AEAkupYj7ofdhFf/95Z3vfXSJ797c7GnPVAhmFnYllu5cXm9GLIUEhiPeJwSx+GxjhDIADKT4R2r3JZcpwigBsqlI/53yfOntaqfJ3IA6uAz/Zc/6Qlr51twGjEWYJkEryP9TVjn94xd+/mk5Supo8ND/u3Dyr9fMkRSu44x97HE+raf6T52xgL597G+3OSE/6mdVXFtDzEk0MQ/3gleXuip92gO2oItRZzagdQytP+n7gAAS7H4o1969V3/ent88r6tL3Zfiw0AfAgfr38BmxYBALrQaZKT+CpHdBZGsPsDb5zAPvj7oZ+99+m1WLkOx0pwPWR68PjvrvzGC1temXtlyU+fIpz6/WH64/YFGy58aezwk6/ec3DL80DCLS6kYLzWuDRhRUat9nM3dCxa3JrPFUVQGQ93mGE2lB5+ZTUL0hbR+SlUX7hrz8dvvLUBtwBXT+GQhj+DTIK2v4f/Ga/yrW25RR8s6U/06v4ykzVGjEOTjLEy6YIjpsnj30wWUyr7JoaDT+2UjY/liZOtOnlmolb/1R3WrN/x5IM5/IThsv9C5t/1c1yLNaJ0j7wF+5ehx0cwihWO9ZtFKPw5lSgcl226Z3BsqPPaItRU6Zzmec7Xbl5vibOD8/HHjobVF175g7cfveq+NvZHtuTsc51uT5pAVd5iw/0dee8zd3whnXps/ovP7j+w+/DwARGUzFGxZotjNZllPYUlhU3i4SdFiQqtlNoRlaJiZ+7Mi7pfr6lApPPtWYNQV3072/srLNfoAzYtQMujMAwlB85VTb3f/Zm+qtGZ+OfZoFSeS5IMVV3tfJHLM6aESnHoiHFozU50UCo/u12SBmpZ2YB4k+aPktM2PbuyoWkM/5+A3tEarH5n5Xj8+qe4/3y869f/4fjqeFbOFLrD7JfuWNW6a/e+9D8/lPeWTJQOn1u0r8s98btvbaRLPjpil64Y/oeRveWRI/1Hw9EhWU57tnseXXPH6idfffalfxkb+OnRY5PPRQi8bEFh1k1/vdHLvpWqmFEF5TXUdviT09/cvvmL7Rue77nVT0K6d7LFw2t94RLvwLutHTY2GQz+AvsUgo/gswNQDbj54qax229z6ndkxSO0fCQMuK5RiflVU+QZzdlltnhECKPmZOWNuwZJNI38HYHTvDPT4LGflfy48XDTzQKz/zVd/u+UB1+JY2/jK/+WwmeQtfG1lvkHGFoi4d36navlgetNrY+lnwyGd8f0Hy//3KZLNzwU1Urj1d0Dk71+TUyNHVOy38s5+VwnJTTS1Zb8qkVOD8muOedzE4SY+mIS1qZ4SifTNYdktAXQ4bjMNK0UTyqmHM9JnNjyItCpQNNh+f2c+MgmH00HNDYQOAvQYyOtgSHwOuQtqeGJSR0fyIjRTkpJJoUzveCSgnVutvaExhNSnFCtVHqrf5iUhNaToX1Z4T9CV4wtKp5+/tmrXeQd/O7/kg6AX+ADwO3kL/Zk4J2Hm+7c0DqR3OBHw0s6l2ce2kAjVp68bf8jj0VjB44+M5gpdAfKf2r0kTgQvhyOamJh2KrrOjM/13VhzmEOqGrmGUprRsZjIVzbztinp1OgdvxBiNccQ5jX7OaOxWWhVcbjzVmXJRuGws3qjUFnVR2EvObLlqaRt1HSWDeH3PEYdZEDCgGKT0KpKyv+JzASpA/fD7zR8N6T+XvOlB84O32yE/5+t/7i/fKUH+l1HxfzylXTnKt1rBXo7LNO4sByhlX/93T+Oz2DJ3+Ks7qsuaXeQYNNSHn53KZi+/JcNu+XxFOP9N520wVP771rqDwSVcWxsXEtawB5uzar7RCpxPHSnDIArpOVRAPK4iaX9hj32k4teoWizRwK1nZqZ1u+W0joQGth1UV6RqmgpIPJWciwPhnG1ckT0afQexz2vY2nGCoW2gQ2hrgQl9nymtj33GPTxUcfm7/+Ur6gizeeRj90pnXGp9Xnr8NTu8VREc5zc3LZReVFn6rYTBBtGpCvo/v/PyAAi1D4Kd6+E2+fg20CsoYm2y0QnmeeMxMEI6XJQIh0LacEamGDzT0CfryXppxzajHuwOIO53bK46CcJoyiIUVBKQictNfVviyfK4ClpeJawp8M4klZmQzEtE6VEL8IPQjsxdTekkz6FmA4xp53YyDC0QQMODdCT4gmvrpWvyhqOA8KTmU0/eTj1uc2k/WXkr+/GPfcqcarltaYl2tX3KmKqvb7kuCZfA6rGZz/EUAAWpB9BA/cj20cubcwSq3WZo92LO8kacq9JpfnFma7F3mFjsX5fKGYaV/onJ1zCi02c4TQhNW4h3SGL8oWGzjXIIbBddI2uJumme7WMzvXpYynQlg1rhWn1POsYgtW6RRkgJlhBDtB+/Hj/pKNsRRKwGCMySnMAMTA9sFIO06/YzS/ZTLTTXiahaZhpIRdz5hjo86IYBqM0jRRElOjqYl+K/+SdYncsBXzCH7+PwXoHS1CYQUuOYQrbGzItbYXF3c/Vg7qCNvyBaVBKeea1xlszSOWceZ73EkryDhUFjON89FWKk6hQyPreZ6NJmqI1lrNF8XsEuZ+Np8rCCtsyeZO7ixYbPlrcsuh4hVT5nFMggqwPrBevHbN8PE55xfoPxG3Aa0Rxl0EMTzHM3LxMD6ywe5sz3hOpHlFBkgISzEwTijAA5JUPb9nqfXHYTq/Ae9WwUZ5Duz/WUAAzsXyABsVooWdn2Xrtpx7381TYckGD1SJ08RO8STF2mh6QoOyhDU5DUiOhWNJTIhkBHxRoRhr3ZCyaA1SCIsSo4HEWsi98vwiY3nutf8CohnQFsnwfAM6ZyrD+RC6BGxH/PnwtRt7T+zhNvoF0IzlBmkPjgUyV2ONmzF3pitOoRmTloicwFREKpUKaBXN2QzxQyfnReqobObZiQcO/PPLPXPY/z8OqBFOChmBkQY4zbmNa3r+4WBv70R1EDLRUBnuaVQVIfUa/MAntM/x8iBsblYC4LZDUxZJUQvUZlQrPRFO2pRLBDNC6FCadKB09pT0hoV2g13yowqbo4pyNANxCfUSdAmVS8LfPvT4B87eYWObh6QZNQMOeI1NnL6oTryk/PJfL6f7l7dkcsKXS0l5HIWQjjqZNMmkRaJZeTw79ay3tLTax6rMXxTe/6fUjc6zEYugn3nMg8e9dsctV5Bjs1TT0CDKwzMqZIYZqccnSySY7Ch2Lsw1WeAaxrXt8VK5sSmtQAnjxpi52VDFTEDFCDL5tOPxqLV6TAYTVn8ZHw+Hww6OiEMDtAa5GPUjOHRX8Ksn+09u6ldwgJwDquHxHNTiaffzGzMngHZlg5m0b0liOyuy3rcLOdE3MK+uqAXa1jWNnqDjhrKPUoTK/zigMZRc5EzCJ4YrI6MjY+UjjSmHJdRtclrSi1uaCoQSwqgiglGl9IypxTNBaSIKtRLEIg7PtRSKSKSmkqcYY1QrMZMMwNEt6UKzk3GzmZaid3pmuSMywWBkcRlxyASUQ58H6wZOrvTQ06ShBPoSHEihopPS0HAga2JRkTuXiQ/cUSGtWLJoSYuXtyy0pL1ca49F3HnMTlhroLhPNQmJouAMzgCO/I9wMUgABAgiRHnkFmRzAERQLVcrIEilsoxmOWeLvKXdfLnWuhKOTUxOAVUF39eH41pAGExiYMBSlPMMBQRUAiahADigTiZ5ZyxBy5EZrjOBJEneQNyPOAQA4wE5y70oZbd7fgCrqlXFHxsdnhms6tB/M9I+Aq9dWMvVGcs9u4kynnJYVmsNKIda8wRAAhoNsImSmilVIwRT6PsfoaNQm0DZR8Bhu3A5bA7HgaWNSFE/MqVGp4HaiYJkjsXSPFZMhebNSvWoPzalhhMkkapEqgZAEqW1JrCo7VBKGDUpkDktJSRonYNLoX9V0QZKhJERZQRIqoh9SEAxOK2EpZFJ3GbpySpmhpOZMTVWCfxqcKwqZiREAjcNkmsgzPIyDYsKnZ6XtzmTmsyjMSYq+rdK2lmhA+5CSpTzyP3/BzQF30fVIGFIZeAJCJMkADMEiaZ+4NcRCRVFUkzISUA1M3BuH8doJHUs5YwSDUwHsiIQWClFAV8FUJoRQiltIKlGyuspo2gDa7IWMO8EkZuraF9OTslqXcFoJAnqElqBScNDfiyKZ4Lg535owyQkUSF+4cvyZOUXvvBRTTpDLJ7gHvPsPGUNrtfUlinmsh6p1cwCd/bEhdGKG6X+VVpgkOGwRv2/Mwr2XybF/jsFCGJoGwxADBUgkDqmJA3ASjChZSCDcTUqEWVUwcjahBpJqHLT6Q9rcK4TnUD99i0ssEDtkNEEKpG+kg5NGyQwcLnn2i40XMdxCjkKJCHXSifQSoBpMA+Yhvs5BKNJRqIBs5GQIsBU0bJRjpNkZra3HIyOTRPV7rPupXF27x+/OFFLZSgxnPnFdu+s7gKpc9nR4zf8KDHV9NBPHx8/ZaxuvXIYf3UxvD8bQlVB1VBT8AF9Lq78vwRkkDQjTUEFZgMET8CYuMZS1AYnlm6glViGWkeGzoogjFF9Kxj1gzFGpt3WJg8sotVIhi4TzEq/FVQamdGoWQkUjNChQQJiualcbMHQLE01MQZbsHtriAOkQ6gETKOi4UzDKksW1Ky00gQK8rVSwhigWKQipkwU+7/A5AKrfIz1iFQPoYHNkfFsf4R2LO4kLlPKClVk4ZvB+P/+eICyxG1P4ORxHAbQCG6DhQgjDE5h0Psztf9LMTAXMYcNQCFUszt9ExgQQ2HpJJRCi5qmqoy9FkNtNn6z0icglqS7oxRLdOxSGBVHQUtz4sRqdiKYyHi2U+NEUwIuYsFAmZfSRoppJPMZCCIIMQkdgKQhNEQTLAt4FSwfLSR2yGOvChnUmdSxJpEkde3Jqp4a9acm08dIxXMVUNfMNCPV5rWy5ZTYTYPxxq2/Un0f7O9d+Zvx5/5wayb3pZswfwDPOeBt+LKHnjwwDrEIq+v/8duIcZQCBB48BjaCHWWoS3F9C7IGyT14cBeOHEani7SG3oVBk1xQVhUlhJIiDNVcTVmmpgJLAYrGnmfH0ghxcSQrx9LklHyBQMVSMmoqGGLppVFNWpJFMJxKpngUCiKJmBXjYy/mndyvc86Hs900RafqfU4K/jA8hpijYTGUgB3ApUgJqx7OxgEXJV2nCQSCUDKCmJoKqSaBK/lkmT96lnOBoYRo+utlanBkiLDiJZrvE3j8O8GXPv6Tf2Hv3n3JLXwEFy/BTR3oASgAB01dWD6EvjxW/VuV+g48/Xz/dduS8gk7hxd6F568vZ2kvWXXXHTVlUfryN1WWVCnK4R3qY1uBmem2u+LA0oIMR3aKVmXpAGWCM3T993uFrxFF7Uzyk2AeNqi81tpzYoqlT8yeooVNHrMBPDlRKSknOVc2YIqI3Vs/Hoo/dGKUIHv1DFajz5l3pJjtzScZdKAh6QE/z5kElj9aFiNqcHZQGuW5hqONlpqQWvUn6RGRcwIDUlnIxuwEBwj/hJvGa01fYCGP2CMoH2F4Bc/OzzMZvbNjW+/+Jdphp7ncO6dOFjEzyle+7OzpBahO/PnMuPHv7jxot4r9JkPHpPVj2wvP3rr2sFDfQYdn3ns5//6wst/v/H1Zz90/+0PP0TsikpVR/o1JCIzlmjicj1UUmLWKJGc9seW2Zd7h05+H3v4C0tbV7rpxrbWrysvPiXt1lUQBuI1gw/UHFDqzw6bFGVKjfVN+ImrjaLAW7W+J/xqMAvBK4Szlw5sV92qgZ4uAWph4hgSD7gelt3UgFrlDb10NWtp96aGA386JCapS8VSMjK0QRKrCr+ml6ZYi5ORiJ6sfNvVBZ6QMgMZ0itnSs2JbCuu3a76BhsW3OCjamEqwIRGGGBfBl4GWfy5PwxAjEu+ufWCifBrx2hpgqdvuPyqu649bdv+t9ns6tNl3+4bHv/auY/f+LMnXu5Z3cxIDEov5428yLLKAmcp7hA1gdmnb995Ze7TT/7s/o4917J7acLAOD3OKdRTJcJTx4H/muhEI5ZoBlcppjVlxHMpoRoijIzARH849aqfpBQFpkpj0qlFwgd/REzCVOGPAhILQbUgKgTL8ol+ZKo8GdbUmKmSxQnxvKyTh93kmllf1SBZfFyrYrez75cqEQa7UOcpTQ6a0zTknD59CdYz7lD20BwqGXT76J/B0y3oZf/li5qHtq9//0d337Ijue97dw2Mhfl3L/n7069bfKdOZ+NGOXDHnfct7TLr9+y47OG7H+pePZIuTmB+S/sYGTUdPWsZN67HHYMT33Ie+Mrmi/+liXHGqWsgIqIIx3GmiYOA8BNpMgdoY81oyQwIqFCyhWdMzTBDoaGkIKkkAbymTDkYTIRUobZScB7xxvcGLAs1i7ikVYnTqqm3JlYBSEimmHbQ6tdgFMmc5GnPSASC0VyadoSFDu+pMO21tCIKlElAkJCZ3KVEfskXRMi0TVceDioJciHGDLYOBruWuJPG6hew/7KGv+2RC34y87OrTrjk+H/ZPPTzYN3f6ic+DhA81XOg2P7Ibb3Pd6HwPvquif6BReu+BKQDFSc5mfeK0lSppDGCKz/2pWeve+bRntOWHF80ByxJTIMyJAULNMYsQmOBnoC0QVg3Oq5JZgwhsKQ+PuW3gCCcOjY5+aQMCIeTUImAcWrpgHGaoIlzizHhdGj/eZhh+MVw4aocAFcqHZuWYkaNpfxUzU1Rm6ZdQCJwHKcZ3vGq7arUWlq8+hsOLZdFPCtrukpO7xn1R+kxTaMIc2Gsf5dya2maCnQyESf7Do495Hduc7HLwbp/A/TUkb3vtU44/rebt3wMT30EM9tRLiEIcO7lcuqaraofHasunHOO5WsqlnFcLrksH4uaoiqWEQh74vYdpZ9PPrW5esnLOze/ef1LrX+ywA00o++0nRpfhplUGnASGDVtAhNQqU5JUxhal8Ep0mhRPjjY509X9XyVzjJtiOuQCNJNOU6SiZ2kYX6qXtdOFkRBb7JYmOYgpubrWhIJiwWghGqZjmeZRhSjZiDAwB3ueYVl521pw1oRYiYM/HBkXkcHhaUTLqENS1xqkbogHjzLyggzNhA8OFJ5XqP0ly5218OvHf5GeMsnoRRUDVpj5Aj8N/DEDkR7kbGwYm1xLlB+JRBBqGsaHEJCBQklzsRYecFc/rT3f9iiw7cevOXabfdmcm0LmuxgthrXYmNELJUlzUwY/iIM6tOqElQQmigUr1WqsUoQ4vXK4GvTk+Vq2fdLWgqbc+JQ6phIBkmic+kCyzpOp53kQFxEEUw1GS+FVKsFcISmupZwm3utae8kh1iW0YnRkELPGNns0LbWzOaLrv7WHc/98NFXvv+NV77zTxPzDNMWRRxCaZJpyjo8tC1oFBN0jNSe4p7UNaJq+yaw0/9zd+I/33jVfS9uueQaXPdQeyYL10NXDyiFSaA1mjehY/XZ2jhvKS1k2FXobuFpRmmLnbM5v/GTNx334/eO9G5XfOz0jSu7Vv3DwtzSaLb8lj/wo+H+o6ODUanq+6MTleEp0e/7w0ZKI0mibP8NGU8aJfTg5PBhf7iuZv2ab2TgB5KROEagaDCnIsK4l56PXAqMBgGCYZRLElIGAY4FkmphUjGybGF71ss7PEfgMJd5lNCaVsaYFq/94rW33X/Lk7d/a9u3Hnj6hw+/TJQCABkA1XQoSfTuFE+yCpkZsNPbF1VGNzSUt++X67p6tnKrm+Fr7wSjCz7xufKP7vvhj738L0Zjja3fxeOT8CvIpHHFkxe9/MpbEUTRyfJMKwBGCeeWneLjvdUtmdO2HL7u0qtb2cPmlVNau366jGPxQLl3bHRgsHf34ckDQkvCwVOWzngEnmOsIFAzs4ENL1ABd1AXiGb9CV1KUr6GpDQAlQF63RybqI3a7W0NJMO0p8TkHEAJbB/iyxjpL2UqoS43rWhPM4tRx5NaayMM6hlOtc7UdSL0bM5b+q6eh/JnZwE4OeAGb56MEIUAKEtlKHii4QMBCgLrZhBEZs9E8uBQaefT+66IZntn0PuOEW26+Kf0XP65qzajQBMKmUbXeWhrB7sI1236bL0mDNTHFi9tMFpIYdGU47icWx+ca/zIh06mntR88qpvf/kTG25rdprGgtGJN+aO9gX+ZGWqPBapwKKQiQ587aY8wHOdfIPCTKk8Pjo2NlyqjJZqgWxw0Z1ZDeJMvDE2I2KBWM1XjWmHastLFfLdGTeTYnk4DFYdRsWW0mISanI2ruroDSWlpOA2sxPWEIQwGg3UehPlETXgZNlfxpN5SkJUqap4otpkqtm2Qr4lm2dIDHojjNNOmvGY0+SI6epEeV+AB33sAZBveuH1py5++eAJp5+9Or+a+/0ISLrjjtz2l7fYnmqggcv01LRfhxgvHxh7dVIjddFJF3+z4/MDe/ecu66wqLX1pju2rb/os1P+5PjkQRhdB+qApAkoa3G8c3NL806nA+45mRSybdmehek80bW3ZABgIijleWvmo07LqQVAurmkLdvd4eU7+GIZVPS0JLTJ+2jGrsPvA2YBAz3pzTxP7RJEIDUU4cnxOcKbqGfmd6QLtvaMhoZkpFYuDfwlIOJXIRRlmVS+p+2F93U/udN8btPNd73wEKwrFK6XaBqKnj/4zBdMn/vcIz3ff6Cqm6oRRl2033fjwW/+89nhh+V3vr/vttu3Fs/LNPcoUx147rEA5+9N821GKK1ks+NClf/uF5uSzs9/qu8cZ3mTrlXWPLHlb1JnNDgtI68e4oAKo2bGHI9ldJpw1sJ7eLrgKUdJapGUbRJWs2c0y7Smo/6+oWr/nPSFXO6BNxd1ZqVnpS0nlSEFdKzyyvcRGKseQ4QSCWwgCEBmM/BYxrXAiY0stRqmpJgzUsxKoxUoGJqMMjOkdFh97CR1BDj674CmAgy8YVkmo4fnH3xs7dP3kk+8gJrl+Ug5wAwei2YxcRNOb/3SBT2ryv7zgj9oLCeDLxd7Kr/b9fQnL17xhUs2/N37is1hgU0qiyWBRLSfTXRNPtq7vetzD1575YYl668buOe7x/pma0kuNV+1XEkev/XRrzy5bqx0NI61RcBTrs3D55r4IuRUis9AFXmBcEdzrsLqVDippRo7coDypsOiX0ZjJpHFVrYwz+s0XLhqg8kq0+OI2hipcZMLDx2ZlFFZTAaQsBxQYCbw3VyORNpUEVVkkA0M6nUqWSoV6EAIEWM6gWE8VWEDb4Xb402n2X9O+0gw1hT7Xt4pUM2mBoz/KDVBq/q9jHPaQi3CqFKWk+r2uONVKrsev0mtvmNpYelU074Mlu7/5zef+tZHe4j8aiFJ/0bec/lgLA1JOY9c23m4624pk4sXtvs71K4dmz/3mT2Z1uzIZMDz4d63Hx3f8kdpHlJhqcUuiFhqqeYgAs5ILae4sgzTmdqCVFFLzAk2NRZF0+FcJZZ6QIjJSEy7+VZwGAQNWV23Ugu9wnEp9mY1PFaJXQ8krab6q9GkNgESAacA7dSWpBvA6IwOvRoTs8IxTRERzNSgfaUFgLTj2cyuq7lfVsr+4NttqzuJBQDzxKiXhBmAJ6AEtboUvgzrNFCQFEohS7yksZsA4djkgSf6vjru94dxABAB0dK5+vGNX5u6q9Gv3H16VvnDpWf3jg59qxSNVirPDE692Du+Z+vU8NYYlY7WQgLF0uKSV1rveHkbo1m/FERBUjeJVWNGVzSqhlCP5zLpvM1yUaBVTRmlomkRvFEZGxsDIOLEBpJ6KlPINXBlkCgSLMi7jdmM6zlwrEQSF3kv7RkoU4dWmKlAR6DgsUZUqaFqfKm1Mo5mNaUIYiMJUQ4xqRbmxcqSk6o8HE70l8J+TIwiqGKeCmyeODAcgLHkRFjSxte6YmOghjTw9TWXfbb4KTti5WPBwLHpwYnhvsPDAwAUBglY2+g/bjzzyJTftyxH1qzmThpLetJtBeRyrKMVzU7Q7PQyrnxVZpAtq3DrK5u61q0NykQFsi5MGESBrAhdI2AElFLPg+ciR02qHsiZ6WCmUp4YHp6qjFbktGVT1trkHscz3KOENTqcp7NuOt3i9eS8gtNUYCluxWkFqwGGJBwAahBVqNgQxJBKBJoZnaZEMm0xKAKKFAFxSIaANsq0CjBxpDIw2D+ze1btwNAjmFefbjImJUNSD6GlSoT+dTUY6Z+cQRko2rhN4562tcudnAxkQAzbf2D70PCeoPq8qpUsUAx+4mj/dsdhbfnUxZ/Kd2Sx+ZqOFT1WVzta0srhOteuOlxWHh61mGhZ5Vxy2Zc93WOmNU8YEl2vzRpT+a2kC2jWS7kAl6HD4DQQDgM/KA0NDw491i/GoqljEzLQlDlWU4OUgaEq1cS6cqsbdD7Ply5Mr/5k7rNtpDMKjaE1g5rSkibQgJLAbAKpNZEOiEYcEGlRwyj3lZSK1nVTnmcdZClsGvLycLD/W4/tuvNu8ZUqvQrzYFgDYRnKpNCBUHGig2DWcQoUa96CijHLsE5bqzUXjBuH58vVwZHJ3l07b5KxsRFA79CGOx53m9ia5dmWXNPtG1vP6WFdOZUvsHwOjU5DWzFbfrUChnyu/eLCvWqyxbNaWnhro5VtSzVluPcBx2HIfsDOncI9h3JLMxXo40J+bGx0cLj/0N7Bif5SPDrrl8tg1M3x2ExKVanoMvUybdkcseB5bR255S0s06CBWkzSyvOQpAAKI6BjMLgAalrOhIkKtBKJFJJJQrW7EAXIHOApY0UijPond92152uf2HzNOZc+++nbCdHcgMfaoiAgaO6Etzybb+/RyEUYBdwppNuwuOu85UtvX6SH57vtnUGlcoE3FoW7uFdF9TFj/951kpaM9d0qcQw5dVXmw1nxpLBJqNsWR/W36qf0OOGN6+hieukFn9Vti03Vy8x3/NrpHH5JTjiOmyDi1OWJI5WA0knNmhmuvqXZp/b2jQ9MRnukFVBjAsWdqziJAQOVcRxGmYTyeCuFrY3SHJmTsjP7qoN9w34oIVBPg7dDV0AkNYanaWtdjeYEIJSggQsGk/FoiqBBGcpMojW0lEN94zPf2fmDoSubdbiI3jrPpgxAksCfjsd7K1N+6VDfzi2PrJ/C9cATEmUXhaV4elnuhubuINNpG0iv4Grop5+5q0UOQm8No2kpOHOSIBSnL0+f0Z2iLOIEjkO5TW0m55Rctnp5s+d+5ZmnU+F8aqjN0l5q/kJeyMAzEinL83QzsxY4WHgicWioSaDGh3uPDQ/7IvIc3sAIasa1UsfKA1Ec2GnueNlP9HztE/NXEcrE7NxMEHhww1nt5ChXbrQd/j74L0JXAA2GtMcdWUES8IqiOuFUcyZa2TRXtRqBdbLjJdKyGIQVHRw+8sSXdz/1oeGDfzN24/t3EEWUzT0AKpaxCM95EZhtZT9ZubBW8VOcowoc0LgFuMQtpO9NvuwWv2QCc+/2K1raty4OLzLOPFU7fSYIAMvj1rIe79dDxJcJgEXpZMVS+3uv8l1n6P2v99039w8Plz8j/OlmJ2s0Y0QHmE6RRmAOwrY9RmCCmnGEN9S/b1fvTl0y0cDCiUnZoBKHG+KGYNqv8MxSF/O9ltZcfnm759hzodEmqBvfn1Sp1HCtokw+cs+mM906M8UDpVVJR37t3FKkHcIc6jHmIq+VpzXj4Cqkc5q87SSJdkwIW7pvHhmvvNS88KO34oMZ+jd0HrNck1jMog4h4RtmaCgUlXC8PExTlEH7ODyFMsXaLvz60vZ5371jINPKhoIdZf1UNDsSmAMhl4orlUhadwljUGjginmK0XhhwVpzHgfwuSu+uugWMvev79UhVC2uU6WMYJR7fP5C3tqWWpRmBNJqrjmxFD86MnJo+56hPX27dj8yMTkrpYxkfGh0OLZk1JwsKha6zlue71zF+XKlIIRwuVHV6kB/nz88efTI2JN7d8reGtWOJrStJccL4B4UjXVSg6lYWhMCo9E8m+HSsU02w9OAJYO6MYZoajEc72Fi+tCP+g/Uw91vhTfPAwCLWBQRDazW6rIv8BXf8hZdafkAAIUGBdaIHgfdLr68NHv1os7Vi/I2qBwZ7r3jQMWb7yEltQ4kgjiWaQdRFFKenNaKrjy7+Ep+w94S2ulP3vrd6ekLomqokSRaxzUhtWTMVoDH04SkAll7KwiVFEPDB0ZG+4PB0tDewbHBsvADJWvUaNuzlp199rlXbllTvGVZYTWBo4Fg2pqY1IG0TEkd7h8c2NtbHq2M752YOhJzQ6PISAmiwTAro8gQ2sARxyoSwYwv/EkT18Co7Ro+MxlGs9N2CsSySIqobJhkAw1foToPrM64DmsBtG5uh3u2xT3moduD44BIGI2mGGVAe+jJ44Ev9Nyb6cnmO1siU75j7z6QbK4glZE/10FCAtfRSkasZnV1uCt6uOvoIb3y53/82+e2TTgkzVKUpqjSMjG6bpRNODGJAWMWMzquiCmpxRQCpZVKRCxULE2sjTaqrSe9dNPKrp4Lz1m15cxTN7bwzgbCqSEAE0GiwySuJlOlYH9ff1SuJno28WdDpRvcWibLCQUANFAn7RjHogxKq4AEAQIYJNqIGiTV77xYaEcaB5RTMAAseeensFoiErOmipnL0LVrlG/r2//5Ky+ppYjHAaOwR+CPAFwsb8BwhL1dhbUXjBWfMh0rJsXG9GcvXvVzwgI/VEoHmSZQmjg2OnJNKy9K2v5w4YfveHOoMLOocH6cijNQU7VJGChNOFylYLTStboCASBIxChbNL845mWQIrbDmhkyHLyVLvlSx+aHr8zkV5/xUEc+5XyA+DYYahRME1AquAr41Ghw7EjVfyMggOWBEhglRJlaWRgONBGDGqFoIJwYCoI8p0Z1G+1QKAMjlfwjS06hDJQBqCsLUBbIPKgklmomjGIRPbu3/OytwfN3KRrkmOdFUAAojhiUCWLAqQPHagMeb1/fedvmjdusJhedazc/mlq6IYZVoVwSxxCG5qZkaSdbv1pP5batuejZorc8w9MOcyil1KL2v/WcUwVAQNqUEgqWooitLqeb0awFK5NmXe25tnWFs77UseaGdel8gaWcN4My4XDSmXTKsVP/50JKWXFN+qUS4ojCsVLcyaedJq6UVjCKIWHwHMa45TDEhuiQsYCbwCHTJKoaW3hEECZ5TVECSh0AWEi9FNAIPi+WCkZqVQsREE/nerL3Pbrl2m9dyOHZSCgWNKAVGNbob4AC0EBzRDGbWSuKG9vyq0/7SvETm5be+fDsmStkplW7rbW8I9asavrEumTpjTmfP4ZZ2mBRSlkzHEJIY5ojZYECqBskihJtapw0pcAg7V9MhzNBgEDahGU6U8WVzoovr+7YuNHJ5sMg8auVicn+qekJExpdM3VWd+AyTSlDIPyRwT4xXWnmqYyXa0nlXXiU83zebgA8xjW1iMfQRFxipRMGTVjIdIxMikVBwiRTgMu47TA2n8ABvABwKeg8YxI/VC6lFkVuMV2ytpAqEJOpSPRzWC5aKbTE5jn0A7DR7VgXGqiZ8qgfzbpN2U/d5f2g/sIjP+DffxVn/CPWnxt8+qLkqqvJ0mvoRPeToPeUB6v1Wi2BxVLUS3uceDGhLoGEJMSihNa10UZqTVRNHYvCZ/v3TEyNKU+4yxw7z4r5ped3XsTQZET9raAi4L88dkCKQEImCI1SlrRtnlFyOiqNDhwaFYFOEkMoKFgj0jZSdSCSuq4SGEMdz806yXwAEJaizEqgaM2KFSihSFHDLDBDmjDTJBNwDT1P1CJCpa6paFSO3K79scTt0XS+OgZWgQZmFfb5EAS5DLwMVrdgXZhUoiSYGu3TNdk8v3jC2d8ey/78f/2s883f4/Xf0b0/4Fd9h9BbXhgMC0S3kiROTAwgnjUAb2G503mHhSbOHKXQkso2g88ENSnUL0qVkSMDE6MTMzIwqsZyhDpcSVVToREMjmyAgAqg9ZtBiSorngRjdoMiMyV/Khi0G7Kc5ZRGnAJoyilw02iXg2oD0JxPEc+KiRbGRJL4obIIpQygiQISKh1wE1PUqCoz/whUzUrADMwcJAFANTlYqmgTOLE19bPgh1eI2hKy6OGmKUwC40AGQBlLDsPqwjiFO3L25Tfe2XV4zxhzivlMzz17WxacduiiW3DuLWeuyGVRyz+n2iJ5mAadeZSuMRC1MMM1NCiow7MWKBDUVWJQUzWjtKy8GsTVaKT//qcOPGJFSgd9hKVQSRlPwQTjwx4IzWeooKaO3ALWniINmiWOyeiKrcTcj0Y3f+ozd5f9Xmqz4qpCc9qrE7D5iWKqDqtegopmO56UC1cV2gqe69UmBiCSqAUElrHsRGkV68CucddyILgZI/gdUSeoCFWJX8+TMpmoCGOU56U6tiSZq0srnkxtfnidhmyEUCAxgiFcP1H7qsFegX0cF6Wwgc/nbcuXXnzeholyf4pTxtijt6cP3RmN3YX9u3871vt0UKIUslwNGpQ9FQQiCUHRrJ0M8hl0cuMoGauakb6ZGA5/NLbvuf7eR/ds9qPesWifsqqyRCeGamJAVqohQUDma5ezjOM2Z10b7vHpJgfO0kJbPpcLpoOB/t06npWRbvY8h7kO9yCSyquRzbGoo01zCAWnlo5rNV+HqRRPO2kZ2COBbxIDY7FUg5BoSWUSSW0YlkUMIfqVwEgMSZSQumY1M8azaPacq/5hyye9B27uF6ynwpFvQd6CdOGly72D/KjJfbQBnovLvrLhkkPn7ZjBKptS1KC1HhkdNLmbH57f02B7FDTf2g3S78wv7He23nb7dewWfsHZmxKdOCkoARFqoolSZGJy7OX+ww8NbyahHBHvmVO+lmBuU6QYrDiCXjAwSC/sbssyh3kGcB3PSXOHp21piyDOe6k0Z9HlQtfgtThti63G+ZwygMBEmhnmeaq5k+eK6WW30OaT4BT0yKSfCNuqMa5RU7UFluuCutyni1MpRqOq/vmpc7eYPQ0QZVRiSOKHQRRYTjrhPLv07JWvHtx17HU0L2pR6KNocpArosBwOpDEIBqRg2EHbRI3LCuSg0Guo7jzyWpwT+/IS/2969lj/qSnEm5b6Qw7vWPxxny79+h91/3w8d3bKg+Ssz+rUDe1RKiKH/pazfqhPz46MhjuAaplUUJ6oLHFc6bgJMzhOgh0LBAzIUSZekaqVCabWeS12zyTAI7VFpTKU0fGyrJ0+uXan7uCH2cZRjNeltIMMFZTSgmRIEnn6JLuwoJiHjwkJNZSE+lCW0KAvCHaPgrAVoZoMAbqA1KY8DdC/E3ZhohRI0IIFVJiDKklHetav/v+tVM/7uH/4gDzDWoukINnoxV8MpqWT4enyZ6bHXy2Bcsja3Kik+uzr2677+JPpZ3mI5SpYarCRFszUjabLy+SE4kSX3n8C9KUldICAaVNIlRC+kbPHn3j8EujB0bGbh2SJVEpR5h0PC8OEqeZ0xgRL/vDgZe2laXcGlwG7rhneMszaY+l4CJPDYz87aN7by/vGfiBv50fB21kXmhj7JZcu02Xa3lgfHBQaQtMwkvSWacx48WpsiMICZ26SDuUZWh7CpaAZOAWIVpxl9PGUxsu+eX94iPBO1nWPBNABnpOwqQsu4FyTgl0MJpQ1DRKMSoSFQIa1KL9/Wu27TlxfHQrxZgFz8GqonfHBT3blnSuPqu4tKO1SK2UH/p+pTruP//S6L5jk4PjkwdqZpJyoWqx0goA48kMqgk1lXopQKWc9M7IgcOjB2aSikeBmja/VUEkk0THREsaOC7z5hMD0pxtb+Q5G41agkoSVv2pUmWif+Dh+zaXJ/u1oZoBmkJb1GKek4fVxBxPiURE2oIEj3maMlK0ZatVa0pmGxKkMtytIUkS4xB2PIWNBkjbTbnN7Vm6mBIUCKx50bSMKpKA1MLk2Zurt31m39Djo6hQu8YBQ8AocnPIgIt7xs4/NPn1Z8d2RrUxjSoBacbqvLe6q3117myvpZjJZHKvieqcCoSsTqnS/v5vPzu4PcWtqDatwoACrIk4Tdz10k4253mZjxfa2fyUVFLMlUMtKtNVRPH4aGV8uFSXtcx8Wnfihoy1MN/OuHd6Op/mLgFzpF2ZnBgfnXhy7/bx0osDz/eWy2Whaq7jNXBXM7CGhKYswM54WZZKJQzasgBorTlgKRvCUUJZIArWXC2qa81SBoCRlphMiWk41GtkGZxEBbx5AEARVOXRF6P990m/jGJ6IyMF6IyLToY0w6YMrsxkXZX2nw52DIV7RirDOhkzUBq1BKwrt7ytmF+4uL2xlTEez0EiYa7nvTS2c9v3rtdEKfhTlTJLwAjh4Bkv05bx2rL5YnZ5Mb+W247bRXnKEUKJQAelSjkMJoJA2WFj3uatXiZbbE7PB1iLk7GQ8ifnJsYmjpWHD72489nhB6NIzPghsSyHpy07BS41LNLAPW++pih251Yua3dzmYW5Nk4dIkwwHQgZUhgHjjEJkYmUs1O1ym9VHKswrujXDkY05I7yrAwx7Xpey6k8V2hiWe+Sz134vR/d8vIPhs4qXto2v5sqj2IUgINugiJFJ+8kSV77snq4vH0mGqRJvwVhQL106yWd29b0nAtHL8gWCbW65/dQoFzd+8MDd2vMEq6O+YNRECplALSl213OwQhJnXimd+GiUwttZ2/Ie4s55ZHQIpBtHtepmKSQ78x6btZx8pS6zHGMSdlggV9+YXjPwdHdM8Hw1FTFJEmiJT+O8lSGgTvaM0o6NGlwiMMhdMJSXluuc1nn6tOddR4pMiBSZQpILQDQFNUmCKTUtSQh1PZsJU1UkzMsCOYHgRcSnk9nCnknU1h/4YYff2/1cb/jz/DCeL/A2rzJpi14Nn4FKB/eFy+8/+ji8UsPeGte7An5WQMbd69sB4Gw3bGV7Zu6X7zitnWfzd0w8bm9vUf7Kwen+4+qsYYgk8nMjxFGqasPDvo28YK0sVn+kCFKgcG08c6lxdWQa6NaKUCfTWiQTUMhkc6StczzPDudcdNNFig1jNX4WGXw7Zr81N4dwbT0xVlRtUo4RaHJ4fY5nat4OsOYxxwWxBEF04o7zHI999yLLlp00UpfB28BXBQa9FgDIIQQs9GCWj4FzlMBLTqctWY6Pee9dvDhZ8u0epgEpEn8P3lfEXIxjJxvAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<PIL.Image.Image image mode=RGB size=96x96 at 0x7FD80CA1BE48>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7Br2zS3bGY8w",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 113
        },
        "outputId": "c05a5360-5eb6-4b2e-d70c-30515c2fcd93"
      },
      "source": [
        "tvf.to_pil_image(ds.raw(idx)[0][1])"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAGAAAABgCAIAAABt+uBvAAA2b0lEQVR4nGW925YcyZIdZtvMPCIyqwpAX86Nw0VKlGaRD3rll+gj9EP6Ol1GQ64h55w+fQNQlRnhbmZbDx5Z3VqqxkJ3A4VCpYe7+bZ9scT//r/9r8dtH2+3/PKFX7760VUVT0+Xv/zDt//4j9/9z//BFtTbL+PXv9f+Cq3K/Pmvf/vyurfr9fn5Irn//K//jaJgqjm8Zclx7Me9N+Cyrdu6aoa5fvzTn8Uvn3/6+Ze//tftafkf//N/1u359hbHUbDldtRPf//xx7/+7ee///3LL78et1FEBo3VlIuqqj3/5c/ry1Nbmy+2bP6Xf/dvPvz5z8t3/255+fPl+rK0DdCsGuPY97evX3/9/PnnftxU2MzNXIBIifmRlVmZmZkQaaZuqgpRKEQAAQAIxQFABCICFVVCCQiFWYyo0RXC6IzOGKKsTDPdLktb3ZWV1dyKIqVqBlMKAShgqmbmzc1UVcxNF18v2+XpWRvGfdfSCkBUQBGSrKqqIkFBJKsoQqMUCZJVJEUgUAAihMwXAgBQVRgBLYMqzs8REaEIhfL4IIVkkVnMpLCELNIUUOhcHZXfLRAJAczoTksKJMkReexxe7Us7q+1v+Vxh0pWLc3X62ZuYPSR29YyqlLhjuZEKuAGN1uXtm6rq6HGujW/rMAHcBzHcfvls66HtAvaRlCQZJ4Pt5DUPkaVOKgUpYBkJosEAFOoMM8FUp3PQtUgJMvM1VxVRcDHh4iAc3WkyExGsI9kpQJmZgrVc20VogAgbtZ0rjSA5pJUb+1yXZ6fzFX6XnHw+IoaNvdWhUksy6qGPBI1HAIwDVAUGSNyhFRpE3dbWlMAlWZwk3Wz+vBke8vK43YXDyxDlhpHZoYo1BzmKdmTJAViQhUxlaqqTBYFUFU5N8Vj+6iqu7DUDGrzVyCQ331QhCIlUiWZHME+qiJEoFY2F14BhamoAoC7L4BiPhg3XLxdny7ffHv95pv1+YLqdf/C/kVVdGtgSR6Vh2JTQeVADGRoFdVFGGPcb/s4DmNhcVN1U4mElqKUw5GXS2tre733/evrcbuJ3/wS+2DEMPflcrEj6+0YpBQVksIUliKqWmVVCamqQvndKdO5IkJ5P1/z5yqZOwgy946QUiVRHFFjVIxicf4J0/c1EjNVwNvlau0LIMKSKgrUdXu5Xr992Z6ac1cr2ZqwVAUQlDHUUaxCDmWpoIrFrJQICmVZmlWuiy+LNQdFTNVBcCjLVMRslSV5kZsckf1+ywKkVKGmMBXzoICSIlmSQAFVzCqyzr0AADJ3//xfAcDfbxkV+d0WmvVLHgsUHFG9x+gZWSICgRn0sX3MVBW+Pj3d11UVyJwbGIz10q4fr+tFcb8B6stz7beKA0q7NOQKZEZHhRIGDUpG9qosdfVt2yzG9bKuqzWDAG5qmqCwCgJALte1ba1ty+fPb1+/7AUzhRk5N3NrWUBWqqRKUQnLZOWs05wrYoABmIVGRCjzz/P96AEiglnj5y+KkDJvoBjZex5H9pEsishcIFeoipmawtu2Ldfrvq7QV2GBxTh43DDe7Hqxy6JFRRWdliIllXSlFAXujlUBHyOV0tq6+KKQ1cUqLk/r5bq15igxEZUSEVUhtJhSw8SWxa7PV9Fl76mRd+3Nbb0s8vUooZCDYoSLBFlzXUjOVyPIcTC6sEhhidjvC865u0oIipIEOC8wSiYza4zso24H74eMEArcxJQ+f1iZwn1d1qdn3y5qVkJFsO9x+1xvmz5hebkYncdrLSZtYQZHlqNGOZTL6k3gcdt3g7Xrh+XpaUV6dtVcL8u6rWoCgQQhCSHcAavRIw6Kmbanl6ftpX3+9ave9q+GdbErN+ILySSlJCAB9qqVpICsqiCFRPZbxc7KKuG5t2btPrcPBUVCpOYO41mPqioiI+IYvB3y9Yb7QBHNpGm50a2aiau4mrV1Xa/XftnquGmWN1sWa9viazOF2VJygZakUFWgksK2ElqFPPaRXC9P6+VD+/S9ryvun+VOM10uS7tebWngqPsrDAIljDBlabFSpEpR4vL0dDH3W68jvn55e73fjxQpkRQZIkbRLEJgEAhJKIQ19nv1XSrPi5y/2z5nuTkXTEgRKbIoVefPY9Te+bbLlzveDhSxKF3VrFzRjKZ0VW3rsl4vx/USt0V7LltbL+t63ZbLZhLWVqgQwVEsga8QgZi0JaOOL8Qxtqfn5Y9/ufzhT2raf4iRuzva1paniz89S+whIaqiE9CIVlkhazCSEmp2edr8st2Dr2/7OI77fQ/Om1VC0CnIKgAOgKyCKavG3rPvrCiyHmDnXJ0Jd89fpBAUVEmVVLGKmTWi9l6vO77s+rZrpbjSVEzFlW5iKh6//CRVBmmttW311a9//MP1T39uzy947FdVpy2kUAxmtj3X6AKhdIGKwC6XyzffrS8f8/4qNbS5uumy6nbRywv4ItrIMZEyqkwXHkOzMOH+7a4txZZ18XXbfFmv10tJ7yNQVMh8+OeemBdZ1f76ulxFakiFcF7fjy30qFUnLpz/J8Jilky4nsVIGYmeGCGRkolKAUShBgKloMcvP2LZjGzN27au7k9/+sv1T/9muSriTWSCDhNfCKc2tOZtHfdfGZ1ZUCVhl8v2zXfL84d9f5Xo1lwbdFl1vdj2rLbpstR4zdEZQyNkXYpvGoGsGtH3YW3oJmtr22Vr6/bychWR17tUpM6XVhSd65OsYuX9dX9qq+SQGsJzD51rcRaauUZFERJFraosyaqsyuJI9kRPRCITkSiCJETxWG6///hje3ohxFjb9fLy6dPztx9bg4Lqi2qzdlFz8UVEaQtZJSxpVSPGGPseOZbLpm3JfozXz5XplwscaJv4WnCIFbV0oYtQQBGBKNAasoCEMMcYdQu01eUvf/yoIn/720/149d97ofKyqqIHKOksvL217DN49kyIzMk84EA5mYpVlWd/SghJLKqR/XB3rP3MXofY8RgJVhzu6hQzpMpE3CVH79+VkKWZiJ2ubx8//3Tpxd3QKi+qKvapm0FE+a0Lfpe417iVYgR4zgyhm4r2pJvX8frF2bBG1zRNrGFVIoWQV3mURFSSkQBd1gKDhHmyDEqEYstf/z+ZW3Gfnx53aOKrIxiSkXE6E28Dt4/f12++5BxmQs0oSpJAfh+wKqyMiIUKEUkj5F98DiiH733Y/Q+Rs0FEirERPDobc8C5te//NmXRZjF9Mu2Pm1mNKXZbG9MW9O2CUlhZlZkjJGRkRxRI3OMMStnHPdxvwlJN2krfS0Kxy6ihBVJokQJ02YagTqgqmbqTSvZB0EFTGW7tI/ffPj02gNf+3EoawK8cQyLNNXWGswICFMqah6nxw6q2bZljDF6HxBRZKQcI/ee+33s977v9733MVhDJCEF0ER0oiqRiTjLr3/5MyrzuEnm8nxdnzY30bOvVVVTa/CVlIqe2SsjY0RkRI2oETn6EAEr5wJZa2ImbaUtRZHRYY0wVhZBgcBhTVtgJKBwNy8WcBwiJaWqujR7+fj06fXYI16Zo8odqoioyKqlybKIuYhKlWTw/aZ/1KDMjIg++tEHSEAj5ei599j3cb/v9/u+H310VoikCM9NxCoRCAmWsPz5z3/O4xZvzszl6aldn62VogAhlGjUJoKqzNGz3yt79nuOPceeMTIyo6qQmf2+jz78+gyYegNUBICSJJSceEVhPvseqMNczaw5KW1bKyIzc9w71dQ+fNjux5URu3CRujxvKmhQv6z2fLV1pXpm5ehaMcu0QN73Tx/Rj9H3LiKqloUY0Y++34/7/b7f7v3e8xAOIksLQoDG3zBlCenX77/vX39SLaEsTx/a5cnRwS5CURN1USeFWZWR0VkjR89x5Dhi9IisYoll5Dj2GCHmpEBNJhOnSkCgFKkiBQonk6KiBrXZiEPVWtOqMUbvOQrWnl5eLr1HHh3MhrxcV4U383ZZ22XzxUU9qzK6ZHDiP5W5fyJzjOg9eg8RwliFETF6HEff933f93707GSUZCkFhFBZ8tiHJRQ3VzB8cV+f15dPbVt0YliWqIsuJTaRQ1Wxqibb1Efsxzj6iKK7LiuL0SOiKiP23S9hS4M2tBXWKvMEJYKabFVJza6qsqJXFlmYfAElybXZy7KypCLAqOjropfrpS0Xb82ar4vBvYjMwuhVWVUQnLxkRESMrJ4UFjKqZIyMrMgakT1GRK8qVqKoFPLs9c9/5g6CJBjW2vL0vD49G0oKoi5Soi5QigqDrLP2Med1O44+jhFVVFNfzhdbzDHifq/ng7mwCdREbfIUJywJYQbr8UvFisy5+JQJ4YqikGVz4JpjxHE/di7NLtdl2Va1pqptMTWloDJz9MqsKgD1XqIjImtkSZWQVTVGjZhrlJFZGfO7gcg7/zZhCDGvJTr7Gyps3dqyKMjYpYYKxJqoFakAKQ8oX9mPcezZ9+i9Rw6CArQGAGowHfvOGO355ssCb2IDNa+WJKsy8zwOMa/lIpIocmQdY+wjIoaoZ/TG5XJZPn58Om5vb8bWFBzK4erqUKWCIBlR/aiKYilB1nmTjBgjYmTNbj8qRozIyFGz/wABmp4cEikJOTl5OSGR5/0LQG/N3FBRcUh0UVdzEWOlVPIEr1WVNUaNkRExRkRSTMzUF0DVTM2iD1RWP3J0xBAbUApQGZWREWPfSVYFVDOZlBLEJB/ifAmmWhnCao7rZXl+vpAhTOSQPFSaqU2OHcKKweMe48gMkcdflKd+MSJiHuaoGBGRc7+ejIiKCaBnXT7Z+t86Xnq+/uLXj94WsGqMHF2il6apizAzpN+FRSbn7p9gjMxMFtXMtou1DaSaAkomtLEqIxCD1iFDzDJHRmTv437PEQURKCcFBowevY8YkY87W5jCZCbA7bJE9xxRObKz3L2Z6gqAJGPUcY/9LddN3JijMjIzcoyIPjKqqqSyYmRWjszkFCqAyTyfyEdM8NtZEwjojDBf1JsIK0flYASUtIHSrBSUKiiPH8V39lsg5r4+v9iyIcLczK2yzB3mFGSRVToliTHi6H3fj9ut76NUBTpfYBElKhPlV2UVI0x19AGYAZdtibsfwV5ZKZWdtSo2hUqRMRh7HrfoV8k2n8QUSCJizA1DTB46KSlCEShgprOVmYwIOEm2/88CoV3QLrA2lRVWVqXAawyRgjYuTnNSqEbYLF2AqZmZ0drTx0/WVgDW3JoBtHWxywW+nASotYgxRvb7vr++3V9vxz5SoKZQQBXmumyLaGYKkcV+9CoSd4heL9vTuozF+hvxaCEy86TEKpGH5sFx5HGnTSQ1MnJEjshMpkgJCkI1llAfnLPzd22pEJNp+400odB12eCrQGehAZSiJBgpIsvlGe2qy0JrStWijkPbbUIYc9Nlu3761trKiim0wEyXVderrk9iC5ZVzKvuKa0H92Pc9rHvPaoAVVP31haY4uy9BaRkZIgO67G0WnxqjxMxiWDSOVIlIgoBU+Ieb5+HmfoaxXHsY/QJ94NFUaoIQFOREihVxVTdpfB7VZEA5P+3QOpOkhkkoQ3KIjIS5liedPvg6wXZaRvVK0v7Dv0KwLyt1+v1m+/Nlzhu81wLoMuK9WrXT2hPaFuRSKftwS/7kH3ULbL3IRQzWxZeRFxlHMfIiYVxtuSzmxq96AREDebzRhRmVZBNVRXMfhvxgwlluXbR/X703iMiZw+rlAeDzwIBAlRFm/vqcb/Pm+tkGwQEhS5VkqNYHEdFh2gVx3HESL808VWXqy6bpBe0KDZ63r/MSm/u7XJtl2c1F6iIklJV4ouuL7p9gF/FnJWFvXQdaEfpHrKPOo65QFWEiBiYoydZgnkJzEqXrDHvacpUeeZvQQQKMzN3FcmMiqPvN0npxHHvx3FERLEKUkKRgqAmGFRoM1NBlZ6y2QmACHnfPhBQ6NX36ndRVD9qDIFlyvH1NZLSnqiLLqu2BaYFrXnE1qf5naKtfrnoskJNYIRWMSLpi24vur6ItaJQJCilS2A5yu6l++BxBEXcWCJVhCQzYDZvFk5NAyhiRFWGVE2RnA8p1Zr7slhrqISWlPQ+JO9Hyr73436PGBSWSghZOctvgWJQcy2bhAiLxZqoqOYemmIaRIQex2HLjuYUIVQEox/jGEE4USJQB1TUVEu9WdvQ1qJUlq9q6wb1yWtSkJRI6vKsyxW+lIA1tQlJkSgZiSiMwijJiI7sI3rLxafZgGBFTAApWRwjmGXz1BWFJaBAzd2sQQ2AmbXGHBgVdVouRowjK1TLT9R3Fhq1s1kUCqlMsFg1FSFQam7Us2aTHvthbVe9wFy8kezH1zGytCWRJNSmhqAKNbO2iS1ZkpkO2LqJ2slWEVmSBdtetF1gxpwdHEnJqsjKYhJBJNFHRiYEy5ovT2try/lJkYAmgSypEUITQlSSqKQCauaLWZuwbhbwLD3RT5+3WK8KtWooh5ScqodO3fKEKphKYOVsDKeweNpCJv73FBsj7bbDXdSFiB4RyeZ18rn4nZKiUCWZxcpz25zKCpCZI5LLxS4v6gtUJUOYFaOqKCAsKUcfPaInj2BGiUgBblaEsphRkWRxhHlrbqpQgYGuVrDZaIlwXrsiAhXJyvttdGaBSelheWw43DotYXWqsSLTbiMybzVhcWLIk8fGaa8RkYlWPdFkZI0Bg7RV0MbIiISdzeepgOPhEQBYOSngqZ9MWgDQGKOP0S6f7PLBWqMIWKyR48jMIkQ9if0YY4ye7MmKCWHzhiEsV2FmZVTmiLKWy7Y0bwpxFTSv8MqjIouVOayWaV6ojHH70u9RoiXQrJZxxSHW4UO9AJnSKs4SoyKqArIypyby2yuhaFGSVULvx2jNi8lechQ1cuQJL393//32AfTjOHrk/B2KQAWaMXJ0irbriy2rCKSSOXIc0e/juPXjGKNHMYpHjxFMIilVFVmVlLLFIGRmVaQAzMKoeSOT1GSJUowyPUNWIqenys0Xk7evElTYSnlGNB+ivflonqqPzhgCUcj0AKHIMr4v0DxlJUhKCin0fu9QRVX2QY7SUZGcSKTq4RJ4by6EwHE/jj6EMnlCQAnN0aMfhLanD3OBmMkYOfZx3Pr+1u/H6D2qgug9MiWpQUQkkyNS0nIxEcmsiloXZxYiVcvMiNKkUAVWUMIKVkSRBLS5b446OIbBNtWG6p6CsVnfPExrLrFAIGqzCRNMiZXvHhnODlFCJEGSfuxdTN1AYowRlSrSLtv68cPzN5+Wy8bKmryCCKWE3Pf99bY3EtbgTtEqxhhHj57MIiNSpMae0TPn8cIYozIvl8sf/vCH62K317f7vu8gKSFZ4BEsKTcIUMBIMpkMgS1q1pbMgtCm2U6gqrMBZSnMfFnVrOpOhpt5U1Oo4IK6SDaUgMKEigKqagoRnJ6f4lnEKTIvUMhk9Pw4upjJtgil9xyj2uLb2q6fPjx9+rhsKytJf5wyCnnf76/349pUzGE+8WH2fozsKZMwTFaNI8dREVUUkYioqsu2PV+vb1v7BX9TVFGiEBysPLKStYi6g4KRMnU+WFoj1aMfpmJ24kWd/HMGy9Gat0VVqyYB4CuWBpjIVfgkuUoJhBqqfNjPVESkwCIeCzSLSlEKwnOBkhxV7BXjuB/wdV03X7e2rO5tPiK0Rc4zVzH6/Xbfj0DpHlRbActx7Pf7fe/3I/d9j+gwi74f++3Yj8zq9736YShrrbk7t7xfiaI57BBBjJ6UI2JkukuDmNoUVzLrvh97L61sDZsLKu+ff13XxUyzLFwmN+TLYqqVQyorosRMAShos0RYqYEmmEdsggf+rnxMj1WJUM6Lz4NSURGZ/RjH2F6uWK7mi3lTRY0jYMt2mW7Iqhqj32+3vQdLj0HYIrDq+36/3Y++99zv9zHCpPqx729v930v2P72FsdOiro1k3Vb+nWLihIUMcZ5/nuPyGxJLNNrJgJkse89RnetZTFZVEO94vp0WdYV5cPF3aDuy2KmEgSLmQKDqNCEzlN+NlUaxQUKFUJAUUzZ/7R4KijnhU+KW9siMipJ03Vp1xddrxGV+61//fnYDywv3lZb1sgRo8d+6/fjmAsUSQBAjr7f7rfbft+P/RgZweLty9eff/z57XZLyk8//nrcDoh406W16+r7vsfozFDWtrq55RgQ3Pd7MaOAKDGbPHpRes67L6VcFq0M//Hnl48f16dLax6RSzN393VFhQPeGtTdqHYJNGYwIkuZIQURc1EQgtn88mF4eJdUpzpRbr4c/T6ioLYtl3Z90e0px9f962vf37789LNsH9vLd9vHT6wx+j72++gjskQYRQogUjGO/diP3iNGVmZFP96+vv76y5evr197j7/97ed+pIHNsDT79Ol5jD1GrxFgLc1taWyjsvoYI2ZtIlUqzwUaUTGiEq4wSPQ0fjFzuHnz1qOZW1t83RDDhW2Z7nBRl2JVDuZxavBQwEE9b3y+g9136XFaIUpYrmpqxjH3kPnlydbrOF5/+Ne//vTzj//yt5/a5cMbnv/dP/7jtrX97a33DtV1XdzU1KbQVUySZr5cn9brE8lj399u97f78eXt+Pr17b//8MvXr7fF8HKx69qqerGqMpJJRVuWtqRqW/tytKqo2UyXhExhpiLFTNvq1jwFWfXlPtrrm7WmZmqu3nRZ1+vziIGxN5Xt0tRdoBmRNKVKtmSOUsmWoiIggIfn8SRS9OGwmeYFqJo1aFSR0HZ98u06vthPP/78T//0z//0wxfxH/T5D8/ffPvd95/6fY8Ib227rE2ttUVUgdPIZc3Xpa3XKyv7cdz247b319vx4y9f/vbjr798fltcPl7bx8vaYxRggLtpWxusLQtUvTV3E2jJKcbXIzMggtbU3aBaIllSkevrvS1N3aDmy7Jum6+Xam+Sh6msq6kvBRN10tSaZlZkikq50OpcoEkJndrLaUGSnNDbVW1Z18g8+qCgXZ7WbTva+nav1xt1++af//6T/5//9G//w//08vyUWUVcnp5enp+a4nrZWlumEqCQbWmyXtdtibGPMUZUH3W/H7/88vXnz69vvWqvr3t8vdZ6j4x42to3H54/ftgAba0RgHmJZgkmdgajKiKrylQVWlURYgpCS/3LbZi9zj7I25kaoLpoK7IqDTQD0JrbxIW5D3aO1KiWtOLkwjkhnjBFpKpqOioBV9Wmtq6rQNd1XbZtvVzX54+2XgVNoW97/+Gv//2XH/++/+XPAgpsvVyuT9fV9Pp0XdZtWnFNdVlclqamEWOMOHrux7i97Z+/vH1920PbEXnvRQHvI0b/9vl6vTypNnd3b4NFICkjicnQqGROzVFMZEYLanp3CFE7erzdDnezZt4Wb80UBaWgj+i9mzd3t8WBBjcmgR6MpPb0US3SRPTswk77gEwfGkRU4QZSbbtctuv18vFTW9f16embv/zlL//+3//013/98l/+5RuPa74ev/50+/Ll8uHJ2+Luy7JdL8vThw/r5dncASyLu9vIjOMgMSKO3u+3/fX17fXtHkVvoBiKxrqPia7N27pcruvlYgZ2VlVk7iMysxncbHZ/OsUNM7gKKzMoYm7Sll5xux/LYr5s62VtrWXW6LH3u1SYqrn72nRxwBglq1dSSipw0AZdqFU6oWJVTpaAkhCYmrfLRbTB3Jd2+fCxLd7W9XJZ/vgP/+a77775r//3//Eshx51//zr7fV1e3nyZV3XbSzLdn26PH9o66aqAHTy6py65HyAox/HPl0mVU2q6Sy9FRHLsl4u2/V63S6XdVvNdFbK2X/2EUIjT9lqZofewUqx5EwctAr2qOPoy3EcRy+iIvuI2g9jXjZfL6tiNRMoCGWZRWlQAyipQpVGWSaYzEKxkjlbcFPzy6dvxRdb1nXblu1i5mq6beu3f/zjd3/6owLLuB9ZX15f7/e7FlvbLpfLWJZ1uyyXJ/NWLIIUFaibbddNlaOPMWnziB55RK5jrAZCemYk//B0+e67jx8+vmyXdVkWoBQCqPvM3UCgUzJWqLuqKcnIABOcfmSYm+jCGj0xIvdjBCE9Y6T0HMjR1+ijYjQ2pZQ1b15LLZGZEpIkjlSG5ECGMFEiSVDUBAB8+/QdvPl6Wbe1tZVCVWutvXz8+N2f//SHP//x6Dccw1UNNLPWfFvaG+BtmfZzVgKnbrA0W7c2g005/QOZEDFFVdWZ4dLL6h9frh9fnp+eLsvS3LWqIOJm67o+XS+SORWqGVJ5hNpO8UZN3czdWmulhpAUjqx9741A5Ow/+8j96Nu+b0erZroscFMXX7QIAmJEBYeMIodWF0aV5LzLRCAQXz58p25t3dZ1bd4iupmp6nK5/OEf/vKP/8t/ehu53e/ff3p+uSzL2hY3V4AJNfMGKFCqTsAMy+q+eB9HZlRmZGRlM7s2r2JPMZWl+ceXpw/P1+tlW9fF3QDJ5Mw2PD8/QWDM6evRMwAoIA1spgaDwr0ty9KWlVkJmQTC3gcxtUEQdkTc78fafG3m01E4/+wyeWy4dYxe+zgYGIoDkpOFpQghCtCXywVqbV2XbXN3OURVBbBl/fDd9//Df/xPO9vx9van7795ed5ac2VIHCrVWvNlhblKmbmZTg0PFTl6xshMEWnuH58v67b2pKpuS3t5vn768PTx5Wlun1l9MlJId7va1txX030/pg9k+pKacl1tMZjC3NqyNTdVS8kCqiSSOs3UUKoWlERkjd77frS1WWtCamtmDlM10ZI8GHscGiEqRIVIniyYTR3Om5Nw8wnSKlXnebF2/fjdv/3H//jhD3/ub2/c39r1qTWTcY/7q7KW1lpb1ZwsdW/NVVij13HLHjFGVanq02VdliU5za/Lpw8v3337Yd0WqF22zZuramZEFEhXNffnp6dPH573+77v974fMXplOmpdzE1cbVmWtl0Ekj2SMbvozIKqmiuY4YQCKmREHX20vbd2R4ZA3NSVBGXhetHYR28jVQQYJYhKFkRc0OBeVVB/55tVdVLRZWrL9vTxu/X6IfZ7f/2cAlEZr7/G/W1Z1/X65G1RYGI4d1fVfuxvX76EmKqs6/Lh48tlbYRlJQTr5frNp4/ffPNR3SLyTCo97E5kAtIc62Vxt8t1O/btuN/HcWR0SC3NFWKK1pq1tViZFKhgZlyr927mdJ0OiIgMlRF59Gj3w024sbmrzayc2tpa2bLntlVeqkLyyJAq1hSAEu45whabivi0PJx2kQk+1ost17Y9t8vT6Mfx9vn4+nPs9/Xjd+vTs3kDxFQBNDc1PY5eP/+6PD2b6dPTdVkahGZLH91Nn16eP3769PT0DODovR/3fT/2/cjMqlGVpqLKddF13eSKcd2Ofe37Hv0Qlrc20xGqKjBGQI2mVCM0sva3O4vrtlhlZR5Hd1E3s2O4zlilwYeZwaBt0XV1VLv37Qk1KgbvtyORMWGjKCV8WqveffwiqMyIpIgJQINOfyUo6LevLGmXp6dv/7A9fTBzUYCqqr5uvm0wz5KM3LZGeGQpYN6O41Dh09P18nTdrteq7DEmSVWczQSn4cKmuCo0d9XTcDCaoQo2HZrElKNUYQ610/dYaedumvIoBFqCLI7Ife8K2rL40m04mqMBblqwxXyBLdRl1JIjqkuxZIiY8EzPPHzGv1ugKVgQKmiLq4h5h6qory/fvnz/5+35o7oJhBR1923z7WLLStXKvF6uvtjUR9zNzCvDfHFvvrSjczzi69MvR9JMzZu5kayMtjaoLeKKVc0gUpMGm4RNJqAwgxmhUUQmlnZmn0lRhzunSpbFoinX3qObLwuSgLoqHOaqC7SVeOSSI3AIMilCFTomNXsqOlOzl6qUwPSxG9AElcFMXy4vf/gHFT59+33bLqpOKZDuS9uuT5++7VF938nwpTXbQqyKVYmeGdFH3G73yMoc47hHjMpkpVDMfJ4dVTMzqJ5HyayqEEnWxOuUmahSSmYJFGpqZjV9adNTAylfahwlQZlitRQZMV1Dw2Jw9HIvgqrS3DZfrnEZtYt0wzGkl1SJq+qpKYLv8eGJtApgpaiRzNErYtkul+d/39z9spk3ilRBpbxty+Xp+ZvvCP3y84/R97Y03TbaOoL7/Q49IiuzV1HfbkDFOLIyMyoDEPM2PaqmOus9SYWqimoCkrOhn/hWYEJiRNaZZXAf01cHMTMVk4XjuE9JWQWmoOB0KI5R/ajmMCttVEVrti1r1FPWzrpDw3Af7EE/Y6/vPs95sFgy42yzDJAiYsu6XJ6WdVFQZi5GHlMczJfLy1V0KvrH7dWWVWaWovmkkKawwUqouGPamiNyshmkViZNhTX3zrm1fx/MlYfCKwBg7qpD5p5pjaxTUos0TDqgsiqkgqUCEVOFe/dm3sxaE3PaBH2LF1tyO8Zl1OSQpp/AoRCeev3D4jWrUsFM1QDMGRG2bpeXT+u2MEfEkZWn2g/Afbm+YNnoSxThDYqMQQG0jUhfFgEiIlhAydpKisWIHCMzS6Qykk5hgTRTqMk0DULPMPO5QAoRNThUtU93rSxNUBWVWRFB1Yf8lIPpJThzz3DrS7PWLN2pLs3hzdZFoJVc7/fLiCVhVBCZczSFTjeJA6cPDqfDUqdTTRV+fVm2bblcVSUZU7uT09OhMAen9Xxt1w8Uq+isr0WZnhAArbUcURWACFTVSkqtvDWoFlVN2qLmDjUCClE1EalUYCoOs+k4n6VC2tKWdc08zb5S8nDiklEsICtYQyDJqCEio+nRhx0uNtyHaqgZzNV1/jDFavXk2FN3UxeIqpq72rySHnhIz+Dw3O3L5bJdrm3dpEYmoPOc1ekTVIeJJOHbcqFqi/01+r0ioISpQNzd3UYkVEWh1shUTXdXs0gq0JqeZjVgIiwRCQD4/U1yKhFQePO2Lkc/ihJZegZ2ITMXQmhJkjF9Y5WqWML6COsBH1gDLUTapDbmTANT2YxX51uKGfxEOWbAQ7tXlentUzFvrVlza8tq3qahP6c9RU5zOScrMedutCYibjZMR9/H25ccw62Z2eQxslRVZtpLIVLGUpBQUYHPdAvIzBIt5CSaWmuswNzMphCQJkIzemvTBa+mSlH3trTWnNXKRHZq7DUF5kQfOUbGyIysTGagcsrPwlkwtTmaRkM1iKtN0t5UTR5AaOYWpifQvHkzb+5tMfMZNcoITiWXpwBwOm9UzVxV0w0Ku13l9poxfN3MzZtbMyszFXcXVVBqkj8sE4XA7BwpwkpCquasCG2tVYbM56yK6cOtMi1zM3c1NzMQ5u7Nl20VkZQS6dyTo8/ARo8aI0bEiFgiZmREziw0IWKGZmhIF7icM2QMEFbOaH6eZgNRNXM3d2turanaRL2zcaoKQAVCqKAgUBWdQs1pGgKsadu0AGDbNkZk5lT3IVqZrMn7KqataX4RNTObHZoWqArA3Lw1CtUUEzdP6vJ88KpqtjSwzM9bZXplYcbWqnKGXjO5Hwntc1u0pVkbsF2LhM2XM4E4AFdsbu84qKrkfPGZxOwwzMzU3LzNS3d+3yQrZ6+mj5IAmXq/cMo0RYqa2qKWJJdlqctlpqVmi/nw1kx2hRCb6O/smqkRg1TytFOb2xzd9NCI6z1eOr+IuaNSp7dh2lUn7a4mqiWoYkjtnQKBqTdf9+ZrhympohOiz0wkFNIMK9VnH1YoTNIvo2Z4fFZqnXeZzsIgLEDUHRCocgIPThtGjogxej/6jBpNoY1AValiWZcxthE5jp4Zc5jExDmnPFWsKmTUfHw43QDyDocew16U56MSyBynMIuhVvH0pzuEaV5UKc5bk5D54OaYF7PuarosYm5aQmYxCpPEdOVqvEj5lKMLhZm7zKjMCQ7nFTbdRngXrkXMfOKTM/FVUmRWRWZExojKKDnrPUUyY7ah3pp7i0ipnJ5HAGoqFMyHXqyMBGaURh6zE/BII+PhMpSUOSfn/XPmljw/AQ9H/XSOndEeyZIiJQoj9RjNdLkctm70FM0sCQqhULhxEW5Crxiws2PKidsiifJsIpiRydMze3o10VrLkKzIek8b8/HvU/4/8QGUVdk73ObMqzlMTc2kKJoyT4NUJaf9lmTEKKab6+9H3ABVKdBpv8TvU2xn1L3OAVvT38mUGoLKqhoDMeYTmfdJkX3U3mM/uh+9tImh5OSqAajSSSd9dkDCLHLypJlJmWqvPNxcv1kUFQIBVSOZj5NUfIQ2K8k8jVqP6UeVFVIQm3/XOSrKDHMsi6QUyDhNbNNuVUWoPKJus8rMDXtaMvCYXXKGawicV55Ow/Wjss4PsPSUimg++2GkMDNjdLFFXAmfuVcRMYgrF3KmNKaXctonTqKOnNor9Tz6dcZhHma+eZ9VZslv/33OQTizx0mZNmuMDPB0tc+HPwG8iJ6DIzBd5I8P1cos0zNYMVO/gsqY9xbOTURmTsqhzNXE5odbJSg9o6bHHjopALGm22VZ1mamiwsgM2APaAFZNWa0U6QpNqHP4UTzb8o4o7d1TqNKYLoTzstiBmB+t12yOOc/1PsazTJZzKoU4XSPZI96fJq+I3SFnO72kgLfv4KwhBWc/fuE0Y9qpbOinEfvEb2dVz3ezasPv+CEsXh4NwBxt9Z8Xdxcm1KAzGKEiBGarIy5QDRI07mDHqbC2VgRIOshS8zvY4IXCmY9PsFATqptQqcJMzJnUZnzR4Rn4iSASp5JP0DNpAoQzAiMkGOe7qjMx1ybZJq4v29agFBIKc/gwxnPkROP8f1RM2taygVSRCVRnEcaDyF7aVAogVGQQakkBkUqk6e3lwb6uR0yJunKc5Ocr3uWEj4C0/MZT02wJjGeERH16Bcn9y5nxAiPiYR6vuQqVomoAsSEThCF1jnwMDMrQ4UqKtPKfq7FuSMgDyz0W8Tr3B4QTjspa+KnqvfNnQQJpU3PlIqpTIaopkmxKFJTNprfpCqmFc9P5DbhX5Gc18lp2ZzU7kmDkJURY8QYPJuyOPXlquh9/jeZ0FRrqj5r51lJp0eyqqQAe39t4EmZ8LRBJucnuL+T5VNafQBonU8N4HvVevi5wTldjRCRrIo+bd8kMfHQyQnOz5RHbmX++dO/8GjYAQr8sdIzcyKcRIPI3PBnAy2TEc6MEWPMhMTEBWOMzJgnbY7lIXN2ZPMQFOt86o97p0hVnRauuW7vL09Ofu50CcrZlZdUir1fW3XiWDljuefXPoHkOUjlkZF45BhlZhnxiLef57GQciLVM5OaWaygKDQF6o8GK2beRGQWQs6zNIGDzheakb3X6GomgFRVjDH6rPnzW2cVZdbW81hVpTxcGgRqBiqlqlKYoqKlc3zUJKSp9n5LnseyQk9LFYoUJjDbt7kZTlG5OFG/CXw2BMKCngdgxgjP+VxRUTMbVlVFiMJUCWGKzJOhJQaYq58oJjMrOVFqSmZaWyZD/G7eqzyZc1PNSqlgJTNnGeE5rOPEWXxcWieu+Q1Jza82ufqcl/ic8yOVU0IS8Tledi7UpCMq3wMSFNTkhck60awaKDonf6qWcPr/z85xzlGkzgxaFLPEJtwqKUnKOIttgRmcgzQy1N3PqXCZVcEZecrMTD0RhZ49wExDk5Mbyeh8DBxQVzWbW3y6VYSYw3vmjcDTC4vzoM9vqxKVhBXmbu2VAZapAiaUOWNCq+aRywgRPY/fTLbykeBUgxmm7u5mM/oOzZyJQlZRWK4okZyjL85CM/uknK4ksChgnQsklcr07LN1LKl5rmaElQuezH2OISA5ZxlUFoCMyBFzMoIq3JtAZwYZ+A224Cw6Z6mRM3Em7zcFq5iCSk46IkZlmAA+S/g5x2dSdyVEzYkJFFF5/C1zVwJqs34BVEhNAUvmlpm4VCb9YJqUyPKqOfCOeVYZZM3KP3+e4T4f/ZhjGqXIOQAnO6mqaHMiB+axnF1omtpZm8cQ1qT7shh80JGqszNQNZxjg6ancLYpc7mrpomixvyWKivGYMacxCsCMjGNAjmdGsqKqhAh1N8xzeQ+ZgpxFnY9KSoCksRsC3yGyt3U5pyQ9JzH8ezn5mAVwRyTozNkNyK939+grm6n5wxKOCFu7mdCcxyvX26ffz5ur2CZWrLu99sYe4JlHlmjj96PHAczCj7jD2OMiD4r12SI3hvaMx8x0a3UhMQVWdHlpFVd3otHJhFip9v4dx28FCcXOCcpD6gikrCTpKoQVpEzbN5mFOq97RYhzxM3ZzHM3zpnz5wjlcv77dXXK+w6nbeqLoJiuTf3RlY/jq+fP3/+4Yfx9tVUzDxFXu9vmcOW5tcnGXEcx+j3GocwC1JSc/zfGOf5nbNrzjU61wkTnYmU4oSmMQZVzVSbnqsjVSREDHoCEYGeVybeT+rsJM0cSFpN3rZyTGlsZInowncO4LzXWMisLMkzTi+zr5xEVQqR6dG7LZfZHImYSqsQxWTp/djvP/63f/nlv/+Xz3//63F7k8kGensbo6SWdbm8vFw+fijWfvuS/a6ALataK0offfQeY8zleRTGc77qft9F6CruNq/+SUJCUcWMFBBVs3PPECLqjKiColO7O3mErIicOUBymClUqhJnIwOhRNTe01TXhhmQGTELFR4d5iyWmpQq6qO78uz9VJwm1SIilVCd+bT9fv/7f/uXn/75//r84w/7/R4x9h6+bfdCqW6X5eXT2zeVttrt66+x7958ybS2UjDmtK04XWIPWfJEf/fbriq6LYBSWCVnPhcgJTPVzmteasYKo1izK53Ae5pizyWaO4EplAxVAytPGU1EKFncR5mmqpkhSyJLTczOoabnIkGS5zymeX27us85wlP4F6WaqzU1j95/+eFv//L//NOXv/3Qb6+ZYy5Qf337coyArtftw/2+R6yXZdxfXeWCS/ZgMSgjsiJO9DTvZJLkGPH3v//y01//+uHjtfnHZTVVm6Sy5IwD4sQCczrdXLWMs/Ocew2zN6rHtKAsctb4HANBqZisiszEKCiCGW3HSaKIqrppzv57pg/09CScI0RKHG4TBk7md6qA7ot6G6P//Le//ut/+a+3L7+CAVRm7L3/+vnrL6+3AV2u19f7fvR+va6o8fR0aW2BdCRGMkuExczzbRYej7z38fe///zTX/9m+v2HD1eR7SS+p8FjNkGsKorMKUMCAZNqStFMRtVjOLS8d6TTIW+lGQUpk3hMmQAFeY4nf3Qzkx1VmCnnxLZJV9tJTs3rtiguFRkDMWBuNlOMsGUx92Pff/zXf91vb3G7SQWUUfn69evt69d+3wdsZO4Zb29vW9OPLxfDt8uyHPsBUwqoZg9qZvbvQsaIr1/e/vrXH8bbTchJ+MwzY2bTTANgQuX52uQxjuJUPE7IK1XF5GkxyhIWRDLTCqWFCnB+iiQlikoSOieQFCWJkTSVKBRByOzKTi+scA5Dc0akdrWFbZ34hYAvi5odb2+//PDDcbuP+73iELBXfv718+vb6957ms975+fIVVl/+f756dKWJjLfXMC0NXGbsyMmagTZ+/j1168//PDTVYaQekrc89EqqZiz8N6b20d7onO8EwBV1AMET7A55RAhBJVZCkqxEvP9OB5xnqKcI9OEKZJE1MPSKirgKficfCwlRYSexx1UW66zM1YzzndegMQYx32v0e/3+/3tyz6OPfLr7b73Pau0Ua1VxOvbbSj3/fnt7U1N27K2dRGKQ5MipsuyqMiclrjvx99++On29vb8stgkj0XImhh4ihITkMuDez6ppDzbfp54fV5ANsHUDKCqGpNZaXO+BNCWpqNUy4yZUxa1Up1vOUJytpYLTEokSMxBbKqSs/Hy2O+KVnGOQ1WADzQco++32367vb6+fv7y5fV+u0fcR0QmIE5ZzEtwu9+laYxx7HtrDTCYiZ7jrqDzghCIVNbtdv/189fso/l23iBykqHvROlvjP+DGZsXIEgmifqtieGZplRIUSDnwL3TCKJqPl9uGGTUFHerXGe/OxfbFFCtQrJKHiqgQCBm8HF/83aNMcZ+h6o3n+KJiOQYb1++/Pjjjz9/+fz57XbvR886ImraiIubwJKV1bbWzHSKfzx7aFNRUVOrGCJWEV+/vn7+8iqs561dt6U1PyXJ+fYYj2T12bdBH/6aU0dFndQhAFKrOGJEBVkQUYAVeqqQoopS5UP4nONPRkRUm+cIKjM44zP7i5OIIPRdJ8acVRa963G31rQ11oWwc99n7ve3v//00y/3+2vvPSKLY6I+smWVYCmpiGZXUz35larMFNUqCGuqVCnS+/j8+euXz18h/PC0PT1dlsXVVN6ZZfltyDqgp7vhhN1FzNHkICbfgdm+ZYyqUhFVSJbKNLiWQsUMWvr4mEJnTd/FHEIiVZTH7n3HBCeZMskHV7M4dtFXa8vy9MxiSWTMd4iRYPz9118+R0yplJSkHFm9aqmaHD4pfhogppmOETGHmySZVWYmxdfb/adfPn/58tXIjx+uHz48L+syg+2nP2SWk1O+PfXHSY7PoyUETEVQMoNdnLPu5HxfHCuRSY+cR1LNLNxhphU6Y0QURDGS7VHRHl9azFBUTvteFaoAOlTnG3BMyWm2gPN1ikixXvf7rTj7WBGZeeSeJcQSaSKiagqIvDejlanNeb4LAecm6WPcbvuxHy68bMu2zRgLMIeGPYrPbx+/T2hXVYlOm9Z79XnoT3McgD6GoODRCM948+OiPOloeZ/XMZXt39j/SZ78pkeSVOH/C3lZNjVZX/edAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<PIL.Image.Image image mode=RGB size=96x96 at 0x7FD80CA1B7B8>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4-FC-zPtGY8y",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 113
        },
        "outputId": "99244bd4-d177-4154-a3cd-6b01c6961218"
      },
      "source": [
        "tvf.to_pil_image(ds.raw(idx)[0][0])"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAGAAAABgCAIAAABt+uBvAAA9IklEQVR4nG39S48kS5YeCH7nIaKqZuaPeN1H3sxkASSb6MXUkAOQu1nOYn5Ir/uvNQj0P+BqMBj0YNAkm6wksyor8z4jwsPdTFVF5Dx6oX6rZjjtAQRs5+bHRI+c872M/u1vsRAWAv/+7F/9i/7VP+3f/t7e/MYvb/PuPklG99vtuo+Xtt/66BGW6MR0Xuoyl6kM4cE5BKY0hEzQODfEprmLb8gebcCDtg5zDITv7tkdeXq8++/+7/e/+Zfzw7uXL3/+5fs//PLnP92+PF+v2+1la+a9dzebpqrzUs/L6XT6L3/64Y//9Y9flb+7nMt8eUh6m+XecAlQxNnlwnzWsohUIhEFFZlmKhNq0amIKpMkERfmDG1uZjZGD3O3HD7S02NkhAgzK5HomFA7AuDrjeYneXjml2fSE7RCS+ri4REj3TIHMghJQCFXbBqr2lDeK90od2UrsQkGZ0c2UOMwtp3QKEeWhgxzWLfmg5Z/Xr/6P9//5q/r6X6/frx+/v7ly6f15eXl+bldm49BnupQLkXKPC/TXAHcL/JXX+uZy1zOMj/66T3zPfQUMoPfkrwRmgLMgAUpE0oiTTk5nQhByQQCACQgRMGsLKkBIFNHDgYHAKIABNCXv/6a//zj9D3mK/zhM33+5MtdlNllMlWrGUBac4/wRAJwxpDs6qP6qLjNfhXfRW6KJraRdrHBcEITGGWntHQ3MwTRSEuAPpT3//359/+a53Nrn18+/fjl4w+3jz/dPv28bTffIiwznaWqlsJShZSkZz+fomZdbJZ5kvmc0z3KPfSC6U0pX5M+AnDnsDRrzAQgsoH2zA6IAEgBUUICIEqhTIYnM4coQBKBCE4QiAms8df/fatYtx/PLyjXzz5/1uc7rycup5ymcIRMbuZu7kGewqbYa75MsVdfJ18rXjSfZFwFm2SjYQxjRASYQMfnxTje7CA4z+Xd/2n66l+Uy1vz/fnzT5+//9svn/789PNPL0+fw/foFEkeqZnETDDOAUtmV4RUnHTiZaZ5Rp1yWqiceLqv8wPpXUaGkUXaLkSEpBYe7vDwTEoVAMiIYGQgmJzIiZI5MwlgZkSEMBEEgNLbb/bvVr8+n/9mO3XQ/hPf7ni6i3LOZQbYakQ4bCACaRrbJM8LfZlwnWyv/FzxWeyzxE0MlKDj+CYokYQggBAF6TCGc/Xynb75PdeL2zpa3r78/Pzxh6ePP12/PPXbC6engxAIgDpZklMOAQepARZQkqmKongWhAaJlAIpJFrCEQz0yDJlIinQMjOBDEoggQAoEAx4Hs/EP/4QgSiFCcRMBECZlv3+zfbd7+r1Pz0+4259jtMn3C5UT6iTOcIswuBdsinawi9zvszxVPNWsNZ4Kv5McZOODCCQBiQijl+IJICQBUHYB7x+Fad/0bym29z3fbvenj6+PH+8fX7q1+cYm5sBIAcRUXoC3tNEiCI9QUyypGxExDmCGzKQA+GwCHJAABAlAPBIs7SR4UdlUpGZRASkJygpEp4AEEBKpCcDTCkMEAGsikGT7ufzD19/NcpPuIL2F1u++LjYNjlTZqf0knvJvfJ19uc5nwuvE18rr4qdYqOBMKQjHZFAgzGIkHa8J2AgGDtqXH4X57/qTTCKON+uH68ff7798rTfrrFfsTXAkJIASAIJRzoASlrAwLyU04mA4MaR7JGUmT7GSNooTyQaRp4jbBiG2W5+i+xCWYkRSEZmAkRJBj+OlVOCQUkgCCWIiCGMBClBNGaeL7fHr5h3oec7W9O+pF0wmHunrHN0oSvHRtwqbUKrSLD3kOhC6meKBkGQOZxFXcwMHHBGNGQiClAoT/8M819FfePZY/jYv+yfP99eftmvT9jXbDt8ZI4AU5AnMRcppiq27+GewpOg4KTndxirxe1oKJmZPvpYgYm8ZlIM26zHGBGrxwpyIk6GEhIJEABHgBKegvBIATJfz7swmJFEAVa4sZZS32xn2+zdE6zcBpdb4FPlRuMTGyWGqlU1YiXqFIkYbbi76PwoIkzEzDIBmo70Ztl6EshucWsjODGYH+av/nWcvvNcpLjndXv27fa531brLfsa5hGWzQ2egQQoDaVAlsTIPoiZFfN5klKT4Y0pAQQBgcw0tpZKmew+wjaP7r5GuEgCYQaZTFmPp19w9Ca3eH2VeVzu8CAVSoLAlaGVkVX25Rzx5iZWp6dKjfLndv37cj7Pl0utNUZH9uV0drexu2XIXGvlo9GAiUmZEei22xgjhjEygoJqS0osc/lmevM7o6k9XzmG2bDtevv487g+W9uQI9LCwgJmsIEEQN4dc9lUF53OlDFGXL980pnrctL5DshMJpFaT1wurHfMxTKaAlR6N4CTGeRAACVCksAUnvn6dGbi14adCc4kImYMB2syoFOpxgD0Mc9R0mbfX67b7cktbcMprqnPlufYImx4bAA8MqlOlGQwa+lOZsxK5pmpdc6IaMNyMxtmCZmXN7+5+/b30/08vryM9QemOWO09cvL83Vd195XHz1XG4ZwhGM48rgQw+355SyzzCcww9m6b7dbqTJfTpkl9MLzpc6PpT5SOQnpyKy9bEy0I3eHOTKRftQC4GPqIEIkGIhMJDgzMomOZxaEpEAStEzQFlJ9ngmYRjs/7XIdeX1CBnrAaFNpIOZgyyGVGIWV97UFmu/NhmUHTygkqjqFIX3s0aOPnhmYL+tydz69fRv901g/+/4JsrhH325mu4UlISK2jraBE54YCXEQQw02fEw3XhbVM4uh7/tTWyY+nRedF54uKO9kfpzm+6KVmS1sUrCQcApsbcO7E0MplAjgzMxMi3R3eLojIoFEupMLE6CgBJIAncSdVxlfmBwcxr0rtblOd2NeLsupqHjrX7IHCOHgRpDkSLM+1jY62CGCiUGaZKOlTefTPF+w5+g/7S2owGNNvLR19b5x7JCIkECKVmIdkdse246xwRMIuEMJleGCGBjLxuuKExikHYN87BY2pio8nbK8mZfLMp1ECkMLzHUQd8bInEdsGWyOiR14PTeZGZHukR7uEU6R4Q4gmVLYiw4XJhKV+JztU25PJBkS1HyZZ3lf44PMpyrkHrfbZ3F4JJLATCByH71bHxiGmVEq5vlUigiCyefT/fzmq7mxzpM8/RiIfb0+//A3RtWtiJYUSQ+iYGZLH/tYN+wdY2AM5EAmJgEEKiiCNMvRwkh4JlZEkhOQRGBRqlOtVetUtTKzx2Ru4C3Txmil1zF2ODyTkYAnCCBGcGIEDSMPT0M3d/fIIPSqXETApnT9C8bG2DmZ+ki0Wex8N+s0E5pte9vW85Qm8EAMQq2ZGOYxHIGaKIpT0fN5KapEKdzmi8x3sZzKdHo4P8q+Dwu6/vTLAJIr6SnLxVxspGGNCCc+3nsPrCuCwIkIuGFm5Axt0NHQKimJsrBQDeQgYlIlFuZStEiZhFWQYuIUPHbWWVmExQIBTkZSJoLgDHiQBzxyGGxkb+g9PDzdSUTZiEjFPosWmpUKw8foMXITRRFyu0VsEmtYhCEDSRndxsh1dR9AQhmToJ55OYlw0DBWr2UU2oKvs0Af6ZSXtWG9jnG7tvVl2BfSxWzaOrtFqWU+37vftrH1a94SmVACHf00QEAHxEy9q00RygIOpoSkC2WCiYhAzEIijHQ4DwIVIiISAgP8OjElEUUmhnNmIigD1rnttu3R94jh3S3DQUQgvfvtt7FfzZ+FnIjmUx0lAXf+kr0rOrOwcKYNQxtws4BWKdNE85zs43yqy+OyLMxMnFnFpQawEzwLEDwyzvM0358e7byu7cvLdv1luO2SPFUZ0+AyIEyl5NT7MyxRCGBkAAwF1GEN0MyTM1mkOuCsLK4I56QkAzihQCalayT9ul4JESlDXpt0ZFCmKGcHeaY57d333bZt366xd+srRpoH2Em1dJQyxTnEKEfsg9OzebiHzrgsecHYnK7rGPsyz3R6V+tjKTbzyuisPp10mmZmZgIESBCM0qVw0ERZcjRzw37N1LnO+tXbx68u663dntdt22LYVrl+cz9w/fS5W8IAT7gjGRRIAzkkiDUtzYxrLQH2YXb9gad3Un9rkWxIzhTAiRzHFkoszEIkFtQRQSgQgDLDnZAZETbSbfR9bLdYr3i66vOLvKwymriTyvlbihcEOB1YhB+kXOUC55K8oN/G9WaxTTm9vcz69nf1zVeip3j+S9x+QH8WHfUy1/NFLh8oA37z8REAs0QQkyRIGlMjMHyEu4U1rricp3l+2x26XGl6+vmHL7cvz59+xp6IBBFA2ALOmHboglPhUoXY3JDlngg5+vr8QvPXMt/cR0RE5HDwMWOCKRmkSGSkwzmTYJJpIHMyYxtsFsNy9LTut80/f9GPH+svX+r6ZUYKANXTmTsyJFGSlLmAGL5irCIyboR1gLosS7l8ff6rf6k6e/synrcoQbrQPPH5zOcHvbxjuSBY7VPi5j7YLaKzB5c3mEL3m2PNxBjdvuzMmjoJz5e7Yvubpy+2XPrb9y/zjvWG4TjW6Uz0QAiyeMJglABytOuVxiiPAbshr3CLCE/jVBAlBQKIhHuER3iGMTwCQWkpGWgjzNkNYWE910brTa6bvqzTeq1IUaBAtPAHTA9AsJ4tKcIp3eLZ6TnH2sdo27W3nR8fytsP5fRVtE/25c+2XVlAy4WUeD5jeht0H/zAwkmZmKL08IExkhs7eQ5oofMZ3HmMjG77MN6cq/k0Vf3um8t5Lh/eXX76/vOPP++/fAIxYMhAJKwh2rCSUpkGxp/WOIvfT6fT7LbFeDFa1S/pE9xTJBOBiAhEmLuZ5fBGW4Wmi0eMgd5pM7eN19W2ra2r9S3X5m0kvBOUIQmo1K9BkWkJTbOILW0dRtaZAvvatm3b93VO8OkEYNw+7r/8fY6bLBeuZ3ANvnO+ixEce2oNrwkOWjMkmCiItDB6IGUPE1BJsWJpsOy3a2Alnh8fTsvpdPcwXS5F5i/Ql62H7XADG0QwGrY0LmDGpiisMtfhYWbwFT6GuUioJsMzM8IzxvBhY7S+37bNsxU2ITYnG2jd90F993Ud60u7Xv15i33TMQpSGEhogjXqBW5u+xhbWnPbbDxZ26xvGH3fx95t33d1pJONvT0/bZ8/ltPCpaYuqOfUS6BQNICCGJgiM0aJcMIEqqDCyhxw3VkGC0khgoOGmg3PiD0j4FyLvPkwd4gs5dMvL+tL6x2VcJ5RJnjAArVwLidaJkwluUYM66vxXqxHifQMwMIjvbv33ve9bet+vW7IrTFTVg+EYW+2d2ur3W7t5aU/Xe36HNtGNgTgQBAoIJoW7uZ9t/EcPtxH7Kv162gv0bZ23fe99519SMQ0xm17flpv1/NyFpQw6FwJSkHEC3ENKIg8kB4eSVSYJTwyNaBENWlQHRzIBIXW05wbet9b23qHeSnT6asPp/luupzK0+eXl+drAZYzL8tMxIVP06wyT7VKWQTBMWDt5nLtZaiPkUWD0t3dw0YffW37uu63tZM3UAEQibC0MfY9X27b9Xl7ue7X276ubo1zAHB67fSi7iPcwyPd3Qeim42xvYzrtfdtv67blikV9Y7L2VuMZn2PaXS+fUEYzSpC0DnlPnRJMEZHwDLgmRzDCbFnBgwjEhHZPdo+DMe6GAkP8pEjyCkXldNpuoe8OU8ffy4fP2Zr+zLR6U6m+UHLzFoYyhNNSjTRCNERtt9Yrl7uUmwUgXt6mJm1bWzbvo+2uxkRPNMzMwzDfDS0Znu3W+uttWEtPcAJDwIl4IAiPcI9LALprW/P4/bZbrfRbtt1uw1voyTY9azT0m83qaWcltZaeovtSoX5koSDIAhAQJzm6Vt4et/HGOSRGUmdqHjw8DK8mI/Ws/fcWu/b2EZPF65l9KsWWpa70/muTJhKXG9bcpNMoaeql1IuVMEsLKnCypHWER9d3/T5QaPqyEAP271vbd/3fd/bvrZhFpkZQXAPDxtjjNzb6O6AEyU4Wb1weInXOTOhPm5wzxg+mg3LjARHYozW9i0NpU6YLvX8huob2UfRk2iJMaBCotm7ty2EE8y+UyhY3a7emkX6fuu3q/cRkEwGp3NiuKv0td1ua99t+OhmNjqoIBjRkT1jyxxzyYf3S61mttu22R49d2Cb9E7qmctMREjHaN1eUn5mmZQytCQ87DbG1vveWtu2tu9772mOdMow83QbEditNTPLTAGJMCcly0HMEBKpoe8St3CknMN6RCIdEJJFF8vNINPl69+c77/SeublZHPp5xKWMs96uZO7d1ln9wLT1EokNsbYhm3bWK/tdlvX62hpHhmCAoVACTwzn8t0Vln3ffP96iHdPNZNS99aLotdLudlLud5fqb1+pluGcSgMPfeW+MyTxMxa1hGXmcmGqfYz13IeQaiDetttza2Zm2PrYWPiFAnTpSgcKYgCy/BRMUpJoV4VFAikw54H9A6/zOLW9Zb9jXLx5QToGY/Yb8y83Se6/xw980/Xx6/FakRlALSwuFSZl7e8elb0keWe9GTC4dF+M2zbpbbbV+f87p5W62NYZbMxFxK4dPsqsyeFlvv1o+jZZ5hGWzoO2XVZMzMnJAsSqICAxA2Ujq8hzmr6BSFIuNq69/ZfsX2nsqdoTbL63Vd1+vW2j7GGN6dQIEUKshU6pzhQeriHlNWp0JixwKXlACDEkqnNxpn2ICu0DNo6Ql+uSKdWUudp8cPy/1XdXoAKqg4ShjFPug0yfSI8+9FH5Puk5gAz9EJDbanX9vty75dn+u+x2jebBOo6qhF9jWlJGVPcwCZTKzM7h4uQUnsaFtE7Mzw7gCYNQwpebC1LCSVpdZS5yKwfUdiGPkVyZtl2UZeb/355da2m7sZw4kMyDSA2NM5PeGFmWZRk8zMkol0BIIyCQQnJZ05+CDJQBQJ9qvjT956Ovj+VC5veLlPVs50FAR7YG++0LzUb6V+m/UeAbMMjwh3w3Aermurt63eRt2btZ3GTpGdKAk611EmqkIsSkjAIpBdjLjYQaNkZuOREoesICjhZAghKlqmMs1al6KLaNGSQkDn3Knb7mHdad2wrm27jd5uTlaqJwkRxXE6Kg5gdQISjGQA4Qc94mkRB+OaqQcFQqwiGTExnwJlePY9MjNdWeeMEkfLYhqOrVmPaSpfyel3Mt+Di9tI2gMR0dzTA92pJffB6xpt973RGNz2jpGAU+3npSxzqdWFI2xEDFgAaGlcQ1UVJJIULAwWASQTCadIViEUAHms7ERlkujR7XM2NUM4snMO57CpbCOM6pjYDxWHO/EhGCAQA2ByBsCJTGRmOCEi4XCo0MGjZRIRCUi8b30fWyepEkHmyIwEIojc9vW2Pm98+pqX35TzV6wTPNwH5RajtXVrfe+228iIuvV4WfvY963bGGRN8iB0evbd1slPcyozicPDO+LYMKXpZGUqhQSKwlRTWNjNIoIi3GBu4kPSkK/AWOIa+5fxskUQkjTKGYCOsw6/NJp2lJ01iEAJohQkH+QGwBAKIcoIZKY7UfgBsCkRkMnJlgZCAOvzy+frum52V6aEZBKRIksE9fWp31560Pn8rp4fpUw5Er6nrd6e237bb21rL9uLrddt3/egQnrybBmZhqQ6QNZ6OJhcBG1BUWdGEmjAElXBgWFuAVNR1+QMJWYCTYAJwbMmTZF1JMwhEWXSsizl/sbri/cdmaesJ8GbUzbJqLuync6YCnQCBuhXlRBzSkLEiAxADByILPx4wqBuIFAG4aiZ98/PLz/9/OX2qU+nN1nPXM6ZNZx9bPttfbn1LytJhw3ztme659X6Sx+bj+FuCN37bd1vRPjqqw9ff/1hu325Pv30/OXTbd1eXsrKtQ2P7pH9yxUsmAWlgggZ2BoikYnafVnGfKIsdxlOtguTFIGkAAeHAu/RWkhBPWudpvmipbbt2vdRsM9nuVT1QixRCBfBecFUgAl08M8KZogcog4AoPGrzGMAQCqUBemICMsctm/b89Mvn5++7Lc1yqe2fCiP5Z757Db69vnjx58+PrWnK/jTy8PLy6XfAHh/advPt9vNXXrE+nzz7aq260nnqc5F6e7uZRmXE3++7qd7W2/jy+dtL3sbFM221Z8cKlgWTAWVIHqw49R2W1ePaEyhknXi84xTDVpXtKu8fWTV5gqI6iCZpMh8XsZeYx8igPmA51AW5hpikH40HTChGpghBNF/aEaAggaQSH4tmfZ2w4iBMca+7y/7y8fr5y+3z9vz1VTbbctMTSLYtl8/f/756eMvT5+fbjSvb9btfuvA7eXjD59/+vPn5+cxRuv49MvnsWV6iKYI1UJ3d4tn2/aV3NityLg7aT3dRzCsr7f1+ctteI4GCgxGCaiCkABHUFsjzJGoc+wnbAueE9v+cXv5cn54WM6XiImJ55mFuUzn6bJhNCarhUVnJoiwlgHKzXp7AhxMqIyJUU7IHUV/LRAQCrZfVU6A/r/+3b979/WHN+/fbesna7f1do2EzGXyqdYKLpEklD19jGbNwKrz/XJ6q7J439b15eMvn3764enTdb1+2Z6/vPzl75+fr9fe7c0Fp0XPEz08nriQZyByBDyYZJrrTCiZg1Ld47bumTEAcgSwD6KEpQ8HDESYFTojWXfL5t5+wcuzvds+3b21U7/ve1zuaD5PcrpfGALN7WNhr4svy0VUiTRB1ryjCe3kNrw7MA2I6FBJUgCRdCgiKB3IlND/6d/+z//63/yb/8u/+lfdVx/dHOf7h7f2eHctd5fl7u6s00xa2JkTUu1yWcry+PDVO5m0tet6vb48908v/vRl//nnT9///dPf/v3tZUMDFLiXfrnDvLTjw7k/4eGynO5O0yxzWbTMY4x+Shl73oZFEqUkImGBGNkd6WDGXCEKBDwDQe48PMZAG/m4f7l/sy3n0xjj3O/m04lz0noZYyfaiUw5SlGahHy2mmaLcnACYbSZBbVWYtXIyU0O5c0BiSAj2fXv/7f/x2/enH//9Ye6FCkicj7f3cPe5Tw9PD68ef/hfHlfyuS9CMtSptOpiqtqZI7e+7puz7f9+Wn/8vn5l788//kvtx/X17PqwM+G2xP6ZwShAN88oMzlod5N5/M8z6w6Iix5GK0txsijOxxDjwciQYAStEC0JKW7WQABZiSjd8RnjN7v7vsYY5jduZVSAY4ow7a+9+h5vvNZJpnKIpJ64mMbHe6T+Rru1G1q62n1amNCChzuyEzyULnuLz/9lx///q/ef/vd/dvHeb4/ne5oXDH07VffPLz97enyDlxFnqhQnatq6f7awGxgXdvttl5vt+fPz5+eXj7dXqszE/aDpQIMYOCu4u3by+Pju/vHN8tlLqW494QlLJGWtHWMgSoQAQvAEAYxSoEcrzg9kIZM1AqtTOBh9nJDJiJvB0g6zUZG3ofdGlluW0/v0PlcRfTME4gSzCFFoIO6cxJTRsZKnSWfOTcbpmGcFhqGPvZtfRZ8VWstp8rtIlZ7s9Pj1/Pdo9bpVXAI9mQimpa8e3Oqi+ytreu2bbu10be9NYxfNZGWmAgAGsCEr8743bcP3/3263fv7u4fT6UWAsZQIVTVOte7hxOw9t1e6T4GJRgQBgo4kdHdUn6t2jRxLYW0wiyjN3dtjHUM3+tMrEFbZHfugEMForvwl4IoEjwJ0exLISGehEpqQVlGvWa72ZrzviM39y42oH/9b/7p73/727/6Znp8PJ/uqipDfU2zxHQ6l3phmYicdVaZE6pLTkrLaQoZfbv11rz3kW03E8FbxdXREwdzdyI8nPDuvrx9+/D+8e7d+8v9ZZkWZYKN7MORNNX67s3j3f3D9nK7Pn/pzQKD/ODn0ykXilKyVloqEYAC5alMUrnKrNlzjJ791pxzQ4s+O9WZJRNZkT120Cs+uFHvYa2el1KDa2EhmlSqxn2ZfVye1/3JF5TnUWlD/9J0V/0f/sf/0banPlY5P4oa+qd+/V785TKfTpc38/le9GS+ayllmk4X6Vv2bc31R9fZrtH3tdkA/O4yzbV8+w0CgEsyL6fT+/f37x4feGKkIKlMpRRVQaaZmY3OFFMVJq3LSb75kLDe+r5tY9+23nI0d1capXJVgFGVS5nqdGFmz+h7ixzpkRGGpCGnU9Eyq0qSxnCgH0NgONYNpJ5Y0xumXeaC+SS8lMopu3sYTBQqq/DL7LxO0n/Z9OG7v47RRnsxx/7yX7aP/6m//FjK3eWrf3q+f190JgpoEeUiclqWq5Tr85e//N33ZTkbz3XRd+/f3t+dKECqRNx7L5LzPJ8e7u7u3p6WEwg+rPXe29ZH23Y7XozRPJoItJRl8WWZVM9I9ex972OsfWzeOjJViUUOpbcQgTkz0wwFaRTDg2q6eYzx8eO6XKdpmgSIRg0DiIG4wAMRsDFsHeWhL+ulPFApXhYFJuaTvrnwHEQr5a3UtZx5eyTV5Z6mIVPZbi9IBeb54bvLm98/fPvP5tM7naeEZbBqkWlaHt5ML896ay5qg1jt/bu7+4fSOgHOpMTU28gwETrfPV4e7uflNHofcWvWPYaFRfS+dxsGQEVKlVoXEclMoNWZSHRaZOxkXkcPIgcAjhygTKR7wi1AJFySCdR6cI4UpCz8ut5ScBaSToYgeMfO8ETr6DMuCJyuIYb5HKlyR6xERRgkXcrIyBjRhEOJio/d985M5ze/Oz18YEY93c2nN6kncFKoipmep9MbG989flfK+fH28uy9B2Galzs9G01hZBa9W2Bt6z72Ad7NXOSaac126zc3MuvDmjuIWKUkVJVVqZSqqof4llOVwVPSADD8qA8JJiIAAaUcZtkcCPOupsXEUnGI0IlURYuSwAXRbhbogObr5euOsaFxCO9QrgPYCRB2Mi7BnKcFIwWYxdTDLQeA6fSmTktKTekA4thMYARCpOo0LW+Si1zenR6u89OP2+15+K4yg8WgyKWNwLra8G1rWzfz2+0K4hQmsIV7UI7d9m149AwGkZtHgEmYU5RUlRgESgGjsjuyg4yZGUokRAdvQiwFiN03YXaRMp+kSLQWNEa4mhMpSEByDFM0UBLiEEVLADAAiJmv4oNhmYPZQqdEpbLog2LqOjeFQ/VcljuvbwMVQKIleqAnEYOF+4EC0oRJFxruuqZeyvIy+pow228YnWcKLsUnmUYpU+FuPjIM6lWUwUGSlpkHflpQEMaqqUJSQAQmZWUGQ8HHrGKH/yYBUElK0GGgIGKhckLYlO7uFmgRHmLp3pPELEBEAY8Y8IQbuCIdJsCAbRgBABHIbJOnerKCZpCAalUupMpcVKcp9DFQXweQf/xRSgSFAJkCEMlMGcmDWabTVOu74av168a/+O1j+CCqRWeRtU4lz3VriUgIiypLYXBiIAHyiHSnrMnEpaBWFa7ElAffysyQFAplLWrslKAUVmJSMNKIKAqYzwIhD5gjMcxJIJlAKHEBH+vmgLdUdACJ4YgdQzEDhGPCxDj3BVqWTUh4YqoEmjgpK2nqw39TmkQHOgMBSAa9kqTwYJAT1zohVJEL26z1HMndervdIFFqmaYpzYDhmREgzkknvPbgY0cfnql6dJYQhVaWQ7dscAZGagGT1FnBS98k4MSpUuhweykh4QlSqyVtmVpvXEqJwsSlVp3LvEyqJTPyVHg8cdt8wAJjAI60VxXSK/SsEFmFWVUJSk1SOyEYof/tuYFzdrIGpMiUBAjDPSlFyE1VBjIT2S2GJZyI59Q3WjTJM+Xu4W6ey+1WRK5jtDQnQnhmdiQRQYpKJhEDyAyRUIWwMlMgwkdkiQiSZKJaKwGRSAoGA4iRmU4kDDenCIKrsJY6syTBtKiUAngkEQUQDMYMBWzAGNERBgtYoHWsJ9x1ZAflNUHi4NqY5kTNY8f///0hp3MWBZIhAdfYesAdFKNbHyN6b615Rk/3TG/DI8Wl9JGcJkJlOV24ArKv2753G3vESKd0B4GJQU6UCc4wR8QISdNpYpFkDYSZuSdzIcIhwwRxMnEKc1jjyOFpYGHySPMwNzcLISNKqPB0KswHwGLDctspnAgKQGADtqKt2CrqilvD1rDteHh4qXdUzsrCnhbxf1AgFswJTjSga/qwCEtgdOfhOSwtAmkBS0QKlSiTztFXjG5ti0IMjWQpylpFnKiGSbBxCZABCrZ0D4eLhEcONwpvxsml1sIFBUScnAmwwC3DkgTJkQkWRCBGJg33TBAoQIPRVURVhAlmzkTZEA3RAm4daFD7hx6LMATDCNhwTSQQBRd+qa/WTE3U//8CxWsjAiT7GGY9Ej6cuzU3D+8+mo8ONgQnzCICnQXEPILQBuARsIjMVNXegwX8qtX1zJEESkZaWLKnwzMiQ1ICHVRTvABEBcyFWBnpbBGREcQAMUNYbLRI7JldyUOEaC6CMmktpdQCUKRkp55XNFCACShIgAIK8AXLjGkBVVSgFhDgI/u6cnXg7P9HPSgTluiMbgPd3DwsRoa7pftw6+YD2SkAHDKAYR5Jk2hoMRvuY5jlMeERkYpGAskMsHBm9eiue8TwYR5bWoKMyckI3IML0aI6AYhIEYJCQjMGgpAJTjghGDwQADlAwiT/4K+kEOqeQjGIOgU6gQ0IkIEORxihKKYFyxllhgLMSEEzUHj2ALUA/psCRWDnHAQn72PYMLe+dwuPAY9u3Xuz2DI9wMdJS9Iq3hMiRCLZORzhjujphMMrBTpOJdFxlCQgDnj6GBFjF2RwqoKzRhKbBTN75XIAoIe7VPgYBTJIkggIxHFRw+j1Q+AIjnAzBiLCzLpbiwbf4QAGCgMVtYALhFEKXrdphw84IYmTEhEB+/8ukAdWGQ0YABKZ3q2t67ZbeGRQWDfz0cxakjFzci1aEmTIzDUO0wwdEvXAQc2mCx16EjuMawwCkiIlyJgP6s4yNcOCRBwh4MwcCUImMTIJSIQl5EDO6PC2k7xOjgDxq3mDiRNiIQQLywwzMzckXv20TtCEAfWgUgMewPH/YbIVTxLmAOU/Fiiwil0pI4lB4DT4PvrWxoZIMCUgIOcIBDIptaoCGmmZGdbd3SIJDGFyimTPkRFGiYykgUgmShIE0gzZSVKZcyowBzkOQ2lQwjIlsrHFcXbCMjIJHcREBP91eqNSuAYbtJOrKLNWLowQdxo+IuMgCQ9xBwAKeGAMjAYT9AIESJGMNCQQhoJOU/5jgRy7ZMfRKKhQrLlvHiNjUCQXYi4AGQXFBjizkFBEZjYbNrxnemZQxiECz2RCIhOI9Ag/vCcjqCaQMczX3ta0hjzc8IWiJifBAw5LgjLX5BxjIH6VpQQAA2lGZI5M4LjUQ8mRFFyLFhERFhgYg72RDZgBHeZAgCaQIBPm2AIywBUSr/dTBGKgFXAbREMBJJq2K2z99bSO0dY+rmM0pJUCLcsAw4f7yExmYSgzAAhxkBINABERwzMDADMxQQjDIy0PaxZRwhroxsTWe4zR9o2iMydPlakQIsOPvRTmQYBYcH21/sRIJhLNfJUSHNt5pgEQEZJMRKQCAiKiwZSsEY5miAFKCIMUoqAKUkRgb0jFUlEmCIM6jJAGB0DQ9ac/7teP7ctHcIgoCWXi1tq+byRdphPVi/luEb0377fIHdazTB6SkEwfo7dmY6xjbx49wnyEu7uneaTlYQe1kExQDiCYIzxHZPRmvgHpbnOZuDAcIYG0JJXYIGDJZIrMQSwAjUx19+LOY8BGmrE7zDYKJxZYyuRSZyZhRvp6TC5tRzhEQQwpqIlwOCOO6wbwHYMQgZHwAAayQP/Dv/u327Zu16cMN/M+gqdpT+0OUizLvDxe7h/e8qwZtj3/fLi7SlmoTkKSLL2P1luM4WHWRyCPKAgDMjmA9NzWfVubak6LlCpCMtLdDpC/qgDJI5x6cIIJFJS/yl1YkEAmmZATUyYFB1kEMjwsbNgYAx5pg8lMsoRLBDDCB8Ca4Q4BPDA6vgSGo59xqqgAAFkRQBHwq9YFmQADA/of/9f/AHhGD/dhvo22bfayj9sOTLKcTqf7+7fv35wvkwqP9jTP9Xw+e00dvRN1g8dhAO2BIx0GmTnykGnl3vynH375/P2PfX35+nfvHvhe9cSFRISVpcwcnQUCQR7LNsLscIvx4Xz2JAHA6dIj0z3ZAbhTRlr0tOZ9jBiIQGSpyWZbtMKDfJDngSKxAI7dII7iKAFn4FeesiREUCYcQqUIHBSTPn3/M5ODHcy3bb9uX6637eXW9+5eJkwXrqdp+bu56Gmihzfz+/dvMsG0EjMgngAXFjBeATsm8gQiM23b9o8/P//Hf//H9dPPM493394RpWiKggczS601AhJJnADSC+AHTUZHuY8+o6/C7mQye42i8BiZOZr34b2bZVA6U/hAT1d2ElPsDIdjGCzRDMPhwBzIhAMOUMIYwyEGZni+Vmc4AtDnj79Y7H20gbxt48t2G613gwNctpjMZF3/3JTjfpF/8k8/EIUNk6KiVViTSUpVEWFoKaxFgCQkkSU+f3z5w3/++7/9w59L+jdfk1IqgYmAZAkpsBAiIRqv/RbIUFAc6pQISkoi8EAqeRAYIuRJgcygcPYYbplpB6RPKBE+xhDKqO5IEJghFbEDCgqEYwCdoAMsQIINzOACDRBBFBXIgA3o3/70w7q129pGYBtY26+Gdca8mM63UL9eb4Vp4mqr7bcVieV0KoVFPUU0klVRSISQcshoYfHyvP7808cf//Jj3/xyh/OpSiFSAnF6HqZKksTIDArJA94CERElEMGgCEeyMRieliDW1yYR7GHuaRaWnThLJl7zeBIJDy9BpAyAJ2iCGgoBQN/RNxRA7qECZ9iAEnwCABGoQhToyID+xz8/94HVAKAnxgG+AADmgdM6Sh19YL7n5VQmFQmgpymIe5JmBiFTU7QA4d4CaLt/ebr+4Q9//uH7j4B98w5v78vbD8t8qlKZlRKgBjrScxKAp0USIwWETI6IoIBbRpizsBIhk0EWoKQpIkf31pvZ7j40k4QIgRhCkAMtRjIzlYIcBKggFRmvQ1U3qMOASQDGYXofgAD8ihYiBfqHGyZAj1kTsHzFtAHsiTZwMgCYOc6TZqYNSzI1M4dF1rkmYWRgeC2VGW756fPzH//443/+mz/267aof/hwevPh8vbD/XKaC1cCwtPDMy3TKDyCIjMpko7AoddREhEZHkFgQEiKe2okItqwMLMxdtubo5FwkVJVA8GegkHRkxgQFK6QGB4zVgIZsiISpHBH7xCFKkJAhgjYIe0AmKkW1gQq4SLYHYb/Jm8IW6ICF8HlXJZlycIdUty2tZ+ncrrMJMLCxMw8iSgTvbw8/9f//Jf/5f/9N59//HyZ8PCV3r978/Du8fTwMC93whrDx9i9b927Ncs0dz/0OAAY5I6MCIDpKIwaEGHiAnKy3JuZZR/m5pYdPkYwExmLIAUDaWZOEqVCVQGUyQ+FwZrYOvpADKSgKCKQx/KpEEAIkuAJnFyC9V+/0WWuIPrTT7drQIEHhgMtXv3rJ8b7r/H4/rI8nPWAdETmh1qmSodDzFOEiCWY17X9+//17/7T3/ztl8+fLyc8zKhitcpynufzSUpxN9s3623dt22/jd6RjowjB0IEwoUFQZwhPSODnDpAgPRhQhrpeUzt0c03+KA0SuUc7E4ewV1iL1MKwbuTSrKiwFrrgW4wQQx4oA90QdnxEogAGMxggc5gFfWJmPXu5Htf1w2s0AEAswKBhbEHhHE+4+3dstRKI6LSVGmpE5IiIjNFIaqsWliePj//6e9++i9/+OPnXz5yQoDLBe+/ub//8Ha+nFU1MEaMPUbf9n3f+r6N3okcSoUoATOkgKUoigtxSEhkDw/3cAcIfkThRDa3UAqUkijI4R5ILwzOAUohiECEMpNYuehUzCcfO2bCqqCOMHjADIUxBH2gNgwGKziT2AOk//L/+n/74fvv/+5Pf4rbS27ed4yEKqpgJiwnXO7lPCkDFJ1TyckJ4uk+2k5ByaJu+en65T/9x7/9D//+b54+fUbiNGFinB8ub756//D4oFW9D8dwjwgYMgKeQAboEOVW0SSShFBoFmFSiEQEqYtxODI84ccg6h7mozKLKoTgBGsUxsJFighUnUUQAo+0yBRVKcXrhH0gd2wdMyMdmegONFCiAKrggcoBDg7oP/kX/6qc30TSLx9/Oa3X27avrQtQCk8SpUAiWmtEgOdMd6mR3UIhDiis5dPty/V5//4vP/3xj9//9P2TFEyKwnjzjt6/v3v88KhF022MAcCD0kyOQCZhKgwKYVZm1QqWPOZPEKvA+MDXHAwxzhJH8JBFhIHcubJS5WPnMR2umlRYqKp4URbhrbElRSBBya/ITvrrNOjH1AakIPkfU+kAkEdK18vj+7fbbawv02m2McYYP/3y+Xq7tbYTR3h2hw1YxuQe2XvXMuW06zwRi+/b88+fv/z049PTl+dta1JeT995xrffvvnw9cOyVIvNPcKCSDIpD5mdcBHxqMDhTxFSYV1AZMkMhGdGxAjPsOHDOyDufYwe+whKYeaahZRByly1lhOLgJgAkJsoqaAEWfMxzM2sYRzXfL6Oa8fEzAwwkuACJ7gjB6I4kDqfpjcf3irZ268e2MXW8V/7H/729sNfmnW35HCAFSfXqciXW2NuLLxovbtr8yQfP336+fPT55cVynJE9hAe7/DtN/O33709353chw1PcLIEDnAwiJwYrEpIoiBJRoCZiaBFswzL8DZ62Bjp2VprfVB6pruNESbEJKUIKUEohUyV6zSpiijCydsOuGWqGvqwYdZ93dB3jIAF6Ii+8FcULRNjwBtcMQTMEENKKIudH06n07eED/b9+vzn//rD5+v0vPLWP+/9RhgAJx4nvqs0fA9zQpxU/P7y9rJYXivLw+Wyh429E/JxwW++mX73+w93dyeA2u6BFGalw+413AY8CCEg6MF8ReQrwv1KdibCyCx772OE2ZHl2fIIQ0rQRKUei1oAUApSiBRVFQYhW6UjPo+gE9DFhwOAH5RhYhhYQApVqIACUhBxsPXIhDsiWN06ZbBnueX1//O//eX/+b88PT+n2V3QDnRCHvxQxOTjyFIhh5hhtDFc2QjpkaMNorib8Pvf4rvf3D+8PQ/U8LQDDRLPZM8kS1gkjDITAoEQEWWCMnlYIiNzWI8+em9ttGbWACE6+nFG5LBStNZalZJBRJmZlWSukyzKnDFcSQfSBpTYi6Q6kZEfeRMIxzAwY4xXYdVIjAAqSsM8IRhSIZTa11upUkKmP/yt/92fto8/ichEfCZqRWPYCsy13JdyFtZID1BmtTiNtkRYIG1Y94KcCh4veP/2dH64kJzN0kZ4OHEi3QF4eBKYmCpnBByHxy9ZGBByRjjcfFhvu6/rtt82SDA7p4C5aKpoYiQi3LkU1aKFpzpxZZNS5SxCoEiUzO7ezSNiIIfh1/i4w6jCrxiuG4xRDi5IXwWNesxEnJpHqo4NPH3J1khVmAtYhe4sBwIjPszTmbUgSx89MpAqkEOnRGS39vyS84TTjLrU5fFrmR8alUOYI0KAEcCE0FQgk5k0YQWcTEmIgEmmAwaQuu1j3fdtv365XdetkFARVVciPpV5lszg9PAY3YW9kpZyKkW0FCYVKaTEpUSubD6MEFBCAUZBMXR6JTO6YzjMUA/cjOC/bhtmEEEkaYT5lvplx/Mt95FETCTABIoiJb0A9yyXA44ozBYIJvaKrEHnWc7n6V53JT4tZT7PrG9GnqKzgiHkKJk9MwOdeaJI5yRWRE3xI5clwoHi5j4yENvq67bebt6GmUdycoscjAnkRZJIKT3IXKrQsZZQMM/C9aAERRLGVbVzNbvu+943g7+GqeqM6YDEgIxXKhkKBkBIw+hQBRycqbfn9UKiTy/x8aNvm2cGMWdqZJqXyBm8sMyAmZcjPi9JFQqIJQHLoneyuOflcXn7/m3qybPGoekEJNhSwT2DMiMJdMRKMWcIUSINCB85ho/h4W3f1nXrvXtE8K9Bq56uoZ59jASZUIpKKVrnZVoWVWauzJKRPgwhgBxApEoCGIZ1w35DM3SHOYKQR0RRIpkSORL0D3zZa2Iv9Pr8aZKZrjdfn31dIy1cggEBDEp8IqrMiMgIAMJUmYrRXCGIlr6NzMz7i7z98ObN+6+TazgTlIg44Qyh8JRkzzhIQwYJURLI4JklPMbofR1b382ybVu7HZ5FIwIbASkc7BZNnF04uWghKbXUZSnLWZkg4hF+5G46BcFj4DVkE0f67UEk87H0VWQD4ljR0wJKvxKHemB+CIju28sQG+tKfTN0dx9kgHKygDQpmV/JMwJHaqIAZ6ZZKDI/r71nzJN++937d+/f63zubRDFKyFDRKAkUacgJGd6JiWSKCk9PdOG9e7btm7bbd+7D+97b3v3I69uIChEwASVECYimWautZQyaZ24FCEOSgHS0yO9W0eCInpY823fx+gJkKIcKOrx8W+vMWkZMENlBAOJfxC8JCRQdKy3jfNm7SRIzgiHGYgTcmSsgYFwomCOaiZMNeksTEK72L6P+XH68O3br3/zWz1VM3N3EQF1OKcIAYCgJI+SHJ7IGBjkwJEX2ntf13V7ft7Xtm7WvFsE7NfWcMj4AJlQJl4utS7LaVItpWiR6QSo0RHo6pzpZj4sMUBhW+zrer2u/ep9wALmeB2NAGMc6W8AYMh6sJIgAx0XvxOY1NrtJlGI7matTMxIWDfycHMDsgTcjCQVCeFKtCAviqvyWvzNm/PD19+++fob0ot3T++SSRFJckRfH8HnmZJqSKGIQ95jZBhIHz66ta3b6H30PvYWB2SlCRIIQYBaMVc+353P93eny5mpEBEk8pCyBh9e956ZyQ7Yjgjrbdu3W2++N7QdIw7pBCJh+ToNpcENrigJLTjY0DgsM7Ak0kJpvm0JLTIvsjzjC6WLDRDENcFOqU7GTDIBM2EiF6X67vLw1f13332r8yNQRCJ9BIcNy8x0P+B1KMAlbUSCGaycA516WgRGsIFTSwgxNJlCGPG6reLgZk8LHt+W+7tzWU7z5aEuZzoUU2wIBSiPhxiI4WFjdHInN7Q+9q2PDWGvWS6pCKAQWDECfnhWFTKBBUWgBTK98vQFSHZ9++5ta1u+rPZ4Nz1/eqP4BKwWI+AOIZSSJcCUhXIhOSNOReTrh+mff3v3u++QnFEBJvUwxABiUEQmv9rygwkDZBkgTyckpYBdQzzhqkohMk3UbYwy1JMBCJgwVywzzg/z/d1pWS5cqpTCxCJCRMQlE3lIYGIcWF5Ck9KBbbR9tb5bD3iCCMsCIozA2F6h9zKB5VAgAwEbmBQwYEICHQBET49f8foynD4Zlak8Vjy8YByzKoMDlEEUVWQBnStdDPNJ7Pfv4/EBVAhCXBLI4EwHQpkDeUjByEpyIAkUDHNSPijAZBB5SreRAwCZcVjJZJGYknRGPWmdZCkyzVVVLSFhsd/SrC5nqUDS8Y9YGAqkuWeGu1vrY92jdSQEqBMAeBztEKiHeRG2g+TVoTZXiKAUQBGJHIAqgpSIWCcsp4993H3zsNjzN307EPwXAjoYebK4U16CFvP5XumfvOGvP/gy9+a1yK9XHDGJSjVyNgScU5KDBAglImK2OEINMg9iJ5GZex9933rbKXrVJCUREtXlfFqWqRQBhYMwIsJI9MBni1WuhfkImwg/LnChUurow2xsL+u+jjTokWZHUMYI/JpIizTgCNdxzA4nCIEVqqiMFFgik3RsI8KTyKb6/PZxp5jKD7/54fpww6eKIZAdDxjnhC5FHpb8q2/xm/f+5vFQ4wkRqxIYSGJmsFBxSjJCEgkoJMGAJhP5keCQnuE9h49t62Pf974O2yMGIeosIsxCokTCDI7EISlwh0YmYNQBEkBKeQW3BJnszunDzGy0I6LjUM0oQRIpUAYIA/ABA8CvlAYER9wLBAdPnQzOdGLd1icQgVPrqWeuQYV1erydtrX2HlCq0zQq14rHBQ8P/v6N3z1EcrZrRHdPZiMukQfopWzJECd3ORJ7kfE6uWXCzS09WnRv27au6zraMOvuRmx0/K0HBxXhwxoHCKpZOD2CmA9hGYBIF5RDHcVB4fDho2/7em3X2xgdgBaooDCISUpSsE6QFnuid4wBYjBQK8qEeYIqREAMDyQiPTTTWCuzmo9gHrXcLg/lcqnQhVVqVdVIdtGcOKeHrDWYYgx/dQcfgqD8B8KTUFOCyMks4SA7ho0wshEWve2W6du+bi/Xva/Zj680sUPwlh2qyZysNghKzEXp+KoHJoiQVkiFHuI9ZlZKSUIcdNJ6XV+u27aPBmKoohQuRYgokaTJGShwh3bsA+G/JsYqtEBmCL+O3ZniSL083CHVzffssDjMs7v7RrzP51LPNBVmeU26iCCzVwAGnMn71iIwT2CZAMoodHzFCY1MREQiScg9R4vhffRt328wbOvzfn0ON3c7stLdJYKccrgXykxaKslyPEFJUJFULaqaosIsykdKiYiAKMg9Rttab68OKqXjz2YWRvKrZilCCcqoFdUOh/Wv2udA+usXIxwqSA/W0+Vd77vHVohNGAaHxWg2btG2VhZRziKUJQtPXHkqVOZ5PgvbML8+vczzjof707nSIQ49RtNIJ4twNwLCMPrex7iNfe3jJXr4uAHt+BYCAEABI50ynQmdLcKJUuukBz2qlRmqpSijFBZCagaZhztYkDiShhLkGWBGKWBGVQLLseYc52WAKFIYU0EpWE4ohYrmkasSduzGaZGZqfPpDfHuQTUpkWYj+m772t2mNPI2VLBRKmGXkVO9XOZFMOH65fnTDz/2sSfdzaNnGHEB8EpheT/C29w8rQ/sY+sjO2Jnd/cuHDTXROhIM3JniUyK7gmAjIkjS3NPZYaUFNQyqxYulbUgGQRIMfPWu9tO6T56hvnIA4oyYBFGnUoplEoOadx4xOaHiFkFdcJpltOp1tmpDMk8JmxP8aie+N8BdZlAcc+rhx0AAAAASUVORK5CYII=\n",
            "text/plain": [
              "<PIL.Image.Image image mode=RGB size=96x96 at 0x7FD80CA1BF28>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zeeFx9xTGY81",
        "colab_type": "text"
      },
      "source": [
        "### SimCLR neural network for embeddings\n",
        "Here I define the `ImageEmbedding` neural network which bases on `EfficientNet-b0` architecture. I swap out last layer of pre-trained `EfficientNet` with identity function and add projection for image embeddings on top of it (following the SimCLR paper) with `Linear-ReLU-Linear` layers."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YDa407DMGY82",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from efficientnet_pytorch import EfficientNet"
      ],
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bWkyjXpJGY84",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class ImageEmbedding(nn.Module):       \n",
        "    class Identity(nn.Module):\n",
        "        def __init__(self): super().__init__()\n",
        "\n",
        "        def forward(self, x):\n",
        "            return x\n",
        "    \n",
        "        \n",
        "    def __init__(self, embedding_size=1024):\n",
        "        super().__init__()\n",
        "        \n",
        "        base_model = EfficientNet.from_pretrained(\"efficientnet-b0\")\n",
        "        internal_embedding_size = base_model._fc.in_features\n",
        "        base_model._fc = ImageEmbedding.Identity()\n",
        "        \n",
        "        self.embedding = base_model\n",
        "        \n",
        "        self.projection = nn.Sequential(\n",
        "            nn.Linear(in_features=internal_embedding_size, out_features=embedding_size),\n",
        "            nn.ReLU(),\n",
        "            nn.Linear(in_features=embedding_size, out_features=embedding_size)\n",
        "        )\n",
        "\n",
        "    def calculate_embedding(self, image):\n",
        "        return self.embedding(image)\n",
        "\n",
        "    def forward(self, X):\n",
        "        image = X\n",
        "        embedding = self.calculate_embedding(image)\n",
        "        projection = self.projection(embedding)\n",
        "        return embedding, projection\n"
      ],
      "execution_count": 34,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iX4yXVQkGY85",
        "colab_type": "text"
      },
      "source": [
        "Next is the implementation of PyTorch-Lightning-based training module that orchestrates everything together:\n",
        "* hyper-parameters handling\n",
        "* SimCLR ImageEmbedding network\n",
        "* STL10 dataset\n",
        "* optimizer\n",
        "* forward step\n",
        "\n",
        "As the `PretrainingDatasetWrapper` I've implemented returns tuple of: `(Image1, Image2), dummy class`, the forward step for this module is straightforward - it needs to produce two batches of embeddings and calculate contrastive loss function:\n",
        "```python\n",
        "(X, Y), y = batch\n",
        "embX, projectionX = self.forward(X)\n",
        "embY, projectionY = self.forward(Y)\n",
        "loss = self.loss(projectionX, projectionY)\n",
        "```"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "raSuHkb3GY85",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from torch.multiprocessing import cpu_count\n",
        "from torch.optim import RMSprop\n",
        "import pytorch_lightning as pl"
      ],
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cBAkMXXmGY87",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class ImageEmbeddingModule(pl.LightningModule):\n",
        "    def __init__(self, hparams):\n",
        "        hparams = Namespace(**hparams) if isinstance(hparams, dict) else hparams\n",
        "        super().__init__()\n",
        "        self.hparams = hparams\n",
        "        self.model = ImageEmbedding()\n",
        "        self.loss = ContrastiveLoss(hparams.batch_size)\n",
        "    \n",
        "    def total_steps(self):\n",
        "        return len(self.train_dataloader()) // self.hparams.epochs\n",
        "    \n",
        "    def train_dataloader(self):\n",
        "        return DataLoader(PretrainingDatasetWrapper(stl10_unlabeled, \n",
        "                                             debug=getattr(self.hparams, \"debug\", False)),\n",
        "                          batch_size=self.hparams.batch_size, \n",
        "                          num_workers=cpu_count(),\n",
        "                          sampler=SubsetRandomSampler(list(range(hparams.train_size))),\n",
        "                         drop_last=True)\n",
        "    \n",
        "    def val_dataloader(self):\n",
        "        return DataLoader(PretrainingDatasetWrapper(stl10_unlabeled,\n",
        "                                            debug=getattr(self.hparams, \"debug\", False)),\n",
        "                          batch_size=self.hparams.batch_size, \n",
        "                          shuffle=False,\n",
        "                          num_workers=cpu_count(),\n",
        "                          sampler=SequentialSampler(list(range(hparams.train_size + 1, hparams.train_size + hparams.validation_size))),\n",
        "                         drop_last=True)\n",
        "    \n",
        "    def forward(self, X):\n",
        "        return self.model(X)\n",
        "    \n",
        "    def step(self, batch, step_name = \"train\"):\n",
        "        (X, Y), y = batch\n",
        "        embX, projectionX = self.forward(X)\n",
        "        embY, projectionY = self.forward(Y)\n",
        "        loss = self.loss(projectionX, projectionY)\n",
        "        loss_key = f\"{step_name}_loss\"\n",
        "        tensorboard_logs = {loss_key: loss}\n",
        "\n",
        "        return { (\"loss\" if step_name == \"train\" else loss_key): loss, 'log': tensorboard_logs,\n",
        "                        \"progress_bar\": {loss_key: loss}}\n",
        "    \n",
        "    def training_step(self, batch, batch_idx):\n",
        "        return self.step(batch, \"train\")\n",
        "    \n",
        "    def validation_step(self, batch, batch_idx):\n",
        "        return self.step(batch, \"val\")\n",
        "    \n",
        "    def validation_end(self, outputs):\n",
        "        if len(outputs) == 0:\n",
        "            return {\"val_loss\": torch.tensor(0)}\n",
        "        else:\n",
        "            loss = torch.stack([x[\"val_loss\"] for x in outputs]).mean()\n",
        "            return {\"val_loss\": loss, \"log\": {\"val_loss\": loss}}\n",
        "\n",
        "    def configure_optimizers(self):\n",
        "        optimizer = RMSprop(self.model.parameters(), lr=self.hparams.lr)\n",
        "        return [optimizer], []"
      ],
      "execution_count": 36,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "V1XtGqPoGY89",
        "colab_type": "text"
      },
      "source": [
        "Initial hyper-parameters. Batch size of 128 works fine with EfficientNet-B0 on GTX1070. Note that I've limitted training dataset to first 10k images from STL10 for convenience of running this blogpost in the form of Jupyter Notebook / Google Colab.\n",
        "\n",
        "> Important! SimCLR greatly benefits from large batch sizes - it should be set to as high value as possible given the GPU / cluster limits."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sCmxMF5xGY89",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from argparse import Namespace\n",
        "hparams = Namespace(\n",
        "    lr=1e-3,\n",
        "    epochs=50,\n",
        "    batch_size=160,\n",
        "    train_size=10000,\n",
        "    validation_size=1000\n",
        ")"
      ],
      "execution_count": 37,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3tdROxusGY9A",
        "colab_type": "text"
      },
      "source": [
        "#### Finding good initial-learning rate using LRFinder algorithm\n",
        "I use pytorch-lightning's built-in LRFinder algorithm to find initial learning rate."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HTpdKsZDGY9A",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 256,
          "referenced_widgets": [
            "66d6c350dc6b4a57927aef2f4bf3065c",
            "4b718bdfdedd440a9308a585af05eaaf",
            "712bf434e0a541a382cc040a6d2b452b",
            "c1c34ea327024c5cbfcfb147c3b2abe4",
            "2020dfc1254e42f8867d99877903bea5",
            "12d98e263dbd4950b82e76d7c28995bd",
            "f5db69016961498ca526b0f389677491",
            "bcd3bff210174fbbaaf6459ff704eefe"
          ]
        },
        "outputId": "9c98e354-5946-4b8c-ad06-3663e3ff07eb"
      },
      "source": [
        "module = ImageEmbeddingModule(hparams)\n",
        "t = pl.Trainer(gpus=1)\n",
        "lr_finder = t.lr_find(module)"
      ],
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "GPU available: True, used: True\n",
            "TPU available: False, using: 0 TPU cores\n",
            "CUDA_VISIBLE_DEVICES: [0]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Loaded pretrained weights for efficientnet-b0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/pytorch_lightning/utilities/distributed.py:25: RuntimeWarning: You have defined a `val_dataloader()` and have defined a `validation_step()`, you may also want to define `validation_epoch_end()` for accumulating stats.\n",
            "  warnings.warn(*args, **kwargs)\n",
            "\n",
            "  | Name  | Type            | Params\n",
            "------------------------------------------\n",
            "0 | model | ImageEmbedding  | 6 M   \n",
            "1 | loss  | ContrastiveLoss | 0     \n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "66d6c350dc6b4a57927aef2f4bf3065c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Finding best initial lr', style=ProgressStyle(description…"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3O-zc0z2GY9C",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 300
        },
        "outputId": "55028365-1fc0-45db-f75b-1efe948f84c5"
      },
      "source": [
        "lr_finder.plot(show=False, suggest=True)\n",
        "lr_finder.suggestion()"
      ],
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.0005248074602497723"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 40
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEKCAYAAAAIO8L1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3xc5Z3v8c9PGvVuFWNLsmTLFRM3hA3GGNu0wNIhJIRkIbDrUJZN29xks3vTdnOzuwkkNyGBcAkphBICMXGoptiATTEy7rhJbnJTcVGz2kjP/UNjIszYluwZnRnN9/16+eXRKXN+Opbnq+c85zyPOecQERE5WpzXBYiISGRSQIiISFAKCBERCUoBISIiQSkgREQkKAWEiIgE5fO6gFDKy8tzpaWlXpchIhI1VqxYUe+cyw+2blAFRGlpKRUVFV6XISISNcxsx7HW6RKTiIgEpYAQEZGgFBAiIhKUAkJERIJSQIiISFAKCBERCWpQ3eYaDQ60dLBy50G6A6OsJ/riGJ6VTGFOCqmJ+ucQkcihTyTgrap6Jhdlk5YUvtPR1e149N0d/PilTTS2+YNuU5idwg3lxdw4o5iCjOSw1SIi0hcxHxANhzv5h99VMCQtkf+5fhIzy/JC8r6Vtc28vXU/Hf5uOvzdPLtmD+v3NDKzLJe7540hPcmHGbR2drHnUCu7D7XyztYD/OSVzdy3eAvzxhcw7rRMSnNTGZ6dQqIvDl+ckeSLJz8jiZzUBMwsJLWKiARjg2lGufLycncyT1JXbD/A159aw7b6Fj539gj+9dIJH2tNOOdwDuLiPvqhXFXXjL/LMe60jA+3e+SdHfzncxvo8Hd/uN3wrGT+7e9O57JPnHbcD/atdc088s4OFq2vYW9D64eXoo6WGB9HYU4Kc8cVcMXkYUwpzj7pwOjwd5MQbwockRhkZiucc+VB1ykgerR2dHHPok38etk2CrNT+J/rJjFzdB7OOV5Yt4/v//UD/N2OKycP55qphdS3tPPw0m28uaUegAnDMrluWiHvbjvAyx/UMGdcPt+/8gyyUhNIjI8jyRf3sXA5kQ5/N7sOHmZvQxudXd34uxxt/i7qmtqpaWxnS00Tb26pp6Orm6KcFC4YX8CccQWcPSqXuDhobPXT3O7HOUecGd3OsePAYSprmtlS28T2/YfZsb+FmsZ28jOSmFyUzaSiLJraOtlc00xlbTNxcZCblkReeiKnD8vkvLH5TCnOJiFe9zeIDAYKiH7o3Zq4cfoIahrbeG1jLROHZ1Kck8prG2vp6OppGRRkJHHzzFIykn08vWIXq3c1kBBvfOOT47n13JH9DoST0dDaycsf1PD82r28VVVPW2c3ZnCif9a89CRG5aUxIjeVwuwUqg8eZlX1IbbWtZDoi2NMQTpjCtIxM+qb26lramdzTRPdDtKTfJw3Jo+LJw5l3rihZKUm0NnVzcGWDhrbOmlp76KlvSecmtv9tLT76ehy+OKM+DijobWTTfua2LSviYbWTnLTE8lL77lslpLoIzUxHl+c0dbZRWtnFy0dXTS1+Wls7cQMZpblMm98AVOKc3DO0dTmp7Wzi4xkX+DSnVpCIn2lgOin3q2JlIR4vnrRWG6ZWYovPo6Gw528tH4fKYnxXDLxNBJ9f/tNurK2mSRfHMVDUk+5hpPR1tnF8m0HqNhxkCRfHJnJPtKTfR+2HpyDopxUxhSkk5OWGPQ9Wtr9JCfEEx8k3BpaO3m7qp7XN9fx6oZaapvaiY8z0pN8NLR29qvWwuwUxp+WQU5aIvub26lv7uDg4Y6eUOjoorPbkZIQT2piPCmJ8WQkJ5CZ7KO1o4uV1Yfo6u4JHP9R1+CO1OOL67lk5osz8jOSOC0rmcLsFM4fm8+5o/M+8u8mEssUECdp074mslMTGJqpO4qO1t3tWLO7gVc+qKGprZMhaUkMSU/sCaUkH2lJPtISfWQk97xOjI+jyzn83d2kJvZsc7IaDnfyZmUda3c3fHiM5IR4mtv8HGrtoKnNT7dzdDvo9HdT29TOvoY2qg8e5nBHT0vjoglDuWZaITPL8oKGoUisUECIAO3+Lt6q3M9za/eyaP0+Gtv8DM9K5tppRcwsy2Xi8CyyUhO8LlNkQCkgRI7S1tnFKxtqeLJiF29uqfuwz6YoJ4WzSodwTlkuM8tyKcrx5nKhyEBRQIgcx4GWDtbtbmD9nkbW7j7Eu1sPsL+lA4BR+WnMGVvAnHH5nFOWq7u3ZNBRQIj0g3OOzTXNLKvs6ZA/8sDj8Kxkbp01ks9MH3FKfSgikUQBIXIKWju6eGNLHQ8v3ca72w6QmezjhvJiPjO9mNEFGV6XJ3JKFBAiIbJy50EeenMbL63fh7/bUV6Sw5cvHMusMaEZokVkoCkgREKsrqmdp9/fxaPv7qD6QCt3zCnjqxeNVR+FRJ3jBURYf5rNbLuZrTWzVWYW9JPbzOYE1q83s9d7Lf+kmW0ys0oz+2Y46xTpr/yMJG4/v4xFXz6fG6cXc/+SKj79q7fZfajV69JEQmYgft2Z65ybEiyhzCwb+CVwpXNuIvCpwPJ44BfApcDpwI1mdvoA1CrSLymJ8fzw2kn87MapbK5p5qr7lrFm1yGvyxIJCa/bw58F/uyc2wngnKsNLJ8OVDrntjrnOoAngKs8qlHkhK6cPJxn7ppJki+OT//qHV75oMbrkkROWbgDwgGLzGyFmc0Psn4skGNmSwLb/H1geSFQ3Wu7XYFlIhFrdEEGC+6ayeiCdOY/UsETy3d6XZLIKQn3zdyznHO7zawAeNnMNjrn3jjq+GcCFwApwNtm9k5/DhAInvkAI0aMCFHZIienICOZP37xbO589H3+dcFakhPiuXqqfreR6BTWFoRzbnfg71pgAT2XjnrbBbzknGtxztUDbwCTgd1Aca/tigLLgh3jQedcuXOuPD8/P9Tfgki/pSb6eOBzZzJj5BC+9qfVutwkUStsAWFmaWaWceQ1cDGw7qjN/gLMMjOfmaUCM4ANwHvAGDMbaWaJwGeAheGqVSTUkhPieejms5g4PJM7H3uft6rqvS5JpN/C2YIYCiw1s9XAcuA559yLZna7md0O4JzbALwIrAls85Bzbp1zzg/8E/ASPYHxpHNufRhrFQm59CQfv/3CdEqGpDL/9ytYt7vB65JE+kUPyomE2d6GVq775Vt0dDmevuMcSnLTvC5J5EOePSgnIjAsK4Xf3zYdf3c3f//wcuqa2r0uSaRPFBAiA2B0QQYP33IWNY1tzH+kAn9gXnORSKaAEBkg00bk8KPrJ7Ny5yF+sbjK63JETkgBITKArpg8nKunDOdnr21hVbWG5JDIpoAQGWDfu+oMhmYk8ZU/ruJwh9/rckSOSQEhMsCyUhL48Q2T2Vbfwg+e2+B1OSLHpIAQ8cDMsjzmzx7Fo+/u5Pm1e70uRyQoBYSIR/7l4nFMLs7mG0+tYef+w16XI/IxCggRjyT64rjvxqlgcPfj79Ph162vElkUECIeKh6Syo+un8TqXQ386KWNXpcj8hEKCBGPffKMYdw0YwS/XrqNjfsavS5H5EMKCJEI8PVLxpGRnMAPntvAYBofTQZGuH5mFBAiESA7NZEvXziGN7fUs3hT7Yl3EAl48r1q7n58JW2dXSF/bwWESIT43NkljMpL4wfPbaBTYzVJH7y5pY5vLVhLQ2sn8XEW8vdXQIhEiIT4OL512QSq6lp47F3NZy3Ht2FvI3f84X1GF6Tzy5umkRAf+o9zBYRIBLlgQgHnjs7lZ69uCcslAxkc9ja08oXfvEdaUjy/+cJZZCQnhOU4CgiRCGJm3DlnNPtbOvSEtQTV0u7ntt9W0NTWycO3nMWwrJSwHUsBIRJhZpblMio/jd+/vcPrUiTCdHU7vvTESjbua+S+m6YxcXhWWI+ngBCJMGbG588uYVX1Idbu0jzW8jc/fH4Dr2yo5TtXTGTuuIKwH08BIRKBrjuziNTEeH7/9navS5EI8cTynTy0dBu3zCzl5pmlA3JMBYRIBMpMTuDqqYUsXL2Hgy0dXpcjHtu4r5FvL1zPeWPy+Pe/mzBgx1VAiESoz59dQru/mz+tqPa6FPFQW2cX//z4SjKTE/jJp6fgC8PtrMeigBCJUBOGZXJWaQ6PvrtTw2/EsP987gM21zRz7w2TyUtPGtBjKyBEIth104rYsf8wG/c1eV2KeOCl9fv4wzs7mT97FLPH5g/48RUQIhFs3vieO1Ve3VDjcSUy0FbuPMhX/riKSUVZ/MvF4zypQQEhEsEKMpOZXJzNKxs0gF8s2VzTxC2/eY+89CQe+vtyEn3efFQrIEQi3IXjC1hVfYjapjavS5EBUH3gMJ//9bsk+eL4w20zKMhM9qwWBYRIhLtgwlAAFm9UK2Kwa2zr5ObfLKe1o4vf3zadEbmpntajgBCJcBOGZVCYnaLLTINcV7fjS4+vZOf+wzz49+WMPy3T65IUECKRzsy4YEIBS7fUa4TXQezHizaxeFMd371yImePyvW6HEABIRIVLpgwlNbOLt6qqve6FAmDhav3cP+SKj47YwSfO7vE63I+FNaAMLPtZrbWzFaZWUWQ9XPMrCGwfpWZfbuv+4rEkrNHDSEtMV6XmQahTfua+MZTazirNIfvXjHR63I+wjcAx5jrnDverz1vOucuP8l9RWJCki+e2WPzeXVDDe7qMzAL/fSSMvCa2/3c8egK0pJ8/OKz0zy7nfVYIqsaETmmCyYMpaaxnfV7Gr0uRULAOcc3nl7D9voW7vvsVE9vZz2WcAeEAxaZ2Qozm3+Mbc4xs9Vm9oKZTeznvpjZfDOrMLOKurq6UNYuElHmjsvHDF7RU9WDwu/e2s5za/by9UvGR0yn9NHCHRCznHPTgEuBu8xs9lHr3wdKnHOTgZ8Dz/RjXwCccw8658qdc+X5+QM/VonIQMlNT2LaiBxeVT9E1Fuz6xA/eH4DF04o4IuzR3ldzjGFNSCcc7sDf9cCC4DpR61vdM41B14/DySYWV5f9hWJRRdMKGDt7gZqGvVUdbRqauvk7sdXkp+exI8/NZm4uMjtTwpbQJhZmpllHHkNXAysO2qb0yzQ22Zm0wP17O/LviKx6ILxPU9VqxURnZxzfGvBOnYdbOX/3jiV7NREr0s6rnDexTQUWBD4/PcBjznnXjSz2wGccw8A1wN3mJkfaAU+45xzZhZ03zDWKhIVxg5NpygnhVc31PDZGSO8Lkf66cmKav66eg//cvFYziod4nU5JxS2gHDObQUmB1n+QK/X9wH39XVfkVhnZlw4YSiPL99Ja0cXKYnxXpckfdTQ2sl/PLuBmWW53DFntNfl9IlucxWJMhdMKKDd362nqqPMH97ZQXO7n3/7uwnER3C/Q28KCJEoM32knqqONm2dXfxm2TbOH5vPxOFZXpfTZwoIkShz5Knq1zbWaK7qKPGnFbuob+7gjjllXpfSLwoIkSg0b3wBNY3tbNiruaojnb+rmwffqGLqiGxmjIz8juneFBAiUej8wAT2r2/W6AGR7rm1e6k+0Mod55dF3RhaCgiRKFSQmcyEYZm8vln9EJHscIef+16rZHRBOhcGZgaMJgoIkSh1/th8KrYfpLnd73UpEkS7v4svPrKCqrpmvnXZ+Ih+YvpYFBAiUer8sfn4ux1vVep210jj7+rmy0+s4s0t9fzXdZOYNz76Wg+ggBCJWmeW5JCWGK9+iAjTM5zGWl5Yt49vX346N5QXe13SSVNAiESpRF8cM0fn8frmOt3uGkEeX17NkxW7uHveaG6dNdLrck6JAkIkip0/Np9dB1vZVt/idSkCbK5p4nt/Xc95Y/L4yoVjvS7nlCkgRKKYbneNHG2dXdz92Eoykn3cc0NkD+PdVwoIkShWPCSVUflpCogI8IPnNrCppol7bphCQUbkTR96MhQQIlHu/LH5vF21n7bOLq9LiVmLN9byyDs7+MfzRn7YqhsMFBAiUe6cUbm0+7tZv6fR61Ji0qHDHXzj6TWMPy2Df7lknNflhJQCQiTKTRmRDcDKnQc9riQ2ffsv6znQ0sE9N0wmyTe45udQQIhEuYKMZAqzU1hVfcjrUmLOc2v2snD1Hr50wZioGsa7rxQQIoPAlBHZrNypgBhIrR1d/O+/rGNyUVbUDePdVwoIkUFganE2uw+1UtvU5nUpMePdbfs50NLBVy8ehy9+cH6UDs7vSiTGTB2RA8AqtSIGzLLKehJ9cVE3x0N/KCBEBoGJwzNJiDdWqh9iwCyt3E95SQ7JCYOrY7o3BYTIIJCcEM/pwzJ1J9MAqWtqZ8PeRs4dned1KWGlgBAZJKYUZ7NmVwNd3Rq4L9zequoZYv28MQoIEYl0VVXc/OiPePe/ryXOFw+ZmXDnnVBV5XVlg9KyynqyUhIG5a2tvSkgRKLdCy/ApEmMfOZxMjpaMeegqQkeeggmTepZLyHjnGPplnpmluUSPwgG5DseBYRINKuqguuvh8OHsc7Oj67r7ITDh3vWqyURMtvqW9jT0Dbo+x9AASES3e65pycIjqezE37yk4GpJwYsC0zxOksBISIR7Q9/6FtAPPLIwNQTA5ZW1lOUk0JJbqrXpYRdnwLCzNLMLC7weqyZXWlmCeEtTUROqLk5tNvJcfm7unmraj+zRudhNrj7H6DvLYg3gGQzKwQWAZ8HfhuuokSkj9LTQ7udHNcDr1fR1ObnotOHel3KgOhrQJhz7jBwLfBL59yngIkn3Mlsu5mtNbNVZlYRZP0cM2sIrF9lZt/ute6TZrbJzCrN7Jt9/YZEYsrnPgcJJ2jMJyTA5z8/MPUMYit2HOQnr2zhysnDmTe+wOtyBkSfA8LMzgFuAp4LLOvr8+VznXNTnHPlx1j/ZmD9FOfc9wMHiwd+AVwKnA7caGan9/F4IrHja1/rW0B85SsDU88g1djWyZeeWMmwrGT+85ozYuLyEvQ9IL4M/CuwwDm33sxGAYvDVxbTgUrn3FbnXAfwBHBVGI8nEp3KyuCppyA19WNB0e1L6Fn+1FM928lJcc7xrT+vZW9DGz+7cSqZybHT/dqngHDOve6cu9I599+Bzup659w/92VXYJGZrTCz+cfY5hwzW21mL5jZkctWhUB1r212BZZ9jJnNN7MKM6uoq9PE7RKDLr0U1qyB+fMhMxNnRlNiKpuv/EzP8ksv9brCqPbS+n08u2YvX71oLNMCo+bGir7exfSYmWWaWRqwDvjAzL7eh11nOeem0XOp6C4zm33U+veBEufcZODnwDP9qB0A59yDzrly51x5fv7gmSxcpF/KyuC++6ChAbq6uOg7C/nF9V9Ry+EUHe7w8/2/fsCEYZl8cfYor8sZcH29xHS6c64RuBp4ARhJz51Mx+Wc2x34uxZYQM+lo97rG51zzYHXzwMJZpYH7AaKe21aFFgmIidgZpxZksP7OzSy66n6+WuV7Glo4z+umjhoJwU6nr5+xwmB5x6uBhY65zrpuXx0TIFnJzKOvAYupqf10Xub0yzQ22Nm0wP17AfeA8aY2UgzSwQ+Ayzs+7clEtvOLMlh96FW9ja0el1K1KqsbeahN7dy/ZlFlJcO3kmBjsfXx+1+BWwHVgNvmFkJ0HiCfYYCCwKf/z7gMefci2Z2O4Bz7gHgeuAOM/MDrcBnnHMO8JvZPwEv0XO31MPOufX9+s5EYlh5ac+18hU7DnL5pBSPq4k+zjm+s3AdKQnxfPPS8V6X45k+BYRz7mfAz3ot2mFmc0+wz1ZgcpDlD/R6fR9w3zH2fx54vi/1ichHTRiWSUpCPBXbD3L5pOFelxN1/rpmL8sq9/P9qyaSl57kdTme6WsndZaZ3XvkbiEzuwdIC3NtInKSEuLjmFycxQr1Q/RbY1sn//HsB0wqyuKmGSVel+OpvvZBPAw0ATcE/jQCvwlXUSJy6s4syeGDvY0c7vB7XUpUuXfRZuqb2/nPq88Y9PM9nEhfA6LMOfedwINrW51z3wNi754vkShyVukQurodFdvViuirdbsb+P3b2/n82SVMKsr2uhzP9TUgWs1s1pEvzOxcejqVRSRCTR85hMT4OJYG5i+Q4+vqdvzbgrUMSUviaxeP87qciNDXu5huB35vZkcmYD0I3ByekkQkFFITfUwryWbpFgVEXzy7Zg+rdzXw009PISsldobTOJ6+DrWxOvC08yRgknNuKjAvrJWJyCk7b0w+H+xtpL653etSIppzjl+9vpWy/DSunKy7vo7o16OBgSefjzz/8NUw1CMiIXRk3uRlusx0XEsr6/lgbyNfnF1GXIx3TPd2Ks+O6yyKRLhPFGaRlZKggDiBB16vYmhmEldNVeuht1MJiOMOtSEi3ouPM2aW5bJ0Sz09gxTI0dbuamBZ5X5uPXckSb6+TnMTG44bEGbWZGaNQf40AYpakSgwa0weexra2Frf4nUpEelXb1SRkeTjxhkjvC4l4hz3LibnXMZAFSIi4TGrVz9EWb7mpu6tqq6Z59fu5R9nj4qpiYD6KvbGrxWJMSW5aRQPSeFN3e76ES3tfu74wwoyUxK47dyRXpcTkRQQIjFg1uh83qnaj7+r2+tSIkJ3t+NrT66msraZ+26cRkFmstclRSQFhEgMOG9MHk3tflZVH/K6lIjwi8WVvLh+H9+6bAKzxuR5XU7EUkCIxIBzR+cRH2cs2aR529+qrOfeVzZzzdRCbpulS0vHo4AQiQFZKQmcWZLDaxtrvS7FU/6ubr771/UU56Tyw2s/QWBCMzkGBYRIjJg7roAP9jayr6HN61I889jynWyuaeZbl00gOUHPPJyIAkIkRswbXwDA65tjsxVx6HAH9768mXNG5XLJxKFelxMVFBAiMWLs0HSGZyWzeGNs9kP89JUtNLZ28u0rTtelpT5SQIjECDNjzvgCllbW0+GPrdtdK2ubeOSdHdw4fQQThmV6XU7UUECIxJC54wpobvdTsf2A16UMqB+/tJmUhHi+etFYr0uJKgoIkRgysyyXxPg4Fm+KnX6ItbsaeHH9Pm6bNZLc9CSvy4kqCgiRGJKW5GPGqCExdbvrPS9vIjs1gdvO0zMP/aWAEIkxc8cVUFXXws79h70uJewqth9gyaY6vji7TIPxnQQFhEiMOTK0xPJB3g/hnOPHizaRl57EzTNLvC4nKikgRGJMWX466Uk+VlUf9LqUsHqraj/vbD3AXXPLSE087swGcgwKCJEYEx9nTCrKGtQD9znnuPflzQzLSubG6ZoI6GQpIERi0OTibDbubaKts8vrUsLizS31rNhxkDvnjtaQGqdAASESg6YUZ+Pvdqzb3eB1KSF3pPVQmJ3CDeVFXpcT1RQQIjFoanE2wKC8zLRkcx2rqg9x19zRJPnUejgVYQ0IM9tuZmvNbJWZVRxnu7PMzG9m1/da1hXYb5WZLQxnnSKxpiAzmeFZyawcZAHhnOOngdbD9Weq9XCqBqJrf65z7piT4ZpZPPDfwKKjVrU656aEtTKRGDZlRDarB1lALNlUx+pdDfzXtZ8g0acLJKcqEs7g3cDTQOw82ikSAaYUZ7PrYCv1ze1elxISzjnuW1xJYXYK16n1EBLhDggHLDKzFWY2/+iVZlYIXAPcH2TfZDOrMLN3zOzqYx3AzOYHtquoq4vNYYxFTsaU4hwAVu0cHK2I5dsOsGLHQb54/igS4iPhd9/oF+6zOMs5Nw24FLjLzGYftf6nwDecc8HGHi5xzpUDnwV+amZlwQ7gnHvQOVfunCvPz88PafEig9knCrOIj7NB01F9/+tV5KYl8qkzi70uZdAIa0A453YH/q4FFgDTj9qkHHjCzLYD1wO/PNJa6LXvVmAJMDWctYrEmpTEeMYNzRgUAbF+TwNLNtVx66yRpCTqzqVQCVtAmFmamWUceQ1cDKzrvY1zbqRzrtQ5Vwo8BdzpnHvGzHLMLCmwbx5wLvBBuGoViVVHOqq7u53XpZyS+5dUkZ7k43Nna8ylUApnC2IosNTMVgPLgeeccy+a2e1mdvsJ9p0AVAT2XQz8l3NOASESYlOKs2lq97O1vtnrUk7a9voWnl+7l5vOHkFWikZsDaWw3eYauDQ0OcjyB46x/S29Xr8FfCJctYlIj/KSno7qt7ceYHRBhsfVnJzHlu8kPs647VzN9xBq6uoXiWEj89IoHpLC61E8w9yrG2o4e1QuBZnJXpcy6CggRGKYmTFnbAHLKvfT7o++gft27j9MVV0Lc8YVeF3KoKSAEIlxc8bl09rZxfJt0TeB0JG5teeNV0CEgwJCJMadU5ZLYnwcSzZF34Omr22sZWReGiPz0rwuZVBSQIjEuNREHzNGDWFJlPVDtHZ08fbW/cwZpwdkw0UBISKcPzafqroWqg8c9rqUPnurqp4Of7cuL4WRAkJEmBv4kF2yOXouMy3eVEtqYjzTRw7xupRBSwEhIoyKsttdnXMs3ljHuaPzNClQGCkgRCTqbnfdXNPM7kOtzNXtrWGlgBAR4G+3u7637aDXpZzQs2v2ADB3vDqow0kBISIAzBiVS5zB8m37vS7luJZsquUXiyv55MTTGJaV4nU5g5oCQkQASE/ycfrwTN7bHrktiE37mvinx1Yy/rRM7rnhY0O9SYgpIETkQ+UlQ1hZfZDOrmBzeHmrvrmdW3/7HqmJ8fz6lnLSksI21qgEKCBE5ENnlQ6hrbOb9XsavS7lY/59wTr2t7Tz0M3lurQ0QBQQIvKh8tKe4b8rtkfWuEyrqg/x4vp93DlnNJOKsr0uJ2YoIETkQ0MzkxkxJJX3IiwgfvTSRnLTErl1luZ8GEgKCBH5iPLSHCq2H8S5yJiGdFllPcsq93Pn3NGkq99hQCkgROQjziodwv6WDrbVt3hdCs45/uelTQzPSuamGSO8LifmKCBE5CPO+rAfwvvbXRd9UMPq6kN86cIxJCdoSI2BpoAQkY8oy08nJzXB836I7m7HT17ezKi8NK6bVuRpLbFKASEiH2FmnFkyhIod3rYgXtlQw8Z9Tdx9wWh88fqo8oLOuoh8zFmlOWyrb6G+ud2T4zvn+NlrWyjJTeWKScM9qUEUECISRHlpzxwL73k0T/XiTbWs293IXXPVevCSzryIfMykoiwykny87sEEQs45/u+rlRTlpHDN1MIBP778jQJCRD4mIT6O2ePyeXVjLQMWlq0AAAt6SURBVN3dA/s8xBtb6lldfYi75o4mQa0HT+nsi0hQF4wvoK6pnXV7GgbsmM457n15M8OzknXnUgRQQIhIUHPGFRBn8MqGgZuG9MV1+1hdfYgvXziWRJ8+nrymfwERCWpIWiLTRuTw2saaATmev6ubHy3axOiCdK6dpr6HSKCAEJFjmjehgHW7G9nX0Bb2Y/1pxS621rXw9UvG6c6lCKF/BRE5pgsnDAXgtY3hvczU2tHFT1/ZzLQR2Vx8+tCwHkv6LqwBYWbbzWytma0ys4rjbHeWmfnN7Ppey242sy2BPzeHs04RCW5MQTpFOSlhv8z027e2U9PYzjc+OR4zC+uxpO8GYuzcuc65+mOtNLN44L+BRb2WDQG+A5QDDlhhZgudc96PHiYSQ8yMC8YX8MeKato6u8IyYF5Dayf3L6lk7rh8ZozKDfn7y8mLhEtMdwNPA73bsJcALzvnDgRC4WXgk14UJxLrLpgwlLbObpZVHvP3vFPy4BtVNLb5+fol48Py/nLywh0QDlhkZivMbP7RK82sELgGuP+oVYVAda+vdwWWicgAmzFqCJnJPp5dszfk713b1MbDS7dzxeThnD48M+TvL6cm3AExyzk3DbgUuMvMZh+1/qfAN5xz3Sd7ADObb2YVZlZRVzfwwwKIDHZJvngunzycF9fto7ndH9L3/uXiKjq6uvnqRWND+r4SGmENCOfc7sDftcACYPpRm5QDT5jZduB64JdmdjWwGyjutV1RYFmwYzzonCt3zpXn5+eH+DsQEYDrphXS2tnFi+v2hew9qw8c5tF3d3BDeREj89JC9r4SOmELCDNLM7OMI6+Bi4F1vbdxzo10zpU650qBp4A7nXPPAC8BF5tZjpnlBPZ9KVy1isjxTRuRQ0luKn9+f1fI3vPnr23BzPjnC8aE7D0ltMLZghgKLDWz1cBy4Dnn3ItmdruZ3X68HZ1zB4D/AN4L/Pl+YJmIeMDMuHZqEW9v3c/uQ62n/H5NbZ38ZdUerj+ziGFZKSGoUMIhbLe5Oue2ApODLH/gGNvfctTXDwMPh6U4Eem3a6YW8pNXNvPMyt3cNXf0Kb3XC+v20e7v1oB8ES4SbnMVkSgwIjeV6aVD+PP7u3Du1IYAX/D+bkpzU5k2IjtE1Uk4KCBEpM+unVZIVV0La3ad/BDgew618s62/Vw9tVBPTUc4BYSI9Nllk4aR5IvjifeqT7zxMTyzajfOodniooACQkT6LDM5gSsmD2fhqt0n9UyEc44F7+/mzJIcSnJ1a2ukU0CISL/cNGMELR1dLFgZ9NGk41q/p5Ettc1qPUQJBYSI9MuU4mwmDs/k0Xd29LuzesHK3STGx3H5pGFhqk5CSQEhIv1iZtw0o4SN+5p4f+ehPu/XGmh1zBtfQHZqYhgrlFBRQIhIv101ZTjpST4efWdHn/d5+v1dHGjp4AvnloavMAkpBYSI9Ftako9rphby7Nq9HGzpOOH23d2Oh5duY1JRFtNHDhmACiUUFBAiclJuOnsEHf5unu7D+Eyvbqxla30L/3DeKD37EEUUECJyUsaflsnUEdn88b3qE3ZW/783t1KYncJlZ5w2QNVJKCggROSk3VBezJbaZlZVH7uzes2uQyzfdoAvnFuKL14fOdFE/1oictIunzSMlIR4nqw49mWmB9/YSkaSj0+fVXzMbSQyKSBE5KRlJCdw2SeG8dfVe2jt6PrY+hU7DvLsmr18/pwSMpITPKhQToUCQkROyQ3lRTS3+3lh3UfnrPZ3dfPvz6xjWFbyKQ8PLt5QQIjIKZk+cgiluan88agB/H739g427G3kO1ecTlpS2KaekTBSQIjIKTEzPlVezLvbDrC9vgWAfQ1t3LtoE3PG5XPJRN25FK0U6yJyyq6bVsQ9izZx+c+XMmJIKm3+Ljq7Hd+7cqKee4hiakGIyCk7LSuZh24u59pphZyWlUxifBzfunS8hvSOcmpBiEhIzBs/lHnjh3pdhoSQWhAiIhKUAkJERIJSQIiISFAKCBERCUoBISIiQSkgREQkKAWEiIgEpYAQEZGg7EQzQUUTM6sD+j6LenTIA+q9LiKK6Hz1j85X/wzG81XinMsPtmJQBcRgZGYVzrlyr+uIFjpf/aPz1T+xdr50iUlERIJSQIiISFAKiMj3oNcFRBmdr/7R+eqfmDpf6oMQEZGg1IIQEZGgFBAiIhKUAkJERIJSQEQxMzvPzB4ws4fM7C2v64l0ZjbHzN4MnLM5XtcT6cxsQuBcPWVmd3hdT6Qzs1Fm9msze8rrWkJFAeERM3vYzGrNbN1Ryz9pZpvMrNLMvnm893DOvemcux14FvhdOOv1WijOF+CAZiAZ2BWuWiNBiH6+NgR+vm4Azg1nvV4L0fna6py7LbyVDizdxeQRM5tNz4fV751zZwSWxQObgYvo+QB7D7gRiAd+eNRb3Oqcqw3s9yRwm3OuaYDKH3ChOF9AvXOu28yGAvc6524aqPoHWqh+vszsSuAO4BHn3GMDVf9AC/H/x6ecc9cPVO3h5PO6gFjlnHvDzEqPWjwdqHTObQUwsyeAq5xzPwQuD/Y+ZjYCaBjM4QChO18BB4GkcNQZKUJ1vpxzC4GFZvYcMGgDIsQ/X4OGLjFFlkKgutfXuwLLjuc24Ddhqyiy9et8mdm1ZvYr4BHgvjDXFon6e77mmNnPAufs+XAXF4H6e75yzewBYKqZ/Wu4ixsIakFEOefcd7yuIVo45/4M/NnrOqKFc24JsMTjMqKGc24/cLvXdYSSWhCRZTdQ3OvrosAyCU7nq390vvon5s+XAiKyvAeMMbORZpYIfAZY6HFNkUznq390vvon5s+XAsIjZvY48DYwzsx2mdltzjk/8E/AS8AG4Enn3Hov64wUOl/9o/PVPzpfwek2VxERCUotCBERCUoBISIiQSkgREQkKAWEiIgEpYAQEZGgFBAiIhKUAkIGPTNrHuDjDejcHGaWbWZ3DuQxJTYoIET6ycyOO4aZc27mAB8zG1BASMgpICQmmVmZmb1oZisCs8yNDyy/wszeNbOVZvZKYO4IzOy7ZvaImS0DHgl8/bCZLTGzrWb2z73euznw95zA+qfMbKOZPWpmFlh3WWDZisCIqc8GqfEWM1toZq8Br5pZupm9ambvm9laM7sqsOl/AWVmtsrMfhTY9+tm9p6ZrTGz74XzXMrgpdFcJVY9CNzunNtiZjOAXwLzgKXA2c45Z2b/APwv4GuBfU4HZjnnWs3su8B4YC6QAWwys/udc51HHWcqMBHYAywDzjWzCuBXwGzn3LbAMA/HMg2Y5Jw7EGhFXOOcazSzPOAdM1sIfBM4wzk3BcDMLgbG0DOfgdEzn8Ns59wbJ322JCYpICTmmFk6MBP4U+AXevjbBEJFwB/NbBiQCGzrtetC51xrr6+fc861A+1mVgsM5eNTmS53zu0KHHcVUErPzGVbnXNH3vtxYP4xyn3ZOXfgSOnA/wnMftZNz9wEQ4Psc3Hgz8rA1+n0BIYCQvpFASGxKA44dOQ37qP8nJ7pSBea2Rzgu73WtRy1bXuv110E///Ul22Op/cxbwLygTOdc51mtp2e+bWPZsAPnXO/6uexRD5CfRASc5xzjcA2M/sUgPWYHFidxd/G/L85TCVsAkb1muLy033cLwuoDYTDXKAksLyJnstcR7wE3BpoKWFmhWZWcMpVS8xRC0JiQaqZ9b70cy89v43fb2b/DiQATwCr6Wkx/MnMDgKvASNDXUygD+NO4EUza6Fn3oG+eBT4q5mtBSqAjYH3229my8xsHfCCc+7rZjYBeDtwCa0Z+BxQG+rvRQY3Dfct4gEzS3fONQfuavoFsMU59xOv6xLpTZeYRLzxj4FO6/X0XDpSf4FEHLUgREQkKLUgREQkKAWEiIgEpYAQEZGgFBAiIhKUAkJERIJSQIiISFD/H3bCPreU5R40AAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qQy0enT0GY9E",
        "colab_type": "text"
      },
      "source": [
        "I also use W&B Logging to keep track of my experiments:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T79_EV1yGY9E",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from pytorch_lightning.loggers import WandbLogger"
      ],
      "execution_count": 41,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j48RY0ZeGY9G",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        },
        "outputId": "2eac8409-f7af-42f0-8bb2-a04ce3975f06"
      },
      "source": [
        "hparams = Namespace(\n",
        "    lr=0.0005248074602497723,\n",
        "    epochs=10,\n",
        "    batch_size=160,\n",
        "    train_size=20000,\n",
        "    validation_size=1000\n",
        ")\n",
        "module = ImageEmbeddingModule(hparams)\n",
        "trainer = pl.Trainer(gpus=1, max_epochs=hparams.epochs)\n",
        "# Uncomment for W&B integration\n",
        "# logger = WandbLogger(project=\"simclr-blogpost\")\n",
        "# logger.watch(module, log=\"all\", log_freq=50)\n",
        "# trainer = pl.Trainer(gpus=1, max_epochs=hparams.epochs, logger=logger)\n"
      ],
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "GPU available: True, used: True\n",
            "TPU available: False, using: 0 TPU cores\n",
            "CUDA_VISIBLE_DEVICES: [0]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Loaded pretrained weights for efficientnet-b0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BgBI2KGbGY9H",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 188,
          "referenced_widgets": [
            "fd93fcfb47284680a4ae25178dd951fa",
            "8171c55f8f1c49f384d9f38784f22c13",
            "3733cce25e7c40cb83c7424036e4f372",
            "c0c72566cfd7404f9e81e6efc397dc8e",
            "eee8913da3b04fcaa27093426b25fd38",
            "27cfa39fbd6441548cf77d3e8735485d",
            "ba588cacf6c94025892f5257e83322cd",
            "53d6d9ee2c2f493a8bfe5269490736d5",
            "3974604f19b840b2a09ebd284495ed53",
            "117c99eccbe94c78b630849a9db450c2",
            "e3917ebbc51e487481f66c8df68316ce",
            "33ba2cf3460542f4a632e05696351dee",
            "01c7832e21554cfda7c0b759a59f4063",
            "e647d0aeeba648d5860ba2c68ad37ba1",
            "fe9475251c4042aca5c598b33549aa40",
            "85d7b051ac3446d9b72a98c4c331599d",
            "e0bd1cbc38b34d2cb633966565325b7d",
            "6b37cdaa12c74de6b86fce96c838f381",
            "e3c18fc7317d44d2944876185d52127a",
            "964d4acb696540afb93260770ae6ce2c",
            "11759fc12080455094624b153eb35dce",
            "678f7c3bb42549d884e3b8995dd47e81",
            "1c1c1eba0d5e436ba66de315449294e3",
            "b2db4f9b5ecd437586c1153c3e21e661",
            "313feecc97084ace93710d8e11237d06",
            "e611467f4c544611aade87ee71d60914",
            "2d139b573db54e6f90b5e545de34f28d",
            "a8a0c1847cec4eef98264aea6b42308c",
            "060fa39300cc4a488d846900165600ad",
            "115e638d13d64a2ba4cd899451a1c005",
            "b88870fb6ffe483fa6fc02d5e729949e",
            "69ba8210c0f747fe9c0b3c8df8f5ca26"
          ]
        },
        "outputId": "652c5f03-6aa2-430a-d916-43799f6eaaa9"
      },
      "source": [
        "trainer.fit(module)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/pytorch_lightning/utilities/distributed.py:25: RuntimeWarning: You have defined a `val_dataloader()` and have defined a `validation_step()`, you may also want to define `validation_epoch_end()` for accumulating stats.\n",
            "  warnings.warn(*args, **kwargs)\n",
            "\n",
            "  | Name  | Type            | Params\n",
            "------------------------------------------\n",
            "0 | model | ImageEmbedding  | 6 M   \n",
            "1 | loss  | ContrastiveLoss | 0     \n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "fd93fcfb47284680a4ae25178dd951fa",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validation sanity check', layout=Layout…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3974604f19b840b2a09ebd284495ed53",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Training', layout=Layout(flex='2'), max…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e0bd1cbc38b34d2cb633966565325b7d",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "313feecc97084ace93710d8e11237d06",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uXuYMpyvGY9K",
        "colab_type": "text"
      },
      "source": [
        "## Image classifier on top of SimCLR embeddings\n",
        "Once the embeddings are trained, they can be used to train classifier on top of them - either by fine tuning the whole network or by freezing the base network with embeddings and learning linear classifier on top of it - I show the latter below.\n",
        "\n",
        "### Get pure weights of neural network with embeddings\n",
        "I extract the learned weights for easier usage later on."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YjVQU5EiGY9K",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "checkpoint_file = \"efficientnet-b0-stl10-embeddings.ckpt\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qeqRY1UcGY9N",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "trainer.save_checkpoint(checkpoint_file)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qRympG2tGY9P",
        "colab_type": "code",
        "colab": {},
        "outputId": "285171d9-1f31-4f03-a524-24182e3d6511"
      },
      "source": [
        "# trainer.logger.experiment.log_artifact(checkpoint_file, type=\"model\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<wandb.artifacts.Artifact at 0x7f7646752e90>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 109
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "C3TkSl9mGY9R",
        "colab_type": "text"
      },
      "source": [
        "### Classifier module\n",
        "Again, I define custom module - this time it uses already existing embeddings and freezes the base model weights on demand."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "plt4u_9pGY9S",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class SimCLRClassifier(nn.Module):\n",
        "    def __init__(self, n_classes, freeze_base, embeddings_model_path, hidden_size=512):\n",
        "        super().__init__()\n",
        "        \n",
        "        base_model = ImageEmbeddingModule.load_from_checkpoint(embeddings_model_path).model\n",
        "        \n",
        "        self.embeddings = base_model.embedding\n",
        "        \n",
        "        if freeze_base:\n",
        "            print(\"Freezing embeddings\")\n",
        "            for param in self.embeddings.parameters():\n",
        "                param.requires_grad = False\n",
        "                \n",
        "        # Only linear projection on top of the embeddings should be enough\n",
        "        self.classifier = nn.Linear(in_features=base_model.projection[0].in_features, \n",
        "                      out_features=n_classes if n_classes > 2 else 1)\n",
        "\n",
        "    \n",
        "    def forward(self, X, *args):\n",
        "        emb = self.embeddings(X)\n",
        "        return self.classifier(emb)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0MawfJuSGY9U",
        "colab_type": "text"
      },
      "source": [
        "### Classifier training code\n",
        "Classifier training code again uses PyTorch lightning, so I'm skipping in-depth explanation."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ST79RozgGY9U",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from torch import nn\n",
        "from torch.optim.lr_scheduler import CosineAnnealingLR"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vljeQe0aGY9X",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class SimCLRClassifierModule(pl.LightningModule):\n",
        "    def __init__(self, hparams):\n",
        "        super().__init__()\n",
        "        hparams = Namespace(**hparams) if isinstance(hparams, dict) else hparams\n",
        "        self.hparams = hparams\n",
        "        self.model = SimCLRClassifier(hparams.n_classes, hparams.freeze_base, \n",
        "                                      hparams.embeddings_path,\n",
        "                                      self.hparams.hidden_size)\n",
        "        self.loss = nn.CrossEntropyLoss()\n",
        "    \n",
        "    def total_steps(self):\n",
        "        return len(self.train_dataloader()) // self.hparams.epochs\n",
        "    \n",
        "    def preprocessing(seff):\n",
        "        return transforms.Compose([\n",
        "                transforms.ToTensor(),\n",
        "                transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n",
        "        ])\n",
        "    \n",
        "    def get_dataloader(self, split):\n",
        "        return DataLoader(STL10(\".\", split=split, transform=self.preprocessing()),\n",
        "                          batch_size=self.hparams.batch_size, \n",
        "                          shuffle=split==\"train\",\n",
        "                          num_workers=cpu_count(),\n",
        "                         drop_last=False)\n",
        "    \n",
        "    def train_dataloader(self):\n",
        "        return self.get_dataloader(\"train\")\n",
        "    \n",
        "    def val_dataloader(self):\n",
        "        return self.get_dataloader(\"test\")\n",
        "    \n",
        "    def forward(self, X):\n",
        "        return self.model(X)\n",
        "    \n",
        "    def step(self, batch, step_name = \"train\"):\n",
        "        X, y = batch\n",
        "        y_out = self.forward(X)\n",
        "        loss = self.loss(y_out, y)\n",
        "        loss_key = f\"{step_name}_loss\"\n",
        "        tensorboard_logs = {loss_key: loss}\n",
        "\n",
        "        return { (\"loss\" if step_name == \"train\" else loss_key): loss, 'log': tensorboard_logs,\n",
        "                        \"progress_bar\": {loss_key: loss}}\n",
        "    \n",
        "    def training_step(self, batch, batch_idx):\n",
        "        return self.step(batch, \"train\")\n",
        "    \n",
        "    def validation_step(self, batch, batch_idx):\n",
        "        return self.step(batch, \"val\")\n",
        "    \n",
        "    def test_step(self, batch, batch_idx):\n",
        "        return self.step(Batch, \"test\")\n",
        "    \n",
        "    def validation_end(self, outputs):\n",
        "        if len(outputs) == 0:\n",
        "            return {\"val_loss\": torch.tensor(0)}\n",
        "        else:\n",
        "            loss = torch.stack([x[\"val_loss\"] for x in outputs]).mean()\n",
        "            return {\"val_loss\": loss, \"log\": {\"val_loss\": loss}}\n",
        "\n",
        "    def configure_optimizers(self):\n",
        "        optimizer = RMSprop(self.model.parameters(), lr=self.hparams.lr)\n",
        "        schedulers = [\n",
        "            CosineAnnealingLR(optimizer, self.hparams.epochs)\n",
        "        ] if self.hparams.epochs > 1 else []\n",
        "        return [optimizer], schedulers"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KUQAMtgfGY9Y",
        "colab_type": "text"
      },
      "source": [
        "> It's worth to mention here that training with frozen base model gives a great performance boost during training as the gradients needs to be calculated only for the small portion of the whole network. Additionaly, by utilizing good embeddings, only a few epochs are required to reach good quality classifier with single linear projection."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MKkq3sw4GY9Y",
        "colab_type": "code",
        "colab": {},
        "outputId": "e198491e-f057-4106-9eae-f471555a9fc0"
      },
      "source": [
        "hparams_cls = Namespace(\n",
        "    lr=1e-3,\n",
        "    epochs=5,\n",
        "    batch_size=160,\n",
        "    n_classes=10,\n",
        "    freeze_base=True,\n",
        "    embeddings_path=\"./efficientnet-b0-stl10-embeddings.ckpt\",\n",
        "    hidden_size=512\n",
        ")\n",
        "module = SimCLRClassifierModule(hparams_cls)\n",
        "trainer = pl.Trainer(gpus=1, max_epochs=hparams_cls.epochs)\n",
        "# Uncomment for W&B integration\n",
        "# logger = WandbLogger(project=\"simclr-blogpost-classifier\")\n",
        "# logger.watch(module, log=\"all\", log_freq=50)\n",
        "# trainer = pl.Trainer(gpus=1, max_epochs=hparams_cls.epochs, logger=logger)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Loaded pretrained weights for efficientnet-b0\n",
            "Freezing embeddings\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "                Logging results to <a href=\"https://wandb.com\" target=\"_blank\">Weights & Biases</a> <a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">(Documentation)</a>.<br/>\n",
              "                Project page: <a href=\"https://app.wandb.ai/egnyte/simclr-blogpost-classifier\" target=\"_blank\">https://app.wandb.ai/egnyte/simclr-blogpost-classifier</a><br/>\n",
              "                Run page: <a href=\"https://app.wandb.ai/egnyte/simclr-blogpost-classifier/runs/r154ystm\" target=\"_blank\">https://app.wandb.ai/egnyte/simclr-blogpost-classifier/runs/r154ystm</a><br/>\n",
              "            "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "GPU available: True, used: True\n",
            "TPU available: False, using: 0 TPU cores\n",
            "CUDA_VISIBLE_DEVICES: [0]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1Tw4L2AOGY9a",
        "colab_type": "code",
        "colab": {
          "referenced_widgets": [
            "4530667ea22b49f3a4e157fb9260e02f"
          ]
        },
        "outputId": "1aa68adf-e0db-476f-9cf2-67a488c2666d"
      },
      "source": [
        "lr_find_cls = trainer.lr_find(module)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "  | Name  | Type             | Params\n",
            "-------------------------------------------\n",
            "0 | model | SimCLRClassifier | 4 M   \n",
            "1 | loss  | CrossEntropyLoss | 0     \n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4530667ea22b49f3a4e157fb9260e02f",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Finding best initial lr', style=ProgressStyle(description…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "LR finder stopped early due to diverging loss.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ev0VcY_9GY9c",
        "colab_type": "code",
        "colab": {},
        "outputId": "0c4162f6-9b4d-4a62-c6ae-3afb6e92ca95"
      },
      "source": [
        "lr_find_cls.plot(show=False, suggest=True)\n",
        "lr_find_cls.suggestion()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.003981071705534969"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 221
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAEKCAYAAAAYd05sAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAAsTAAALEwEAmpwYAAAegUlEQVR4nO3deXCcd53n8fe3D7VuybIVx/ER2c49Ic6hJCRhIJwFCwMs10ARYCAzLtjdYcPMMgVTWzNQW7Vs1Q6wQJYZXAEGEu5M2M0GwjFkQkgc7MiO7cRJcCLLjuVT961WH9/9o1u24si2LOvpp/vR51XVZXX38zy/bz+2P/3o9/ye32PujoiIRE8s7AJERCQYCngRkYhSwIuIRJQCXkQkohTwIiIRpYAXEYmoRNgFzLRs2TJva2sLuwwRkYqxbdu2Xndvne29QAPezD4F/DngwFPAR9198lTLt7W10dHREWRJIiKRYmb7T/VeYF00ZrYS+CTQ7u5XAnHg/UG1JyIiLxV0H3wCqDGzBFALHAq4PRERKQos4N39IPAPwIvAYWDI3X918nJmttHMOsyso6enJ6hyREQWnSC7aJYA7wDWAhcAdWZ228nLufsmd2939/bW1lnPE4iIyDwE2UXzBqDL3XvcPQPcB9wcYHsiIjJDkAH/IvBKM6s1MwNeDzwbYHsiIjJDkH3wW4B7ge0UhkjGgE1BtSciUomePjjE754P5vxjoKNo3P3v3f0yd7/S3T/k7ukg2xMRqTT3/H4/n/rRzkC2rakKRERC1DuaZll9VSDbVsCLiISoZ3SK1oZUINtWwIuIhKh3JM2yegW8iEikuLu6aEREomg0nSWdzauLRkQkanpHpwDURSMiEjU9I4WR4wp4EZGI6R1VwIuIRNLxgG/QSVYRkUjpHUljBi21CngRkUjpGZ2ipbaKRDyYKFbAi4iEpDAGPpj+d1DAi4iEpnc0HVj/OyjgRURC0zuaplVH8CIi0dM7MqUuGhGRqBlLZ5nI5FgW0DQFoIAXEQlF0Bc5gQJeRCQUJwJeJ1lFRCKlZyTYicZAAS8iEorpI/igpgoGBbyISCimA76lTl00IiKR0juaZkltkmRA0xSAAl5EJBQ9Ad6LdZoCXkQkBL2jU4H2v0OAAW9ml5rZjhmPYTO7I6j2REQqSdATjQEkgtqwu/8BuBrAzOLAQeCnQbUnIlJJeiPURfN6oNPd95eoPRGRsjUxlWNsKhfoTJJQuoB/P/CD2d4ws41m1mFmHT09PSUqR0QkPKWYpgBKEPBmVgW8HfjJbO+7+yZ3b3f39tbW1qDLEREJXc/0RU6VHvDAW4Dt7n60BG2JiJS93pGIHMEDH+AU3TMiIotR72hxHppK7oM3szrgjcB9QbYjIlJJpvvgl9ZV6DBJAHcfA5YG2YaISKXpHU3TXJukKhFsJ4quZBURKbFSXOQECngRkZIrzEMTbP87KOBFREru4MAEFzTXBN6OAl5EpIQyuTxHhidZtaQ28LYU8CIiJXRkaJK8w6olOoIXEYmUAwPjgAJeRCRyugcmAFitLhoRkWjpHpggZnB+U3XgbSngRURKqHtgnPMbqwO9F+s0BbyISAkdHJgoyQgaUMCLiJRU98BESU6wggJeRKRkssfHwCvgRUQi5fDQJLm8s1IBLyISLdNDJNUHLyISMd0lvMgJFPAiIiVzcHACM1jRpIAXEYmU7oEJzm+sDvxGH9MU8CIiJdI9MF6y7hlQwIuIlEz3wAQrSzAP/DQFvIhICWRzeQ4PlWYe+GkKeBGREjgyXBgDry4aEZGIOVjiMfCggBcRKYkTFznpCF5EJFKmA35Fc/DzwE8LNODNrNnM7jWz58zsWTO7Kcj2RETKVffAOMsbU6QS8ZK1mQh4+18BfuHu7zGzKqB0nU8iImWku4TzwE8L7AjezJqAVwPfBHD3KXcfDKo9EZFy1j1Y2oucINgumrVAD/BtM3vSzO4ys7qTFzKzjWbWYWYdPT09AZYjIhKOTC7PocHJktxoe6YgAz4BXAv8o7tfA4wBnzl5IXff5O7t7t7e2toaYDkiIuE40D9OLu+sXfayY9xABRnw3UC3u28pPr+XQuCLiCwq+/rGAGiLSsC7+xHggJldWnzp9cAzQbUnIlKu9vYUAn5diQM+6FE0fwl8rziCZi/w0YDbExEpO/v6xmiqSbKkrqqk7QYa8O6+A2gPsg0RkXLX1TtW8v530JWsIiKB29c7roAXEYmayUyOg4MTCngRkajZ31e40XapR9CAAl5EJFBdvaNA6UfQgAJeRCRQXb06ghcRiaSu3lFaG1LUp4Ielf5yCngRkQDt6x1n7dLSH72DAl5EJFB7QxoDDwp4EZHAjExm6B1Nh9L/Dgp4EZHA7CueYNURvIhIxHQVZ5FUwIuIRMy+3jHM4MKl4dytVAEvIhKQrt4xLmiqoTpZuhttz6SAFxEJSFizSE5TwIuIBKSrd4y2ZeF0z4ACXkQkEH2jaYYmMrSFdJETKOBFRAKxbf8AAFetag6tBgW8iEgAtnb1U5WIsWF1U2g1KOBFRAKwpaufa1Y3k0qEM4IGFPAiIgtuZDLD7kND3Lhuaah1KOBFRBZYx/4B8g43rm0JtQ4FvIjIAtva1U8iZlyzpjnUOhTwIiILbMvePq5a1URtVelv8jGTAl5EZAFNTOXY1T3EDWvD7X8HmNPXi5nVARPunjezS4DLgAfdPXOG9fYBI0AOyLp7+znWKyJS1ra/OEA276H3v8Pcj+AfAarNbCXwK+BDwD/Pcd3XuvvVCncRWQy2dPUTM7iubUnYpcw54M3dx4F3AV939/cCfxRcWSIilWlrVx9XXNBIY3Uy7FLmHvBmdhPwQeBnxdfmMnrfgV+Z2TYz23iKDW80sw4z6+jp6ZljOSIi5SedzfHki4PcWAb97zD3gL8D+CzwU3ffbWbrgH+bw3qvcvdrgbcA/9HMXn3yAu6+yd3b3b29tbV1rnWLiJSdp7qHSGfzXN8Wfv87zPEkq7v/FvgtgJnFgF53/+Qc1jtY/POYmf0UuIFCf76ISOQ8e3gYgKtWhTf/zExzOoI3s++bWWNxNM3TwDNm9ukzrFNnZg3TPwNvKq4rIhJJe46O0pBKsKKpOuxSgLl30Vzh7sPAO4EHgbUURtKcznLgUTPbCWwFfubuv5hvoSIi5W7P0REuWl6PmYVdCjDHLhogaWZJCgF/p7tnzMxPt4K77wU2nGN9IiIV4/ljo7zx8uVhl3HcXI/gvwHsA+qAR8zsQmA4qKJERCpN72ia/rEpLl5eH3Ypx831JOtXga/OeGm/mb02mJJERCrPnqMjAFyyvCHkSk6Y60nWJjP70vR4dTP7IoWjeRERAZ4/OgpUYMAD36Iwp8z7io9h4NtBFSUiUmn2HB2hoTrB8sZU2KUcN9eTrOvd/d0znn/ezHYEUI+ISEV6/ugolyxvKJsRNDD3I/gJM3vV9BMzuwWYCKYkEZHK4u7sOTbCJWV0ghXmfgT/ceC7ZjZ9edYA8JFgShIRqSw9o2kGxzNcfF759L/D3EfR7AQ2mFlj8fmwmd0B7AqwNhGRilCOJ1jhLO/o5O7DxStaAf4qgHpERCrOiSGS5dVFcy637CufMwkiIiHac3SUppokrQ3lM4IGzi3gTztVgYjIYvH80cIJ1nIaQQNn6IM3sxFmD3IDagKpSESkgrg7e46O8NarLgi7lJc5bcC7e3mdMRARKTPHRtIMT2bLrv8dzq2LRkRk0SvHOWimKeBFRM7BH44UAr6cZpGcpoAXETkHu7qHOL+xmvMayuMuTjMp4EVEzsHO7kE2rC6Pe7CeTAEvIjJP/WNT7O8b5+rVS8IuZVYKeBGRedrZPQigI3gRkajZ8eIgZnDVquawS5mVAl5EZJ52dg9yyXkN1KfmOjFvaSngRUTmwd3ZeaB8T7CCAl5EZF5e7B9nYDxTtidYQQEvIjIvOw4MAuV7ghVKEPBmFjezJ83sgaDbEhEplR0HBqlOxri0DKcomFaKI/j/DDxbgnZEREpmx4FBXrGyiUS8fDtCAq3MzFYBbwXuCrIdEZFSmsrm2X1omKtXN4ddymkF/dXzv4C/AfKnWsDMNppZh5l19PT0BFyOiMi5e+7IMFPZPBsWa8Cb2duAY+6+7XTLufsmd2939/bW1tagyhERWTA7iydYF/MR/C3A281sH/BD4HVmdk+A7YmIlMSTBwZZVl/FyubyvrFdYAHv7p9191Xu3ga8H3jI3W8Lqj0RkVLZ1T3EhlXNZXcP1pOV7+lfEZEyNJrO0tkzWrbzz8xUkgkU3P1h4OFStCUiEqSnuodwh6vK+AKnaTqCFxE5C8enCK6AI3gFvIjIWdjVPcjqlhpa6qrCLuWMFPAiImdh54Ghiuh/BwW8iMic9Y2mOTg4wYZV5d//Dgp4EZE529U9BJTvHZxOpoAXEZmjnd2FW/RduVJH8CIikbLzwCAXn1dftrfoO5kCXkRkDtydXd2Vc4IVFPAiInNycHCCvrGpijnBCgp4EZE5qbQTrKCAFxGZk53dgyTjxmUryvcWfSdTwIuIzMHOA4NcvqKRVCIedilzpoAXETmDyUyO7S8Ocn1bS9ilnBUFvIjIGWzfP8BUNs8tFy0Nu5SzooAXETmDxzp7icdMR/AiIlGzubOPDauaaKhOhl3KWVHAi4icxshkhl3dQ9xy0bKwSzlrCngRkdPYsrefXN65aX1l9b+DAl5E5LQ2d/aRSsS4ds2SsEs5awp4EZHT2NzZS3vbEqqTlTP+fZoCXkTkFHpH0zx3ZISb11de/zso4EVETunxzj6AijzBCgp4EZFT2tzZS0MqwZUXNIZdyrwo4EVETmFzZx83rltKIl6ZURlY1WZWbWZbzWynme02s88H1ZaIyELrHhhnf984N1fg8MhpQd53Kg28zt1HzSwJPGpmD7r77wNsU0RkQWyu8P53CDDg3d2B0eLTZPHhQbUnIrKQHu/sY2ldFZcsrw+7lHkLtGPJzOJmtgM4Bvza3bfMssxGM+sws46enp4gyxERmRN357EXerlp/VLMLOxy5i3QgHf3nLtfDawCbjCzK2dZZpO7t7t7e2tra5DliIjMSWfPGMdG0hXdPQMlGkXj7oPAvwFvLkV7IiLn4vHOXoCKPsEKwY6iaTWz5uLPNcAbgeeCak9EZKFs7uxjZXMNa1pqwy7lnAQ5imYF8B0zi1P4Ivmxuz8QYHsiIucsn3ce39vHGy9fXtH97xDsKJpdwDVBbV9EJAjPHB5mcDzDzRV2e77ZVOblWSIiAZmef6ZSJxibSQEvIjLD5s5e1rfWsbyxOuxSzpkCXkSkKJPLs7WrPxJH76CAFxE57rEXehmbyvGqixXwIiKRct/2gzTVJLn10mhcdKmAFxEBhicz/HL3Ed6+4QJSicq7Pd9sFPAiIsCDTx0mnc3zrmtXhl3KglHAi4gA/7L9IOuW1XH16uawS1kwCngRWfQO9I+ztaufd1+3quKvXp1JAS8ii95PnzwIwDuviU73DCjgRWSRc3fu297NTeuWsrK5JuxyFpQCXkQWta1d/ezrG4/UydVpCngRWdS+/nAnLXVVvPWqFWGXsuCCnC64ZP7bA8+QiBkN1Qkaa5I0VCdoSCVprEnSXJuktT5Fc20yUidPROTc7TwwyG/39PA3b76U2qpIxOFLROITPfjUYfrGpkhn86dcJhk3WutTnNdYzfLGFC11KdKZHCPpLBNTOVKJWOGLoTpJPFb4Isi7k8s7eXeyOccBA2JmxGJgZhQXZSydY2Qyw2g6S3UyTlNNksbqJKlEjFjMMAN3mMrmyeTy5N2Jx4xELEZsxhePU2grk8szlcuTiBnJeIyqRIxEzArLGsTNSMSMeCxGIm7FbRXez+bzZHLOVDaPu5P3wmeZZga5fGHejWwuT95PvG5A3iHnjrvjM9Y1jFis8JnjsRNt5vKQzuaYyhZqzuadXM7J5gvbyBW3kyiuk4zHCuvGjWQsdnybMSu8n0pMf94Y8RjF9418cTs+XY9ZsebC+lbcRqz495JMxEgl4lQnYyTjsZf93cWL7VnhwxGzQtvVyTg1yTjVyTipRIxUMkZ1Ik4spgOEqPnaQ8/TXJvkwze1hV1KICIR8Js/+3qgEDIjk1lGJrMMT2QYmczSPz5F70iantE0x4bTHBuZZF/vONv2D1KdjFGfSlBbFad/LM+eY4V1cnknVgyPuBVCLVEMAqcQ1LkZYeNAXSpOQypJfSpB3+gUXb1jDE1kyGTzxwPWDKqKYW1m5POFEMzlnZnRkUzESMYL4Z/3QlBPZfPkvPBlk3eOr3sm06FnxZ/dC58hblZooxi205/DvRDehcAs/klhX8xsO1f88svmnETMqErESCViJItfRIlicMdjJ9qf/qyZXL74p5PNF/aPT28v78e/KPzMH6+kaqvi1KUS1KcShd8Wq5M01iRYUlvF0roqltRV0VA9/RtkgubaKpbWV7GktoqqhHpDy83TB4f412eP8V/edAn1qUhE4ctE6lOlEnFS9XGW1afCLqVk8nknk8+Tz1P800nGY8WHVWy31PRvHtO/QeXdjx+dn/iimv6SLfw8/UXq+cIXcCaXJ53JM1n87aKwXU58URa/cKa/2NyddC7P5FSO8akc6WyedDbHZCbPRCbHWDrL+FT2+EHEyGSGQ0MTDI5nGBifOu0XUkMqQUt9FS11VbTWp1jeWM35TdWc31jNiqbCzyuaaqipisYl8pXgaw89T2N1gg/f3BZ2KYGJVMAvRrGYkYoVQqGG6IRDoQuF491l5S6Xd4YmMoxMFn4LHJ7MMDieoX9s6mWP/X3jbOnqZ2gi87LtLKuvYtWSWta01NK2tJa1rXW0La1jXWs9TTXJED5ZND1zaJhf7j7KHW+4mMbq6O5XBbzIAojHjJa6whH6XE1M5TgyPMnhoQmODE1yaHCCA/0TdA+O8+SBAR7YdYiZvXCtDSnWt9Zx6fIGLl/RyOUrGrn0/Aaqk9H5Yi+FofEMf/mD7TTXJvnozWvDLidQCniRkNRUxVm7rI61y+pmfT+dzXGgf5y9PWPs7R2j89goL/SMcu+2bsamcgDEDNYuq+OyFY1csaKRV6xs4hUrm1hyFl80i8lUNs/H79nGi/3j3H37jTTVRvfoHRTwImUrlYhz0XkNXHRew0tez+edAwPjPHt4mGcOj/Dc4WF2dQ/ys12Hjy+zuqWG69Ysob2thfa2JVx8XkPFdHcFxd35zH27eHxvH1/+0w28cl3l31T7TBTwIhUmFjMuXFrHhUvrePOVJy7OGZrIsPvgELsODrHzwCCPdfbxf3YcAgojgK68oImrVjVx3YWF4G9tWDyDEQC+8pvnuW/7QT71hkv499esCruckjAvo7Fo7e3t3tHREXYZIpHg7hzon6Bjfz+7uofY2T3I7kPDx0cUrVtWxw1rW7hp/VJuWreU8+Z6k+nOTvjiF+Gee2B0FOrr4bbb4K//GtavD/ATzd/3t7zI3/70Kd597Sr+4b1XVezostmY2TZ3b5/1PQW8yOIxlc3z9KEhnujq54l9/Wzp6mdkMgsUAv/GdUt55boWXrluKctnC/wHH4T3vAcymcJjWjJZeNx7L7zlLSX6NHPzy91H+MQ923jNJa1s+nA7yXi0rkkIJeDNbDXwXWA5hWtrNrn7V063jgJepLRyeWf3oSEe7+xjS1c/T3T1M5IuBP5l5zfwmktbec3FrVx74RKqX9wHV10F4+On3mBtLezaVTZH8lu7+rntm1u4YkUj3/+LGyM5HUFYAb8CWOHu282sAdgGvNPdnznVOgp4kXDl8s4zh4Z59IVeHtnTQ8f+fjI5J5WIceejd/G6R+8nnsueegPJJGzcCHfeWbqiT+HR53vZeHcH5zdVc+/Hbz6rIayVpCy6aMzs/wJ3uvuvT7WMAl6kvIyls/x+bx+bO/v4q/dcT136NEfv0xobYWgo+OJO4+dPHeaOH+5g7bI6vnv7DbN3N0VE6AFvZm3AI8CV7j580nsbgY0Aa9asuW7//v2B1yMi8xCLMZcJgjwWw3K5EhT0ctlcnnt+v5/PP/AM165Zwrc+cn3kx7qfLuAD75Ays3rgX4A7Tg53AHffBGyCwhF80PWIyDzV18PIyBkXG0tW83c/2kF7WwvXty1hfWt94DNxDk9m+NHWA/zz5n0cHJzgtZe28vUPXrfo5/YJNODNLEkh3L/n7vcF2ZaIBOy22+Cuu146euYkuUSCbX/8Vh55vof7ivc5ba5Ncn1bCzeubeHGtUu5fEUDiQUaydLZM8p3N+87fnXvjWtb+Ls/uYI3XL580V/YBcGeZDXgO0C/u98xl3XUBy9Sxjo75zyKxtetY3/fOE/s62drVz9b9/Wzv6+wXn0qwbUXLuGGtiVcdF4DbcsKk6vNZYTL4PgUuw8Ns/vQEI++0Mcje3qoisd424YVfOyWtVy5smmhPm3FCGsUzauA3wFPAdN34vhbd//5qdZRwIuUuXMYB39kaJItXX3HQ3/P0dGXvN9SV8WK4rTJqWSMdKYwXfPwZJb+sTT9o1PH5+ABWLWkhve1r+YDN6xZdFflzhT6Sda5UsCLVIDOTvjyl+Huu09cyfqhD8GnPnVW49+HJzPs7x1nX98YL/aP0z0wweGhCQ4PTpLJ5Ukl48dvytNSV8XSuhTLG1NccUEjf3RBU2SHPZ4tBbyISESdLuCjdc2uiIgcp4AXEYkoBbyISEQp4EVEIkoBLyISUQp4EZGIUsCLiESUAl5EJKLK6kInM+sBojhf8DKgN+wiKpj23/xp381fpey7C929dbY3yirgo8rMOk51pZmcmfbf/GnfzV8U9p26aEREIkoBLyISUQr40tgUdgEVTvtv/rTv5q/i95364EVEIkpH8CIiEaWAFxGJKAW8iEhEKeBDZmZ/bGb/ZGZ3mdnmsOupJGZ2q5n9rrj/bg27nkpjZpcX9929ZvaJsOupJGa2zsy+aWb3hl3L6Sjgz4GZfcvMjpnZ0ye9/mYz+4OZvWBmnzndNtz9d+7+ceAB4DtB1ltOFmLfAQ6MAtVAd1C1lqMF+rf3bPHf3vuAW4Kst5ws0L7b6+63B1vpudMomnNgZq+mEDDfdfcri6/FgT3AGymEzhPAB4A48IWTNvExdz9WXO/HwO3uPlKi8kO1EPsO6HX3vJktB77k7h8sVf1hW6h/e2b2duATwN3u/v1S1R+mBf5/e6+7v6dUtZ+tRNgFVDJ3f8TM2k56+QbgBXffC2BmPwTe4e5fAN4223bMbA0wtFjCHRZu3xUNAKlACi1TC7X/3P1+4H4z+xmwKAJ+gf/tlTV10Sy8lcCBGc+7i6+dzu3AtwOrqHKc1b4zs3eZ2TeAu4E7A66tEpzt/rvVzL5a3Ic/D7q4Mne2+26pmf0TcI2ZfTbo4uZLR/BlwN3/PuwaKpG73wfcF3YdlcrdHwYeDrmMiuTufcDHw67jTHQEv/AOAqtnPF9VfE3OTPvu3Gj/zV8k950CfuE9AVxsZmvNrAp4P3B/yDVVCu27c6P9N3+R3HcK+HNgZj8AHgcuNbNuM7vd3bPAfwJ+CTwL/Njdd4dZZznSvjs32n/zt5j2nYZJiohElI7gRUQiSgEvIhJRCngRkYhSwIuIRJQCXkQkohTwIiIRpYCXsmdmoyVur6Tz8ptZs5n9h1K2KYuDAl4WHTM77RxM7n5zidtsBhTwsuAU8FKRzGy9mf3CzLYV7+p0WfH1PzGzLWb2pJn9a3GueMzsc2Z2t5k9BtxdfP4tM3vYzPaa2SdnbHu0+OetxffvNbPnzOx7ZmbF9/5d8bVtxRkZH5ilxj8zs/vN7CHgN2ZWb2a/MbPtZvaUmb2juOj/ANab2Q4z+5/FdT9tZk+Y2S4z+3yQ+1IizN310KOsH8DoLK/9Bri4+PONwEPFn5dw4grtPwe+WPz5c8A2oGbG880U5pFfBvQByZntAbcCQxQmnopRuLz9VRTuIHUAWFtc7gfAA7PU+GcUpp1tKT5PAI3Fn5cBLwAGtAFPz1jvTcCm4nsxCnf7enXYfw96VN5D0wVLxTGzeuBm4CfFA2o4ccOPVcCPzGwFUAV0zVj1fnefmPH8Z+6eBtJmdgxYzstv/bfV3buL7e6gEMajwF53n972D4CNpyj31+7eP1068N+LdxTKU5hvfPks67yp+Hiy+LweuBh45BRtiMxKAS+VKAYMuvvVs7z3NQq377vfCjfi/tyM98ZOWjY94+ccs/9/mMsypzOzzQ8CrcB17p4xs30Ufhs4mQFfcPdvnGVbIi+hPnipOO4+DHSZ2XsBrGBD8e0mTszj/ZGASvgDsG7Gbd/+dI7rNQHHiuH+WuDC4usjQMOM5X4JfKz4mwpmttLMzjv3smWx0RG8VIJaM5vZdfIlCkfD/2hm/xVIAj8EdlI4Yv+JmQ0ADwFrF7oYd58oDmv8hZmNUZhLfC6+B/w/M3sK6ACeK26vz8weM7OngQfd/dNmdjnweLELahS4DTi20J9Fok3TBYvMg5nVu/tocVTN/waed/cvh12XyEzqohGZn78onnTdTaHrRf3lUnZ0BC8iElE6ghcRiSgFvIhIRCngRUQiSgEvIhJRCngRkYhSwIuIRNT/B+6MuNbV2nE9AAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CLvAVjDRGY9e",
        "colab_type": "code",
        "colab": {},
        "outputId": "d061a1eb-de50-42a9-e63d-e862b3f44f04"
      },
      "source": [
        "hparams_cls = Namespace(\n",
        "    lr=0.003981071705534969,\n",
        "    epochs=5,\n",
        "    batch_size=160,\n",
        "    n_classes=10,\n",
        "    freeze_base=True,\n",
        "    embeddings_path=\"./efficientnet-b0-stl10-embeddings.ckpt\",\n",
        "    hidden_size=512\n",
        ")\n",
        "module = SimCLRClassifierModule(hparams_cls)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Loaded pretrained weights for efficientnet-b0\n",
            "Freezing embeddings\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pr0fPPunGY9g",
        "colab_type": "code",
        "colab": {
          "referenced_widgets": [
            "",
            "4030a285e27945cba4e9c1bfa12e4e25"
          ]
        },
        "outputId": "1d7aabc9-13da-48c4-873d-7bde1f02f431"
      },
      "source": [
        "trainer.fit(module)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "  | Name  | Type             | Params\n",
            "-------------------------------------------\n",
            "0 | model | SimCLRClassifier | 4 M   \n",
            "1 | loss  | CrossEntropyLoss | 0     \n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validation sanity check', layout=Layout…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4030a285e27945cba4e9c1bfa12e4e25",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Training', layout=Layout(flex='2'), max…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 228
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VvejFEk9GY9j",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.metrics import classification_report"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_hMSIaArGY9l",
        "colab_type": "text"
      },
      "source": [
        "### Evaluation\n",
        "Here I define utility function for evaluating the model using provided data loader:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sHUngTyiGY9l",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def evaluate(data_loader, module):\n",
        "    with torch.no_grad():\n",
        "        progress = [\"/\", \"-\", \"\\\\\", \"|\", \"/\", \"-\", \"\\\\\", \"|\"]\n",
        "        module.eval().cuda()\n",
        "        true_y, pred_y = [], []\n",
        "        for i, batch_ in enumerate(data_loader):\n",
        "            X, y = batch_\n",
        "            print(progress[i % len(progress)], end=\"\\r\")\n",
        "            y_pred = torch.argmax(module(X.cuda()), dim=1)\n",
        "            true_y.extend(y.cpu())\n",
        "            pred_y.extend(y_pred.cpu())\n",
        "        print(classification_report(true_y, pred_y, digits=3))\n",
        "        return true_y, pred_y"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FLyle_SGGY9n",
        "colab_type": "code",
        "colab": {},
        "outputId": "e5bcf38c-afcf-456b-9128-5e8e258444c2"
      },
      "source": [
        "_ = evaluate(module.val_dataloader(), module)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0      0.856     0.864     0.860       800\n",
            "           1      0.714     0.701     0.707       800\n",
            "           2      0.903     0.919     0.911       800\n",
            "           3      0.678     0.599     0.636       800\n",
            "           4      0.665     0.746     0.703       800\n",
            "           5      0.633     0.564     0.597       800\n",
            "           6      0.729     0.781     0.754       800\n",
            "           7      0.678     0.709     0.693       800\n",
            "           8      0.868     0.910     0.888       800\n",
            "           9      0.862     0.801     0.830       800\n",
            "\n",
            "    accuracy                          0.759      8000\n",
            "   macro avg      0.759     0.759     0.758      8000\n",
            "weighted avg      0.759     0.759     0.758      8000\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uNbRlDOEGY9q",
        "colab_type": "text"
      },
      "source": [
        "## Summary\n",
        "I hope you find my explanation of SimCLR framework useful. Feel free to post any comment, fork the notebook and share with ML community!\n",
        "\n",
        "## Additional links and resources\n",
        "* SimCLR paper - <a href=\"https://arxiv.org/pdf/2002.05709.pdf\" target=\"blank\">https://arxiv.org/pdf/2002.05709.pdf</a>\n",
        "* EfficientNet paper - <a href=\"https://arxiv.org/abs/1905.11946\">https://arxiv.org/abs/1905.11946</a>\n",
        "* this notebook on GitHub - \n",
        "* this notebook on Google Colab - \n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RImDfAeBGY9q",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}