Art_of_RAG_Evaluation_LangChain_&_Ragas.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "c4bc64d47f2e4a239cf7156e7812887d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b4a71cf676584200b46922fb976d1d50",
              "IPY_MODEL_65f75f3d72cd415faffb32999893738d",
              "IPY_MODEL_5367cbe7ca4d43b089ff0d5d7cd17d3c"
            ],
            "layout": "IPY_MODEL_31390facfbeb455fa83067fc23d30718"
          }
        },
        "b4a71cf676584200b46922fb976d1d50": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_22fc25617c664e4aa02b7457832c1bef",
            "placeholder": "​",
            "style": "IPY_MODEL_b556054df18d47aa8cb58ac482ca31bb",
            "value": "Creating CSV from Arrow format: 100%"
          }
        },
        "65f75f3d72cd415faffb32999893738d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0fff4a6672c848a38945e19101c46168",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_de662cb67d054f08a15d191923d40369",
            "value": 1
          }
        },
        "5367cbe7ca4d43b089ff0d5d7cd17d3c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b53ebcf672a244978391cae559db5bf2",
            "placeholder": "​",
            "style": "IPY_MODEL_f2de6a2c1f4b4a2fa5196028c0da0754",
            "value": " 1/1 [00:00&lt;00:00, 18.79ba/s]"
          }
        },
        "31390facfbeb455fa83067fc23d30718": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "22fc25617c664e4aa02b7457832c1bef": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b556054df18d47aa8cb58ac482ca31bb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0fff4a6672c848a38945e19101c46168": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "de662cb67d054f08a15d191923d40369": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b53ebcf672a244978391cae559db5bf2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f2de6a2c1f4b4a2fa5196028c0da0754": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "86761a36bdb04fed9f1dae6e74da54ce": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_e079c0eb3d9c4ae3a40491f059a864f7",
              "IPY_MODEL_d4bde97f3895450782030cad7808ea59",
              "IPY_MODEL_1d17b8847ab34a1d8fc79430e8b64a12"
            ],
            "layout": "IPY_MODEL_fb953d8d116e4cc389dbc23a0055873f"
          }
        },
        "e079c0eb3d9c4ae3a40491f059a864f7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bf27f78edb15477492fb2d5dd8dc5137",
            "placeholder": "​",
            "style": "IPY_MODEL_ee96a569b8a54944bcac1b1bc164e53e",
            "value": "Creating CSV from Arrow format: 100%"
          }
        },
        "d4bde97f3895450782030cad7808ea59": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e556e254882340168833ecf1a7153a90",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c077837b8b044c2fb14dcd8fbc44a37b",
            "value": 1
          }
        },
        "1d17b8847ab34a1d8fc79430e8b64a12": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e5c8b5933c754a1192456f43682276c5",
            "placeholder": "​",
            "style": "IPY_MODEL_84abf77081a04bb686f794b49e04761d",
            "value": " 1/1 [00:00&lt;00:00, 35.79ba/s]"
          }
        },
        "fb953d8d116e4cc389dbc23a0055873f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bf27f78edb15477492fb2d5dd8dc5137": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ee96a569b8a54944bcac1b1bc164e53e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e556e254882340168833ecf1a7153a90": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c077837b8b044c2fb14dcd8fbc44a37b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "e5c8b5933c754a1192456f43682276c5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "84abf77081a04bb686f794b49e04761d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bf7f045bdbe24360ad2aa2f4c8f02e79": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_761e3c6035bf49429b3035145451d2df",
              "IPY_MODEL_ffb7c97e7af648aaa13a43427154140e",
              "IPY_MODEL_95d92c5c74e845779337eb727c2bbfc0"
            ],
            "layout": "IPY_MODEL_6e7fb9a1d1454fcd9bcf5b5f748fb975"
          }
        },
        "761e3c6035bf49429b3035145451d2df": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_207785da1f404d6ea0e8c63655a7aa51",
            "placeholder": "​",
            "style": "IPY_MODEL_71237d176b2c4138a5e0346d10482257",
            "value": "Creating CSV from Arrow format: 100%"
          }
        },
        "ffb7c97e7af648aaa13a43427154140e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ed457547dc154f6bbce4ec970bb09c76",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c9f479f81119450bb451d5830361467c",
            "value": 1
          }
        },
        "95d92c5c74e845779337eb727c2bbfc0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_56098a347ea94ea3b10bd8d2ec0d4288",
            "placeholder": "​",
            "style": "IPY_MODEL_7940d9e3f5fa4592b58dec3fdb55595a",
            "value": " 1/1 [00:00&lt;00:00, 33.86ba/s]"
          }
        },
        "6e7fb9a1d1454fcd9bcf5b5f748fb975": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "207785da1f404d6ea0e8c63655a7aa51": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "71237d176b2c4138a5e0346d10482257": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ed457547dc154f6bbce4ec970bb09c76": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c9f479f81119450bb451d5830361467c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "56098a347ea94ea3b10bd8d2ec0d4288": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7940d9e3f5fa4592b58dec3fdb55595a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "632135599e39470aac1a3bb3d3de0ca4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_567176b50d074a1c9d384a8df8c3ff4c",
              "IPY_MODEL_6bfa39f2c84e4f08b5ffb9a416398824",
              "IPY_MODEL_911b40169865413b98643789150e5495"
            ],
            "layout": "IPY_MODEL_2add9d30edd84152bb6d7bfa4ed2d910"
          }
        },
        "567176b50d074a1c9d384a8df8c3ff4c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f34ffd11b98045b9bae326dd48a54896",
            "placeholder": "​",
            "style": "IPY_MODEL_e3e19fc9963c4bf39293bda7c2030d5a",
            "value": "Creating CSV from Arrow format: 100%"
          }
        },
        "6bfa39f2c84e4f08b5ffb9a416398824": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c15c27b0523a429e9d77f439d47ada90",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_256a0dbb2f104d928e181d2a882a9867",
            "value": 1
          }
        },
        "911b40169865413b98643789150e5495": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_68790cfe4ea14fc4a63275f3e99c468f",
            "placeholder": "​",
            "style": "IPY_MODEL_e7df092ac205443e8baa3646ff1eae5b",
            "value": " 1/1 [00:00&lt;00:00, 33.41ba/s]"
          }
        },
        "2add9d30edd84152bb6d7bfa4ed2d910": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f34ffd11b98045b9bae326dd48a54896": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e3e19fc9963c4bf39293bda7c2030d5a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c15c27b0523a429e9d77f439d47ada90": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "256a0dbb2f104d928e181d2a882a9867": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "68790cfe4ea14fc4a63275f3e99c468f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e7df092ac205443e8baa3646ff1eae5b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Evaluation with RAGAS and Advanced Retrieval Methods Using LangChain\n",
        "\n",
        "In the following notebook we'll discuss a major component of LLM Ops:\n",
        "\n",
        "- Evaluation\n",
        "\n",
        "We're going to be leveraging the [RAGAS]() framework for our evaluations today as it's becoming a standard method of evaluating (at least directionally) RAG systems.\n",
        "\n",
        "We're also going to discuss a few more powerful Retrieval Systems that can potentially improve the quality of our generations!\n",
        "\n",
        "Let's start as we always do: Grabbing our dependencies!"
      ],
      "metadata": {
        "id": "wa8ykQk92aLX"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5BN13TZlSCv4"
      },
      "outputs": [],
      "source": [
        "!pip install -U -q langchain openai ragas arxiv pymupdf chromadb wandb tiktoken"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "import openai\n",
        "from getpass import getpass\n",
        "\n",
        "openai.api_key = getpass(\"Please provide your OpenAI Key: \")\n",
        "os.environ[\"OPENAI_API_KEY\"] = openai.api_key"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8Lhqp5rUThG-",
        "outputId": "c33f7eee-b819-40bd-dc75-ce90721a6a94"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Please provide your OpenAI Key: ··········\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Data Collection\n",
        "\n",
        "We're going to be using papers from Arxiv as our context today.\n",
        "\n",
        "We can collect these documents rather straightforwardly with the `ArxivLoader` document loader from LangChain.\n",
        "\n",
        "Let's grab and load 5 documents.\n",
        "\n",
        "- [`ArxivLoader`](https://api.python.langchain.com/en/latest/document_loaders/langchain.document_loaders.arxiv.ArxivLoader.html)"
      ],
      "metadata": {
        "id": "DV_BOewX8CW0"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.document_loaders import ArxivLoader\n",
        "\n",
        "base_docs = ArxivLoader(query=\"Retrieval Augmented Generation\", load_max_docs=5).load()\n",
        "len(base_docs)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DTDNFXaBSO2j",
        "outputId": "3b24521d-5c6f-466b-d818-46ce68d359ee"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "5"
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for doc in base_docs:\n",
        "  print(doc.metadata)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nNPAWPgNSyGP",
        "outputId": "b2f80fc8-792c-489a-b8d4-9f98678c679a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'Published': '2022-02-13', 'Title': 'A Survey on Retrieval-Augmented Text Generation', 'Authors': 'Huayang Li, Yixuan Su, Deng Cai, Yan Wang, Lemao Liu', 'Summary': 'Recently, retrieval-augmented text generation attracted increasing attention\\nof the computational linguistics community. Compared with conventional\\ngeneration models, retrieval-augmented text generation has remarkable\\nadvantages and particularly has achieved state-of-the-art performance in many\\nNLP tasks. This paper aims to conduct a survey about retrieval-augmented text\\ngeneration. It firstly highlights the generic paradigm of retrieval-augmented\\ngeneration, and then it reviews notable approaches according to different tasks\\nincluding dialogue response generation, machine translation, and other\\ngeneration tasks. Finally, it points out some important directions on top of\\nrecent methods to facilitate future research.'}\n",
            "{'Published': '2023-08-08', 'Title': 'Hybrid Retrieval-Augmented Generation for Real-time Composition Assistance', 'Authors': 'Xuchao Zhang, Menglin Xia, Camille Couturier, Guoqing Zheng, Saravan Rajmohan, Victor Ruhle', 'Summary': \"Retrieval augmented models show promise in enhancing traditional language\\nmodels by improving their contextual understanding, integrating private data,\\nand reducing hallucination. However, the processing time required for retrieval\\naugmented large language models poses a challenge when applying them to tasks\\nthat require real-time responses, such as composition assistance.\\n  To overcome this limitation, we propose the Hybrid Retrieval-Augmented\\nGeneration (HybridRAG) framework that leverages a hybrid setting that combines\\nboth client and cloud models. HybridRAG incorporates retrieval-augmented memory\\ngenerated asynchronously by a Large Language Model (LLM) in the cloud. By\\nintegrating this retrieval augmented memory, the client model acquires the\\ncapability to generate highly effective responses, benefiting from the LLM's\\ncapabilities. Furthermore, through asynchronous memory integration, the client\\nmodel is capable of delivering real-time responses to user requests without the\\nneed to wait for memory synchronization from the cloud. Our experiments on\\nWikitext and Pile subsets show that HybridRAG achieves lower latency than a\\ncloud-based retrieval-augmented LLM, while outperforming client-only models in\\nutility.\"}\n",
            "{'Published': '2023-05-27', 'Title': 'Augmentation-Adapted Retriever Improves Generalization of Language Models as Generic Plug-In', 'Authors': 'Zichun Yu, Chenyan Xiong, Shi Yu, Zhiyuan Liu', 'Summary': \"Retrieval augmentation can aid language models (LMs) in knowledge-intensive\\ntasks by supplying them with external information. Prior works on retrieval\\naugmentation usually jointly fine-tune the retriever and the LM, making them\\nclosely coupled. In this paper, we explore the scheme of generic retrieval\\nplug-in: the retriever is to assist target LMs that may not be known beforehand\\nor are unable to be fine-tuned together. To retrieve useful documents for\\nunseen target LMs, we propose augmentation-adapted retriever (AAR), which\\nlearns LM's preferences obtained from a known source LM. Experiments on the\\nMMLU and PopQA datasets demonstrate that our AAR trained with a small source LM\\nis able to significantly improve the zero-shot generalization of larger target\\nLMs ranging from 250M Flan-T5 to 175B InstructGPT. Further analysis indicates\\nthat the preferences of different LMs overlap, enabling AAR trained with a\\nsingle source LM to serve as a generic plug-in for various target LMs. Our code\\nis open-sourced at https://github.com/OpenMatch/Augmentation-Adapted-Retriever.\"}\n",
            "{'Published': '2023-02-07', 'Title': 'Augmenting Zero-Shot Dense Retrievers with Plug-in Mixture-of-Memories', 'Authors': 'Suyu Ge, Chenyan Xiong, Corby Rosset, Arnold Overwijk, Jiawei Han, Paul Bennett', 'Summary': 'In this paper we improve the zero-shot generalization ability of language\\nmodels via Mixture-Of-Memory Augmentation (MoMA), a mechanism that retrieves\\naugmentation documents from multiple information corpora (\"external memories\"),\\nwith the option to \"plug in\" new memory at inference time. We develop a joint\\nlearning mechanism that trains the augmentation component with latent labels\\nderived from the end retrieval task, paired with hard negatives from the memory\\nmixture. We instantiate the model in a zero-shot dense retrieval setting by\\naugmenting a strong T5-based retriever with MoMA. Our model, MoMA, obtains\\nstrong zero-shot retrieval accuracy on the eighteen tasks included in the\\nstandard BEIR benchmark. It outperforms systems that seek generalization from\\nincreased model parameters and computation steps. Our analysis further\\nillustrates the necessity of augmenting with mixture-of-memory for robust\\ngeneralization, the benefits of augmentation learning, and how MoMA utilizes\\nthe plug-in memory at inference time without changing its parameters. We plan\\nto open source our code.'}\n",
            "{'Published': '2023-10-17', 'Title': 'Self-RAG: Learning to Retrieve, Generate, and Critique through Self-Reflection', 'Authors': 'Akari Asai, Zeqiu Wu, Yizhong Wang, Avirup Sil, Hannaneh Hajishirzi', 'Summary': \"Despite their remarkable capabilities, large language models (LLMs) often\\nproduce responses containing factual inaccuracies due to their sole reliance on\\nthe parametric knowledge they encapsulate. Retrieval-Augmented Generation\\n(RAG), an ad hoc approach that augments LMs with retrieval of relevant\\nknowledge, decreases such issues. However, indiscriminately retrieving and\\nincorporating a fixed number of retrieved passages, regardless of whether\\nretrieval is necessary, or passages are relevant, diminishes LM versatility or\\ncan lead to unhelpful response generation. We introduce a new framework called\\nSelf-Reflective Retrieval-Augmented Generation (Self-RAG) that enhances an LM's\\nquality and factuality through retrieval and self-reflection. Our framework\\ntrains a single arbitrary LM that adaptively retrieves passages on-demand, and\\ngenerates and reflects on retrieved passages and its own generations using\\nspecial tokens, called reflection tokens. Generating reflection tokens makes\\nthe LM controllable during the inference phase, enabling it to tailor its\\nbehavior to diverse task requirements. Experiments show that Self-RAG (7B and\\n13B parameters) significantly outperforms state-of-the-art LLMs and\\nretrieval-augmented models on a diverse set of tasks. Specifically, Self-RAG\\noutperforms ChatGPT and retrieval-augmented Llama2-chat on Open-domain QA,\\nreasoning and fact verification tasks, and it shows significant gains in\\nimproving factuality and citation accuracy for long-form generations relative\\nto these models.\"}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Creating an Index\n",
        "\n",
        "Let's use a naive index creation strategy of just using `RecursiveCharacterTextSplitter` on our documents and embedding each into our `VectorStore` using `OpenAIEmbeddings()`.\n",
        "\n",
        "- [`RecursiveCharacterTextSplitter()`](https://api.python.langchain.com/en/latest/text_splitter/langchain.text_splitter.RecursiveCharacterTextSplitter.html)\n",
        "- [`Chroma`](https://api.python.langchain.com/en/latest/vectorstores/langchain.vectorstores.chroma.Chroma.html?highlight=chroma#langchain.vectorstores.chroma.Chroma)\n",
        "- [`OpenAIEmbeddings()`](https://api.python.langchain.com/en/latest/embeddings/langchain.embeddings.openai.OpenAIEmbeddings.html?highlight=openaiembeddings#langchain-embeddings-openai-openaiembeddings)"
      ],
      "metadata": {
        "id": "Z7ht6bJX9PAY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.vectorstores import Chroma\n",
        "from langchain.embeddings import OpenAIEmbeddings\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "\n",
        "text_splitter = RecursiveCharacterTextSplitter(chunk_size=250)\n",
        "\n",
        "docs = text_splitter.split_documents(base_docs)\n",
        "\n",
        "vectorstore = Chroma.from_documents(docs, OpenAIEmbeddings())"
      ],
      "metadata": {
        "id": "xne8P5dQTUiR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "len(docs)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cnRzYx4c_2mZ",
        "outputId": "59d9bdd8-0414-4e8b-c285-bf3a2760e26a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "5382"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(max([len(chunk.page_content) for chunk in docs]))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WyUh8EVI_6TZ",
        "outputId": "643fca9d-77c0-4296-d953-ec62d6de8954"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "249\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Let's convert our `Chroma` vectorstore into a retriever with the `.as_retriever()` method."
      ],
      "metadata": {
        "id": "0f9kNIUUTxdT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "base_retriever = vectorstore.as_retriever(search_kwargs={\"k\" : 2})"
      ],
      "metadata": {
        "id": "bwbdftltT29h"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now to give it a test!"
      ],
      "metadata": {
        "id": "DBPZQUt4UBPl"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "relevant_docs = base_retriever.get_relevant_documents(\"What is Retrieval Augmented Generation?\")"
      ],
      "metadata": {
        "id": "r0Pie4xqUCkW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "len(relevant_docs)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Z_CiHTD0UKj7",
        "outputId": "fab040cf-971f-440a-a6aa-93873c8e152f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "2"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Creating a Retrieval Augmented Generation Prompt\n",
        "\n",
        "Now we can set up a prompt template that will be used to provide the LLM with the necessary contexts, user query, and instructions!"
      ],
      "metadata": {
        "id": "D8MKsT6JTgCU"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.prompts import ChatPromptTemplate\n",
        "\n",
        "template = \"\"\"Answer the question based only on the following context. If you cannot answer the question with the context, please respond with 'I don't know':\n",
        "\n",
        "### CONTEXT\n",
        "{context}\n",
        "\n",
        "### QUESTION\n",
        "Question: {question}\n",
        "\"\"\"\n",
        "\n",
        "prompt = ChatPromptTemplate.from_template(template)"
      ],
      "metadata": {
        "id": "ijSNkTAjTsep"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Setting Up our Basic QA Chain\n",
        "\n",
        "Now we can instantiate our basic RAG chain!\n",
        "\n",
        "We'll follow *exactly* the chain we made on Tuesday to keep things simple for now - if you need a refresher on what it looked like - check out last week's notebook!"
      ],
      "metadata": {
        "id": "BYHnPaXl-cvJ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from operator import itemgetter\n",
        "\n",
        "from langchain.chat_models import ChatOpenAI\n",
        "from langchain.schema.output_parser import StrOutputParser\n",
        "from langchain.schema.runnable import RunnableLambda, RunnablePassthrough\n",
        "\n",
        "primary_qa_llm = ChatOpenAI(model_name=\"gpt-3.5-turbo\", temperature=0)\n",
        "\n",
        "retrieval_augmented_qa_chain = (\n",
        "    # INVOKE CHAIN WITH: {\"question\" : \"<<SOME USER QUESTION>>\"}\n",
        "    # \"question\" : populated by getting the value of the \"question\" key\n",
        "    # \"context\"  : populated by getting the value of the \"question\" key and chaining it into the base_retriever\n",
        "    {\"context\": itemgetter(\"question\") | base_retriever, \"question\": itemgetter(\"question\")}\n",
        "    # \"context\"  : is assigned to a RunnablePassthrough object (will not be called or considered in the next step)\n",
        "    #              by getting the value of the \"context\" key from the previous step\n",
        "    | RunnablePassthrough.assign(context=itemgetter(\"context\"))\n",
        "    # \"response\" : the \"context\" and \"question\" values are used to format our prompt object and then piped\n",
        "    #              into the LLM and stored in a key called \"response\"\n",
        "    # \"context\"  : populated by getting the value of the \"context\" key from the previous step\n",
        "    | {\"response\": prompt | primary_qa_llm, \"context\": itemgetter(\"context\")}\n",
        ")"
      ],
      "metadata": {
        "id": "-TsjUWjbUfbW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Let's test it out!"
      ],
      "metadata": {
        "id": "zO69de-F-oMD"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "question = \"What is RAG?\"\n",
        "\n",
        "result = retrieval_augmented_qa_chain.invoke({\"question\" : question})\n",
        "\n",
        "print(result)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2FS5NxC6UyU2",
        "outputId": "2520bf73-9e62-435c-a213-c26d0655a913"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'response': AIMessage(content='RAG stands for Retrieval-Augmented Generation.'), 'context': [Document(page_content='RAG (7B and 13B parameters) significantly outperforms state-of-the-art LLMs\\nand retrieval-augmented models on a diverse set of tasks. Specifically, SELF-RAG\\noutperforms ChatGPT and retrieval-augmented Llama2-chat on Open-domain QA,', metadata={'Authors': 'Akari Asai, Zeqiu Wu, Yizhong Wang, Avirup Sil, Hannaneh Hajishirzi', 'Published': '2023-10-17', 'Summary': \"Despite their remarkable capabilities, large language models (LLMs) often\\nproduce responses containing factual inaccuracies due to their sole reliance on\\nthe parametric knowledge they encapsulate. Retrieval-Augmented Generation\\n(RAG), an ad hoc approach that augments LMs with retrieval of relevant\\nknowledge, decreases such issues. However, indiscriminately retrieving and\\nincorporating a fixed number of retrieved passages, regardless of whether\\nretrieval is necessary, or passages are relevant, diminishes LM versatility or\\ncan lead to unhelpful response generation. We introduce a new framework called\\nSelf-Reflective Retrieval-Augmented Generation (Self-RAG) that enhances an LM's\\nquality and factuality through retrieval and self-reflection. Our framework\\ntrains a single arbitrary LM that adaptively retrieves passages on-demand, and\\ngenerates and reflects on retrieved passages and its own generations using\\nspecial tokens, called reflection tokens. Generating reflection tokens makes\\nthe LM controllable during the inference phase, enabling it to tailor its\\nbehavior to diverse task requirements. Experiments show that Self-RAG (7B and\\n13B parameters) significantly outperforms state-of-the-art LLMs and\\nretrieval-augmented models on a diverse set of tasks. Specifically, Self-RAG\\noutperforms ChatGPT and retrieval-augmented Llama2-chat on Open-domain QA,\\nreasoning and fact verification tasks, and it shows significant gains in\\nimproving factuality and citation accuracy for long-form generations relative\\nto these models.\", 'Title': 'Self-RAG: Learning to Retrieve, Generate, and Critique through Self-Reflection'}), Document(page_content='ric knowledge they encapsulate. Retrieval-Augmented Generation (RAG), an ad\\nhoc approach that augments LMs with retrieval of relevant knowledge, decreases\\nsuch issues. However, indiscriminately retrieving and incorporating a fixed number', metadata={'Authors': 'Akari Asai, Zeqiu Wu, Yizhong Wang, Avirup Sil, Hannaneh Hajishirzi', 'Published': '2023-10-17', 'Summary': \"Despite their remarkable capabilities, large language models (LLMs) often\\nproduce responses containing factual inaccuracies due to their sole reliance on\\nthe parametric knowledge they encapsulate. Retrieval-Augmented Generation\\n(RAG), an ad hoc approach that augments LMs with retrieval of relevant\\nknowledge, decreases such issues. However, indiscriminately retrieving and\\nincorporating a fixed number of retrieved passages, regardless of whether\\nretrieval is necessary, or passages are relevant, diminishes LM versatility or\\ncan lead to unhelpful response generation. We introduce a new framework called\\nSelf-Reflective Retrieval-Augmented Generation (Self-RAG) that enhances an LM's\\nquality and factuality through retrieval and self-reflection. Our framework\\ntrains a single arbitrary LM that adaptively retrieves passages on-demand, and\\ngenerates and reflects on retrieved passages and its own generations using\\nspecial tokens, called reflection tokens. Generating reflection tokens makes\\nthe LM controllable during the inference phase, enabling it to tailor its\\nbehavior to diverse task requirements. Experiments show that Self-RAG (7B and\\n13B parameters) significantly outperforms state-of-the-art LLMs and\\nretrieval-augmented models on a diverse set of tasks. Specifically, Self-RAG\\noutperforms ChatGPT and retrieval-augmented Llama2-chat on Open-domain QA,\\nreasoning and fact verification tasks, and it shows significant gains in\\nimproving factuality and citation accuracy for long-form generations relative\\nto these models.\", 'Title': 'Self-RAG: Learning to Retrieve, Generate, and Critique through Self-Reflection'})]}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Ground Truth Dataset Creation Using GPT-3.5-turbo and GPT-4\n",
        "\n",
        "The next section might take you a long time to run, so the evaluation dataset is provided.\n",
        "\n",
        "The basic idea is that we can use LangChain to create questions based on our contexts, and then answer those questions.\n",
        "\n",
        "Let's look at how that works in the code!"
      ],
      "metadata": {
        "id": "wyazkAIu85KL"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.output_parsers import ResponseSchema\n",
        "from langchain.output_parsers import StructuredOutputParser\n",
        "\n",
        "question_schema = ResponseSchema(\n",
        "    name=\"question\",\n",
        "    description=\"a question about the context.\"\n",
        ")\n",
        "\n",
        "question_response_schemas = [\n",
        "    question_schema,\n",
        "]"
      ],
      "metadata": {
        "id": "V24T_gpPatAO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "question_output_parser = StructuredOutputParser.from_response_schemas(question_response_schemas)\n",
        "format_instructions = question_output_parser.get_format_instructions()"
      ],
      "metadata": {
        "id": "ebbmazGrdPap"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "question_generation_llm = ChatOpenAI(model=\"gpt-3.5-turbo-16k\")\n",
        "\n",
        "bare_prompt_template = \"{content}\"\n",
        "bare_template = ChatPromptTemplate.from_template(template=bare_prompt_template)"
      ],
      "metadata": {
        "id": "qorL4TPGXJQ7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.prompts import ChatPromptTemplate\n",
        "\n",
        "qa_template = \"\"\"\\\n",
        "You are a University Professor creating a test for advanced students. For each context, create a question that is specific to the context. Avoid creating generic or general questions.\n",
        "\n",
        "question: a question about the context.\n",
        "\n",
        "Format the output as JSON with the following keys:\n",
        "question\n",
        "\n",
        "context: {context}\n",
        "\"\"\"\n",
        "\n",
        "prompt_template = ChatPromptTemplate.from_template(template=qa_template)\n",
        "\n",
        "messages = prompt_template.format_messages(\n",
        "    context=docs[0],\n",
        "    format_instructions=format_instructions\n",
        ")\n",
        "\n",
        "question_generation_chain = bare_template | question_generation_llm\n",
        "\n",
        "response = question_generation_chain.invoke({\"content\" : messages})\n",
        "output_dict = question_output_parser.parse(response.content)"
      ],
      "metadata": {
        "id": "oPqC1_MXdRuh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for k, v in output_dict.items():\n",
        "  print(k)\n",
        "  print(v)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aKFw9kyZd7eB",
        "outputId": "bbcf9e15-58be-4899-f102-6cae59c45eb0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "question\n",
            "What is the aim of the paper 'A Survey on Retrieval-Augmented Text Generation'?\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -q -U tqdm"
      ],
      "metadata": {
        "id": "dtASDdhLfd89"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from tqdm import tqdm\n",
        "\n",
        "qac_triples = []\n",
        "\n",
        "for text in tqdm(docs[:10]):\n",
        "  messages = prompt_template.format_messages(\n",
        "      context=text,\n",
        "      format_instructions=format_instructions\n",
        "  )\n",
        "  response = question_generation_chain.invoke({\"content\" : messages})\n",
        "  try:\n",
        "    output_dict = question_output_parser.parse(response.content)\n",
        "  except Exception as e:\n",
        "    continue\n",
        "  output_dict[\"context\"] = text\n",
        "  qac_triples.append(output_dict)"
      ],
      "metadata": {
        "id": "Zolpr3CYeEYm",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a7962cf2-4cdf-4c7a-b776-a0c2478042e1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 10/10 [00:35<00:00,  3.55s/it]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "qac_triples[5]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mKBdQHK7Y2Vw",
        "outputId": "73c7d139-be2d-483f-9f70-b6c4aae91506"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'question': 'What is the main focus of this paper?',\n",
              " 'context': Document(page_content='thisisjcykcd@gmail.com, brandenwang@tencent.com\\nlemaoliu@gmail.com\\nAbstract\\nRecently, retrieval-augmented text generation\\nattracted increasing attention of the compu-\\ntational linguistics community.\\nCompared', metadata={'Published': '2022-02-13', 'Title': 'A Survey on Retrieval-Augmented Text Generation', 'Authors': 'Huayang Li, Yixuan Su, Deng Cai, Yan Wang, Lemao Liu', 'Summary': 'Recently, retrieval-augmented text generation attracted increasing attention\\nof the computational linguistics community. Compared with conventional\\ngeneration models, retrieval-augmented text generation has remarkable\\nadvantages and particularly has achieved state-of-the-art performance in many\\nNLP tasks. This paper aims to conduct a survey about retrieval-augmented text\\ngeneration. It firstly highlights the generic paradigm of retrieval-augmented\\ngeneration, and then it reviews notable approaches according to different tasks\\nincluding dialogue response generation, machine translation, and other\\ngeneration tasks. Finally, it points out some important directions on top of\\nrecent methods to facilitate future research.'})}"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "answer_generation_llm = ChatOpenAI(model=\"gpt-4-1106-preview\", temperature=0)\n",
        "\n",
        "answer_schema = ResponseSchema(\n",
        "    name=\"answer\",\n",
        "    description=\"an answer to the question\"\n",
        ")\n",
        "\n",
        "answer_response_schemas = [\n",
        "    answer_schema,\n",
        "]\n",
        "\n",
        "answer_output_parser = StructuredOutputParser.from_response_schemas(answer_response_schemas)\n",
        "format_instructions = answer_output_parser.get_format_instructions()\n",
        "\n",
        "qa_template = \"\"\"\\\n",
        "You are a University Professor creating a test for advanced students. For each question and context, create an answer.\n",
        "\n",
        "answer: a answer about the context.\n",
        "\n",
        "Format the output as JSON with the following keys:\n",
        "answer\n",
        "\n",
        "question: {question}\n",
        "context: {context}\n",
        "\"\"\"\n",
        "\n",
        "prompt_template = ChatPromptTemplate.from_template(template=qa_template)\n",
        "\n",
        "messages = prompt_template.format_messages(\n",
        "    context=qac_triples[0][\"context\"],\n",
        "    question=qac_triples[0][\"question\"],\n",
        "    format_instructions=format_instructions\n",
        ")\n",
        "\n",
        "answer_generation_chain = bare_template | answer_generation_llm\n",
        "\n",
        "response = answer_generation_chain.invoke({\"content\" : messages})\n",
        "output_dict = answer_output_parser.parse(response.content)"
      ],
      "metadata": {
        "id": "vNB9Z2DrX2TC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for k, v in output_dict.items():\n",
        "  print(k)\n",
        "  print(v)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Rk-_lRR6fn5U",
        "outputId": "fb014a65-a56f-49be-8ecf-9ca5527aa803"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "answer\n",
            "The focus of this paper is on retrieval-augmented text generation, which has gained increasing attention in the computational linguistics community. The paper conducts a survey of this field, highlighting the generic paradigm of retrieval-augmented generation, reviewing notable approaches across various tasks such as dialogue response generation and machine translation, and discussing future research directions.\n",
            "question\n",
            "What is the focus of the paper titled 'A Survey on Retrieval-Augmented Text Generation'?\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for triple in tqdm(qac_triples):\n",
        "  messages = prompt_template.format_messages(\n",
        "      context=triple[\"context\"],\n",
        "      question=triple[\"question\"],\n",
        "      format_instructions=format_instructions\n",
        "  )\n",
        "  response = answer_generation_chain.invoke({\"content\" : messages})\n",
        "  try:\n",
        "    output_dict = answer_output_parser.parse(response.content)\n",
        "  except Exception as e:\n",
        "    continue\n",
        "  triple[\"answer\"] = output_dict[\"answer\"]"
      ],
      "metadata": {
        "id": "yCdH0e9rrAKd",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "629d8791-dedb-47c7-b5a0-adaa26f142cd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 10/10 [01:10<00:00,  7.09s/it]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -q -U datasets"
      ],
      "metadata": {
        "id": "rrHXgH9Qs1ep"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "from datasets import Dataset\n",
        "\n",
        "ground_truth_qac_set = pd.DataFrame(qac_triples)\n",
        "ground_truth_qac_set[\"context\"] = ground_truth_qac_set[\"context\"].map(lambda x: str(x.page_content))\n",
        "ground_truth_qac_set = ground_truth_qac_set.rename(columns={\"answer\" : \"ground_truth\"})\n",
        "\n",
        "\n",
        "eval_dataset = Dataset.from_pandas(ground_truth_qac_set)"
      ],
      "metadata": {
        "id": "uAvGGTyXsoHQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "eval_dataset"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "q_FHUnAPVseB",
        "outputId": "1a907389-e62b-4686-b3d7-e8707acfbd47"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Dataset({\n",
              "    features: ['question', 'context', 'ground_truth'],\n",
              "    num_rows: 10\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "eval_dataset[0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "W8CaCUeBVu4l",
        "outputId": "72cbf3c8-c698-4682-821a-8566f36f6adb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'question': 'What is the focus of this paper?',\n",
              " 'context': 'A Survey on Retrieval-Augmented Text Generation\\nHuayang Li♥,∗\\nYixuan Su♠,∗\\nDeng Cai♦,∗\\nYan Wang♣,∗\\nLemao Liu♣,∗\\n♥Nara Institute of Science and Technology\\n♠University of Cambridge\\n♦The Chinese University of Hong Kong\\n♣Tencent AI Lab',\n",
              " 'ground_truth': 'The focus of this paper is on retrieval-augmented text generation, which has gained increasing attention in the computational linguistics community. The paper surveys the paradigm of retrieval-augmented generation, reviews notable approaches across various NLP tasks such as dialogue response generation and machine translation, and discusses future research directions in this area.'}"
            ]
          },
          "metadata": {},
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "eval_dataset.to_csv(\"groundtruth_eval_dataset.csv\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 66,
          "referenced_widgets": [
            "c4bc64d47f2e4a239cf7156e7812887d",
            "b4a71cf676584200b46922fb976d1d50",
            "65f75f3d72cd415faffb32999893738d",
            "5367cbe7ca4d43b089ff0d5d7cd17d3c",
            "31390facfbeb455fa83067fc23d30718",
            "22fc25617c664e4aa02b7457832c1bef",
            "b556054df18d47aa8cb58ac482ca31bb",
            "0fff4a6672c848a38945e19101c46168",
            "de662cb67d054f08a15d191923d40369",
            "b53ebcf672a244978391cae559db5bf2",
            "f2de6a2c1f4b4a2fa5196028c0da0754"
          ]
        },
        "id": "Nhp5X4M8zqrm",
        "outputId": "7e3c36df-12a3-4ea7-a772-dc1e0bc61568"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Creating CSV from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "c4bc64d47f2e4a239cf7156e7812887d"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "7359"
            ]
          },
          "metadata": {},
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Evaluating RAG Pipelines\n",
        "\n",
        "If you skipped ahead and need to load the `.csv` directly - uncomment the code below.\n",
        "\n",
        "If you're using Colab to do this notebook - please ensure you add it to your session files."
      ],
      "metadata": {
        "id": "7Al5cagr-rvL"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# from datasets import Dataset\n",
        "# eval_dataset = Dataset.from_csv(\"groundtruth_eval_dataset.csv\")"
      ],
      "metadata": {
        "id": "QJhes58R66-P"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "eval_dataset"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5fAD8c_kthWc",
        "outputId": "e722498d-3179-4cf1-e206-29a27163ace5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Dataset({\n",
              "    features: ['question', 'context', 'ground_truth'],\n",
              "    num_rows: 10\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Evaluation Using RAGAS\n",
        "\n",
        "Now we can evaluate using RAGAS!\n",
        "\n",
        "The set-up is fairly straightforward - we simply need to create a dataset with our generated answers and our contexts, and then evaluate using the framework."
      ],
      "metadata": {
        "id": "IqFYbjLK-6X7"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from ragas.metrics import (\n",
        "    answer_relevancy,\n",
        "    faithfulness,\n",
        "    context_recall,\n",
        "    context_precision,\n",
        "    context_relevancy,\n",
        "    answer_correctness,\n",
        "    answer_similarity\n",
        ")\n",
        "\n",
        "from ragas.metrics.critique import harmfulness\n",
        "from ragas import evaluate\n",
        "\n",
        "def create_ragas_dataset(rag_pipeline, eval_dataset):\n",
        "  rag_dataset = []\n",
        "  for row in tqdm(eval_dataset):\n",
        "    answer = rag_pipeline.invoke({\"question\" : row[\"question\"]})\n",
        "    rag_dataset.append(\n",
        "        {\"question\" : row[\"question\"],\n",
        "         \"answer\" : answer[\"response\"].content,\n",
        "         \"contexts\" : [context.page_content for context in answer[\"context\"]],\n",
        "         \"ground_truths\" : [row[\"ground_truth\"]]\n",
        "         }\n",
        "    )\n",
        "  rag_df = pd.DataFrame(rag_dataset)\n",
        "  rag_eval_dataset = Dataset.from_pandas(rag_df)\n",
        "  return rag_eval_dataset\n",
        "\n",
        "def evaluate_ragas_dataset(ragas_dataset):\n",
        "  result = evaluate(\n",
        "    ragas_dataset,\n",
        "    metrics=[\n",
        "        context_precision,\n",
        "        faithfulness,\n",
        "        answer_relevancy,\n",
        "        context_recall,\n",
        "        context_relevancy,\n",
        "        answer_correctness,\n",
        "        answer_similarity\n",
        "    ],\n",
        "  )\n",
        "  return result"
      ],
      "metadata": {
        "id": "1eBoHaf5t4w8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Lets create our dataset first:"
      ],
      "metadata": {
        "id": "J4c4Jd8G_lXY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from tqdm import tqdm\n",
        "import pandas as pd\n",
        "\n",
        "basic_qa_ragas_dataset = create_ragas_dataset(retrieval_augmented_qa_chain, eval_dataset)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "R7oXgcjkuopO",
        "outputId": "6db1a904-90a2-4e47-85da-a442ebdc56b1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 10/10 [00:12<00:00,  1.30s/it]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "basic_qa_ragas_dataset"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nfzaFWEMZ5l_",
        "outputId": "60ca3e05-209a-4375-f24c-a23ad06e525e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Dataset({\n",
              "    features: ['question', 'answer', 'contexts', 'ground_truths'],\n",
              "    num_rows: 10\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "basic_qa_ragas_dataset[0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8Vv1NsRGZ6m5",
        "outputId": "2ef4fbd9-011e-42b7-88d9-11c79974d87e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'question': 'What is the focus of this paper?',\n",
              " 'answer': 'The focus of this paper is on improving the zero-shot generalization ability of language models through the use of Mixture-Of-Memory Augmentation (MoMA), a mechanism that retrieves augmentation documents from multiple information corpora.',\n",
              " 'contexts': ['paper are used for illustration only, they do not represent\\nthe ethical attitude of the authors.\\nReferences\\nPayal Bajaj, Daniel Campos, Nick Craswell, Li Deng,\\nJianfeng Gao, Xiaodong Liu, Rangan Majumder,\\nAndrew McNamara, Bhaskar Mitra, Tri Nguyen,',\n",
              "  '2Microsoft Research, Redmond, USA\\n3Beijing National Research Center for Information Science and Technology, Beijing, China\\n{yuzc19, yus21}@mails.tsinghua.edu.cn; chenyan.xiong@microsoft.com\\nliuzy@tsinghua.edu.cn\\nAbstract'],\n",
              " 'ground_truths': ['The focus of this paper is on retrieval-augmented text generation, which has gained increasing attention in the computational linguistics community. The paper surveys the paradigm of retrieval-augmented generation, reviews notable approaches across various NLP tasks such as dialogue response generation and machine translation, and discusses future research directions in this area.']}"
            ]
          },
          "metadata": {},
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Save it for later:"
      ],
      "metadata": {
        "id": "Obbgw3im_n01"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "basic_qa_ragas_dataset.to_csv(\"basic_qa_ragas_dataset.csv\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 66,
          "referenced_widgets": [
            "86761a36bdb04fed9f1dae6e74da54ce",
            "e079c0eb3d9c4ae3a40491f059a864f7",
            "d4bde97f3895450782030cad7808ea59",
            "1d17b8847ab34a1d8fc79430e8b64a12",
            "fb953d8d116e4cc389dbc23a0055873f",
            "bf27f78edb15477492fb2d5dd8dc5137",
            "ee96a569b8a54944bcac1b1bc164e53e",
            "e556e254882340168833ecf1a7153a90",
            "c077837b8b044c2fb14dcd8fbc44a37b",
            "e5c8b5933c754a1192456f43682276c5",
            "84abf77081a04bb686f794b49e04761d"
          ]
        },
        "id": "6FG8x4i3yZ2B",
        "outputId": "52eb909f-69be-4c80-d9bc-77c2842bf14d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Creating CSV from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "86761a36bdb04fed9f1dae6e74da54ce"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "11397"
            ]
          },
          "metadata": {},
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "And finally - evaluate how it did!"
      ],
      "metadata": {
        "id": "A5I_d_RT_pFr"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "basic_qa_result = evaluate_ragas_dataset(basic_qa_ragas_dataset)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ywp3Rwavy9pc",
        "outputId": "7a3948ce-b743-4d95-a581-17e035215f3f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "evaluating with [context_precision]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1/1 [00:00<00:00,  1.04it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "evaluating with [faithfulness]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1/1 [00:07<00:00,  7.44s/it]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "evaluating with [answer_relevancy]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1/1 [00:05<00:00,  5.91s/it]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "evaluating with [context_recall]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1/1 [00:07<00:00,  7.35s/it]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "evaluating with [context_relevancy]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1/1 [00:02<00:00,  2.26s/it]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "evaluating with [answer_correctness]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1/1 [00:09<00:00,  9.06s/it]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "evaluating with [answer_similarity]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1/1 [00:00<00:00,  1.74it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "basic_qa_result"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "m4oYnKTn15gY",
        "outputId": "ba48e4d2-5559-4748-8bc6-0101074a5c0e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'context_precision': 0.5000, 'faithfulness': 0.4000, 'answer_relevancy': 0.9535, 'context_recall': 1.0000, 'context_relevancy': 0.0559, 'answer_correctness': 0.6167, 'answer_similarity': 1.0000}"
            ]
          },
          "metadata": {},
          "execution_count": 63
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Testing Other Retrievers\n",
        "\n",
        "Now we can test our how changing our Retriever impacts our RAGAS evaluation!\n",
        "\n",
        "We'll build this simple qa_chain factory to create standardized qa_chains where the only different component will be the retriever."
      ],
      "metadata": {
        "id": "SwhBxlxYAdno"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def create_qa_chain(retriever):\n",
        "  primary_qa_llm = ChatOpenAI(model_name=\"gpt-3.5-turbo\", temperature=0)\n",
        "  created_qa_chain = (\n",
        "    {\"context\": itemgetter(\"question\") | retriever,\n",
        "     \"question\": itemgetter(\"question\")\n",
        "    }\n",
        "    | RunnablePassthrough.assign(\n",
        "        context=itemgetter(\"context\")\n",
        "      )\n",
        "    | {\n",
        "         \"response\": prompt | primary_qa_llm,\n",
        "         \"context\": itemgetter(\"context\"),\n",
        "      }\n",
        "  )\n",
        "\n",
        "  return created_qa_chain"
      ],
      "metadata": {
        "id": "qnfy4VNkzZi2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### Parent Document Retriever\n",
        "\n",
        "One of the easier ways we can imagine improving a retriever is to embed our documents into small chunks, and then retrieve a significant amount of additional context that \"surrounds\" the found context.\n",
        "\n",
        "You can read more about this method [here](https://python.langchain.com/docs/modules/data_connection/retrievers/parent_document_retriever)!\n",
        "\n",
        "The basic outline of this retrieval method is as follows:\n",
        "\n",
        "1. Obtain User Question\n",
        "2. Retrieve child documents using Dense Vector Retrieval\n",
        "3. Merge the child documents based on their parents. If they have the same parents - they become merged.\n",
        "4. Replace the child documents with their respective parent documents from an in-memory-store.\n",
        "5. Use the parent documents to augment generation."
      ],
      "metadata": {
        "id": "vOPp4Xq7AvEx"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.retrievers import ParentDocumentRetriever\n",
        "from langchain.storage import InMemoryStore\n",
        "\n",
        "parent_splitter = RecursiveCharacterTextSplitter(chunk_size=1500)\n",
        "child_splitter = RecursiveCharacterTextSplitter(chunk_size=200)\n",
        "\n",
        "vectorstore = Chroma(collection_name=\"split_parents\", embedding_function=OpenAIEmbeddings())\n",
        "\n",
        "store = InMemoryStore()"
      ],
      "metadata": {
        "id": "67I6QJAJ0Un7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "parent_document_retriever = ParentDocumentRetriever(\n",
        "    vectorstore=vectorstore,\n",
        "    docstore=store,\n",
        "    child_splitter=child_splitter,\n",
        "    parent_splitter=parent_splitter,\n",
        ")"
      ],
      "metadata": {
        "id": "zfk5RYUt00Pw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "parent_document_retriever.add_documents(base_docs)"
      ],
      "metadata": {
        "id": "68c1t4o104AK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Let's create, test, and then evaluate our new chain!"
      ],
      "metadata": {
        "id": "MTH0MDolBndm"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "parent_document_retriever_qa_chain = create_qa_chain(parent_document_retriever)"
      ],
      "metadata": {
        "id": "KMjLfqOC09Iw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "parent_document_retriever_qa_chain.invoke({\"question\" : \"What is RAG?\"})[\"response\"].content"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "Rv8bAHPN1H4P",
        "outputId": "faa6bf43-1604-4468-9faf-bbefd8e48281"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'RAG stands for Retrieval-Augmented Generation.'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pdr_qa_ragas_dataset = create_ragas_dataset(parent_document_retriever_qa_chain, eval_dataset)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OQJRIQmU1WTw",
        "outputId": "295a9011-684d-4c38-e409-867022603608"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 10/10 [00:17<00:00,  1.80s/it]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pdr_qa_ragas_dataset.to_csv(\"pdr_qa_ragas_dataset.csv\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 66,
          "referenced_widgets": [
            "bf7f045bdbe24360ad2aa2f4c8f02e79",
            "761e3c6035bf49429b3035145451d2df",
            "ffb7c97e7af648aaa13a43427154140e",
            "95d92c5c74e845779337eb727c2bbfc0",
            "6e7fb9a1d1454fcd9bcf5b5f748fb975",
            "207785da1f404d6ea0e8c63655a7aa51",
            "71237d176b2c4138a5e0346d10482257",
            "ed457547dc154f6bbce4ec970bb09c76",
            "c9f479f81119450bb451d5830361467c",
            "56098a347ea94ea3b10bd8d2ec0d4288",
            "7940d9e3f5fa4592b58dec3fdb55595a"
          ]
        },
        "id": "d9vfKnCL1jtB",
        "outputId": "1d7421a8-b564-4da3-cd74-d1eb3f8311f7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Creating CSV from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "bf7f045bdbe24360ad2aa2f4c8f02e79"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "55620"
            ]
          },
          "metadata": {},
          "execution_count": 46
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pdr_qa_result = evaluate_ragas_dataset(pdr_qa_ragas_dataset)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qfB1H9S_1mW3",
        "outputId": "426d7b1b-2b0c-40d3-e7d7-39da43363f06"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "evaluating with [context_precision]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1/1 [01:01<00:00, 61.80s/it]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "evaluating with [faithfulness]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1/1 [01:09<00:00, 69.23s/it]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "evaluating with [answer_relevancy]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1/1 [01:04<00:00, 64.76s/it]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "evaluating with [context_recall]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1/1 [00:06<00:00,  6.54s/it]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "evaluating with [context_relevancy]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1/1 [00:04<00:00,  4.02s/it]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "evaluating with [answer_correctness]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1/1 [00:06<00:00,  6.83s/it]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "evaluating with [answer_similarity]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1/1 [00:00<00:00,  1.54it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pdr_qa_result"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5nFyYCdL2Nco",
        "outputId": "bdde7173-c649-40bc-cbc3-e38a70c9f50a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'context_precision': 0.6972, 'faithfulness': 0.3500, 'answer_relevancy': 0.9439, 'context_recall': 1.0000, 'context_relevancy': 0.0134, 'answer_correctness': 0.6000, 'answer_similarity': 1.0000}"
            ]
          },
          "metadata": {},
          "execution_count": 48
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### Ensemble Retrieval\n",
        "\n",
        "Next let's look at ensemble retrieval!\n",
        "\n",
        "You can read more about this [here](https://python.langchain.com/docs/modules/data_connection/retrievers/ensemble)!\n",
        "\n",
        "The basic idea is as follows:\n",
        "\n",
        "1. Obtain User Question\n",
        "2. Hit the Retriever Pair\n",
        "    - Retrieve Documents with BM25 Sparse Vector Retrieval\n",
        "    - Retrieve Documents with Dense Vector Retrieval Method\n",
        "3. Collect and \"fuse\" the retrieved docs based on their weighting using the [Reciprocal Rank Fusion](https://plg.uwaterloo.ca/~gvcormac/cormacksigir09-rrf.pdf) algorithm into a single ranked list.\n",
        "4. Use those documents to augment our generation.\n",
        "\n",
        "Ensure your `weights` list - the relative weighting of each retriever - sums to 1!"
      ],
      "metadata": {
        "id": "JaNk6o7_BqX8"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -q -U rank_bm25"
      ],
      "metadata": {
        "id": "zz7dl1GD5-L-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.retrievers import BM25Retriever, EnsembleRetriever\n",
        "\n",
        "text_splitter = RecursiveCharacterTextSplitter(chunk_size=450, chunk_overlap=75)\n",
        "docs = text_splitter.split_documents(base_docs)\n",
        "\n",
        "bm25_retriever = BM25Retriever.from_documents(docs)\n",
        "bm25_retriever.k = 2\n",
        "\n",
        "embedding = OpenAIEmbeddings()\n",
        "vectorstore = Chroma.from_documents(docs, embedding)\n",
        "chroma_retriever = vectorstore.as_retriever(search_kwargs={\"k\": 3})\n",
        "\n",
        "ensemble_retriever = EnsembleRetriever(retrievers=[bm25_retriever, chroma_retriever], weights=[0.75, 0.25])"
      ],
      "metadata": {
        "id": "Vs8wxT9b5pRA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "ensemble_retriever_qa_chain = create_qa_chain(ensemble_retriever)"
      ],
      "metadata": {
        "id": "cv69YDpF6PrJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "ensemble_retriever_qa_chain.invoke({\"question\" : \"What is RAG?\"})[\"response\"].content"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "6lSszzrf6UmP",
        "outputId": "8a5893d5-4095-42e5-aecf-66d849512321"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'RAG stands for Retrieval-Augmented Generation.'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 52
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "ensemble_qa_ragas_dataset = create_ragas_dataset(ensemble_retriever_qa_chain, eval_dataset)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "abUgTGDT6UrV",
        "outputId": "749ae6db-75b9-48a7-e743-a8aecdcbd802"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 10/10 [00:20<00:00,  2.07s/it]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "ensemble_qa_ragas_dataset.to_csv(\"ensemble_qa_ragas_dataset.csv\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 66,
          "referenced_widgets": [
            "632135599e39470aac1a3bb3d3de0ca4",
            "567176b50d074a1c9d384a8df8c3ff4c",
            "6bfa39f2c84e4f08b5ffb9a416398824",
            "911b40169865413b98643789150e5495",
            "2add9d30edd84152bb6d7bfa4ed2d910",
            "f34ffd11b98045b9bae326dd48a54896",
            "e3e19fc9963c4bf39293bda7c2030d5a",
            "c15c27b0523a429e9d77f439d47ada90",
            "256a0dbb2f104d928e181d2a882a9867",
            "68790cfe4ea14fc4a63275f3e99c468f",
            "e7df092ac205443e8baa3646ff1eae5b"
          ]
        },
        "id": "bGHipYsf7phk",
        "outputId": "a70b0d3e-870f-49b8-a16a-5b7d4623c33f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Creating CSV from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "632135599e39470aac1a3bb3d3de0ca4"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "22820"
            ]
          },
          "metadata": {},
          "execution_count": 54
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "ensemble_qa_result = evaluate_ragas_dataset(ensemble_qa_ragas_dataset)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ozo0jkvx7r1d",
        "outputId": "f5770c52-4614-4172-8834-d48bd4005218"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "evaluating with [context_precision]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1/1 [01:01<00:00, 61.76s/it]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "evaluating with [faithfulness]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1/1 [01:08<00:00, 68.62s/it]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "evaluating with [answer_relevancy]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1/1 [00:05<00:00,  5.37s/it]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "evaluating with [context_recall]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1/1 [00:11<00:00, 11.67s/it]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "evaluating with [context_relevancy]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1/1 [01:02<00:00, 62.45s/it]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "evaluating with [answer_correctness]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1/1 [00:08<00:00,  9.00s/it]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "evaluating with [answer_similarity]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1/1 [00:00<00:00,  1.57it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "ensemble_qa_result"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hvabdcbh793a",
        "outputId": "56daa44b-841b-4924-9242-77c2bc93f86e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'context_precision': 0.8858, 'faithfulness': 0.7000, 'answer_relevancy': 0.8918, 'context_recall': 0.9800, 'context_relevancy': 0.0192, 'answer_correctness': 0.7750, 'answer_similarity': 1.0000}"
            ]
          },
          "metadata": {},
          "execution_count": 56
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Conclusion\n",
        "\n",
        "Observe your results in a table!"
      ],
      "metadata": {
        "id": "O4vXVWqiCcSI"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "basic_qa_result"
      ],
      "metadata": {
        "id": "PmBoVQ5hV3Kc",
        "outputId": "12196187-5cbf-40b2-f35f-ab45616e71a4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'context_precision': 0.5000, 'faithfulness': 0.4000, 'answer_relevancy': 0.9535, 'context_recall': 1.0000, 'context_relevancy': 0.0559, 'answer_correctness': 0.6167, 'answer_similarity': 1.0000}"
            ]
          },
          "metadata": {},
          "execution_count": 64
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pdr_qa_result"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ax1JLXKxUsXF",
        "outputId": "b83ba792-7e66-44ff-b219-0f3890a5fe8b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'context_precision': 0.6972, 'faithfulness': 0.3500, 'answer_relevancy': 0.9439, 'context_recall': 1.0000, 'context_relevancy': 0.0134, 'answer_correctness': 0.6000, 'answer_similarity': 1.0000}"
            ]
          },
          "metadata": {},
          "execution_count": 59
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "ensemble_qa_result"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "drxLlO3zUpyQ",
        "outputId": "3b595607-2fa5-4590-d2e6-9707aa7bb283"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'context_precision': 0.8858, 'faithfulness': 0.7000, 'answer_relevancy': 0.8918, 'context_recall': 0.9800, 'context_relevancy': 0.0192, 'answer_correctness': 0.7750, 'answer_similarity': 1.0000}"
            ]
          },
          "metadata": {},
          "execution_count": 58
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We can also zoom in on each result and find specific information about each of the questions and answers."
      ],
      "metadata": {
        "id": "r6YPGf-2l0Kx"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "ensemble_qa_result_df = ensemble_qa_result.to_pandas()"
      ],
      "metadata": {
        "id": "SkxLtk43ikka"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "ensemble_qa_result_df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "3JZMhUg3jSvE",
        "outputId": "95d1cf44-88ee-4056-b698-64237b119fe0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                            question  \\\n",
              "0                   What is the focus of this paper?   \n",
              "1                    What is the title of the paper?   \n",
              "2                     What is the aim of this paper?   \n",
              "3  What is the main focus of the paper 'A Survey ...   \n",
              "4              What is the main focus of this paper?   \n",
              "5              What is the main focus of this paper?   \n",
              "6  What are the advantages of retrieval-augmented...   \n",
              "7  What is the main focus of the paper 'A Survey ...   \n",
              "8  What are the advantages of retrieval-augmented...   \n",
              "9  What are the advantages of retrieval-augmented...   \n",
              "\n",
              "                                            contexts  \\\n",
              "0  [has to make an important career decision.\\nNe...   \n",
              "1  [of War. The game was released worldwide in\\nG...   \n",
              "2  [has to make an important career decision.\\nNe...   \n",
              "3  [A Survey on Retrieval-Augmented Text Generati...   \n",
              "4  [example of completions of the prompt by diffe...   \n",
              "5  [example of completions of the prompt by diffe...   \n",
              "6  [attracted increasing attention of the compu-\\...   \n",
              "7  [A Survey on Retrieval-Augmented Text Generati...   \n",
              "8  [attracted increasing attention of the compu-\\...   \n",
              "9  [attracted increasing attention of the compu-\\...   \n",
              "\n",
              "                                              answer  \\\n",
              "0  The focus of this paper is on a framework call...   \n",
              "1  Title: Self-RAG: Learning to Retrieve, Generat...   \n",
              "2  The aim of this paper is to introduce a new fr...   \n",
              "3  The main focus of the paper 'A Survey on Retri...   \n",
              "4                                      I don't know.   \n",
              "5                                      I don't know.   \n",
              "6  The advantages of retrieval-augmented text gen...   \n",
              "7  The main focus of the paper 'A Survey on Retri...   \n",
              "8  The advantages of retrieval-augmented text gen...   \n",
              "9  The advantages of retrieval-augmented text gen...   \n",
              "\n",
              "                                       ground_truths  context_precision  \\\n",
              "0  [The focus of this paper is on retrieval-augme...           1.000000   \n",
              "1  [The title of the paper is 'A Survey on Retrie...           0.500000   \n",
              "2  [The aim of this paper is to conduct a compreh...           1.000000   \n",
              "3  [The main focus of the paper 'A Survey on Retr...           0.679167   \n",
              "4  [The main focus of this paper is to conduct a ...           1.000000   \n",
              "5  [The main focus of this paper is to conduct a ...           1.000000   \n",
              "6  [The advantages of retrieval-augmented text ge...           1.000000   \n",
              "7  [The main focus of the paper 'A Survey on Retr...           0.679167   \n",
              "8  [The advantages of retrieval-augmented text ge...           1.000000   \n",
              "9  [The advantages of retrieval-augmented text ge...           1.000000   \n",
              "\n",
              "   faithfulness  answer_relevancy  context_recall  context_relevancy  \\\n",
              "0      0.666667          0.784617             1.0           0.000000   \n",
              "1      1.000000          0.976911             1.0           0.000000   \n",
              "2      0.333333          0.800732             1.0           0.078947   \n",
              "3      1.000000          0.982435             0.8           0.017857   \n",
              "4      0.000000          0.742601             1.0           0.000000   \n",
              "5      0.000000          0.742601             1.0           0.000000   \n",
              "6      1.000000          0.968712             1.0           0.025641   \n",
              "7      1.000000          0.982422             1.0           0.017857   \n",
              "8      1.000000          0.968731             1.0           0.025641   \n",
              "9      1.000000          0.968692             1.0           0.025641   \n",
              "\n",
              "   answer_correctness  answer_similarity  \n",
              "0                0.50               True  \n",
              "1                0.50               True  \n",
              "2                0.75               True  \n",
              "3                1.00               True  \n",
              "4                0.50               True  \n",
              "5                0.50               True  \n",
              "6                1.00               True  \n",
              "7                1.00               True  \n",
              "8                1.00               True  \n",
              "9                1.00               True  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-a1e0be28-2fa7-4b13-8d13-5b1c1554cdd9\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>question</th>\n",
              "      <th>contexts</th>\n",
              "      <th>answer</th>\n",
              "      <th>ground_truths</th>\n",
              "      <th>context_precision</th>\n",
              "      <th>faithfulness</th>\n",
              "      <th>answer_relevancy</th>\n",
              "      <th>context_recall</th>\n",
              "      <th>context_relevancy</th>\n",
              "      <th>answer_correctness</th>\n",
              "      <th>answer_similarity</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>What is the focus of this paper?</td>\n",
              "      <td>[has to make an important career decision.\\nNe...</td>\n",
              "      <td>The focus of this paper is on a framework call...</td>\n",
              "      <td>[The focus of this paper is on retrieval-augme...</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.666667</td>\n",
              "      <td>0.784617</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.50</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>What is the title of the paper?</td>\n",
              "      <td>[of War. The game was released worldwide in\\nG...</td>\n",
              "      <td>Title: Self-RAG: Learning to Retrieve, Generat...</td>\n",
              "      <td>[The title of the paper is 'A Survey on Retrie...</td>\n",
              "      <td>0.500000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.976911</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.50</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>What is the aim of this paper?</td>\n",
              "      <td>[has to make an important career decision.\\nNe...</td>\n",
              "      <td>The aim of this paper is to introduce a new fr...</td>\n",
              "      <td>[The aim of this paper is to conduct a compreh...</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.333333</td>\n",
              "      <td>0.800732</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.078947</td>\n",
              "      <td>0.75</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>What is the main focus of the paper 'A Survey ...</td>\n",
              "      <td>[A Survey on Retrieval-Augmented Text Generati...</td>\n",
              "      <td>The main focus of the paper 'A Survey on Retri...</td>\n",
              "      <td>[The main focus of the paper 'A Survey on Retr...</td>\n",
              "      <td>0.679167</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.982435</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.017857</td>\n",
              "      <td>1.00</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>What is the main focus of this paper?</td>\n",
              "      <td>[example of completions of the prompt by diffe...</td>\n",
              "      <td>I don't know.</td>\n",
              "      <td>[The main focus of this paper is to conduct a ...</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.742601</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.50</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>What is the main focus of this paper?</td>\n",
              "      <td>[example of completions of the prompt by diffe...</td>\n",
              "      <td>I don't know.</td>\n",
              "      <td>[The main focus of this paper is to conduct a ...</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.742601</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.50</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>What are the advantages of retrieval-augmented...</td>\n",
              "      <td>[attracted increasing attention of the compu-\\...</td>\n",
              "      <td>The advantages of retrieval-augmented text gen...</td>\n",
              "      <td>[The advantages of retrieval-augmented text ge...</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.968712</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.025641</td>\n",
              "      <td>1.00</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>What is the main focus of the paper 'A Survey ...</td>\n",
              "      <td>[A Survey on Retrieval-Augmented Text Generati...</td>\n",
              "      <td>The main focus of the paper 'A Survey on Retri...</td>\n",
              "      <td>[The main focus of the paper 'A Survey on Retr...</td>\n",
              "      <td>0.679167</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.982422</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.017857</td>\n",
              "      <td>1.00</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>What are the advantages of retrieval-augmented...</td>\n",
              "      <td>[attracted increasing attention of the compu-\\...</td>\n",
              "      <td>The advantages of retrieval-augmented text gen...</td>\n",
              "      <td>[The advantages of retrieval-augmented text ge...</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.968731</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.025641</td>\n",
              "      <td>1.00</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>What are the advantages of retrieval-augmented...</td>\n",
              "      <td>[attracted increasing attention of the compu-\\...</td>\n",
              "      <td>The advantages of retrieval-augmented text gen...</td>\n",
              "      <td>[The advantages of retrieval-augmented text ge...</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.968692</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.025641</td>\n",
              "      <td>1.00</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a1e0be28-2fa7-4b13-8d13-5b1c1554cdd9')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-a1e0be28-2fa7-4b13-8d13-5b1c1554cdd9 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-a1e0be28-2fa7-4b13-8d13-5b1c1554cdd9');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-9d17e429-a85f-4eed-ab52-253849a35ce1\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-9d17e429-a85f-4eed-ab52-253849a35ce1')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-9d17e429-a85f-4eed-ab52-253849a35ce1 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 77
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We'll also look at combining the results and looking at them in a single table so we can make inferences about them!"
      ],
      "metadata": {
        "id": "0jXR7ckel-v8"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def create_df_dict(pipeline_name, pipeline_items):\n",
        "  df_dict = {\"name\" : pipeline_name}\n",
        "  for name, score in pipeline_items:\n",
        "    df_dict[name] = score\n",
        "  return df_dict"
      ],
      "metadata": {
        "id": "BE5KKE_JkcD3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "basic_rag_df_dict = create_df_dict(\"basic_rag\", basic_qa_result.items())"
      ],
      "metadata": {
        "id": "L1mPqYdqk4iQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pdr_rag_df_dict = create_df_dict(\"pdr_rag\", pdr_qa_result.items())"
      ],
      "metadata": {
        "id": "ntJPzwy9lI46"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "ensemble_rag_df_dict = create_df_dict(\"ensemble_rag\", ensemble_qa_result.items())"
      ],
      "metadata": {
        "id": "R0fkbIQElPza"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "results_df = pd.DataFrame([basic_rag_df_dict, pdr_rag_df_dict, ensemble_rag_df_dict])"
      ],
      "metadata": {
        "id": "Bc4T1E83lVbE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "results_df.sort_values(\"answer_correctness\", ascending=False)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 163
        },
        "id": "cv_9wNYGlibg",
        "outputId": "6580c17c-543a-4d54-8577-104c0173f368"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "           name  context_precision  faithfulness  answer_relevancy  \\\n",
              "2  ensemble_rag           0.885833          0.70          0.891845   \n",
              "0     basic_rag           0.500000          0.40          0.953475   \n",
              "1       pdr_rag           0.697222          0.35          0.943909   \n",
              "\n",
              "   context_recall  context_relevancy  answer_correctness  answer_similarity  \n",
              "2            0.98           0.019158            0.775000                1.0  \n",
              "0            1.00           0.055904            0.616667                1.0  \n",
              "1            1.00           0.013386            0.600000                1.0  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-b21809b4-1506-4413-b95a-a7bdc7cf6816\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>name</th>\n",
              "      <th>context_precision</th>\n",
              "      <th>faithfulness</th>\n",
              "      <th>answer_relevancy</th>\n",
              "      <th>context_recall</th>\n",
              "      <th>context_relevancy</th>\n",
              "      <th>answer_correctness</th>\n",
              "      <th>answer_similarity</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>ensemble_rag</td>\n",
              "      <td>0.885833</td>\n",
              "      <td>0.70</td>\n",
              "      <td>0.891845</td>\n",
              "      <td>0.98</td>\n",
              "      <td>0.019158</td>\n",
              "      <td>0.775000</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>basic_rag</td>\n",
              "      <td>0.500000</td>\n",
              "      <td>0.40</td>\n",
              "      <td>0.953475</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.055904</td>\n",
              "      <td>0.616667</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>pdr_rag</td>\n",
              "      <td>0.697222</td>\n",
              "      <td>0.35</td>\n",
              "      <td>0.943909</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.013386</td>\n",
              "      <td>0.600000</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-b21809b4-1506-4413-b95a-a7bdc7cf6816')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-b21809b4-1506-4413-b95a-a7bdc7cf6816 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-b21809b4-1506-4413-b95a-a7bdc7cf6816');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-43d2deee-1b4d-4900-9ce8-743ac419b9d1\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-43d2deee-1b4d-4900-9ce8-743ac419b9d1')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-43d2deee-1b4d-4900-9ce8-743ac419b9d1 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 90
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### ❓QUESTION❓\n",
        "\n",
        "What conclusions can you draw about the above results?\n",
        "\n",
        "Describe in your own words what the metrics are expressing."
      ],
      "metadata": {
        "id": "YPocfrNFiYWi"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "retrieval_augmented_qa_chain = (\n",
        "    RunnableParallel({\n",
        "        'context': itemgetter('question') | base_retriever,\n",
        "        'question': RunnablePassthrough()\n",
        "    }) | {\n",
        "        'response': prompt | primary_qa_llm | parser,\n",
        "        'context': itemgetter('context')\n",
        "    }\n",
        ")"
      ],
      "metadata": {
        "id": "fbhz-vD4JPtN"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}