LLM_as_a_judge.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "ea0f693848c545a1b722692225243d47": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_04fbb6e110194292bedd4ba9170d46bb",
              "IPY_MODEL_059b18a979034523824f71a3c8fbc19c",
              "IPY_MODEL_47e38f90d19741db833e6adf49bb8288"
            ],
            "layout": "IPY_MODEL_7f7d4760368a43658cb70fa52e43880d"
          }
        },
        "04fbb6e110194292bedd4ba9170d46bb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_70f12b0132364f608af4aeb71afee274",
            "placeholder": "​",
            "style": "IPY_MODEL_d35aedb568704646b0e8838de1b4181e",
            "value": "100%"
          }
        },
        "059b18a979034523824f71a3c8fbc19c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6b79b2acd65d48d28aa2b23e46ff65bd",
            "max": 2,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_de9b8adac7094b6b834a46b7df1c2cf1",
            "value": 2
          }
        },
        "47e38f90d19741db833e6adf49bb8288": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b349f8d7fb784c42bf3ee1d11770d8fe",
            "placeholder": "​",
            "style": "IPY_MODEL_b6191fc85be743da9a369ef36a806afb",
            "value": " 2/2 [03:45&lt;00:00, 116.50s/it]"
          }
        },
        "7f7d4760368a43658cb70fa52e43880d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "70f12b0132364f608af4aeb71afee274": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d35aedb568704646b0e8838de1b4181e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6b79b2acd65d48d28aa2b23e46ff65bd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "de9b8adac7094b6b834a46b7df1c2cf1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b349f8d7fb784c42bf3ee1d11770d8fe": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b6191fc85be743da9a369ef36a806afb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "08d9cc1ec1084189b8755654995042e6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_14361477e6d54c53a3880ea8805559a9",
              "IPY_MODEL_7c04689ccf0c492b9fdf9fcdd1ec3d08",
              "IPY_MODEL_5e0007d21592421cbd960c7a18d918b9"
            ],
            "layout": "IPY_MODEL_2db49dca74df421187cbf0cd767b12a8"
          }
        },
        "14361477e6d54c53a3880ea8805559a9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_39d0925e43f749d6b335963219d4f6f8",
            "placeholder": "​",
            "style": "IPY_MODEL_bc6d3838870545369a2bc8cdb7535283",
            "value": "100%"
          }
        },
        "7c04689ccf0c492b9fdf9fcdd1ec3d08": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c9a0b3506806417b84685ab6b26ad193",
            "max": 15,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c226380da836476fbc77dbb290bfa773",
            "value": 15
          }
        },
        "5e0007d21592421cbd960c7a18d918b9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bbdb9728cde643cca95cf7d6535bba2c",
            "placeholder": "​",
            "style": "IPY_MODEL_95c8056af87c4709baf4d7dca5ccb84c",
            "value": " 15/15 [01:31&lt;00:00,  6.72s/it]"
          }
        },
        "2db49dca74df421187cbf0cd767b12a8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "39d0925e43f749d6b335963219d4f6f8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bc6d3838870545369a2bc8cdb7535283": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c9a0b3506806417b84685ab6b26ad193": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c226380da836476fbc77dbb290bfa773": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "bbdb9728cde643cca95cf7d6535bba2c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "95c8056af87c4709baf4d7dca5ccb84c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6fd44662b4664fac8159ac992b3ee617": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_640f067893b04478a798434f96fd9815",
              "IPY_MODEL_c29e695c818e49a3bed8fa3a90e53904",
              "IPY_MODEL_38bebd46c4e4457c8375c6ac7b0eed06"
            ],
            "layout": "IPY_MODEL_c8eaa8ca1db943c48c9c97fe864abf0f"
          }
        },
        "640f067893b04478a798434f96fd9815": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c0dc1c963681423b8da39836f980a9c2",
            "placeholder": "​",
            "style": "IPY_MODEL_3df484403862488fbdf8463806ed7d3d",
            "value": "100%"
          }
        },
        "c29e695c818e49a3bed8fa3a90e53904": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c3933a8f7784478ea41eb44e68fca069",
            "max": 21,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_468244301dc74f5f95f54fd45d1a4202",
            "value": 21
          }
        },
        "38bebd46c4e4457c8375c6ac7b0eed06": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8362933b46804f1da4446fa2ffa782d7",
            "placeholder": "​",
            "style": "IPY_MODEL_cccdd9b1dfb444f5a4fbe180aa778949",
            "value": " 21/21 [02:14&lt;00:00,  5.82s/it]"
          }
        },
        "c8eaa8ca1db943c48c9c97fe864abf0f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c0dc1c963681423b8da39836f980a9c2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3df484403862488fbdf8463806ed7d3d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c3933a8f7784478ea41eb44e68fca069": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "468244301dc74f5f95f54fd45d1a4202": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "8362933b46804f1da4446fa2ffa782d7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cccdd9b1dfb444f5a4fbe180aa778949": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c37bd7d344c040478b88433efeb7ca7d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_1f57a25faf8f46bea09eac107407a95c",
              "IPY_MODEL_2b9634561f0d44ab83cf5d073d4fa88f",
              "IPY_MODEL_f934a15f197648e38a28caa06eba445f"
            ],
            "layout": "IPY_MODEL_24477365073343a1ba14223f4e36a955"
          }
        },
        "1f57a25faf8f46bea09eac107407a95c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_348b9b5caa114236b46b07e4628557e2",
            "placeholder": "​",
            "style": "IPY_MODEL_a6c88eb2239649b98896baa242faba31",
            "value": "100%"
          }
        },
        "2b9634561f0d44ab83cf5d073d4fa88f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d0a293d7ddbe4498a75461b62f590f7e",
            "max": 2,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_d904e0a728df49699f909d2359bc35ac",
            "value": 2
          }
        },
        "f934a15f197648e38a28caa06eba445f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6d9d7ae0059648a79922f03925c099ac",
            "placeholder": "​",
            "style": "IPY_MODEL_3ef3c0899ea24b2ea5d23db85a58b9f9",
            "value": " 2/2 [04:54&lt;00:00, 151.64s/it]"
          }
        },
        "24477365073343a1ba14223f4e36a955": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "348b9b5caa114236b46b07e4628557e2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a6c88eb2239649b98896baa242faba31": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d0a293d7ddbe4498a75461b62f590f7e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d904e0a728df49699f909d2359bc35ac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "6d9d7ae0059648a79922f03925c099ac": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3ef3c0899ea24b2ea5d23db85a58b9f9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "65d10277e4b7417798a70778ecb62bbd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_7a9b7280abe4436687a59cf02c480aa8",
              "IPY_MODEL_7622fb3b89fe4b8fa7c13f873245f44b",
              "IPY_MODEL_e3174ed225d7471b863d4a99adf36d57"
            ],
            "layout": "IPY_MODEL_1d3d12fa02bf464ba3972c9be3911999"
          }
        },
        "7a9b7280abe4436687a59cf02c480aa8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_27f9c2711d8446a4b4a75b014f386628",
            "placeholder": "​",
            "style": "IPY_MODEL_83ad9a056d94419992b6f6eaf2bf7369",
            "value": "100%"
          }
        },
        "7622fb3b89fe4b8fa7c13f873245f44b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1c4de8e6c3f140b19eb279198c86fb20",
            "max": 15,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_b0cf0bcb1f794a6c889c0ce97b860977",
            "value": 15
          }
        },
        "e3174ed225d7471b863d4a99adf36d57": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fb6701efde37483c8e970e221406d5ed",
            "placeholder": "​",
            "style": "IPY_MODEL_e8e72b83670f4757bc69c3a00c56d43e",
            "value": " 15/15 [02:03&lt;00:00,  9.66s/it]"
          }
        },
        "1d3d12fa02bf464ba3972c9be3911999": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "27f9c2711d8446a4b4a75b014f386628": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "83ad9a056d94419992b6f6eaf2bf7369": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1c4de8e6c3f140b19eb279198c86fb20": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b0cf0bcb1f794a6c889c0ce97b860977": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "fb6701efde37483c8e970e221406d5ed": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e8e72b83670f4757bc69c3a00c56d43e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a36f7cfe2b214a779bf26fd4c4953872": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_37b2ab070f564406bd1d76e41b6748de",
              "IPY_MODEL_307fc2978c294e72b49f9a787bf88bc8",
              "IPY_MODEL_441d078617284ae28fe96f315ae61921"
            ],
            "layout": "IPY_MODEL_6d4499a8e3fb4679a4cd6803792b5323"
          }
        },
        "37b2ab070f564406bd1d76e41b6748de": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3f61823e40a1461f9b45c17f7f192c22",
            "placeholder": "​",
            "style": "IPY_MODEL_6d2dc61b414a4803881b5ebfad9298d5",
            "value": "100%"
          }
        },
        "307fc2978c294e72b49f9a787bf88bc8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8621f7eb61e6472db68cf7f125dd6582",
            "max": 21,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_f1eda8a875ac46f883b925225b174d42",
            "value": 21
          }
        },
        "441d078617284ae28fe96f315ae61921": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_15b6239a2b284b1396a11f5e11838008",
            "placeholder": "​",
            "style": "IPY_MODEL_d84a44640cbe4deca53885c0618d098f",
            "value": " 21/21 [02:51&lt;00:00,  7.09s/it]"
          }
        },
        "6d4499a8e3fb4679a4cd6803792b5323": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3f61823e40a1461f9b45c17f7f192c22": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6d2dc61b414a4803881b5ebfad9298d5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8621f7eb61e6472db68cf7f125dd6582": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f1eda8a875ac46f883b925225b174d42": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "15b6239a2b284b1396a11f5e11838008": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d84a44640cbe4deca53885c0618d098f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Wittgenbot/fine-tuning/blob/main/LLM_as_a_judge.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Set Up"
      ],
      "metadata": {
        "id": "awE4aFuBD0Q3"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Imports"
      ],
      "metadata": {
        "id": "KTMbr15F9nrl"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import re\n",
        "import ast\n",
        "import json\n",
        "import requests\n",
        "from tqdm.auto import tqdm\n",
        "from google.colab import userdata, drive"
      ],
      "metadata": {
        "id": "P8WUySVK9nUv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Google Drive"
      ],
      "metadata": {
        "id": "hqigzGFVMv_3"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "drive.mount('/content/drive')\n",
        "data_dir = '/content/drive/My Drive/LLM as a Judge/'"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hxVjwhl0Mx80",
        "outputId": "81ddc567-def0-40dc-f42d-c8b44668a188"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Mixtral API Set Up"
      ],
      "metadata": {
        "id": "dhn237oN8_HQ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "MIXTRAL_API_KEY = userdata.get('MIXTRAL_API_KEY')"
      ],
      "metadata": {
        "id": "vqYZSVRx9loc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def query_mixtral(system_prompt, user_prompt):\n",
        "\n",
        "    url = \"https://api.mistral.ai/v1/chat/completions\"\n",
        "\n",
        "    payload = {\n",
        "        \"model\": \"open-mixtral-8x7b\",\n",
        "        \"stop\": [\"</s>\"],\n",
        "        \"stream\": False,\n",
        "        \"messages\": [\n",
        "            {\n",
        "                \"role\": \"system\",\n",
        "                \"content\": system_prompt\n",
        "            },\n",
        "            {\n",
        "                \"role\": \"user\",\n",
        "                \"content\": user_prompt\n",
        "            }\n",
        "        ]\n",
        "    }\n",
        "\n",
        "    headers = {\n",
        "        \"accept\": \"application/json\",\n",
        "        \"content-type\": \"application/json\",\n",
        "        \"Authorization\": f\"Bearer {MIXTRAL_API_KEY}\"\n",
        "    }\n",
        "\n",
        "    answer = ''\n",
        "    max_attempts = 3\n",
        "    attempt = 0\n",
        "\n",
        "    while attempt < max_attempts:\n",
        "        try:\n",
        "            response = requests.post(url, json=payload, headers=headers)\n",
        "            response.raise_for_status()\n",
        "            data = response.json()\n",
        "            answer = data.get(\"choices\", [{}])[0].get(\"message\", {}).get(\"content\", \"\")\n",
        "            break\n",
        "        except requests.RequestException as e:\n",
        "            print(f\"Attempt {attempt + 1} failed: {e}\")\n",
        "            attempt += 1\n",
        "            if attempt < max_attempts:\n",
        "                sleep_time = 2 ** attempt\n",
        "                print(f\"Retrying in {sleep_time} seconds...\")\n",
        "                sleep(sleep_time)\n",
        "            else:\n",
        "                print(\"Maximum retry attempts reached, failing.\")\n",
        "\n",
        "    return answer"
      ],
      "metadata": {
        "id": "5eYbe4Gv6PQS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Test Set Up"
      ],
      "metadata": {
        "id": "Ena7XVlr8gjW"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Test Questions"
      ],
      "metadata": {
        "id": "cUtgL3Mw8ias"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "TLP_test_questions = [\n",
        "    'How does Wittgenstein define the concept of a \"proposition\" in the Tractatus?',\n",
        "    'What role does logic play in the structure of reality as presented in the Tractatus?',\n",
        "    'Can you explain the picture theory of language proposed in the Tractatus?',\n",
        "    'How does Wittgenstein distinguish between what can be said and what can only be shown?',\n",
        "    'How are facts and states of affairs conceptualized in the Tractatus?',\n",
        "    'What is the significance of the limit of language in the Tractatus?',\n",
        "    'How does the Tractatus address the relationship between language and reality?',\n",
        "    'What is the purpose of the ladder metaphor in the conclusion of the Tractatus?',\n",
        "    'What is meant by \"propositional logic\" in the Tractatus, and how is it significant to the work’s overall argument?',\n",
        "    'How does the Tractatus critique the possibility of metaphysical propositions?',\n",
        "    'What is the role of silence in Wittgenstein’s philosophy as expressed in the Tractatus?',\n",
        "    'What did Wittgenstein mean by \"philosophy is not a body of doctrine but an activity\" in the Tractatus?',\n",
        "    'How does Wittgenstein treat the problem of solipsism in the Tractatus?',\n",
        "    'What does Wittgenstein mean by \"The limits of my language mean the limits of my world\"?',\n",
        "    'How does the Tractatus conceptualize the idea of sense and nonsense in language?',\n",
        "]\n",
        "\n",
        "PI_test_questions = [\n",
        "    'In Wittgenstein\\'s Philosophical Investigations, what determines the meaning of a word?',\n",
        "    'How did Wittgenstein use the example of \"games\" in Philosophical Investigations to illustrate the family resemblance concept?',\n",
        "    'In Philosophical Investigations, what is the concept of family resemblance?',\n",
        "    'In the context of Philosophical Investigations, is the existence of a private language possible?',\n",
        "    'How can we confirm that someone is following a rule, according to Philosophical Investigations?',\n",
        "    'In Philosophical Investigations, how can the concept of sameness be used to teach a rule?',\n",
        "    'What is the role of language-games in Wittgenstein\\'s Philosophical Investigations?',\n",
        "    'How is the idea that mental processes form the basis of our understanding of language critiqued in Philosophical Investigations?',\n",
        "    'Do private mental objects exist according to Wittgenstein’s Philosophical Investigations?',\n",
        "    'What is the relationship between forms of life and language in Philosophical Investigations?',\n",
        "    'In Philosophical Investigations, how is it explained that the meaning of a word is its use in language?',\n",
        "    'How do philosophical problems arise from misunderstandings of language, as discussed in Philosophical Investigations?',\n",
        "    'What is problematic about the Augustinian view on meaning in Philosophical Investigations?',\n",
        "    'In Philosophical Investigations, what is said about the misguided nature of philosophical questions?',\n",
        "    'What is the purpose of the analogy with the toolbox in Philosophical Investigations?',\n",
        "    'In what ways does the notion of \"language-games\" in Philosophical Investigations challenge traditional epistemology?',\n",
        "    'What implications does the critique of private language in Philosophical Investigations have for theories of consciousness?',\n",
        "    'How is the concept of pain used in Philosophical Investigations to argue against the possibility of a private language?',\n",
        "    'In what ways does the examination of pain\\'s sameness in Philosophical Investigations challenge the understanding of subjective experiences?',\n",
        "    'What are the limitations of language, as discussed in Wittgenstein’s Philosophical Investigations?',\n",
        "    'In Philosophical Investigations, what is the relationship between forms of life and understanding philosophy?'\n",
        "]\n",
        "\n",
        "test_questions = {\n",
        "    'TLP': TLP_test_questions,\n",
        "    'PI': PI_test_questions\n",
        "}"
      ],
      "metadata": {
        "id": "KVccSdjC6e_I"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Retrieve Wittgenbot \"To-Be-Judged\" Answers From Google Drive"
      ],
      "metadata": {
        "id": "HCCexm7xNZpW"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "wittgenbot_answers_file_path = data_dir + 'wittgenbot_answers.txt'\n",
        "\n",
        "with open(wittgenbot_answers_file_path, 'r') as file:\n",
        "    content = file.read()\n",
        "    wittgenbot_answers = ast.literal_eval(content)\n",
        "\n",
        "print(wittgenbot_answers)"
      ],
      "metadata": {
        "id": "du8RA68LNbf3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Generate Reference \"Golden\" Answers Through Mixtral 8x7b"
      ],
      "metadata": {
        "id": "_10iOx_Y8kQT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "system_prompt = '''\n",
        "You are an AI assistant specialized in the philosophy of Ludwig Wittgenstein. \\\n",
        "Answer the user's question. Ensure your answer is accurate, thorough, coherent and relevant. \\\n",
        "Use a tone that is conversational and clear.\n",
        "'''\n",
        "\n",
        "user_prompt = '''\n",
        "Answer this question about Ludwig Wittgenstein's philosophy:\n",
        "{question}\n",
        "'''"
      ],
      "metadata": {
        "id": "oAFe7hulBOJV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "golden_answers = {}\n",
        "\n",
        "for work, questions in tqdm(test_questions.items()):\n",
        "\n",
        "  golden_answers[work] = {}\n",
        "\n",
        "  for question in tqdm(questions):\n",
        "\n",
        "    formatted_user_prompt = user_prompt.format(question=question)\n",
        "\n",
        "    answer = query_mixtral(system_prompt, formatted_user_prompt)\n",
        "\n",
        "    golden_answers[work][question] = answer\n",
        "\n",
        "print(golden_answers)"
      ],
      "metadata": {
        "id": "VSt4V0F28dC1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Save Results to Google Drive"
      ],
      "metadata": {
        "id": "oVaQZQ8SNK6E"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "golden_answers_file_name = 'golden_answers.txt'\n",
        "golden_answers_file_path = data_dir + golden_answers_file_name\n",
        "\n",
        "golden_answers_json = json.dumps(golden_answers, indent=4)\n",
        "\n",
        "with open(golden_answers_file_path, 'w') as file:\n",
        "    file.write(golden_answers_json)\n",
        "\n",
        "print(f'File saved as {golden_answers_file_name}')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ny5QtYxmC5PZ",
        "outputId": "266fd7ab-cc4e-45a7-9596-0f7d62434cc3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "File saved as golden_answers.txt\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# LLM-as-a-Judge"
      ],
      "metadata": {
        "id": "LSuw_Wi1m7Yb"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Single Answer Grading\n",
        "### Prompts\n",
        "Prompts based on prompt templates from [LLM-as-a-Judge](https://arxiv.org/abs/2306.05685) paper."
      ],
      "metadata": {
        "id": "t2q9JwQDm_gS"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "system_prompt = '''\n",
        "Please act as an impartial judge and evaluate the quality of the response provided by an \\\n",
        "AI assistant specialized in Ludwig Wittgenstein's philosophy to the user's question about Ludwig Wittgenstein's philosophy. \\\n",
        "Your evaluation should consider factors such as the accuracy, relevance, depth, coherency, and level of detail of \\\n",
        "the response. Begin your evaluation by providing a short explanation. Be as objective as \\\n",
        "possible. After providing your explanation, please rate the response on a scale of 1 to 100 \\\n",
        "by strictly following this format: \"[[rating]]\", for example: \"Rating: [[53]]\".\n",
        "'''\n",
        "\n",
        "user_prompt = '''\n",
        "[User Question]\n",
        "{question}\n",
        "\n",
        "[The Start of Assistant’s Answer]\n",
        "{answer}\n",
        "[The End of Assistant’s Answer]\n",
        "'''"
      ],
      "metadata": {
        "id": "RaQ4PUm8nOL6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Grading"
      ],
      "metadata": {
        "id": "EZRJB-r2q9vj"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "single_grade_eval = {}\n",
        "\n",
        "for work, questions in tqdm(wittgenbot_answers.items()):\n",
        "\n",
        "    single_grade_eval[work] = {}\n",
        "\n",
        "    for question, answer in tqdm(questions.items()):\n",
        "\n",
        "      formatted_user_prompt = user_prompt.format(question=question, answer=answer)\n",
        "\n",
        "      response_grade = query_mixtral(system_prompt, formatted_user_prompt)\n",
        "\n",
        "      single_grade_eval[work][question] = response_grade"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 113,
          "referenced_widgets": [
            "ea0f693848c545a1b722692225243d47",
            "04fbb6e110194292bedd4ba9170d46bb",
            "059b18a979034523824f71a3c8fbc19c",
            "47e38f90d19741db833e6adf49bb8288",
            "7f7d4760368a43658cb70fa52e43880d",
            "70f12b0132364f608af4aeb71afee274",
            "d35aedb568704646b0e8838de1b4181e",
            "6b79b2acd65d48d28aa2b23e46ff65bd",
            "de9b8adac7094b6b834a46b7df1c2cf1",
            "b349f8d7fb784c42bf3ee1d11770d8fe",
            "b6191fc85be743da9a369ef36a806afb",
            "08d9cc1ec1084189b8755654995042e6",
            "14361477e6d54c53a3880ea8805559a9",
            "7c04689ccf0c492b9fdf9fcdd1ec3d08",
            "5e0007d21592421cbd960c7a18d918b9",
            "2db49dca74df421187cbf0cd767b12a8",
            "39d0925e43f749d6b335963219d4f6f8",
            "bc6d3838870545369a2bc8cdb7535283",
            "c9a0b3506806417b84685ab6b26ad193",
            "c226380da836476fbc77dbb290bfa773",
            "bbdb9728cde643cca95cf7d6535bba2c",
            "95c8056af87c4709baf4d7dca5ccb84c",
            "6fd44662b4664fac8159ac992b3ee617",
            "640f067893b04478a798434f96fd9815",
            "c29e695c818e49a3bed8fa3a90e53904",
            "38bebd46c4e4457c8375c6ac7b0eed06",
            "c8eaa8ca1db943c48c9c97fe864abf0f",
            "c0dc1c963681423b8da39836f980a9c2",
            "3df484403862488fbdf8463806ed7d3d",
            "c3933a8f7784478ea41eb44e68fca069",
            "468244301dc74f5f95f54fd45d1a4202",
            "8362933b46804f1da4446fa2ffa782d7",
            "cccdd9b1dfb444f5a4fbe180aa778949"
          ]
        },
        "id": "gnMYmOpuq5vj",
        "outputId": "391c4399-63fe-4321-88f6-322ac6b0e87c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/2 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "ea0f693848c545a1b722692225243d47"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/15 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "08d9cc1ec1084189b8755654995042e6"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/21 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "6fd44662b4664fac8159ac992b3ee617"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Save Results to Google Drive"
      ],
      "metadata": {
        "id": "faOcEIbqVn1c"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "single_grade_eval_file_name = 'single_grade_eval.txt'\n",
        "single_grade_eval_file_path = data_dir + single_grade_eval_file_name\n",
        "\n",
        "single_grade_eval_json = json.dumps(single_grade_eval, indent=4)\n",
        "\n",
        "with open(single_grade_eval_file_path, 'w') as file:\n",
        "    file.write(single_grade_eval_json)\n",
        "\n",
        "print(f'File saved as {single_grade_eval_file_name}')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Vc7FzwRBVsaz",
        "outputId": "6012a60f-98f3-4d9b-b207-051b322c4759"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "File saved as single_grade_eval.txt\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Reference-Guided Grading\n",
        "### Prompts\n",
        "\n"
      ],
      "metadata": {
        "id": "vfMwU8Q_nClp"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "system_prompt = '''\n",
        "Please act as an impartial judge and evaluate the quality of the response provided by an AI \\\n",
        "assistant specialized in Ludwig Wittgenstein's philosophy to the user's question about Ludwig \\\n",
        "Wittgenstein's philosophy. You will be given a reference answer and the assistant's answer. \\\n",
        "Your job is to evaluate the assistant's answer in reference to the reference answer. Your \\\n",
        "evaluation should consider factors such as the accuracy, relevance, depth, coherence, and \\\n",
        "level of detail of the response. Begin your evaluation by comparing the assistant's answer \\\n",
        "with the reference answer. Identify and correct any mistakes. Do not allow the length of the \\\n",
        "responses to influence your evaluation. Be as objective as possible. After providing your \\\n",
        "explanation, please rate the assistant's answer relative to the reference answer on a scale \\\n",
        "of 1 to 100 by strictly following this format: \"[[rating]]\", for example: \"Rating: [[53]]\".\n",
        "'''\n",
        "\n",
        "user_prompt = '''\n",
        "[User Question]\n",
        "{question}\n",
        "\n",
        "[The Start of Reference Answer]\n",
        "{ref_answer}\n",
        "[The End of Reference Answer]\n",
        "\n",
        "[The Start of Assistant Answer]\n",
        "{wittgenbot_answer}\n",
        "[The End of Assistant Answer]\n",
        "'''"
      ],
      "metadata": {
        "id": "sDFeuOnpnNbA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Grading"
      ],
      "metadata": {
        "id": "SLRhiDFuxnHh"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "ref_grade_eval = {}\n",
        "\n",
        "for work, questions in tqdm(wittgenbot_answers.items()):\n",
        "\n",
        "    ref_grade_eval[work] = {}\n",
        "\n",
        "    for question, answer in tqdm(questions.items()):\n",
        "\n",
        "      wittgenbot_answer = answer\n",
        "      ref_answer = golden_answers[work][question]\n",
        "\n",
        "      formatted_user_prompt = user_prompt.format(question=question,\n",
        "                                                 wittgenbot_answer=wittgenbot_answer,\n",
        "                                                 ref_answer=ref_answer)\n",
        "\n",
        "      response_grade = query_mixtral(system_prompt, formatted_user_prompt)\n",
        "\n",
        "      ref_grade_eval[work][question] = response_grade"
      ],
      "metadata": {
        "id": "bJ_y00y-xn4V",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 113,
          "referenced_widgets": [
            "c37bd7d344c040478b88433efeb7ca7d",
            "1f57a25faf8f46bea09eac107407a95c",
            "2b9634561f0d44ab83cf5d073d4fa88f",
            "f934a15f197648e38a28caa06eba445f",
            "24477365073343a1ba14223f4e36a955",
            "348b9b5caa114236b46b07e4628557e2",
            "a6c88eb2239649b98896baa242faba31",
            "d0a293d7ddbe4498a75461b62f590f7e",
            "d904e0a728df49699f909d2359bc35ac",
            "6d9d7ae0059648a79922f03925c099ac",
            "3ef3c0899ea24b2ea5d23db85a58b9f9",
            "65d10277e4b7417798a70778ecb62bbd",
            "7a9b7280abe4436687a59cf02c480aa8",
            "7622fb3b89fe4b8fa7c13f873245f44b",
            "e3174ed225d7471b863d4a99adf36d57",
            "1d3d12fa02bf464ba3972c9be3911999",
            "27f9c2711d8446a4b4a75b014f386628",
            "83ad9a056d94419992b6f6eaf2bf7369",
            "1c4de8e6c3f140b19eb279198c86fb20",
            "b0cf0bcb1f794a6c889c0ce97b860977",
            "fb6701efde37483c8e970e221406d5ed",
            "e8e72b83670f4757bc69c3a00c56d43e",
            "a36f7cfe2b214a779bf26fd4c4953872",
            "37b2ab070f564406bd1d76e41b6748de",
            "307fc2978c294e72b49f9a787bf88bc8",
            "441d078617284ae28fe96f315ae61921",
            "6d4499a8e3fb4679a4cd6803792b5323",
            "3f61823e40a1461f9b45c17f7f192c22",
            "6d2dc61b414a4803881b5ebfad9298d5",
            "8621f7eb61e6472db68cf7f125dd6582",
            "f1eda8a875ac46f883b925225b174d42",
            "15b6239a2b284b1396a11f5e11838008",
            "d84a44640cbe4deca53885c0618d098f"
          ]
        },
        "outputId": "0bc05c4f-aaa2-474a-d64b-4603185b8fe9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/2 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "c37bd7d344c040478b88433efeb7ca7d"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/15 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "65d10277e4b7417798a70778ecb62bbd"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/21 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "a36f7cfe2b214a779bf26fd4c4953872"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Save Results to Google Drive"
      ],
      "metadata": {
        "id": "4K-0a7_6qj67"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "ref_grade_eval_file_name = 'ref_grade_eval.txt'\n",
        "ref_grade_eval_file_path = data_dir + ref_grade_eval_file_name\n",
        "\n",
        "ref_grade_eval_json = json.dumps(ref_grade_eval, indent=4)\n",
        "\n",
        "with open(ref_grade_eval_file_path, 'w') as file:\n",
        "    file.write(ref_grade_eval_json)\n",
        "\n",
        "print(f'File saved as {ref_grade_eval_file_name}')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "q52bLV-gqnQH",
        "outputId": "8cb1a813-19de-4e00-b05f-b3868378463a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "File saved as ref_grade_eval.txt\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Parsing Results\n"
      ],
      "metadata": {
        "id": "Eu5N61AFqHmZ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "pattern = r\"\\[\\[(\\d+)\\]\\]\"\n",
        "\n",
        "def extract_rating(evaluation):\n",
        "\n",
        "    match = re.search(pattern, evaluation)\n",
        "\n",
        "    return int(match.group(1)) if match else -1"
      ],
      "metadata": {
        "id": "ogtqwZkDsg--"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Single Answer Grading"
      ],
      "metadata": {
        "id": "R4wloRm4siha"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "single_grade_eval_total = 0\n",
        "num_scores = 0\n",
        "question_counter = 1\n",
        "\n",
        "for work, questions in single_grade_eval.items():\n",
        "\n",
        "    for question, evaluation in questions.items():\n",
        "\n",
        "      score = extract_rating(evaluation)\n",
        "\n",
        "      if score >= 0:\n",
        "        single_grade_eval_total += score\n",
        "        num_scores += 1\n",
        "\n",
        "      print(f'Question {question_counter}: {score}%')\n",
        "      question_counter += 1\n",
        "\n",
        "single_grade_eval_avg_score = round(single_grade_eval_total / num_scores, 1)\n",
        "print(f'\\nSingle grade evaluation average score: {single_grade_eval_avg_score}%')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xd9rra9fqN7L",
        "outputId": "a434589b-fa1c-450b-f6c7-bc85b16f35a6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Question 1: 92%\n",
            "Question 2: 90%\n",
            "Question 3: 82%\n",
            "Question 4: 92%\n",
            "Question 5: 92%\n",
            "Question 6: 88%\n",
            "Question 7: 89%\n",
            "Question 8: 85%\n",
            "Question 9: 92%\n",
            "Question 10: 89%\n",
            "Question 11: 92%\n",
            "Question 12: 92%\n",
            "Question 13: 88%\n",
            "Question 14: 87%\n",
            "Question 15: 88%\n",
            "Question 16: 89%\n",
            "Question 17: 87%\n",
            "Question 18: 85%\n",
            "Question 19: 92%\n",
            "Question 20: 92%\n",
            "Question 21: 92%\n",
            "Question 22: 95%\n",
            "Question 23: 90%\n",
            "Question 24: 92%\n",
            "Question 25: 92%\n",
            "Question 26: 95%\n",
            "Question 27: 92%\n",
            "Question 28: 90%\n",
            "Question 29: 92%\n",
            "Question 30: 90%\n",
            "Question 31: 92%\n",
            "Question 32: 87%\n",
            "Question 33: 90%\n",
            "Question 34: 92%\n",
            "Question 35: 95%\n",
            "Question 36: 88%\n",
            "\n",
            "Single grade evaluation average score: 90.2%\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Reference-Guided Grading"
      ],
      "metadata": {
        "id": "UU8FYh3BqJCe"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "ref_grade_eval_total = 0\n",
        "num_scores = 0\n",
        "question_counter = 1\n",
        "\n",
        "for work, questions in ref_grade_eval.items():\n",
        "\n",
        "    for question, evaluation in questions.items():\n",
        "\n",
        "      score = extract_rating(evaluation)\n",
        "\n",
        "      if score >= 0:\n",
        "        ref_grade_eval_total += score\n",
        "        num_scores += 1\n",
        "\n",
        "      print(f'Question {question_counter}: {score}%')\n",
        "      question_counter += 1\n",
        "\n",
        "ref_grade_eval_avg_score = round(ref_grade_eval_total / num_scores, 1)\n",
        "print(f'\\nReference-guided evaluation average score: {ref_grade_eval_avg_score}%')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dZYnv-0JvYlH",
        "outputId": "9f87a73c-9e15-4856-8cec-a4a49ee90f5e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Question 1: 78%\n",
            "Question 2: 83%\n",
            "Question 3: 92%\n",
            "Question 4: 95%\n",
            "Question 5: 85%\n",
            "Question 6: 78%\n",
            "Question 7: 90%\n",
            "Question 8: 78%\n",
            "Question 9: 75%\n",
            "Question 10: 88%\n",
            "Question 11: 85%\n",
            "Question 12: 78%\n",
            "Question 13: 78%\n",
            "Question 14: 60%\n",
            "Question 15: 65%\n",
            "Question 16: 78%\n",
            "Question 17: 78%\n",
            "Question 18: 78%\n",
            "Question 19: 85%\n",
            "Question 20: 85%\n",
            "Question 21: 75%\n",
            "Question 22: 72%\n",
            "Question 23: 72%\n",
            "Question 24: 92%\n",
            "Question 25: 85%\n",
            "Question 26: 88%\n",
            "Question 27: 72%\n",
            "Question 28: 75%\n",
            "Question 29: 85%\n",
            "Question 30: 85%\n",
            "Question 31: 85%\n",
            "Question 32: 80%\n",
            "Question 33: 88%\n",
            "Question 34: 78%\n",
            "Question 35: 70%\n",
            "Question 36: 72%\n",
            "\n",
            "Reference-guided evaluation average score: 80.2%\n"
          ]
        }
      ]
    }
  ]
}