GenAI-Workshop-Demo6.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[{"file_id":"1gMGVSHvrHDh8Mf3okBAyK8JMEJJKE4lU","timestamp":1712772336096},{"file_id":"1tRDG7JrUiqRAJOQShKk_3hDNZvOpqN2T","timestamp":1705687451345}],"authorship_tag":"ABX9TyOj7G3QcZfuGif+Utb/DD16"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"},"widgets":{"application/vnd.jupyter.widget-state+json":{"80bd747ce9ec472288d73c8fd0986b23":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_0cdb0f3fc0bd41309f0df20fb14f48f0","IPY_MODEL_f7467fdc4ff444a89c764512cdc48262","IPY_MODEL_93c051bdb4cd4b65a201363f6334cb41"],"layout":"IPY_MODEL_3cf73d3f9a544ee1a2c4b7e097f0ccfb"}},"0cdb0f3fc0bd41309f0df20fb14f48f0":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_9944b5224bac4654838d60c11d02dfa5","placeholder":"​","style":"IPY_MODEL_40ea864c6574458b82c780f7bd6b84da","value":"run_evals "}},"f7467fdc4ff444a89c764512cdc48262":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"","description":"","description_tooltip":null,"layout":"IPY_MODEL_cb27a7c41ac14aeca5ae69666290f170","max":2,"min":0,"orientation":"horizontal","style":"IPY_MODEL_02995a3e82dc41af8fbf073b315b87e9","value":2}},"93c051bdb4cd4b65a201363f6334cb41":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_f37566ea00094393aeca205c182e1c70","placeholder":"​","style":"IPY_MODEL_bcb91fa935e34e279af633456263ebd5","value":" 2/2 (100.0%) | ⏳ 00:21&lt;00:00 |  9.74s/it"}},"3cf73d3f9a544ee1a2c4b7e097f0ccfb":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"9944b5224bac4654838d60c11d02dfa5":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"40ea864c6574458b82c780f7bd6b84da":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"cb27a7c41ac14aeca5ae69666290f170":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"02995a3e82dc41af8fbf073b315b87e9":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"f37566ea00094393aeca205c182e1c70":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"bcb91fa935e34e279af633456263ebd5":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"29e4236e6d7445feab525f065b4b048d":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_fd2b8688f41d4be2ad3280a6e58bc5e9","IPY_MODEL_3e7cd6f103c74b5ba56925b6c87b5acb","IPY_MODEL_435f4cce46c24c0ab0997260e99f6e2d"],"layout":"IPY_MODEL_fbfcc54ffe254e6886f97564d3287304"}},"fd2b8688f41d4be2ad3280a6e58bc5e9":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_8126225ebd3a41b2a84ab18497a42992","placeholder":"​","style":"IPY_MODEL_41e0e748ebeb4293a6f6c5e2b1929dff","value":"run_evals "}},"3e7cd6f103c74b5ba56925b6c87b5acb":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_f583b40533cd49b8a541bfdbaf7b0bd6","max":4,"min":0,"orientation":"horizontal","style":"IPY_MODEL_2e61666fba864740abbd3272d9046660","value":4}},"435f4cce46c24c0ab0997260e99f6e2d":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_c200bade8ca34e1faba6dca8de317db8","placeholder":"​","style":"IPY_MODEL_052873ae7348481e9b80c223c65b4f26","value":" 4/4 (100.0%) | ⏳ 00:17&lt;00:00 |  1.48s/it"}},"fbfcc54ffe254e6886f97564d3287304":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"8126225ebd3a41b2a84ab18497a42992":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"41e0e748ebeb4293a6f6c5e2b1929dff":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"f583b40533cd49b8a541bfdbaf7b0bd6":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"2e61666fba864740abbd3272d9046660":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"c200bade8ca34e1faba6dca8de317db8":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"052873ae7348481e9b80c223c65b4f26":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}}}}},"cells":[{"cell_type":"code","execution_count":1,"metadata":{"id":"Sl3lgsKvh47O","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1712775375034,"user_tz":240,"elapsed":31977,"user":{"displayName":"John Willis","userId":"15271974867993570949"}},"outputId":"6ed82fbf-40af-460c-9b39-4c9ad2df93fd"},"outputs":[{"output_type":"stream","name":"stdout","text":["\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m814.5/814.5 kB\u001b[0m \u001b[31m10.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m290.4/290.4 kB\u001b[0m \u001b[31m21.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m676.9/676.9 kB\u001b[0m \u001b[31m9.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.8/1.8 MB\u001b[0m \u001b[31m22.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m178.7/178.7 kB\u001b[0m \u001b[31m5.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.9/1.9 MB\u001b[0m \u001b[31m17.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m278.4/278.4 kB\u001b[0m \u001b[31m14.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m104.2/104.2 kB\u001b[0m \u001b[31m4.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m307.7/307.7 kB\u001b[0m \u001b[31m15.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m268.3/268.3 kB\u001b[0m \u001b[31m12.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.4/49.4 kB\u001b[0m \u001b[31m1.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m53.0/53.0 kB\u001b[0m \u001b[31m3.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m144.8/144.8 kB\u001b[0m \u001b[31m4.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.6/75.6 kB\u001b[0m \u001b[31m5.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.9/77.9 kB\u001b[0m \u001b[31m5.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m5.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25h"]}],"source":["%pip install --quiet langchain pypdf pymongo langchain-openai tiktoken google-cloud-secret-manager"]},{"cell_type":"code","source":["%pip install --upgrade google-auth"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":421},"id":"A48y0kGZcaGd","executionInfo":{"status":"ok","timestamp":1712775399583,"user_tz":240,"elapsed":7558,"user":{"displayName":"John Willis","userId":"15271974867993570949"}},"outputId":"33dadfbc-2c76-499a-955d-0e94b3691c27"},"execution_count":2,"outputs":[{"output_type":"stream","name":"stdout","text":["Requirement already satisfied: google-auth in /usr/local/lib/python3.10/dist-packages (2.27.0)\n","Collecting google-auth\n","  Downloading google_auth-2.29.0-py2.py3-none-any.whl (189 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m189.2/189.2 kB\u001b[0m \u001b[31m4.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from google-auth) (5.3.3)\n","Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from google-auth) (0.4.0)\n","Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.10/dist-packages (from google-auth) (4.9)\n","Requirement already satisfied: pyasn1<0.7.0,>=0.4.6 in /usr/local/lib/python3.10/dist-packages (from pyasn1-modules>=0.2.1->google-auth) (0.6.0)\n","Installing collected packages: google-auth\n","  Attempting uninstall: google-auth\n","    Found existing installation: google-auth 2.27.0\n","    Uninstalling google-auth-2.27.0:\n","      Successfully uninstalled google-auth-2.27.0\n","\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n","google-colab 1.0.0 requires google-auth==2.27.0, but you have google-auth 2.29.0 which is incompatible.\u001b[0m\u001b[31m\n","\u001b[0mSuccessfully installed google-auth-2.29.0\n"]},{"output_type":"display_data","data":{"application/vnd.colab-display-data+json":{"pip_warning":{"packages":["google"]},"id":"898bc7d615b94cda90bf241ab5be6b63"}},"metadata":{}}]},{"cell_type":"code","source":["import os\n","\n","%pip show langchain\n","\n","from platform import python_version\n","print(python_version())"],"metadata":{"id":"WlkitefoiJ24","executionInfo":{"status":"ok","timestamp":1712775415272,"user_tz":240,"elapsed":6181,"user":{"displayName":"John Willis","userId":"15271974867993570949"}},"colab":{"base_uri":"https://localhost:8080/"},"outputId":"0170ea4f-1ac6-4a17-f953-989b9281e061"},"execution_count":3,"outputs":[{"output_type":"stream","name":"stdout","text":["Name: langchain\n","Version: 0.1.15\n","Summary: Building applications with LLMs through composability\n","Home-page: https://github.com/langchain-ai/langchain\n","Author: \n","Author-email: \n","License: MIT\n","Location: /usr/local/lib/python3.10/dist-packages\n","Requires: aiohttp, async-timeout, dataclasses-json, jsonpatch, langchain-community, langchain-core, langchain-text-splitters, langsmith, numpy, pydantic, PyYAML, requests, SQLAlchemy, tenacity\n","Required-by: \n","3.10.12\n"]}]},{"cell_type":"code","source":["!pip install arize-phoenix"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"vRd6LDTiFidh","executionInfo":{"status":"ok","timestamp":1712775515537,"user_tz":240,"elapsed":95419,"user":{"displayName":"John Willis","userId":"15271974867993570949"}},"outputId":"0f77017d-dde0-4925-d90f-6bb1d4801ae8"},"execution_count":4,"outputs":[{"output_type":"stream","name":"stdout","text":["Collecting arize-phoenix\n","  Downloading arize_phoenix-3.19.4-py3-none-any.whl (1.3 MB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m9.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hCollecting hdbscan>=0.8.33 (from arize-phoenix)\n","  Downloading hdbscan-0.8.33.tar.gz (5.2 MB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.2/5.2 MB\u001b[0m \u001b[31m59.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n","  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n","  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n","Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from arize-phoenix) (3.1.3)\n","Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from arize-phoenix) (1.25.2)\n","Collecting openinference-instrumentation (from arize-phoenix)\n","  Downloading openinference_instrumentation-0.1.1-py3-none-any.whl (6.5 kB)\n","Collecting openinference-instrumentation-langchain>=0.1.12 (from arize-phoenix)\n","  Downloading openinference_instrumentation_langchain-0.1.14-py3-none-any.whl (13 kB)\n","Collecting openinference-instrumentation-llama-index>=1.2.0 (from arize-phoenix)\n","  Downloading openinference_instrumentation_llama_index-1.2.1-py3-none-any.whl (16 kB)\n","Collecting openinference-instrumentation-openai>=0.1.4 (from arize-phoenix)\n","  Downloading openinference_instrumentation_openai-0.1.4-py3-none-any.whl (21 kB)\n","Collecting openinference-semantic-conventions>=0.1.5 (from arize-phoenix)\n","  Downloading openinference_semantic_conventions-0.1.5-py3-none-any.whl (8.4 kB)\n","Collecting opentelemetry-exporter-otlp (from arize-phoenix)\n","  Downloading opentelemetry_exporter_otlp-1.24.0-py3-none-any.whl (7.0 kB)\n","Collecting opentelemetry-proto (from arize-phoenix)\n","  Downloading opentelemetry_proto-1.24.0-py3-none-any.whl (50 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m50.8/50.8 kB\u001b[0m \u001b[31m6.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hCollecting opentelemetry-sdk (from arize-phoenix)\n","  Downloading opentelemetry_sdk-1.24.0-py3-none-any.whl (106 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m106.1/106.1 kB\u001b[0m \u001b[31m13.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from arize-phoenix) (2.0.3)\n","Requirement already satisfied: protobuf<6.0,>=3.20 in /usr/local/lib/python3.10/dist-packages (from arize-phoenix) (3.20.3)\n","Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from arize-phoenix) (5.9.5)\n","Requirement already satisfied: pyarrow in /usr/local/lib/python3.10/dist-packages (from arize-phoenix) (14.0.2)\n","Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from arize-phoenix) (2.31.0)\n","Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (from arize-phoenix) (1.2.2)\n","Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from arize-phoenix) (1.11.4)\n","Requirement already satisfied: sortedcontainers in /usr/local/lib/python3.10/dist-packages (from arize-phoenix) (2.4.0)\n","Collecting starlette (from arize-phoenix)\n","  Downloading starlette-0.37.2-py3-none-any.whl (71 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m71.9/71.9 kB\u001b[0m \u001b[31m10.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hCollecting strawberry-graphql==0.208.2 (from arize-phoenix)\n","  Downloading strawberry_graphql-0.208.2-py3-none-any.whl (269 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m269.0/269.0 kB\u001b[0m \u001b[31m1.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from arize-phoenix) (4.66.2)\n","Requirement already satisfied: typing-extensions>=4.5 in /usr/local/lib/python3.10/dist-packages (from arize-phoenix) (4.10.0)\n","Collecting umap-learn (from arize-phoenix)\n","  Downloading umap_learn-0.5.6-py3-none-any.whl (85 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m85.7/85.7 kB\u001b[0m \u001b[31m10.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hCollecting uvicorn (from arize-phoenix)\n","  Downloading uvicorn-0.29.0-py3-none-any.whl (60 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.8/60.8 kB\u001b[0m \u001b[31m7.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: wrapt in /usr/local/lib/python3.10/dist-packages (from arize-phoenix) (1.14.1)\n","Collecting graphql-core<3.3.0,>=3.2.0 (from strawberry-graphql==0.208.2->arize-phoenix)\n","  Downloading graphql_core-3.2.3-py3-none-any.whl (202 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m202.9/202.9 kB\u001b[0m \u001b[31m21.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: python-dateutil<3.0.0,>=2.7.0 in /usr/local/lib/python3.10/dist-packages (from strawberry-graphql==0.208.2->arize-phoenix) (2.8.2)\n","Collecting cython<3,>=0.27 (from hdbscan>=0.8.33->arize-phoenix)\n","  Using cached Cython-0.29.37-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl (1.9 MB)\n","Requirement already satisfied: joblib>=1.0 in /usr/local/lib/python3.10/dist-packages (from hdbscan>=0.8.33->arize-phoenix) (1.3.2)\n","Collecting opentelemetry-api (from openinference-instrumentation-langchain>=0.1.12->arize-phoenix)\n","  Downloading opentelemetry_api-1.24.0-py3-none-any.whl (60 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.1/60.1 kB\u001b[0m \u001b[31m6.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hCollecting opentelemetry-instrumentation (from openinference-instrumentation-langchain>=0.1.12->arize-phoenix)\n","  Downloading opentelemetry_instrumentation-0.45b0-py3-none-any.whl (28 kB)\n","Collecting opentelemetry-semantic-conventions (from openinference-instrumentation-langchain>=0.1.12->arize-phoenix)\n","  Downloading opentelemetry_semantic_conventions-0.45b0-py3-none-any.whl (36 kB)\n","Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->arize-phoenix) (3.4.0)\n","Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->arize-phoenix) (2.1.5)\n","Collecting opentelemetry-exporter-otlp-proto-grpc==1.24.0 (from opentelemetry-exporter-otlp->arize-phoenix)\n","  Downloading opentelemetry_exporter_otlp_proto_grpc-1.24.0-py3-none-any.whl (18 kB)\n","Collecting opentelemetry-exporter-otlp-proto-http==1.24.0 (from opentelemetry-exporter-otlp->arize-phoenix)\n","  Downloading opentelemetry_exporter_otlp_proto_http-1.24.0-py3-none-any.whl (16 kB)\n","Collecting deprecated>=1.2.6 (from opentelemetry-exporter-otlp-proto-grpc==1.24.0->opentelemetry-exporter-otlp->arize-phoenix)\n","  Downloading Deprecated-1.2.14-py2.py3-none-any.whl (9.6 kB)\n","Requirement already satisfied: googleapis-common-protos~=1.52 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-exporter-otlp-proto-grpc==1.24.0->opentelemetry-exporter-otlp->arize-phoenix) (1.63.0)\n","Requirement already satisfied: grpcio<2.0.0,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-exporter-otlp-proto-grpc==1.24.0->opentelemetry-exporter-otlp->arize-phoenix) (1.62.1)\n","Collecting opentelemetry-exporter-otlp-proto-common==1.24.0 (from opentelemetry-exporter-otlp-proto-grpc==1.24.0->opentelemetry-exporter-otlp->arize-phoenix)\n","  Downloading opentelemetry_exporter_otlp_proto_common-1.24.0-py3-none-any.whl (17 kB)\n","Collecting importlib-metadata<=7.0,>=6.0 (from opentelemetry-api->openinference-instrumentation-langchain>=0.1.12->arize-phoenix)\n","  Downloading importlib_metadata-7.0.0-py3-none-any.whl (23 kB)\n","Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->arize-phoenix) (3.3.2)\n","Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->arize-phoenix) (3.6)\n","Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->arize-phoenix) (2.0.7)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->arize-phoenix) (2024.2.2)\n","Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->arize-phoenix) (2023.4)\n","Requirement already satisfied: tzdata>=2022.1 in /usr/local/lib/python3.10/dist-packages (from pandas->arize-phoenix) (2024.1)\n","Requirement already satisfied: anyio<5,>=3.4.0 in /usr/local/lib/python3.10/dist-packages (from starlette->arize-phoenix) (3.7.1)\n","Requirement already satisfied: numba>=0.51.2 in /usr/local/lib/python3.10/dist-packages (from umap-learn->arize-phoenix) (0.58.1)\n","Collecting pynndescent>=0.5 (from umap-learn->arize-phoenix)\n","  Downloading pynndescent-0.5.12-py3-none-any.whl (56 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.8/56.8 kB\u001b[0m \u001b[31m6.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: click>=7.0 in /usr/local/lib/python3.10/dist-packages (from uvicorn->arize-phoenix) (8.1.7)\n","Requirement already satisfied: h11>=0.8 in /usr/local/lib/python3.10/dist-packages (from uvicorn->arize-phoenix) (0.14.0)\n","Requirement already satisfied: sniffio>=1.1 in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.4.0->starlette->arize-phoenix) (1.3.1)\n","Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.4.0->starlette->arize-phoenix) (1.2.0)\n","Requirement already satisfied: llvmlite<0.42,>=0.41.0dev0 in /usr/local/lib/python3.10/dist-packages (from numba>=0.51.2->umap-learn->arize-phoenix) (0.41.1)\n","Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil<3.0.0,>=2.7.0->strawberry-graphql==0.208.2->arize-phoenix) (1.16.0)\n","Requirement already satisfied: setuptools>=16.0 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-instrumentation->openinference-instrumentation-langchain>=0.1.12->arize-phoenix) (67.7.2)\n","Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.10/dist-packages (from importlib-metadata<=7.0,>=6.0->opentelemetry-api->openinference-instrumentation-langchain>=0.1.12->arize-phoenix) (3.18.1)\n","Building wheels for collected packages: hdbscan\n","  Building wheel for hdbscan (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n","  Created wheel for hdbscan: filename=hdbscan-0.8.33-cp310-cp310-linux_x86_64.whl size=3039296 sha256=e4890402334deabf16c3228d4847acc419baaddbd1f0213b0a0f04a7a49220fd\n","  Stored in directory: /root/.cache/pip/wheels/75/0b/3b/dc4f60b7cc455efaefb62883a7483e76f09d06ca81cf87d610\n","Successfully built hdbscan\n","Installing collected packages: uvicorn, opentelemetry-semantic-conventions, opentelemetry-proto, openinference-semantic-conventions, importlib-metadata, graphql-core, deprecated, cython, strawberry-graphql, starlette, opentelemetry-exporter-otlp-proto-common, opentelemetry-api, pynndescent, opentelemetry-sdk, opentelemetry-instrumentation, openinference-instrumentation, hdbscan, umap-learn, opentelemetry-exporter-otlp-proto-http, opentelemetry-exporter-otlp-proto-grpc, openinference-instrumentation-openai, openinference-instrumentation-llama-index, openinference-instrumentation-langchain, opentelemetry-exporter-otlp, arize-phoenix\n","  Attempting uninstall: importlib-metadata\n","    Found existing installation: importlib_metadata 7.1.0\n","    Uninstalling importlib_metadata-7.1.0:\n","      Successfully uninstalled importlib_metadata-7.1.0\n","  Attempting uninstall: cython\n","    Found existing installation: Cython 3.0.10\n","    Uninstalling Cython-3.0.10:\n","      Successfully uninstalled Cython-3.0.10\n","Successfully installed arize-phoenix-3.19.4 cython-0.29.37 deprecated-1.2.14 graphql-core-3.2.3 hdbscan-0.8.33 importlib-metadata-7.0.0 openinference-instrumentation-0.1.1 openinference-instrumentation-langchain-0.1.14 openinference-instrumentation-llama-index-1.2.1 openinference-instrumentation-openai-0.1.4 openinference-semantic-conventions-0.1.5 opentelemetry-api-1.24.0 opentelemetry-exporter-otlp-1.24.0 opentelemetry-exporter-otlp-proto-common-1.24.0 opentelemetry-exporter-otlp-proto-grpc-1.24.0 opentelemetry-exporter-otlp-proto-http-1.24.0 opentelemetry-instrumentation-0.45b0 opentelemetry-proto-1.24.0 opentelemetry-sdk-1.24.0 opentelemetry-semantic-conventions-0.45b0 pynndescent-0.5.12 starlette-0.37.2 strawberry-graphql-0.208.2 umap-learn-0.5.6 uvicorn-0.29.0\n"]}]},{"cell_type":"code","source":["from urllib.request import urlopen\n","\n","import nest_asyncio\n","import numpy as np\n","import pandas as pd\n","import phoenix as px\n","from langchain.chains import RetrievalQA\n","from langchain.chat_models import ChatOpenAI\n","from langchain.embeddings import OpenAIEmbeddings\n","from langchain.retrievers import KNNRetriever\n","from phoenix.experimental.evals import (\n","    HallucinationEvaluator,\n","    OpenAIModel,\n","    QAEvaluator,\n","    RelevanceEvaluator,\n","    run_evals,\n",")\n","from phoenix.session.evaluation import get_qa_with_reference, get_retrieved_documents\n","from phoenix.trace import DocumentEvaluations, SpanEvaluations\n","from phoenix.trace.langchain import LangChainInstrumentor\n","from tqdm import tqdm\n","\n","nest_asyncio.apply()  # needed for concurrent evals in notebook environments"],"metadata":{"id":"mDx-ckXTFtSf","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1712775549981,"user_tz":240,"elapsed":29365,"user":{"displayName":"John Willis","userId":"15271974867993570949"}},"outputId":"b24d240b-3ebf-4027-d27e-352ad39626cd"},"execution_count":5,"outputs":[{"output_type":"stream","name":"stderr","text":["WARNING:phoenix.experimental.evals:Evals are moving out of experimental. Install the evals extra with `pip install arize-phoenix[evals]` and import `phoenix.evals`. For more info, see the [migration guide](https://github.com/Arize-ai/phoenix/blob/main/MIGRATION.md).\n"]}]},{"cell_type":"code","source":["import os\n","from google.cloud import secretmanager\n","from google.colab import auth\n","from google.colab import drive"],"metadata":{"id":"5Ca6p8tnd-wL","executionInfo":{"status":"ok","timestamp":1712775565704,"user_tz":240,"elapsed":390,"user":{"displayName":"John Willis","userId":"15271974867993570949"}}},"execution_count":6,"outputs":[]},{"cell_type":"code","source":["def load_secrets(secrets_name, project_id):\n","  # Build a client\n","  auth.authenticate_user()\n","  client = secretmanager.SecretManagerServiceClient()\n","  secret_name = secrets_name\n","  # Create path to latest secret\n","  resource_name = f\"projects/{project_id}/secrets/{secret_name}/versions/latest\"\n","  # Get your secret :\n","  response = client.access_secret_version(request={\"name\": resource_name})\n","  secret_string = response.payload.data.decode('UTF-8')\n","  return secret_string"],"metadata":{"id":"sCa55hgW2ikm","executionInfo":{"status":"ok","timestamp":1712775581618,"user_tz":240,"elapsed":181,"user":{"displayName":"John Willis","userId":"15271974867993570949"}}},"execution_count":7,"outputs":[]},{"cell_type":"code","source":["project_id = 'botchagalupep1'\n","openai_api_key = load_secrets(\"openai_api_key\",project_id)\n","os.environ['OPENAI_API_KEY'] = openai_api_key\n","MONGODB_ATLAS_CLUSTER_URI = load_secrets(\"MDB_CLUSTER0_URI\",project_id)\n","langsmith_api_key = load_secrets(\"langsmith_api_key\",project_id)"],"metadata":{"id":"mWn3YgC12nlg","executionInfo":{"status":"ok","timestamp":1712775606825,"user_tz":240,"elapsed":22248,"user":{"displayName":"John Willis","userId":"15271974867993570949"}}},"execution_count":8,"outputs":[]},{"cell_type":"code","source":["# connect notebook to langsmith\n","os.environ['LANGCHAIN_TRACING_V2'] = 'true'\n","os.environ['LANGCHAIN_ENDPOINT'] = 'https://api.smith.langchain.com'\n","\n","# This key is sourced from vars.env\n","os.environ['LANGCHAIN_API_KEY'] = langsmith_api_key  # Uncomment and replace  with your actual API key\n","\n","os.environ['LANGCHAIN_PROJECT'] = 'opencontext-1'\n","\n","# To verify, you can print the variables\n","print(os.environ.get('LANGCHAIN_TRACING_V2'))\n","print(os.environ.get('LANGCHAIN_ENDPOINT'))\n","#print(os.environ.get('LANGCHAIN_API_KEY'))  # Uncomment if you want to print your API key (be careful with sharing your notebook)\n","print(os.environ.get('LANGCHAIN_PROJECT'))"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"NpU8CL2r22vD","executionInfo":{"status":"ok","timestamp":1712775611014,"user_tz":240,"elapsed":222,"user":{"displayName":"John Willis","userId":"15271974867993570949"}},"outputId":"385b4fda-9146-43e7-b2fd-e1bbb68cec30"},"execution_count":9,"outputs":[{"output_type":"stream","name":"stdout","text":["true\n","https://api.smith.langchain.com\n","opencontext-1\n"]}]},{"cell_type":"code","source":["from pymongo import MongoClient\n","\n","# initialize MongoDB python client\n","client = MongoClient(MONGODB_ATLAS_CLUSTER_URI)\n","\n","DB_NAME = \"Cluster0\"\n","COLLECTION_NAME = \"Langchain_demo\"\n","ATLAS_VECTOR_SEARCH_INDEX_NAME = \"vector_index\"\n","\n","MONGODB_COLLECTION = client[DB_NAME][COLLECTION_NAME]"],"metadata":{"id":"aPqh0syziQ2k","executionInfo":{"status":"ok","timestamp":1712775735591,"user_tz":240,"elapsed":241,"user":{"displayName":"John Willis","userId":"15271974867993570949"}}},"execution_count":14,"outputs":[]},{"cell_type":"code","source":["from langchain_community.vectorstores import MongoDBAtlasVectorSearch\n","from langchain_openai import OpenAIEmbeddings\n","\n","vector_search = MongoDBAtlasVectorSearch.from_connection_string(\n","    MONGODB_ATLAS_CLUSTER_URI,\n","    DB_NAME + \".\" + COLLECTION_NAME,\n","    OpenAIEmbeddings(disallowed_special=()),\n","    index_name=ATLAS_VECTOR_SEARCH_INDEX_NAME,\n",")"],"metadata":{"id":"7himXISricVE","executionInfo":{"status":"ok","timestamp":1712775737857,"user_tz":240,"elapsed":337,"user":{"displayName":"John Willis","userId":"15271974867993570949"}}},"execution_count":15,"outputs":[]},{"cell_type":"markdown","source":["#Similarity Search with Score"],"metadata":{"id":"jddBa70YjpAt"}},{"cell_type":"code","source":["session = px.launch_app()\n","LangChainInstrumentor().instrument()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":106},"id":"dYv2b5P1F95o","executionInfo":{"status":"ok","timestamp":1712775745536,"user_tz":240,"elapsed":1424,"user":{"displayName":"John Willis","userId":"15271974867993570949"}},"outputId":"14ac16b3-cac9-47ab-c22d-818e811eeade"},"execution_count":16,"outputs":[{"output_type":"stream","name":"stderr","text":["WARNING:phoenix.session.session:Existing running Phoenix instance detected! Shutting it down and starting a new instance...\n","WARNING:opentelemetry.instrumentation.instrumentor:Attempting to instrument while already instrumented\n"]},{"output_type":"stream","name":"stdout","text":["🌍 To view the Phoenix app in your browser, visit https://g0qkfncqrqm2-496ff2e9c6d22116-6006-colab.googleusercontent.com/\n","📺 To view the Phoenix app in a notebook, run `px.active_session().view()`\n","📖 For more information on how to use Phoenix, check out https://docs.arize.com/phoenix\n"]}]},{"cell_type":"markdown","source":[],"metadata":{"id":"8fsP9JtfWkqB"}},{"cell_type":"code","source":["query = \"What is GPT-4?\"\n","\n","results = vector_search.similarity_search_with_score(\n","    query=query,\n","    k=5,\n",")\n","\n","# Display results\n","for result in results:\n","    print(result)"],"metadata":{"id":"5pflRaNwjh8w","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1712775760586,"user_tz":240,"elapsed":480,"user":{"displayName":"John Willis","userId":"15271974867993570949"}},"outputId":"c498e123-8ad8-4bf4-869d-fea41dee30f5"},"execution_count":17,"outputs":[{"output_type":"stream","name":"stdout","text":["(Document(page_content='GPT-4 Technical Report\\nOpenAI∗\\nAbstract\\nWe report the development of GPT-4, a large-scale, multimodal model which can\\naccept image and text inputs and produce text outputs. While less capable than\\nhumans in many real-world scenarios, GPT-4 exhibits human-level performance\\non various professional and academic benchmarks, including passing a simulated\\nbar exam with a score around the top 10% of test takers. GPT-4 is a Transformer-\\nbased model pre-trained to predict the next token in a document. The post-training\\nalignment process results in improved performance on measures of factuality and\\nadherence to desired behavior. A core component of this project was developing\\ninfrastructure and optimization methods that behave predictably across a wide\\nrange of scales. This allowed us to accurately predict some aspects of GPT-4’s\\nperformance based on models trained with no more than 1/1,000th the compute of\\nGPT-4.\\n1 Introduction', metadata={'_id': ObjectId('6616def76182e3cd8e9a220f'), 'embedding': [-0.007480065685650684, -0.01595565552119774, -0.015423800523422316, -0.010412087782435717, 0.025665420897754337, 0.012246306029520163, -0.010384813071644926, 0.014810121393274518, -0.016160214920806173, -0.016828442541213053, 0.031065796870525957, 0.03752988283884843, -0.03635707586277942, -0.018478557887443407, -0.003220110414605576, 0.02839288452625344, 0.022487927335174684, -0.006481132429736088, -0.002013208187403074, -0.016378412607132502, 0.00354911056578168, -0.0001317918401462636, -0.02869290448230714, -0.014387364307339758, -0.0001248666424374104, 0.00275303238929703, 0.02738372208963917, -0.0376117088338658, -0.0028246282722922313, -0.003740032842825342, 0.01680116783042226, -0.005720852427579664, -0.03251135095450288, -0.0021529906145446275, -0.0016458531488003888, -0.010507549270203486, -0.00015352630254810486, -0.005553795522477944, 0.016610246717531724, -0.007241412897553763, 0.002875768122194339, 0.013057726347239946, 0.022678850310710225, -0.004357121078991363, -0.00937565249767566, 0.0026405246729462673, 0.020578705030399318, -0.0400391487810212, -0.00916427395470828, 0.018273998487834977, 0.02172423915832254, 0.03079305162526305, -0.004360530650670838, -0.0012171301273337363, -0.00928019100990789, -0.0031791983484193895, -0.010848482223765875, 0.026933692155558624, 0.009048356899508668, 0.00017217107344604795, -0.00020786246582866507, -0.025365400941700637, -0.013391841088765887, 0.03570248280380043, -0.009450657486689084, -0.007241412897553763, -0.04917614895626119, 0.03477514636220354, -0.004046872407899241, 0.0009068812816186449, 0.02138330620476015, 0.01307818284599429, -0.0004589807873348737, -0.01316682519040311, 0.032565900376084464, -0.0010321740186525908, -0.005703805966166045, -0.009607486608074882, -0.030165735139719856, 0.002391643635472174, 0.024738084456157446, -0.0069891227540614465, 0.0160783907884338, 0.029538418654176665, 0.009450657486689084, 0.0013688452404462613, 0.011053041623374307, 0.0031672657788637307, -0.02256975146754706, -0.007861910705399258, 0.011584896621149731, 0.00527763977421306, 0.022910684421109447, 0.03322049343506345, -0.006682283188987548, 0.007834635994608468, -0.0034229652612048963, 0.03690256635330524, 0.0038184474035182414, -0.035675209955654645, -0.0012427000522847902, -0.004244613595471226, -0.015519261079867584, -0.01592838081040695, -0.006286800813843577, 0.007336873919660281, 0.012075839552738969, 0.010800751014220738, 0.014223716973917512, 0.017442122602683352, -0.01749667202426493, 0.0324568015329213, -0.0057447175666909805, -0.013944151653973154, -0.020415056765654572, 0.002775192975399235, 0.024574436191412696, -0.013412296656197729, -0.006655008478196756, 0.0022058352502864727, 0.019610455591293735, -0.00010451721666695696, 0.011803093376153559, 0.0026337059952485694, 0.023974394416660296, 0.012205394894656477, -0.014100980775358951, -0.02016958436853745, 0.0013526509391218855, -0.003988913880299435, 0.018969500819032644, 0.014360089596548966, 0.01110077190159694, -0.00496057242542324, -0.03444784983271404, 0.020196859079328243, -0.02776556804071025, 0.007786905250724583, -0.039302733452314845, -0.019855926125765852, 0.01944680732654899, 0.02814741212913632, -0.02241974148952021, -0.0031706751177125797, -0.009784771296892524, 0.009389289853071055, 0.02113783380764303, 0.013466846077779312, 0.007445972297162195, 0.009689310740447257, 0.012580420771046104, -0.009355195998921316, -0.024042582124959775, 0.022283367935566255, 0.014169167552335929, 0.00027743407571023356, 0.005533339489384851, 0.016582972006740936, -0.02666094504765071, -0.022951597418618137, -0.009184729522140122, 0.01722392491635702, 0.0020609386984563327, 0.0044764477058704495, 0.013296379600998116, 0.02978388918864878, 0.022828860288737072, 0.019351346770103722, -0.004885566970748564, -0.015846556678034574, -0.02127420736159699, 0.026347286804879112, -0.05135811836894447, 0.009764315729460682, -0.0009708061522039361, 0.01298272135822652, 0.009662036029656465, -0.02090599969724381, -0.002389939082463062, -0.007425516729730353, 0.007752811862236094, 0.026224551537643054, 0.000892391591511037, 0.01083484486837048, -0.008284666860011518, -0.03510244289169303, 0.019569542593785048, -0.00015118239664143533, -0.004742375204758161, -0.027410996800429962, 0.028720179193097932, 0.01174172574253553, -0.003651390265585896, -0.01498740608209216, -0.6148791429167925, -0.02071507672170827, -0.011632626899372365, -0.022487927335174684, 0.024588074478130596, 0.0027035970924040335, 0.0007146804151951801, 0.006273163458448181, -0.016937541384376215, 0.01331683516842996, -0.008748336012132466, -0.01868311728705184, -0.021165108518433822, -0.0052810493458925346, 0.005390147723394448, -0.03722986288279473, -0.017278474337938603, -0.016392049031205395, 0.010050699261441486, 0.012546326916896363, -0.018724030284560527, 0.012539508704859917, -0.007398242018939561, -0.0015060707216588349, -0.02135603149396936, -0.0045446344828474275, -0.008871071279368524, -0.019405896191685305, -0.005710624643863743, 0.007711900261711158, -0.03349324054297136, 0.015532898435262978, 0.009273371866548943, 0.0012725317090045618, 0.04044826758023806, -0.008005102471389662, -0.03466604751904037, 0.016051116077643007, -0.0061879302200575845, 0.026347286804879112, -0.018505832598234198, -0.02386529557349713, 0.0031774937954102776, 0.0056628938999798584, -0.011584896621149731, 0.029565693364967456, 0.013310016956393512, -0.005587888910966433, -0.00764371348473418, -0.011278057521737083, -0.03232042984161235, 0.007793723928422281, 0.023783473303769762, 0.0023831204047653647, 0.003221814967614688, 0.012784980170654535, 0.04194837108579657, 0.01674662027148568, 0.0038150380646693924, -0.024915369144975087, 0.0027206437866482782, 0.011803093376153559, -0.008141475094021116, -0.031229445135270706, -0.021710600871604643, 0.008475589835547057, 0.007391423341241864, 0.011400792788973142, -0.0006754731348487305, -0.045548621734310984, 0.0021649234169309117, 0.024969918566556667, 0.010562098691785069, -0.008032377182180453, 0.003431488724742331, 0.01052800483763533, 0.023851659149424238, -0.0203195943465643, 0.02127420736159699, 0.0203195943465643, 0.027438271511220753, -0.0227470361563647, 0.00214617193684693, 0.004810561981735139, 0.007711900261711158, 0.01686935553872174, -0.01037799485960848, 0.011878099296489485, 0.019214973216149765, 0.003965048741188118, 0.017455759026756244, 0.017728506134664156, -0.029156572703105586, -0.03575703222538201, 0.01471465990550675, 0.009709766307879099, -0.027861028597155515, -0.0021308300284424227, 0.001003194754852688, -0.009389289853071055, -0.012157663685111342, -0.022324280933074942, 0.04197564765923236, 0.03403873103349718, 0.04085738637945493, 0.023619825039025013, -0.018273998487834977, 0.0019637728905100775, -0.0034689912192490437, -0.0151646907709098, -0.016405687317923294, -0.0011779228469872866, -0.019174060218641077, -0.013412296656197729, 0.020824175564871435, -0.02222881851398467, 0.008482408047583503, -0.006184521114039361, -0.02046960618723615, -0.010432544281190061, 0.024192592102986625, -0.0016526718264980866, 0.007971008617239922, 0.003695711670620932, 0.010943942780211142, 0.012614513693873342, 0.008782428934959705, -0.032265880420030764, -0.0016194308309294661, 0.008046014537575849, -0.006873205233200584, -0.016760256695558574, 0.057440358386195864, -0.006116334337062383, 0.020619616165263, 0.02237882849201152, 0.047676045450702684, 0.00703685303228408, 0.010248440449013472, 0.0005510326743193403, -0.022856134999527863, 0.01838309733099814, -0.0034263746000537452, 0.014060068709172764, -0.027301899819911803, -0.009832502506437659, -0.007486884363348382, 0.0009230755829430208, 0.006890252160275454, -0.017878516112691006, -0.007752811862236094, 0.0050696708029251535, -0.0058129043436679585, 0.010487092771449142, 0.0011395679595607056, -0.038539045275462695, -0.012866804303026908, -0.04715782408303265, -0.005625391405473146, -0.025160841542092208, 0.0039036806419088382, 0.01037799485960848, -0.0018035345466907429, 0.0010935420015165582, 0.001019389056177064, -0.014060068709172764, -0.006862977449484662, 0.040666465266564394, -0.040666465266564394, -0.032184058150303396, 0.0028740635691852276, -0.046339586484598926, 0.007030034354586382, 0.008121019526589272, -0.029156572703105586, 0.007091402453865662, 0.005904956259756253, -0.008155112449416512, 0.008482408047583503, -0.0294293198110135, -0.012559964272291758, 0.02983843861023036, -0.013753229609760116, -0.0209332744080346, 0.013903239587786966, 0.029892988031811944, 0.04143015344341654, 0.02105600967527066, -0.012969084002831125, 0.0018717210908370955, 0.001670570797246887, 0.015082867569859928, -0.024001669127451088, 0.03594795333827255, -0.0012631560853778837, 0.023019783264272613, 0.011523528987531702, 0.0009102906204674937, 0.013732773111005772, 0.04044826758023806, 0.00621861450252785, 0.019501356748130572, 0.00626293567473226, -0.02022413379011903, -0.002585975251364685, -0.008959714555099847, 0.009907507495451084, -0.019105874372986602, 0.0007257607082462826, 0.003085441879321983, 0.008202843658961647, -0.011700813676349342, -0.02479263387773903, -0.005475380961785045, 0.00992114485084648, 0.015028318148278347, -0.008802885433714049, 0.013275923102243773, -0.016378412607132502, -0.023224342663881042, 0.005448106250994254, 0.004203701529285039, 0.0034570586496933853, 0.003189426364965936, -0.01538288845723613, 0.032102232155286015, -0.014551012572084503, 0.00876879157956431, 0.023169793242299463, -0.02051051732209984, 0.007868728917435706, 0.016937541384376215, 0.01665115785239541, 0.011475798709309069, 0.022610664465055746, -0.005471971390105571, 0.019965024968929018, -0.002357550246983685, 0.031802212199232315, -0.010909849857383904, -0.011489435133381961, 0.005560614200175642, -0.0041014218294808235, -0.03752988283884843, 0.022856134999527863, -0.011905374007280277, 0.02363346146309791, -0.007343692597357979, -0.029920262742602736, -0.010418906925794667, -0.0057447175666909805, 0.013391841088765887, -0.025338126230909846, 0.00027168083095471514, 0.016378412607132502, -0.006515225818224577, -0.006395899657006742, -0.0019245657265789407, -0.009634761318865674, 0.026006353851316725, 0.010937124568174694, 0.016269313763969336, 0.01617385320752407, -0.005475380961785045, 0.013930514298577758, -0.021246932650806197, 0.001663752119549189, -0.02326525379874473, -0.010050699261441486, -0.004108240507178521, -0.013289560457639168, -0.005737899354654534, 0.011803093376153559, -0.04222111819370448, 0.020796900854080643, 0.02220154380319388, 0.00928019100990789, 0.020333232633282197, 0.015055592859069137, 0.01580564554317089, -0.01571018312408062, -0.04775786772043006, 0.02021049736604614, 0.017755780845454948, -0.0026149545151645877, -0.002287659149828221, -0.0415392522865797, 0.00858468774738772, -0.005901546688076779, -0.0104666372040173, -0.013050908135203498, 0.009962056917032667, -0.004316209478466428, 0.008557413036596928, -0.0009145522358208985, 0.016051116077643007, 0.02978388918864878, -0.016664796139113307, -0.006900479943991375, -0.00012699746466602688, 0.019037688527332126, 0.008571050391992324, -0.011482616921345515, -0.012423591649660304, 0.019978663255646914, -0.022515202045965476, -0.029292946257059544, -0.0030939651100287922, -0.007861910705399258, -0.03084760104684463, 0.025474499784863804, -0.0032661363726497234, 0.004411670500572945, -0.010043880118082538, -0.002009798848554225, 0.020278683211700614, -0.01853310730902499, -0.0020080942955451133, 0.022897047997036554, 0.01474193461629754, 0.012744068104468348, -0.005751536244388678, -0.014046431353777368, 0.026170002116061474, 0.05722216069986953, 0.04680325470539737, -0.015205602837095987, -0.004854883153939549, -0.019242247926940556, 0.010630284537439545, -0.022951597418618137, -0.026251826248433845, -0.004227566668396356, 0.02520175267695589, 0.009655216886297517, -0.023919844995078713, -0.009423382775898293, -0.0011037699016477923, 0.03174766277765073, 0.006314075524634369, -0.010091611327627674, -0.016296588474760128, -0.008407403058570078, -0.026674583334368607, -0.00842785862600192, -0.0166920708499041, 0.005529929917705376, 0.007425516729730353, 0.009552937186493301, -0.022801585577946284, 0.016937541384376215, 0.03199313331212285, -0.02724735039833022, -0.030111185718138273, -0.012737249892431902, -0.008325578926197705, -0.00351842651614204, 0.003975276524904039, 0.001439588730521594, -0.0047764685932466504, 0.008748336012132466, -0.017660318426364677, 0.012219031318729371, 0.01779669198031863, 0.006890252160275454, 0.008632418956932855, -0.0021632186310911743, -0.01738757318110177, 0.0018717210908370955, 0.001651819433578218, 0.0005373953189239447, 0.020524155608817735, -0.004469629028172752, -0.029538418654176665, -0.005427650217901161, 0.006014054637258168, -0.04028461931549332, -0.0007930949176804229, 0.019978663255646914, 0.021192383229224614, 0.007261868930646857, 0.00010547608594223867, -0.018355822620207348, -0.015437437878817711, -0.02222881851398467, -0.03695711577488681, -0.0012222441356070097, -0.013794141675946303, -0.007330055241962583, 0.003208177612219292, -0.009587030109320539, -0.01798761495585417, -0.02541995036328222, 0.008877890422727474, -0.009239278943721703, -0.01774214255873705, -0.03952093113864117, -0.0148646698835336, 0.00764371348473418, 0.006624324195726491, 0.010773476303429948, -0.007684625550920367, 0.015601085212239957, -0.007091402453865662, -0.005536748595403074, -0.022733399732291805, 0.0024291461299788867, -0.03000208687497511, 0.01969227972366611, -0.005318551840399247, -0.01285998515966796, -0.022937959131900238, -0.007043671709981778, 0.009627942175506726, -0.007111858486958756, -0.0061026974473282385, -0.00527763977421306, -0.009089268965694855, 0.013235011967380087, 0.008502864546337847, 0.01815126322059892, -0.00642999257983398, 0.012246306029520163, -0.026988241577140203, -0.025010831564065358, -0.029020201011796636, -0.0032831828340633426, -0.044921305248767786, 0.008209661870998093, 0.01049391191480809, 0.02284249857545497, 0.016419323741996186, 0.008598325102783116, -0.03510244289169303, 0.01816489964467181, -0.011612171331940521, -0.007780086573026885, -0.015123779636046115, -0.0009247802523674451, -0.004012779019410752, 0.0033172762225518316, 0.018464921463370514, -0.01738757318110177, -0.02921112212468717, -0.008148294237380064, -0.022365192067938626, 0.03878451580993481, 0.018192174355462602, -0.0006064342559901658, 0.02983843861023036, -0.0005535896202483205, -0.000617088352581334, -0.04863065474044537, 0.009116543676485645, -0.0007696558004060712, 0.014250991684708303, -0.007800542606119979, -0.008489227190942451, -0.015341976391049943, -0.017060278514257277, -0.001458340094190263, -0.009048356899508668, -0.018369459044280244, -0.010050699261441486, -0.026947328579631516, -0.005546976844780247, -0.012232668674124767, 0.0084960454029789, 0.011400792788973142, -0.025583598628026966, -0.011966741640898306, 0.014210079618522116, 0.012382679583474119, 0.016201126055669857, 0.010064336616836882, 0.012696337826245714, -0.009443838343330136, -0.018819490841005797, -0.014796484037879122, -0.03221133099844918, 0.009096087177731301, 0.00270871121709262, 0.02787466688387341, 0.019951388544856126, 0.027929216305454994, -0.006586821701219778, 0.0029388407744827313, 0.045739546572491525, -0.0009494978426062869, 0.00453781580514973, -0.004629867721238024, -0.01362367519916511, -0.03199313331212285, -0.0031843124731079754, -0.013460026934420362, 0.00035201313019817774, 0.021396942628833047, 0.0008033229342269695, -0.00910290632109025, 0.006665236261912678, -0.007984645972635316, 0.012048565773270679, -0.013125913124216923, -0.029374770389431915, -0.023919844995078713, 0.024560799767339804, -0.021192383229224614, -0.004967391103120938, -0.030874875757635423, -0.01774214255873705, 0.03158401451290599, 0.006481132429736088, 0.014237354329312907, 0.014932856660510576, 0.044075794802188276, -0.016378412607132502, -0.018724030284560527, 0.007030034354586382, 0.026906417444767832, -0.0016825033668025453, 0.00515490404131575, -0.05040350535391178, 0.0066993296504011665, 0.029892988031811944, 0.001129339943014159, 0.024451700924176638, -0.016010204942779323, -0.0004432126533666039, 0.0024410789323651705, 0.009143817455953935, 0.0073641486304510725, 0.008761973367527862, -0.0024990374599649762, -0.010575736047180465, -0.009034719544113272, -0.013466846077779312, -0.0034911520381818743, -0.015369251101840734, -0.016405687317923294, 0.02775192975399235, -0.008577869535351272, 0.03829357101570057, -0.02090599969724381, -0.03627524986776204, 0.014673747839320562, 0.022515202045965476, 0.03553883453905568, 0.015437437878817711, 0.00644703950690885, 0.03024755927209223, 0.028856552747051886, 0.0012640083618824394, 0.01684208082793095, 0.005264002418817665, -0.007180044798274483, 0.014373726951944362, 0.011782637808721717, 0.007193682153669879, -0.025869982160007774, 0.01737393489438387, 0.022160632668330196, -0.026388199802387803, -0.051030821839454975, 0.026170002116061474, 0.016596608430813828, 0.018123988509808127, -0.016555697295950144, -0.003515017177293191, -0.037475333417266846, 0.005826541699063354, 0.0026251825317111345, 0.009027900400754324, 0.002785420991945782, -0.008830159213182339, -0.007766449217631489, 0.015969291945270633, 0.007950553049808079, -0.004026416374806148, 0.0002904321364157277, -0.005195816107501938, 0.0236607361738887, 0.008993807477927085, -0.010514367482239934, 0.006406127440722663, -0.00785509156204031, 0.02779284275150104, -0.0036445715878881983, 0.011216688956796552, 0.009512025120307114, -0.015901106099616157, 0.013930514298577758, -0.002875768122194339, -0.002253565761339732, 0.03073850220368147, -0.03952093113864117, -0.0257881580276354, -0.0033462554863517343, -0.008080107460403087, 0.002920089527229375, -0.024969918566556667, -0.0009009149386331592, 0.005649256544584463, -0.012464503715846491, 0.019678641436948214, 0.018355822620207348, 0.011346243367391559, -0.016964816095167007, 0.013241830179416535, -0.009109724533126697, -0.025215390963673787, -0.01071210866981192, -0.0041798363901737225, 0.01407370606456816, -0.05135811836894447, -0.004380986683763931, 0.005533339489384851, -0.01538288845723613, 0.006873205233200584, -0.0005936492353069824, -0.017360298470310977, 0.00709822113156336, 0.04148470286499812, -0.016405687317923294, -0.014878307238928995, 0.014060068709172764, -0.004060509763294636, -0.02011503494695587, 0.017333023759520186, -0.007800542606119979, 0.004138924323987536, 0.03433875098955088, -0.022678850310710225, -0.016610246717531724, 7.132953203561358e-05, -0.020428693189727464, 0.006562956562108461, 0.005175360074408844, 0.0069311637608003894, -0.013978245508122893, -0.007057309065377174, 0.0038389034366113346, -0.006044738454067182, 0.00429234387369386, -0.014060068709172764, 0.007527796429534568, 0.005284458451910758, 0.024915369144975087, -0.017973976669136273, 0.004793515054660269, 0.00014095440919871681, 0.008291486003370468, -0.013091820201389685, -0.059895074906787055, -0.013125913124216923, -0.0066993296504011665, 0.008898345990159316, 0.002253565761339732, -0.004820789765451061, -0.01419644226312672, 0.0072141381867629716, -0.02090599969724381, 0.008291486003370468, 0.017155739070702544, 0.004462810350475054, -0.0020507109147404117, 0.0006119744025157171, -0.0021973120195796633, 0.004363939756689061, -0.0333841416998082, -0.028529258080207398, -0.015451075234213107, 0.00992114485084648, -0.00940292720846645, -0.028747453903888723, -0.01778305555624574, 0.050458054775493366, 0.007698262906315763, -0.019896839123274543, -0.019501356748130572, -0.021015098540406972, -0.04273933583608451, -0.014551012572084503, -0.030302108693673814, 0.02645638564804228, 0.004394624039159327, 0.03341141454795399, 0.0006290210385523051, 0.03311139459190029, 0.010241621305654524, 0.009805227795646868, -0.0069107081933685475, 0.006559546990428987, -0.006576593917503857, 0.0032507942314145906, -0.014905581949719785, -0.0052196812466132544, -0.018342184333489452, -0.007466428330255288, 0.004963981531441463, -0.018628567865470257, -0.0026320012094088324, 0.030274833982883022, 0.012021291062479888, -0.006426583473815756, 0.004616230365842629, 0.025065379123001934, 0.003097374448877641, 0.010132523393813861, 0.012703156038282162, -0.04481220640560463, -0.0022740217944328253, 0.009771133941497128, -0.018123988509808127, -0.012989539570262967, -0.014264629040103699, 0.006518635389904052, 0.015328339035654547, 0.004909432575521132, 0.0065527283127312895, -0.015001043437487555, -0.00032005066580170386, -0.003545701226932831, 0.034584225249313004, -0.006433402151513454, -0.0026200686398531736, 0.01232131194985609, -0.0052912771296084555, -0.01095076192357009, -0.014496463150502922, 0.021846974425558597, -0.01037117571624953, 0.00816193159277546, 0.0324568015329213, -0.04361212285609982, -0.004166199034778327, 0.0004649471012165311, 0.003917317997304234, -0.026920053868840724, -0.016664796139113307, 0.010500730126844538, -0.007848273350003864, -0.022706125021501013, 0.036384348710925205, 0.02948386923259508, -0.01045981806065835, 0.0032439755537168927, -0.00737096730814877, 0.01725119962714781, -0.00889152777812287, -0.013569125777583527, -0.0045480435888656506, -0.02105600967527066, -0.001594713182482968, 0.02284249857545497, -0.0112235081001555, -0.011959922497539357, 0.017346662046238085, 0.02866562977151635, -0.03212950872872181, 0.016923904960303323, 0.2056506711621042, -0.03234770641504814, 0.005390147723394448, 0.021028734964479868, 0.023087969109927088, -0.007861910705399258, 0.014237354329312907, 0.013194099901193902, -0.004299162551391558, -0.012109933406888708, 0.00313146783736613, -0.0026081360702975153, -0.004295753445373334, -0.00494352549834837, 0.014510100505898318, -0.021519677896069105, -0.030411205674191973, -0.037175313461213146, -0.00582313212738388, 0.0065527283127312895, 0.004902613897823434, -0.011823549874907902, -0.009539299831097905, -0.011898554863921327, 0.016637521428322515, 0.0014285084374704915, -0.01091666806942035, -0.007186863475972181, 0.020128673233673764, 0.017333023759520186, -0.024615349188921387, 0.0009392698842673967, 0.01350093900060655, 0.011571259265754336, -0.015423800523422316, -0.00638567140762957, 0.014482825795107526, 0.006873205233200584, 0.0058026765599520375, 0.012007653707084492, 0.03371143450400768, 0.008284666860011518, 0.01977410199339348, -0.03616615102459887, 0.005737899354654534, 0.022624300889128642, -0.010957580135606538, -0.0015708479269563385, -0.0031263539455081697, 0.0022944778275259185, -0.02074235143249906, 0.017210288492284127, 0.027656469197547082, 0.021206019653297506, -0.022433377913593105, 0.007248231575251461, -0.019133149083777393, 0.014428276373525943, 0.007323236564264886, 0.006228842286243772, -0.023306166796253417, -0.012362223084719773, -0.016923904960303323, 0.038702693540207445, -0.016160214920806173, 0.021233294364088297, -0.011366699866145902, -0.010630284537439545, 0.016023841366852216, -0.0055912980169846565, -0.01568290841328983, 0.0019041096934858473, -0.023169793242299463, -0.01007115482887333, -0.05324006782557405, -0.04363939942953562, 0.0342842015679693, 0.027533733930311024, 0.0239607579925874, 0.0409392123744723, 0.0018120578938128652, -0.026742769180023083, 0.014660110483925166, -0.024697173321293762, -0.024683535034575863, -0.041293781752107586, -0.0023115245217701632, -0.006835702738693872, -0.01113486575574668, -0.020060485525374285, -0.006978894504684275, -0.009034719544113272, -0.000506285189239683, 0.010725746025207315, 0.0003326225591510919, 0.008448315124756266, 0.006014054637258168, -0.0016390344711026909, 0.009389289853071055, -0.02978388918864878, -0.014905581949719785, 0.06884115396913651, 0.004230976240075831, 0.006215204930848376, 0.0024973329069558643, 0.009252916299117099, -0.010296170727236105, 0.01118941424600576, -0.0026217731928622856, -0.02037414376814588, 0.0012188347967581608, -0.021574227317650685, 0.012512233994069125, 0.01419644226312672, 0.0034536493108445364, 0.022869773286245763, -0.0026183638540134366, 0.005243546385724571, 0.020728715008426168, -0.013016814281053758, -0.00934155864352592, -0.0203195943465643, -0.008952895411740899, 0.009171092166744726, -0.015151054346836907, -0.046339586484598926, -0.012116751618925154, 0.009450657486689084, -0.030902150468426214, 0.02059234145447221, 0.015969291945270633, -0.04235749361030344, 0.0011446818514186663, 0.009109724533126697, -0.010473455416053747, -0.0031689703318728427, -0.013903239587786966, 0.009559756329852249, -0.01689663024951253, 0.017346662046238085, 0.009621123963470278, 0.024601710902203488, -0.013337291667184304, -0.010296170727236105, 0.011053041623374307, -0.02323798095059894, 0.017510308448337827, 0.006682283188987548, -0.004524178449754334, -0.017673956713082573, -0.02480627030181192, -0.009662036029656465, 0.0148646698835336, 0.02014230965774666, 0.029402045100222707, 0.005642437866886765, -0.01920133492943187, -0.012573601627687154, 0.03172038992950495, -0.008216481014357043, -0.04762149602912111, 0.009382470709712107, 0.018001251379927065, -0.007998283328030712, -0.017851241401900215, 0.0011668425539361842, -0.17303021391385814, -0.015751096121589307, 0.001676537082024716, -0.03529336400458356, 0.021219657940015405, -0.0019876382624520197, 0.023851659149424238, 0.004091193580103651, -0.010296170727236105, 0.009962056917032667, 0.013112275768821527, -0.008605144246142064, -0.0072209568644606694, 0.0022740217944328253, -0.005993598604165074, 0.006944801116195785, -0.009941600418278322, -0.009662036029656465, -0.004428716961986564, 0.0042514322731689235, 0.03572975937723622, -0.03212950872872181, 0.01722392491635702, -0.004667370215744737, -0.007745993184538396, -0.0038491314531578814, -0.013637312554560506, 0.04503040409193095, 0.007009578321493289, -0.037311685152522096, -0.007561889818023058, -0.007711900261711158, 0.03057485393893672, -0.008973351910495243, 0.0043332559398800465, -0.013487301645211154, 0.02812013741834553, -0.009539299831097905, -0.03444784983271404, 0.012055383985307125, 0.031529465091324406, 0.018792216130215006, 0.0188058544169329, -0.017973976669136273, -0.031802212199232315, -0.018246723777044185, 0.023838022725351342, -0.020183222655255347, 0.008816522789109445, -0.02130148207238778, 0.028256510972299486, -0.01052800483763533, 0.008564232179955876, -0.0013782208640729394, 0.009205186020894466, 0.003930955352699629, 0.010112066895059516, 0.0074323349417668, -0.0005024497121385562, -0.004060509763294636, -0.008182387160207302, -0.010991673989756277, 0.010364357504213084, 0.019174060218641077, -0.018123988509808127, -0.021533316182787, -0.022542476756756268, 0.013637312554560506, -0.023306166796253417, 0.014250991684708303, -0.01556017314605377, -0.001389301157124042, -0.014250991684708303, -0.010705290457775471, 0.025733608606053816, 0.02105600967527066, -0.024519886769831117, 0.022433377913593105, 0.020524155608817735, -0.00855059482456048, -0.014441913728921339, 0.02059234145447221, -0.007077765098470267, 0.0028382656276876267, -0.004599183438767759, -0.008421040413965474, 0.011584896621149731, 0.016814806117140157, -0.009927963062882926, -0.00629702906322075, 0.022801585577946284, -0.014782846682483728, -0.0103098080826315, 0.010316626294667949, 0.015341976391049943, 0.012553146060255312, 0.019555906169712155, -0.016201126055669857, 0.03202040988555865, -0.01909223794891371, 0.008775610722923257, 0.016678432563186203, -0.0023268664301746705, -0.01161899047529947, 0.04402124538060669, 0.026129090981197787, -0.018055800801508648, 0.013480483433174706, 0.04077556410972756, 0.006208386253150678, -0.027124613268449158, -0.02181969971476781, 0.0010475160434724109, 0.02299250855348182, 0.00030875727452063414, 0.02359255032823422, -0.02098782382961618, -0.029620242786549036, 0.022501565621892584, -0.0024888094434184294, 0.03845721928044532, -0.009321103076094078, -0.0324568015329213, 0.023838022725351342, -0.008468770692188108, -0.02869290448230714, -0.11122591670229573, -0.012089476908134365, 0.02348345148507106, 0.03313866744004607, -0.004394624039159327, 0.015369251101840734, -0.0029627061464246734, 0.038211748745973204, -0.021028734964479868, 0.036384348710925205, -0.00429234387369386, -0.020005937966437706, 0.030111185718138273, -0.009968875129069113, 0.01277134281525914, -0.009177911310103674, 0.017087551362403065, -0.023306166796253417, -0.005468562284087347, 0.03755715941228422, 0.004940116392330146, 0.0030496439378243823, 0.030056636296556694, -0.020428693189727464, -0.010296170727236105, -0.009941600418278322, -0.037638981682011595, 0.024274416235359, 0.003736623503976493, 0.010043880118082538, 0.017292112624656502, -0.03709348746619577, 0.015464711658286001, -0.02747918450872944, 0.026470023934760174, 0.02237882849201152, -0.011987197208330148, -0.0324568015329213, 0.01944680732654899, -0.014973768726696764, -0.010207528382827286, -0.011428067499763932, 0.019242247926940556, -0.017728506134664156, -0.005315142268719772, -0.017292112624656502, -0.010227983950259128, 0.02389257028428792, 0.0032985249752984754, -0.023851659149424238, -0.029320220967850332, 0.015341976391049943, -0.032293156993466555, 0.030656678071309094, 0.01920133492943187, 0.010112066895059516, 0.012280399883669902, 0.009641579530902122, 0.019910475547347435, 0.0004559976158421309, -0.02408349325982346, -0.010275715159804263, 0.0038286754200647878, 0.021560590893577793, 0.0030735093097663244, 0.014551012572084503, -0.005598116694682354, -0.0148646698835336, 0.015846556678034574, -0.014496463150502922, 0.007773267895329187, 0.015792007256452995, 0.00031472358840229156, 0.00737096730814877, -0.014182804907731324, -0.019555906169712155, -0.008557413036596928, -0.012287218095706348, 0.007486884363348382, 0.014019156642986579, -0.027901941594664203, -0.03207495930714023, 0.018192174355462602, -0.01247132192788294, -0.019978663255646914, -0.0009503501773184992, 0.009089268965694855, -0.012819073093481773, 0.016787531406349365, -0.03267499921924762, 0.0071323145200518495, 0.05220363254081399, -0.011687176320953946, -0.005199225213520161, -0.004370758434386759, -0.012164482828470291, -0.014537375216689108, 0.0021785605394956816, -0.014755571971692936, 0.010248440449013472, -0.014114618130754347, -0.0012512233994069126, -0.07385968585348206, 0.027697380332410766, 0.021083284386061447, -0.023551637330725534, 0.0104666372040173, 0.00906881246694051, 0.014782846682483728, 0.0004508836366726857, -0.01892858968416896, -0.005448106250994254, -0.024328965656940583, -0.0024973329069558643, -0.0203195943465643, -0.01434645224115357, -0.0014813530732123367, -0.01371231754357393, 0.005598116694682354, -0.007405060696637259, 0.006740241716587353, -0.0013688452404462613, -0.001430213106894916, 0.0002262941530485551, 0.025979079140525933, 0.011141683967783127, -0.011871280153130536, -0.004282116089977938, -0.04699417954357791, 0.02893837687942426, -0.009689310740447257, -0.01371231754357393, 0.005669712577677556, -0.013678224620746691, -0.021519677896069105, 0.014401001662735153, -0.005008302703645873, 0.010064336616836882, 0.004684416677158356, 0.02374256030626107, 0.003763898214767284, 0.03605705218143571, -0.01358958134501537, -0.03952093113864117, 0.005669712577677556, -0.022092444960030714, 0.014673747839320562, 0.003327504239098378, 0.008652874524364697, -0.02814741212913632, 0.02168332616081385, 0.020442331476445363, 0.025092653833792725, 0.013841871954168937, -0.017578496156637306, -0.028774728614679515, -0.007595982740850295, -0.017878516112691006, 0.02296523384269103, 0.010691653102380076, 0.006528863173619973, -0.002367778263530232, 0.04197564765923236, -0.00752097775183687, 0.006402717869043189, 0.0063993087630249655, -0.010957580135606538, 0.002255270547179469, -0.01776941726952784, -0.005908365365774477, 0.011530347199568148, -0.005236727708026873, 0.006651598906517282, -0.01556017314605377, 0.019542267882994256, 0.025583598628026966, 0.020974185542898285, -0.007595982740850295, -0.00512762933052496, 0.010323445438026897, -0.01763304557821889, 0.03742078399568526, 0.019965024968929018, -0.00332920879210749, -0.007268687608344554, -0.005410603756487542, 0.016732981984767782, -0.0031587425481569213, 0.0013714021863752417, 0.013937333441936706, -0.02831106039388107, -8.32621816267995e-05, 0.012553146060255312, 0.004230976240075831, -0.004657141966367564, 0.023674374460606596, 0.004807152410055665, -0.0009725108216283606, -0.0044764477058704495, 0.0019092237017591207, 0.03401145818535139, 0.013507758143965497, 0.008468770692188108, -0.01526015225867757, -0.006126562586439555, -0.0020848040703982753, -0.03308411801846449, 0.01331683516842996, -0.0454940723127294, -0.03736623457410368, 0.01088939428995206, 0.016787531406349365, 0.01749667202426493, 0.013194099901193902, 0.008386947491138236, 0.007050490387679476, -0.024138042681405042, 0.032756825214265005, -0.0012137207884848874, -0.02205153382516703, -0.041457430016852335, 0.02220154380319388, 0.02945659452180429, 0.012382679583474119, 0.056840318474088464, -0.012703156038282162, 0.026551846204487545, 0.007561889818023058, 0.03133854397843387, -0.016978454381884903, 0.030056636296556694, 0.02059234145447221, -0.014455551084316735, -0.008005102471389662, 0.00278371643893667, -0.018464921463370514, -0.008966532767136295, -0.011359880722786955, -0.02132875678317857, 0.015819281967243786, 0.023428902063489476, 0.08967896595808084, 0.01746939731347414, -0.00187342576026152, 0.008543775681201533, -0.003192835703814785, -0.01262133283723229, 0.004415079606591169, 0.02738372208963917, -0.00584017905445875, 0.006252707891016339, 0.009580211897284092, -0.0035832037214395436, -0.003552519904630529, -0.01710118964912096, 0.013937333441936706, -0.009034719544113272, 0.005396966401092146, -0.006665236261912678, -0.008721061301341676, -0.00473214742104224, 0.039630029981804336, -0.012934991080003887, 0.026974603290422308, 0.02021049736604614, -0.021465130337132526, -0.007902822771585445, 0.02997481216418432, 0.009573392753925143, -0.00690388951567085, -0.028556530928353183, 0.02948386923259508, 0.0019416123044078723, -0.04361212285609982, -0.02220154380319388, -0.0075073403964414755, -0.003148514531610375, -0.006089060091932843, -0.027370085665566278, 0.010043880118082538, -0.0016927313833490919, 0.014128255486149743, 0.0074391536194644975, -0.0038934526253622914, -0.0023030010582327284, -0.0019791150317452102, -0.006170883758643966, -0.008236936581788885, -0.018192174355462602, -0.022324280933074942], 'source': 'https://arxiv.org/pdf/2303.08774.pdf', 'page': 0}), 0.937362551689148)\n","(Document(page_content='these evaluations it performs quite well and often outscores the vast majority of human test takers.\\nFor example, on a simulated bar exam, GPT-4 achieves a score that falls in the top 10% of test takers.\\nThis contrasts with GPT-3.5, which scores in the bottom 10%.\\nOn a suite of traditional NLP benchmarks, GPT-4 outperforms both previous large language models\\nand most state-of-the-art systems (which often have benchmark-specific training or hand-engineering).\\nOn the MMLU benchmark [ 35,36], an English-language suite of multiple-choice questions covering\\n57 subjects, GPT-4 not only outperforms existing models by a considerable margin in English, but\\nalso demonstrates strong performance in other languages. On translated variants of MMLU, GPT-4\\nsurpasses the English-language state-of-the-art in 24 of 26 languages considered. We discuss these\\nmodel capability results, as well as model safety improvements and results, in more detail in later\\nsections.', metadata={'_id': ObjectId('6616def76182e3cd8e9a2211'), 'embedding': [-0.007733211456855665, -6.984724797683271e-05, -0.0003024558455137633, -0.023123752064577202, 0.0053463325813108325, 0.016984091385250547, -0.005736097625038841, 0.002279952326364106, -0.030905252600270966, -0.03371983627555475, 0.013058848652141408, 0.036837956625378816, -0.033830212909452075, -0.016680558436000813, 0.013038153731777639, 0.02603491576018246, 0.023468676717199752, -0.002971526208306185, 0.0036286078578166695, -0.004018372668714018, 0.012520766752843816, 0.002343763293967014, -0.022737435708581836, -0.030546531334072573, 0.002992221827161934, 0.008312685059526075, 0.024779390397165438, -0.022861608956055008, 0.0021575038884185734, 0.024006759547819983, 0.005546389066096439, -0.0053256371952857436, -0.03465803207574619, 0.0023765311825322878, -0.015356047211136642, -0.001604762155874004, 0.0014210897317863647, -0.011817119902700231, 0.008402365841736995, 0.001299503815019982, 0.018432774926265254, 0.03035337315607489, 0.028339013557288257, -0.005587779838146617, -0.0038666057284278323, 0.004911727612138685, 0.02052991793179752, -0.026159089007655633, -0.022433902759332103, 0.0036527526300663797, -0.0005846473327613529, 0.03286442388210894, -0.013300295908977192, 0.0045116151082287915, -0.004501267182385587, -0.01539743798318682, 0.008961143592720995, -0.003576869159923287, 0.011230747993241898, 0.007457271734757626, 0.0040563142873702345, -0.015300859825510618, -0.020171196665599123, 0.04773757937354193, 0.006594960103201253, -0.016266648852853755, -0.02545544308883469, 0.015093905033937089, 0.011741236665387799, -0.009726876135278527, -0.00328195839566648, 0.016252852239277908, -0.002274778363442504, 0.001262424368296926, 0.044481489162668836, -0.008085034416266138, -0.005680909773751497, -0.005932704956430486, -0.012920878791092388, 0.00668119126635689, 0.02173025497786588, -0.015894129110434235, 0.0038252149563776545, 0.003176756423182734, 0.0010615057115781592, -0.003980431050057801, 0.017011686475047517, -0.003269886125956954, -0.02196480485923638, -0.0010528825952625955, 0.013631423948023896, -0.0013150253545387988, 0.016652963346203843, 0.014238491709168638, -0.014500633886368191, 0.007650429912755309, -0.005915458723799358, 0.015797550952758033, -0.0037148390209723065, -0.049862313743580605, -0.00825059843578949, -0.007457271734757626, -0.02388258630034681, -0.007112347082135076, -0.018501759856789764, 0.013603829789549564, 0.005291144730023489, -0.007346896032182938, -0.0031008729530396413, 0.0197020987655034, -0.016542588574951795, 0.026572998590802692, 0.00346821780121492, -0.017811910551544664, -0.020571309635170336, -0.006384556158233762, 0.02908404931683674, -0.019191609162034863, 0.006719132885013106, 0.017011686475047517, 0.03879712790721678, -0.0038045193375219055, 0.0316778853122617, 0.00033716388286314085, 0.026021119146606615, 0.013776292115860839, 0.0023834297221508706, -0.026241870551755992, -0.01336238253271378, 0.005791285476326185, 0.01796367888881481, 0.011410108626341096, 0.032836830654957246, -0.0056188231500149104, -0.03567901128268327, 0.010872026727043502, -0.027814727340243872, -0.012824299702093547, -0.02753878761814583, -0.0024196467641101065, 0.011465296943289759, 0.01717724956324823, 0.0003576437841126047, -0.00723651986394693, -0.007995354565377859, 0.028394200011591642, 0.028339013557288257, 0.016846121524201528, 0.007271012329209185, 0.0013883218432210905, 0.008119526881528392, -0.021909616542287718, -0.024048149388547522, 0.03148472527161874, 0.0036217093181980862, -0.0183499933821649, 0.0038114178771404884, 0.019370970726456698, -0.027318036212996454, -0.027897508884344228, -0.024751795307368468, 0.014817965311839048, 0.0002800357372725311, 0.0014297128481019284, 0.006688090038806133, 0.01812924197701552, 0.015218077350087622, 0.015066310875462756, -0.0021143883068407547, -0.01574236263580937, -0.005763691783513173, 0.03694833325927614, -0.04177727839599183, 0.007919471328065425, 0.0039148952729272535, 0.012872589712254287, 0.0255244280193592, -0.013320991760663602, -0.01014768495653615, -0.009043925136821352, 0.015107701647512935, 0.016418415327478623, 0.022337322739010623, -0.011589470190762933, -0.0061741522132662706, -0.023786006280025328, 0.01844657340248638, 0.008381669990050584, -0.011065183973718546, -0.012769112316467523, 0.027635365775822034, 0.012493172594369483, 0.009540617195391406, -0.003102597529736622, -0.62472759043461, -0.017011686475047517, -0.008843869583358385, 0.007747008536092831, 0.02433788572422141, 0.00020846386894296892, 0.004549556726885008, 0.0015193932810668572, -0.016128678991804736, 0.037582996110217856, -0.010513304529522467, -0.01467999545079003, -0.016473603644427286, -0.023910179527498503, -0.008919752820670817, -0.03148472527161874, -0.013452063314924698, -0.029222021040531038, -7.954826110780003e-05, 0.009926932620064133, -0.02189581992871187, 0.020447136387697164, -0.01601830235790741, 0.0024558638060693423, -0.02760777254867034, 0.006522526019282781, 0.0011943017261209065, -0.016721948276728352, -0.0013822857083663279, 0.007353794338970861, -0.014224694164270153, 0.02563480279061125, -0.004546107573491046, 0.003828664109771616, 0.03863156481901607, -0.011893003140012664, -0.03421652926544744, 0.003325073977244298, 0.010416725440523626, 0.03606532763867864, -0.03523750847238452, -0.008381669990050584, -0.01209305962479827, 0.020378151457172654, -0.001979867832016356, 0.024613825446319446, 0.002040229646225302, 0.0017634275193634428, -0.011499789408552013, -0.015825144179909725, -0.015949317427382897, 0.007153738319846574, 0.020364354843596808, -0.001655638565418896, -0.012155146248534858, 0.006191398445897398, 0.034740815482491826, -0.018970857756885486, -0.004325355702680351, -0.022337322739010623, 0.002166127004734137, -0.000802381077937002, 0.0006786393238840635, -0.038576378364712685, -0.02582796096860893, 0.02531747322778567, 0.0015995883093677318, -0.0010744403860515049, 0.0028421794635727295, -0.03131916218341803, -0.007691820684805487, 0.017494580988719085, -0.002078171264881519, -0.006891595211324381, 0.0027783684959698214, 0.011389413705977326, 0.012086161318010347, -0.022861608956055008, -0.00032336689093747235, 0.02559341294988371, 0.019453752270557054, -0.023565254874875954, 0.019246797478983526, -0.002036780260000681, 0.040480363192247265, 0.007346896032182938, -0.01086512748893294, 0.0110996764389808, 0.007691820684805487, 0.014900746855939406, 0.02182683499818736, 0.03151231849877043, -0.025993524056809644, -0.03247810752611357, 0.01248627428758156, -0.009312966552131468, -0.025924539126285135, -0.009678587056440426, 0.008761087107935389, -0.0305189362442756, -0.012293116109583876, -0.02395157123087132, 0.046192313949560464, 0.007064057537635656, 0.019481347360354024, 0.023634239805400463, -0.007574546209781556, 0.015452626300135483, -0.0037458821000099403, -0.012913980484304465, -0.012244827030745776, -0.0033975082939934295, -0.010761650093146176, -0.010616781925309232, 0.023275518539202067, -0.026186682234807326, 0.005839575020825606, -0.012024074694273761, -0.019453752270557054, -0.028421795101388612, 0.014321273253268995, -0.01597691251717987, 0.021633676820189678, -0.0003733809830300415, 0.008871462810510078, 0.00846445246547358, 0.020557511158949212, -0.015314656439086465, 0.007912572089954863, 0.011568774339076524, -0.015176686578037444, -0.0035803183133172488, 0.040783896141497, -0.015507814617084148, 0.03763818256452124, 0.015093905033937089, 0.0399008905208995, 0.0008657609595733295, 0.012120653783272602, -0.01532845305266231, -0.012824299702093547, -0.0001892774293200731, -0.004342601935311478, 0.004252921618761879, -0.024862171941265793, -0.011382514467866764, -0.01601830235790741, -0.0007415881195538107, -0.0030042940968714594, -0.004697874513777232, -0.006463888548940156, -0.019039842687409995, 0.005384274665628369, 0.01581134756633388, -0.017701535780292616, -0.025924539126285135, -0.0036562017834603413, -0.03959735384635921, -0.017673940690495646, -0.02640743550260198, 0.019715895379079248, 0.008188511812052901, 0.001415053596931602, 0.010016613402275052, 0.006781219508749693, -0.006115514742923645, -0.01703927970219921, 0.04390201649132107, -0.03341630332630501, -0.030160214978077204, 0.0009632022205053316, -0.02836660678443995, 0.03785893583231589, 0.0073744901906572695, -0.026697171838275868, 0.00042813777745463765, -0.002700760449129748, 0.0014943862204686565, -0.0005738684373668982, -0.026241870551755992, -0.008360974138364176, 0.032284951210761165, 0.013865972898071757, -0.008816275424884053, 0.030877659373119273, 0.02980149371187881, 0.02251668430343246, 0.023247925312050374, -0.017563565919243598, 0.00848514738583735, 0.013348585919137934, 0.02284781234247916, -0.03551344819448256, 0.017563565919243598, -0.00497036461681999, 0.015507814617084148, -0.0005126443348094618, -0.0018643179336894065, 0.009354357324181646, 0.020088415121498768, 0.0033785374846653214, 0.015880332496858388, 0.0037251867139848514, -0.027649164252043158, 0.007326200646157848, -0.00927157578008129, 0.00781599393227866, -0.010740955172782406, -0.005015205007925449, 0.0044909197222037026, 0.0028094115750074552, 0.007215824477921841, -0.01974348860623094, -0.00874039218757162, 0.0038217655701530333, 0.00769871899159341, -0.0010640926930389602, 0.0005277347883616983, -0.00841616245531284, -0.016915106454726037, 0.01310713866230215, 0.017949880412593683, 0.005215261492711056, 0.01012698910484974, 0.005839575020825606, 0.01519048319161329, -0.01356933732428731, 0.009588906274229508, 0.02448965406149155, -0.023979164458023013, -0.01583894265613085, 0.011237646300029821, 0.03465803207574619, 0.01142390617123958, 0.02046093300127301, -0.012962269563142568, 0.027842320567395565, -0.0027680205701266173, 0.018294805065216233, 0.0013115762011448372, -0.002811136151704436, -0.021702661750714187, -0.011775729130650054, -0.028449388328540305, 0.027249051282471945, -0.005170421101605596, 0.016170068832532275, -0.01049950791594662, -0.02185442822533905, 0.013700408878548405, -0.007912572089954863, 0.013120935275877996, -0.0221027747202854, 0.015645784478133168, 0.02778713411309218, -0.008754188801147466, -0.00420808122765642, 0.005925806183981243, 0.0030991483763426605, 0.02395157123087132, 0.008609320633310524, 0.0065880617964133295, 0.02455863899201606, 0.004749613211670614, 0.035044348431741555, -0.03234013766506455, 0.0010615057115781592, -0.009354357324181646, 0.0015530234579806217, -0.0037803745652721953, -0.013527946552237132, 0.004735816132433448, 0.00695713098845493, -0.027938900587717044, 0.01662537011905215, 0.004666831201908938, -0.004432282251861077, 0.009816556917489446, 0.0168737166139985, 0.006129311822160811, -0.025731382810932728, -0.04575081113926171, 0.037141489574628546, 0.0036493032438417584, -0.01745318928534627, 0.004083908445844566, -0.04599915763420805, -0.016721948276728352, -0.005680909773751497, -0.005287695576629527, -0.01645980703085144, 0.01308644281061574, 0.005863720025905976, 0.01341067254287452, 0.00723651986394693, -0.0014555821970486196, 0.016004505744331564, -0.011327327082240738, -0.0012167218052583063, -0.02115078230651811, 0.008726594642673134, 0.00279733930529793, -0.02658679520437854, -0.015687174318860707, 0.016239053763056784, -0.026931719857001088, -0.03606532763867864, -0.02556581786008674, 0.007588343289018722, -0.00381486703053445, 0.014597212975367034, 0.008988737751195328, 0.005491201214809095, -0.015093905033937089, -0.006143108901397977, 0.02399296107159886, -0.004835844374826251, 0.005270449343998399, 0.0235790514884518, 0.007215824477921841, -0.010237364807424428, -0.011127270597455132, -0.00242826988042567, 0.021081797375993597, 0.048317052044889695, 0.028242433536966777, -0.0050255529337686535, -0.0022230396655491215, -0.012362101040108387, -0.002838730310178768, -0.010361538054897602, -0.025358863068513208, 0.013707307185336328, 0.01796367888881481, 0.018322400155013206, -0.02726284789604779, -0.003973532277608559, -0.012189638713797113, 0.03460284562144281, 0.01942615904340536, -0.000446246327538088, -0.029967056800079522, -0.0008041057128416477, -0.001860868780295445, 0.019605518745181922, -0.004611643350621595, 0.008954245285933072, 0.008036745337428037, 0.006663945033725763, -0.018060257046491012, 0.007609038675043811, 0.004839293528220213, -0.022806420639106345, -0.02333070685615073, -0.024848375327689947, -0.010140785718425586, 0.008609320633310524, 0.011092778132192877, -0.009561312115755175, 0.02374461643929779, 0.010651274390571487, -0.015342250597560796, 0.020778264426743867, 0.005960298649243498, 0.013024356186879154, 0.0012244826332253797, 0.011955089763749252, 0.0036665494764728857, 0.008899056968984409, -0.006567366410388241, 0.01296916880125313, 0.028311418467491287, 0.0027524991470231305, -0.03523750847238452, -0.00335266813571863, 0.0020660989951719936, -0.03948697721246188, 0.009099113453770015, 0.007719414843279818, 0.019191609162034863, -0.0009045649829933662, 0.02276502893573353, -0.020557511158949212, -0.022861608956055008, -0.02196480485923638, -0.03620329749972766, 0.0021644024280371562, 0.007208926171133918, -0.01621146053590509, 0.0016340807746299867, -0.020640294565694845, -0.014155709233745643, -0.028421795101388612, 0.003949387738189508, -0.007202027398684675, -0.00631212207431529, -0.04230156461303622, -0.0009769992415348326, 0.016597776891900458, 0.003987329356845724, 0.01176883082386213, 0.0022402858981802488, -0.0009114635226119492, 0.004383992707361656, -0.004321906549286389, -0.038024498920516604, 0.008712798029097288, -0.019729691992655094, 0.026159089007655633, -0.004063213059819477, -0.01467999545079003, -0.00855413231636186, -0.0031353654183018964, -0.008029847030640113, -0.001819477775414607, -0.007919471328065425, -0.03289201710926063, 0.0013055399498747447, 0.012748416464781115, 0.0183499933821649, 0.02767675747919485, 0.01676333998010117, 0.020474729614848856, -0.02251668430343246, -0.003563072080686121, -0.028863297911687364, -0.022875405569630854, -0.040314800104046554, 0.011244544606817744, 0.022971983727307056, 0.018943264529733793, 0.01826721183806454, 0.007533155437731378, -0.024820780237892977, 0.030905252600270966, -0.008002252872165782, 0.0162804454664296, 0.00048375688933155676, 0.0068295090532491136, 0.01467999545079003, -0.005356680507154037, 0.02149570695914066, 0.007477967586444034, -0.030850064283322303, -0.03396818277050109, -0.03308517714990359, 0.025703787721135758, 0.009395749027554464, 0.0006230202120071445, 0.014845558538990741, 0.005974095728480664, 0.002654195481327308, -0.04001126342950627, 0.01576995772560634, -0.018791498055108928, 0.014817965311839048, -0.0013710756105899632, -0.002486907117937636, -0.017163452949672385, -0.0255244280193592, -0.028145855379290572, -0.0017056525701999677, -0.023468676717199752, 0.017232437880196894, -0.011713642506913466, 0.00261280447644647, 0.002538645815831018, -0.0003800638923538369, 0.00855413231636186, -0.016915106454726037, -0.009947628471750543, 0.008526538157887528, -7.99255195362227e-05, 0.018598339877111243, -0.006094819356898556, 0.021164578920093956, -0.03239532784465849, -0.012941574642778798, -0.002504153350568763, -0.014514431431266678, 0.01645980703085144, 0.007119245854584319, 0.018418978312689408, 0.01750837760229493, 0.02836660678443995, -0.018060257046491012, -0.0036217093181980862, 0.037582996110217856, -0.005053146626581666, -0.0075538508237564675, -0.008057441189114445, -0.0074641705072068685, -0.007146839547397331, 0.0031405393812234984, -0.003192278079116881, -0.007650429912755309, 0.01699788986147167, 0.013728003037022738, -0.005853372100062771, 0.019715895379079248, -0.004897930532901519, 0.015797550952758033, -0.03623089072687935, -0.029856682028827474, -0.00880937711809613, 0.017798113937968818, -0.01130663123055433, -0.00201091091105399, -0.02785611904361669, -0.027524991004569985, 0.01570097279508183, 0.009278474086869213, 0.011389413705977326, 0.017329016037873096, 0.023096156974780232, -0.017218641266621048, -0.024434465744542888, -0.005970646575086703, 0.015645784478133168, -0.007705617764042653, -0.00723651986394693, -0.04713050974975191, -0.003659650936854303, 0.018832887895836467, -0.017439392671770422, 0.04519893169506563, -0.013355484225925857, -4.776398658314058e-06, -0.014169506778644128, 0.008678305563835033, -0.02149570695914066, -0.004325355702680351, 0.0020350556833037, -0.011603266804338779, 0.02087484258442007, -0.01639082210032693, 0.01052020283631039, -0.0008748152782708034, -0.023344503469726576, 0.005242855185524068, -0.0005536041373083895, 0.01810164874986383, -0.03200901148866313, -0.009782064452227191, 0.03435449912649646, 0.014707588677941722, 0.03843840850366367, 0.006070674817479506, 0.0176601440769198, 0.03222976475645778, 0.014804167766940563, 0.004718569899802321, 0.0045426584200970845, 0.018239618610912847, -0.007039912998216605, 0.010078700026011638, 0.009554413808967252, 0.00023239301089789176, -0.025579614473662585, 0.003371638945046738, 0.020861045970844223, -0.028614953279386297, -0.04183246857558577, 0.024061946002123368, 0.012941574642778798, -0.002348937256888616, -0.030325779928923196, -0.008823173731671977, -0.03708630312032516, 0.022116571333861246, -0.006839856513430998, 0.002905990664006297, 0.0029094398174002584, -0.0031405393812234984, -0.004859988914245302, 0.03639645381508006, 0.010754751786358253, -0.005556736991939643, -0.015383641369610974, 0.004725468206590244, 0.013286499295401346, 0.0043288048560743125, -0.009188794235980935, 0.009071519295295684, -0.005939603263218409, 0.01655638518852764, 0.0038459103424027434, -0.015218077350087622, -0.004908278458744723, -0.0012477651171265997, 0.012500070901157406, 0.005984443654323868, -0.012989863721616899, 0.03711389634747685, -0.03291961406170288, -0.010678868549045819, 0.007802196387380175, -0.001976418445791735, -0.0007752182382599103, -0.019936646784228625, -0.005394622125810254, 0.014790371153364717, 0.0058706183326938995, 0.02713867464857462, 0.02106800076241775, 0.010761650093146176, -0.02351006655792729, 0.022088978106709553, -0.009678587056440426, -0.032284951210761165, -0.01454202465841837, -0.010292553124373091, 0.014735182836416054, -0.04908968103158987, 0.004001126436082891, -0.004966915463426028, -0.004766859444301741, 0.01236899934689631, -0.0057395467784328025, -0.036424047042231754, 0.0032198720047605526, 0.018212023521115877, -0.014376461570217658, 0.014169506778644128, -0.01389356705654609, 0.009913136006488287, -0.0152594681221378, 0.0016038998675255137, -0.014928341014413737, 0.0009468182762226946, 0.02665578013490305, -0.03689314680497276, 0.01761875423619226, -0.0022816769030610867, -0.020447136387697164, 0.00014001786863626555, 0.018432774926265254, -0.012396593505370642, 0.0031940026558138616, -0.007505561279257046, 0.0017125511098185506, -0.005342883427916871, 0.006388005311627723, 0.005097987017687125, 0.0041770379157881265, 0.017260031107348587, 0.0273318328265723, -0.014141912620169797, -0.0022109673958395962, -0.014128115075271312, 0.010789244251620507, -0.018625933104262936, -0.05549148481943872, -0.011830917447598717, -0.002790440765679347, 0.006974377221086057, 0.0018781150129265722, -0.004204632074262459, -0.028394200011591642, 0.01655638518852764, -0.01861213649068709, 0.02080585765389556, -0.0023610095265981412, 0.015880332496858388, 0.01447303972789386, -0.0018418978545520067, 0.007540053744519301, 0.017301422810721404, -0.029967056800079522, -0.019757287082452064, -0.010258060659110836, -0.003400957447387391, -0.014390258183793504, -0.010809940103306916, 0.016680558436000813, 0.047268479610800926, -0.01142390617123958, -0.027276644509623638, -0.019964241874025596, -0.028421795101388612, -0.03170547853941339, -0.0070088696863483116, -0.019550332290878537, 0.03948697721246188, 0.0077263131500677415, 0.029387584128731752, 0.0054774041355719295, 0.027773335636871052, 0.005615373996620949, 0.009444038106392565, -0.025772772651660267, -0.01204477054596017, -0.007615937447493054, -0.0110513873601427, -0.008002252872165782, 0.010851330875357093, -0.020571309635170336, -0.004263269078943764, 0.0007463308218858378, -0.019315782409508035, 0.008105730267952546, 0.02480698362431713, 0.008471350772261504, -0.006791567434592897, -0.0016073490209194752, 0.00014562288842061549, 0.01748078437514324, -0.0058326767140376825, 0.008126426119638954, -0.032643674339604835, 0.015204280736511776, 0.015728566022233523, -0.028752923140435316, -0.020585106248746182, -0.015866535883282542, 0.0197020987655034, 0.0028283823843355633, -0.01341067254287452, 0.004677178662090823, -0.01902604607383415, 0.007402083883470282, -0.0007674574685005019, 0.027662960865619004, 0.0029767001712277872, -0.011465296943289759, 0.026973111560373905, 0.0014719661413312567, 0.002116113116368395, -0.013624525641235972, 0.010347740509999115, -0.02046093300127301, 0.009650992897966094, 0.03620329749972766, -0.039045474402163126, -0.01149289110176409, -0.007760805615329997, -0.005042799166399781, -0.012789807236831293, -0.02168886513713834, -0.007622835754280977, -0.02168886513713834, -0.024268900793696896, 0.026669576748478897, 0.04230156461303622, -0.01236899934689631, 0.019564128904454383, 0.0007195991263829912, 0.016142475605380582, -0.01875010635173611, -0.002838730310178768, 0.004104603831869656, -0.01447303972789386, -0.0025369212391340374, 0.0086024223265226, -0.008360974138364176, -0.03424412249259913, 0.011623962656025187, 0.017053076315775056, -0.024199915863172387, 0.021081797375993597, 0.19569647177355665, -0.012610446603732095, 0.008788681266409722, 0.0394041975310068, 0.052400961422056906, 0.008360974138364176, 0.014611010520265519, 0.008305786752738152, -0.008595523088412039, -0.01810164874986383, 0.011989582229011506, -0.005580881531358694, -0.020405744684324347, -1.8243282637901952e-05, 0.01375559719549707, -0.03636886058792837, -0.02242010428311098, -0.04045276996509557, -0.0024955302342531994, 0.0016125229838410773, -0.00769871899159341, -0.01197578561543566, 0.002540370392527999, -0.015369844756035127, 0.01519048319161329, 0.010671970242257895, -0.0020885189578940633, -0.0186811214212116, 0.01650119687157898, 0.01583894265613085, -0.0026731662906554162, 0.008029847030640113, 0.007947064555217117, -0.00372173756059089, -0.008340279218000407, -0.004521962568410676, -0.004987611315112437, 0.004225327460287548, 0.004922075537981889, -0.004052865133976273, 0.010761650093146176, -0.0007174433473041003, 0.014390258183793504, -0.04230156461303622, -0.006015486500530842, 0.018143038590591368, -0.013320991760663602, 0.016308038693581293, 0.006912291063010789, 0.004949669230794901, -0.015949317427382897, 0.005487752061415134, 0.035982544231933006, 0.02173025497786588, -0.025331269841361515, 0.006867450671905331, -0.0263108554822805, -0.019205407638255987, 0.011906800684911149, 0.012982965414828976, -0.015425032141661152, -0.007312403566920683, -0.015300859825510618, 0.040673519507599676, -0.027124878034998773, 0.02640743550260198, -0.007346896032182938, 0.008995636057983251, 0.0305189362442756, -0.008747290494359542, -0.01731521942429725, -0.0037251867139848514, -0.028670139733689683, -0.006581163023964087, -0.04690976020724781, -0.03592735777762962, 0.045309308328962954, 0.03429931267219308, 0.017604955759971137, 0.03910066458175707, 0.007188230785108829, -0.01981247353675545, 0.003556173541067538, -0.008712798029097288, -0.016363227010529956, -0.041722091941688445, 0.005463607056334763, -0.009775166145439268, -0.0075538508237564675, -0.02379980475624645, -0.01539743798318682, -0.0093405607106058, 0.00473926528582741, 0.009057722681719838, -0.0007045086728307547, 0.009485428878442743, 0.014997325944938246, 0.0007135629333205637, -0.0005186805278718893, -0.04332254009468274, -0.017439392671770422, 0.061920881834439265, 0.0025213995831998904, 0.019660707062130585, 7.157187560552034e-05, -0.002326517061335886, 0.004818598142195124, 0.00022032065387686905, -0.008105730267952546, -0.005035900393950538, 0.011568774339076524, -0.036286077181182735, 0.016942701544523008, 0.0008394604664523931, 0.01519048319161329, 0.010934112419457449, 0.004566802959516135, -0.0035596229272921594, 0.021578488503241015, -0.003271610702653935, 0.01345896162171262, -0.0005436875302624252, -0.0001795764234650639, 0.009671688749652503, -0.0040977055250817325, -0.04050795641939896, -0.013610728096337487, 0.002702485025826729, -0.040480363192247265, 0.007898775476379016, -0.009926932620064133, -0.04942081093328186, 0.011016894894880445, -0.0026369492486961804, 0.00038049506563191457, -0.004822047295589086, -0.001969519906173152, 0.0035354781550424492, -0.032754047248211605, 0.004039068054739107, 0.03520991151994227, 0.022433902759332103, -0.009975222630224875, 0.015756159249385216, 0.01601830235790741, -0.00020889501311721411, 0.014024637679484546, 0.026366043799229164, -0.0008890433852668845, -0.018143038590591368, -0.024103337705496185, -0.014900746855939406, -0.009602703819127993, 0.006970928067692095, 0.03620329749972766, -0.0009951077625144503, -0.028256230150542624, -0.010299451431161014, 0.024227510952969357, -0.003645854090447797, -0.06043080659005174, 0.007112347082135076, 0.024889765168417486, -0.025841757582184776, -0.008561030623149782, -0.01204477054596017, -0.17604955759971136, -0.005191116487630686, 0.012741518157993192, -0.044950585200119283, 0.010575391153259055, 0.004211530381050382, 0.03907306762931482, -0.008947346979145149, -0.018943264529733793, 0.002004012371435407, 0.011506687715339936, 0.0034233776429401204, -0.013210616058088914, -0.0033388710564814642, -0.016321837169802417, -0.0034182036800185184, -0.006508728940045615, -0.0037562300258531444, 0.008519639851099605, 0.0022506335911927937, 0.032450514298961876, -0.02284781234247916, -0.0022782275168364654, 0.0002218296875905597, 0.00992003431327621, -0.011472195250077682, -0.0033509433261909895, 0.03225735798360947, 0.003918344426321215, -0.04014923329055529, 0.0019453752503387715, 0.0026697171372614546, 0.019964241874025596, -0.016418415327478623, 0.005404970051653458, 8.235077099997499e-05, 0.023868789686770964, -0.0183499933821649, -0.01662537011905215, 0.005674011001302254, 0.030905252600270966, 0.016859918137777374, 0.012224131179059367, -0.019205407638255987, -0.010478812064260212, -0.00853343739599809, 0.018487963243213917, -0.018653528194059906, 0.016818528297049835, -0.003061206757686444, 0.014928341014413737, -0.0032905815119820435, 0.016308038693581293, -0.005577432377964733, 0.0028473534264943316, 0.018791498055108928, 0.0017953330031648966, -0.0020540264926318086, -0.004080459292450604, -0.0014840385274561118, -0.011196255527979643, -0.0140108410659087, 0.012175842100221266, 0.021523300186292352, 0.0030508588318432397, -0.03446487576039379, -0.009050824374931914, 0.013438265770026213, -0.01615627221895643, 0.0033854357914532445, -0.02115078230651811, -0.019867661853704113, -0.0033854357914532445, -0.01888807621278513, 0.030574124561224266, 0.03035337315607489, -0.0013900465363334011, 0.009568211353865739, 0.02767675747919485, 0.0021281853860779204, -0.0052980435024727315, 0.01426608493632033, 0.0032009011954324445, -0.00559467861059586, 0.0005988754979650991, -0.003240567390785642, 0.024889765168417486, 0.029856682028827474, -0.011679150041651211, -0.013741799650598584, 0.02908404931683674, -0.019605518745181922, -0.006060326891636301, 0.01782570902776579, 0.026048712373758307, 0.008471350772261504, 0.004928973844769812, -0.009885541848013955, 0.00982345522427737, -0.016846121524201528, 0.007408982655919524, 0.00971307952170268, -0.013452063314924698, -0.016887513227574345, 0.047792765827845314, 0.011203153834767566, -0.0065880617964133295, 0.007802196387380175, 0.03322314701095261, -0.006450091935364309, -0.012789807236831293, -0.018625933104262936, 0.004194284148419254, 0.013921160283697782, -0.010458117143896443, 0.023109955451001356, -0.01505251333056427, -0.028614953279386297, 0.022461495986483795, -0.017949880412593683, 0.0353202881538396, -0.03134675541056972, -0.008995636057983251, 0.015356047211136642, 0.013231310978452683, -0.025552021246510893, -0.11109334254748421, -0.020474729614848856, 0.02062649608947372, 0.04199803166378648, -0.022061383016912583, 0.004283964930630173, -0.006401802390864889, 0.04428832912202588, -0.01992285017065278, 0.047102916522600215, -0.008050541951003883, -0.016666761822424967, 0.02121976723704262, -0.011637759269601033, 0.020102211735074614, -0.014611010520265519, 0.04588878100031073, -0.020709279496219358, -0.019784880309603757, 0.040866679548242635, 0.011844714061174563, -0.0029904970176342937, 0.0194675488841329, -0.019798676923179603, -0.00571195308561979, -0.013286499295401346, -0.03832803186976634, 0.02658679520437854, 0.0070709563100848985, 0.004280515311574891, 0.017494580988719085, -0.022406307669535133, 0.015204280736511776, -0.018087850273642705, 0.00404596682718835, 0.001620283695392821, -0.009230185008031112, -0.02261326246110866, 0.019660707062130585, 0.004352949861154682, -0.015149092419563113, -0.015356047211136642, 0.0080919336543767, -0.01153428187381427, -0.02980149371187881, -0.014252288322744484, -0.007043362151610566, 0.01107208228050647, 0.0027335281048643622, -0.01930198579593219, -0.03498916197743817, 0.001812579235796024, -0.0443987057559232, 0.017770520710817125, 0.020916234287792886, 0.010106293253163331, 0.018212023521115877, 0.015880332496858388, 0.009002534364771175, -0.003019815752805606, -0.026738561679003407, -0.01724623449377274, 0.0002489925127157352, -0.00695713098845493, 0.002783542226060764, 0.02090243581157176, 0.0037320852536034343, -0.022999578817104026, 0.0050841899384499595, -0.01341067254287452, 0.005215261492711056, 0.020847249357268376, -0.017811910551544664, 0.018156835204167214, -0.014969731786463915, 0.017867098868493327, -0.006053428584848378, -0.022806420639106345, -0.004939321770613016, 0.009423342254706157, -0.02613149578050394, -0.01098240242961819, 0.0025438195459219605, -0.012458680129107228, -0.013707307185336328, 0.004394340633204861, -0.01218274040700919, -0.013838378739597425, 0.025276081524412852, -0.03625848395403104, -0.000776942873164556, 0.030574124561224266, -0.006836407360037037, -0.018239618610912847, 0.010996199043194037, 0.011734338358599876, -0.019453752270557054, -0.01741179944461873, 0.015149092419563113, 0.0013952203828396736, -0.01738420435482176, -0.016846121524201528, -0.06363171034662143, 0.026435028729753673, 0.006039631505611212, -0.01488695024236356, 0.015245671508561954, 0.0018091299659867325, 0.012024074694273761, 0.005705054313170548, -0.015963114040958744, -0.003659650936854303, -0.01949514397392987, -0.006270730836603793, -0.020102211735074614, -0.025607209563459556, -0.024931156871790303, -0.014845558538990741, 0.012927777097880311, 8.72012739274796e-05, 0.006536323098519947, 0.017370407741245913, 0.0031474379208420813, -0.010458117143896443, 0.01785330225491748, 0.01567337770528486, -0.0043598481679426055, -0.007381388497445193, -0.045861187773159034, 0.036120514092982024, -0.020005631714753135, -0.02864254650653799, -0.004615092504015556, -0.005049697473187704, -0.005222159799498979, -0.0018815641663205338, -0.014804167766940563, -0.0018160285056053156, -0.0021816486606682835, 0.009802759372590961, 0.012203435327372959, 0.026641983521327205, -0.030049840206825155, -0.047958328916046025, 0.017908490571866144, -0.03038096638322658, -0.002585210550802798, -0.004252921618761879, -0.002736977491088984, -0.0161838673087534, 0.014307476639693149, 0.01248627428758156, 0.0203919480707485, 0.006839856513430998, -0.002626601555683636, -0.02924961426768273, -0.025013938415890658, -0.003495811726858592, 0.0010649549813874506, 0.009995917550588644, 0.014307476639693149, -0.007395185576682358, 0.0360101374590847, 0.010292553124373091, 0.017439392671770422, 0.006450091935364309, 0.01243798427742082, 0.010885823340619348, -0.03082247105617061, 0.008636914791784855, -0.004653034122671772, 0.014431648955843682, 0.0028490780031913123, -0.000853257487481894, 0.020861045970844223, 0.045502464644315364, 0.021164578920093956, -0.02531747322778567, -0.0010925490234464524, 0.012920878791092388, -0.01679093320725286, 0.014528228044842524, 0.017329016037873096, 0.007933267941641271, -0.009195692542768858, 0.0012779460242310727, 0.01091341749909368, -0.0032336690839977187, -0.00769871899159341, 0.008816275424884053, -0.014183303392219975, -0.005722300545801675, 0.025124315049787987, 0.009933830926852056, -0.011920597298486995, 0.014003942759120777, 0.0006557881005724186, 0.014817965311839048, -0.015894129110434235, 0.003049134255146259, 0.02767675747919485, 0.02497254857516312, -0.006632901721857469, -0.023661834895197433, -0.010175278183687842, -0.013852175353173271, -0.016680558436000813, -0.007284809408446351, -0.05662283693498257, -0.03123638063931767, 0.022778827411954652, 0.012727721544417345, 0.0380520921476683, 0.005677460620357536, -0.0029266860500313857, 0.004311558623443185, 0.003959735664032712, 0.002726629798076439, -0.001852245663979881, -0.013217514364876837, -0.03678277017107543, 0.02399296107159886, 0.031429538817315354, 0.01803266381933932, 0.048289458817738, -0.0001798997815957478, 0.02433788572422141, -0.015204280736511776, 0.04724088638364923, -0.0035354781550424492, 0.01331409345387568, 0.02668337522470002, -0.019439955656981207, 0.012982965414828976, -0.01611488237822889, -0.0028318317705601846, 8.714738090569895e-05, -0.013624525641235972, -0.03253329770570751, 0.01669435504957666, 0.029497960762629078, 0.08272673762568954, 0.024034352774971676, -0.007036463844822643, 0.005263551037210476, -0.00443918102431032, 0.005267000190604438, 0.009237083314819036, 0.009126707612244347, -0.0055084474474402225, -0.01810164874986383, 0.004342601935311478, -0.01761875423619226, 0.004587498345541225, -0.019964241874025596, 0.0051669719482116344, -0.012776010623255446, -0.0011425630282275241, 0.01343136746323829, -0.00843685830699925, 0.01005800417432523, 0.04936562075368792, -0.007402083883470282, 0.01981247353675545, -0.019191609162034863, -0.013114036969090073, 0.005573983224570771, 0.0350995386113355, 0.0013771118618600556, 0.0038631565750338707, -0.02924961426768273, 0.028145855379290572, 0.013500352393762799, -0.050855695998075434, -0.005149725715580507, -0.023096156974780232, -0.0073744901906572695, -0.005222159799498979, -0.030022245117028185, 0.013521048245449209, 0.007312403566920683, 0.014183303392219975, 0.002260981284205338, -0.004173588762394165, -0.011506687715339936, 0.02374461643929779, -0.023206533608677558, -0.022116571333861246, -0.003442348452268229, -0.021771646681238696], 'source': 'https://arxiv.org/pdf/2303.08774.pdf', 'page': 0}), 0.9312931299209595)\n","(Document(page_content='2 GPT-4 Observed Safety Challenges\\nGPT-4 demonstrates increased performance in areas such as reasoning, knowledge retention, and\\ncoding, compared to earlier models such as GPT-2[ 22] and GPT-3.[ 10] Many of these improvements\\nalso present new safety challenges, which we highlight in this section.\\nWe conducted a range of qualitative and quantitative evaluations of GPT-4. These evaluations\\nhelped us gain an understanding of GPT-4’s capabilities, limitations, and risks; prioritize our\\nmitigation eﬀorts; and iteratively test and build safer versions of the model. Some of the speciﬁc\\nrisks we explored are:6\\n•Hallucinations\\n•Harmful content\\n•Harms of representation, allocation, and quality of service\\n•Disinformation and inﬂuence operations\\n•Proliferation of conventional and unconventional weapons\\n•Privacy\\n•Cybersecurity\\n•Potential for risky emergent behaviors\\n•Interactions with other systems\\n•Economic impacts\\n•Acceleration\\n•Overreliance', metadata={'_id': ObjectId('6616defa6182e3cd8e9a22af'), 'embedding': [0.00014502545819620116, -0.011435680564455451, -0.0064793044898813, -0.03413554144209096, -0.004651104844993017, 0.02041546918599732, -0.01886510028345129, 0.004236072597778897, -0.023214363494106764, -0.022624400469051124, 0.025683977543284862, 0.02960791692702783, -0.04703241025688862, -0.02052522927230777, 0.017150092182762124, 0.03262633327040325, 0.03822412188662214, -0.008094843152635868, -0.007182458127813522, -0.0050592769703974175, 0.0030990216104864595, 0.01927670200409928, -0.024984252569273696, -0.015627163767454974, 0.007141297862616469, 0.010873158026638689, 0.031830570316346146, -0.030568322804518218, 0.0027783148740028215, 0.011744382320602714, 0.009953912880006624, -0.019523664526604135, -0.027604787435620563, -0.004808885783971509, -0.018522097918255428, -0.019921546003632686, 0.0030904464582243094, -0.0209505512365752, 0.0025707987503957616, -0.0005938219017812694, 0.00344716836854771, 0.017877255781367098, 0.019098342562336706, 0.008629926134536286, 0.00675370610264123, 0.024901932970202125, 0.002028855879516257, -0.02683646310615727, -0.026603220827271856, 0.018247696771156768, 0.019070902075097824, 0.020717310820334862, -0.003930800902536495, 0.001984265553414344, -0.00302527599952387, -0.012526426893685449, -0.0016241136986014002, 0.009933332514577462, 0.0165326886704676, 0.017890976024986537, 0.01253328701549517, -0.005793300690812111, -0.01791841464958034, 0.04014493364160292, -0.020566390003166093, -0.0156957631229071, -0.021320994089009945, 0.02068987033309598, -0.007943922335467096, -0.012245165624777067, 0.0011747813079999476, 0.02064870960223766, 0.007429419718995839, -0.016875689173018387, 0.032516575046737876, -0.01747937244169347, 0.0021952117379262642, -0.019208102648647155, -0.0085544657259519, 0.012800828040784109, 0.024792172883891676, -0.0220069969567566, 0.0034060081033506573, 0.023914086605472857, 0.005107297357404191, 0.00922674928140165, 0.00634553397723683, 0.012066804320369415, -0.00854074548233246, -0.001714151633200807, 0.009830432550076732, -0.007895902414121593, 0.013260450614100142, 0.02700110416694548, -0.008067402665396987, 0.021211232140054423, -0.008753406464466176, 0.018192817659324078, -0.029388396754406933, -0.021595395236108608, -0.011449400808074892, 0.002383862759387228, -0.0352331460304856, -0.0126704875890445, -0.013534851761198804, -0.008883746916205786, 0.016848250548424582, 0.007045257554264191, 0.006647375611574372, 0.014899999237527462, -0.0050009668663373325, 0.02571141616787867, -0.008870027603908882, 0.005347398361115518, -0.02930607529269029, 0.003790170268082305, 0.016093646462580727, 0.007813581883727487, -0.002244946922554833, -0.016258287523368938, 0.023914086605472857, -0.01584668394007587, 0.018604419379972068, -0.003285957601494993, 0.009748112019682627, -0.006215193059835951, 0.004767725518774456, -0.015201840871865003, -0.005621799973875448, -0.01570948336652654, 0.002963535834558925, 0.008595625525487685, 0.028084990374365758, -0.019811785917322237, -0.04469999864390492, 0.0339983427311867, -0.04019981461608068, -0.015942725645411954, -0.027563626704762242, -0.0184534985628033, 0.002339272433285315, 0.022363719565571902, -0.006270073568652445, 0.00895234813430299, 0.0036289592681989537, 0.017534253416171233, 0.019523664526604135, 0.05101123247775442, -0.003073296386530643, 0.002915515447552152, -0.0030853013668670193, -0.014131674908064168, -0.013445672040317515, 0.017397052842621902, -0.02255579925095392, -0.01666988924401693, -0.00870538654312067, 0.01872789970990196, -0.020319427480661238, -0.03926684922582917, -0.008636786256346006, -0.007484299762151063, -0.009055248098803717, 0.01017343398395006, 0.0017424492864199514, 0.017767493832411572, 0.016189686305271737, 0.009535451037548912, 0.007751841253101273, -0.0030029810693035478, -0.015050920054696233, 0.026671822045369057, -0.01516068107232922, 0.029415837241645815, 0.005865331038491637, 0.012711647388580284, 0.012094244807608297, -0.007251058414588187, -0.01639548809691827, -0.01019401434937922, -0.008897467159825227, -0.000649988450606451, 0.022871361128910903, -0.0004218922526085231, -0.00674341591992665, -0.032104971463444806, 0.034025779493135434, 0.014488397516879472, -0.028290790303367216, -0.0068531764718983675, 0.017383332599002464, -0.007196178371432963, -0.03822412188662214, -0.008184023804839693, -0.6142202369608868, -0.019605984125675706, 0.015736923853765423, -0.020854511393884192, 0.010859437783019248, 0.02586233698504744, 0.017602854634268434, 0.018330018232873408, -0.00481917550102482, 0.05158747525919062, -0.0060642722426671805, -0.009556030471655534, -0.013136970284170253, -0.02540957453354113, -0.002881215304164819, -0.02875727299849297, 0.0028966503454060555, -0.03018416157110911, 0.017794934319650454, 0.02390036822449849, -0.013088949431502211, 0.014639318334048242, -0.0018607848742385026, -0.0073059389234046805, -0.008835726994860281, 0.02565653705604598, 0.0001686068358788216, -0.010591896757730307, -0.011367080277680786, 0.01102407884380746, -0.009569750715274975, 0.02274788079898101, 0.0023341273419280246, 0.01697173087835447, 0.039623571834644475, -0.028002668912649114, -0.02427080921428816, 0.015393922419892096, 0.010722237209469918, 0.04030957283974605, -0.024654970447697273, -0.01020087447118894, 0.023255524224965088, 0.032928174904740794, -0.0037387198201706716, 0.031199446560432186, 0.02365340570199364, -0.0008806571942830239, -0.02040174894237788, -0.0013265595239796164, -0.00013034069912215915, 0.006455294529208548, 0.01544880246304732, 0.00924046952502109, -0.006259783385937865, 0.006633655367954932, 0.03172080836739063, -0.011538581460278717, 0.0010881733178900838, -0.015270441158639668, -0.008074262787206707, 0.02067615008947654, -0.029827438962293805, -0.05158747525919062, -0.028427990876916547, 0.0340806604676132, -0.013267310735909863, 0.003961670985018968, 0.003320257744882326, -0.02247347965188235, 0.007676380844516887, 0.02892191405928118, -0.016203406548891176, -0.01031749467930911, -0.0003327117750276729, 0.03048600320544665, 0.031940328540011526, -0.036028912709832855, 0.008033102987670923, 0.02709714400963649, 0.015736923853765423, -0.019551105013843016, -0.004579074497313492, 0.015736923853765423, 0.02579373762959531, -0.005662960239072501, -0.0009338224980241115, 0.019592263882056263, 0.024010128310808943, -0.008801427317134217, 0.015791804828243185, 0.018000736111296985, -0.03097992452516621, -0.032983055879218556, 0.020223387637970228, -0.006836026633035335, -0.011970763546355869, -0.007477439640341343, 0.0008540745715163095, -0.015750644097384862, -0.008719106786740111, -0.02052522927230777, 0.023036002189699114, 0.03254401180868661, 0.02424336872704928, 0.00825948421342408, -0.004994106744527612, -0.009741251897872907, -0.002862350202018723, -0.014529557316415255, -0.017685174233340005, -0.0025296384851987088, -0.007470579984192892, -0.014433516542401709, 0.009549171281168353, -0.03682467566388996, 0.023077162920557434, -0.013308470535445647, -0.037291160221660784, -0.02739898564397403, 0.030376241256491125, -0.0029000804063109157, 0.011922743625010366, -0.028373111765083857, 0.011161279417356791, 0.023049722433318553, 0.025340975178089, -0.019564823394817382, 0.01875534019714084, 0.022322558834713582, -0.009748112019682627, -0.022816482017078216, 0.01102407884380746, 0.005570349525963815, 0.014886278993908021, 0.0019019451394355555, 0.01983922640456112, -0.0017613143885660478, 0.023118323651415758, -0.008280063647530702, -0.02630137919293431, -0.001720154123368995, -0.0034060081033506573, 0.004750575214250155, -0.019084622318717263, -0.011078958886962685, -0.006956076902060365, 0.002037430798947773, 0.0007743266450085074, -0.015887844670934192, -0.017877255781367098, -0.020854511393884192, 0.0029961209474938274, 0.029498156840717382, -0.004037131393603353, -0.021279833358151624, 0.008767126708085616, -0.031007365012405093, -0.012505846528256289, -0.03152872868200861, 0.0004917788704301695, 0.02288508137253034, 0.006218623120740812, 0.000673998585902179, -0.013109529796931372, -0.009809852184647572, -0.0168208100611857, 0.030348800769252247, -0.03462946276181052, -0.023639685458374197, -0.010008792923161848, -0.044727439131143805, 0.02536841566532788, 0.028812153972970732, -0.006126012407632125, 0.018261417014776207, 0.000625978315310723, -0.0007014387238951084, 0.005470879156706677, 0.001192788894919829, 0.0073059389234046805, 0.00563894981273848, -0.011456260929884611, -0.007388259453798786, 0.02274788079898101, 0.023406445042133857, 0.026809022618918387, 0.021307273845390506, -0.00675370610264123, 0.02862007242494364, 0.02064870960223766, 0.01724613202545313, -0.018426058075564418, 0.013322190779065088, -0.004205202515296425, 0.006513604633268633, 0.0033648478381536045, 0.007326518823172572, 0.009137568629197823, 0.038580844495437444, 0.029553037815195145, 0.0019293852774284852, 0.0023427024941901752, -0.021471914906178717, 0.015915285158173073, -0.004287523045690531, -0.008650506499965446, -0.01294488873614316, -0.0004238216327636776, 0.009329649245902379, -0.009391390342189861, -0.015901564914553634, -0.017355892111763582, -0.022693001687148325, 0.004726565253577403, 0.005745280769466607, -0.018604419379972068, 0.01458443735957048, -0.016875689173018387, -0.016875689173018387, 0.016889409416637826, 0.01893370150154849, 0.0184534985628033, 0.017671453989720563, -0.018302577745634527, 0.028867033084803418, -0.0011696362166426575, 0.011476841295313773, 0.020991711967433522, -0.0239278068490923, -0.010619336313646652, 0.015791804828243185, 0.021046592941911284, 0.014886278993908021, 0.01859069913635263, 0.00688404655438084, 0.020168506663492465, 0.0063112338338494976, 0.02244603916464347, -0.004493323906014526, -0.0018556398992965297, -0.007477439640341343, -0.010358655410167431, -0.02067615008947654, 0.02048406854144945, -0.006462154651018268, 0.019290422247718722, -0.003989111472257849, -0.03600147222259397, 0.013425091674888354, 0.004942656296615979, 0.0009183873985752167, -0.03213241195068369, -0.00784788156145355, 0.004719705131767683, -0.0011713512470950876, -0.005131307318076942, 0.0039959711284063, -0.00633867385542711, 0.02390036822449849, 0.009768692385111789, 0.0067056857156344565, 0.021224952383673862, 0.0189611401261423, 0.01525672091502023, -0.02014106803889866, -0.009556030471655534, -0.021705155322419057, -0.011229879704131456, -0.0025193483024841283, -0.012979189345191761, -0.01666988924401693, 0.013267310735909863, -0.02106031132288565, 0.005659530178167641, 0.012389226320136119, -0.003435163388211334, 0.014707918620822907, 0.013088949431502211, 0.009542311159358632, -0.010125414062604554, -0.043904235689847824, 0.009391390342189861, 0.0204429096732362, -0.006348964038141691, 0.003934230963441356, -0.0443707165223285, -0.003424873205496754, 0.005090147052879889, 0.011209299338702294, -0.017493092685312912, -0.0008407832019252937, -0.0039959711284063, 0.015146960828709779, 0.005549769626195923, -0.008444704708318913, 0.014529557316415255, -0.02204815768761492, 0.005055846909492557, -0.019578543638436825, -0.003333977755671132, 0.002754304680499435, -0.022789041529839335, -0.0010598756646709394, 0.008883746916205786, -0.004856905705317013, -0.01378867347419084, -0.014090515108528382, -0.0018059045982526437, -0.036495393542313535, 0.035617309126539785, 0.0013265595239796164, 0.0013702923348553144, -0.016107366706200166, -0.0032928174904740793, 0.01172380288649609, -0.0220755981748538, -0.011716942764686369, 0.024654970447697273, 0.014001334456324557, 0.0004664824913989482, -0.013452531230804698, -0.006650805206817964, 0.028510312338633187, 0.05059962889446135, 0.0346569032490494, -0.015860404183695314, 0.014515837072795815, -0.003930800902536495, -0.0036701195333960065, -0.021979556469517717, -0.030431122230968888, 0.016052485731722403, 0.01525672091502023, 0.021856076139587826, -0.016615008269539166, 0.007093277941270964, -0.010139134306223995, 0.021293553601771067, 0.009130708507388103, -0.00025510760748157917, -0.029223755693618722, -0.012704787266770564, -0.024805891264866042, 0.020840791150264753, -0.014200275194838833, 0.01458443735957048, 0.008506444873283859, 2.5939513631893172e-05, -0.005820741178050992, 0.023914086605472857, 0.02572513641149811, -0.017534253416171233, -0.01459815760318992, -0.013136970284170253, 0.005536049382576482, 0.006105432507864233, -0.00716187822804563, -0.00922674928140165, 0.011751242442412433, 0.014502116829176374, -0.021677714835180176, 3.148649431088776e-05, 0.0007996229949358993, 0.007443139496954011, 0.016711048112230176, 0.017438213573480223, -0.01699917136559335, 0.0034454533380952803, -0.0012931168958184988, 0.016368047609679387, 0.02687762383701559, -0.02420220799619096, -0.02630137919293431, 0.0019396753437277484, 0.015970164270005763, -0.030101840109392465, 0.0053645486656398195, 0.02027826861244799, -0.0008017667830014369, -0.00908954870785232, 0.00895920732479017, -0.026959943436087156, -0.005899631647540237, -0.01282826852802299, -0.022665561199909444, 0.008197744048459134, -0.0085544657259519, -0.02554677696973553, 0.018947421745167933, -0.01566832263566822, -0.014529557316415255, -0.037346037470848394, 0.01697173087835447, -0.0035706489313082346, -0.004157182128289652, -0.021430754175320397, -0.009116988263768663, 0.011579741259814503, 0.005992241894987655, 0.011126978808308189, -0.003394003123014281, 0.003635819390008674, -0.005855041321438325, 0.002145476320467061, -0.03583682929916068, -0.009459990628964526, -0.027467586862071233, 0.0288944735720423, -0.018124216441226877, -0.008650506499965446, -0.019784345430083356, -0.008849447238479722, 0.01115441929554707, 0.009075828464232879, -0.012039363833130534, -0.006805156084891594, 0.004483033723299946, 0.012704787266770564, 0.022693001687148325, 0.009123848385578384, 0.01679336957394682, 0.015215561115484444, -0.016217126792510615, -0.004915216275038366, -0.030211600195702914, -0.01676592908670794, -0.04700496976964973, 0.016861968929398948, 0.0076695207227071675, 0.002922375569361872, 0.012883148571178215, 0.022789041529839335, -0.02427080921428816, 0.02674042326346626, -0.012059944198559696, 0.005433148952414485, -0.004942656296615979, 0.0035569289205194285, 0.007813581883727487, 0.007779281274678885, 0.015874124427314753, -0.006126012407632125, -0.024298249701527042, -0.026918782705228836, -0.026932502948848278, 0.03172080836739063, 0.0028469151607774866, 0.00689433673709542, 0.024147328884358273, 0.025148893630061907, -0.009741251897872907, -0.06618563565634815, 0.01529788164587855, -0.019441343064887495, 0.02368084618923252, -0.0019997005946555805, -0.014982319767921567, 0.005762430608329638, -0.010399815209703216, -0.023571086102922068, 0.011909023381390925, -0.006417564324916356, 0.00784788156145355, -0.031748248854629506, 0.003920510719821915, 0.0042223523541594565, -0.0009355374702688829, 0.0346569032490494, -0.03213241195068369, -0.017973295624058104, 0.011792403173270755, -0.00344030824673799, 0.015435082219427881, 0.009844152793696173, 0.013164409840086596, -0.030348800769252247, -0.01278710779716467, -0.016272005904343304, -0.0239278068490923, 0.011408241008539108, 0.0021043162881006425, 0.028098708755340124, 0.019153221674169392, 0.032736095219358774, -0.01697173087835447, 0.005186187826893436, 0.03476666519800493, 0.0012665342730517845, 0.0011190435167878737, 0.011758102564222155, -0.007525460027348116, -0.023351564067656098, 0.006565055081180267, -0.008911187403444668, -0.015586003036596652, 0.015366481932653215, 0.014282595725232938, 0.0006426996875989404, 0.017836095050508774, -0.014063074621289503, 0.010091113453555954, -0.037044195836510856, -0.0411053395190933, -0.03550755090287441, 0.019386463953054805, -0.013020349144727546, -0.01127789962547696, -0.033010496366457434, -0.003903360880958883, 0.020195947150731346, 0.0008579333318266186, 0.004407573314715561, -0.001140481048197298, 0.028126149242579005, -0.0016849964647554473, -0.015174400384626124, 0.008623066012726565, 0.019208102648647155, -0.010639916679075812, -0.028290790303367216, -0.04530368191258001, -0.014954879280682686, 0.024682410934936155, 0.00676056622445095, 0.037812522028619226, -0.001881365006837029, -0.0006800011342780256, -0.024174769371597155, 0.017712614720578886, -0.006640515489764652, -0.00027268646589218227, 0.011456260929884611, -0.02630137919293431, 0.013171269961896317, -0.01665616900039749, -0.0011559162058538512, 0.006424424446726076, -0.009830432550076732, 0.005776150851949079, -0.022830202260697655, 0.03841620529729431, -0.028812153972970732, -0.01983922640456112, 0.007751841253101273, 0.015586003036596652, 0.01868673897904364, 0.016381767853298826, 0.013088949431502211, 0.023228083737726207, 0.0061328725294418455, 0.0008510732682245569, 0.015462522706666761, 0.03062320377899598, -0.013150689596467155, 0.0074637198623831716, 0.01144254068626517, -0.011229879704131456, -0.0101871542275695, 0.0038244704114696374, 0.008904327281634946, -0.03482154617248268, -0.0476360935255637, 0.015832963696456433, 0.009219889159591928, 0.002596523741520944, -0.024984252569273696, -0.014008194578134276, -0.03235193212330459, -0.0038656306766666902, 0.002443888126730377, 0.024929373457441006, 0.015586003036596652, -0.0029275206607191624, -0.008863167482099163, 0.023941527092711738, -0.00220550168781021, -0.002551933648249665, -0.008053683353100083, -0.005748710830371467, 0.006194613160068059, 0.011730662076983273, -0.003484898572839903, -0.0005187902508099915, 0.003375138020868185, 0.02572513641149811, -0.0019105201752823886, -0.01265676734542506, 0.019153221674169392, -0.020360588211519558, 0.025148893630061907, 2.4090518253238977e-05, -0.019784345430083356, 0.030156721083870228, -0.012759668241248325, -0.017726334964198325, -0.006493024733500741, -0.011325920478145002, -0.0010538730580874343, -0.025560495350709898, -0.0059064913036886895, 0.007271638780017348, 0.007889042292311872, 0.01090059851387757, 0.01841233783194498, 0.03698931858732324, -0.016861968929398948, 0.011497421660742933, 0.0007052974842054175, -0.03331233800079498, 1.400144208956123e-05, -0.004579074497313492, 0.0339983427311867, -0.042422464280108846, 0.012306905789742013, 0.003275667418780413, -0.015832963696456433, 0.011106399374201565, 0.01086629790482897, -0.016121085087174532, 0.008245763969804639, 0.023667125945613078, -0.007106997719229137, 0.008787707073514776, 0.0037833101462725846, 0.011511140973039838, -0.022939962347008104, 0.004321823189077863, -0.019180662161408273, -0.012636186979995898, 0.016752208843088496, -0.018508377674635985, -0.018535818161874867, -0.029937199048604254, -0.020291988856067433, 0.02177375654051626, 0.011614041868863103, 0.001460330269454721, 0.0269736636797066, -0.02077218993216755, -0.0018059045982526437, 0.006520464755078353, -0.009048387976993998, 0.006445004346493968, -0.003591229063906761, 0.021403313688081516, 0.016052485731722403, -0.007868461926882711, -0.001980835492509484, -0.022898801616149784, 0.019180662161408273, -0.01294488873614316, -0.052163718040626826, -0.006685105815866565, 0.002196926768378694, -0.0030321361213335903, -0.008801427317134217, -0.015572282792977211, -0.006805156084891594, 0.002092311074933632, -0.013342771144494248, 0.0209505512365752, 0.030156721083870228, 0.0019053752003404156, 0.003807320339775971, -0.015393922419892096, 0.002920660538909442, 0.0053748388483544, -0.01709521120828436, -0.026754141644440625, -0.005728130464942306, -0.010063673897639609, -0.01857697889273319, -0.007799861640108046, 0.010434115818751817, 0.05523701535848001, 0.0020683011142608797, -0.014241435925697155, -0.02233627907833302, -0.019592263882056263, -0.032297051148826826, -0.017397052842621902, -0.023241803981345645, 0.02538213590894732, 0.016848250548424582, 0.024023848554428382, 0.002867495293376013, 0.01731473138090526, 0.01697173087835447, 0.0031607617754514047, -0.029772557987816042, 0.003069866325625783, 0.002505628291695322, -0.021279833358151624, 0.00509357711378475, -0.0014714778509801992, -0.04703241025688862, -0.005608079730256008, 0.01267734771085422, -0.011620901990672823, -0.008918047525254387, 0.01817909741570464, -1.6667744844770976e-05, -0.012732227754009444, -0.0069320669413876134, -0.006445004346493968, 0.02218535826116425, 0.0008858022274326555, 0.008465285073748075, -0.054852852262425825, 0.010585036635920588, 0.006321523550902809, -0.010159713740330619, -0.019770625186463917, -0.013390791065839753, 0.01584668394007587, 0.0035980891857164813, -0.004146891945575071, 0.0015923859844773956, -0.016491527939609278, 0.00647244436807158, -0.014611877846809361, 0.03048600320544665, 0.019249263379505475, 0.010639916679075812, 0.02218535826116425, 0.0028434850998726264, 0.007333378944982293, -0.016450367208750954, 0.012217725137538186, 0.013287891101339024, -0.005326818461347626, 0.04066629544856135, -0.04505672125272023, 0.012327485223848637, 0.005940791447076022, 0.004716275070862822, -0.019825506160941676, -0.016985451121973912, 0.015901564914553634, -0.031062245986882852, -0.03419042241656872, 0.03358673914789364, 0.014172835638922488, -0.017959575380438665, 0.007614640679551942, -0.016450367208750954, 0.0023118324117077024, 0.015860404183695314, -0.02053894951592721, -0.006081422547191481, -0.021238672627293304, 0.006650805206817964, -0.002596523741520944, 0.006527324876888074, 0.016189686305271737, 0.001764744449470908, 0.015119520341470898, -0.01572320361014598, 0.01699917136559335, 0.191971264537516, -0.009171869238246425, 0.0035191987162272354, 0.02588977747228632, 0.03627587336969263, 0.007751841253101273, 0.026246500081101624, 0.004088581841514987, -0.00495980660114028, -0.016477807695989836, 0.008479005317367515, -0.0028229049672741, -0.0008892322301298571, 0.00021266112765286232, 0.013287891101339024, -0.011250460069560616, -0.031748248854629506, -0.022939962347008104, -0.008828866873050562, 0.012231445381157627, -0.013548572004818245, -0.01750681292893235, -0.009281629324556874, -0.01693057014749615, 0.021128912540982855, 0.004791735479447208, -0.016148525574413413, -0.0070109574108768585, 0.012354925711087518, 0.01268420783266394, -0.018206536040298444, -0.010255754514344165, 0.003676979655205727, 0.0003740863608236209, -0.005865331038491637, -0.01365147290064151, 0.009014088299267933, -0.006537614593941385, -0.0029052254976682057, 0.004328682845226315, 0.029937199048604254, 0.019359023465815924, 0.014076794864908941, -0.023159482519629005, -0.00851330499509358, 0.018645580110830392, 0.003323687805787186, 0.0032688072969706927, -0.030293921657419558, 0.009144428751007544, -0.04047421576317934, 0.012780247675354949, 0.03314769507736169, 0.009665791489288521, -0.026315099436553753, -0.005316528278633046, -0.02064870960223766, 0.0005976806620915785, 0.007271638780017348, -0.005909921364593549, -0.015613443523835532, -0.019345303222196485, -0.019070902075097824, 0.03303793685369632, -0.031748248854629506, 0.026356260167412073, 0.009741251897872907, 0.010564456270491428, 0.02090939050571688, -0.008931767768873828, -0.01171008264287665, 0.01376809310876168, -0.03976077054554873, 0.009302209689986034, -0.0377027600796637, -0.039815651520026495, 0.02000386746534933, 0.019208102648647155, 0.0004501899058276677, 0.029415837241645815, 0.004205202515296425, -0.011126978808308189, -0.0010976058689631321, -0.019468783552126372, -0.011634622234292263, -0.039541250372927834, 0.02683646310615727, -0.01336335150992341, -0.01676592908670794, 0.017877255781367098, -0.03067808289082867, 0.0002043004560606683, -0.006115722690578813, 0.0029292356911715923, 0.007964502700896258, 0.028729832511254088, 0.03715395778546638, 0.0072373386366300155, 0.005820741178050992, -0.04587992469401621, -0.021320994089009945, 0.09307697787433811, 0.003172766988618415, 0.0008163443672241167, -0.0016892840408865226, 0.00675713616354609, 0.003385427970752131, 0.024723571665794475, -4.0597480812083965e-05, -0.024490331249554135, -0.005350828422020379, -0.021156353028221733, 0.0020134208382750207, 0.019153221674169392, 0.013267310735909863, 0.020470348297830007, 0.0006752848587415015, 0.007539180270967557, 0.012396086441945838, -0.015791804828243185, 0.011216159460512016, -0.010468415496477881, 0.008918047525254387, 0.02023710788158967, -0.017383332599002464, -0.03816924463743453, -0.0009724101593348607, 0.0043698431104233675, -0.04335542967036035, -0.0237082866764714, 0.006661095389532544, -0.04994106465130851, 0.011531721338468998, 0.017671453989720563, -0.0058276008341994435, -0.007415699475376398, -0.00344030824673799, -0.005145027561696383, -0.00771068098790422, 0.018878820527070732, 0.011600321625243663, 0.007916481848228216, 0.004805455723066648, -0.008719106786740111, 0.0076420807011295545, -0.015146960828709779, 0.005968231934314903, 0.008437844586509194, 0.0024370280049206567, -0.006908056980714861, -0.012238305502967348, -0.002966965895463785, -0.00839668478697341, 0.016560127295061403, 0.029498156840717382, 0.017685174233340005, -0.018563258649113748, -0.021567954748869727, 0.022020717200376037, -0.0011044658743575352, -0.05427660948098962, 0.017836095050508774, 0.005734990586752026, -0.005062707031302277, -0.011010358600188019, -0.010502716105526482, -0.17397052842621902, -0.01005681377582989, 0.007895902414121593, -0.04338287015759923, 0.01719125105097537, -0.010139134306223995, 0.03520570926853687, 0.0007790429205450315, -0.03394346175670894, 0.007422559597186119, 0.03495874488338694, -0.006256353325033004, 0.000520934038875529, -0.015572282792977211, -0.0107496776967088, -0.005062707031302277, -0.024311969945146485, 0.006462154651018268, 0.028345671277844976, -0.004009691372025741, 0.03622099239521487, -0.022665561199909444, 0.012725367632199724, -0.0003305680160659646, 0.004705984888148242, -0.0058584713823431845, -0.021663994591560733, 0.027055983278778166, -0.009693231976527402, -0.03855340400819856, -0.0012622466969207091, 0.005072997214016858, 0.03215985243792257, 0.0028246199977265303, -0.012890008692987936, 0.008348664865627905, 0.01307522918788277, -0.004932366113901398, -0.04047421576317934, 0.006726265615402349, 0.0272617850704247, 0.036166111420737106, 0.011058378521533524, -0.00924732964683081, -0.03427274387828536, -0.012210865015728467, 0.008773986829895337, -0.03484898665972157, 0.006568485142085126, -0.009823572428267013, 0.026658101801749615, -0.016546408914087037, 0.011895303137771485, -0.002695994343608716, 0.0027371546088057687, 0.009151288872817263, 0.004778015235827767, -0.018673018735424197, 0.00227238717696308, -0.00364953940079748, -0.005566919465058955, -0.015146960828709779, 0.020881951881123073, -0.007868461926882711, -0.004191482271676985, -0.023776886031923527, -0.004719705131767683, -0.003735289992096446, -0.028427990876916547, 0.016230847036130057, -0.02564281681242654, -0.0038313303004487236, -0.014406076986485366, -0.01724613202545313, 0.03430018436552425, 0.02878471348573185, -0.013095809553311931, 0.0061843229773534784, -0.0009149573958780151, 0.013253590492290423, 0.01841233783194498, 0.0025742288113006218, -0.016203406548891176, -0.01584668394007587, -0.00785474168326327, -0.0038244704114696374, 0.004599654397081384, 0.03366906060961028, -0.007943922335467096, -0.021691435078799615, 0.028373111765083857, -0.02180119702775514, 0.013061509875585867, 0.0088082865076214, 0.021074031566505093, 0.00825948421342408, 0.0209505512365752, -0.009370809976760701, 0.013027209266537266, -0.02067615008947654, 0.018014456354916428, 0.004345833149750616, -0.01680708981756626, -0.002610243985140384, 0.0330653773409352, -0.00551889954371345, 0.009995073610864944, 0.01376123298695196, 0.0319677690272504, -0.0009295349218930363, -0.016560127295061403, -0.011428820442645732, 0.02503913354375146, 0.005477739278516397, -0.010557596148681707, 0.012697928076283381, -0.02026454836882855, -0.020977991723814083, 0.010351795288357711, -0.008725966908549833, 0.060039041020641795, -0.02351620512844431, -0.024586371092245145, 0.01128475974728668, -0.009185588550543328, -0.03967845280912224, -0.12007808204128359, -0.03556242815206202, 0.02919631520637984, 0.0220069969567566, -0.005594359952297835, 0.0220069969567566, -0.011387660643109946, 0.03295561539197967, -0.018014456354916428, 0.043492632106554754, 0.012094244807608297, -0.024257088970668722, 0.022926242103388665, -0.012025643589511095, 0.00537140878744954, -0.005961371812505183, 0.030925045413333522, -0.022981123077866428, -0.01584668394007587, 0.04826721728147766, 0.015037199811076792, -0.0008630783649762502, 0.016079926218961284, -0.015229281359103885, -0.017630293258862242, -0.015599723280216093, -0.032242170174349064, 0.012595027180460114, -0.015956445889031397, 0.014001334456324557, 0.016587567782300285, -0.003632389329103814, -0.0018539248688440996, -0.019359023465815924, -0.012128544485334361, 0.011367080277680786, 0.007388259453798786, -0.017726334964198325, 0.017012889746567717, -0.005752140425615058, 0.004736855436291983, 0.0019122352057348187, 0.017355892111763582, 0.0034197281141394634, -0.029498156840717382, -0.015201840871865003, -0.023804326519162408, 0.023914086605472857, 0.01003623341040073, -0.008993507933838773, -0.029854879449532686, -0.006691965472015017, -0.0275499064611428, 0.0007717540993298623, 0.012382366198326398, 0.011929603746820085, 0.023132043895035197, 0.016724768355849615, 0.00130940945228595, -0.00716873834985535, -0.018906261014309613, -0.0168208100611857, -0.004359553393370056, -0.0021694865139704478, -0.00882200675124084, 0.007895902414121593, -0.01020087447118894, -0.012306905789742013, 0.01699917136559335, -0.01212168436352464, 1.052452856701674e-05, 0.008156583317600812, 0.00021512645482440126, 0.02081335066302587, -0.024819611508485485, -0.006150022833966146, 0.01113383893011791, -0.004812315844876369, 0.02106031132288565, -0.002562223598133611, -0.025903497715905763, -0.006647375611574372, 0.0206624298458571, -0.026054418533074532, 0.0030527162539321167, 0.008636786256346006, 0.003179626877597501, -0.019153221674169392, 0.01253328701549517, -0.018535818161874867, 0.0020511508097365793, 0.03065064240358979, -0.015805525071862624, -0.01841233783194498, 0.0036563995226072004, 0.0046065145188911045, -0.015860404183695314, -0.024161049127977712, 0.008561325847761621, 0.013253590492290423, -0.014515837072795815, -0.0076420807011295545, -0.04631896503925801, 0.02427080921428816, 0.007525460027348116, -0.010687936600421317, -0.004218922293254597, 0.02090939050571688, 0.024778452640272237, 0.0013882999217751956, -0.023571086102922068, 0.011380800521300227, -0.01859069913635263, 0.004966666257288731, -0.033010496366457434, -0.029717678875983356, -0.017712614720578886, -0.022857640885291464, 0.001090745863568729, 0.0031659068668086947, 0.006393553898582335, 0.016038765488102964, -0.0022226517595038764, -0.009590331080704137, 0.006366113877004722, 0.023022281946079675, 0.024984252569273696, 0.015284161402259109, -0.0366874769529857, 0.022363719565571902, -0.023872927737259613, -0.019372743709435366, 0.02848287185139431, -0.002870925354280873, -0.004798595601256928, 0.00044418735745182104, -0.01031063455749939, 0.006856606532803228, 0.005292518317960294, 0.012848848893452152, 0.026123019751171733, 0.042504785741825486, -0.015064640298315673, -0.03161105014372525, 0.01365147290064151, -0.037208838759944136, 0.015778084584623743, -0.02757734694838168, -0.009789271819218412, -0.0039651010459238285, 0.021224952383673862, 0.012739087875819165, 0.026685542288988496, 0.0020631560229035896, -0.026383700654650954, -0.02674042326346626, -0.008849447238479722, -0.0010615906951233695, 0.007353959310411454, -0.002493623311358946, 0.016038765488102964, -0.01927670200409928, 0.03937661117478469, 0.008341804743818185, 0.0064278540419696675, 0.010797697618054304, 0.016848250548424582, 0.0008377819568411996, -0.026905062461609397, -0.01680708981756626, 0.02395524733633118, -0.0072167582712008545, 0.0009484000240391327, -0.01199820403359475, 0.021211232140054423, 0.026850183349776707, 0.008300644012959863, -0.022487199895501794, 0.0088082865076214, 0.024380569300598613, -0.02683646310615727, 0.0030510012234796864, 0.025972098934002964, -0.0013188420033589984, -0.014543277560034696, -0.006150022833966146, 0.013136970284170253, 0.0006345533511575561, -0.022253957616616377, 0.002673699180557759, -0.01432375645609126, 0.0007893329286366361, 0.017973295624058104, -0.001956825531836732, -0.014707918620822907, 0.008225183604375477, 0.012162845094382962, 0.03111712509871554, -0.02409244790988051, 0.005566919465058955, 0.023667125945613078, 0.016477807695989836, 0.0010495855983716762, -0.0040954419633247065, -0.013418231553078634, 0.0024198779332269904, -0.01926298176047984, 0.01885138003983185, -0.046236647302831516, -0.03852596352095968, 0.032022650001728166, 0.03847108627177207, 0.016738488599469057, 0.01898858061338118, -0.007662660600897447, -0.0014320326162355764, -0.009933332514577462, 0.006691965472015017, -0.0035123385944175155, -0.01912578118693051, -0.019372743709435366, 0.029964639535843135, 0.01683453030480514, 0.01690312966025727, 0.035260586517724485, -0.009199308794162768, 0.0421206226457713, -0.015188120628245563, 0.06437458585032291, -0.013864133882775227, 0.007196178371432963, 0.025121453142823026, -0.0010109978788532684, 0.005628660095685168, 0.009562890593465255, -0.003999401189311161, 0.012910589058417096, -0.04069373593580024, -0.022624400469051124, 0.011662061790208608, 0.020374308455139, 0.07117973727860182, 0.029443277728884696, -0.0014697628205277693, -0.002021995757706537, -0.006849746410993508, -0.013747513674655057, 0.0030132710191874937, 0.016052485731722403, 0.0007344526526507771, -0.026520901228200285, 0.02058011024678553, -0.015133240585090338, 0.015325321201794894, -0.026850183349776707, 0.008094843152635868, -0.022596959981812242, 0.01144254068626517, 0.01683453030480514, 0.008101703274445588, 0.0012819693142930206, 0.04810257435804438, -0.015750644097384862, 0.04807513759609565, 0.011895303137771485, -0.0159838845136252, -0.01017343398395006, 0.040858378859233525, -0.006163742611924318, -0.014035635065373158, -0.02189723687044615, 0.005199907604851608, 0.02602697804583565, -0.04917274218449029, -0.023941527092711738, -0.015023479567457351, -0.014529557316415255, -0.004956376540235419, -0.018906261014309613, 0.011394520764919668, -0.007799861640108046, -0.0012365215893802098, -0.0036975597878042533, 0.004483033723299946, -0.009823572428267013, -0.0027062845263232964, -0.007189318249623242, -0.026342539923792634, -0.006568485142085126, -0.005141597500791523], 'source': 'https://arxiv.org/pdf/2303.08774.pdf', 'page': 43}), 0.9295841455459595)\n","(Document(page_content='performance based on models trained with no more than 1/1,000th the compute of\\nGPT-4.\\n1 Introduction\\nThis technical report presents GPT-4, a large multimodal model capable of processing image and\\ntext inputs and producing text outputs. Such models are an important area of study as they have the\\npotential to be used in a wide range of applications, such as dialogue systems, text summarization,\\nand machine translation. As such, they have been the subject of substantial interest and progress in\\nrecent years [1–34].\\nOne of the main goals of developing such models is to improve their ability to understand and generate\\nnatural language text, particularly in more complex and nuanced scenarios. To test its capabilities\\nin such scenarios, GPT-4 was evaluated on a variety of exams originally designed for humans. In\\nthese evaluations it performs quite well and often outscores the vast majority of human test takers.', metadata={'_id': ObjectId('6616def76182e3cd8e9a2210'), 'embedding': [-0.025839819974608345, -0.004514352466209241, -0.003384033370396938, -0.014692416917054007, 0.017545044660491678, 0.017586587231409004, -0.011216642227954031, 0.00689961976756956, -0.01833436282114578, -0.027404611255804078, 0.011825941339536397, 0.03107425282805573, -0.035228565799137766, -0.02081310370103755, -0.008433253654916054, 0.022917953992093468, 0.016700333300872842, -0.00767162929977685, 0.0006980109546459682, -0.017475806421414473, 0.01408311780547164, 0.008481720608534594, -0.0182374289139087, -0.018292820250228454, 0.010136521414266195, 0.011777474385917856, 0.03115733983253536, -0.037028767465724984, -0.01477550299021115, 0.0030707290347211044, 0.015094000380082271, -0.016894202977991983, -0.02208709139787706, -0.006927315435729438, 0.0028682062075316684, -0.007387751495105575, 0.009319506654129726, -0.010545028328673184, 0.004355103771273679, -0.004687448529563496, -0.001184844418633161, 0.015647909086667373, 0.023679578347232673, -0.011583606571443694, 0.001659128312089237, 0.0028197392539131274, 0.018015865431274657, -0.028443188567252097, 0.001694613095880802, 0.004659753327064863, 0.01822358201115125, 0.03398227377045813, -0.0037873476962698846, 0.0002944801253982979, -0.018722098450094107, 0.0077339440874753295, -0.00019851985761517728, 0.009457984063606623, 0.017988171625759757, 0.00114416681347693, -0.011957494366312082, -0.008135527550503594, -0.024704308755923242, 0.033788405955983966, -0.00579526501141121, -0.015080152546002333, -0.03700107366021008, 0.01592486204297619, -0.003652332710974216, -0.0017647171156262803, -0.0026570286004575817, 0.008668664040307543, 0.002052922543070503, -0.025036653048551814, 0.03725032908571405, -0.010406551850518775, 0.001853861674401052, -0.009873414429392338, -0.023014887899330552, 0.01250447869019597, 0.018735947215496533, -0.009471830966364073, 0.0140346508518531, 0.03403766138148793, 0.005681021544404826, -0.004614748099135685, 0.015980253379295945, -0.003053419242121181, -0.017018830690743964, -0.0018036637415225188, 0.017115764597981045, -0.0024475820890587988, 0.017683521138646085, 0.014041574303231824, -0.015315563862716312, 0.01991300286774394, 0.003224784675461377, 0.02874783905436577, -0.014387767361262823, -0.01980222019510443, -0.011015850962101142, 0.0029132112026302244, -0.02036997673576947, -0.01444315776626009, -0.023194908811047264, 0.008426329272214839, 0.018195886342991373, 0.00010748200753298275, 0.0036661803122235324, 0.005694869378484765, -0.02024534716037251, 0.021810138441568244, -0.011493596115585336, -0.0107596683599285, -0.01591101514021874, 0.00623839197667929, 0.02085464627195488, -0.008080137145506329, 0.0008001377996630267, -0.0020581153644351692, 0.025798277403691016, -0.01672802896903272, 0.014401615195342762, 0.002134277753382965, 0.02682300781238159, 0.006608818045858315, -0.018195886342991373, -0.03107425282805573, 0.003498276837403321, -0.000425600611085778, 0.012670651767832743, 0.0025756733901451203, 0.027349219919484323, -0.007706248884976697, -0.03079729987174691, 0.010150369248346133, -0.0244827452732892, 0.00600990457700528, -0.03838584216704413, -0.0182374289139087, 0.01753119589508925, 0.02782004255291228, -0.015966406476538495, -0.009001009264258604, -0.007900116233789617, 0.02838779909357732, 0.015606365584427557, 0.010953536174402662, 0.011819017888157672, 0.011749778717757979, 0.02023150025761506, -0.01783584638220292, -0.008938694476560125, 0.02773695554843265, 0.01167361702730205, -0.011431282259209346, 0.00423739811291669, 0.022765628748536635, -0.009485678800444012, -0.024662766185005916, -0.017766608143125716, 0.01881903235733119, -0.00507172243282246, 0.0069930919491172795, 0.018583621971939697, 0.019885307199584062, 0.009070247503335809, 0.02027304282853239, -2.4503949594601142e-05, -0.005383295905653613, -0.008987161430178666, 0.024551983512366406, -0.03387149109781862, 0.023416472293681306, -0.013238407377175293, 0.000709694977339433, 0.013259178662633958, -0.007366979743985667, 0.002184475802676809, -0.0163818377736467, 0.008107831882343718, 0.018749794118253983, -0.003207475115692076, -0.0042512459469966285, -0.015661755989424823, -0.02712765643685028, 0.01397233606415462, -0.00151026549271363, -0.018002018528517207, -0.019636048048790145, 0.019885307199584062, 0.0018140496170824727, 0.004171621599528848, -0.007422370614644177, -0.6279104366730462, -0.01042732313597744, -0.006279935013257861, -0.015163238619159476, 0.025812124306448466, 0.013916944727834866, 0.01430468128810568, -0.0011346464857546277, -0.02427503055605759, 0.020605387121160963, -0.008807141449784442, -0.015315563862716312, -0.012629108265592928, -0.0251751313893512, -0.009305658820049788, -0.03345605793806544, -0.01737887251417739, -0.018154343772074043, 0.003873895960777634, 0.020508453213923878, -0.02919096601963385, 0.014720111653891395, 0.0012990880020548547, 0.019774524526944556, -0.029163270351473973, 0.0003435962107912497, -0.008800217998405717, -0.013702305627902039, 0.005546006326278536, 0.01613257862285278, -0.03594864572071466, 0.02318106004564484, 0.0035346270526172264, -0.004334332020153772, 0.04140464205679609, -0.0022035162252907916, -0.0295510059804223, 0.013965411681453406, -0.008447101488995992, 0.026629139997907423, -0.029606397316742052, -0.012670651767832743, -0.0017586587464239628, 0.008675588423008756, -0.013875402156917538, 0.03719494147468425, 0.0015968136408060687, -0.014900132565608109, -0.0064080263143441825, -0.0144154630294227, -0.012075200490330316, 0.010489637923675918, 0.02427503055605759, -0.002167166010076886, -0.010967384008482602, 0.00441049464193219, 0.03539473980809703, 0.0019334860218688324, -0.015467888174950659, -0.028498579903571852, -0.0023177599251277964, 0.005947589789306801, -0.0035519368452171496, -0.04021374136443621, -0.018846728025491064, 0.01912368284444486, 0.00801782235780785, -0.0029080183812655587, 0.0032680588077152522, -0.041072297764167515, -0.010358084896900235, 0.019982241106821143, 0.0026535668747682192, -0.008938694476560125, 0.003943134665516082, 0.005334828952035072, 0.021837834109728123, -0.01697728811982664, 0.000915679588425887, 0.026961484290535995, 0.011715159598219378, -0.01419389954678866, 0.00532444330930574, -0.00041997498669845844, 0.019331397561676473, 0.00915333450781544, -0.005251742878877929, 0.01499706647284519, 0.02449659217604665, 0.006958472363917433, 0.007851649280171074, 0.024579679180526285, -0.020453061877604126, -0.023748816586309878, 0.012227524802564662, -0.0008806275785531478, -0.0288586198643603, -0.005269052438647229, 0.0050751846241730665, -0.010448094421436103, -0.009437211846825472, -0.02473200442408312, 0.03542243361361193, 0.028360103425417442, 0.035007000453858746, 0.017434261987852168, -0.01466472124889413, 0.0012713925667255995, -0.00595451370634677, -0.006826919337141749, -0.008142451933204809, 0.0022692929715092554, -0.006556888900889167, -0.009610308375840971, 0.016838811641672228, -0.029135574683314097, 0.013813087369219058, -0.006563812817929137, -0.022710239274861856, -0.03173894513860283, 0.019885307199584062, -0.016340293340084393, 0.012442164833819977, -0.007477761485302687, 0.009873414429392338, 0.010787363096765888, 0.01476165515613121, -0.027626174738438116, -0.00020998749334922922, 0.014470853434419966, -0.01557866991626768, -0.01811279933851174, 0.05727411276345252, -0.01047579008959598, 0.02963409298490193, 0.02143624971537737, 0.033511449274385194, 0.0036454087939342466, 0.006086067198783698, -0.013646915222904772, -0.021020818418269166, 0.00293398272091951, 0.00021712771003712814, 0.004171621599528848, -0.018057409864836962, -0.014235442117705987, -0.011237413513412694, 0.006047985887894489, 0.00945105968090541, -0.0007330629645187072, -0.0005911240039754139, -0.008073212762805116, -0.0045732050625571135, 0.01419389954678866, -0.0016461461422622612, -0.023513406200918387, -0.01080813531354704, -0.05303671465053583, -0.008557882298990523, -0.019151378512604737, 0.011348195254729715, 0.003217860758421408, -0.009001009264258604, 0.008793293615704502, 0.006657284999476856, -0.02577058173553114, -0.01694959245166676, 0.038524318645198535, -0.04794076013788783, -0.028013910367386444, 0.0057641080832232135, -0.031295818173334745, 0.018611317640099576, 0.025950602647247852, -0.02016226015589288, -0.0023887294927109263, 0.004358565496963042, -0.001409004195534223, 0.0018296281975917815, -0.025493628779222323, -0.005715641129604673, 0.01110586048663701, -0.01105739353301847, -0.020217651492212635, 0.019843762766021757, 0.03265289287465389, 0.04126616557864168, 0.015744842993904454, -0.022931802757495897, 0.00018434758861525814, 0.017794303811285595, 0.018569775069182247, -0.028581666908051483, 0.02830471208909769, 0.0011986921362977885, 0.030603432057272745, 0.01208212394170904, 0.006511884138621233, 0.015232476858236681, 0.03265289287465389, 0.012012885702631836, 0.010184988367884736, 0.014263137785865864, -0.019968392341418718, -0.011652844810520898, -0.012456011736577429, 0.017226547270620556, -0.020425368072089226, 0.0010965652912807299, 0.003345952059507729, 0.006799224134643116, -0.011403586591049468, -0.016824962876269802, -0.002205247320966095, -0.005061336790093128, 0.018348211586548206, -0.0025756733901451203, 0.0018642475499610059, -0.025382846106582813, -0.02944022330778279, 0.005130575494831577, -0.0007252736160563973, 0.016783420305352476, 0.00922949619827137, -0.012850671748226968, 0.020522301979326307, -0.022294807977753652, 0.01855592630377982, 0.02506434871671169, -0.0270030287240983, 0.0006633917186920548, 0.017558891563249128, 0.01833436282114578, 0.016229512530089862, 0.013293798713495049, 0.0017725064640885901, 0.011763626551837918, -0.007027711068655881, 0.037361111758353556, 0.0031711249004781703, -0.0048051541879204854, 0.007498533236422595, -0.0011519560455239288, -0.02874783905436577, 0.02509204438487157, -0.009444136229526685, 0.011188946559794154, -0.018874423693650944, -0.02693378862237612, 0.0014938213643666694, -0.0122413726366446, 0.018472841161945166, -0.022045548826959735, 0.005767569808912576, 0.03292984583096271, -0.0085855779671504, 0.006234929785328683, 0.003555398570906512, 0.006054909804934458, 0.030298781570159077, 0.004407032450581583, 0.01339073262073213, 0.02529975910210318, 0.006878848482110896, 0.022170178402356695, -0.02863705638172626, 0.002959947293404084, -0.015204782121399292, -0.021574726193531776, -0.013051463014079856, -0.025922906979087976, -0.003494814878883336, 0.019456027137073433, -0.026379880847113506, 0.01844514549378529, 0.008135527550503594, 0.015232476858236681, 0.019386788897996228, 0.03079729987174691, 0.016284903866409618, -0.028235473850020485, -0.052039681772650115, 0.009644927495379573, 0.001973298195602723, -0.007851649280171074, -0.0032784444504445837, -0.054532269555299336, 0.011853637007696274, -0.0029720640318087192, -0.014180051712708721, -0.014062346520012977, 0.018002018528517207, -0.00212216101497833, 0.009818024024395073, -0.002641450136363584, 0.018860576790893493, 0.027196896538572463, -0.004445113761470792, -0.012656803933752805, -0.001361402673338023, 0.012089048324410254, -0.0025445159962958807, -0.01288529086776557, -0.03184972408595238, 0.015758689896661904, -0.018708251547336657, -0.035256259604652666, -0.01014344486564492, 0.010205759653343398, -0.015634060321264947, 0.031461988457004056, 0.00043663550200483185, 0.010565800545454336, -0.017115764597981045, 0.0025877901285497554, 0.016797267208109926, -0.016797267208109926, 0.022294807977753652, 0.03514548065730311, 0.01264987955105159, 0.01600794904745582, -0.026075232222644813, -0.007699324967936728, 0.018140495006671618, 0.0401029586917967, 0.04057377946257968, -0.012580641311974387, -0.012456011736577429, -0.028360103425417442, 0.0022242879764106994, -0.030852691208066663, -0.01721269850521813, 0.01252525090697712, 0.02096542708194941, 0.01264987955105159, -0.023084126138407754, -0.010884297004002971, -0.0007785007335360893, 0.026739920807901957, 0.007318512790367126, -0.0023506481818217173, -0.013411503906190794, -0.014401615195342762, -0.010572723996833061, 0.007366979743985667, -0.020882341940114756, 0.013653838674283499, -0.005840270239340388, 0.011368967471510867, -0.025341303535665487, 0.025825973071850895, 0.020480757545764002, -0.013231483925796568, -0.0280693017037062, -0.0262414043689591, -0.012386773497500224, -0.006006442851315918, 0.005310595475225801, 0.005390219822693582, 0.00532444330930574, 0.001995800809567312, -0.01466472124889413, 0.012386773497500224, 0.01693574554890931, 0.010835830050384429, 0.011666692644600837, 0.003745804659691313, -0.01310685435039961, -0.017946627192197452, -0.023291842718284345, 0.0032040131571720912, 0.023485710532758507, -0.008668664040307543, -0.03165585627147822, 0.01384770648875766, 0.020023783677738473, -0.02474585132684057, -0.0006607952498020663, 0.02200400625604241, 0.002556632734700516, 0.002707226649751426, 0.002579135348665105, -0.019359093229836352, -0.01959450547787282, -0.006567275009279744, -0.036142513535188825, 0.005556392434669112, -0.0026449118620529466, -0.01161130223960357, 0.0006036734580912195, -0.021519336719857, -0.010468866638217255, -0.034813132639384584, 0.012892215250466783, -0.01067658135544887, -0.012421392617038826, -0.048827014068424, -0.010655810069990205, 0.015370954267713578, 0.01476165515613121, 0.005483692004241301, -0.0026501049162482348, 0.0013406310386334262, 0.0034550027051494458, -0.014650873414814192, -0.02529975910210318, -0.0038150431315991393, -0.030824995539906787, 0.019192921083522066, 0.006411488040033545, -0.022931802757495897, -0.01444315776626009, -0.008481720608534594, 0.009478755349065287, -0.0007975413307730382, -0.010898144838082909, -0.007837802377413624, -0.008994085812879879, 0.015855623803898985, 0.019968392341418718, 0.014747807322051273, -0.003911977271666843, 0.007664705382736881, -0.027640021641195566, -0.021380858379057614, -0.027044571295015626, -0.006754218906713938, -0.03885666293782711, 0.012760661292368611, 0.005670635901675495, 0.020605387121160963, 0.012781433509149764, 0.020093021916815675, -0.023582644439995592, 0.0138615543228376, -0.003330373362583109, 0.006619203688587647, -0.007325436707407096, 0.008377862318596298, 0.000662958944773229, 0.005130575494831577, 0.022322503645913528, 0.013930792561914804, -0.017129613363383474, 0.001431506693083501, -0.017891237718522676, 0.02506434871671169, 0.01592486204297619, -0.00825323274319934, 0.03148968598780891, 0.008183994504122135, 0.0019317550426088401, -0.04434035680471339, 0.009887262263472278, -0.014076193422770427, 0.01316916913809809, -0.0011709967009685333, -0.008087060596885054, -0.026947637387778545, -0.013162244755396877, -0.014789350824291088, 0.004400108999202858, -0.013646915222904772, 0.0018607855914410212, -0.003641946835414262, -0.006449569350922754, -0.00762316234615831, 0.005874889358878989, 0.0029668712104440535, -0.02597829831540773, -0.011417434425129408, 0.0037111853073220884, 0.021283924471820533, 0.022973345328413223, -0.005144422863250271, 0.016340293340084393, -0.011957494366312082, -0.027169200870412587, -0.012691423053291406, -0.04193085416389882, -0.004545509860058481, 0.01087045010124552, 0.028526275571731728, 0.016991136885229064, 0.03542243361361193, -0.011244337896113907, 0.0022121712380060643, 0.03996448221364229, -0.01105739353301847, -0.006286858930297831, -0.010974307459861327, -0.015481736009030599, -0.025798277403691016, -0.0012134052853847565, -0.007290817587868493, -0.0020373438461458834, 0.009866490978013613, -0.001942140801753483, -0.010704277023608745, 0.014173127330007508, -0.008640969303470155, 0.008509415345371982, -0.0144154630294227, -0.015121696048242149, -0.025729039164613814, 0.020453061877604126, -0.021713204534331162, 0.0024475820890587988, -0.0251751313893512, -0.009028704932418481, 0.031600464935158466, 0.004275479423805899, -0.0030845766359704208, 0.014076193422770427, 0.04251245760732131, -0.01663109506179564, -0.028526275571731728, -0.001091372353500753, 0.020605387121160963, 0.004036606381402558, -0.00973493795123793, -0.053369062668454356, 0.016908049880749433, 0.02201785315879986, 0.0022727549300292403, 0.028775534722525645, -0.004642443767295562, 0.002603368825474375, -0.0013648646318580077, 0.012642956099672866, 0.004628595933215623, -0.009361049225047054, -0.0029737948946534005, -0.015855623803898985, -0.009811100573016348, -0.004940169406046776, 0.011645921359142173, -0.0026916479528268063, -0.003416921627090859, 0.01970528628786735, -0.011368967471510867, 0.021270077569063083, -0.03387149109781862, -0.021907072348805325, 0.027390764353046628, 0.009852643143933675, 0.03198820056410679, 0.011867484841776214, 0.013792316083760395, 0.02063308278932084, 0.020466910643006552, -0.006352635443685673, 0.0064011023973042135, 0.00687538629076029, -0.0041750833252182104, 0.0103788561823589, 0.005923356312497531, 0.013266103045335171, -0.018569775069182247, 0.015066304711922395, 0.011098937035258286, -0.040047567355476946, -0.038441233503363884, 0.017918931524037576, 0.014678569082974068, 0.022862564518418692, -0.030963472018061197, -0.007823954543333686, -0.025742886067371264, -0.00024384946225668058, -0.011112784869338223, 0.020093021916815675, 0.007997050141026698, -0.00983187185847501, 0.0066711328335567945, 0.03010491375568491, 0.005549468517629143, -0.0026258714394389645, -0.006390716754574882, 0.005483692004241301, 0.021228534998145754, 0.015620213418507495, -0.00720773151471135, 0.013072235230861009, -0.00828785279406043, 0.02773695554843265, 0.0057571841661832445, 0.005504463289699965, -0.0003390524338895115, -0.006345711526645703, 0.020522301979326307, 9.136240869344307e-05, -0.0005266455795134274, 0.030658821530947525, -0.03226515724570556, -0.014276985619945804, 0.018472841161945166, 0.010101902294727592, -0.005546006326278536, -0.03290215202544781, -0.0066330515226675855, 0.0017846232024932255, 0.006148381520820934, 0.02462122175144361, 0.01946987590247586, 0.002459698827463434, -0.016215663764687437, 0.009326430105508451, -0.004199316802027482, -0.026019840886325057, -0.01346689431118806, -0.015426345604033331, 0.009721090117157992, -0.04663907677288844, 0.008128604099124869, 0.0020113792736613095, -0.013944640395994743, 0.023748816586309878, 0.008412481438134901, -0.02085464627195488, 0.01773891247496584, 0.038662795123352946, -0.007193883680631411, -0.012075200490330316, 0.00036436778728819096, -0.013633067388824834, -0.02577058173553114, 0.010884297004002971, -0.018140495006671618, -0.009097943171495686, 0.02932944249778826, -0.02565980092553661, -0.02073001669655792, -0.010898144838082909, -0.017392719416934842, -0.004407032450581583, 0.0028145464325484617, 0.004358565496963042, -0.0010844485528760945, -0.007941659736029432, 0.0014592021284127563, -0.0017794302647132483, 0.006345711526645703, -0.009021780549717268, 0.014207747380868599, 0.0015007451649913278, 0.018666708976419328, -0.015052456877842455, -0.0027383840436006656, -0.006681518476286127, 0.015744842993904454, -0.021006971515511716, -0.060763737149277415, -0.009077171886037022, -0.017032679456146393, 0.0076370101802382484, 0.002310836008087827, 6.0800086258545866e-05, -0.01636798900824427, 0.004407032450581583, -0.0163818377736467, 0.0078101067092537476, 0.0032213227169413925, 0.012456011736577429, 0.0020373438461458834, -0.005732950689373974, -0.004022758547322619, 0.010503485757755858, -0.03700107366021008, -0.020079175014058225, -0.014117736925010243, 0.0020771560198797737, -0.025618256491974304, -0.01402080301777316, -0.011403586591049468, 0.05605551454028779, -0.002463160785983419, -0.012109819609868917, -0.009367973607748267, -0.013134550018559487, -0.045891299320506694, -0.016658790729955516, -0.023084126138407754, 0.022419437553150613, -0.005175580257099511, 0.033400666601745684, -0.0003249883583692988, 0.01980222019510443, 0.003273251629079918, 0.002065039281475138, -0.013487666527969212, 0.0007105604669694292, -0.007879344948330952, -0.004846697224499057, 0.0027383840436006656, 0.0031849725017274867, -0.03791502139626115, -0.0051651946143701786, 0.007990126689647973, -0.020702321028398044, 0.003257672932155298, 0.023984226971701366, 0.0015561360356498377, 0.004334332020153772, 0.0015803695124591082, 0.015454040340870721, 0.016769573402595023, 0.01811279933851174, 0.025576713921056978, -0.03533934847177727, -0.006141458069442209, 0.005739874606413943, -0.018278973347471004, -0.018777489786413862, -0.02316721314288739, 0.017558891563249128, 0.013944640395994743, 0.003593479881795721, -0.0011580144147262464, -0.007086563665003754, 0.008744826662085961, -0.016908049880749433, 0.030880385013581566, 0.0013068773505171645, -0.007643934097278217, 0.017129613363383474, -0.0005244818845422649, -0.019289854990759147, -0.024455049605129325, 0.008841760569323042, -0.02246098012406794, 0.014706264751133945, 0.03171124760779797, -0.04677755325104285, -0.010690429189528807, 0.006591508486089014, 0.002594714045589725, -0.02736306868488675, -0.011306652683812387, -0.0033424901009877447, -0.0019057905865395774, -0.019885307199584062, 0.028498579903571852, 0.04337101773234257, -0.022613305367624775, 0.011244337896113907, -0.004178545516568818, 0.015592517750347618, -0.0019767601541227074, -0.015329410765473762, 0.00695501063822807, -0.021782442773408368, 0.014006955183693222, 0.023111821806567633, -0.023624187010912918, -0.02077155926747525, 0.017808150714043045, 0.010240378772882001, -0.03918901095574564, 0.020605387121160963, 0.19907463734111963, -0.022765628748536635, 0.010171140533804796, 0.030188000760164542, 0.029938741609370625, 0.0055252350408198725, 0.010974307459861327, 0.014221594283626049, 0.0017707754848285979, -0.02014841325313543, 0.006228005868288714, -0.016451076012723904, -0.0027487696863299975, -0.00223294275629535, 0.019982241106821143, -0.026712225139742078, -0.03201589809491164, -0.05522465567136133, -0.013356112569871039, 0.008897151905642798, -0.0031970892401321223, -0.023042583567490428, -0.004701296363643435, -0.02268254360670198, 0.028803230390685525, 0.002028689066261233, -0.002487394262792689, -0.006830381062831112, 0.018708251547336657, 0.013598448269286231, -0.011542063069203877, -0.005390219822693582, 0.01339073262073213, 0.0037042616231127415, -0.01764197856772876, 0.0033026779272538544, 0.00817707105274341, 0.0066261276056276165, 0.00595451370634677, 0.010420398753276225, 0.031268120642529894, 0.0052621285216072605, 0.03683489965125082, -0.026642986900664873, 0.004562819419827781, 0.01902674893720778, -0.011002003128021205, -0.002239866440504697, -0.005258666795917898, -0.004971326799896016, -0.03301293097279736, 0.019303703756161573, 0.024690459990520816, 0.021754747105248492, -0.024718155658680692, 0.004455499404200124, -0.025022806145794364, -0.00767855321681682, 0.021782442773408368, 0.0006698828036055428, -0.03387149109781862, -0.01672802896903272, -0.015052456877842455, 0.028803230390685525, -0.01247678395335858, 0.02211478706603694, -0.015135542950999599, -0.022308654880511102, 0.013009920443162529, -0.0012359077829340347, -0.023998075737103795, 0.010101902294727592, -0.021228534998145754, 0.00046433090823025917, -0.02701687562685575, -0.04999022095526898, 0.022405588787748183, 0.02014841325313543, 0.02233635054867098, 0.04298328210339424, 0.0025618257888958035, -0.03021569642832442, 0.003245556193750663, -0.02093773327643451, -0.02326414705012447, -0.03550551875544658, 0.017586587231409004, -0.011583606571443694, -0.008073212762805116, -0.011936723080853417, -0.01753119589508925, -0.006103376758553, -0.005518311123779903, -0.004417418558972159, -0.004722068114763342, 0.004604362456406353, 0.0067576806324033, 0.004344718128544348, 0.012386773497500224, -0.03306832230911711, -0.014637025580734252, 0.08125834159779886, 0.015412497769953393, 0.009610308375840971, 0.013916944727834866, 0.011064317915719683, 0.0020979275381690595, 0.015384802101793516, -0.009160257959194165, -0.011396662208348255, 0.0004664946032014218, -0.022945649660253347, 0.011382814374268318, 0.00537983417996425, -0.0004928918331895103, 0.012234449185265875, 0.004652829410024894, -0.014623177746654314, 0.025964449550005302, -0.017434261987852168, 0.01444315776626009, -0.012940682204085324, -0.017004983787986514, 0.002724536209520727, -0.016880354212589557, -0.04791306633237293, 0.0039812155107440475, 0.010988155293941265, -0.030077218087525035, 0.009298735368671063, -0.010849677884464368, -0.047137595074476284, 0.01592486204297619, 0.011458976996046734, -0.00586796544183902, -0.003849662483968364, 0.0062141584998700195, -0.003207475115692076, -0.020314585399449716, 0.01836205848930566, 0.01190902741269354, 0.024828938331320202, -0.013826935203298998, -0.00462513420752626, 0.005407529382462883, -0.013556904767046416, 0.01190902741269354, 0.02095158017919196, 0.0026708764345375205, -0.004427804201701491, -0.00012257384107488046, -0.019054444605367656, 0.009333354488209664, 0.004005448987553318, 0.04497735158445563, 0.008578654515771675, -0.01946987590247586, -0.011251261347492632, 0.020494606311166428, 0.0004539451199817885, -0.043454102874177225, -0.0017032678757654526, 0.017046526358903843, -0.009354125773668329, -0.006591508486089014, -0.007616238429118341, -0.17680752130623592, -0.0057191028552940355, 0.007408523246225483, -0.037527285767312825, 0.016201816861929987, -0.00911871445695435, 0.011355119637430928, -0.0050821085412130354, -0.01477550299021115, -0.008087060596885054, 0.013868477774216325, -0.004479732881009394, 0.00212216101497833, -0.002416424695209559, -0.010794287479467103, -0.003659256395183563, -0.017351176846017512, 0.004396646807852251, 0.012926834370005386, 0.008107831882343718, 0.03902283694678637, -0.006961934555268039, 0.010655810069990205, -0.023638035776315344, -0.00537291026292428, 0.00231256710376313, -0.0007049348134782819, 0.032348242387540214, 0.000660362534090896, -0.04409109765391947, -0.002947830554999449, -0.003780424012060537, 0.021200839329985878, -0.0014107350583789043, 0.01649261858364123, -0.00042192232381403595, 0.019552961044310514, -0.002191399486886156, -0.02863705638172626, -0.002205247320966095, 0.018029714196677087, 0.02316721314288739, 0.010309617943281694, -0.017600435996811433, -0.016298750769167068, -0.01908214027352753, 0.014581635175736987, -0.016174121193770107, 0.015855623803898985, -0.0138615543228376, 0.022516371460387694, 0.00022307790031165318, 0.014470853434419966, 0.0033026779272538544, 0.011452053544668009, 0.011140479606175613, 0.011375890922889592, -0.011244337896113907, -0.011756703100459193, -0.0003526837645947261, -0.003766576177980599, -0.0070311732600064885, 0.006345711526645703, 0.022239416641433897, -0.019276008088001697, -0.021270077569063083, -0.013494589979347937, 0.019843762766021757, -0.028498579903571852, 0.010891221386704184, -0.011770550934539131, -0.0004262497137563612, -0.008523263179451922, -0.019996088009578594, 0.03049264938463324, 0.00956184142222243, -0.014180051712708721, 0.026642986900664873, 0.03226515724570556, -0.003344221196663048, -0.00507172243282246, 0.01661724815903819, 0.007311588873327157, 0.001184844418633161, 0.005400605465422914, -0.0011355119171769683, 0.016520314251801105, 0.022724086177619306, 3.8033862668697026e-07, -0.0024337344878094824, 0.03819197435256996, -0.021103905422748797, -0.0017387526595570176, 0.010392704016438837, 0.021173143661826002, 0.01639568467640415, 0.0028941707800162423, -0.01577253866206433, 0.016797267208109926, -0.009063324051957084, 0.005642940233515617, 0.00445896159555073, 0.002485663399948008, -0.009084095337415747, 0.04622364361313527, 0.012462936119278642, -0.0036211753171249764, 0.025729039164613814, 0.042069334367343184, 0.012573717860595662, -0.033760708425179115, -0.013854630871458875, -0.0039292868314361445, 0.004067763775251797, 0.01431852819086313, 0.015232476858236681, -0.012989149157703866, -0.024898176570397404, 0.01069735357223002, -0.004566281145517144, 0.03952135524837421, -0.02277947751393906, -0.013882325608296263, 0.020660778457480714, -0.003493084016038655, -0.03791502139626115, -0.12185981486589345, -0.006411488040033545, 0.020549995784841207, 0.017101917695223595, -0.010046510958407837, 0.0313512095096545, -0.0006188193810970135, 0.03971522306284837, -0.013840783037378936, 0.04522661073524955, -0.009187953627354042, -0.006716137595824729, 0.016326446437326943, -0.0057502602491432755, 0.004164697682488879, -0.014900132565608109, 0.025562867018299528, -0.010067282243866502, -0.01413158475909018, 0.0437587533612909, 0.0034480790209400988, -0.01250447869019597, 0.02439965826880957, -0.022807173182098937, -0.010309617943281694, -0.0056740976273648575, -0.031101948496215604, 0.025742886067371264, 0.004725529840452705, 0.007193883680631411, 0.02474585132684057, -0.03569938656992074, 0.003124388809704311, -0.02991104594121075, 0.018763642883656412, 0.008696359708467421, 0.0005032775923341532, -0.02563210525737673, 0.03342836413255054, 0.008793293615704502, -0.011431282259209346, -0.003970829868014716, 0.008557882298990523, -0.00778933495813384, -0.005597935471247683, -0.020549995784841207, -0.017004983787986514, 0.02655990175883022, 0.015786385564821783, -0.010863525718544306, -0.029966437277530504, 0.012220601351185937, -0.043315626396022815, 0.016104882954692902, 0.0070311732600064885, 0.023194908811047264, 0.02689224605145879, 0.017032679456146393, 0.015274020360476497, 0.010635038784531542, -0.024261181790655163, -0.019317550658919023, -0.005286361998416531, 0.012497555238817244, -0.01019191181926346, 0.005874889358878989, -0.003179779680362821, -0.01958065671247039, 0.011964418749013295, -0.015966406476538495, -0.00847479622583338, 0.015412497769953393, -0.004673601161144802, -0.003494814878883336, -0.00891792319110146, -0.00795550757010937, -0.02246098012406794, -0.006442645433882785, -4.6979427877780365e-05, 0.010212684036044613, -0.031185035500695235, -0.01433237602494307, 0.016824962876269802, -0.0058091128454911475, -0.016894202977991983, 0.008267080577279278, 0.006958472363917433, -0.017448110753254597, 0.012802204794608427, -0.037527285767312825, -0.00800397452372791, 0.05137499318739297, -0.0007321974748887111, -0.015647909086667373, 0.0011303190958123024, -0.004091997252061067, -0.01661724815903819, -0.0013596716940780307, -0.003652332710974216, 0.014941675136525437, -0.011825941339536397, -0.00949952663452395, -0.06652438024718256, 0.02711380953409283, 0.021242381900903204, -0.021602421861691656, 0.015841776901141535, 0.022350199314073407, 0.016852658544429678, 0.007360055826945698, -0.021117752325506247, -0.004330870294464409, -0.025950602647247852, -0.0055390824092385665, -0.0251751313893512, -0.008571730133070462, -0.008952542310640063, -0.021574726193531776, 0.0047982302708805165, -0.013459970859809335, 0.007796258875173809, 0.007609314977739615, 0.003873895960777634, -0.010351160514199022, 0.035865560578880004, 0.02174090020249104, -0.004542047668707873, -0.010863525718544306, -0.05359062428844342, 0.0262414043689591, -0.010752743977227286, -0.019414484566156104, -0.01100892657939993, -0.012137515278028794, -0.006878848482110896, -0.0038046574888698074, 0.002464891881658722, 0.009838795309853736, 0.006099914567202392, 0.024593526083283735, 0.00624531589371926, 0.04251245760732131, -0.0138615543228376, -0.043287932590507915, -0.004074687692291767, -0.033788405955983966, 0.0006690173139755466, 0.00034857273250798597, 0.003801195530349823, -0.016783420305352476, 0.01901290203445033, 0.009790328356235195, 0.02383190359078951, 0.008329395364977758, -0.006158767629211509, -0.03533934847177727, -0.011853637007696274, -0.013923869110536079, 0.010787363096765888, 0.010641962235910267, 0.014803198658371026, 0.005400605465422914, 0.04159850987127025, -0.0024129629695201966, 0.013376884786652192, -0.0019940697138920082, -0.0020875416626091052, 0.0019646434157180722, -0.018168190674831493, -0.0014972833228866541, 0.009804176190315135, -0.0016651866812915546, 0.012289839590263143, -0.00383927660840841, 0.020896188842872206, 0.021907072348805325, 0.024676613087763366, -0.02768156607475787, 0.0010654078974314902, 0.005698331104174128, -0.01293375782138411, 0.027889280791989487, 0.02372112091815, 0.01658955249087831, -0.0135153612648066, 0.010011891838869236, 0.018528230635619942, -0.003472312497749369, 0.004728992031803311, 0.023762665351712304, -0.021103905422748797, 0.0006417546525651174, 0.01855592630377982, 0.006941162804148132, -0.0039292868314361445, 0.005798727202761816, -0.011715159598219378, 0.012151362180786244, -0.005061336790093128, 0.004088535526371705, 0.025452084345660018, 0.02023150025761506, 0.019192921083522066, -0.005147885054600878, -0.0011216641995123408, 0.0013129357197194822, -0.032348242387540214, 0.011272033564273785, -0.05414453392635101, -0.0343423118686016, 0.009312582271428513, 0.01165976919322211, 0.014069269971391702, 0.007630086263198279, 0.0060341380538145506, 0.0026587596961328854, -0.023582644439995592, 0.02577058173553114, 0.01080813531354704, -0.02086849317471233, -0.030298781570159077, 0.027044571295015626, 0.02678146524146426, 0.010371931799657685, 0.04935322617552673, -0.016451076012723904, 0.03808119167993046, 0.00038384112934013805, 0.03669642317309641, -0.02370727401539255, 0.028360103425417442, 0.023651882679072794, -0.007990126689647973, -0.011943646532232143, 0.0034792361819587165, -0.020397672403929347, -0.00584373196502975, -0.010531180494593246, -0.016894202977991983, 0.024108856547098326, 0.027072266963175506, 0.08441562169099519, 0.013093006516319672, 0.0010775246358361253, -0.008502491893993257, 0.003377109453356969, -0.0038912055205469354, 0.009222572746892645, 0.02403961830802112, -0.007498533236422595, 0.0031867035974027904, 0.007110797141813024, -0.004832849390419118, -0.010828906599005704, -0.020328434164852142, 0.006148381520820934, -0.006404564122993576, -0.0005742471017096277, -0.0011121439882053497, -0.01881903235733119, -0.007103873690434299, 0.04891009921025865, -0.013099930899020885, 0.022031701924202285, 0.004074687692291767, -0.028124693040025954, 0.0032057440200167725, 0.036004037057034415, 0.011348195254729715, -0.017337328080615087, -0.024538136609608956, 0.0393828787702198, 0.00784472582879235, -0.031295818173334745, -0.021117752325506247, -0.014913980399688047, -0.012864519582306907, 0.005743336332103306, -0.015869472569301414, 0.0029512925135194335, 0.011964418749013295, 0.007443142365764085, -0.009534145754062553, -0.010482713540974705, 0.00418200724225818, 0.00905639966925587, -0.00038297563971014187, -0.014720111653891395, -0.015121696048242149, -0.019303703756161573], 'source': 'https://arxiv.org/pdf/2303.08774.pdf', 'page': 0}), 0.9294002056121826)\n","(Document(page_content='Anna Katariina Wisakanto, Akhila Yerukola, Lexin Zhou, Xuhui Zhou.\\nWe thank Brian Christian, Heidy Khlaaf, Katya Klinova, Haydn Belﬁeld, Owain Evans, Andrew\\nReddie, Paul Scharre, Jason Matheny, Jacob Hilton, Vishal Maini, Sam Manning, Julian Hazell,\\nJason Wei, and Erol Can Akbaba for valuable input on drafts.\\nGPT-4 was used in the following ways: to help us iterate on LaTeX formatting; for text\\nsummarization; and as a copyediting tool.\\nWe thank Microsoft for their partnership, especially Microsoft Azure for supporting model\\ntraining with infrastructure design and management, and the Microsoft Bing team and Microsoft’s\\nsafety teams for their partnership on safe deployment.\\n70', metadata={'_id': ObjectId('6616defc6182e3cd8e9a2320'), 'embedding': [0.006444291819050019, -0.006031376937834323, -0.009705258976823861, -0.029221650606754457, 0.011512201632036914, 0.025579533185220608, -0.021570377830190512, 0.01503432745908896, -0.018252943656233606, -0.028642865320692455, 0.02947575264523952, 0.027104139828195806, -0.013622653917157368, -0.0006273126030725955, 0.0018387052936083702, 0.02676533773109741, 0.025240731088122213, 0.008335934532876408, -0.0037691696049295385, 0.003329786071812492, -0.003839753142327739, 0.006737214174461384, -0.016516585935402544, 0.005946676413559724, -0.002387493495957133, 0.009641733467202596, 0.034106044246960815, -0.024069041666476755, -0.00406915022094847, -0.008597094133477139, 0.0007406876748632989, -0.007538339209859077, -0.02048338846715839, 0.009140588814993201, -0.012154512427720181, -0.006398412077362988, -0.0014152031378966397, -0.012634481599614977, 0.01617778383830415, 0.01631895184442308, 0.005869034382723325, 0.028233480151857122, 0.02572069932869448, 0.0094511569383388, 0.008794729155779134, 0.013170917787692839, 0.0013234443530144736, -0.012069812369106847, -0.006169015231572888, 0.014667292319560295, 0.025805399387307817, 0.021556260843314114, 0.0015943092656774136, -0.007383054682525014, -0.012338030463145777, -0.005533761998005293, 0.013403844345863303, 0.02718883988680914, 0.0046161736835223675, -0.0006304006357441497, 0.009931126110233597, -0.019043481882796528, -0.02142921168671664, 0.016544818046510285, -0.030209822924296848, -0.014540242231640292, -0.03421897641668189, 0.017547107350929073, 0.007132482822081581, -0.004418539592542901, 0.025748933302447277, -0.006101960940893787, 0.0019304641949058523, -0.0189023138766776, 0.03263790368884616, -0.020370454434792257, 0.004404422605666502, -0.024944278089007958, -0.039950372505666645, 0.0005232016875042127, 0.0274570589121706, -0.0015316663005665553, 0.015119028449024822, 0.014624942290253627, 0.0030439219770086946, -0.009380573866601863, 0.01688362014360868, 0.0017090078049877791, -0.022657367193222637, -0.010121702639097391, 0.010171111161842258, 0.008618269613791738, 0.02231856509612424, 0.018507045694718664, -0.0020875128799382014, 0.022290331122371443, -0.007926549364041075, 0.015514296630983755, -0.016925971104237877, -0.027979378113372064, -0.011646310213395116, -0.003493893017452408, -0.009211172818052666, -0.005794921598606025, -0.03873633398603705, 0.005621991768998991, 0.006811326958578684, -0.006211365260879556, 0.002177507041460818, 0.031056827235720307, -0.021542143856437716, 0.041192644999049045, -0.0177023923439244, -0.012154512427720181, -0.009006480233635, -0.01568369861085548, 0.03136739722171096, -0.00399856668355027, -0.0027827623764079628, 0.006857206234604451, 0.022530316173980106, 0.004002095464608105, 0.0191987650131468, -0.008279467516693341, 0.007721856779623409, 0.015500180575429885, 0.009754667499568727, -0.03388017618222855, -0.00984642605162026, -0.016022498845308823, 0.030350990930415777, 0.0033033171870805087, 0.032129699611876034, -0.009761725061684398, -0.03594121901328161, 0.03757875968862293, -0.031395627470173645, -0.009789959035437196, -0.030096888891930716, -0.01715183916897014, 0.0040373874661378375, 0.01816824359762027, -0.00030328934302574566, -0.002242796941611001, -0.00779244031702161, 0.014610825303377229, 0.0026539469666365147, 0.013735587949523499, 0.0023380849732122667, 0.0019198765711638695, 8.012352515980932e-05, -0.01259919006374651, 0.005872563629442424, 0.024774877971781288, 0.0013093275989687072, -0.012923875173968506, 0.00028939316689992285, 0.009218231311490865, -0.011900411320557648, -0.022219748050634508, -0.004760870470699132, -0.006345474307899022, 0.004739695456045798, -0.011829828248820711, 0.01629071787067028, 0.016671869065752815, 0.034388376533908555, 0.015514296630983755, 0.00042703137332700107, 0.0015166672348410138, -0.01571193072196322, 0.018196477571373066, -0.02536778024471969, -0.006684276404997417, 0.003195677024793026, -0.003765640358210439, 0.01688362014360868, -0.0038997494052299044, -0.010919298427775986, -0.0029221650606754456, 0.008554744104170472, 0.0031445036458579777, 0.006444291819050019, 0.005629049796775927, -0.004686757686581832, -0.025706582341818083, 0.034444844481414154, -0.015401362598617624, -0.001960462326356935, -0.005734925335703859, 0.012789766592610304, 0.024958395075884356, 0.003980920449954772, -0.004305605560176769, -0.6401094472507559, -0.026525354542133804, -0.0035256557722630408, -0.016544818046510285, 0.012246271911094243, 0.00871708665928147, 0.013742646442961698, 0.013650886959587636, -0.024506660809064884, 0.024718411886920748, -0.01300857523258184, 0.003578593541727007, -0.0013190329110309151, -0.019946952279080526, -0.011053407009134188, -0.025650116256957546, -0.013629712410595567, -0.011448676122415648, 0.01627660088379388, 0.012987400683589771, -0.024069041666476755, 0.012458022988950108, -0.0002653505810796627, -0.0013234443530144736, -0.0062501865091283875, 0.007446580192146279, 0.011046348515695989, -0.015937798786695486, 0.011392209106232582, 0.023574954576383034, -0.025537182224591413, 0.02913695054814112, -0.002336320349852717, -0.0032556730548645597, 0.04006330840067784, -0.0015846040700305216, -0.030661559053761375, 0.016968320202222015, 0.004347955589483437, 0.023560837589506636, -0.051751968643379616, -0.022050347933407835, 0.012528606060687045, 0.005113789089012662, 0.00522672265571753, 0.03255320176758776, 0.014893160384292559, -0.002955692438845628, -0.01273329957642724, 0.00550199924319466, -0.02141509469984024, -0.006130193983324057, 0.019184649888915457, -0.01774474144190854, 3.686344094837863e-05, 0.012655657079929576, 0.03498128253213707, -0.018267060643110004, 0.01130750811629672, -0.02089277549863878, -0.018549396655347862, 0.020935124596622917, 0.005378477936332494, -0.04057150875235784, -0.01726477320133627, 0.0012819764025569491, 0.004203259267967936, 0.01173101027200845, 0.006874852002538684, -0.021937413901041706, 0.0027757038829697633, 0.01842234563610533, -0.004189142281091538, -0.008886487707830668, 0.0007305412651688614, 0.01691185411736148, 0.010778130421657057, -0.023207920368176895, 0.007834790811989541, 0.027852327094129534, -0.0014257907616386227, -0.03060509110625578, 0.022713833278083174, 0.015951915773571884, 0.018676445811945337, 0.0027562934916759794, -0.024054924679600357, 0.003980920449954772, -0.0014469657762919564, 0.022911469231707695, 0.013792054965706565, 0.01445554124170443, -0.022290331122371443, -0.027979378113372064, 0.01256389852787804, -0.0026645347067938132, -0.0003350519814210622, -0.0037515236041646724, 0.028459346353944332, -0.03004042280707018, 0.003987978943392971, -0.021979762999025845, 0.02701943976958247, 0.004019741698203604, 0.017970609506640804, 0.022657367193222637, -0.03043569098902911, 0.003599768556380341, 0.012189804894911177, -0.016304834857546677, -0.019085832843425722, -0.005629049796775927, -0.005057322072829596, -0.012090987849421444, 0.02385729058862089, -0.02219151407688171, 0.002128098518715951, 0.003931511927209906, -0.03788932967461359, -0.014526125244763894, 0.013185034774569239, 0.01644600100102055, -0.00859003564003894, -0.006550167357977952, 0.02387140757549729, 0.012465081482388308, -0.003386252855164926, -0.03376724028721736, -0.012952108216398776, 0.02032810533680812, -0.012119220891851712, -0.015359012569310956, 0.039357470232728245, -0.009408806909032133, 0.015133144504578692, 0.005159668365038429, 0.020088120285199455, -0.005498469996475561, 0.023264386453037435, -0.0006423116687981369, -0.04181378124574024, 0.0030368637164011275, -0.008808845211333004, 0.002076925372611535, -0.014335548715900098, -0.013403844345863303, 0.011406326093108982, -0.001896936933150986, 0.022502082200227307, -0.018196477571373066, 0.004527944378189933, -0.0029398110614403117, -9.699744499721438e-05, 0.03797402787058187, 0.003301552563720959, -0.042011419062009815, -0.006592517387284619, -0.036364721168993336, -0.004594998668869034, -0.019918720167972786, 0.014074389115299367, 0.026610054600747137, 0.01671422002638201, -0.005819625859978458, 0.005039675839234098, -0.02014458823270505, -0.006585458893846419, 0.060363177901088104, -0.0231796863944241, -0.028064078171985397, 0.013206210254883836, -0.03893396807701652, 0.013488544404476638, -0.004887921024280398, -0.012090987849421444, 0.008406518535935872, 0.003889161665072606, -0.000818770919747244, -0.008978246259882203, -0.007898315390288279, -0.001664010724226471, 0.01599426673420108, -0.011222808057683385, -0.006652513650186784, 0.000775979559977715, 0.019622269031503586, 0.022953818329691837, 0.017843558487398273, -0.0034850701334852912, 0.00943704088278493, -0.0027757038829697633, -0.00155195911995543, -0.01989048619421999, 0.017928258546011606, 0.010813422888848054, 0.026087735399545675, -0.006807797711859584, -0.00636664978821362, -0.01600838185843242, 0.021626845777696108, 0.002521602542976599, 0.026624171587623535, 0.014469658228580828, -0.019791669148730255, 0.02389963968660503, -0.008900604694707067, -0.010531087807932724, -0.02005988631144666, 0.005322010920149427, -0.011173399534938518, 0.01231685498283118, -0.007270121115820146, -0.03057685899514804, 0.007100720067270948, 0.010545204794809122, 0.009027654782627069, -0.015598997620919618, 0.00708660308039455, -0.03755052571487014, -0.02062455647327732, 0.008230058993948476, 0.003079213745707795, 0.012006286859485581, 0.003691527341262507, -0.019961069265956925, 0.030181588950544052, -0.008830020691647603, 0.007453638685584478, 0.04621820664537433, -0.0074818721936760115, -0.011780418794753318, 0.00779244031702161, 0.013269734833182573, 0.010933414483329856, -0.002225151173676767, -0.02761234204252087, 0.03060509110625578, -0.007270121115820146, 0.02601715046516368, -0.012027462339800178, 0.00784184930542774, -0.01561311367647349, -9.617029234933991e-05, -0.03650588731246721, 0.010756955872664987, -0.02127392669372131, 0.02343378843290916, -0.006345474307899022, -0.013022692219458239, 0.023984341607863422, -0.019932837154849184, 0.03181912962588538, -0.011533376181028983, 0.0012202156327105497, 0.018563511779579205, -0.030407457015276314, 0.009684083496509262, -0.004016212451484504, 0.004150321498503969, 0.03913160216799599, 0.0028780504080092285, 0.012048636888792248, 0.008357110013191005, 0.0012502137641616326, 0.007132482822081581, -0.02432314184231676, -0.005727867307926924, -0.004365601823078934, -0.02532543114673555, -0.0011893353059950081, -0.010382862239698124, -0.019636384155734928, 0.01198511231049351, -0.013170917787692839, 0.004107971469197302, 0.008512394074863804, -0.0182952946168628, 0.006401941324082088, -0.0024033748733624494, 0.026567703640117943, -0.013389727358986903, -0.0377763937796024, 0.018972898811059593, 0.018027077454146397, -0.010975765443959052, 0.007594805760380879, -0.025226614101245815, -0.0036703523266091733, -0.02970162070997178, -0.011900411320557648, -0.012422730521759111, 0.018027077454146397, -0.012810941141602375, 0.009034713276065268, -0.004309134806895869, -0.02269971815385183, 0.02044103936917425, -0.024972512062760754, 0.007227770620852215, 0.003458601248753308, -0.01641776888991281, 0.007453638685584478, -0.02947575264523952, -0.008526511061740204, 0.03258143574134056, -0.020822190564256784, -0.022092697031391977, -0.007926549364041075, 0.001560782003922547, -0.02416785871196649, 0.033202571988031754, 0.010037001649161529, -0.00887237072095427, -0.004954975780620763, 0.009768783555122597, 0.019099947967657065, -0.01756122433780547, 0.00988171758748873, 0.011893352827119449, -0.0006255480379207038, -0.035235380845332016, -0.013114450771509774, -0.02426667575745622, 0.021556260843314114, 0.04729107902153005, 0.06533226787461768, -0.02141509469984024, 0.012987400683589771, -0.010298161249762261, 0.02185271384242837, -0.03181912962588538, -0.01241567295964344, 0.000614078160706604, 0.022290331122371443, -0.0031568560093748264, -0.02546659915285448, 0.017900026434903866, -0.007983016380224141, 0.037945797622119186, 0.007757148315491878, -0.00787008234785801, -0.013312085793811768, 0.002465135759624165, -0.01605073281906162, -0.004986738069770132, 0.0064725253271415515, 0.023659656497641422, -0.007203066359479781, 7.289972924888278e-05, -0.020102237272075853, 0.012669774066805974, 0.0063172407998074885, -0.022530316173980106, -0.011272216580428252, -0.002632771951983181, 0.015344895582434558, 0.01928346693440519, 0.017589458311558268, -0.016502468948526146, 0.011547493167905381, 0.016685986052629213, 0.00185811580131747, 0.027852327094129534, 0.022205631063758106, 0.010135818694651261, 0.014392015732083164, 0.013537952927221503, 0.00325214380814546, -0.0017513580671250787, -0.014596709247823358, 0.006229011494475054, 0.026370069549138477, 0.006458408340265153, -0.01987636920734359, 0.008498277087987406, 0.004312664053614968, -0.024619594841431017, -0.012140396372166311, 0.001477846219422814, 0.006730155681023184, -0.0008505335581425605, -0.00275099962159733, -0.017024788149727608, -0.020709256531890652, 0.007347763146656546, -0.028247595276088465, -0.0029645153228127453, -0.007227770620852215, -0.015231962481390953, 0.0137214709626471, -0.02244561611536677, -0.02896755043091445, -0.040006840453172245, -0.007905373883726478, -0.014144973118358831, -0.018690562798821735, -0.027908795041635126, -0.004344426808425601, 0.006677217911559218, 0.019057598869672926, 0.0271606077757014, -0.009422923895908531, 0.0138696965308817, 0.007111307807428247, -0.025410131205348883, -0.015500180575429885, 0.018817613818064266, -0.04568176859465141, 0.011575726210335651, -0.008597094133477139, -0.012945050654283103, -0.005720808814488725, 0.010326395223515058, 0.0038009321269095392, 0.0048314540080973325, -0.013495602897914837, -0.017222422240707077, -0.01631895184442308, 0.02183859685555197, 0.01614954986455135, 0.011272216580428252, 0.010255811220455593, -0.004566765160777501, -0.031056827235720307, 0.01216862941459658, -0.037098793310695724, -0.0010534617520313086, -0.03766346160988133, -0.0019181119478043197, 0.019481101025384657, 0.012817999635040574, 0.015401362598617624, 0.00635606204805632, -0.03475541446740481, 0.00859003564003894, -0.008971187766444004, 0.01445554124170443, -0.00815947592221154, 0.009211172818052666, 0.005360831702736995, 0.02665240369873128, 0.02299616929032103, 0.009112355772562931, -0.045371202333950866, -0.01774474144190854, -0.03856693386881038, 0.025283080186106352, 0.016685986052629213, -0.002936281814721212, 0.030322756956662977, 0.004005624711327206, 0.006041964677991622, -0.029786320768585117, 0.021076292602741846, -0.020370454434792257, 0.014822576381233094, 0.001851057540709903, 0.008152417428773338, -0.02987102082719845, -0.0037974028801904397, -0.030661559053761375, -0.0007658331502832776, -0.002835700145871929, 0.01274035713854291, -0.021203343621984377, 0.009507623954521866, -0.0011955113713381165, 4.640326863892804e-05, -0.026737105619989668, -0.03602592093454, -0.023617305537012228, 0.011025173966703918, 0.01801296046727, 0.014370841183091095, -0.0067478014489574184, 0.02118922663510798, -0.011067523996010586, -0.01743417331856294, -0.0322990997291027, -0.027259424821191133, 0.006641925910029486, 0.014709642348866962, 0.035178916623116535, 0.004372660316517134, 0.041418513063781304, -0.012126279385289911, -0.00184223454032747, 0.035037746754352554, -0.0012158041907269913, 0.00523025190243663, -0.004937329547025265, -0.0050467343326722975, -0.024661943939415155, -0.008477102538995336, -0.0044750066087259665, 0.0013543246797300154, 0.013220326310437706, -0.013763820991953769, -0.001776944640177287, 0.004887921024280398, -0.027104139828195806, 0.011498084645160516, -0.010255811220455593, -0.05745513075861158, -0.03266613393730884, 0.020765724479396248, -0.00943704088278493, -0.017222422240707077, -0.033908410155981346, -0.017547107350929073, 0.012973283696713373, -0.0007190714461624197, 0.013601478436842769, 0.013234443297314104, 0.026779454717973807, -0.008053600383283606, -0.022855001284202103, 0.010756955872664987, 0.012323913476269379, -0.01658716900713948, -0.006257245002566587, -0.03557418480507547, -0.0029892195841851785, 0.02913695054814112, -0.001524607923543672, 0.023405554459156364, 0.005074967840763831, 0.0013710883688151066, -0.005449061473730694, 0.010538146301370923, 0.0004640878235933091, -0.0032856711863156426, 0.0053961237042667275, -0.03348490800026961, 0.0031286225012832937, -0.009712316538939532, 0.007305412651688614, 0.01486492641053976, -0.009133530321555002, 0.002225151173676767, -0.024280792744332622, 0.023391437472279963, -0.009571149464143131, -0.0323273337028555, 0.022332682083000637, 0.01542959657237042, 0.049295653905077515, 0.0023204389724474006, 0.01487904339741616, 0.03774816353113972, 0.02089277549863878, 0.0008148005753959149, 0.042604321334948216, -0.010467562298311459, 0.0004003419106750742, 0.006624280142095252, 0.023207920368176895, -0.004287959792242535, -0.0188881968898012, -0.0012299209447727577, 0.03585651709202321, -0.03244026959786669, -0.02062455647327732, 0.01688362014360868, 0.018097660525883335, 0.020102237272075853, -0.013354435823118436, -0.0038715158971383717, -0.03825636388281973, -0.001896936933150986, -0.0028021727677017467, 0.002260442942375867, 0.0022833825803887505, -0.0013719706804948815, -0.01925523296065239, 0.017518873377176277, 0.02097747555725211, 0.004062091727510271, -0.004418539592542901, 0.0002953486543230875, 0.026977088808953276, 0.018083543539006937, -0.010481679285187857, -0.008660619643098404, 0.0010340511279068928, 0.017645924396418808, -0.007926549364041075, 0.012549781541001642, 0.005921972152187292, -0.010368745252821726, 0.006310182772030553, 0.005985497661808556, -0.008314759983884338, 0.021386860726087444, -0.017885909448027468, -0.0023557309739771328, -0.010975765443959052, 0.008844137678524001, 0.02632771858850928, -0.01932581603238933, 0.009684083496509262, 0.003172737153949511, -0.006684276404997417, 0.010199344204272527, 0.022530316173980106, 0.00044423616004432174, 0.007545397237636013, 0.022487967075995964, -0.004965563055116798, -0.01542959657237042, 0.004287959792242535, -0.0011363975365310417, 0.01502021140353509, -0.07103543371513975, 0.014808460325679224, -0.0010711076363808588, -0.019551684097121592, 0.005022030071299864, -0.0034286031173022253, -0.008780612168902736, 0.014610825303377229, 0.029503986618992315, -0.0068219142330747185, -0.015641347650226286, -0.003783286126144673, -0.010227578178025325, -0.0420678832842253, -0.005650225277090524, -0.009839367558182061, -0.007707739792747011, 0.029306350665367794, -0.01045344531143506, -0.018930547850430395, -0.005043205085953198, 0.0006930436590626659, 0.015443713559246819, 0.004743224702764898, -0.0033721361011191595, -0.011963936830178913, -0.0057808050773908905, 0.009853484545058461, 0.00014215998102635208, -9.324768584178629e-05, 0.02128804368059771, 0.0018545866710136865, -0.0007093661923078697, 0.0045420608994050675, -0.005851388614789091, 0.015500180575429885, 0.000995230112488693, 0.017321239286196808, -0.01171689421645458, -0.06905909280534508, 0.0021969176655852337, 0.012119220891851712, 0.017420056331686543, 0.0013534423680502405, -0.005805509338763323, -0.014822576381233094, 0.012359205943460374, -0.026835922665479402, 0.01590956481294269, 0.012246271911094243, -0.004834983254816432, 0.023320854400543028, 0.006229011494475054, 0.013220326310437706, 0.0008522981815021104, -0.03210146563812324, -0.025791284263076475, 0.007256004128943748, -0.002276324319781183, -0.029532218730100056, -0.011667485693709713, 0.008215942938394604, 0.04345132564637168, 0.003896220158510805, -0.011942761349864316, -0.032722601884814434, -0.014264965644163161, -0.030689791164869116, -0.026101852386422073, -0.021951530887918104, 0.018323528590615597, -0.007453638685584478, 0.030350990930415777, 0.012062753875668646, 0.027301773919175272, 0.0045491193928432665, 0.008611211120353538, -0.015768398669468817, -0.022530316173980106, 0.014483775215457226, -0.005597287507626558, 0.003560947540962141, -0.017236539227583475, -0.034501312428919746, -0.0067019221729316516, 0.004602057162307233, -0.001236097010115866, -0.004909096038933732, 0.01558488063404322, 0.02373023956937836, -0.013065042248764907, 0.0007600981825723988, 0.012606248557184709, 0.0013534423680502405, -0.001366676810416232, 0.03004042280707018, -0.055394087927558525, 0.010411095282128392, 0.01873291375945093, -0.0019286995715463026, -0.0189023138766776, -0.01188629433368125, 0.02274206725183597, -0.003238027054099694, 0.01214745486560451, 0.0031939124014334764, -0.013305027300373569, 0.00650781686301002, -0.01843646262298173, 0.03890573410326372, -0.0027015910988524633, -0.007707739792747011, 0.03325903993553735, 0.02269971815385183, 0.009041771769503468, -0.00743952169870808, 0.020074003298323057, -0.014624942290253627, -0.001208745813704108, 0.03481187868962029, -0.018027077454146397, 0.005473765735103127, 9.561885482543784e-05, 0.013488544404476638, -0.02891108248340886, -0.020709256531890652, -0.00543141570579646, -0.02028575437617892, -0.02483134591928688, 0.027781744022392595, 0.03388017618222855, -0.017363590246826006, 0.011046348515695989, -0.013954397520817563, 0.019862252220467193, -0.009006480233635, -0.02433725882919316, -0.029532218730100056, -0.02085042453800958, -0.012352147450022175, 0.010382862239698124, -0.01259213157030831, 0.010439329255881188, 0.01750475639029988, 0.0030527448609758115, -0.012839175115355171, 0.026779454717973807, 0.18871257250607545, -0.03210146563812324, 0.015062561432841756, 0.018591745753332, 0.021923296914165308, 0.01129339206074285, 0.017674158370171604, 0.009677025003071063, -0.0066136924019379525, -0.00736893816130988, 0.019424633077879064, -0.004203259267967936, -0.0036703523266091733, -0.005085555580921129, 0.013234443297314104, -0.02056809038841678, -0.03915983614174878, -0.029447518671486723, -0.012359205943460374, -0.014667292319560295, -0.0013984395652268647, 0.0029433403081594116, 0.0047255784691694, -0.02175389493429358, -0.002874521161290129, -0.014949627400475625, -0.021372743739211046, -0.002140450649402168, 0.007630097761910612, 0.004429126867038936, -0.010199344204272527, 0.00708660308039455, 0.008053600383283606, 0.008498277087987406, -0.007954782406471345, -0.013072100742203106, 0.01614954986455135, 0.0031021536165513103, 0.03811519773934585, -0.0028286416524337297, 0.006952494499036348, 0.00037674048069470147, 0.010615787866546059, -0.01688362014360868, -0.013417961332739701, 0.035969452987034405, -0.014794343338802824, -0.006716038694146786, -0.018069426552130535, 0.011236925044559783, -0.027555875957660333, 0.0030245115857149107, 0.04124911294655464, 0.007305412651688614, -0.005240839642593928, 0.00902059628918887, -0.03063332508000858, 0.0030086302083095944, -0.0014302022036221812, 0.02329262042679023, 0.0037868153728637726, 0.005717279567769625, -0.024774877971781288, 0.02490192899102382, -0.02439372677669875, 0.005773746583952691, -0.005717279567769625, -0.0140249805925545, 0.01015699417496586, -0.03060509110625578, -0.013912046560188369, 0.0095993825065734, -0.009084121798810135, 0.01603661583218522, -0.044072461893062874, -0.041757317023524754, 0.04404422791931008, 0.01998930323970972, 0.006059610445925856, 0.028628748333816057, -0.004806749746724899, -0.005191431119849062, 0.026483003581504606, -0.027118256815072204, -0.017222422240707077, -0.05330480739746256, -0.009987593126416661, -0.0013984395652268647, -0.013834404995013231, -0.010354628265945326, -0.01743417331856294, -0.012479197537942178, 0.003850340649654406, 0.016643636954645075, 0.0052761316441236605, -0.0012749180255340658, 0.012690948615798043, 0.0011681601749263585, 0.00026204197048816484, -0.011356916639041587, -0.03029452298291018, 0.05096142855417164, -0.005883151369599723, 0.0051737848862535634, -0.0003551242228899933, 0.014469658228580828, -0.006274890770500821, 0.003308610824328526, -0.0015784280046874132, -0.018634096713961195, -0.002046927241160452, -0.023066752362057966, 0.008766495182026338, 0.011378092119356184, -0.004806749746724899, 0.012408614466205241, 0.0094511569383388, 0.003552124656995024, 0.012189804894911177, 0.0038362238956086395, 0.008632386600668136, -0.033569606196237896, -0.007566572252289346, 0.032044997690617645, -0.015274312510697621, -0.01670010303950561, -0.017165956155846537, 0.017349473259949608, -0.025664233243833944, -0.0015254902352234468, 0.021796245894922777, -0.01976343517497746, 0.009211172818052666, -0.01216157092115838, -0.028812265437919125, 0.014264965644163161, -0.0046373491638369655, -0.009571149464143131, -0.017942375532888008, -0.0017804737704810705, 0.015768398669468817, -0.0007137776342914283, -0.010587554824115789, -0.004750282730541833, -0.0016296011507918298, -0.019001130922167334, 0.003003336338230945, 0.0027792331296888633, 0.013043867699772838, -0.03582828684356053, -0.021217460608860775, 0.01256389852787804, -0.0010384626863057674, 0.018704679785698133, 0.050340292307480446, -0.004697344961077867, -0.011801594275067915, -0.019099947967657065, 0.00771479828618521, 0.00787008234785801, -0.036703521403446675, -0.015528413617860153, 0.01627660088379388, -0.027146490788825, -0.010206402697710726, -0.009041771769503468, -0.18272707437860564, -0.015175495465207888, 0.00586197635494639, -0.04616173869786873, 0.016078966792814415, 0.011462792177969519, 0.038086963765593056, 0.0026257136913756134, -0.004958505027339862, -0.0028851086686167956, 0.040148006596646114, 0.010869889905031118, -0.01271918258955084, -0.0023486724805389333, -0.012394497479328843, 0.003906807665837472, -0.027894678054758728, 0.008258292967701272, 0.011512201632036914, -0.0010446387516488756, 0.005367890196175194, -0.008371227000067403, 0.009204114324614467, 0.002332791335964249, -0.0013075629756091574, -0.009938184603671796, 0.005251427382751227, 0.016629519967768677, -0.008787670662340935, -0.046782874944559925, -0.012098046342859643, 0.009479390912091596, 0.01644600100102055, -0.011349859076925916, -0.0011734539285896918, 0.012705065602674441, 0.027245307834314735, -0.007919490870602876, -0.038623398091025864, 0.018351760701723337, 0.032779069832320026, 0.01558488063404322, -0.00415385074522307, -0.014187323147665498, -0.0036138853104261074, 0.020398688408545054, 0.010248752727017394, -0.028289946236717663, 0.010255811220455593, -0.004005624711327206, 0.0007146599459712031, -0.009189997337738069, 0.028416997255960193, 0.007961840899909544, -0.003590945672413224, 0.016925971104237877, -0.0034109571165373592, 0.002652182576107597, 0.00011977171505375537, 0.006550167357977952, -0.0026080676906107473, -0.02011635425895225, 0.011321625103173118, -0.011773361232637647, -0.00860415262691534, -0.0007887728465038192, -0.022770301225588766, 0.005067909347325631, -0.020963358570375713, 0.016234249923164686, -0.007383054682525014, -0.0023963166127548823, -0.01113810799907005, -0.004986738069770132, 0.02244561611536677, 0.010848714424716521, -0.013933222040502966, 0.02243149912849037, 0.02134450976545825, -0.014751993309496158, -0.015570763647166821, 0.022233865037510906, -0.0059819684150894566, -0.006899556729572382, -0.017532990364052675, 0.00022366209635793795, -0.0046514656850521, 0.003086272239145994, -0.012112162398413513, -0.021951530887918104, 0.012182746401472978, -0.022657367193222637, -0.0016278365274322798, 0.01074283888578859, 0.029814554742337913, -0.00018682622401157366, -2.662397812719626e-06, 0.0064725253271415515, 0.01713772218209374, 0.003980920449954772, 0.00026204197048816484, -0.014766109365050028, -0.018069426552130535, -0.006730155681023184, 0.03571535094854934, 0.0007499517728779614, 0.012345088956583976, 0.012966225203275174, 0.02675122074422101, 0.005911384877691256, -0.028699331405552992, 0.007160716330173114, 0.004834983254816432, 0.001561664315602322, 0.0015775456930076383, 0.02003165420033892, -0.027555875957660333, -0.02044103936917425, 0.006461937586984253, 0.006765447682552916, 0.04043034260888397, -0.0038362238956086395, 0.00020204583855322976, 0.012373321999014246, -0.0040585624807911716, -0.019791669148730255, -0.10372980162620139, -0.011251041100113653, 0.01002994315572333, 0.041333814867813026, 0.007488930221452947, 0.012846232677470842, -0.0049267422725292305, 0.02483134591928688, -0.020158703356936394, 0.039018669998274906, 0.00721012485291798, -0.004686757686581832, 0.01688362014360868, 0.003121564007845094, -0.007587747732603944, -0.003474482626158624, 0.010524029314494525, -0.039809206362192776, 0.00371976084935404, 0.03692938946817894, 0.004009153958046305, -0.003031569846322478, 0.002083983633219102, -0.020102237272075853, 0.007940666350917475, -0.017547107350929073, -0.04415716008903116, 0.01718007128007788, 0.008046541889845406, 0.007524222222982679, 0.017490641266068536, -0.008103008906028473, 0.017716507468155743, -0.02632771858850928, 0.0022498554350492, -0.0052549561638090625, -0.003271554432269876, -0.0225867841214857, 0.0038538698963735056, 0.00843475157836614, 0.008314759983884338, -0.008371227000067403, 0.02219151407688171, -0.021993879985902243, -0.014893160384292559, 0.018097660525883335, -0.011921586800872247, 0.027061790730211664, 0.012775649605733906, -0.023250269466161037, -0.020539856414663982, -0.00764421474878701, -0.03323080596178456, 0.0004887921140695714, 0.00799007487366234, 0.00701249029627725, 0.013206210254883836, 0.014406132718959562, 0.0018404699169679202, -0.01885996477869346, 0.002267501202983434, -0.031113293320580847, -0.0020239876031475684, 0.0026257136913756134, 0.00028718741680431464, 0.008223000500510275, -0.015189611520761759, -0.0335131382487323, 0.008745320633034268, -0.012634481599614977, -0.013065042248764907, 0.009062947249818066, -0.011942761349864316, 0.012436847508635511, -0.004566765160777501, -0.012895642131538237, -0.009959360083986393, -0.010764014366103187, 0.014822576381233094, -0.017885909448027468, -0.02810642913261459, -0.02731589090605167, 0.004044445959576037, -0.00815947592221154, -0.012196863388349376, 0.0182952946168628, -0.006564283879193086, 9.418512017367537e-05, 0.02429490973120902, -0.03997860647941944, -0.014109681582490362, 0.020935124596622917, -0.00787008234785801, -0.023518488491522493, -0.0020610439952062185, -0.0008196531732193609, -0.008985304753320402, -0.023927873660357826, -0.003924453433771706, 0.004947917287182564, -0.015627230663349888, -0.006236069522251989, -0.08797551994360897, 0.0329484699495467, 0.004033858219418738, -0.021669194875680246, 0.006652513650186784, 0.010905181440899588, 0.004951446533901663, 0.012408614466205241, -0.03608238515675548, -0.00038821032880497223, -0.03043569098902911, 0.025523065237715015, -0.023123220309563562, -0.020765724479396248, -0.004213847008125235, -0.029419284697733923, 0.006783093450487151, -0.006119606708828022, 0.02104805862898905, 0.01686950315673228, 0.0020186937330689187, 0.006169015231572888, 0.022205631063758106, 0.018803496831187864, -0.0054702364883840275, 0.005304365152215194, -0.035884751065776016, 0.007898315390288279, -0.007312471145126813, 0.002929223554113645, -0.0027404121142706635, -0.027541758970783935, 0.0012802117791973994, -0.001097576521112842, -0.004669111452986334, -0.022092697031391977, 0.008215942938394604, 0.01213333787872811, 0.013185034774569239, 0.02672298863311327, -0.016248366910041084, -0.022544433160856504, 0.012507431511694974, -0.023151452420671303, 0.01785767547427467, -0.003562712164321691, 0.017928258546011606, -0.0061972487396644215, 0.014427308199274161, 0.012479197537942178, 0.030463924962781906, 0.0021969176655852337, -0.011032232460142118, -0.040995010908069576, 0.0073336461597801475, 0.005039675839234098, 0.01000876860673126, 0.0001767901032771081, 0.007072486559179416, -0.0033791945945573585, 0.02600303534093234, -0.008519452568302003, 0.012895642131538237, -0.0055655247528159256, 0.005466707241664928, -0.0038926909117917054, -0.019099947967657065, -0.018690562798821735, 0.003635060557910073, -0.024930162964776615, -0.017984726493517202, 0.009521740941398264, 0.02116099266135518, 0.046444074710106587, 0.011554551661343582, -0.009133530321555002, -0.008780612168902736, 0.015556646660290423, -0.020299871363055322, 0.030774493086127508, 0.033061405844557884, -0.012888583638100038, -0.010636963346860656, -0.014780226351926426, 0.013516778378229434, -0.01718007128007788, -0.00012903582762033237, 0.025579533185220608, -0.005318481673430328, 0.01502021140353509, 0.021386860726087444, 0.0013234443530144736, 0.013566186900974301, 0.01542959657237042, 0.01804119257837774, 0.024661943939415155, -0.009486449405529796, -0.0031445036458579777, 0.022106814018268375, 0.0408538447645957, -0.0017892967708635036, -0.01332620184936564, 0.003513303641576824, -0.0021704487808532507, -0.029503986618992315, 0.0234620205440169, -0.038171665686851444, -0.025946567393426746, 0.029306350665367794, -0.005967851893874322, 0.03314610404052616, 0.007764206808930077, 0.003497422264171508, 0.010220519684587126, -0.00593608913906369, 0.025904216432797552, 0.005925501398906391, -0.003246850170897443, -0.017123605195217342, 0.03325903993553735, 0.016544818046510285, 0.013283851820058971, 0.026807688691726606, -0.01058049633067759, 0.024083158653353153, -0.011491026151722317, 0.030209822924296848, -0.015669581623979082, 0.021965646012149447, -0.0037444651107264733, -0.013707353975770702, -0.0003985773164193534, -0.014229673176972166, -0.008279467516693341, -0.0039773912032356725, 0.00721718334635618, -0.012585073076870111, 0.013192093268007438, 0.017109488208340944, 0.0891048639925604, 0.018944664837306793, -0.01657305202026308, 0.008385343055621275, -0.0003562271124897119, -0.00036130031733693066, 0.011808652768506114, 0.0073336461597801475, -0.012203921881787575, -0.012154512427720181, -0.005971380674932158, -0.006716038694146786, 0.01502021140353509, -0.02916518452189392, 0.005819625859978458, -0.04051504453014236, 0.008258292967701272, 0.008350051519752806, -0.010206402697710726, -0.005703163046554491, 0.029532218730100056, 0.011568667716897452, 0.030407457015276314, 0.017518873377176277, -0.019226998986899595, -0.008342993026314607, 0.02586186733481341, 0.008053600383283606, 0.010354628265945326, -0.03560241877882827, 0.0231655694075477, 0.004834983254816432, -0.040006840453172245, -0.008688853616851202, -0.003949157695144139, -0.01744829030543934, -0.00018473077742785175, -0.019099947967657065, 0.025226614101245815, 0.00032292042865478927, 0.012726241082989039, -0.008886487707830668, -0.006959552526813284, -0.014653175332683897, 0.013269734833182573, -0.021245692719968515, -0.016206017812056946, -0.007707739792747011, -0.006740742955519219], 'source': 'https://arxiv.org/pdf/2303.08774.pdf', 'page': 69}), 0.9281438589096069)\n"]}]},{"cell_type":"markdown","source":["#Question Answering"],"metadata":{"id":"43GA0yPQjw6T"}},{"cell_type":"code","source":["qa_retriever = vector_search.as_retriever(\n","    search_type=\"similarity\",\n","    search_kwargs={\"k\": 20},\n",")"],"metadata":{"id":"aEn15wD-iN5n","executionInfo":{"status":"ok","timestamp":1712775766494,"user_tz":240,"elapsed":361,"user":{"displayName":"John Willis","userId":"15271974867993570949"}}},"execution_count":18,"outputs":[]},{"cell_type":"code","source":["from langchain.prompts import PromptTemplate\n","\n","prompt_template = \"\"\"Use the following pieces of context to answer the question at the end. If you don't know the answer, just say that you don't know, don't try to make up an answer.\n","\n","{context}\n","\n","Question: {question}\n","\"\"\"\n","PROMPT = PromptTemplate(\n","    template=prompt_template, input_variables=[\"context\", \"question\"]\n",")"],"metadata":{"id":"ovtUEpElj8n9","executionInfo":{"status":"ok","timestamp":1712775770240,"user_tz":240,"elapsed":587,"user":{"displayName":"John Willis","userId":"15271974867993570949"}}},"execution_count":19,"outputs":[]},{"cell_type":"markdown","source":["## https://github.com/Arize-ai/phoenix/tree/main/tutorials"],"metadata":{"id":"GUyp708odpt8"}},{"cell_type":"markdown","source":["# Use this example to evaluate your model"],"metadata":{"id":"MbmCqoDzKEtA"}},{"cell_type":"code","source":["query = \"What is an LLM?\"\n","\n","from langchain.chat_models import ChatOpenAI\n","\n","llm = ChatOpenAI(model_name=\"gpt-3.5-turbo-16k\", temperature=0)\n","\n","qa_chain = RetrievalQA.from_chain_type(\n","    llm, retriever=vector_search.as_retriever(), return_source_documents=True\n",")\n","answer = qa_chain({\"query\": query})\n","\n","print(answer)"],"metadata":{"id":"GJYM9rSm1CWm","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1712775818007,"user_tz":240,"elapsed":2576,"user":{"displayName":"John Willis","userId":"15271974867993570949"}},"outputId":"afe5d9db-f852-40ad-c10b-dd94c4e2ce82"},"execution_count":20,"outputs":[{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.10/dist-packages/langchain_core/_api/deprecation.py:117: LangChainDeprecationWarning: The class `langchain_community.chat_models.openai.ChatOpenAI` was deprecated in langchain-community 0.0.10 and will be removed in 0.2.0. An updated version of the class exists in the langchain-openai package and should be used instead. To use it run `pip install -U langchain-openai` and import as `from langchain_openai import ChatOpenAI`.\n","  warn_deprecated(\n","/usr/local/lib/python3.10/dist-packages/langchain_core/_api/deprecation.py:117: LangChainDeprecationWarning: The function `__call__` was deprecated in LangChain 0.1.0 and will be removed in 0.2.0. Use invoke instead.\n","  warn_deprecated(\n"]},{"output_type":"stream","name":"stdout","text":["{'query': 'What is an LLM?', 'result': 'An LLM stands for Large Language Model. It refers to a type of artificial intelligence model that is trained on a large amount of text data and is capable of generating human-like text. LLMs have various applications, including web browsing, voice assistants, and coding assistance tools. They have the potential to significantly impact society in multiple ways.', 'source_documents': [Document(page_content='1 Introduction\\nLarge language models, also known as LLMs, have become an increasingly prevalent part of our\\nday-to-day lives, with their use extending to a wide range of domains including web browsing, voice\\nassistants, and coding assistance tools.[ 1,2,3,4] These models have the potential to signiﬁcantly\\nimpact society in numerous ways.[ 5,6,7] This system card analyzes GPT-4, the latest large language\\nmodel in the GPT family of models.[ 8,9,10] Since it ﬁnished training in August of 2022, we have\\nbeen evaluating, adversarially testing, and iteratively improving the model and the system-level\\nmitigations around it. Our mitigations and processes alter GPT-4’s behavior and prevent certain\\nkinds of misuses, though they have limitations, pointing to the need for anticipatory planning and\\ngovernance[ 11] and further safety research. Our approach to deployment balances minimizing risk\\nfrom deployment, enabling positive use cases, and learning from deployment.', metadata={'_id': ObjectId('6616defa6182e3cd8e9a22a5'), 'embedding': [-0.011843956774537794, 0.008242524067261262, -0.003200518370206373, -0.03750926182581863, 0.003005157631530234, 0.009350134807647823, -0.01687916707656034, 0.030904368822370794, -0.02149987364843186, -0.03816159367189241, 0.030252033251006782, 0.018931304664492454, -0.03495428270062136, -0.0026552071273882073, 0.0015841207621856185, 0.022328882971379013, 0.02238324426899268, 0.007549417988348278, 0.01414751419977993, -0.017613043663022295, 0.019284652167658737, 0.0019043424025913407, -0.01744995977018129, -0.025780820713234124, 0.016090929192484713, 0.011572150286469457, 0.027996042194007247, -0.03846058174009014, -0.013277734369283804, 0.01721892472098448, 0.0024836295728333678, -0.0035504688743483994, -0.027153443012317955, 0.01383493720416262, -0.008772546253333244, -0.015425003762378563, 0.0249925828291585, -0.0020130648814033563, -0.0007725242606590676, -0.012299231943560343, 0.023769454564173537, 0.012387568819351914, 0.026011855762430935, -0.011708053530503627, -0.0011772607455004538, 0.0063398805134280084, 0.009771434398492469, -0.023837407583174455, -0.012387568819351914, 0.013359276315704306, 0.03150234309612114, 0.031040271135082405, -0.006295712075532223, 0.0049570662859488536, -0.02303557797771158, 0.006224362988829604, 0.0003225575428961768, 0.01838769168835578, 0.018537185722454642, -0.015302691308409088, -0.011477018481306818, -0.006010315263060468, -0.015316281167151228, 0.03386705674834801, -0.0005427630860963546, -0.0011381886210482899, -0.024353838979181742, 0.0203446965398027, 0.019298242026400878, -0.002400388894070099, 0.01923029087004507, 0.01227884622412458, 0.01749073120905282, -0.007923152142272882, 0.022274521673765344, -0.004022732298121178, -0.005215281983952496, -0.022750181630901103, -0.02436742883792388, 0.016485048134522524, 0.01339325189388221, -0.0038086848051826815, 0.023986899754628206, 0.0285668348876282, 0.005670557618636324, -0.017857670433606355, 0.022573507879317962, -0.007202865414553065, -0.011443042903128914, 0.0085143305553296, -0.010145167621094516, -0.0019060412513494276, 0.01774894783837902, 0.022328882971379013, -0.0072096603439241345, 0.008310475223617069, -0.01681121592020453, 0.012360388170545081, -0.015846304284545764, -0.024353838979181742, -0.00883370248031798, 0.0015492955850056926, -0.04049912760663501, -0.005575425347812406, -0.025767230854491986, 0.006530144589414567, 0.008317271084310695, 0.0015654340079232605, 0.006261736031693042, 0.010349022021484492, -0.02656905859730975, 0.030578201036688787, -0.004280947996124821, -0.019162339713689264, -0.009044352741401581, -0.0022984614610444725, 0.013141831125249635, -0.00508957159963621, 0.006373856091605912, 0.008595872501750102, 0.031964413194514756, 0.002283172404298288, 0.020032118612862835, 0.0043998634510700385, 0.024163573506211348, -0.0019060412513494276, 0.000611564016077163, -0.018469232703453724, -0.013583516435530046, 0.005928773316639967, -0.0003389083946680803, 0.014731897683465578, 0.01360390215496581, -0.016960708091658283, -0.03269829164362183, 0.028838641375696538, -0.01989621630015122, -0.0077328871349637675, -0.026501107440953943, -0.004776994604018899, 0.010063625674674016, 0.036123047805347544, -0.008500739765264905, 0.001817704142727218, 0.0031427596079071705, 0.01637632553929519, 0.015628859094091094, 0.015112426766761252, 0.007155299046310466, -0.007902766422837118, 0.002325642109851306, -0.02189399259046967, -0.008303680294246, 0.012754507112582893, 0.011103284327382213, -0.0017072829315724347, 0.008860883129124815, 0.029599699542287885, -0.039085737593969876, -0.0236607319689462, -0.022002715185697005, 0.02750679051548424, 0.0006234555499301528, 0.021146524282620465, 0.016457866554393134, 0.017164563423370815, 0.029953047045454168, 0.008344451733117528, -0.005558437558723454, -0.007372744236765137, -0.025359522053712034, 0.030904368822370794, -0.03647639530851383, 0.017110202125757146, -0.010002469447689278, 0.009717073100878802, 0.025780820713234124, -0.0034094694332859284, 0.001230772560527418, -0.01566962867031751, 0.003961576071136441, 0.0036150227988493103, 0.010586852931374928, -0.002826784914773686, -0.027330116763901097, -0.023117120855454636, 0.011300343798401119, 0.006271928891410924, 0.004070298200702498, -0.025536195805295175, 0.02058932144774165, 0.035008642135589914, 0.005099764459354092, -0.02173090869762867, -0.6384184831284765, -0.01074993682421593, 0.020480598852514312, 0.0026263277462386064, 0.0020980042925093927, 0.0016716083882211252, -0.0008459968792054655, 0.008500739765264905, -0.00714170872190705, 0.04204842365730198, -0.014609585229496106, -0.013216578142299068, -0.008004693157370828, -0.018890533225620926, 0.0015552412646206977, -0.021934764029341202, -0.009452061542181532, -0.026052627201302463, 0.009771434398492469, 0.010743140963522304, -0.004454224748683707, 0.026582650318697, -0.010158758411159211, -0.00941808596400363, -0.02742524950038629, 0.012768097902647588, 0.004297936250875051, 0.0028828449447301212, -0.008004693157370828, 0.018645908317681977, -0.011354705096014786, 0.0011781101116718374, -0.009023967021965817, -0.0007143407570665137, 0.04550036326180221, -0.005989930009285982, -0.03171978828657581, 0.01744995977018129, 0.002371509280089859, 0.013733009538306354, -0.03487273982287831, -0.01792562158996216, -0.01111008018807584, 0.026202121235401325, 0.004457622213369241, 0.04229304856524093, 0.01746354962892343, -0.0035402762474611564, -0.03166542512631703, -0.0018720654403408855, -0.006825734261604204, 0.02171731883888653, 0.016009386314741657, -0.01538423325482959, -0.00508957159963621, 0.01471830782472344, 0.028159126086848247, -0.008371632381924362, -0.006472385827115365, -0.017572272224150764, 0.0011925498022466378, 0.009982083728253514, -0.007399924885571971, -0.02231529124999176, -0.026949587680605423, 0.010967382014670601, -0.000808198862218037, 0.011979860018571964, -0.008194957699018665, -0.02699035911947695, -0.003421361025346578, 0.022260929952378092, -0.012197305209026635, -0.009975288798882445, 0.0022933650311855315, 0.018768220771651453, 0.014256236795007265, -0.025712869556878317, 0.007739682529996115, 0.0212552468778478, 0.014351369531492462, -0.014460091195397241, 0.008758955463268549, -0.004433839029247942, 0.03655793818625689, 0.017952801307446437, 0.010546081492503398, 0.010919815646428003, -0.0052696432815661646, 0.01280886841019656, 0.01990980615889336, 0.021431922492076053, -0.034628114914939355, -0.044603402782499246, 0.004518778440353978, 0.002521002848527445, -0.013875708643034149, 0.011626511584083125, 0.011245982500787451, -0.0023783044422915674, 0.0033160361276354166, -0.00898319558309429, 0.03984679576056122, 0.013474793840302712, 0.03593278233237716, 0.013563130716094282, -0.022845314367386297, 0.01841487326848517, -0.00047098922899655387, -1.777092585745166e-05, 0.005959351895793614, 0.003496107576734732, 0.0011662186360265074, 0.000648087983807516, 0.007080552494922313, -0.028430932574916586, -0.013277734369283804, -0.011612921725340985, -0.004668272008791564, -0.026011855762430935, 0.030143310655779448, -0.0018618726970383228, 0.00898319558309429, -0.007902766422837118, 0.01471830782472344, 0.01705584082814348, 0.02124165701910566, -0.02344328677849153, 0.006418024529501697, 0.024462561574409077, -0.012041016245556702, -0.02628366225049927, 0.02455769244824916, -0.02457128416963641, 0.022016305044439147, 0.01966518125095441, 0.05881886813863498, -0.003434951349749995, 0.024190755086340738, 0.002121787476630692, -0.018890533225620926, -0.003271867456908992, 0.0029507963339165666, 0.009023967021965817, -0.035525075394242316, -0.011646897303518889, -0.00687669762887106, 0.0008680811563610179, 0.012360388170545081, -0.015982206597257378, -0.00620397726939384, -0.019597230094598607, -0.002770724884817251, 0.031746968004060086, -0.002582159308342821, -0.02255991802057582, -0.004539164159789742, -0.026650601475052806, -0.0021268836736589936, -0.021377561194462387, 0.010797502261135971, 0.0068970833483068235, -0.009941313220704542, -0.0008689305807400614, 0.01969236096843869, -0.01459599443943141, 0.00023464521922306338, 0.014894981576306582, -0.03606868837037899, -0.03294291655156077, 0.004124659498316166, -0.03693846540690745, 0.011483813410677886, 0.022668640615803155, 0.00033657255038528557, 0.017259696159856012, -0.0025430870674753377, 0.022723001913416824, 0.004046515482242477, -0.004596922922088945, -0.0017293671505203274, 0.01494934287392025, 0.003757721205085188, -0.00806585031567812, 0.021622186102401332, 0.028621196185241865, 0.03761798442104596, 0.027357298344030487, -0.021350379614332997, 0.009696687381443038, 0.010437359828598619, 0.019406964621628213, -0.01932542360653027, 0.005429329709721632, -0.0010787310099910007, 0.010009264377060347, -0.008847293270382675, -0.01195267936976513, 0.014038791604552596, 0.023524829656234588, 0.005738509240653409, 0.0003998524547329507, 0.016294784524197244, -0.02921917045899221, -0.008806521831511148, -0.009710278171507733, 0.008826907550946912, -0.007433900463749874, 0.004060105806645895, 0.009907337642526638, 0.008860883129124815, 0.0004300059433472871, -0.012638989587984489, -0.01704225096940134, 0.0005945761105728919, 0.029518156664544826, -0.00798430836925762, 0.014976523522727084, -0.013549540857352143, -0.004783789533389969, 0.007556213383380626, 0.004549357019507625, 0.028131944506718857, -0.014922162225113415, -0.024448969853021824, 0.010355817882178117, -0.026079808781431853, 0.017572272224150764, 0.005368173017075617, -0.03171978828657581, -0.012074991823734605, 0.019651591392212273, 0.010980971873412739, 0.03354088896266601, 0.003239590611073856, -0.019121568274817737, 0.02919198887886282, -0.007535827663944862, 0.013658263452579478, -0.00025588008655148733, 8.64258892174472e-05, 0.003351710670986726, -0.015805532845674236, -0.0156152683040264, 0.0315838859738642, -0.009452061542181532, 0.02745242921787057, 0.006458795502711948, -0.02478872936009108, 0.004189213655647716, -0.018754630912909315, 0.018211017936772638, -0.013420432542689043, 0.0023749069776060326, 0.011850751703908865, -0.027968860613877856, 0.01485421106875761, 0.009540398417973103, 0.013821347345420482, 0.03030639454862045, -0.0075018520857669585, 0.007447490788153291, 0.0035130955986543227, -0.010260685145692922, 0.02503335426803003, -0.02053496015012798, -0.007107733143729146, -0.009662711803265134, -0.01017234826990135, -0.011368295886079481, -0.0181974280780305, -0.010607238650810692, 0.01017914319927242, -0.03604150492760449, 0.02030392510093117, 0.01724610630111387, 0.0012605014242637225, 0.014297008233878795, 0.01860513687881045, 0.025862363590977183, -0.0077328871349637675, -0.046098335672907435, 0.007515442410170376, -0.013250553720476971, -0.007766863178802949, -0.019189519431173543, -0.036123047805347544, 0.004233382093543501, -0.008344451733117528, -0.007658140583575614, -0.006414627064816163, 0.012971952303037564, 0.006125832787658873, 0.01093340550517014, -0.0013624287408740296, 0.006431614853905115, 0.0183605119708715, -0.00861625728986331, -0.001821101723828072, -0.012034221316185631, -0.0009377315107208698, 0.011905113001522532, -0.02414998364746921, -0.026011855762430935, 0.03688410597193889, -0.013454409052189503, -0.038080054519439584, -0.005079379205579606, -0.007311587544119122, -0.00861625728986331, 0.016675311744847807, 0.02544106306880998, 0.020100071631863753, -0.01637632553929519, 0.008643437938670144, 0.02058932144774165, 0.009112303897757388, 0.00829009043550386, 0.024897451955318416, 0.013841732133533688, 0.014827030419950775, -0.03076846650965918, -0.001953607386761387, 0.006649060044359785, 0.05112675104555891, 0.019855444861279693, -0.01817024649790111, -0.002638219338299256, -0.02222016037615168, -0.010206323848079254, -0.026215711094143467, -0.010851863558749638, 0.009662711803265134, 0.03576970030218126, 0.013671853311321617, -0.01621324164645419, -0.0002650110639321607, 0.009105508968386319, 0.02522361787835531, 0.02364714211020406, 0.01413392434103779, -0.01382814227479155, -0.026949587680605423, -0.016688903466235055, 0.004909499917706255, -0.01249629141457925, -0.0018312944671306348, 0.013896093431147357, 0.006248145707289625, -0.021567824804787666, 0.02455769244824916, 0.012061401964992466, -0.03335062348969561, -0.02609339864017399, -0.009587964786215701, 0.00097595426900165, -0.007800838756980852, 0.011184826273802715, -0.003051025034599426, 0.019203111152560792, -0.014759078332272413, -0.0006111393329914711, 0.00972386803024987, -0.005347787763301131, 0.028159126086848247, 0.012924386866117521, 0.005099764459354092, -0.0019570048514469217, 0.006700023877287918, -0.018020752463802244, 0.009669506732636203, 0.04449468018727191, 0.014894981576306582, -0.03093154853985507, 0.008779341182704313, -0.003829070291787807, -0.03381269731337946, 0.002731652643949768, 0.03223621781993798, -0.003547071409662865, -0.007100937748696799, 0.009649121013200439, -0.018482824424840977, -0.01240115960941661, -0.018510004142325252, -0.048762037393332035, -0.02128242845797719, -0.021975533605567615, -0.010566467211939162, 0.014229056146200432, -0.023293794607037777, 0.0070058054778728805, -0.03294291655156077, 0.008779341182704313, -0.005035210302022542, -0.03427476741177307, -0.03541635279901498, -0.013943659799389954, 0.023755864705431395, 0.0005860822160284161, 0.020072890051734363, -0.0020589322844725482, 0.005837038976162862, 0.008276499645439165, -0.009547194278666729, -0.011660487162261028, 0.004321718969335072, -0.027370888202772625, 0.016933528374174004, -0.0066116865358350685, -0.023089939275325246, -0.013563130716094282, -0.013631082803772645, 0.016512227852006803, 0.004617308175863431, -0.002208425620079495, -0.014419320687848269, -0.004630898500266848, -0.005008029653215709, 0.019148749854947127, 0.02057573158899951, 0.005517666585513203, 0.0038120825026988553, -0.034628114914939355, -0.008636643009299075, -0.015533726357605897, -0.021785269995242336, -0.03343216636743867, -0.0012987241825445025, 0.0014821933291599917, 0.02826784868207558, 0.024870270375189026, 0.006808746006853975, -0.012638989587984489, 0.017191745003500205, -0.01004323995523825, 0.021431922492076053, -0.00887447391918951, 0.01084506862937857, 0.0028794472472139475, 0.026039037342560325, 0.014759078332272413, 0.012251665575317747, -0.03726463319258946, -0.01316901177405647, -0.046152698833166216, 0.031148993730309743, 0.0025702677162821714, -0.00973066295962094, 0.025128487004515226, 0.003937792887015143, -0.006424819924534045, -0.050610321512196736, 0.005391956201196917, -0.012095377543170369, 0.022655048894415906, -0.015601677513961704, -0.00789597149346605, -0.02636520512824233, -0.0018126078292835962, -0.028675557482855534, 0.004926488172456485, -0.03171978828657581, -0.005058993486143842, -0.014663946527109773, 0.0023918947666949843, -0.0048347533663181015, 0.0015365546267736593, 0.027996042194007247, -0.02878428007808287, -0.010369407740920256, 0.0045119835109829086, 0.021350379614332997, 0.032344940415165314, -0.023307384465779915, 0.012353593241174013, -0.02593031474733299, -0.013141831125249635, 0.002089510397964917, -0.019148749854947127, 0.012217689997139843, 0.010668394877795428, 0.02436742883792388, 0.026609830036181278, 0.03310599858175666, -0.025808002293363514, 0.001136489772290203, 0.029164809161378542, -0.010641214228988595, -0.013916479150583121, -0.001059194889557259, -0.01373980539899998, -0.0009037556997123276, -0.010022855167125042, 0.0007627562586498565, -0.0009538699918460975, 0.019298242026400878, 0.002104799454711101, 0.006519952195357963, 0.0011611222061675662, -0.015126016625503391, 0.0021251849413162264, -0.027588333393227294, -0.011742029108681528, -0.015995796455999516, 0.010070420604045085, -0.005259450421848282, -0.002826784914773686, -0.015465774269927535, -0.01197306415787834, 0.009907337642526638, 0.008602667431121171, 0.017626633521764433, 0.009941313220704542, 0.020263155524704757, -0.010410179179791786, -0.007712501881189281, 0.02171731883888653, 0.023171482153068305, -0.02152705336591614, -0.007393129490539624, -0.04672349152678717, 0.0023086540879317156, 0.02364714211020406, -0.004036322622524595, 0.022573507879317962, 0.0021336788358607022, -0.009968493869511375, 0.0030629163938294367, -0.0008901654335165705, 0.005205089589895893, 0.012034221316185631, -0.01284284491969702, -0.015506545708799065, -0.002933808544827615, -0.023144300572938915, -0.01261860479987128, 0.0004646187789843671, -0.0020980042925093927, 0.0006756933157000416, 0.0022899675664999966, 0.023348155904651446, -0.0428366615413776, -0.017164563423370815, 0.013794166696613647, 0.009961698940140306, 0.04077093595734846, 0.004838150831003636, 0.014745488473530273, 0.0018244993049289263, -0.0007058468043143781, -0.00602730305214942, 0.012577833360999751, 0.011925498720958296, 0.001197646232105579, 0.002363015385545383, 0.002019860043605065, 0.0037475283453673055, -0.0050759812752327925, 0.028893002673310204, 0.011409066393628453, -0.010736346034151235, -0.04789225663151336, 0.02654187887982547, 0.0028335800769753944, 0.011150850695624812, -0.030877189104886515, -0.02120088558023413, -0.02124165701910566, -0.0003344490767491666, 0.008629848079928006, 0.030850007524757125, 0.01182357105510203, -0.020127251349348032, 0.019828265143795414, 0.017150973564628677, 0.01995057759776489, -0.007264021641537802, -0.016281192802809992, -0.004124659498316166, 0.03294291655156077, -0.007902766422837118, 0.00018304450718554212, 0.0065709155626248175, -0.012428340258223444, 0.02826784868207558, 0.0121701245602198, -0.0001469452516447108, -0.006910673672710241, 0.02368791354907559, 0.02900172526853754, -0.007264021641537802, -0.011089694468640074, 0.027778597003552574, -0.03432912684674163, -0.005758894494427895, 0.003310939697776475, 0.002724857481748059, 0.005021619977619125, -0.02614775993778766, -0.000528323395521554, -0.011531379778920485, 0.0065743134929716306, 0.0118303659844731, 0.03489991954036258, 0.01744995977018129, -0.02457128416963641, 0.015329871957215923, -0.008779341182704313, -0.035008642135589914, -0.014460091195397241, -0.006112242463255456, 0.019406964621628213, -0.035008642135589914, 0.0036727817939791515, -0.0027435442360104175, -0.013984430306938928, 0.016919938515431866, 0.004015936903088831, -0.02881146165821226, -0.009601555576280396, 0.016688903466235055, -0.021377561194462387, 0.013814551484726855, 0.011008152522219573, -0.0026789903115095066, -0.00367957695618086, 0.02037187811993209, 0.0014813439629886081, -0.008351246662488598, 0.01361069708433688, -0.017857670433606355, -0.030578201036688787, -0.013440818262124808, -0.016729673042461472, -0.0037203479293911107, 0.006360265767202495, 0.007311587544119122, 0.011042128100397475, -0.008106620823227094, 0.004797379857793386, 0.009044352741401581, -9.247782691536504e-05, 0.0225870977380601, -0.00973745788899201, 0.0011755620131576863, 0.010464540477405453, -0.009683097522700898, -0.0021744498090709528, -0.006822336331257391, -0.0036693840964629777, -0.021119344565136186, -0.0451741954761202, -0.0076649359786079615, 0.004026129762806713, 0.0045935254574034105, 0.01240795453878768, -0.007182479695117301, -0.021595004522271942, -0.0016834998638664551, -0.0156152683040264, 0.004600320386774479, -0.004991041864126757, 0.025182848302128892, 0.014663946527109773, 0.014867800927499748, 0.021187295721491993, -0.001321657767663779, -0.030197671953393113, -0.00580306293232368, -0.010736346034151235, -0.0142834174438141, -0.026745734211538003, -0.01637632553929519, -0.023266613026908387, 0.05441560861986324, -0.0004047364848413861, -0.01859154702006831, -0.014514452493010909, -0.004892512128617304, -0.05180627005969742, -0.02083394821832571, -0.000534269133344219, 0.01963799967082502, 0.010580058002003857, 0.03704718800213479, 0.018374101829613642, 0.01186434249397356, -0.006285519215814341, -0.00170133713554211, -0.005711328591846574, -0.0014575608952826285, -0.0018075113994246552, -0.009587964786215701, 0.0004002771378186425, 0.005395353665882451, -0.02390535873953026, 0.003346614241127785, 0.0017752344371741997, -0.026528289021083333, -0.003440047546778297, 0.03416604481654574, -0.0026823877761950415, -0.018727449332779925, 0.007257226246505454, 0.008045464596242356, 0.0005856574747350645, 0.01164010237414782, 0.017327647316211815, -0.013304915018090639, 0.0045119835109829086, 0.017259696159856012, -0.015982206597257378, -0.018278969093128445, -0.011694463671761487, 0.005171113546056711, 0.013590311364901115, -0.0002800878373431588, 0.0026738938816505656, -0.013726214608935285, 0.010117986972287683, -0.003764516367286896, 0.04400543037139401, -0.0009351832957913993, 0.0035130955986543227, 0.024476151433151214, -0.009017172092594748, 0.004827958436947032, -0.004301333715560586, 0.004080491060420381, 0.0005916032707653892, 0.010097601252851918, 0.026609830036181278, -0.03957498740984777, -0.020399057837416367, -0.003163145094512296, -0.0019926793947982312, -0.018088705482803162, -0.02628366225049927, -0.003818877664900564, -0.005860821694622882, -0.033513709245181725, 0.03449221260222774, 0.024856680516446888, -0.010607238650810692, 0.0012214292998115586, -0.0034859149498474885, 0.008663823658105908, 0.0012800374282821445, -0.021771680136500198, 0.005854026765251813, -0.01280886841019656, 0.012231280787204538, 0.015180377923117059, -0.0023035578909034135, -0.00906473752951479, 0.004821163041914684, 0.007753272854399532, -0.01638991539803733, -0.0013293024124521905, 0.19189519803702565, -0.038813929243256426, 0.013447613191495877, 0.025712869556878317, 0.010661599948424359, 0.006591301282060582, 0.028757100360598593, 0.015778351265544846, -0.010457744616711827, -0.009030761951336886, 0.012550652712192918, -0.0033296264520388335, -0.008154187191469691, 0.004372682802263205, 0.01744995977018129, -0.041260185773226356, -0.03288855339130199, -0.03655793818625689, -0.0038970219138048916, 0.007182479695117301, -0.00852792041407174, -0.004433839029247942, -0.011171236415060576, -0.022464785284090624, 0.01164010237414782, -0.011551764567033693, 0.0039955516493143444, 0.0026653999871060897, 0.008344451733117528, 0.01814306678041683, -0.012632194658613418, 0.0010252191949640362, 0.0025549786595359873, 0.013794166696613647, -0.008602667431121171, -0.02457128416963641, 0.01704225096940134, 0.001780330867033141, 0.016063747612355322, 0.0107091653853444, 0.008432788608909099, -0.005938966176357849, 0.007352358517329373, -0.03718309404013662, 0.00024505031862038693, 0.027466019076612707, -0.009295773510034156, -0.014867800927499748, 0.00041853912168381905, 0.006353470837831426, -0.01658018087100772, 0.018319740531999973, 0.02834938969717353, 0.040526311049409514, -0.014745488473530273, -0.005762291959113429, -0.029246350176476487, 0.0170286611106592, -0.004019334833435644, -0.011769209757488363, -0.03364961155789334, -0.026052627201302463, 0.003907214773522774, 0.03386705674834801, -0.00898319558309429, 0.027860139881295633, 0.0032735664220823984, -0.01337966203514007, 0.004073696131049311, -0.012149738840784036, -0.01539782311357173, -0.0037135527671894025, -0.011354705096014786, 0.005310414254776415, -0.032861373673817716, -0.04066221336212113, 0.021853221151598143, 0.035579434829210874, 0.014691127175916606, 0.035035821853074196, -0.00735915391236172, -0.029490976947060547, 0.001681801131523688, -0.019977757315249166, -0.0028947363039601316, -0.03449221260222774, 0.006299109540217758, -0.02231529124999176, -0.02410921220859768, -0.01799357274631797, -0.010009264377060347, -0.011776004686859432, 0.0037203479293911107, 0.00185167995373576, 0.01097417694404167, 0.007658140583575614, 0.013760190187113188, -0.00794353693038609, 0.0029423024393720907, -0.02680009550915167, -0.02651469729969608, 0.06604891140802721, 0.025862363590977183, -0.0032684699922234575, 0.006407832135445093, 0.012455520907030277, -0.006999010548501812, 0.020956260672295182, -0.027914501178909302, -0.01004323995523825, -0.001089773119464947, -0.049658997872634995, 0.020480598852514312, -0.0007716748944876839, -0.020698044042968985, 0.0016257409851519334, 0.007107733143729146, 0.020507780432643702, 0.006071471955706484, -0.0029134230582224894, 0.005425931779374819, -0.0027537368628976606, 0.011884727282086768, 0.00018622974674355043, 0.006397639275727211, -0.036585117903741166, -0.022043484761923422, 0.004800777788140198, -0.04960463843766644, 0.008711390026348506, -0.003061217661486669, -0.04006423722572567, 0.012740917253840755, 0.004576537668314459, -0.01250308634395032, -0.00340267427108422, -0.004491597791547145, -0.00224070269874527, -0.017585863945538016, -0.0028097968928540956, 0.007121323468132564, 0.015873484002030043, -0.011409066393628453, -0.015520135567541202, 0.017871260292348493, -0.011898318072151463, 0.017205334862242343, 0.026881636524249616, 0.0035334810852594477, 0.002967784355836157, -0.025400293492583565, -0.01414751419977993, 0.003947985746733025, -0.0012392665714872134, 0.02788731959877991, 0.01814306678041683, -0.024408200276795407, -0.004484802862176075, 0.03074128492952979, 0.007311587544119122, -0.05137137967878808, 0.008147392262098622, 0.025835182010847793, -0.010063625674674016, -0.006051086236270719, -0.012081786753105674, -0.17286876994927355, -0.007107733143729146, 0.009207436634242585, -0.046152698833166216, 0.02370150340781773, 0.001691993758410931, 0.01526191986953756, 0.0017336140977925654, -0.02593031474733299, -0.0037475283453673055, 0.01947491577798402, -0.02718062272980223, -0.0009156472335653174, -0.0018669691268972638, -0.007800838756980852, 0.002408882788614575, -0.0014065972951851347, -0.0047939823931078505, 0.013943659799389954, -0.003248084505618332, 0.04612551911568194, -0.030877189104886515, -0.011646897303518889, -0.021173705862749855, -0.008358041591859667, 0.000823487977171881, 0.00043382823663766306, 0.0327254713611061, -0.009261797000533697, -0.044141332684105625, -0.01526191986953756, -0.006530144589414567, 0.025495424366423648, -0.009085123248950555, 0.00972386803024987, 0.012985543093102259, 0.03033357612874984, 0.002835278809318162, -0.028213487384461916, 0.0024360634374214086, 0.036123047805347544, 0.01859154702006831, -9.311487337177521e-05, -0.005374968412107965, -0.021173705862749855, -0.005833641511477327, 0.02035828639854484, -0.03215467866748515, 0.001554391898449314, -0.020467008993772174, 0.015329871957215923, -0.0026993755652839927, 0.013413637613317974, -0.006553927773535867, -0.0028013029983096197, 0.02075240534058265, 0.0010039842257722077, -0.004702248052630745, -0.016457866554393134, -0.003693167280584277, -0.00185167995373576, -0.0025804603431694145, -0.0003533481143467108, 0.006448603108655344, 0.006686433552884501, -0.028213487384461916, -0.013957250589454651, 0.022519146581704293, -0.03199159291199903, 0.0063398805134280084, -0.022464785284090624, 0.00555504009403792, -0.011300343798401119, -0.01040338331909816, 0.037916968763953465, 0.022723001913416824, -0.013970840448196789, 0.009961698940140306, 0.035008642135589914, -0.0006574313027310352, -0.00466487454410603, 0.009825795696106136, -0.0003970919273644641, 0.0017854271804767625, -0.01160612586464736, -0.007508647015138028, -0.007026191197308645, 0.009621940364393605, -0.0035368785499449826, -0.009153075336628918, 0.025740051137007707, -0.023837407583174455, 0.0009377315107208698, -0.015832712563158515, 0.007148504116939397, 0.0047158383770341625, 0.014487271844204074, -0.00452897130007186, -0.001043056466639691, 0.010464540477405453, 0.003000061201671293, -0.005130342572846461, -0.008758955463268549, -0.0059253758519544324, 0.020630092886613178, -0.013508769418480613, -0.0038596486381108144, 0.011184826273802715, 0.030904368822370794, -0.021173705862749855, -0.018700269615295646, -0.005504076261109787, -0.0044304415645624075, -0.002845471436205405, -0.013671853311321617, 0.006713614201691334, -0.018713859474037784, -0.02766987440832524, 0.014174694848586765, -0.0008927135902383812, 0.04914256833927282, -0.008942425075545315, -0.005622991250393726, 0.004974054075037805, 0.010960586153976975, -0.03008894935816578, -0.1227476945276645, -0.01749073120905282, 0.008779341182704313, 0.015723989967931177, -0.02173090869762867, 0.03237212385793982, -0.00972386803024987, 0.04995798594083272, -0.019284652167658737, 0.04378798518093934, -0.021336789755590856, -0.007270816570908872, 0.004454224748683707, -0.014174694848586765, 0.01373980539899998, -0.0034655294632423635, 0.020208794227091088, -0.028050403491620912, -0.026419566425855998, 0.031067452715211795, -0.002592351935230064, -0.015221149361988588, -0.001558638845721552, -0.018401281547097918, -0.014378550180299297, -0.0060884597447954355, -0.025821592152105652, 0.028648377765371255, 0.007794043827609783, -0.001408296027527902, 0.029300711474090156, -0.02675932407028014, -0.006431614853905115, -0.005143932897249877, 0.01641709697816672, -0.0076649359786079615, 0.010226709567515018, -0.015873484002030043, 0.035307630203787646, -0.0012817362770402314, -0.0021965340280188455, 0.008242524067261262, 0.009880156993719804, -0.02144551235081819, -0.00985297634491297, -0.01591425544090157, -0.023144300572938915, 0.019733132407310217, 0.007576598637155113, -0.02881146165821226, -0.035035821853074196, -0.02014084120809017, -0.05419816342940857, 0.016702493324977197, 0.01426982758507196, 0.0345465720371963, 0.007950332791079717, 0.026745734211538003, -0.0066048916064639996, 0.0131622168446854, -0.02568568983939404, -0.004508586046297374, -0.030061769640681503, 0.0009037556997123276, 0.0014770970157163702, 0.013841732133533688, 0.009166665195371055, -0.03623177040057488, 0.01704225096940134, -0.021377561194462387, -0.0026161351193513633, 0.015316281167151228, -0.006258338567007507, 0.0018244993049289263, -0.02041264769615851, 0.01481343962988608, -0.011979860018571964, -0.01504447467908289, -0.004223189233825619, 0.00027690259778515047, -0.01700147953052981, -0.0176809948193781, 0.008691004306912742, -0.013766986047806814, 0.0012452123675175382, -0.0023069553555889483, -0.0007466177775246288, -0.008500739765264905, -0.010151962550465585, -0.030387937426363507, 0.0023664130830615567, 0.04210278681756076, 0.007488261761363542, -0.0156152683040264, -0.005109957319071975, 0.01636273568055305, -0.02298121668009791, -0.018469232703453724, 0.02234247283012115, 0.013264144510541667, -0.01361069708433688, 0.009683097522700898, -0.05599208438801449, 0.036639481063999946, 0.008657028728734839, -0.01772176625824963, -0.004515380975668443, -0.012217689997139843, 0.017653815101893823, -0.015316281167151228, -0.00012072019071238592, 0.01721892472098448, -0.0048245605066002195, 0.0023018591585606462, -0.005976339684882565, -0.002797905533624085, -0.01681121592020453, -0.02769705598845463, 0.016770444481333004, 0.0034791197876457803, 0.015343461815958062, 0.027017538836961227, 0.008262909786697028, -0.013501974489109544, 0.01708302240827287, 0.028648377765371255, 0.030143310655779448, -0.006818938866571857, -0.030632562334302456, 0.018550775581196784, -0.015574496865154872, -0.021132934423878324, -0.021826041434113864, -0.02656905859730975, -0.0008247620264289565, -0.003035735977853242, -0.010940201365863766, -0.01205460610429884, 0.012027425455492007, 0.016430686836908858, 0.008922039356109551, 0.04155917384142408, -0.0036591914695757346, -0.041586353558908364, -0.001950209689245213, -0.03579688001966554, 0.010675189807166498, -0.00807264524504919, 0.0036693840964629777, -0.011619716654712056, 0.025413883351325703, -0.0023783044422915674, 0.0353891730815307, 0.00024356386961280573, -0.039738073165333884, -0.04000987779075711, -0.008242524067261262, 0.021798859853984474, 0.02053496015012798, 0.00906473752951479, -0.0010014361272580566, 0.010586852931374928, 0.02144551235081819, 0.005025017907965938, 0.012781687761389727, -0.00020024475423265916, 0.00794353693038609, -0.011674077952325723, -0.011008152522219573, -0.008704595096977437, 0.012149738840784036, -0.006414627064816163, 0.02149987364843186, -0.002597448365089005, 0.014174694848586765, 0.012632194658613418, 0.0032599760976789816, -0.022070666342052812, -0.007372744236765137, 0.007175684765746231, -0.026677781192537085, 0.003645601145172318, 0.013671853311321617, 0.00015745650726154236, 0.008181367840276526, 0.0014813439629886081, 0.019379784904143934, 0.007997898227999758, -0.002009667416717822, 0.013080674898264899, -0.019801083563666024, -0.00789597149346605, -0.001778632018275054, -0.006465590897744296, -0.010580058002003857, 0.013046699320086995, 0.0023409311665974907, 0.008935630146174246, 0.002915121790565257, 0.010124781901658752, 0.030714105212045515, -0.0011662186360265074, 0.01735482889634121, -0.024068442632371262, -0.017436369911439153, 0.0019841855002537554, -0.027370888202772625, -0.004124659498316166, -0.0285668348876282, -0.017382008613825484, 0.029953047045454168, 0.03585124317992432, 0.01641709697816672, 0.005344390298615597, -0.00020873869243287978, 0.008691004306912742, -0.00785520005459452, 0.027615513110711573, -0.004002346578685414, -0.02521002801961317, -0.031094632432696074, 0.03511736473081725, 0.029164809161378542, 0.006635469719956368, 0.05444278833734752, -0.002859061760608822, 0.015900665582159433, -0.0033941803765397444, 0.03237212385793982, -0.03199159291199903, 0.03166542512631703, 0.0273980679202569, -0.0052900290010019285, -0.013501974489109544, -0.02152705336591614, 0.00304253114005495, -0.014677536385851911, -0.007060166775486548, -0.01197306415787834, 0.022260929952378092, 0.015873484002030043, 0.07730169442612107, 0.008575486782314339, -0.023715093266559868, 0.012958362444295424, 0.0059967249386570515, 0.007739682529996115, 0.01566962867031751, 0.014663946527109773, -0.007128118397503633, -0.020195202505703835, -0.00874536560452641, 0.005422534314689285, -0.002718062319546351, -0.02900172526853754, -0.006519952195357963, -0.024625645467250077, -0.00839201717003757, 0.015723989967931177, -0.006074869420392018, -0.010145167621094516, 0.04071657279708968, 0.0058744120190262995, 0.028865822955825928, -0.0013420432542689043, -0.00949962791042413, 0.0051167522484430435, 0.01971954254856808, -0.0008162680736768209, -0.0013097662920184489, -0.006295712075532223, 0.02572645941562046, 0.03294291655156077, -0.038188777114666915, -0.01138188574482162, -0.002184642668788835, 0.005718123521217644, 0.005218679914299309, -0.027615513110711573, 0.0120885826137993, -0.006587903817375048, 0.0017752344371741997, 0.0037339382537945275, -0.006149615971780172, -0.02832220997968925, 0.00735915391236172, -0.013807756555355787, -0.012163328699526176, -0.008548306133507504, -0.024353838979181742], 'source': 'https://arxiv.org/pdf/2303.08774.pdf', 'page': 41}), Document(page_content='[28] Hugo Touvron, Thibaut Lavril, Gautier Izacard, Xavier Martinet, Marie-Anne Lachaux, Timo-\\nthée Lacroix, Baptiste Rozière, Naman Goyal, Eric Hambro, Faisal Azhar, et al. LLaMA: Open\\nand efficient foundation language models. arXiv preprint arXiv:2302.13971 , 2023.\\n[29] Alec Radford, Rafal Józefowicz, and Ilya Sutskever. Learning to generate reviews and discover-\\ning sentiment. arXiv preprint arXiv:1704.01444 , 2017.\\n19', metadata={'_id': ObjectId('6616def76182e3cd8e9a225e'), 'embedding': [-0.012870163283624456, 0.012273487279109633, 0.005356208265996507, -0.01767826215331909, -0.005352739089506647, 0.019107508850433757, -0.012682834272429834, 0.017164842659270724, -0.022437793483478227, -0.02615661284487138, 0.01953766997137057, 0.02468573602987839, -0.017261975875696618, -0.01522217646810769, -0.015388691351685697, -0.007097669671328026, 0.026559020553889316, -0.009088902470704008, -0.006244283919789021, -0.0005255605640250114, 0.0186634710235375, -0.015569081078578049, -0.029389764261489957, -0.011871077707446605, 0.005633731674976031, 0.009192974040109625, 0.033746886019986065, -0.025587688389630346, -0.01627676700547821, -0.012502444545537481, 0.018635719474263714, 0.001764010237608381, -0.010899744611865088, 0.016193510495011755, -0.004953798694333479, 0.005699643699977007, 0.023200984371571056, -0.006459364945918702, 0.025532183428437678, -0.0022358006615162906, 0.009734145083431778, 0.02099467008040412, 0.023534012276081973, -0.017456242308548413, -0.01218329148434091, -0.002780440881328304, 0.02002333791614515, -0.005567820115636329, -0.026905925164359672, 0.03041660138694159, 0.04437604417886753, 0.03616134344996424, -0.018455327884726265, 0.01634614867263032, -0.0035505691910783063, 0.0032817182576621597, 0.003746570445005667, 0.03621684654851181, -0.003600870387536185, -0.0010398467701192506, 0.005227853392484416, 0.00010970860285185536, -0.029362010849571074, 0.025684821606056243, 4.967783244533551e-05, 0.005290296240995532, -0.0073405027123927685, 0.025351793701545323, 0.015707842550237175, -0.011267463349952062, 0.024824497501537517, -0.007874736334057749, 0.002714529089157965, -0.011170330133526165, 0.04021318885322321, 0.0008529518716471361, -0.015027910035255896, -0.034856981518549254, -0.017817023624978213, -0.00012651180074227934, 0.0072503073832853185, -0.002065817299940334, -0.011808635324596762, 0.010337759440926322, 0.009137469078916955, -0.007687407322863128, 0.01864959431757806, 0.005047462734269516, -0.029806048676467328, -0.01896874737877463, 0.0007514822428165732, 0.007576397866139064, 0.02379766037608588, 0.010316944381987159, -0.024824497501537517, 0.005626793787657584, -0.026461887337463418, 0.0076457790676299, 0.013480715062776172, -0.01821943319664124, -0.02031473756542284, 0.0007991816151147157, -0.017830900330937656, -0.013848434732185692, -0.014105143547887328, -0.012349806367918915, 0.028917973022674816, 0.003226213296469491, -0.00044165300089011445, 0.020869785314704434, -0.01371661068218374, 0.019052003889241085, 0.010365511921522656, -0.02969503875408199, -0.010781797267822578, -0.03435743873046033, 0.035634047249956426, -0.02368665045370054, 0.00020424012608524954, -0.0074306980415002185, 0.02268756487752269, 0.0305831162705196, 0.0014119019788185496, -0.030028068521238006, 0.027669116052452505, -0.007375193080307549, -0.02543505021201178, -0.037854238247082804, -0.02303446948799305, -0.005880033892530633, -0.0003540595832538891, 0.013001987333626407, 0.011794758618637323, 0.013376644424693102, -0.007909426236311256, 0.046679491686460364, -0.019038129045926742, -0.01143397823353007, -0.008131445149759385, -0.009269293128918908, 0.002074490008334348, 0.02508814560154142, -0.003892270269644512, 0.014458986511337408, -0.018594089356385388, 0.016068623866666975, -0.018816109201156065, 0.003628622868132519, 0.032581287422875305, 0.004308556081605708, 0.007347441065372489, -0.033080830210964224, -0.013903938762055814, 0.019482166872822997, 0.007957992844524205, -0.01622126204428554, 0.002032861287439846, 0.03582831368280822, -0.018594089356385388, 0.0039061465099426798, -0.0021143839081532827, 0.008186950110952053, 0.0002783909941402846, 0.017456242308548413, 0.016498786850248887, 0.021313823141600692, 0.04296067232506721, 0.0049503295178436185, -0.0023676242458569685, 0.003231417061204281, 0.0028723705658500473, 0.01796966180259678, -0.010316944381987159, -0.0054533414824224045, 0.0010866789065025874, 0.019204642066859654, 0.040768234739859716, -0.013452962582179836, -0.02504651734630819, -0.02164685104611161, 0.006792393316090895, 0.012925668244817124, 0.01382761967324653, 0.004960736581651926, -0.021521966280411926, 0.010060235566285524, -3.7969798349501317e-05, -0.002515058891571381, -0.00010797408009054186, -0.004943391630525172, 0.01237062049553553, 0.030888390763111638, 0.02481062265822317, 0.007215617015370537, -0.645631235000439, -0.02007884287733782, -0.021452584613259818, 0.001927055479035254, 0.00329559426512969, 0.005876565181702047, -0.014389605775507845, -0.006733419644069639, -0.007798416779587192, 0.024921630717963415, -0.0402409422651421, 0.009442745434154088, 0.004336308562202042, -0.016429405183096775, -0.00214387074416391, -0.033386106566201355, 0.008492226466189185, -0.022826326349181813, 0.0050231794301630415, -0.0017327888133528233, -0.014361853294911511, 0.018150051529489133, -0.01826106145187447, -0.011995963404468835, -0.004180201208093617, 0.012100034973874454, 0.018816109201156065, -0.013841496379205972, 0.0069415623172196, 0.002525465955379688, -0.01924627032209288, 0.019204642066859654, 0.008735059507253928, -0.009595382680450106, 0.037881987933711496, -0.002591377980380664, -0.008388154896783568, 0.03752120847992679, 0.0037292252610482764, 0.014153710156100277, -0.03247027750048996, -0.018205556490681802, 0.014653252944189203, 0.016373900221904106, -0.010046358860326083, 0.025809706371755926, 0.0333305997423636, 0.011496420616379911, -0.026531269004615526, -0.021091803296830015, -0.021868870890882287, -0.0066917909231751374, 0.003415276430247768, -0.023117727861104602, 0.0031273454917986636, -0.004232236992796425, 0.01960705163852268, -0.024047431770130342, 0.01253713444779099, 0.0016894257370440283, -0.005043994023440929, 0.013563972504565175, -0.02242391864016388, 0.007167050407157588, -0.0021039768443449755, 0.004353653513328796, 0.013862310506822587, 0.006327541361578024, -0.003923491926730707, -0.00711848379894464, -0.01161436796042242, 0.0017171781012250444, -0.0007389069437021036, -0.0055851650667630825, 0.012502444545537481, 0.025920716294141263, 0.008617112163211416, 0.00040739616362124733, 0.0005728263474695807, 0.011100949397696602, 0.018427576335452476, -0.015014034260619003, -0.0036980038367927185, 0.002856759970137587, 0.02986155363766, 0.011038506083524214, -0.010268377773774212, 0.0016746823190387145, -0.019287900439971202, -0.002022454223631539, 0.008388154896783568, 0.03835378103517173, -0.01558295778453749, -0.028349048567433782, 0.005900848485808521, 0.013022801461243022, -0.0012401841454129814, 0.021216689925174795, 0.020037212759459497, -0.049260462137371445, -0.009824339946877954, -0.008311835807974285, 0.051480653134497824, -0.010289192832713373, 0.026323125865804292, -0.0048636033652260284, 0.01389700134039864, 0.015152795732278127, 0.01336970607171338, -0.01694282467714514, -0.02274306983871536, 0.003951243941665768, -0.008859945204276158, -0.030971649136223186, 0.0037639156289630575, -0.03435743873046033, 0.004100412942794473, -0.006011857942532586, 0.002603519632433901, -0.018829985907115505, -0.0026746349565083937, 0.005238260456292723, 0.00846447398559285, -0.005245198809272444, 0.003968589358453795, 0.02093916511921145, -0.01720647091450395, -0.01694282467714514, -0.014112081900867048, 0.024824497501537517, 0.025532183428437678, 0.007416821801202051, 0.019135262262352636, -0.01586048072785574, 0.008242455072144722, 0.01119114426114278, 0.028362925273393222, 0.006757702948176113, 0.00437446764094541, -0.01592986239500785, -0.038853323823260656, -0.02177173767445639, 0.014819767827767208, -0.010774858914842858, -0.053312307540630426, -0.032803303542355784, -0.007104607558646473, 0.006906871949304819, 0.0025983158676991104, 0.006216531439192686, -0.023339745843230182, 0.01300892475528358, -0.037104922202304325, 0.034912484617096826, 0.012904853185877963, -0.012814658322431787, -0.00569270581265856, -0.024227821497022694, -0.01623513875024498, -0.026586773965808195, -0.0014153710388930913, 0.016457156732370564, 0.007624964474352012, 0.012315115534342861, -0.009616197739389267, -0.002175092168419468, 0.00747232676239472, 0.03752120847992679, -0.00950518781700393, -0.035634047249956426, 0.011919644315659552, -0.03449619833947436, -0.015680091000963386, 0.030166829992897132, -0.01044183007900939, 0.009810464172241061, -0.019842946326607703, 0.00703522682281691, 0.004429972602138079, -0.004083067991667719, 0.0067056671634733046, 0.03913084676657891, 0.008256330846781616, -0.00969251682819855, 0.014542243953126411, 0.004447318018926107, 0.016165757083092872, 0.010455706784968832, 0.009213788167726238, 0.005304172481293699, -0.003691065716643635, -0.001998170919525065, -0.015485824568111593, 0.01964867989375591, -0.011156454358889272, 0.02068939372516699, 0.01429247255908195, 0.0033337538095343316, -0.009116654951300342, 0.003160301504299152, 0.004662398579394515, 0.010622220737224292, 0.026225992649378395, -0.006480179073535316, -0.007347441065372489, -0.020162099387804274, 0.0013546627786269058, -0.013931691242652148, 0.0075625216258408965, 0.004700558123799156, 0.0058939101328288005, -0.010983002053654092, -0.01734523424880817, -0.004856665477907582, -0.0021594815727070077, 0.039241856688964245, 0.00875587363487054, 0.007458450522096553, -0.014035762812057765, -0.0030579647559691014, 0.013799868123972743, 0.005921662613425135, 0.006882588645198345, 0.0002766564641030137, -0.013057491363496529, 0.000465719510569802, -0.008214702591548387, 0.013605600759798402, 0.027058563341978238, -0.013217067894094815, -0.014181462636696611, 0.01423696759788928, 0.014223091823252387, 0.006625879363835435, 0.027807877524111627, -0.0219382506953893, 0.026697783888193535, -0.007187864534774203, 0.007569459978820617, 0.006625879363835435, 0.0085130405938058, 0.018427576335452476, 0.0018854268745560707, -0.011163391780546445, 0.03790974134563038, -0.02712794500913035, 0.013973320429207924, 0.024533097852259825, -0.012426125456728198, -0.009102778245340901, -0.01505566251585223, 0.01727585258165606, 0.001927055479035254, 0.016790186499526575, -0.0025011826512732135, -0.0031221419598945104, 0.004877479605524196, 0.001032041355847702, 0.0031984610487037933, 0.016873443009993033, 0.0016434608947831567, 0.025990097961293375, 0.01767826215331909, -0.0023988456701125264, 0.012530197026133817, -0.027336088147941584, -0.028127030585308202, -0.012093096620894733, -0.01440348155014474, -0.03119366711834877, -0.01999558450422627, -0.008360402416187233, -0.00933173551176875, -0.04681825502076458, 0.022854079761100696, 0.015749472668115497, -0.014160648509079997, 0.02308997444918572, 0.013133810452305812, 0.021341574690874478, -0.011052382789483653, -0.04354347534891278, 0.015360938871089363, 0.006067362438063981, -0.0013165032342222643, -0.009775773338665006, -0.0203008608594634, -0.00986596913343373, -0.02805764891815609, -0.009706392602835444, 0.008346526641550339, 0.012849349156007842, -0.016776309793567135, -0.010774858914842858, -0.011121763525313216, 6.238863535250421e-05, 0.022798574799908027, -0.009907597388666959, -0.01136459656637796, 0.007111545911626193, -0.008311835807974285, -0.001764010237608381, -0.00875587363487054, -0.03321958981997826, 0.03244252408857108, -0.02264593662228946, -0.028293545468886207, -0.00577596278878629, 0.0051654105439733, -0.018330443119026582, 0.017303604130929847, 0.009616197739389267, 0.0021143839081532827, 0.004190608271901924, 0.03432968531854145, 0.011753130363404094, 0.002261818321037058, 0.012759153361239117, 0.024283326458215362, -0.012245734798513298, -0.018399822923533597, -0.004044908447263078, -0.0028619635020417406, 0.011121763525313216, 0.050009778182149925, 0.04343246542652744, -0.03316408672143068, 0.026212115943418955, -0.005526191394741827, -0.006216531439192686, -0.018496956139959494, -0.004190608271901924, -0.00045054244550325566, 0.009852092427474288, -0.030388849837667806, 0.008797502821426316, 0.0268504220658121, 0.004530574995053836, 0.03879781699942289, 0.009130530725937235, -0.016901194559266818, -0.01858021451307104, -0.01964867989375591, -0.018885490868308177, -0.010649973217820626, 0.008339588288570619, 0.017081586148804267, -0.0007154908755104351, 0.015263805654663466, -0.02550443187916389, 0.024172316535830025, 0.016554289948796458, 0.0022548804337186113, -0.024546974558219265, 0.002834211021445406, 0.009810464172241061, 0.009553754425216878, 0.007101138382156613, -0.006136743639554816, 0.028335173724119436, 0.02578195482248214, 0.013432148454563224, 0.0334971164885867, -0.0016347883028044613, 0.026642278927000867, 0.03199848812431992, 0.0060153271190224455, 0.01773376711451176, -0.017178719365230164, 0.004454255906244553, -0.015236053174067132, 0.0256709449000968, 0.006636286427643742, -0.030194581542170917, 0.00382288953381495, -0.00886688355725588, -0.033358353154282476, -0.022784698093948588, 0.009872906555090903, 0.013203192119457922, -0.010525087520798395, 0.008138383502739105, -0.007111545911626193, -0.04465356712218578, 0.0013928222066162288, -0.03049985789740805, -0.008360402416187233, -0.013744363162780075, -0.011968210923872501, -0.01171843952982804, -0.03396890586475674, 0.0017952316618639389, -0.041267777527948635, 0.004599955730883399, -0.007264183623583486, -0.02456085126417871, -0.04093475148608281, 0.019482166872822997, 0.018691224435456382, 0.009047273284148232, 0.02216027054015998, -0.00028381137867888396, -0.017817023624978213, 0.0030492920475750874, -0.014805891121807769, 0.013154624579922426, 0.00702828846983719, -0.0190242523399673, 0.02504651734630819, -0.012495506192557761, -0.03041660138694159, -0.007860859628098307, 0.0005836671165467793, 0.0130158631082633, -0.00382288953381495, -0.020453499037081966, -0.015097290771085459, -0.0017163108071025792, -0.0016946292689481819, 0.024519221146300386, 0.03169321176908279, -0.001231511553434286, -0.011524173096976245, -0.025934593000100706, 0.007160112519839142, -0.020217604348996946, -0.03896433374564599, -0.0022999778654416996, 0.0007783673710827834, 0.005626793787657584, 0.011350720791741065, -0.010337759440926322, -0.006358762553002945, 0.0018403292100023455, 0.0274054679524486, 0.008166135983335439, 0.014306348333718842, -0.007007474342220576, 0.006754233771686253, 0.003632091811791743, 0.00868649289904098, 0.030832887664564063, 0.021521966280411926, -0.032581287422875305, -0.010233687871520703, -0.00399634183905013, 0.01154498722459286, 0.012391434623152144, 0.00028793088024144494, 0.04001892242037142, -0.0006409063749460823, 0.0003054929168332814, -0.015069539221811671, 0.019398908499711445, -0.012460815358981707, 0.03321958981997826, 0.010892806258885368, -0.002700652848859798, -0.05322905103016397, -0.0111980826141225, -0.017997413351870568, -0.010649973217820626, -0.04476457704457112, -0.011239710869355728, 0.0023780315424959124, 0.005626793787657584, -0.010192058684964929, -0.007687407322863128, 0.0028931849262972984, -0.028917973022674816, -0.009935349869263293, 0.013161562932902146, 0.01864959431757806, 0.022798574799908027, -0.028446183646504773, 0.023173230959652173, -0.03593932360519356, -0.010740168081266802, -0.007909426236311256, -0.012384496270172424, -0.0004566132715298742, 0.008853006851296438, 0.019829071483293357, 0.020758775392319097, 0.042211360005578916, -0.030527611309326932, 0.02002333791614515, 0.027211201519596804, 0.0063067267683001365, 0.012051468365661505, -0.00892932594010572, 0.003009397914925516, -0.017234224326422833, 0.005210507975696389, 0.013085243844092863, 0.0035523037793232363, 0.004957267870823339, -0.040768234739859716, 0.008630987937848309, -0.001986029267471828, -0.0006304992529301162, -0.0027214672093070487, -0.006969314797815935, -0.02714182171508979, -0.008256330846781616, -0.01126052592829489, 0.0008351729824208538, -0.015138919957641234, -0.01878835765188228, -0.0005476757492406414, 0.015485824568111593, 0.014528367247166971, 0.005880033892530633, -0.004780346389098299, 0.022881831310374485, -0.022951212977526593, -0.0009140937789745542, 0.013570910857544895, 0.021480336162533604, -0.038076258091853485, -0.009907597388666959, -0.03224825765571929, 0.007104607558646473, 0.02013434783853049, -0.023423002353696636, 0.00338405500599221, 0.02379766037608588, -0.022965089683486033, 0.01762275719212642, -0.0022756945613352252, 0.029028982945060153, -0.013792929770993024, -0.005703112876466867, -0.01769213885927853, -0.012779968420178279, -0.03208174463478638, -0.018122299980215344, 0.010747106434246522, -0.002643413532252836, -0.0038055441170269226, -0.011642120441018755, 0.014986281780022669, -0.04201709357272712, -0.030333344876475137, 0.02070327043112643, 0.03566180066187531, 0.042932922638438516, -0.005130720176058519, 0.009470496983427874, 0.00312040737164958, 0.019565423383289454, -0.0039338989905390144, 0.04026869567706098, -0.006795862492580755, -0.019065880595200528, 0.006584250642940934, 0.0085130405938058, -0.017928033547363553, 0.006612003123537268, 0.024644107774645162, 0.05089785383594244, -0.015347062165129922, -0.010337759440926322, 0.021577471241604595, 0.016471033438330004, 0.015791100923348726, -0.01787252858617088, -0.009935349869263293, -0.008825254370700104, 0.02199375565658197, -0.015305433909896693, 0.019398908499711445, -0.01143397823353007, -0.010483459265565166, -0.012294301406726247, 0.024005803514897114, 0.02615661284487138, -0.0004683213056257084, 0.01418840098967633, -0.010538964226757836, 0.044875586966956454, -0.021660727752071052, -0.01622126204428554, 0.011975149276852223, -0.0016278501826553777, 0.03929735978751182, -0.017470119014507856, 0.009081964117724287, -0.0009574568552833492, 0.021660727752071052, 0.0009435806731928412, 0.005099498518972324, -0.005432526889144517, 0.023561765688000856, -0.026933678576278555, 0.010421015951392778, 0.0065599673388344595, -0.0006716941521404076, 0.010074111340922417, -0.02071714713708587, 0.0156523394516896, 0.008485288113209465, -0.01021287374390409, 0.0037604466853038343, 0.022548803405863564, -0.0029365480026060935, -0.0018576745103750546, 0.02526853532843377, -0.004603424907373259, -0.031804221691468126, 0.005827998573489098, -0.029722792166000874, -0.013265634502307763, -0.02068939372516699, 0.004207953688689951, -0.011336844085781624, -0.02000946121018571, -0.005286827064505672, 0.012988110627666966, -0.018455327884726265, 0.005175817607781607, 0.009130530725937235, -0.01622126204428554, 0.0052035700883779415, 0.020772652098278537, -0.03174871486763036, -0.016734681538333906, -0.0018975685266093077, 0.0062373460324705735, 0.010941372867098316, 0.026212115943418955, -0.014167586862059718, -0.018399822923533597, -0.0013234413543713479, 0.010289192832713373, 0.0029729729587658045, -0.0050231794301630415, -0.009179097334150184, 0.003704941956941802, 0.005033586959632622, 0.000508648968921194, 0.0047144343640973235, -0.0021698886365153144, 0.01959317493256324, -0.019398908499711445, -0.008943202646065162, 0.025296288740352654, -0.0169289479711857, 0.01429247255908195, 0.0039477752308371816, 0.007729036043757629, -0.015610710265133824, -0.035578544151408854, 0.010046358860326083, 0.0162073853383261, 0.007909426236311256, 0.00045531238855383584, -0.011933521021618994, -0.030971649136223186, 0.02099467008040412, -0.015569081078578049, 0.015125043251681793, -0.0033528335817366523, -0.005855750588424159, 0.025809706371755926, 0.02714182171508979, -0.0027422813369236626, -0.007458450522096553, 0.0003219708939796958, 0.0021334636803556034, 0.019343403538518777, 0.01691507126522626, -0.013022801461243022, -0.02195212740134874, -0.01895487067281519, 0.03391339904091897, 0.008214702591548387, -0.027044688498663892, -0.004145510840178835, -0.003326815689385248, -0.03577281058426065, 0.0008585890506125222, -0.02274306983871536, 0.013924753820994975, -0.00010081916551467159, 0.01694282467714514, -0.013036677235879915, 0.02135545139683392, -0.012675896850772661, 0.008547731427381853, -0.001087546084209734, -0.016831814754759804, 0.0005550474582432983, -0.005873096005212187, 0.019163013811626425, -0.004808098869694633, -0.032886563778112436, -0.018760604239963397, 0.01878835765188228, 0.007708221916141016, -0.010622220737224292, 0.0025566873796352457, -0.01189883018804294, -0.010171244557348314, -0.002612192107997278, -0.0010962187926037478, -0.019149137105666982, 0.0068132079093687825, 0.024588602813452494, -0.012842410803028122, -0.01773376711451176, 0.009949225643900186, -0.005314579545102006, -0.0039477752308371816, -0.010060235566285524, 0.016401651771177892, -0.005279889177187224, 0.032886563778112436, 0.0031100003078412732, -0.022840203055141256, 0.026239869355337838, -0.021924375852074955, 0.03316408672143068, -0.006282443464193662, -0.008894636037852214, 0.018108423274255905, 0.01114257765292983, 0.0006560834400126288, 0.0057794319652761505, -0.003430887258790865, -0.0031984610487037933, 0.0023762969542509824, 0.013661105720991072, -0.012856286577665015, -0.003614746627834352, -0.016956699520459487, -0.012023715885065171, 0.004211422865179812, -0.015180548212874461, -0.01049733504020206, -0.014486738991933742, 0.0011491217550137032, 0.03158220184669745, 0.026531269004615526, -0.018705099278770728, 0.015208300693470796, -0.011982087629831942, 0.008658740418444645, -0.0014864864793829023, -0.015277681429300359, -0.012828535028391227, -0.004749124732012104, -0.006483648250025177, -0.011475606488763297, -0.026933678576278555, 0.022077012167048427, 0.0021334636803556034, 0.005279889177187224, -0.022798574799908027, -0.0034811884552487437, 0.17783720275733594, -0.03502349453948216, -0.0012280424933597443, 0.01522217646810769, -0.00014797651536606053, -0.006917279013113126, 0.03247027750048996, 0.00560251048355111, -0.015416443832282032, 0.012315115534342861, 0.007853921275118587, -0.015680091000963386, -0.016956699520459487, -0.005505377267125213, 0.01120502096710222, -0.01044183007900939, -0.0442372845698535, -0.031054905646689644, -0.0044195655383297726, 0.01831656641306714, 0.001176874119194719, 0.0010259705298210835, 0.000993014633735914, -0.04468132053410466, 0.006695260099664997, -0.003892270269644512, -0.0001270538304649904, -0.012557948575407603, 0.001878488754406987, 0.005290296240995532, -0.01827493815783391, -0.007312750231796433, 0.010178182910328034, 0.008159197630355719, -0.023520135570122534, -0.008964016773681777, 0.006497524490323344, -0.009637011867005881, 0.014542243953126411, 0.007208679128052091, -0.00221672088931397, 0.0003809446824162697, 0.010608344962587397, -0.01505566251585223, 0.0032366205931084347, 0.016138005533819087, -0.016956699520459487, -0.003031946863617697, -0.02363114549250787, 0.010282254479733651, -0.021022423492323, 0.033996659276675625, 0.042683149381748955, 0.01591598568904841, -0.00747232676239472, 0.006195717311576072, -0.014958529299426334, 0.019759689816141245, 0.0004982418469052279, -0.02308997444918572, -0.009033397509511338, 0.012273487279109633, -0.0001799567928749457, 0.021313823141600692, 0.0011257056286143755, -0.0007770664298990857, -0.026531269004615526, -0.02270144158348213, 0.015458072087515259, -0.0400466758322903, -0.014105143547887328, -0.00781923137286508, -0.008304897454994565, 0.016845691460719247, -0.031471191924312115, -0.038131761190401056, 0.024186193241789465, 0.017039957893571038, -0.01382761967324653, 0.041101264507015726, -0.0012237062555780559, -0.00799962203107998, 0.01354315837694856, -0.01604087231739319, -0.030250086503363586, -0.030361096425748923, -0.005134188886887106, 0.014174524283716891, -0.01769213885927853, -0.023964175259663885, -0.007402945560903884, -0.020397994075889297, 0.0013737424344139081, 0.011302154183528116, 0.006351824665684498, -0.003370178765694043, 0.01300892475528358, 0.0017067709210014188, 0.006001450878724278, 0.012224920670896686, -0.025657070056782454, 0.04012993234275676, 0.038603550566571096, 0.008610173810231696, -0.006674445972048384, -0.007402945560903884, -0.01218329148434091, 0.012238796445533578, 0.014347977520274618, 0.004846258414099275, 0.0038645180218788147, -0.025226907073200546, -0.005043994023440929, -0.005914724726106688, -0.016790186499526575, 0.0156523394516896, -0.009741083436411498, -0.0008581554035512896, 0.01558295778453749, -0.0037708537491121415, -0.0068132079093687825, -0.01586048072785574, -0.010559778354374449, 0.0005212242680356637, 0.012162477356724296, -0.007957992844524205, -0.04318269216983788, -0.0035106752912593716, -0.02310385115514516, -0.01365416736801135, 0.013106057971709478, -9.285119907877733e-05, 0.020953041825170892, -0.01691507126522626, 0.0015185751977609253, -0.014805891121807769, -0.014986281780022669, -0.019884576444486025, -0.005706581587295453, 0.003099593244032966, -0.014500614766570637, -0.004235706169286285, 0.0006547825570365904, -0.01660979490998913, -0.0010780061981085737, -0.0193850336563971, 0.005137658063376966, 0.0099700397715168, 0.004603424907373259, -0.0064558957694288415, -0.009345612217728191, 0.0023884386063042196, 0.01184332522685027, -0.021105680002789458, 0.04198934016080824, 0.011177268486505885, -0.019482166872822997, -0.004336308562202042, 0.007451512169116832, 0.022007632362541413, -0.03316408672143068, 0.001064130074225725, 0.030000315109319123, -0.0025133243033264507, -0.002105711432589906, -0.005009303189864874, -0.17850326974222835, 0.0070005359892408555, 0.005810653156701071, -0.026170487688185726, 0.028390678685312105, 0.02605947776580039, 0.02510202230750086, -0.009401116247598313, -0.026170487688185726, -0.01540256712632259, 0.010303068607350266, 0.006667507619068663, -0.021966004107308184, 0.00542905817831593, -0.036744142748519615, 0.0015593365081176434, -0.00452016793124553, -0.015596833559174385, 0.03521776097233396, 0.01598536735620052, 0.008069002766909543, -0.0406017217189268, 0.011815572746253935, 0.005009303189864874, -0.012620391889579993, -0.003172443156352389, 0.0014405216371220306, 0.037132675614223204, -0.026947555282238, -0.024949384129882294, -0.019620928344482123, -0.006500993666813204, 0.02643413578818963, 0.013217067894094815, -0.0034551705628973394, 0.030888390763111638, 0.008270207552741056, -0.0003755242978776703, -0.020411870781848737, 0.010670787345437239, 0.020203727643037503, 0.038131761190401056, -0.012981172274687246, -0.010046358860326083, -0.014861396083000437, 0.013279510276944658, 0.04795610020595646, -0.029667287204808206, 0.0035453656591741527, -0.01558295778453749, 0.0015506639161389482, -0.012211043964937244, 0.02164685104611161, -0.0004145510781971176, -0.016554289948796458, 0.021133431552063243, -0.01248162948659832, 0.020592260508741092, 0.010649973217820626, -0.020370242526615512, -0.00863792629082803, 0.0031256111363843703, 0.0020276577555356926, 0.006837491213475257, -0.003075309939926492, -0.026670030476274652, -0.02601784951056716, -0.009414992953557752, -0.04651297866552745, 0.013112996324689197, -0.011045444436503933, 0.013293386982904097, -0.021660727752071052, -0.007604150346735398, -0.008325711582611178, 0.014694882130744978, -0.013869248859802306, 0.0023103851620806435, 0.03721593212468966, -0.013785991418013304, -0.012322053887322581, 0.01014349207675198, -0.0032799836694172298, 0.014514491472530077, -0.02526853532843377, -0.004204484512200091, -0.0022045792372607327, -0.01137153491935768, -0.02433883141940803, -0.02164685104611161, 0.027349964853901027, -0.020453499037081966, 0.005266012936889057, 0.006636286427643742, -0.011933521021618994, -0.0006699596221031367, -0.012037591659702064, -0.005904317662298381, 0.005179286784271467, -0.0029521588311491906, -0.0032331514166185744, -0.02035636582065607, -0.015152795732278127, -0.0015948941701548897, 0.030028068521238006, 0.0021161184963982126, -0.0440707678236304, 0.006157557767171431, 0.046540732077446335, -0.020175976093763717, 0.0015905579323732013, 0.0007935444361493297, 0.0036841275964945514, 0.010726292306629909, -0.016179633789052312, 0.025532183428437678, -0.029251000927185734, -0.02339525080442285, 0.020509003998274634, 0.010740168081266802, 0.044181777746015734, -0.018427576335452476, 0.019842946326607703, 0.012488567839578042, -0.0156523394516896, -0.029306505888378406, -0.09768835171949795, -0.011961272570892781, 0.010941372867098316, 0.016762433087607696, -0.02105017504159679, 0.04093475148608281, -0.01190576854102266, 0.020980795237089775, -0.012002901757448557, 0.032886563778112436, -0.012599577761963379, -0.020633888763974317, 0.006077769967533561, 0.009317859737131857, 0.012335929661959476, 0.016484910144289443, -0.002485572055560753, -0.017747641957826105, -0.012141663229107681, 0.03341385625283005, 0.010622220737224292, -0.00828408332737795, 0.007874736334057749, -0.010240626224500424, -0.014445110736700515, -0.004031032206964911, -0.03591157019327468, 0.013001987333626407, 0.009942288222243013, -0.002534138663773702, 0.022174147246119418, -0.007583336219118784, 0.010476520912585446, -0.022090888873007867, -0.004742186844693658, -0.01715096595331128, 0.009782711691644727, -0.013022801461243022, 0.030305591464556254, -0.00329559426512969, 0.0061471507033631235, -0.007125421686263087, 0.020522880704234078, -0.013196253766478202, -0.0071809266474557555, -0.017095462854763707, -0.01724809916973718, 0.028251915351007885, 0.018940993966855748, -0.03396890586475674, -0.02743322136436748, -0.025351793701545323, -0.019981709660911923, -0.01599924406215996, 0.02969503875408199, -0.015569081078578049, 0.00560251048355111, 0.025226907073200546, -0.008478349760229743, -0.013626415818737565, 0.007187864534774203, -0.003972058534943656, -0.030250086503363586, -0.0066189410108557145, 0.02242391864016388, 0.006379576680619559, 0.022840203055141256, -0.04012993234275676, 0.015513577048707929, -0.007500078777329781, -0.0014639376471060398, 0.016762433087607696, -0.012280425632089354, 0.004013686790176884, -0.0264896407493823, 0.010316944381987159, -0.027641364503178716, -0.030777382703371395, 0.012322053887322581, -0.003965120181963935, 0.004044908447263078, -0.030943895724304307, -0.011045444436503933, -0.002341606586336201, 0.012037591659702064, 0.024519221146300386, -0.006643224314962189, 0.008450597279633409, -0.010351635215563215, -0.025934593000100706, 0.003534958595365846, 0.026933678576278555, 0.019634805050441562, -0.020397994075889297, -0.018413699629493036, 0.004485477563330748, -0.002922671762307926, -0.013494591768735612, 0.006889526532516791, 0.02034248911469663, -0.04015768575467564, 0.003505471759355218, -0.08031537150935128, 0.030694124330259843, 0.005179286784271467, -0.020148222681844834, 0.002006843395088442, -0.030028068521238006, 0.01114951600590955, -0.004003279726368577, 0.005939008030213163, 0.016706928126415024, -0.027016935086745013, 0.02343687905965608, -0.01919076536090021, 0.001405831152791931, 0.0016235138284583707, -0.012100034973874454, 0.023423002353696636, 0.009137469078916955, 0.03166545835716391, 0.019620928344482123, 0.026448012494149072, 0.015416443832282032, 0.0054533414824224045, 0.030028068521238006, 0.0004501087984420231, -0.006497524490323344, -0.01183638687387055, 0.010643034864840905, 0.0008824387658654231, 0.0033927277143862237, -0.014264720078485615, -0.021674604458030492, -0.000760588481856501, 0.02374215541489321, -0.013598663338141229, -0.032636790521422876, 0.00702481929334733, 0.010351635215563215, 0.013265634502307763, 0.03505124795140105, -0.01966255659971535, -0.030943895724304307, 0.021313823141600692, -0.004461194259224274, 0.001208095543450277, 0.0008681289367136826, -0.00027340422755908815, -0.018108423274255905, 0.03122142053026765, -0.006136743639554816, 0.021299946435641252, 0.008859945204276158, -0.005869626828722326, -0.058890534720075056, 0.00857548297665564, -0.0006903402772814958, 0.0083326499355909, 0.01505566251585223, 0.017109337698078052, 0.018774480945922836, 0.03330284633044471, 0.00507174603837599, 0.030250086503363586, 0.0018195149659704132, -0.002504651594932437, -0.01358478663218179, 0.0019339934827690192, -0.0052035700883779415, -0.002659024360795933, -0.016873443009993033, -0.0078261697258448, 0.0050578702637390964, 0.007576397866139064, 0.04690151153123104, 0.012294301406726247, -0.0022566147891329045, -0.014278595853122508, 0.016498786850248887, -0.03119366711834877, 0.010414078529735605, -0.0037604466853038343, -0.021258318180408024, 0.010809549748418912, 0.02450534630298604, 0.024144564986556236, 0.0017735501237095413, -0.02619824110010461, -0.013168501285881868, -0.020772652098278537, -0.004499353337967642, -0.004239174880114872, 0.0028307420777861827, -0.004357122689818657, 0.012349806367918915, -0.01627676700547821, 0.005196631735398221, -0.0016720805530866377, -0.004266927360711207, 0.028168658840541427, 0.02297896452680038, 0.024519221146300386, 0.0016651424329375542, -0.014500614766570637, -0.03527326779617172, -0.008839131076659545, 0.01253713444779099, -0.026586773965808195, -0.017664385447359647, 0.012807719969452066, 0.010185121263307756, 0.012044530012681784, -0.006095114918660315, -0.004756063084991825, 0.012072282493278118, -0.012502444545537481, 0.0058210602205093775, -0.013557034151585455, -0.0022791637378250856, -0.028029897368882305, 0.01991232799375981, 0.03890882692180823, -0.0016980983290227237, 0.05167491956734962, 0.018455327884726265, 0.01964867989375591, 0.007666593195246514, 0.02889022147340103, -0.03843703754563819, 0.0260317262165266, -0.004530574995053836, 0.0015610709799472552, 0.006667507619068663, -0.026406384238915843, -0.008957078420702055, -0.01626289029951877, 0.0006539153211217846, -0.0010649973683481899, 0.011496420616379911, 0.008672617124404085, 0.0910277861786992, 0.0029382825908510234, -0.005432526889144517, 0.023020594644678705, -0.004547920411841864, 0.003240089536767658, 0.0029382825908510234, 0.012953420725413459, 0.011579678058168914, -0.010601406609607678, -0.023131602704418944, 0.01790028013544467, -0.015152795732278127, -0.04212810349511246, -0.011052382789483653, -0.008922388518448548, -0.004658929868565928, 0.008582421329635362, 0.005099498518972324, 0.003099593244032966, 0.010705478179013295, 0.01505566251585223, -0.005345800736526927, 0.0017353905793049, -0.015971490650241078, 0.0203008608594634, -0.000884173295902694, 0.01660979490998913, -0.007708221916141016, -0.03907533994274114, 0.0302223349540898, 0.00933867386474847, -0.025615439938904132, -0.006251222272768741, -0.005873096005212187, -0.002979911078914888, -0.0027960517098714016, -0.013508467543372506, 0.007673531082564961, 0.009227663942363133, -0.006827083684005675, 0.010629159090204012, -0.010830363876035526, -0.02988930704957888, 0.009609259386409547, -0.0034967990509612045, 0.002208048180919956, 0.007243369495966871, -0.029084487906252822], 'source': 'https://arxiv.org/pdf/2303.08774.pdf', 'page': 18}), Document(page_content='GPT-4 System Card\\nOpenAI\\nAbstract\\nLarge language models (LLMs) are being deployed in many domains of our lives ranging\\nfrom browsing, to voice assistants, to coding assistance tools, and have potential for vast societal\\nimpacts.[ 1,2,3,4,5,6,7] This system card analyzes GPT-4, the latest LLM in the GPT family\\nof models.[ 8,9,10] First, we highlight safety challenges presented by the model’s limitations\\n(e.g., producing convincing text that is subtly false) and capabilities (e.g., increased adeptness\\nat providing illicit advice, performance in dual-use capabilities, and risky emergent behaviors).\\nSecond, we give a high-level overview of the safety processes OpenAI adopted to prepare GPT-4\\nfor deployment. This spans our work across measurements, model-level changes, product- and\\nsystem-level interventions (such as monitoring and policies), and external expert engagement.\\nFinally, we demonstrate that while our mitigations and processes alter GPT-4’s behavior and', metadata={'_id': ObjectId('6616defa6182e3cd8e9a22a3'), 'embedding': [-0.0063128476399484535, -0.0010789729116658334, -0.02278516622951611, -0.03071818559006837, 0.002800191509844157, 0.014098578323601221, -0.013804001901124888, 0.024470418776814423, -0.024963664015399022, -0.032060908843028965, 0.026484502724932666, 0.025511713866571847, -0.04499487937024264, -0.0031872515309704333, 0.009460707948167443, 0.03836347840622584, 0.01951056874128755, -0.00048510961137879693, 0.00422340804801339, -0.006405330772937073, 0.002890962173312391, -0.0003752855999164838, -0.021716469764787225, -0.025470610872792003, 0.02012712482385698, 0.00037464336563867377, 0.03381466762172388, -0.0343627174728967, -0.010502002106602218, 0.020168229680282118, 0.011173363733082515, -0.018277458438793998, -0.018935117515143266, -0.0006242628939031466, -0.007720650322619584, -0.008727691831017381, 0.020812187137823006, -0.007056139971204801, 0.018894014521363425, 0.008419413789732669, 0.01897622050892311, 0.02085329199424815, 0.03195129961785252, -0.0034647018612589398, 0.016605905647658768, 0.011296674577067341, 0.0009847768434955045, -0.011358329999059754, -0.017537591512239745, 0.01812674435719241, 0.01220095627270891, 0.031047015128243004, 0.0027162714229494508, -0.0018085640897751245, -0.020579267068661738, -0.006973932586661142, -0.012536637551610384, 0.018907714277526508, 0.027073655569885332, -0.004973551188673927, -0.006059374676913288, -0.010282782724926678, -0.01127612308017742, 0.0460087730850286, -0.0015294011735509014, 0.0015131309668772745, -0.027882028727836187, 0.03940477535862856, 0.015180977338461136, -0.009042819750383636, 0.01907212997793647, 0.013194296627959652, 0.0017203622470027995, -0.0005656044099867392, 0.026374891637110924, -0.003182113656747953, -0.012605142851684337, -0.021702768145978844, -0.03551361992518527, 0.016468893184865562, 0.0048159867633294755, 0.000593435066491609, 0.01783901781279762, 0.01997641260490069, 0.01501656163805117, -0.002183635660105393, 0.014756238331273139, -0.0021630836975541474, -0.002688868883824677, 0.002270980918871532, 0.0009162706120989016, -0.004343293952957445, 0.018387067663970445, 0.015934544952501118, -0.010255380418632566, 0.019044726740319713, -0.008734542640421571, 0.018715897202145077, -0.013995819907828966, -0.039706199796541136, -0.003163274396547755, 0.005874407945274725, -0.029676892362956245, -0.01175566669995364, -0.021826078989963672, 0.00310846955112887, 0.005723694329334464, 0.009515513492078315, 0.020538162212236595, 0.020209332674061962, -0.01618116850311607, 0.03515738901203918, 0.0026392019126282723, -0.009762135180047966, -0.010467748990903917, 0.000138189872899969, 0.018277458438793998, -0.011214466726862358, -0.01148849165244877, 0.009542915798372426, 0.027717614890071518, 0.003555472524727174, 0.013804001901124888, 0.00369248498752038, 0.018578884739351872, -0.0018753576188206798, -0.0014471937890072428, -0.017647200737416192, -0.007179451280850951, -0.000647383735357967, 0.016647010504083907, -0.0028841115967388627, 0.005559279094585822, -0.010077264030736869, -0.04107632442447319, 0.017962328656782448, -0.023209905236704107, 0.001420647601057993, -0.025662427948173434, -0.009481260376380014, 0.013194296627959652, 0.027525795952044792, -0.0026220753547791216, 0.006429308140190414, 0.006593722909277731, 0.010495152228520677, 0.017195060355256732, 0.020990304457041354, -0.005227024151709093, 0.000426451185670055, -0.0026511903634242806, -0.017647200737416192, -0.023059191155102524, -0.001641580150745905, -0.0015902004771984528, 0.0009702192227575939, 0.011598101808947865, 0.02229192285357681, -0.040939311961679985, -0.032307530530998614, -0.01501656163805117, 0.01730466958043318, -0.009440156451277522, 0.013310757593862937, 0.00959086960155646, 0.024662237714841148, 0.021195823151231162, 0.003747289832939265, -0.009953953186752044, 0.0034629891589078926, -0.021004004213204436, 0.032581555456585025, -0.0244156141642262, 0.02040114974944339, -0.0019404384455151877, 0.007001334892955254, 0.016304479347100893, -0.013687541866544252, -0.00044400589000313486, -0.02225081799715167, 0.0059018102515688365, -0.0011269271805111906, 0.01708544926743499, 0.008131687245337877, -0.014386304867996013, -0.028197156647202442, 0.018359664426353683, 0.010502002106602218, -0.00275737511522128, -0.015290586563637581, 0.00959086960155646, 0.0320335056054122, -0.018839208046129905, -0.012105047735018196, -0.6256535551268062, -0.024826651552605817, -0.001861656349258293, 0.0004335158791412966, -0.0004219554584138864, 0.011139110617384214, 0.008508471518019194, 0.010728073229004596, 0.0029115141358636363, 0.039322565645778275, -0.021963091452756878, -0.03326661777055106, -0.017195060355256732, -0.011474790964963039, 0.01131037619587572, -0.02579944041096664, -0.004654997398944253, -0.018674794208365236, 0.007535683590981021, 0.015194678025946868, -0.006586872565534865, 0.02175757275856707, -0.006943104875664936, -0.006734160776773032, -0.030498967139715477, 0.024826651552605817, 0.007611040166120489, -0.001955852301013291, -0.024703340708620992, 0.007761753782060751, -0.003928831159875732, 0.010132068643325092, 0.0015071367414792704, -0.004963274974567642, 0.045625138934265744, -0.01195433458473926, -0.03236233700623214, 0.018085639500767272, -0.0032163667724462545, 0.017263566586653335, -0.025936452873759844, -0.026388593255919305, -0.00749457966587853, 0.009816940723958837, -0.0018325412241978031, 0.030334551439305513, 0.018455573895367047, -0.006374503061940867, -0.03214311483058865, -0.0014052337455598897, -0.007521982437833965, 0.01825005520117724, 0.023552436393687124, -0.005802476309176027, -0.010474599800308107, 0.009138729219396999, 0.034664145636099875, -0.008117986557852145, 0.002724834701874026, -0.01757869450601959, 0.00410694754777143, 0.014495915024495107, -0.03184168666738548, -0.03123883406626973, -0.022428933453724716, 0.016318180965909274, 0.0008974314101063689, 0.009419604023064953, 0.0010935304159884129, -0.028937025436401993, -0.0055558541555450515, 0.022442635072533097, -0.006977357991363237, -0.010193724996640154, -0.00351094342775325, 0.025635024710556672, 0.018633689351940094, -0.02981390482574945, 0.001680114905906494, 0.029074037899195198, 0.021250627763819384, -0.017044346273655146, 0.018606287976968634, -0.0070629907806089905, 0.010282782724926678, 0.02233302584735665, 0.006490963562182827, 0.020346345136855168, 0.017907524044194222, 0.00912502760058862, 0.028553389422993836, 0.020469655980839992, -0.038089453480639426, -0.04414540508115724, 0.014893250794066344, -0.003481828419108091, -0.03197869913017869, -0.009501811873269934, 0.00039926276344299513, -0.021223224526202625, 0.0010549956608278236, 0.0014797342023544968, 0.0380346507306965, 0.009077072866081937, 0.03830867565628291, 0.01131037619587572, -0.02274406323573627, 0.0035931508122969084, -0.01195433458473926, -0.01788012266922276, -0.003647955890546455, -0.0012742156245800192, -0.018455573895367047, -0.0017537591279409084, 0.0027745016730704305, -0.03145805251662262, 0.008823600368708096, -0.011378882427272325, -0.00796042166684637, -0.02863559727319882, 0.028800012973608787, -0.010467748990903917, -0.00022500010135631865, -0.012269462504105513, 0.02023673591167872, 0.01407117601730711, 0.0225111413039297, -0.025223987322177056, 0.004172028374465938, 0.031595064979415825, -0.004545387242445158, -0.027799820877631207, 0.028827414348580247, -0.021606860539610778, 0.03003312327610234, 0.004411800184354048, 0.043542548754750894, 0.0030176986548299737, 0.021784975996183828, 0.0032197919443176875, -0.013797151091720698, -0.010762326344702899, -0.009940251567943663, 0.00797412328565475, -0.040939311961679985, -0.017756809962592636, 0.016318180965909274, 0.0009727882180764996, -0.0007325883490159587, -0.007631592128671735, -0.00938535090736665, -0.016331880722072357, 0.004617318878543856, 0.028580792660610598, -0.004011039010080716, -0.027895730346644568, 0.005918936809417987, -0.028608194035582058, -0.009337397104182618, -0.018469275514175425, 0.006792391259724674, 0.013899910438815603, -0.014358902561701902, 0.00845366690543097, 0.012728454626991813, -0.003993912452231565, -0.010488301419116487, 0.019825698523299105, -0.03381466762172388, -0.040500875060974204, -0.004514559531448952, -0.05069459633232376, 0.009296293179080126, 0.012399625088817178, -0.010378691262617392, 0.002747099133945657, 0.006843770933272126, 0.01124871984256066, 0.009947102377347854, -0.012584591354794417, 0.009275741682190204, 0.009042819750383636, -0.0025261665842577453, -0.01918173920311292, 0.011611803427756244, 0.022223416622180206, 0.04450163599430334, 0.026594111950109113, -0.02156575568318564, 0.01456442125589171, 0.008152239673550447, 0.02841637696020063, -0.024703340708620992, 0.02262075239175144, -0.007802857707163243, 0.024991067253015784, -0.004192580337017184, -0.008398861361520099, 0.015372794413842565, 0.028525988048022373, 0.007857662319751465, -0.008816749559303906, 0.017277266342816417, -0.018496676889146888, -0.0017220749493538471, -0.00987174533654706, 0.006525216677881128, -0.014386304867996013, 0.01550980687663577, 0.020387449993280307, 0.008035778707647162, -0.004833113321178626, -0.009070222988000396, -0.01951056874128755, 0.006134731252052756, 0.02063407168124996, -0.005946339115712099, 0.009309994797888506, -0.020113425067693896, -0.011673458849748658, 0.01972978905428574, 0.013475172362950254, 0.020113425067693896, -0.019620179829109297, -0.011906379850232577, 0.014023222214123077, -0.017962328656782448, 0.014632927487288313, 0.0035794496591498525, -0.029457672049958056, -0.0032780224272693295, 0.005240725304856148, 0.020798487381659923, 0.02301808816132268, 0.016989541661066924, -0.02492256102161918, 0.03148545575423938, 0.0041172232962163905, 0.020866993613056526, -0.01244757889200121, -0.00480913641958661, -0.00516879366875745, -0.0008079451569860894, -0.0234976317810989, 0.022908477073500938, -0.019222844059538058, 0.026429698112344444, 0.0052030467844557515, -0.03778802904272685, 0.0005189345514893365, -0.02303178978013106, 0.01036499057513166, -0.022264519615960047, 0.010173172568427584, 0.01881180667115844, -0.009097625294294507, 5.6303543649570795e-05, -0.0027385358550210816, -0.007624741784928869, 0.0267448260317107, -0.0018119893780618884, 0.02579944041096664, 0.002216176073452647, -0.0047543313413370624, 0.012098196925614006, -0.026224179418154636, -0.014386304867996013, -0.018880312902555044, -0.0064464346980395645, -0.001777736262363587, -0.00027552342372824705, -0.001971266389342056, 0.008241297401836972, -0.037705819329876565, 0.02534729816616188, 0.022648153766722905, 0.005528451383589616, 0.01734577257421302, 0.019400959516111108, 0.022771464610707733, -0.006114179755162835, -0.05957300317626538, 0.007350716859342458, 0.0073712688218937035, -0.015139873413358644, 0.005939488771969233, -0.049050450504095885, -0.004994103151225173, -0.001865081637545057, 0.002442246497363039, -0.004377547068655747, 0.004004188200676525, 0.0068780240489704275, 0.013988969098424776, -0.0015045676879526992, 0.005761372384073536, 0.01172826346233688, -0.004976976593376022, -0.0013829691970729276, -0.018661092589556855, 0.0038774517191589425, -0.0006448147400390612, -0.016784022966877116, -0.018606287976968634, 0.04080229949888678, -0.019606478210300916, -0.02745728972064819, 0.0012493820225664858, 0.0010567083631788713, -0.020976602838232973, 0.019798295285682343, 0.012618844470492718, 0.005018080052817189, -0.02045595622467691, -0.0003339677907469408, 0.019606478210300916, -0.012769558552094303, 0.0026100866711524515, 0.015866038721104515, 0.021880885465197192, 0.012125600163230766, -0.01127612308017742, -0.011296674577067341, 0.013468321553546063, 0.05644911976963841, 0.01855148336438041, -0.01193378215652669, 0.00518249528756583, -0.03145805251662262, -0.007754903438317885, -0.014975458644271328, -0.006932828661558651, 0.016647010504083907, 0.03830867565628291, 0.006131306313011986, -0.01761979749979943, -0.0075630858972751324, -0.0017066610938557439, 0.0337324579088736, 0.013276504478164636, 0.012351670354310496, -0.007343866049938268, -0.013290205165650366, -0.035541023162802034, 0.009056521369192015, -0.0022761190259246744, -0.005812752057620988, 0.011050051957775042, 0.0042165572386092, -0.01944206250989095, 0.025032170246795625, 0.023620942625083727, -0.035815048088388445, -0.01990790637350409, -0.010769177154107088, 0.006586872565534865, 0.0008730260418882629, 0.002216176073452647, -0.009268890872786015, 0.01384510582622738, 0.0025484312491600384, -0.004442627895350254, -0.001271646571053448, 0.009131878409992808, 0.02249743968512132, 0.016345582340880738, -0.0029920088180562477, -0.017153955498831593, 0.009357948601072539, -0.01861998959577701, 0.011145961426788404, 0.03249934946902534, 0.015222080332240978, -0.035842451326005206, 0.00341160948536044, 0.0017691729834390115, -0.04685825109940459, 0.013331309090752858, 0.031129222978447987, 0.009693628948651363, -0.01812674435719241, 0.008049480326455543, -0.016756619729260354, -0.0091729823350953, -0.01384510582622738, -0.032663761444144714, 0.0012433877971684817, -0.01879810505235006, -0.013783450404234968, 0.020538162212236595, -0.010248529609228377, -0.008487920021129271, -0.03806205396831326, 0.015866038721104515, 0.0014942918230924074, -0.00843996528662259, -0.04250125552763877, -0.00014332784898586406, 0.01851037850795527, -0.0073712688218937035, 0.016715516735480513, 0.0033482411281863175, 0.010186874187235963, -0.007330164896791213, 0.006439583888635375, -0.0333762269957275, -0.00564148647912948, -0.023771656706685313, 0.018414470901587203, -0.015071367181962041, -0.004154901816616787, -0.013906761248219792, 0.00155080937086234, 0.017496486655814606, 0.006117604694203606, -0.0010164609056672346, -0.011406284733566436, 0.005720268924632369, 0.0013358711629877632, 0.04189839920123243, 0.017510188274622987, 0.005364036614502298, 0.005179069882863736, -0.026881838494503905, -0.01640038695346896, -0.02348393016229052, -0.019866801517078946, -0.030553771752303702, 0.001060990002641159, -0.007247957512247554, 0.02473074394623775, 0.025909049636143083, 0.01918173920311292, -0.010974695848296898, 0.020003813979872154, -0.0014146532592446577, 0.012577740545390226, -0.01359163239753089, -0.002053473773885715, -0.004987252341820982, 0.029485075287574814, 0.007939870169956448, 0.003029687338456644, -0.03236233700623214, -0.01946946574750771, -0.026443397868507527, 0.030170135738895546, 0.01708544926743499, -0.0029663189812825213, 0.039432174870954725, 0.013927313676432363, -0.00869343871531908, -0.055764057455672374, 0.013680691057140062, 0.0028241686442668354, 0.008296102014425193, -0.009378501029285108, -0.013207998246768032, -0.011221317536266548, 0.0039322565645778275, -0.028937025436401993, -0.0038877277004345655, -0.018154145732163875, 4.17459738433676e-05, -0.02442931578303458, 0.0047337793787858165, 0.0017880121272238786, 0.014742537643787407, 0.026936643107092127, -0.028306767735024184, -0.0043227419904061995, 0.013207998246768032, -0.005442818594343862, 0.027621705421058157, -0.013749197288536666, 0.01861998959577701, -0.031074418365859766, -0.02562132309174829, -0.014290396330305299, -0.01897622050892311, 0.008816749559303906, -0.006121030098905701, 0.0234976317810989, 0.030855198052861577, 0.04548812647147254, -0.01523578195104936, 0.007796006897759052, 0.02997831866351412, -0.016085259034102704, -0.01408487763611549, 0.006408756177639168, -0.02162056029577386, 0.010522554534814788, -0.0021870608319768258, 0.0007402952767650103, -0.017441682043226384, 0.02223711637834329, 0.006538918296689508, 0.001388107187710739, 0.0073575672030853235, -0.010043010915038567, 0.020264137286650184, -0.018688495827173614, -0.02442931578303458, -0.019757192291902502, 0.0029834455391316724, -0.0044255013375011035, 0.0048570906884319665, -0.03504777978686273, -0.006997909953914483, 0.027758717883851362, 0.00844681609602678, 0.019209142440729677, 0.017715706968812795, 0.02138764022661259, -0.02203159768415348, -0.009789537486342077, 0.018277458438793998, 0.013687541866544252, 0.001803426099137313, -0.015674221645723088, -0.04387138015557083, -0.00021151293413972916, 0.016098960652911085, 0.0073575672030853235, 0.03307479883252433, -0.0027299725760965062, -0.0025775462578051973, -0.01614006364669093, 0.01009781552762679, 0.011522744768147072, 0.02445671902065134, 0.0010113230314447543, -0.01996271098609231, 0.004041866721076922, -0.03030714820168875, -0.010899338807496104, -0.013666989438331683, -0.008871555103214777, 0.009748434492562234, 0.0007921030676925588, 0.037157769478703744, -0.030581173127275162, -0.010412944378315693, 0.017893822425385845, 0.015139873413358644, 0.025073275103220768, 0.016098960652911085, 0.01929135029093466, 0.009419604023064953, 0.008049480326455543, -0.009494961063865744, 0.010488301419116487, 0.006552619449836564, 0.0027196965948208837, -0.003743864661067832, 0.007001334892955254, -0.006357376504091716, -0.006672505354780619, 0.017414278805609622, 0.01244757889200121, -0.01970238767931428, -0.04789954432651672, 0.018852909664938283, 0.005172219073459545, 0.006038822714362043, -0.02412788761983141, -0.028772609735992025, -0.0191954408219213, -0.0017606097045144362, 0.013714944172838363, 0.011241869964479118, 0.01668811349786375, -0.014687732099876536, 0.00959086960155646, 0.0010678405792146869, 0.002695719460398205, -0.007261658665394609, 0.002740248557372129, -0.002318935420547551, 0.026909239869475368, -4.1076967677385146e-05, 0.004750905936634967, -0.004261086568413786, -0.0034201727642850155, 0.02347022854348214, 0.013797151091720698, -0.0016467181413837163, 0.000317911642763362, -0.004278213126262937, 0.03425310824772026, -0.012625695279896907, -0.008755094137311493, 0.03244454299379182, -0.03417089853486998, -0.024073083007243187, 0.00964567514546733, -0.006422457330786224, 0.0056894412136361625, -0.03030714820168875, -0.0023292111689925117, 0.006525216677881128, -0.0034492877729301744, 0.002133968456078326, 0.020757382525234784, 0.019647581204080757, -0.018167847350972256, 0.01949686898512447, -0.005857281387425574, -0.04768032587616383, -0.008789347253009795, -0.005792200560731067, 0.032417139756175065, -0.04209021813925915, -0.006641677643784412, 0.002000381397987215, -0.018894014521363425, 0.0030930554628001046, 0.0024285453442159832, -0.021415041601584053, -0.011618654237160434, 0.04082970273650354, -0.013947865173322284, 0.007419223090739061, -0.0019678409846399614, 0.006384779276047151, -0.023826461319273535, 0.009721031254945474, 0.004771457899186213, -0.00867288628710651, 0.013276504478164636, -0.02416899247625655, -0.016838827579465337, -0.03354064269613747, -0.01690733381086194, 0.007501430475282719, 0.005922362214120082, 0.008337205939527685, 0.003497242274606194, -0.020373748374471926, 0.0050728851310667365, 0.003269459148344754, 0.016044156040322863, 0.01788012266922276, -0.012666799204999399, -0.0018411045031223785, 0.022401532078753256, -0.015427599026430788, 0.0041686029697638426, -0.007686397206921282, 0.0019250245900170845, -0.0036376799092708327, -0.03847309135669288, -0.004007613605378621, 0.02112731691983456, -0.0048947687431710385, 0.012187255585223178, -0.021510951070597417, -0.02514178133461737, 0.016537399416262165, -0.017290967961624798, 0.004237109201160445, 0.011817322121946054, 0.00846051771483516, 0.015317989801254341, 0.004798860205480325, 0.01128297295825896, -0.014057475329821378, -0.03222532454343893, -0.0010978120554507005, -0.022004196309182017, -0.000677783328974077, -0.015619417033134865, -0.015427599026430788, -0.009262040063381825, 0.047872141088899964, 0.001745195849016333, -0.015413898338945057, -0.03126623730388649, -0.003367080388386516, -0.03751400411714043, -0.023826461319273535, -0.01941466113491949, 0.029868709438337672, 0.0025826843648583397, 0.02934806282478161, 0.021963091452756878, 0.015427599026430788, 0.0007205997235969539, -0.002159658292852052, -0.02301808816132268, -0.0019472891385040466, 0.0032266427537218775, -0.019359856522331263, 0.004922171515126475, -0.001242531445992958, -0.02423749870765315, -0.005929213023524272, -0.012159853278929067, -0.021319133995215987, -0.002390866823815587, 0.036719328852707366, -0.0016681263386951548, -0.012954524818071541, 0.004298765088814183, 0.0031050441464267747, 0.002354901005766238, 0.014893250794066344, 0.003606852198274626, -0.03362284868369715, -0.00023078031172002375, 0.01718135873644835, -0.01383825501682319, -0.01195433458473926, -0.02623787917431772, 0.019537971978904313, 0.0011235018922244266, 0.003175263080174425, 0.004035015911672732, -0.008748243327907303, 0.00798782490446313, -0.0003468127091338038, 0.0450222826078594, 0.006025121561214986, -0.003850049180034169, 0.02348393016229052, -0.0021134167263577425, 0.00587098254057263, -0.014742537643787407, 0.007302762590497101, 0.0031187452995738306, 0.0015002860484904115, 0.032965189607347886, -0.03562323287565231, -0.022963283548734458, 0.002020933360538461, 0.0014677456351431576, -0.024045681632271724, -0.02014082644266536, 0.008145388864146256, -0.008248148211241162, -0.03828127241866615, 0.03551361992518527, 0.005110563651467132, -0.009919700071053743, 0.003178688252045858, -0.017003241417230006, 0.010275931915522488, 0.008419413789732669, -0.023785356462848396, 0.008542724633717495, -0.006970507181959047, 0.003596576216999003, 0.019222844059538058, -0.005014654648115094, -0.011865275925130085, 0.020264137286650184, 0.017208760111419814, -0.012611993661088528, -0.0005330639966394852, 0.19543452626427665, -0.026278984030742858, 0.006100478136354455, 0.02278516622951611, 0.01244757889200121, 0.006145007466159042, 0.020935497981807834, 0.010412944378315693, -0.0011962897630833171, -0.020949199600616215, -0.006898575545860349, 0.015002860950565439, -0.0030108480782564456, -0.004442627895350254, 0.014509615711980839, -0.03137584652906294, -0.03499297331162921, -0.03378726438410712, -0.00846051771483516, -0.012687350701889321, -0.002467936334136765, -0.010769177154107088, -0.0015533784243889112, -0.021908286840168652, 0.011152811304869946, -0.0003110610370860014, 0.002906376028810494, -0.010981545726378439, 0.01855148336438041, 0.015646818408106326, -0.017263566586653335, -0.002270980918871532, 0.008398861361520099, 0.011652907352858736, 0.013187445818555462, -0.016770321348068735, 0.010447197494013995, 0.015852337102296138, 0.010036160105634379, 0.00035280699273947347, 0.0230043865425143, 0.002652903065775328, 0.016112662271719466, -0.034225705010103497, 0.0010686969303902105, 0.031896493142619, -0.00234805042919271, -0.01740057904944654, -0.009419604023064953, -0.00037014763838250515, -0.027635407039866538, 0.013242251362466333, 0.030279746826717288, 0.02255224616035484, -0.012886018586674939, -0.0029337785679352676, -0.02696404634470889, 0.01364643794144176, 0.010680118494497915, 0.0014215039522335168, -0.025498012247763466, -0.012303715619803814, -0.005213322998562037, 0.03386947037166681, -0.017606097743636348, 0.03375986114649036, 0.009775836798856347, -0.0076658452443700365, 0.007576787050422188, -0.013776599594830777, -0.01811304273838403, 0.004504283317342667, -0.019154337828141455, 0.0038911528723059984, -0.044035795855980796, -0.04039126211050716, 0.02253854454154646, 0.02467593747100423, 0.006857472086419182, 0.029950917288542656, 0.0020294966394630363, -0.020716279531454943, -0.008083733442153844, -0.0013084686238629896, -0.015372794413842565, -0.03828127241866615, 0.016290777728292512, -0.018825508289966823, -0.033951680084517086, -0.006425882735488319, -0.011543297196359641, -0.009344247913586807, -0.0006619412978882119, 0.005603808424390408, 0.00825499902064535, 0.008371459055225986, 0.02486775640903096, 9.28580340104895e-05, 0.008789347253009795, -0.036828941803174406, -0.02262075239175144, 0.07639813286221293, 0.017770511581401017, -0.020086021830077135, -0.004490582164195612, 0.01812674435719241, -0.006360801908793811, 0.0300605265137191, -0.017647200737416192, -0.018222651963560477, -0.01408487763611549, -0.04263826799043197, 0.007891915435449766, 0.005237299900154054, -0.018578884739351872, 0.012495533626507892, 0.016838827579465337, 0.024210095470036393, 0.011543297196359641, -0.01240647496689872, -0.004151476411914692, -0.0029697443859846165, -0.009659375832953062, 0.009035969872302095, 0.0042165572386092, -0.0332392145329343, -0.013653288750845951, 0.00411379835717562, -0.03899373797024894, 0.002353188536245853, 0.003041675789252652, -0.03173207744220903, -0.006973932586661142, 0.00620323748344936, -0.009816940723958837, 0.010255380418632566, 0.003819221469037963, -0.0017263565888161348, -0.01431779863659941, 0.012995628743174033, 0.014838446181478122, 0.024744443702400837, -0.017893822425385845, -0.01054310603170471, 0.02514178133461737, -0.025881648261171623, 0.01861998959577701, 0.0168662289544368, 0.0018428172054734259, -0.009810089914554648, -0.013235400553062144, -0.0062272148507027, 0.000595147710634991, 0.00865233479021659, 0.03406128930969353, 0.028882218961168472, -0.003969935084978224, 0.0011491917289981528, 0.03389687360928356, -0.004110372952473525, -0.04578955463467571, 0.020812187137823006, 0.015126171794550265, -0.015550910801738262, -0.019537971978904313, -0.0024987640451329717, -0.17460863191970938, -0.0033722182626089964, 0.002245291082097806, -0.04540591675862226, 0.01106375357658342, -0.014646628174774044, 0.022908477073500938, 0.011584401121462133, -0.026018660723964828, 0.002721409297171931, 0.021456146458009192, -0.0012528074272685809, 0.00821389509554286, -0.01059106076621139, -0.010001906989936077, -0.002053473773885715, -0.026868136875695524, -0.004165177565061747, 0.009953953186752044, 0.00016495011953926702, 0.04236424306484556, -0.023209905236704107, -0.0003784968295131996, -0.013687541866544252, 0.003000572096980823, -0.0032317806279443576, -0.013043583477680715, 0.04110372766208995, -0.01788012266922276, -0.03915814994536831, -0.010022459418148647, -0.01946946574750771, 0.04154216828808633, 0.0029834455391316724, 0.011330927692765643, 0.004130924915024771, 0.03688374455311733, -0.0086797370965107, -0.029074037899195198, 0.003647955890546455, 0.03828127241866615, 0.023525033156070362, 0.007823409204053163, -0.01619486825927915, -0.021880885465197192, -0.003033112510328077, 0.011173363733082515, -0.020058618592460376, 0.006073075830060344, -0.030142734363924083, 0.012714753008183432, -0.010769177154107088, 0.006316273044650549, -0.003658231871822078, -0.003286585706193905, 0.02863559727319882, 0.0022230266500261747, -8.798766766694563e-05, -0.010460899112822376, -0.0026237880571301692, -0.0070218868555065, -0.004206281490164239, -0.0053366338425468625, 0.0002733826039971032, 0.003915130006728677, -0.019346154903522882, -0.016989541661066924, 0.027635407039866538, -0.038527894106635804, 0.011666608040344468, -0.009604571220364838, -0.0022333026313017977, -0.013057284165166447, -0.021236926145011006, 0.040309056122947474, 0.028046444428246154, -0.02585424502355486, 0.0024473843715855192, 0.03252674898135151, 0.0024251199395138884, 0.004976976593376022, 0.0040281655679298665, -0.0029303533960638346, -0.010960994229488517, -0.0037575658142148874, -0.0009907710688935086, 0.008611230865114098, 0.008172791170440369, -0.003367080388386516, -0.025004768871824165, 0.022209715003371825, -0.02492256102161918, 0.008775645634201414, -0.0074534762064373625, 0.012310566429208004, 0.004309040837259143, 0.012187255585223178, -0.003440724494005599, 0.006093627792611589, -0.007604189822377624, 0.0018582310609715292, 0.003012560780607493, 0.006552619449836564, -0.01456442125589171, 0.023840162938081916, -0.012331118857420574, -0.0013084686238629896, 0.013310757593862937, 0.02510067647819223, -0.01622227149689591, -0.02729287588288352, -0.009953953186752044, 0.0054325428458989015, 0.018373366045162064, -0.017852719431606, 0.010406093568911503, -0.023936070544449982, -0.04058308104853389, 0.015194678025946868, -0.010625313881909692, 0.04019944689777103, -0.0012904858312536463, -0.014824744562669741, 0.018140445976000792, 0.005785349751326876, -0.03924035965821859, -0.10676008567648991, -0.017606097743636348, 0.011324076883361452, 0.016729216491643596, -0.022264519615960047, 0.019428362753727866, -0.008316654442637765, 0.04485786690744943, -0.023429125549702296, 0.042035411664025635, -0.016468893184865562, -0.02651190409990413, 0.009001716756603793, -0.012077645428724085, 0.01707174764862661, -0.01059106076621139, 0.013324458281348667, -0.033595445446080396, -0.021099913682217798, 0.02701885095729711, 0.011899529040828387, -0.01007041322133268, 0.01103635127028931, -0.012502384435912083, -0.012803811667792604, -0.008487920021129271, -0.03129363681621265, 0.022401532078753256, 0.005764797788775631, 0.02051076083726513, 0.033595445446080396, -0.019332453284714505, -0.00868658790591489, -0.01407117601730711, -0.0018171273686996999, -0.002089439591935064, 0.0086797370965107, -0.015139873413358644, 0.02653930733752089, -0.007309612934239967, 0.0009950527083557963, -0.00031234550564162146, 0.022319324228548272, -0.019647581204080757, -0.018044536506987428, -0.009919700071053743, -0.031896493142619, 0.02673112441290232, 0.00914557909747854, -0.030005721901130877, -0.032334933768615376, -0.019716087435477363, -0.030690784215096908, 0.02530619517238204, 0.018948819133951647, 0.016838827579465337, 0.024963664015399022, 0.03318440805770077, -0.0007527120195641114, -0.015222080332240978, -0.015550910801738262, -0.01830485981376546, -0.013434068437847762, -0.007309612934239967, 0.0003290439170068431, 0.010358139765727472, -0.01311893958715886, -0.02118212153242278, 0.019839400142107486, -0.0034784030144059956, 0.004603617725396801, 0.011324076883361452, -0.011659757230940277, 0.017756809962592636, -0.008172791170440369, -0.006580021756130676, -0.009967653874237774, -0.003363654983684421, -0.0018308285218467555, -0.0061381566567548515, -0.026498204343741047, -0.018167847350972256, 0.0168662289544368, -0.016071557415294323, -0.006792391259724674, 0.007391820318783625, -0.005535302192993806, -0.010207425684125885, 0.0025381552678844154, -0.02560762333558521, 0.01623597311570429, 0.050557583869530555, 0.014427408793098504, -0.020538162212236595, -0.005405140073943465, 0.009200384641389411, -0.031869089905002236, -0.019811996904490724, 0.0020260712347609415, 0.015167275719652757, -0.014756238331273139, -0.010913039494981836, -0.06373817981000447, 0.04524150478350288, 0.004637870841095102, 0.00028537120031227526, -0.005264702672109489, -0.007186301624593817, 0.012543487429691925, -0.010762326344702899, -0.00328829817571429, 0.009056521369192015, -0.01738687743063816, -0.007432924243886117, -0.009131878409992808, -0.0029406291445087953, -0.010885637188687724, -0.028772609735992025, 0.022483739928958236, 0.010330737459433359, 0.009241488566491903, 0.011372031617868135, -0.002604948796929971, -0.012639395967382639, 0.014646628174774044, 0.030855198052861577, 0.02910143927416666, -0.0032197919443176875, -0.03603426653874134, 0.015674221645723088, -0.01222835951032567, -0.015893441958721277, -0.002680305604900102, -0.017660902356224573, -0.009947102377347854, 0.00865233479021659, -0.02044225460586853, 0.004740630188190007, 0.009659375832953062, 0.00867288628710651, 0.004942723477677721, 0.030444160664481957, -0.006919127508411595, -0.036280891952001584, 0.0053880135160943145, -0.03312960530775785, 0.00014953622038541464, -0.022730361616927888, -0.006713608814221787, -0.011303525386471532, 0.035294401474832385, 0.011666608040344468, 0.03477375486127632, 0.0019164613110925091, -0.04702266679981456, -0.04535111400867933, -0.010138919452729283, 0.018921415896334885, 0.02367574723767195, 0.0040281655679298665, -0.00749457966587853, 0.000510799448152523, 0.034828557611219245, 0.012529786742206193, 0.018729598820953458, 0.0015157000204038457, 0.016770321348068735, -0.015372794413842565, -0.011145961426788404, -0.0037609912189169826, 0.016277076109484135, -0.00358630023572338, 0.009015417444089525, -0.020017515598680532, 0.0035058055535307693, 0.014276694711496918, 0.0007852524911190309, -0.02368944885648033, 0.008775645634201414, 0.011714562774851148, -0.033047399320198165, 0.006717034218923881, 0.02396347378206674, -0.003168412270770235, -0.00015713613334135857, -0.011481641774367229, 0.019222844059538058, 0.0022487164867999007, -0.015824935727324674, 0.012598292973602796, -0.024566328245827787, -0.0009239775398479531, -0.003038250617381219, -0.005309231536252752, -0.006809517817573825, 0.008070031823345465, 0.01949686898512447, 0.0119611844628208, -0.007247957512247554, 0.0018565183586204815, 0.03762361334231688, 0.003084492183875529, 0.02262075239175144, -0.0053982897302006, -0.01763349911860781, -0.014509615711980839, -0.024538925008211025, 0.002805329384066637, -0.029923514050925894, -0.021702768145978844, 0.025539117104188606, 0.005860706792127669, 0.021469848076817573, 0.006309422235246359, 0.010077264030736869, 0.007206853587145062, -0.0021630836975541474, 0.01944206250989095, 9.007496544961924e-05, -0.03312960530775785, -0.03913075043304214, 0.021716469764787225, 0.029485075287574814, 0.008809898749899715, 0.04337813305434093, 0.0011175076668264224, 0.029704293737927705, -0.011303525386471532, 0.049050450504095885, -0.009029119062897904, 0.0234976317810989, 0.01734577257421302, -0.005309231536252752, -0.00434671935765954, -0.006333399602499699, -0.008638633171408209, -0.009241488566491903, -0.008988015137795412, -0.03307479883252433, 0.025169182709588834, 0.014194487792614584, 0.08620821625750903, 0.023072892773910902, -0.021963091452756878, 0.012337968735502115, -0.0018239779452732276, -0.006007995003365836, 0.01663330888527553, 0.012125600163230766, -0.007275359818541665, -0.016372985578497496, 0.01812674435719241, -0.0006842057881675086, 0.005394864325498505, -0.02912884251178342, 0.007474027703327284, -0.0023634642846908135, -0.00987174533654706, 0.015578313108032373, -0.006384779276047151, -0.012564039857904495, 0.03992542197218462, 0.007967272476250559, 0.033211811295317535, 0.0023103721416229757, -0.016564802653878927, -0.001115794964475375, 0.03280077390693792, -0.0068780240489704275, -0.004011039010080716, -0.010296484343735058, 0.02656671057513765, 0.024292303320241376, -0.03778802904272685, -0.010522554534814788, 0.001174025214596355, 0.005939488771969233, -0.00369248498752038, -0.022442635072533097, 0.007802857707163243, 0.0011312088199734783, -0.004452904109456539, 0.01757869450601959, -0.009097625294294507, -0.031595064979415825, -0.005055758573217585, -0.010947292610680137, -0.020086021830077135, -0.015331690488740073, -0.028882218961168472], 'source': 'https://arxiv.org/pdf/2303.08774.pdf', 'page': 40}), Document(page_content='multiple languages. For instance, preliminary results from red teaming indicate some proﬁciency\\nof the model to generate text that favors autocratic regimes when prompted to do so in multiple\\nlanguages, and ﬁnd that the model does an especially good job of “following the lead” of the user\\nby picking up on even subtle indicators in the prompt. Additional testing is necessary to verify\\nthe extent to which - and in fact, whether - the language choice can inﬂuence diﬀerences in model\\noutputs.\\nThe profusion of false information from LLMs - either because of intentional disinformation, soci-\\netal biases, or hallucinations - has the potential to cast doubt on the whole information environment,\\nthreatening our ability to distinguish fact from ﬁction.[ 55] This could disproportionately beneﬁt\\nthose who stand to gain from widespread distrust, a phenomenon scholars Chesney and Citron refer\\nto as “Liar’s Dividend” in the context of deep fakes.[56]\\n51', metadata={'_id': ObjectId('6616defa6182e3cd8e9a22d1'), 'embedding': [-0.024638138053406557, 0.009708451581806573, 0.008122691350446087, -0.037725852784229866, -0.016134586974063485, 0.02425035394082876, -0.024804332574972214, 0.005131214642313704, -0.020704899995536853, -0.04645099345458508, 0.008233487077274778, 0.031521306982985095, -0.01455575298902905, 0.002691983376928677, 0.004421430856093669, 0.0018939097406571708, 0.03595312488026129, -0.007852627348377791, 0.004012872661150819, -0.007077059588883505, -0.0007037242520073604, 0.019389203766244662, -0.012187500148832157, -0.006391512542530237, -0.018932172091568278, -0.0017060768693349638, 0.019527698890441838, -0.024818181342333837, -0.007478692934484234, -0.005477450124500087, 0.010220880188574684, -0.005640181057241409, -0.021120383505483134, -0.005539772837256554, -0.011349608745258784, -0.012499112315630558, 0.015829901053591135, -0.02089879205182575, 0.0150820312944814, -0.004397194581888211, 0.011072620359509677, 0.017090198488146358, 0.003263272271782193, -0.01355859331021607, -0.010020063748604974, 0.010435546327228635, 0.02148046822069245, 0.004656871542773982, -0.012464488534581264, 0.02332244368675961, 0.031410509393511164, 0.00600372842995244, -0.021245027999673444, -0.009639204019707987, -0.02024786832086047, -0.02598153368006723, -0.0037047229190234785, 0.012720702837965319, 0.009216797057403515, 0.004258700389013658, 0.015289772118131921, 0.0142649149045957, -0.020137074456677017, 0.042074574352045854, -0.011024147811098763, -0.02972087968164804, -0.019319956204146078, 0.02587073795323854, 0.010546342054057325, 0.0006851140977502254, 0.015719105326762444, 0.012381392205121055, -0.0018575549801030024, -0.0024011452924953282, 0.02190980049800035, -0.009992365282559111, -0.005830610921689902, 0.002382102306050479, -0.027768110351397427, -0.00025167014748969126, 0.01587144921832124, -0.02596768491270561, -0.006831232326681975, 0.03257386173124456, 0.013884055175698714, 0.009279119304498672, 0.00978462352758597, 0.021051135943384546, 0.01170969439179072, -0.007146306685320782, -0.0007911487869293851, 0.011578124582596976, 0.0020254793170202587, 0.014417257864831876, -0.0017407004175536021, 0.0028408646109857566, 0.0035177557120767006, 0.008489701380658833, -0.008108842583084466, -0.02433345213293421, 0.005352805164648465, 0.002743918582841307, -0.041299006126890254, -0.002717950979884992, -0.024693536848143523, -0.0009711913773646978, 0.0015701793203407141, -0.002359596894934569, 0.007665660607092323, -0.018004261837499126, -0.0405511354364579, 0.023973365555079652, 0.014763493812679569, -0.03409730064683251, 0.007284800878195335, -0.02425035394082876, -0.00168097481433875, -0.007340198741609681, -0.022491476666867048, -0.004539151432264481, 0.010317826216719133, 0.001242986475353197, 0.006993962793761987, 0.00146371179897384, 0.029776278476385004, -0.0025898435953623095, 0.003777432440131816, -0.010712535644300362, -0.0301640625889628, 0.0013953302504966644, 0.019527698890441838, 0.004611860720542163, 0.02160511271488276, 0.013627840872314658, -0.015608309599933753, 0.02739417500618125, -0.03340482875113712, -0.005155450916519162, 0.0015251686145242222, -0.002466929964261545, 0.020940340216555855, 0.013510120761805156, -0.017450283203355673, -0.009798473226270213, 0.002612349006478219, 0.04376420220126374, 0.002319779826124668, -0.0026140801023984217, -0.010871803919539968, -0.006349964377800134, 0.003052068557799302, -0.03415269944156947, 0.008191938912544675, 0.011640446829692133, 0.0015545986421515977, 0.029997868067397144, 0.014611149921120774, 0.02974857907901652, -0.01566370653202548, -0.008524325161708127, -0.008718217217997025, 0.00947301136078757, -0.0003085826616265503, 0.012810724482428957, 0.023183948562562437, 0.02087109265445727, 0.02537215811383205, -0.003683948836658427, 0.011328834662893732, 0.001382346449018507, -0.024098011911915206, 0.03958167701765865, -0.01915376354522566, 0.015968394315143068, 0.0017190606708131213, 0.0011425781389530147, 0.014985085266336951, -0.008683593436947733, -0.006727361449195405, -0.03484517133726486, 0.003372336669860026, 0.02250532729687391, -0.003313476614605275, -0.004151367319686683, 0.0027681550898774193, 0.004476828719508014, 0.003052068557799302, -0.010823330439806433, 0.011425780691038182, -0.014251065205911458, 0.025510653238029226, 0.015400567844960612, 0.013053089087128772, -0.009452237278422518, -0.6510340889704812, -0.029610083954819347, 0.0006548184639550849, -0.007561789729605753, -0.02250532729687391, 0.011972834010178205, 0.0029897460778734912, 0.004210227374941433, -0.027034090290971934, 0.04030184644807728, -0.019513848260434976, -0.011543500801547684, -0.006744673339720052, -0.017228693612343533, -0.017699574054381537, -0.006755060380902578, -0.00498579560009703, -0.017367186873895466, 0.017491831368085776, 0.013433948816025758, -0.011640446829692133, 0.027671165254575598, -0.014008699669889026, 0.007520241564875648, -0.03370951653425471, 0.013634765255995468, 0.02299005650627354, -0.008205788611228915, 0.0035627663014778646, 0.010317826216719133, -0.01703480155605463, 0.014119495396717717, 0.005847922346553239, -0.006415749282397005, 0.043930398585474636, 0.0038085937965100488, -0.026715553740492722, 0.00937606533264312, 0.00610759930743901, 0.01007546161201932, -0.03830752709045132, 0.006194158294400933, 0.00945916166210333, 0.018392045950076926, 0.004816139818013588, 0.021051135943384546, 0.009708451581806573, 0.01263760650850511, -0.012969992757668563, -0.024901277671794043, 0.0020375976869536423, -0.00422753879980477, 0.0032182616823810285, -0.011550426116551113, 0.02495667646653101, 0.012575284261409955, 0.03232457274286393, -0.030911931416749916, -0.010026988132285784, 0.010110085393068612, -0.019707740316723874, 0.00753409079789858, -0.018475142279537133, -0.01750568199809264, -0.010643288082201776, 0.01873828189792462, -0.010366299696452667, -0.009549183306566967, -0.0077418320872104105, -0.021106532875476272, -0.005006569682462081, 0.02534446057910881, -0.010255503969623977, -0.02587073795323854, -0.009756924130217488, 0.022948508341543432, 0.037864344183136556, -0.017865766713301954, -0.01211132820305276, 0.008732066916681267, 0.0028218216245409072, -0.02780965851612753, -0.004085582415089811, 0.003374067765780229, 0.01567755716203234, -0.0037116477683655992, 0.006519620159883575, -0.0018315873771466873, -0.001657603855262739, 0.014064097533303371, 0.007388671755681905, 0.03005326686213411, -0.00269717689751994, -0.014292613370641563, -0.007070134739541384, 0.03528835051928914, -0.006204545335583459, 0.004438742746618316, 0.015912997383051342, -0.02304545343836526, -0.016397726592450968, -0.005508611248047666, 0.02290696017681333, 0.011938210229128911, 0.01783806731593347, 0.01027627805198903, -0.008988281220065323, -6.995044874231272e-05, 0.011134942606604832, -0.02171590844171145, 0.009417613497373224, -0.001319158304717267, 0.009535333607882727, -0.02055255610397806, 0.010954900249000175, -0.019167614175232522, 0.018378195320070064, -0.0006634743510097445, 0.024582741121314835, -0.025219814222273255, 0.00432448482794922, 0.010269353668308218, 0.008531249545388938, -0.011280362114482818, 0.011792790721250928, 0.02056640673398492, 0.014133345095401957, -0.02171590844171145, 0.019707740316723874, 0.01956924705517194, -0.009736150047852436, -0.01973543971409236, 0.03667329431067992, -0.03310014096801953, 0.011730468474155771, 0.017228693612343533, 0.034235795771029684, -0.00041223697750010393, 0.015026633431067055, -0.004757279762758837, -0.020137074456677017, -0.011751242556520825, 0.009196022975038463, -0.005221235821116032, -0.026854047002044655, -0.03708877968327144, -0.029000710251229408, 0.0032598100799417877, 0.0060695133345493105, -0.013288529308147775, 0.0001259432672398583, -0.012589133028771576, -0.028585226741283127, 0.023848721060889342, -0.00611452415678113, -0.0058029119899827295, 0.002828746473883028, -0.010982599646368659, -0.02159126394752114, -0.024499644791854625, 0.01283149856479401, 0.02035866404768916, 0.0037635829742782294, 0.01304616470344796, -0.00803959502098588, -0.012997691223714426, 0.01008238599570013, 0.00549822420686514, -0.012658380590870162, -0.040384942777537486, -0.004816139818013588, -0.016425425989819453, 0.01943075193097477, 0.02386256982825096, -0.008081143185715983, 0.014832741374778156, -0.013877130792017902, 0.01742258566863243, -0.00590678240180799, -0.029388494363807204, 0.0012265402492039792, 0.022145240719019355, -0.006533469392906506, -0.0001282154470504518, 0.02844673161708595, 0.017450283203355673, 0.03426349330575292, 0.0005029074627916681, 0.0011832607557230174, 0.0129422942916227, 0.0028391335150655538, 0.005789062291298488, -0.0035056373421433165, -0.006478071529492161, -0.0052974077668954305, 0.013925603340428817, 0.008309659023054175, -0.012914594894254217, 0.008621271189852576, 0.01764417525964457, 0.004736505680393785, 0.02618927450371775, 0.016259233330899035, -0.02006782689457843, 0.014292613370641563, -0.012755326619014613, 0.014583451455074913, -0.01200745685990488, -0.005051580039032591, 0.001282803660578426, 0.00966690341707647, -0.021189631067581722, -0.0020826082763548068, -0.01283149856479401, -0.022782315682623018, 0.037421165001112276, 0.013586292707584553, 0.01682705886975887, -0.013683238735729004, -0.0012421209273930957, -0.016439274757181075, -0.005186612040066739, 0.02171590844171145, 0.0026002308693754903, -0.014541903290344807, 0.0010291858846593472, -0.009466086045784139, 0.002807972158687321, 0.0287237218654803, -0.02229758461057815, 0.01883522699474645, 0.01692400582922594, 0.012582208645090765, 0.027034090290971934, 0.005470525275157967, 0.0038674538517647996, 0.008697443135631974, -0.0034952503009607907, 0.012568358946406524, -0.018004261837499126, -0.006761984764583389, 0.013489346679440104, 0.00453222658292236, 0.005529385796074028, 0.04922088103736663, -0.00028001818195927625, 0.03783664664841332, -0.009999289666239922, -0.034485084759410305, 0.00027136229490461666, -0.01934765560151456, 0.034900566406711346, -0.01974929034409922, -0.016536221716648144, 0.01456960175639067, -0.017187145447613426, 0.010428621943547824, -0.0072986505768795765, 0.022643820558425842, 0.03157670577772206, 0.01221519861487802, 0.009466086045784139, 0.014735795346633707, -0.001507856840414903, 0.0038051313718389886, -0.014915837704238365, 0.0036700993708048404, -0.01100337372873371, -0.01161274836364627, -0.027477273198286696, -0.016550070484009766, -0.014348011234055909, 0.009902343638095472, -0.03567613463186694, 0.00865589497090187, 0.012713778454284508, -0.005287020725712904, 0.012242897080923881, 0.025330609949101947, 0.009099076946894014, -0.014611149921120774, -0.015843749820952754, 0.027588067062470148, 0.005809836373663541, -0.005875621278260412, -0.026133877571626024, -0.02974857907901652, 0.010373224080133478, 0.0010032181652877047, -0.013828657312284368, -0.0033879172316338153, -0.008164239515176192, 0.014112571013036906, -0.012679154673235216, 0.0002527521406474817, 0.02883451572966375, 0.008496626695662264, -0.023031604671003642, 0.013579367392581122, -0.011197265785022609, 0.0006353427035301848, -0.0007244983925800762, -0.05102130647605845, -0.028668323070743334, 0.04046803910699769, -0.002669477965812767, -0.024305752735565726, 0.009507635141836864, 0.012277520861973175, -0.02452734418922311, -0.0012447176876887271, -0.002191672441601985, -0.01721484298233667, 0.017768821616480125, 0.031216619199867505, 0.019693891549362255, -0.01007546161201932, -0.010560191752741567, 0.03534374931402611, 0.011024147811098763, 0.003749733508424643, -0.023876420458257823, 0.018475142279537133, 0.018142755099051062, 0.051852273495951014, 0.03296164584382236, -0.016536221716648144, 0.005522460946731907, -0.02423650517346714, -0.019915483003019634, -0.021328124329133655, -0.033155539762756495, 0.01069176156193531, -0.008129616665449518, -0.02444424599711766, -0.0034433148622175056, 0.0001063052092964813, -0.011952059927813154, 0.03024715891842301, 0.02199289682746056, -0.008337357489100038, -0.011633522446011322, -0.014209517041181355, -0.013496271063120915, 0.0050688919295572375, -0.005013494066142892, 0.012139026669098621, 0.012866122345843303, 0.007499467016849286, -0.009978515583874869, 0.026964842728873346, 0.02107883534075303, -0.013884055175698714, -0.028335937752902503, -0.007025124382970874, 0.01324005675973686, -0.005948331032199404, 0.0035229492326679635, 0.009528409224201916, 0.02444424599711766, 0.012028230942269931, -0.01038014939513691, 0.0045460758159452915, -0.011044921893463814, 0.02281001321734626, 0.02253302483159715, 0.003503906246223114, -0.00351429328740564, 0.0139533027377973, -0.030995027746210123, 0.009936967419144765, 0.028474431014454436, -0.005733664893545453, -0.04157599264999412, 0.0008461137017406887, -0.00864896965589844, -0.023308593056752747, -0.011488103869455958, 0.0072986505768795765, -0.005716353003020806, 0.005065429737716833, 0.013897904874382954, -0.015566761435203648, -0.0035021749174722564, -0.017879615480663576, -0.022491476666867048, 0.02243607973477532, -0.018073507536952475, -0.01713174665287646, -0.009452237278422518, -0.009479935744468381, 0.0012637606741335765, -0.020289416485590572, 0.0037324218507306514, -0.014943537101606848, -0.03634090899283909, -0.039553975757644924, 0.012575284261409955, -0.0010361106175861405, 0.019804687276190943, 0.02473508501287363, -0.0001494223761551342, 0.022699217490517568, -0.0026036930612158954, -0.006169921554534166, -0.014929687402922606, 0.01058789021878743, -0.03974786967657906, 0.005255859136504016, 0.0071740056170279546, -0.03232457274286393, -0.03880610879250305, -0.015829901053591135, -0.016439274757181075, -0.0009426269559050875, 0.004913085846158037, -0.004771128995781768, -0.0069004794231192525, 0.016370029057727727, 0.008925958972970166, 0.015705254696755582, 0.0072640267958302834, 0.016453125387187937, -0.02282386384735312, 0.009306817770544533, -0.00803959502098588, -0.030330255247883218, -0.012201348916193778, 0.027546518897740045, -0.0006916059984893041, 1.0332974448346307e-05, -0.0008309659139469504, 0.005699041112496159, -9.110329565140472e-05, 0.010477095423281358, -0.004390269732546091, 0.01181356480361598, 0.003429465396363919, 0.002293812223168353, -0.004352183759656392, -0.013350852486565552, 0.020594104268708162, 0.025524502005390845, -0.038750709997766086, -0.004414506472412858, -0.028668323070743334, 0.017173294817606564, 0.003760120782437824, -0.013163884813957463, 0.022131391951657732, -0.004833451708538235, -0.014008699669889026, -0.02994247113530542, 0.019915483003019634, -0.01436186093274015, 0.04376420220126374, -0.008323508721738418, -0.010096235694384371, -0.025621448964857914, -0.012893820811889166, -0.01966619215199377, -0.016647017443476835, -0.04127130486687653, 0.002234951935082947, 0.0039886363869453614, 0.0021847478250905197, 0.004421430856093669, -0.019915483003019634, 0.037559660125309445, -0.03539914810876307, -0.02243607973477532, 0.01822585142851127, -0.0006249556041399951, 0.03337713121641388, -0.010200106106209632, 0.01743643443599405, -0.04282244131718772, -0.019486150725711734, 0.014514203892976325, -0.01659161864873987, 7.465709042056602e-06, -0.012789950400063905, 0.030413351577343425, 0.013018465306079478, 0.03321093855749346, -0.009202947358719274, -0.006613103530526309, 0.011294211813167058, -0.01274147692033037, -0.00226611329146118, 0.003483132163858062, -0.0015926846150412963, -0.011529652034186062, -0.0014005237710879275, 0.01242294036985116, 0.01263760650850511, 0.02578764162377833, 0.00453222658292236, 0.026410865957375133, 0.038279829555728076, -0.018115055701682578, 0.006228782075450226, -0.01324698114341767, -0.009888493939411231, -0.014998934033698572, 0.004224076607964365, 0.001308771263534741, 0.005768288208933436, -0.018170454496419543, -0.001738969205218072, 0.008815163246141476, 0.0024011452924953282, -0.0067515977234008625, 0.00743714476975413, 0.016259233330899035, -0.010712535644300362, -0.005512073905549381, 0.0015165127856772264, 0.029249999239610032, -0.0429609364413849, -0.017353338106533844, -0.02096803961392434, 0.0033792612863714918, 0.02994247113530542, 0.006453835255286704, 0.004005948277470008, 0.014306463069325804, -0.005868696428918291, -0.00017106212289561508, 0.02362712960723196, 0.006744673339720052, -0.007402520988704837, -0.02474893378023525, -0.021231179232311825, 0.00918217327635422, -0.0012005726462476641, -0.0019960492893928835, 0.02024786832086047, -0.01303924031976715, 0.004878462065108744, -0.014278763671957321, 0.02304545343836526, -0.01933380683415294, -0.030718039360461014, 0.007450994002777061, 0.018682883103187654, 0.0336264202047945, -0.0028737570632841923, 0.009313743085547964, 0.03044105097471191, -0.0150820312944814, 0.0006760253785078514, 0.016023793109880034, 0.009549183306566967, -0.003344637738152853, 0.03185369230082593, 0.002406338813086591, -0.014680397483219362, -0.007104758520590677, 0.013205432978687566, 0.0258845885832454, -0.02024786832086047, -0.028058949367153394, 0.028031249969784913, 0.01069868594561612, 0.007603337894335856, -0.017574929560191226, -0.013994850902527403, -0.020607954898715024, -0.00875976538272713, -0.01149502825313677, 0.012616832426140059, -0.007908025211792137, -0.0037254970013885307, 0.010103160078065183, 0.03260156299125828, 0.005553622070279486, 0.007811079649308997, -0.02822514202607381, -0.00407865756574769, 0.0015658513477095524, -0.021840552935901765, -0.0018246626442198941, 0.011834338885981031, -0.006370738460165185, 0.036091620004458466, -0.0023007368396798186, 0.01974929034409922, -0.01069176156193531, 0.00014909778111818025, 0.014708095949265225, 0.009189097660035032, -0.01772727158910478, 0.018765979432647864, -0.022643820558425842, 0.0007703746463566694, -0.0028304775698032303, -0.012083628805684277, 0.005377041904515233, -0.04196377676257192, -0.002636585513514331, 0.002743918582841307, -0.009763849445220919, 0.006467684488309635, 0.018350497785346823, 0.028100497531883498, -0.013295454623151206, 0.01846129164953027, -0.011737392857836582, -0.02024786832086047, 0.0023890269225619446, -0.010809481672444811, 0.029776278476385004, -0.009022905001114617, 0.004445667595960437, 0.0052800958763707835, 0.006336115144777202, -0.008399680667517815, -0.00652308235172398, -0.03226917394812697, 0.008607421491168336, 0.023959516787718033, -0.03271235685544173, 0.021189631067581722, 0.0016878994308502156, 0.0005453213501048648, -0.004314097786766694, 0.014431107563516116, 0.01976313911146084, -0.002695445568769082, 0.01190358644807962, -0.029166902910149825, -0.008586647408803282, -0.007610262743677977, 0.0016177867864528375, 0.01058789021878743, 0.010303977449357512, -0.0012126908997657203, 0.019403054396251524, -0.02578764162377833, -0.00585484719589536, -0.019818536043552566, -0.015719105326762444, 0.011044921893463814, -0.011730468474155771, 0.00966690341707647, 0.018959871488936762, -0.012512961082992179, -0.006318803254252555, -0.011723544090474962, 0.006052201444024664, -0.013420099117341518, -0.03365411773951774, 0.00048429730853453317, -0.0012135564477258217, 0.005889470976944654, -0.015123579459211505, -0.02222833704847956, -0.027227982347260832, -0.00021455798881468623, -0.011335759977897163, 0.00555015987843908, -0.012769176317698853, 0.011314985895532112, 0.012568358946406524, 0.01733948747652698, 0.013593217091265365, 0.015123579459211505, -0.016937854596587563, -0.024582741121314835, -0.003850141961240153, -0.0029239611732766197, -0.022352983405315115, 0.0017190606708131213, -0.02681249883731455, 0.0588323837970609, 0.008690518751951164, -0.006235706459131038, -0.008635120888536818, -0.02364098023723882, -0.029803976011108246, -0.016231533933530554, 0.006481534186993877, 0.013877130792017902, 0.03013636319159432, 0.011280362114482818, -0.011121093839243212, 0.015012783732382814, -0.00224360788034527, 0.004930397736682684, -0.02364098023723882, -0.024388849065025933, -0.0011512339678000105, -0.03647940411703626, 0.0048992366131351064, 0.01650852231927966, -0.02800355057241643, -0.0018419744183292134, 0.005238547711640679, -0.008905184890605114, 0.003062455598981828, 0.01840589471743855, -0.005830610921689902, -0.024679686218136664, 0.008628195573533387, 0.025621448964857914, 0.010477095423281358, -0.010054687529654268, -0.003420809683932251, -0.011086470058193918, -0.0026469725546968575, 0.0060937500744160785, -0.010629438383517533, -0.021702059674349832, -0.0006907403923215391, -6.145685193017212e-05, 0.01843359411480703, 5.642561430547015e-05, -0.0007625843072621111, -0.009618429937342934, 0.009583807087616261, -0.012305220259341658, 0.021826704168540143, 0.014624999619805016, 0.011730468474155771, 0.015539062037835165, -0.006654651695256413, 0.0012161532080214531, -0.01598224494514993, 0.003451970807479829, 0.019776987878822462, -0.0026867896235067586, 0.03395880552263533, -0.021674360276981348, -0.009507635141836864, -0.012776100701379665, 0.0056436432490818145, -0.017187145447613426, -0.0007383478002259987, -0.02660475801366403, -0.01018625733884801, -0.025510653238029226, 0.01486043984082402, 0.00927219492081786, -0.01751953076545426, 0.00029754637216245264, -0.009078302864528962, 0.002922230077356417, -0.0015398836283379098, -0.02311470100046385, -0.013572443008900313, -0.01162659806233051, 0.026230822668447853, 0.01660546927874673, -0.0069628016702144085, 0.005937943991016878, 0.013378550952611413, 0.018087358166959337, -0.030662642428369292, 0.005072354121397643, 0.1859147639572333, -0.0305518467015406, 0.02089879205182575, 0.026646306178394134, -0.0004955499558848243, -0.006117986348621536, 0.02474893378023525, 0.01355859331021607, -0.006294566514385788, -0.015262073652086058, 0.01692400582922594, -0.009812321993631834, -0.0017744584178121393, 0.006353426569640538, 0.0075825638119708045, -0.006897016765617537, -0.04766974458705544, -0.03886150386194954, -0.005681729221971513, -0.00494078477786521, -0.0142649149045957, -0.003429465396363919, -0.003573153342660391, -0.02190980049800035, 0.028349786520264122, 0.015829901053591135, -9.16983817018483e-05, 0.006277255089522451, 0.035620739562420456, 0.0006154341109175705, -0.020732599392905334, -0.005106977902446936, 0.0074579188521191825, 0.021951348662730456, -0.023308593056752747, -0.006948952437191477, 0.01771342282174316, -0.0020549093446476344, 0.019264559272054352, -0.00448721576069054, 0.013156960430276652, -0.016342329660359246, 0.013108486950543117, -0.04448437535697285, 0.006685813284465302, 0.019001419653666866, -0.014133345095401957, -0.01039399816249853, -0.011695844693106479, -0.0031386275447612254, -0.021868252333270246, 0.013122336649227358, 0.04401349491493484, 0.02160511271488276, 0.0037912819059854022, 0.01689630643185746, -0.0010741965904758391, 0.011017222496095332, -0.018516690444267236, -0.0009659978567734349, -0.013773259448870022, -0.0042448506903294166, 0.00021271861208797525, 0.010553266437738136, -0.02026171895086733, 0.023959516787718033, -0.007025124382970874, -0.005775213058275557, -0.006737748490377932, -0.03514985912038245, 0.011058771592148057, -0.014638849318489256, -0.012886896428208356, -0.00041634853403740835, -0.010047762214650837, -0.04066193302593183, 0.019915483003019634, 0.02699254212624183, 0.026355469025283407, 0.03872301246304284, -0.006931640546666831, -0.014846591073462397, -0.012686079056916025, -0.01436186093274015, -0.018876775159476552, -0.028696022468111818, 0.012512961082992179, -0.018516690444267236, -0.004030184551675466, -0.0037739700154607557, -0.005702503304336565, -0.012803799167425528, -0.019139914777864038, 0.007631036826043029, 0.0029897460778734912, 0.006647727311575603, 0.02557990080012781, -0.014334161535371667, 0.0006552512379351356, -0.037005679628520756, -0.021466619453330827, 0.09024289691586254, 0.021563564550152656, 0.017935014275400542, 0.013551668926535261, -0.002181285400419459, -0.0016506791223359458, 0.002783735651651208, 0.012762251002695423, 0.016051490644603275, -0.003739346467242117, -0.04229616580570323, 0.023848721060889342, -0.010352449997768427, -0.012693004371919456, 0.021258876767035067, -0.001049094419064298, -0.0036735617954759005, 0.015912997383051342, -0.006845081559704907, 0.0004639559710456999, -0.005248934752823205, 0.030496447906803635, 0.016120738206701863, -0.0031040037637119324, -0.00498579560009703, -0.03251846666179807, 0.003985173729443646, -0.027061789688340415, 0.002833939994474291, 0.004289861512561236, -0.030801135689921225, 0.02783735791349601, 0.013281604924466964, 0.015829901053591135, -0.022339132775308253, 0.005840997962872428, -0.004826526859196114, -0.007492542633168476, -0.01121111455238423, 0.004781516036964294, -0.0026469725546968575, 0.004511452500557308, -0.004196377676257192, 0.0004267356043237663, -0.002413263429598057, 0.01345472289839081, 0.012436789137212781, -0.003377530190451289, 0.006197620486241338, -0.0005955255765126197, -0.014929687402922606, -0.00017181950646453562, -0.007575638962628683, 0.0289176139217692, 0.004906160996815917, -0.006374201117666901, 0.00028174936519097455, 0.0009002130685918909, 0.02290696017681333, -0.044567471686433054, 0.013551668926535261, 0.024070312514546725, 0.0009850408432182842, 0.011668146227060616, -8.980490822763101e-05, -0.1783806768846252, -0.0009763849561636246, -0.000950417236791982, -0.03944318189346147, 0.02545525444329226, 0.006128373389804062, 0.01840589471743855, -0.002397682867824268, -0.02618927450371775, 0.00977769914390516, 0.019680042782000633, 0.004127130579819915, -0.021771307236448417, -0.01740873503862557, -0.011176491702657557, 0.0025742630335885202, -0.01785191794594033, 0.004400656773728617, 0.009085227248209772, 0.007257101946488162, 0.03185369230082593, -0.05678266936998846, 0.0027421874869211042, 0.0007824928998747256, 0.0031542081065350147, -0.0041652165527096136, -0.012748402235333802, 0.015428267242329095, -0.01853053921162886, -0.025330609949101947, -0.00957688177261283, 0.003015713680829806, 0.019610795219902045, 0.00641921147423741, 0.02181285540117852, 0.016037641877241656, 0.003110928613054053, 0.0039886363869453614, -0.02740802563618811, 0.01659161864873987, 0.00978462352758597, 0.04332102301923945, 8.504416889237663e-05, 0.009507635141836864, -0.01375248536650497, 0.0289176139217692, 0.021134232272844757, -0.024167257611368554, -0.005117364943629462, -0.008738991300362079, 0.007894176444430514, -0.02646626288946686, 0.011280362114482818, -0.001513916025381595, -0.012616832426140059, 0.013981001203843163, 0.017754970986473263, 0.004317560444268409, 0.0005790793503634018, -0.009452237278422518, 0.006921253505484304, -0.02801739933977805, 0.03229687520814069, 0.0025344459647786187, -0.004930397736682684, -0.03002556746476563, -0.013011540922398668, 0.005920632100492231, -0.02160511271488276, 0.014105645698033475, -0.023544033277771752, 0.00783877858101617, 0.0009651322506056697, -0.010130859475433665, 0.018627486171095928, 0.019098366613133935, 0.0075894886613129256, 0.001817737911293101, 0.011024147811098763, 0.012665304974550973, 0.0012568359412067835, 0.004674182967637319, -6.454051205719251e-05, 0.0009798473226270213, -0.027892754845587737, -0.016660866210838454, -0.004719193789869138, 0.01803195937222237, -0.00743021992041201, -0.012132102285417811, 0.03836292588518829, -0.008212712994909726, -0.01772727158910478, -0.007326349508586749, 0.002801047542175855, 0.0009988903090718706, 0.032795453184901936, -0.0038362924953865667, -0.006654651695256413, -0.002120694249244506, -0.0076171875930200975, -0.0003174549356712352, -0.007025124382970874, 0.007450994002777061, 0.020829544489727164, -0.0023890269225619446, -0.01638387782508935, 0.0009824440829226528, 0.03947087942818472, -0.022768465052616156, -0.009999289666239922, -0.009923117720460525, 0.01772727158910478, -0.005754438975910505, -0.006765447422085104, 0.022796164449984637, 0.005030805956667539, -0.009756924130217488, -0.0005223832232165681, 0.014528053591660567, 0.0655355113003574, -0.016134586974063485, -0.0067585225727429836, 0.014694247181903602, 0.0001894558901627247, -0.02578764162377833, -0.1399346509446862, -0.02555220140275933, -0.0009028098288875224, 0.02199289682746056, 0.00402672235983506, 0.03947087942818472, -0.0040682705245651644, 0.014749645045317948, -0.017658025889651433, 0.043293321759225734, -0.01436186093274015, -0.0008880948150738347, 0.009382989716323932, -0.014680397483219362, 0.02595383428269875, -0.009036753768476237, 0.028889914524400716, -0.01823970205851813, -0.010726384411661983, 0.04387499979073767, 0.002098188838128596, -0.015552911736519407, 0.0032234552029722914, -0.02985937480584521, 0.0021103072080619796, -0.0021051136874707167, -0.03201988495974634, 0.026964842728873346, 0.011841264200984462, -0.004975408093253193, 0.015829901053591135, -0.022643820558425842, -0.005761363359591315, -0.00866281935458268, -0.0009296430962192662, -0.003007057735567483, -0.01703480155605463, -0.03481747007725114, 0.016730113772937042, -0.0002152071788885941, -0.00815731513149538, 0.007956498691525671, 0.0037878194813143417, -0.006436523364762057, 0.005730202236043738, -0.023142400397832334, -0.017630326492282952, 0.017367186873895466, -0.00016597678124744878, -0.023779473498790754, -0.02941619189853045, -0.03498366273617155, -0.01843359411480703, 0.007180930466370075, 0.017990411207492268, 0.030330255247883218, 0.003121315654236579, 0.010677911863251068, -0.017491831368085776, 0.013433948816025758, -0.022491476666867048, 0.00033671430910610996, -0.020926491449194236, 0.0036181639320615553, 0.016536221716648144, 0.00896058182269684, -0.018198153893788028, -0.027740410954028943, 0.01029012775067327, -0.013766335065189211, -0.004978870750754909, 0.004681107816979439, -0.01221519861487802, 0.0035281427532592264, -0.03481747007725114, 0.003739346467242117, -0.014721945647949465, -0.0307457387578295, 0.004099431648112742, -0.01833664715533996, -0.011910510831760429, -0.0036181639320615553, -0.003940163372873137, -0.030385652179974944, 0.0016974209240726403, 0.0006483265050083424, 0.014832741374778156, -0.01610688943934024, -0.020053976264571567, -0.009362215633958879, -0.01017933202384458, 0.03249076540178435, -0.004303710745584168, -0.026133877571626024, -0.015539062037835165, 0.01782421854857185, 0.00044318180136915225, -0.009244495523449378, 0.013974076820162351, -0.012464488534581264, -0.019790838508829324, 0.0023267044426361332, -0.034762075007804656, 0.01883522699474645, -0.022726916887886053, -0.025593749567489433, -0.003992098578785767, 0.00022656804883772975, 0.016231533933530554, -0.019209162339962626, -0.0038709160436052052, 0.023280893659384266, -0.019292258669422833, 0.012305220259341658, -0.01070561032929693, -0.01140500660867313, -0.031022727143578607, -0.02322549672729254, 0.027435723170911353, -0.013316228705516258, 0.0150820312944814, 0.020317115882959053, 0.011675070610741425, -0.00013016301727217542, -0.008538174860392368, 0.016023793109880034, -0.004781516036964294, -0.020040127497209948, -0.020497159171886332, 0.02433345213293421, 0.00713245745229785, -0.005986417005089103, 0.00043885385784182243, -0.00605566410152638, -0.015912997383051342, -0.021951348662730456, -0.015954545547781446, -0.011335759977897163, -0.0003501309427719819, 0.025607598334851055, 0.0016151899097418786, 0.06138067992618507, 0.001983931152290155, -0.029776278476385004, 0.007506391866191407, -0.03282315444491566, -0.013281604924466964, -0.0008738125752401977, -0.019652343384632148, 0.0046534088852722665, 0.01110031975687816, -0.011453480088406664, 0.035870028550801084, 0.014735795346633707, -0.04755894699758151, -0.021245027999673444, -0.013205432978687566, -0.010303977449357512, 0.03188138983554917, 0.010373224080133478, 0.00025989323146046833, -0.0013840775449387097, 0.01121111455238423, 0.0014065829560546193, 0.010677911863251068, 0.0011304598854349585, 0.012568358946406524, 0.007894176444430514, -0.02985937480584521, -0.01578835288886103, 0.004210227374941433, 0.0012776100235718354, -0.012055930339638414, 0.012810724482428957, 0.016231533933530554, 0.01222212299855883, 0.004151367319686683, 0.006890092381936727, 0.007066672547700979, 0.028945311456492442, -0.002517134307084627, 0.013690163119409814, 0.005428977110427863, -0.01789346611067044, -0.02329474428939113, 0.0041479046621849675, 0.0035229492326679635, 0.01466654778453512, -0.01840589471743855, -0.00804651940466669, -0.010117009776749423, -0.00361470174022115, 0.02639701719001351, 0.014167968876451251, -0.006014115471134966, 0.022754616285254534, -0.016577769881378247, -0.007811079649308997, -0.020344815280327538, 0.010310901833038323, 0.005626331824218477, 0.015165127623941609, 0.017768821616480125, -0.018558238608997343, 0.008226562693593968, -0.03462357988360748, -0.037005679628520756, 0.0005167569286452545, -0.043237926689779244, -0.020594104268708162, 0.020053976264571567, 0.016951703363949185, -0.010657137780886016, 0.002461736443670282, 0.020330966512965915, 0.017367186873895466, -0.016937854596587563, 0.03334942995640015, -0.014791193210048053, -0.0020895328928662725, -0.030662642428369292, 0.008503551079343075, 0.02974857907901652, 0.012852272647159062, 0.037725852784229866, 0.008025745322301638, 0.01943075193097477, 0.007596413044993736, 0.05406818058194387, -0.046921873896623084, 0.02783735791349601, 0.008697443135631974, 0.014417257864831876, 0.014985085266336951, -0.003860529002422679, -0.015843749820952754, -0.013309303390512827, -0.005338955931625534, 0.00252232782767589, -0.005120827135469868, -0.0025984995406246325, 0.05694886202890887, 0.03221377887868048, -0.0026036930612158954, 0.01140500660867313, -0.005640181057241409, -0.009445311963419087, 0.010837180138490674, 0.019375354998883043, -0.014749645045317948, -0.03013636319159432, 0.01974929034409922, 0.0062634053908382096, 0.018142755099051062, -0.024984374001254254, -0.011169566387654126, 0.00315767029837542, -0.016217685166168932, 0.04686647510188612, -0.022879260779444847, -0.013572443008900313, 0.04969176147940464, -0.0071740056170279546, 0.03143821065352489, 0.008690518751951164, -0.014541903290344807, 0.005518998289230192, 0.035565340767683494, 0.009992365282559111, -0.02455504172394635, -0.009431463196057466, 0.0030122514889894007, 0.0332386360922167, -0.02545525444329226, -0.014514203892976325, -0.007146306685320782, -0.006737748490377932, 0.008316583406734987, -0.023267044892022644, 0.02120347983494334, 0.005487837165682614, -0.0038986149753123776, 0.02534446057910881, 0.005969105114564456, -0.029499290090635896, -0.0015433460530089702, -0.008628195573533387, -0.0014169699972371454, 0.001602206108263721, -0.015885297985682858], 'source': 'https://arxiv.org/pdf/2303.08774.pdf', 'page': 50})]}\n"]}]},{"cell_type":"code","source":["queries_df = get_qa_with_reference(px.Client())\n","retrieved_documents_df = get_retrieved_documents(px.Client())"],"metadata":{"id":"wlBbSyHyQ1Fy","executionInfo":{"status":"ok","timestamp":1712775823872,"user_tz":240,"elapsed":186,"user":{"displayName":"John Willis","userId":"15271974867993570949"}}},"execution_count":21,"outputs":[]},{"cell_type":"code","source":["eval_model = OpenAIModel(\n","    model_name=\"gpt-4-turbo-preview\",\n",")\n","hallucination_evaluator = HallucinationEvaluator(eval_model)\n","qa_correctness_evaluator = QAEvaluator(eval_model)\n","relevance_evaluator = RelevanceEvaluator(eval_model)\n","\n","hallucination_eval_df, qa_correctness_eval_df = run_evals(\n","    dataframe=queries_df,\n","    evaluators=[hallucination_evaluator, qa_correctness_evaluator],\n","    provide_explanation=True,\n",")\n","relevance_eval_df = run_evals(\n","    dataframe=retrieved_documents_df,\n","    evaluators=[relevance_evaluator],\n","    provide_explanation=True,\n",")[0]\n","\n","px.Client().log_evaluations(\n","    SpanEvaluations(eval_name=\"Hallucination\", dataframe=hallucination_eval_df),\n","    SpanEvaluations(eval_name=\"QA Correctness\", dataframe=qa_correctness_eval_df),\n","    DocumentEvaluations(eval_name=\"Relevance\", dataframe=relevance_eval_df),\n",")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":99,"referenced_widgets":["80bd747ce9ec472288d73c8fd0986b23","0cdb0f3fc0bd41309f0df20fb14f48f0","f7467fdc4ff444a89c764512cdc48262","93c051bdb4cd4b65a201363f6334cb41","3cf73d3f9a544ee1a2c4b7e097f0ccfb","9944b5224bac4654838d60c11d02dfa5","40ea864c6574458b82c780f7bd6b84da","cb27a7c41ac14aeca5ae69666290f170","02995a3e82dc41af8fbf073b315b87e9","f37566ea00094393aeca205c182e1c70","bcb91fa935e34e279af633456263ebd5","29e4236e6d7445feab525f065b4b048d","fd2b8688f41d4be2ad3280a6e58bc5e9","3e7cd6f103c74b5ba56925b6c87b5acb","435f4cce46c24c0ab0997260e99f6e2d","fbfcc54ffe254e6886f97564d3287304","8126225ebd3a41b2a84ab18497a42992","41e0e748ebeb4293a6f6c5e2b1929dff","f583b40533cd49b8a541bfdbaf7b0bd6","2e61666fba864740abbd3272d9046660","c200bade8ca34e1faba6dca8de317db8","052873ae7348481e9b80c223c65b4f26"]},"id":"Tr_nRl46RCg5","executionInfo":{"status":"ok","timestamp":1712775858491,"user_tz":240,"elapsed":31660,"user":{"displayName":"John Willis","userId":"15271974867993570949"}},"outputId":"713d0040-bfa2-4fe8-dc94-fe19e9554dd3"},"execution_count":22,"outputs":[{"output_type":"stream","name":"stdout","text":["The `model_name` field is deprecated. Use `model` instead.                 This will be removed in a future release.\n"]},{"output_type":"display_data","data":{"text/plain":["run_evals |          | 0/2 (0.0%) | ⏳ 00:00<? | ?it/s"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"80bd747ce9ec472288d73c8fd0986b23"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["run_evals |          | 0/4 (0.0%) | ⏳ 00:00<? | ?it/s"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"29e4236e6d7445feab525f065b4b048d"}},"metadata":{}}]}]}