LAB2.1_Explainability.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Explainability Lab.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "ad906fc1d7e74cae9357c30c779aab42": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_1f749a46d1794e04bb336dc7da273a8f",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_287f090ca73e425d8a0dab05bb04aec1",
              "IPY_MODEL_65c59bf033d94e06a58364dff88be987"
            ]
          }
        },
        "1f749a46d1794e04bb336dc7da273a8f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "287f090ca73e425d8a0dab05bb04aec1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_5a81bcd34be64c968c166d0eae814928",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_487bed7eba1c4b16ada6a934ce7d3700"
          }
        },
        "65c59bf033d94e06a58364dff88be987": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_8e7f6cfaf07149bea8a6a20ce4e29b24",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 232k/232k [00:01&lt;00:00, 218kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_74d96d9eaed44dc7b1fb722a90045f3c"
          }
        },
        "5a81bcd34be64c968c166d0eae814928": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "487bed7eba1c4b16ada6a934ce7d3700": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8e7f6cfaf07149bea8a6a20ce4e29b24": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "74d96d9eaed44dc7b1fb722a90045f3c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0d4a3a14c45e413dac84ea1b9cd7f34d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_0ef2c01097fc4ed99e50d34f1e0b5c90",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_7354ff1e0fcb498388d320a10f6284e8",
              "IPY_MODEL_72d27b6301da4cac90d98dd8b939a0f5"
            ]
          }
        },
        "0ef2c01097fc4ed99e50d34f1e0b5c90": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7354ff1e0fcb498388d320a10f6284e8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_95eff4722e7445e8bcf9adc92525bc8f",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 466062,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 466062,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a498de796f6342548898365b469d2996"
          }
        },
        "72d27b6301da4cac90d98dd8b939a0f5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d38b3289ab314590b69646a7decaf9ec",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 466k/466k [00:00&lt;00:00, 1.22MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_77fa6c2180f447beba55b6dd5a4953c4"
          }
        },
        "95eff4722e7445e8bcf9adc92525bc8f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a498de796f6342548898365b469d2996": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d38b3289ab314590b69646a7decaf9ec": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "77fa6c2180f447beba55b6dd5a4953c4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "261496b5302b44f6b9b662490e4f2b55": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_48ff09d268cd49f98d96d2785c39c3ee",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_548fb78ab387403a970d5312c5e36358",
              "IPY_MODEL_06234dd7145b4c398dfa048a0d1937e6"
            ]
          }
        },
        "48ff09d268cd49f98d96d2785c39c3ee": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "548fb78ab387403a970d5312c5e36358": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_1b13d31979c04480baf250834fd45ed7",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 433,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 433,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_184c74a697ce4f719498cf2794bf8e76"
          }
        },
        "06234dd7145b4c398dfa048a0d1937e6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_58f3523b9f3a4ec28868a75c08b7d2fb",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 433/433 [00:23&lt;00:00, 18.8B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_aec6d3a6d64b45f8a551ea780736512d"
          }
        },
        "1b13d31979c04480baf250834fd45ed7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "184c74a697ce4f719498cf2794bf8e76": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "58f3523b9f3a4ec28868a75c08b7d2fb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "aec6d3a6d64b45f8a551ea780736512d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "23d18a55ccfc4a41bdafe5002953a891": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_7b975f237d354750b1ad4db967c9196d",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_e743075f0129463a974a019d2f8abf2f",
              "IPY_MODEL_b9262859fe79453dba9325e00c73abc9"
            ]
          }
        },
        "7b975f237d354750b1ad4db967c9196d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e743075f0129463a974a019d2f8abf2f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_5c41d5c1185045b6ba252fe1191552d5",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 440473133,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 440473133,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_bc08252a1d7f4c6bbd6a5fd891ca69d0"
          }
        },
        "b9262859fe79453dba9325e00c73abc9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_3f947ebeac584c18bef7e8f3b9cf876a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 440M/440M [00:22&lt;00:00, 20.0MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_cab8e701f6834ba0adefb379d81b0b64"
          }
        },
        "5c41d5c1185045b6ba252fe1191552d5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "bc08252a1d7f4c6bbd6a5fd891ca69d0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3f947ebeac584c18bef7e8f3b9cf876a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "cab8e701f6834ba0adefb379d81b0b64": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "37467989caf54aa89936e7c8ca9584e1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_bb1bc3cdfa2d4a488fad9227ef4df011",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_83a508ad828242e68844344077560b21",
              "IPY_MODEL_291ef6aedc6d40b68733dfa231cc928d"
            ]
          }
        },
        "bb1bc3cdfa2d4a488fad9227ef4df011": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "83a508ad828242e68844344077560b21": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_d89760a24d7244278856ac405319def1",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 13,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 13,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4c40a712ce874a0a9cace124e048b5fe"
          }
        },
        "291ef6aedc6d40b68733dfa231cc928d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_01f462b832dc4057b7000fd99faa6699",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 13/13 [00:04&lt;00:00,  3.15it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9e1f76ce613a42bab787a7828ec1e67e"
          }
        },
        "d89760a24d7244278856ac405319def1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4c40a712ce874a0a9cace124e048b5fe": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "01f462b832dc4057b7000fd99faa6699": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9e1f76ce613a42bab787a7828ec1e67e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3abea2fbddff483fb7d340106ce5b79e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_8d5e9e04823c4472b7712a510b1ebd54",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_f56c6a0905274da0ad94437dc5a6721b",
              "IPY_MODEL_02a4c5fd5a664f7aade917d7d66a1578"
            ]
          }
        },
        "8d5e9e04823c4472b7712a510b1ebd54": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f56c6a0905274da0ad94437dc5a6721b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a5318d44129546f296d5d036bb952c15",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 13,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 13,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9b854947f2fa42ff83c2f8b1a14a8dba"
          }
        },
        "02a4c5fd5a664f7aade917d7d66a1578": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_54f50efa178c45da9f66803b2e73c9d1",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 13/13 [00:01&lt;00:00, 12.56it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a7f923bb32f74ab09b981ec4708bbfc6"
          }
        },
        "a5318d44129546f296d5d036bb952c15": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9b854947f2fa42ff83c2f8b1a14a8dba": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "54f50efa178c45da9f66803b2e73c9d1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a7f923bb32f74ab09b981ec4708bbfc6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8d8158cf9fa84af2a4ac632110e2f883": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_3cb04339779840c9931e966dc75ef277",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_5e65464d57a04687ba6e69a57cc8b5fb",
              "IPY_MODEL_7cb94b9b0cc5434eacf005f1af80f76f"
            ]
          }
        },
        "3cb04339779840c9931e966dc75ef277": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5e65464d57a04687ba6e69a57cc8b5fb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_5e56debe576a4ab287d8ec88d5a3cae9",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 13,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 13,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_52bf7c646aac4afc91d0380d71e2ea48"
          }
        },
        "7cb94b9b0cc5434eacf005f1af80f76f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_8027ceca8e0240b998ac585fca7074e6",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 13/13 [00:00&lt;00:00, 13.31it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_00cb837ec6244e08a6abe05930632300"
          }
        },
        "5e56debe576a4ab287d8ec88d5a3cae9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "52bf7c646aac4afc91d0380d71e2ea48": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8027ceca8e0240b998ac585fca7074e6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "00cb837ec6244e08a6abe05930632300": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5oDLx_YTmbTG"
      },
      "source": [
        "# Welcome \n",
        "\n",
        "Welcome to this lab on explainability at the winter school [ALPS 2021](http://lig-alps.imag.fr/)!"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "51j0S8W1Rb1u"
      },
      "source": [
        "In this tutorial, we will lay the foundations of post-hoc explainability techniques and ways of evaluating them.\n",
        "\n",
        "We've already prepared the modelling part for you in the **Modelling** section so that you can concentrate on exploring the explainability techniques.\n",
        "\n",
        "As you already know from the accompanying lecture, *post-hoc explainability techniqes* generate saliency maps over the input for an already trained model, thus providing information which parts were most important for the prediction.\n",
        "\n",
        "We'll review two types of explainability techniques in the **Explainability techniques** section and then we'll explore how those can be evaluated in the **Properties evaluation** section.\n",
        "\n",
        "You will have the chance to *define explainability techniques on your own* as well as to see *what parameters of a model affect the performance of the above*. You will also take part in a *human-in-the-loop experiment* where you'll be able to use the explainability techniques to detect blind spots of the trained models.\n",
        "\n",
        "For this notebook of the lab, we encourage you to work in groups, so that you could split the work and discuss the outcomes.\n",
        "\n",
        "We also provide a [notebook](https://colab.research.google.com/drive/1-aZ9-Kzkb_BVb-8vcvHBAAYy2iBk0khV?usp=sharing) with **solutions**, which we encourage you to consult after every task to make sure you're on track."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lJ7h2NYz5ckw"
      },
      "source": [
        "## Set-up\n",
        "\n",
        "This notebook can be run on Google Colab. One way to start working on the lab's notebook is to make your own copy of it (**File->Save a copy to Drive**) \n",
        "\n",
        "First, make sure, you've selected a GPU runtime from the menu: **Runtime -> Change Runtime Type**.\n",
        "\n",
        "This lab notebook is largely dependent on the external code from the explainability tutorial [repo](https://github.com/copenlu/ALPS_2021), which we'll download shortly."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P4jGTlAlbwdv"
      },
      "source": [
        "# Execute this line first as you might have to restart the runtime after this\n",
        "!pip install -U scikit-learn "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CN4hEPHvNmYs"
      },
      "source": [
        "# magic commands to make sure changes to external packages are automatically loaded and plots are displayed in the notebook\n",
        "# thus, if you make any change on the imported files with code, \n",
        "# you can upload(overwrite) them and re-import the package without restarting the kernel\n",
        "%reload_ext autoreload\n",
        "%autoreload 2\n",
        "%matplotlib inline"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Bj-4wEVFZHwU"
      },
      "source": [
        "import torch\n",
        "import os\n",
        "import torchtext\n",
        "import nltk\n",
        "import argparse\n",
        "import random\n",
        "import numpy as np\n",
        "\n",
        "from argparse import Namespace\n",
        "from tqdm.notebook import tqdm\n",
        "from torch.utils.data import DataLoader\n",
        "from functools import partial"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JRqOhXC7n0Ot",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "373953c6-af24-435e-d146-342ff35e1c35"
      },
      "source": [
        "!pip install transformers\n",
        "!pip install captum\n",
        "nltk.download('punkt')\n",
        "!git clone https://github.com/copenlu/ALPS_2021\n",
        "!export PYTHONPATH='ALPS_2021/tutorial_src/'"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: transformers in /usr/local/lib/python3.6/dist-packages (4.1.1)\n",
            "Requirement already satisfied: tokenizers==0.9.4 in /usr/local/lib/python3.6/dist-packages (from transformers) (0.9.4)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.8)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers) (20.8)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.19.5)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.6/dist-packages (from transformers) (0.0.43)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.41.1)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.12.5)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.0.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Collecting captum\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/bf/27/e6d97c600cabc38b860ead2f6be243d819ff3d259bc3195b7d2ed943ba5d/captum-0.3.0-py3-none-any.whl (5.7MB)\n",
            "\u001b[K     |████████████████████████████████| 5.7MB 6.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: matplotlib in /usr/local/lib/python3.6/dist-packages (from captum) (3.2.2)\n",
            "Requirement already satisfied: torch>=1.2 in /usr/local/lib/python3.6/dist-packages (from captum) (1.7.0+cu101)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from captum) (1.19.5)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->captum) (2.8.1)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib->captum) (0.10.0)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->captum) (2.4.7)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->captum) (1.3.1)\n",
            "Requirement already satisfied: dataclasses in /usr/local/lib/python3.6/dist-packages (from torch>=1.2->captum) (0.8)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.6/dist-packages (from torch>=1.2->captum) (3.7.4.3)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from torch>=1.2->captum) (0.16.0)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.6/dist-packages (from python-dateutil>=2.1->matplotlib->captum) (1.15.0)\n",
            "Installing collected packages: captum\n",
            "Successfully installed captum-0.3.0\n",
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n",
            "Cloning into 'ALPS_2021'...\n",
            "remote: Enumerating objects: 13, done.\u001b[K\n",
            "remote: Counting objects: 100% (13/13), done.\u001b[K\n",
            "remote: Compressing objects: 100% (8/8), done.\u001b[K\n",
            "remote: Total 50 (delta 6), reused 12 (delta 5), pack-reused 37\u001b[K\n",
            "Unpacking objects: 100% (50/50), done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8-_JyO_ebC19"
      },
      "source": [
        "from transformers import BertTokenizerFast\n",
        "from ALPS_2021.tutorial_src.data_loaders import TwitterDataset, get_embeddings, EmbeddingsVocabTokenizer, collate_tweet\n",
        "from ALPS_2021.tutorial_src.model_builders import get_model\n",
        "from ALPS_2021.tutorial_src.training_utils import enforce_reproducibility, train_model, eval_model\n",
        "from ALPS_2021.tutorial_src.args_utils import ALL_ARGUMENTS, get_model_args"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DTlzO3wIqddK"
      },
      "source": [
        "# Modeling"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bFeA6gCZ3gZP"
      },
      "source": [
        "## Arguments"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HkmZjwh-3Czn",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9d97c5da-7f00-4911-d927-a697d7b796e6"
      },
      "source": [
        "args = ALL_ARGUMENTS\n",
        "args['model'] = 'transformer' # cnn/rnn/transformer are possible models\n",
        "args.update(get_model_args(ALL_ARGUMENTS['model']))\n",
        "\n",
        "model_args = Namespace(**args)\n",
        "enforce_reproducibility(seed=model_args.seed)\n",
        "\n",
        "# if you don't want to train a model, but use one the pre-trained one, you can set the following:\n",
        "model_args.mode = 'train'\n",
        "# check the folder for other models\n",
        "model_args.epochs = 3\n",
        "model_args"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Namespace(batch_size=8, epochs=3, gpu=True, init_only=False, labels=3, lr=3e-05, mode='train', model='transformer', model_path='tweet_model', seed=73)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5WxepR9dxjZ9",
        "outputId": "7755e9ef-4a17-4fa9-dacf-4b8d925f4498"
      },
      "source": [
        "# in case you don't want to train a model, download one of the models and set the mode to test:\n",
        "# download pre-trained lstm model:\n",
        "#!wget --load-cookies /tmp/cookies.txt \"https://docs.google.com/uc?export=download&confirm=$(wget --quiet --save-cookies /tmp/cookies.txt --keep-session-cookies --no-check-certificate 'https://docs.google.com/uc?export=download&id=1KBc0v-Iin5CYWEcpOckUgrBwVW01BHhg&authuser' -O- | sed -rn 's/.*confirm=([0-9A-Za-z_]+).*/\\1\\n/p')&id=1KBc0v-Iin5CYWEcpOckUgrBwVW01BHhg&authuser\" -O tweet_model_lstm && rm -rf /tmp/cookies.txt\n",
        "\n",
        "# download pre-trained cnn model:\n",
        "#!wget --load-cookies /tmp/cookies.txt \"https://docs.google.com/uc?export=download&confirm=$(wget --quiet --save-cookies /tmp/cookies.txt --keep-session-cookies --no-check-certificate 'https://docs.google.com/uc?export=download&id=1jk2PzoULNwwLKPlyZ8RN8RtAj8zeHMTz&authuser' -O- | sed -rn 's/.*confirm=([0-9A-Za-z_]+).*/\\1\\n/p')&id=1jk2PzoULNwwLKPlyZ8RN8RtAj8zeHMTz&authuser\" -O tweet_model_cnn && rm -rf /tmp/cookies.txt\n",
        "\n",
        "# download pre-trained transformer model:\n",
        "!wget --load-cookies /tmp/cookies.txt \"https://docs.google.com/uc?export=download&confirm=$(wget --quiet --save-cookies /tmp/cookies.txt --keep-session-cookies --no-check-certificate 'https://docs.google.com/uc?export=download&id=1FZwDnsxgearvPo8XlqJ-zszfUGq_H8o_&authuser' -O- | sed -rn 's/.*confirm=([0-9A-Za-z_]+).*/\\1\\n/p')&id=1FZwDnsxgearvPo8XlqJ-zszfUGq_H8o_&authuser\" -O tweet_model_transformer && rm -rf /tmp/cookies.txt\n",
        "\n",
        "model_args.mode = 'test'\n",
        "model_args.model_path = 'tweet_model_transformer'"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2021-01-17 23:31:43--  https://docs.google.com/uc?export=download&confirm=I92t&id=1FZwDnsxgearvPo8XlqJ-zszfUGq_H8o_&authuser\n",
            "Resolving docs.google.com (docs.google.com)... 74.125.195.101, 74.125.195.102, 74.125.195.139, ...\n",
            "Connecting to docs.google.com (docs.google.com)|74.125.195.101|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Moved Temporarily\n",
            "Location: https://doc-0k-74-docs.googleusercontent.com/docs/securesc/d7hum75s3huioouvps2pqmpvk6iapj9n/rnv1nmmsi4k591n7j337rq76u91shjsc/1610926275000/09251033333050931776/09755432298029098590Z/1FZwDnsxgearvPo8XlqJ-zszfUGq_H8o_?e=download [following]\n",
            "--2021-01-17 23:31:43--  https://doc-0k-74-docs.googleusercontent.com/docs/securesc/d7hum75s3huioouvps2pqmpvk6iapj9n/rnv1nmmsi4k591n7j337rq76u91shjsc/1610926275000/09251033333050931776/09755432298029098590Z/1FZwDnsxgearvPo8XlqJ-zszfUGq_H8o_?e=download\n",
            "Resolving doc-0k-74-docs.googleusercontent.com (doc-0k-74-docs.googleusercontent.com)... 173.194.203.132, 2607:f8b0:400e:c05::84\n",
            "Connecting to doc-0k-74-docs.googleusercontent.com (doc-0k-74-docs.googleusercontent.com)|173.194.203.132|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://docs.google.com/nonceSigner?nonce=qfnonegjhkg30&continue=https://doc-0k-74-docs.googleusercontent.com/docs/securesc/d7hum75s3huioouvps2pqmpvk6iapj9n/rnv1nmmsi4k591n7j337rq76u91shjsc/1610926275000/09251033333050931776/09755432298029098590Z/1FZwDnsxgearvPo8XlqJ-zszfUGq_H8o_?e%3Ddownload&hash=7l5khu1hht1fs59ps17rh020lnujiu6j [following]\n",
            "--2021-01-17 23:31:43--  https://docs.google.com/nonceSigner?nonce=qfnonegjhkg30&continue=https://doc-0k-74-docs.googleusercontent.com/docs/securesc/d7hum75s3huioouvps2pqmpvk6iapj9n/rnv1nmmsi4k591n7j337rq76u91shjsc/1610926275000/09251033333050931776/09755432298029098590Z/1FZwDnsxgearvPo8XlqJ-zszfUGq_H8o_?e%3Ddownload&hash=7l5khu1hht1fs59ps17rh020lnujiu6j\n",
            "Connecting to docs.google.com (docs.google.com)|74.125.195.101|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://doc-0k-74-docs.googleusercontent.com/docs/securesc/d7hum75s3huioouvps2pqmpvk6iapj9n/rnv1nmmsi4k591n7j337rq76u91shjsc/1610926275000/09251033333050931776/09755432298029098590Z/1FZwDnsxgearvPo8XlqJ-zszfUGq_H8o_?e=download&nonce=qfnonegjhkg30&user=09755432298029098590Z&hash=k0nik53m9sjmat3fu2lktbjusb2b3skn [following]\n",
            "--2021-01-17 23:31:43--  https://doc-0k-74-docs.googleusercontent.com/docs/securesc/d7hum75s3huioouvps2pqmpvk6iapj9n/rnv1nmmsi4k591n7j337rq76u91shjsc/1610926275000/09251033333050931776/09755432298029098590Z/1FZwDnsxgearvPo8XlqJ-zszfUGq_H8o_?e=download&nonce=qfnonegjhkg30&user=09755432298029098590Z&hash=k0nik53m9sjmat3fu2lktbjusb2b3skn\n",
            "Connecting to doc-0k-74-docs.googleusercontent.com (doc-0k-74-docs.googleusercontent.com)|173.194.203.132|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: unspecified [application/x-zip]\n",
            "Saving to: ‘tweet_model_transformer’\n",
            "\n",
            "tweet_model_transfo     [        <=>         ] 417.74M   212MB/s    in 2.0s    \n",
            "\n",
            "2021-01-17 23:31:45 (212 MB/s) - ‘tweet_model_transformer’ saved [438031856]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "up61BlS78YkJ"
      },
      "source": [
        "The arguments contain some training-specific parameters as well as **model hyper-parameters**. Later, if you want to change a hyper-parameter of a model and train it again, you can just change it's parameters."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zHi4HtJK8pxX",
        "outputId": "3dfe4cc7-b847-419e-fcf8-188d9cf6d5d9"
      },
      "source": [
        "# reviewing the parameters for a model:\n",
        "get_model_args('cnn')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'activation': 'relu',\n",
              " 'batch_size': 64,\n",
              " 'dropout': 0.05,\n",
              " 'embedding_dim': 300,\n",
              " 'epochs': 3,\n",
              " 'in_channels': 1,\n",
              " 'kernel_heights': [2, 3, 4],\n",
              " 'lr': 0.001,\n",
              " 'out_channels': 50,\n",
              " 'padding': 0,\n",
              " 'pooling': 'max',\n",
              " 'stride': 1}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jR_Ub_PU_y9c"
      },
      "source": [
        "## Prepare Data\n",
        "\n",
        "This might take a while for the embeddings to be downloaded for the CNN and the LSTM models."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 114,
          "referenced_widgets": [
            "ad906fc1d7e74cae9357c30c779aab42",
            "1f749a46d1794e04bb336dc7da273a8f",
            "287f090ca73e425d8a0dab05bb04aec1",
            "65c59bf033d94e06a58364dff88be987",
            "5a81bcd34be64c968c166d0eae814928",
            "487bed7eba1c4b16ada6a934ce7d3700",
            "8e7f6cfaf07149bea8a6a20ce4e29b24",
            "74d96d9eaed44dc7b1fb722a90045f3c",
            "0d4a3a14c45e413dac84ea1b9cd7f34d",
            "0ef2c01097fc4ed99e50d34f1e0b5c90",
            "7354ff1e0fcb498388d320a10f6284e8",
            "72d27b6301da4cac90d98dd8b939a0f5",
            "95eff4722e7445e8bcf9adc92525bc8f",
            "a498de796f6342548898365b469d2996",
            "d38b3289ab314590b69646a7decaf9ec",
            "77fa6c2180f447beba55b6dd5a4953c4"
          ]
        },
        "id": "LvGaiy_fo311",
        "outputId": "8191735b-9a94-4275-dfd8-8d4a0ddbfcf6"
      },
      "source": [
        "device = torch.device(\"cuda\") \n",
        "\n",
        "if model_args.model == 'transformer':\n",
        "  tokenizer = BertTokenizerFast.from_pretrained('bert-base-uncased')\n",
        "  embeddings = None\n",
        "else:\n",
        "  embeddings, word_to_index = get_embeddings('glove', model_args.embedding_dim)\n",
        "  tokenizer = EmbeddingsVocabTokenizer(word_to_index, {v: k for k, v in word_to_index.items()})\n",
        "\n",
        "collate_fn = partial(collate_tweet,\n",
        "                     tokenizer=tokenizer,\n",
        "                     device=device,\n",
        "                     return_attention_masks=model_args.model == 'trans',\n",
        "                     pad_to_max_length=False,\n",
        "                     return_seq_lens = True)\n",
        "\n",
        "train = TwitterDataset(split='train')\n",
        "dev = TwitterDataset(split='val')\n",
        "train_dl = DataLoader(batch_size=model_args.batch_size, dataset=train, collate_fn=collate_fn, shuffle=True)\n",
        "dev_dl = DataLoader(batch_size=model_args.batch_size, dataset=dev, collate_fn=collate_fn, shuffle=False)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "ad906fc1d7e74cae9357c30c779aab42",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0d4a3a14c45e413dac84ea1b9cd7f34d",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=466062.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "12VxY3Iq_8AT"
      },
      "source": [
        "## Load/Train Models"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8aCBqoVd25SV",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 218,
          "referenced_widgets": [
            "261496b5302b44f6b9b662490e4f2b55",
            "48ff09d268cd49f98d96d2785c39c3ee",
            "548fb78ab387403a970d5312c5e36358",
            "06234dd7145b4c398dfa048a0d1937e6",
            "1b13d31979c04480baf250834fd45ed7",
            "184c74a697ce4f719498cf2794bf8e76",
            "58f3523b9f3a4ec28868a75c08b7d2fb",
            "aec6d3a6d64b45f8a551ea780736512d",
            "23d18a55ccfc4a41bdafe5002953a891",
            "7b975f237d354750b1ad4db967c9196d",
            "e743075f0129463a974a019d2f8abf2f",
            "b9262859fe79453dba9325e00c73abc9",
            "5c41d5c1185045b6ba252fe1191552d5",
            "bc08252a1d7f4c6bbd6a5fd891ca69d0",
            "3f947ebeac584c18bef7e8f3b9cf876a",
            "cab8e701f6834ba0adefb379d81b0b64"
          ]
        },
        "outputId": "d6e7a4d0-16f6-4aed-cdd0-af9cb3848b20"
      },
      "source": [
        "enforce_reproducibility(model_args.seed)\n",
        "model, optimizer, scheduler = get_model(model_args, device, embeddings)\n",
        "\n",
        "if model_args.mode == 'train':\n",
        "  if model_args.init_only:\n",
        "      best_model_w, best_perf = model.state_dict(), {'val_f1': 0}\n",
        "  else:\n",
        "      best_model_w, best_perf = train_model(model, train_dl, dev_dl, optimizer, scheduler, model_args.epochs)\n",
        "      print('F1', best_perf['val_f1'])\n",
        "      checkpoint = {\n",
        "        'performance': best_perf,\n",
        "        'args': vars(model_args),\n",
        "        'model': best_model_w\n",
        "      }\n",
        "      torch.save(checkpoint, f'{model_args.model_path}_{args[\"model\"]}')\n",
        "      model.load_state_dict(best_model_w)\n",
        "else:\n",
        "  checkpoint = torch.load(f'{model_args.model_path}_{args[\"model\"]}')\n",
        "  model.load_state_dict(checkpoint[\"model\"])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "261496b5302b44f6b9b662490e4f2b55",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=433.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "23d18a55ccfc4a41bdafe5002953a891",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=440473133.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']\n",
            "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "833IJ10hdfzX"
      },
      "source": [
        "# Interpretability Techniques\n",
        "\n",
        "In this section we'll get familiar with two types of explainability approaches -- gradient-based and perturbation-based. Most of the available explainability approaches are available through the [captum](https://captum.ai/) package, which we'll also make use of."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Xnm67Fk9mkAL"
      },
      "source": [
        "## Gradient - Based\n",
        "\n",
        "Gradient-based approaches compute saliency map based on the gradient of the input with respect to the output.\n",
        "\n",
        "In Natural Language Processing such approaches are harder to apply as the input layer with the input token ids is not differentiable. To alleviate this, a common trick is to *patch the model so that it would receive as input the differentible embeddings* for the token ids.\n",
        "\n",
        "We configure the input needed for the saliency approaches by using the `get_embeddings_input_captum` and `get_tokens_input_captum` functions. The first returns the embeddings of the input instaces' tokens, which then can be used to receive the gradient for.\n",
        "\n",
        "The baseline gradient-based technique (*Saliency*) takes the gradient of the input [(see paper)](https://arxiv.org/abs/1312.6034):"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ovZeS7wwUIpp"
      },
      "source": [
        "from captum.attr import Saliency\n",
        "from typing import Callable, Union, Tuple, Any, List\n",
        "from captum._utils.gradient import _run_forward"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Mm4RtSiQdsYz"
      },
      "source": [
        "def compute_gradients(\n",
        "    forward_fn: Callable,\n",
        "    inputs: Union[torch.Tensor, Tuple[torch.Tensor, ...]],\n",
        "    target_ind = None,\n",
        "    additional_forward_args: Any = None,\n",
        ") -> Tuple[torch.Tensor, ...]:\n",
        "    r\"\"\"\n",
        "    https://github.com/pytorch/captum/blob/45f3339b58bca9773e09273589db2e95298b33e4/captum/_utils/gradient.py#L94\n",
        "    Computes gradients of the output with respect to inputs for an arbitrary forward function.\n",
        "    Args:\n",
        "        forward_fn: forward function. This can be for example model's forward function.\n",
        "        input:      Input at which gradients are evaluated, will be passed to forward_fn.\n",
        "        target_ind: Index of the target class for which gradients must be computed (classification only).\n",
        "        additional_forward_args: Additional input arguments that forward function requires. It takes an empty tuple (no additional\n",
        "                    arguments) if no additional arguments are required\n",
        "    \"\"\"\n",
        "    with torch.autograd.set_grad_enabled(True):\n",
        "        # runs forward pass, configures some specifics about the layers that require gradients\n",
        "        outputs = _run_forward(forward_fn, inputs, target_ind, additional_forward_args)\n",
        "        assert outputs[0].numel() == 1, (\n",
        "            \"Target not provided when necessary, cannot\"\n",
        "            \" take gradient with respect to multiple outputs.\"\n",
        "        )\n",
        "        # torch.unbind(forward_out) is a list of scalar tensor tuples and\n",
        "        # contains batch_size * #steps elements\n",
        "        grads = torch.autograd.grad(torch.unbind(outputs), inputs)\n",
        "    return grads"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7ACmqGEypr3x"
      },
      "source": [
        "def get_embeddings_input_captum(model: torch.nn.Module, \n",
        "                         model_type: str, \n",
        "                         collate_fn: Callable, \n",
        "                         instances: List[Any], \n",
        "                         pad_token_id: int = None,\n",
        "                         batch=None):\n",
        "  if batch == None:\n",
        "    batch = collate_fn(instances)\n",
        "  token_ids = batch[0]\n",
        "  sequence_lengths = batch[-1]\n",
        "\n",
        "  if model_type == 'transformer':\n",
        "    input_embeddings = model.transformer.bert.embeddings(token_ids)\n",
        "  else:\n",
        "    input_embeddings = model.embedding(token_ids)\n",
        "  if model_type == 'lstm':\n",
        "    additional_forward_args = (sequence_lengths,)\n",
        "  elif model_type == 'cnn':\n",
        "    additional_forward_args = None\n",
        "  elif model_type == 'transformer':\n",
        "    additional_forward_args = (token_ids != pad_token_id, )\n",
        "  return input_embeddings, additional_forward_args\n",
        "\n",
        "def get_tokens_input_captum(model_type: str, \n",
        "                         collate_fn: Callable, \n",
        "                         instances: List[Any], \n",
        "                         pad_token_id: int = None):\n",
        "  batch = collate_fn(instances)\n",
        "  token_ids = batch[0]\n",
        "  sequence_lengths = batch[-1]\n",
        "\n",
        "  if model_type == 'lstm':\n",
        "    additional_forward_args = (sequence_lengths,)\n",
        "  elif model_type == 'cnn':\n",
        "    additional_forward_args = None\n",
        "  elif model_type == 'transformer':\n",
        "    additional_forward_args = (token_ids != pad_token_id, )\n",
        "  return token_ids, additional_forward_args"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZC9tY3vsuBjL",
        "outputId": "fbccf2b1-2693-42af-940f-727f3f18f805"
      },
      "source": [
        "input_embeddings, additional_forward_args = get_embeddings_input_captum(model, \n",
        "                                                                 model_args.model, \n",
        "                                                                 collate_fn,\n",
        "                                                                 dev.dataset[:2],\n",
        "                                                                 tokenizer.pad_token_id)\n",
        "\n",
        "attributions = compute_gradients(model, \n",
        "                                 inputs=input_embeddings, \n",
        "                                 target_ind=0,\n",
        "                                 additional_forward_args=additional_forward_args)\n",
        "attributions[0]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[[-9.3983e-03,  1.1500e-03,  1.4011e-03,  ...,  3.3202e-03,\n",
              "           7.9773e-03,  9.5162e-04],\n",
              "         [ 2.8178e-03, -1.6889e-03, -8.3604e-03,  ...,  2.6754e-03,\n",
              "          -4.5492e-04,  2.4837e-03],\n",
              "         [ 2.9172e-03,  7.0193e-03, -4.9965e-03,  ..., -7.8309e-03,\n",
              "           1.1074e-02,  2.3271e-03],\n",
              "         ...,\n",
              "         [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
              "           0.0000e+00,  0.0000e+00],\n",
              "         [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
              "           0.0000e+00,  0.0000e+00],\n",
              "         [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
              "           0.0000e+00,  0.0000e+00]],\n",
              "\n",
              "        [[-6.8152e-04,  3.0616e-04, -5.2241e-04,  ...,  1.3034e-04,\n",
              "          -2.6643e-04,  1.1583e-05],\n",
              "         [ 4.4404e-05, -7.2836e-04,  1.4775e-04,  ...,  2.0612e-03,\n",
              "           1.7945e-04, -3.4629e-04],\n",
              "         [ 2.6430e-04,  8.7656e-04,  2.1136e-04,  ...,  5.6390e-04,\n",
              "          -8.8478e-04,  3.2502e-04],\n",
              "         ...,\n",
              "         [ 1.3351e-04,  1.0961e-04, -2.6417e-04,  ...,  5.9816e-05,\n",
              "           1.2834e-04, -6.3132e-05],\n",
              "         [ 5.0724e-05,  2.5922e-04,  1.3696e-04,  ...,  6.3485e-05,\n",
              "           2.3820e-04, -3.4960e-04],\n",
              "         [ 3.0778e-04,  9.2026e-05, -3.2322e-04,  ...,  1.9106e-04,\n",
              "           2.9063e-04, -1.3783e-04]]], device='cuda:0')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tpiFlFSsI_vk"
      },
      "source": [
        "Now we have the attributions at the embedding layer, which are not quite informative and we are looking for saliency maps at the word level.\n",
        "\n",
        "### Task 1\n",
        "Can you think how the gradients of the embeddings can be used to produce **one saliency score** per word? What are the consequences of the different approaches?\n",
        "\n",
        "*(check the solutions notebook for answers)*"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "idLiuBMO7BIx"
      },
      "source": [
        "You can aggregate the embedding gradients by taking the mean or the L2 norm of the vector. In our work ([see paper](https://www.aclweb.org/anthology/2020.emnlp-main.263.pdf)), we found L2 norm aggregation to be working better as the mean was making the attributions more uniform and the informantion from the separate dimentions was lost."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qRYBOkj4tNb-"
      },
      "source": [
        "def summarize_attributions(attributions, type='mean'):\n",
        "    # YOUR CODE HERE"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mic-XqVW5Zwn",
        "outputId": "5739dbd0-93d7-49c9-db58-bddfde513223"
      },
      "source": [
        "summarized_saliency = summarize_attributions(attributions[0], type='l2').detach().cpu().numpy()\n",
        "summarized_saliency"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[2.4415631 , 4.2119226 , 4.190755  , 1.4050797 , 1.1381679 ,\n",
              "        2.9951453 , 2.3997874 , 0.9750633 , 0.6543945 , 0.61226463,\n",
              "        0.64614785, 0.6857956 , 1.2404237 , 0.61521757, 0.4497512 ,\n",
              "        1.015996  , 0.5740527 , 0.9215991 , 0.99524164, 0.558179  ,\n",
              "        0.9547747 , 0.        , 0.        , 0.        , 0.        ,\n",
              "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "        0.        , 0.        ],\n",
              "       [0.7337796 , 0.6752617 , 0.31475812, 0.35597217, 0.69898075,\n",
              "        2.6614914 , 0.35795587, 0.6978358 , 0.1671758 , 0.15031673,\n",
              "        0.5202161 , 0.20650557, 0.5049865 , 0.37323612, 0.17771149,\n",
              "        0.29355967, 0.5930433 , 0.27872103, 0.45088464, 0.22605383,\n",
              "        0.16704844, 0.30293605, 0.20800115, 0.13248089, 0.17115802,\n",
              "        0.10573292, 0.09743957, 0.08023961, 0.10284176, 0.08306251,\n",
              "        0.13217077, 0.18541744]], dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ys1DXSA5J_nQ"
      },
      "source": [
        "### Task 2\n",
        "There are a lot of approaches the improve over the baseline by accounting for the specifics of different layers and their back-propagation proccess. One of them, **Input X Gradient**, multiplies the gradient with the embeddings (see paper [link text](http://proceedings.mlr.press/v70/shrikumar17a.html)). It was proposed as a technique to improve the sharpness of the attribution maps.\n",
        "\n",
        "Can you quickly modify the above approach to Input X Gradient and compare the differences between the two?"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tm9O0rslePN9",
        "outputId": "87001d55-c945-448a-d1a9-acc0535b64af"
      },
      "source": [
        "gradients = compute_gradients(model, \n",
        "                              inputs=input_embeddings, \n",
        "                              target_ind=0,\n",
        "                              additional_forward_args=additional_forward_args)[0]\n",
        "\n",
        "attributions = # YOUR CODE HERE\n",
        "summarized_inputx = summarize_attributions(attributions, type='l2').detach().cpu().numpy()\n",
        "summarized_inputx"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0.4184738 , 1.9442306 , 2.1370826 , 0.57326627, 0.42939132,\n",
              "        1.4509289 , 1.2539201 , 0.41862074, 0.29001504, 0.2806509 ,\n",
              "        0.3043002 , 0.3496624 , 0.6471158 , 0.29934034, 0.15684427,\n",
              "        0.53703433, 0.26753074, 0.38561398, 0.5373926 , 0.24556574,\n",
              "        0.290793  , 0.        , 0.        , 0.        , 0.        ,\n",
              "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "        0.        , 0.        ],\n",
              "       [0.12156984, 0.32546508, 0.12648456, 0.1506634 , 0.31653205,\n",
              "        1.17974   , 0.15311517, 0.3511062 , 0.06942554, 0.06161044,\n",
              "        0.26566064, 0.09083129, 0.24966733, 0.19913176, 0.07825722,\n",
              "        0.10958775, 0.26920623, 0.1329355 , 0.25202718, 0.11766723,\n",
              "        0.0648167 , 0.14561571, 0.09213648, 0.0571308 , 0.08479664,\n",
              "        0.04895854, 0.04220755, 0.03769324, 0.05185024, 0.03864675,\n",
              "        0.04872997, 0.05726779]], dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mXgBlKJu9KQH"
      },
      "source": [
        "We can see that the saliency scores from the Saliency approach have a larger deviation from the mean, but also a higher mean. Normalising the attributions scores first might help to compare them better."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tPMUDfiD8TfY",
        "outputId": "9bbf2f7b-b3da-42df-e145-e44244ede65d"
      },
      "source": [
        "for i in range(len(summarized_inputx)):\n",
        "  print(np.mean(summarized_inputx[i]), \n",
        "        np.std(summarized_inputx[i]), \n",
        "        np.mean(summarized_saliency[i]), \n",
        "        np.std(summarized_saliency[i]))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.41305542 0.538557 0.9275414 1.1308007\n",
            "0.16845421 0.20347749 0.381468 0.45472315\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ig7uiINzcZ3v"
      },
      "source": [
        "## Local-Approximation Based\n",
        "\n",
        "Another type of explainability techniques perturbs the input to find which regions from it change the prediction to a larger degree. One such method is LIME ([see paper](https://arxiv.org/abs/1602.04938)), which build a linear local approximator for each instance. It perturbs the input and tries to predict how the output is being changed with each local perturbation. The weights of the linear model for each token are used as saliency scores.\n",
        "\n",
        "See this [book on interpretability](https://christophm.github.io/interpretable-ml-book/lime.html) for more information on LIME. It discusses some of the disadvantages of the approach, which are important to take into account: \n",
        "\n",
        "*   The correct definition of the neighborhood is a very big, unsolved problem.\n",
        "*   Sampling could be improved in the current implementation of LIME. Data points are sampled from a Gaussian distribution, ignoring the correlation between features. This can lead to unlikely data points which can then be used to learn local explanation models.\n",
        "* The complexity of the explanation model has to be defined in advance. \n",
        "* The instability of the explanations. If you repeat the sampling process, then the explantions that come out can be different. Instability means that it is difficult to trust the explanations, and you should be very critical."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "C7eNFlv_s3lv"
      },
      "source": [
        "### Task 3 Version 1\n",
        "\n",
        "For this task, you have to train a linear classifier that will approximate the decision of the model in the neighbourhood of one instance. Here are some guidelines how to approach the task:\n",
        "\n",
        "\n",
        "*   Implement a sampling function that will perturb a given instance randomly to another instance with a few tokens changed/removed/added.\n",
        "*   Collect the predictions of the original model on the perturbations.\n",
        "*   Train a linear model with the different perturbations to predict the confidence of the original model for the target class.\n",
        "*   Use the weights of the linear model to explain the predictions of the original model.\n",
        "\n",
        "***Note.*** If this task seems difficult and/or you won't have enough time to complete it, continue below to use the implementation of the captum package and implement only the sampling function.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U7_DujB9fX6s"
      },
      "source": [
        "from captum.attr import LimeBase\n",
        "from sklearn.metrics import mean_squared_error\n",
        "from captum._utils.models.linear_model import SkLearnLinearModel"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "90Zg9XeNgFT8"
      },
      "source": [
        "def similarity_kernel(\n",
        "     original_input: torch.Tensor,\n",
        "     perturbed_input: torch.Tensor,\n",
        "     perturbed_interpretable_input: torch.Tensor,\n",
        "     **kwargs)->torch.Tensor:\n",
        "         # kernel_width will be provided to attribute as a kwarg\n",
        "         return torch.sum(original_input==perturbed_input)\n",
        "\n",
        "def to_interp_rep_transform_custom(curr_sample, original_input, **kwargs: Any):\n",
        "  return curr_sample"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1sD61LUcDEm_"
      },
      "source": [
        "### Task 3 Version 2\n",
        "\n",
        "*   Can you implement the sampling function that perturbs the input?\n",
        "* Then, experiment with LIME to see how the explanation varies based on the number of samples and the different seed for the samples.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xgD0ZqaBDMaY"
      },
      "source": [
        "# Define sampling function\n",
        " # This function samples in original input space\n",
        "def perturb_func(\n",
        "     original_input: torch.Tensor,\n",
        "     **kwargs: Any)->torch.Tensor:\n",
        "         return # YOUR CODE HERE\n",
        "\n",
        " # For this example, we are setting the interpretable input to\n",
        " # match the model input, so the to_interp_rep_transform\n",
        " # function simply returns the input. In most cases, the interpretable\n",
        " # input will be different and may have a smaller feature set, so\n",
        " # an appropriate transformation function should be provided."
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zPNb9xmHvonc",
        "outputId": "4fd70366-8f93-456b-f1fe-1dda2883a1ba"
      },
      "source": [
        "sklearnmodel = SkLearnLinearModel(\"linear_model.Ridge\")\n",
        "\n",
        "# The LimeBase attributor will need another wrapper for \n",
        "# the model if we try to explain more than one instance\n",
        "batch = collate_fn(dev[:1])\n",
        "input, additional_forward_args = get_tokens_input_captum(model_args.model, collate_fn, dev[:1], tokenizer.pad_token_id)\n",
        "\n",
        " # Defining LimeBase interpreter\n",
        "lime_attr = LimeBase(model,\n",
        "                    sklearnmodel,\n",
        "                    similarity_func=similarity_kernel,\n",
        "                    perturb_func=perturb_func,\n",
        "                    perturb_interpretable_space=False,\n",
        "                    from_interp_rep_transform=None,\n",
        "                    to_interp_rep_transform=to_interp_rep_transform_custom)\n",
        "\n",
        " # Computes interpretable model, returning coefficients of linear model.\n",
        "attr_coefs = lime_attr.attribute(input, \n",
        "                                 n_perturb_samples=100, \n",
        "                                 target=1,\n",
        "                                 additional_forward_args=additional_forward_args)\n",
        "attr_coefs"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[ 4.6955e-04,  4.0859e-05,  3.8338e-05,  1.2541e-04,  1.6611e-04,\n",
              "          2.0025e-05,  1.9535e-05,  2.4987e-04,  1.3605e-04,  4.0213e-05,\n",
              "          1.3477e-04,  7.3199e-06,  9.3516e-07,  4.5949e-05,  2.3781e-05,\n",
              "          4.6775e-05,  3.4967e-04,  1.7209e-05, -1.2891e-06, -1.9280e-05,\n",
              "          8.7523e-03]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ehU-NmNVz2fL",
        "outputId": "e3effe76-05c8-411e-baab-5f6b60f5bd91"
      },
      "source": [
        "# You can evaluate the convergence of the local explanation:\n",
        "pred, true = [], []\n",
        "enforce_reproducibility(seed=model_args.seed)\n",
        "\n",
        "for i in range(20):\n",
        "  instance = dev[i]\n",
        "  if len(instance[0].split()) < 5:\n",
        "    continue\n",
        "  instance_input, additional_args = get_tokens_input_captum(model_args.model, \n",
        "                                                   collate_fn, \n",
        "                                                   [instance], \n",
        "                                                   tokenizer.pad_token_id)\n",
        "  \n",
        "  lime_attr.attribute(instance_input, \n",
        "                      n_perturb_samples=10, \n",
        "                      target=1,\n",
        "                      additional_forward_args=additional_args)\n",
        "  pred.append(sklearnmodel(instance_input.to('cpu').float()).item())\n",
        "  true.append(model(instance_input, additional_args[0] if additional_args else None)[0][1].item())\n",
        "mean_squared_error(true, pred)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1.2863891527193987"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TxQsYH7rLb0a"
      },
      "source": [
        "# YOUR CODE HERE"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ov1rM_dlL-gI"
      },
      "source": [
        "# Visualising Saliency Maps\n",
        "\n",
        "Humans are good at understanding complex patterns when presented with good visualisations. We will visualize the saliency maps and the predictions of a model using the "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jM4C0e4lUjKD"
      },
      "source": [
        "from ALPS_2021.tutorial_src.explainability_utils import GradientBasedVisualizer"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hq8czemfpO67"
      },
      "source": [
        "ablator = Saliency(model)\n",
        "visualizer = GradientBasedVisualizer(collate_fn, tokenizer, ablator)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 559
        },
        "id": "oYC80UXT0zap",
        "outputId": "fd63cdb8-259a-4334-c3f3-88da8fcca0ab"
      },
      "source": [
        "# we add the instances to the visuliser and then show all of them together:\n",
        "for i in range(3):\n",
        "  visualizer.interpret_sentence(model, model_args.model, dev[i], target=0)\n",
        "  visualizer.interpret_sentence(model, model_args.model, dev[i], target=1)\n",
        "  visualizer.interpret_sentence(model, model_args.model, dev[i], target=2)\n",
        "visualizer.visualize()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/captum/_utils/gradient.py:58: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the gradient for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations.\n",
            "  if input.grad is not None:\n",
            "/usr/local/lib/python3.6/dist-packages/captum/_utils/gradient.py:87: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the gradient for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations.\n",
            "  if input.grad is not None:\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table width: 100%><div style=\"border-top: 1px solid; margin-top: 5px;             padding-top: 5px; display: inline-block\"><b>Legend: </b><span style=\"display: inline-block; width: 10px; height: 10px;                 border: 1px solid; background-color:                 hsl(0, 75%, 60%)\"></span> Negative  <span style=\"display: inline-block; width: 10px; height: 10px;                 border: 1px solid; background-color:                 hsl(0, 75%, 100%)\"></span> Neutral  <span style=\"display: inline-block; width: 10px; height: 10px;                 border: 1px solid; background-color:                 hsl(120, 75%, 50%)\"></span> Positive  </div><tr><th>True Label</th><th>Predicted Label</th><th>Attribution Label</th><th>Attribution Score</th><th>Word Importance</th><tr><td><text style=\"padding-right:2em\"><b>neutral</b></text></td><td><text style=\"padding-right:2em\"><b>neutral (0.99)</b></text></td><td><text style=\"padding-right:2em\"><b>negative</b></text></td><td><text style=\"padding-right:2em\"><b>3.59</b></text></td><td><mark style=\"background-color: hsl(120, 75%, 86%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> [CLS]                    </font></mark><mark style=\"background-color: hsl(120, 75%, 75%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> last                    </font></mark><mark style=\"background-color: hsl(120, 75%, 75%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> session                    </font></mark><mark style=\"background-color: hsl(120, 75%, 92%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> of                    </font></mark><mark style=\"background-color: hsl(120, 75%, 94%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> the                    </font></mark><mark style=\"background-color: hsl(120, 75%, 82%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> day                    </font></mark><mark style=\"background-color: hsl(120, 75%, 86%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> http                    </font></mark><mark style=\"background-color: hsl(120, 75%, 95%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> :                    </font></mark><mark style=\"background-color: hsl(120, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> /                    </font></mark><mark style=\"background-color: hsl(120, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> /                    </font></mark><mark style=\"background-color: hsl(120, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> t                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##wi                    </font></mark><mark style=\"background-color: hsl(120, 75%, 93%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##tp                    </font></mark><mark style=\"background-color: hsl(120, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##ic                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> .                    </font></mark><mark style=\"background-color: hsl(120, 75%, 94%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> com                    </font></mark><mark style=\"background-color: hsl(120, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> /                    </font></mark><mark style=\"background-color: hsl(120, 75%, 95%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> 67                    </font></mark><mark style=\"background-color: hsl(120, 75%, 94%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##ez                    </font></mark><mark style=\"background-color: hsl(120, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##h                    </font></mark><mark style=\"background-color: hsl(120, 75%, 95%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> [SEP]                    </font></mark></td><tr><tr><td><text style=\"padding-right:2em\"><b>neutral</b></text></td><td><text style=\"padding-right:2em\"><b>neutral (0.99)</b></text></td><td><text style=\"padding-right:2em\"><b>neutral</b></text></td><td><text style=\"padding-right:2em\"><b>3.55</b></text></td><td><mark style=\"background-color: hsl(120, 75%, 85%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> [CLS]                    </font></mark><mark style=\"background-color: hsl(120, 75%, 72%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> last                    </font></mark><mark style=\"background-color: hsl(120, 75%, 76%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> session                    </font></mark><mark style=\"background-color: hsl(120, 75%, 91%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> of                    </font></mark><mark style=\"background-color: hsl(120, 75%, 95%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> the                    </font></mark><mark style=\"background-color: hsl(120, 75%, 85%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> day                    </font></mark><mark style=\"background-color: hsl(120, 75%, 88%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> http                    </font></mark><mark style=\"background-color: hsl(120, 75%, 94%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> :                    </font></mark><mark style=\"background-color: hsl(120, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> /                    </font></mark><mark style=\"background-color: hsl(120, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> /                    </font></mark><mark style=\"background-color: hsl(120, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> t                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##wi                    </font></mark><mark style=\"background-color: hsl(120, 75%, 93%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##tp                    </font></mark><mark style=\"background-color: hsl(120, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##ic                    </font></mark><mark style=\"background-color: hsl(120, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> .                    </font></mark><mark style=\"background-color: hsl(120, 75%, 95%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> com                    </font></mark><mark style=\"background-color: hsl(120, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> /                    </font></mark><mark style=\"background-color: hsl(120, 75%, 95%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> 67                    </font></mark><mark style=\"background-color: hsl(120, 75%, 94%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##ez                    </font></mark><mark style=\"background-color: hsl(120, 75%, 95%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##h                    </font></mark><mark style=\"background-color: hsl(120, 75%, 94%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> [SEP]                    </font></mark></td><tr><tr><td><text style=\"padding-right:2em\"><b>neutral</b></text></td><td><text style=\"padding-right:2em\"><b>neutral (0.99)</b></text></td><td><text style=\"padding-right:2em\"><b>positive</b></text></td><td><text style=\"padding-right:2em\"><b>3.75</b></text></td><td><mark style=\"background-color: hsl(120, 75%, 85%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> [CLS]                    </font></mark><mark style=\"background-color: hsl(120, 75%, 79%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> last                    </font></mark><mark style=\"background-color: hsl(120, 75%, 76%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> session                    </font></mark><mark style=\"background-color: hsl(120, 75%, 92%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> of                    </font></mark><mark style=\"background-color: hsl(120, 75%, 92%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> the                    </font></mark><mark style=\"background-color: hsl(120, 75%, 81%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> day                    </font></mark><mark style=\"background-color: hsl(120, 75%, 85%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> http                    </font></mark><mark style=\"background-color: hsl(120, 75%, 94%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> :                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> /                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> /                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> t                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##wi                    </font></mark><mark style=\"background-color: hsl(120, 75%, 93%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##tp                    </font></mark><mark style=\"background-color: hsl(120, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##ic                    </font></mark><mark style=\"background-color: hsl(120, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> .                    </font></mark><mark style=\"background-color: hsl(120, 75%, 94%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> com                    </font></mark><mark style=\"background-color: hsl(120, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> /                    </font></mark><mark style=\"background-color: hsl(120, 75%, 94%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> 67                    </font></mark><mark style=\"background-color: hsl(120, 75%, 93%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##ez                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##h                    </font></mark><mark style=\"background-color: hsl(120, 75%, 95%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> [SEP]                    </font></mark></td><tr><tr><td><text style=\"padding-right:2em\"><b>positive</b></text></td><td><text style=\"padding-right:2em\"><b>positive (0.97)</b></text></td><td><text style=\"padding-right:2em\"><b>negative</b></text></td><td><text style=\"padding-right:2em\"><b>3.64</b></text></td><td><mark style=\"background-color: hsl(120, 75%, 90%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> [CLS]                    </font></mark><mark style=\"background-color: hsl(120, 75%, 90%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> shanghai                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> is                    </font></mark><mark style=\"background-color: hsl(120, 75%, 95%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> also                    </font></mark><mark style=\"background-color: hsl(120, 75%, 90%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> really                    </font></mark><mark style=\"background-color: hsl(120, 75%, 61%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> exciting                    </font></mark><mark style=\"background-color: hsl(120, 75%, 95%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> (                    </font></mark><mark style=\"background-color: hsl(120, 75%, 90%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> precisely                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> -                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> -                    </font></mark><mark style=\"background-color: hsl(120, 75%, 93%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> skyscraper                    </font></mark><mark style=\"background-color: hsl(120, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##s                    </font></mark><mark style=\"background-color: hsl(120, 75%, 93%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> gal                    </font></mark><mark style=\"background-color: hsl(120, 75%, 95%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##ore                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> )                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> .                    </font></mark><mark style=\"background-color: hsl(120, 75%, 92%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> good                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> t                    </font></mark><mark style=\"background-color: hsl(120, 75%, 94%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##wee                    </font></mark><mark style=\"background-color: hsl(120, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##ps                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> in                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> china                    </font></mark><mark style=\"background-color: hsl(120, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> :                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> (                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> sh                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> )                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> (                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> b                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##j                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> )                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> .                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> [SEP]                    </font></mark></td><tr><tr><td><text style=\"padding-right:2em\"><b>positive</b></text></td><td><text style=\"padding-right:2em\"><b>positive (0.97)</b></text></td><td><text style=\"padding-right:2em\"><b>neutral</b></text></td><td><text style=\"padding-right:2em\"><b>3.77</b></text></td><td><mark style=\"background-color: hsl(120, 75%, 90%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> [CLS]                    </font></mark><mark style=\"background-color: hsl(120, 75%, 91%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> shanghai                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> is                    </font></mark><mark style=\"background-color: hsl(120, 75%, 95%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> also                    </font></mark><mark style=\"background-color: hsl(120, 75%, 88%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> really                    </font></mark><mark style=\"background-color: hsl(120, 75%, 63%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> exciting                    </font></mark><mark style=\"background-color: hsl(120, 75%, 93%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> (                    </font></mark><mark style=\"background-color: hsl(120, 75%, 91%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> precisely                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> -                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> -                    </font></mark><mark style=\"background-color: hsl(120, 75%, 91%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> skyscraper                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##s                    </font></mark><mark style=\"background-color: hsl(120, 75%, 94%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> gal                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##ore                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> )                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> .                    </font></mark><mark style=\"background-color: hsl(120, 75%, 88%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> good                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> t                    </font></mark><mark style=\"background-color: hsl(120, 75%, 95%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##wee                    </font></mark><mark style=\"background-color: hsl(120, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##ps                    </font></mark><mark style=\"background-color: hsl(120, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> in                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> china                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> :                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> (                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> sh                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> )                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> (                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> b                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##j                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> )                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> .                    </font></mark><mark style=\"background-color: hsl(120, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> [SEP]                    </font></mark></td><tr><tr><td><text style=\"padding-right:2em\"><b>positive</b></text></td><td><text style=\"padding-right:2em\"><b>positive (0.97)</b></text></td><td><text style=\"padding-right:2em\"><b>positive</b></text></td><td><text style=\"padding-right:2em\"><b>3.71</b></text></td><td><mark style=\"background-color: hsl(120, 75%, 90%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> [CLS]                    </font></mark><mark style=\"background-color: hsl(120, 75%, 91%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> shanghai                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> is                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> also                    </font></mark><mark style=\"background-color: hsl(120, 75%, 89%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> really                    </font></mark><mark style=\"background-color: hsl(120, 75%, 62%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> exciting                    </font></mark><mark style=\"background-color: hsl(120, 75%, 94%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> (                    </font></mark><mark style=\"background-color: hsl(120, 75%, 90%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> precisely                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> -                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> -                    </font></mark><mark style=\"background-color: hsl(120, 75%, 91%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> skyscraper                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##s                    </font></mark><mark style=\"background-color: hsl(120, 75%, 95%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> gal                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##ore                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> )                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> .                    </font></mark><mark style=\"background-color: hsl(120, 75%, 88%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> good                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> t                    </font></mark><mark style=\"background-color: hsl(120, 75%, 94%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##wee                    </font></mark><mark style=\"background-color: hsl(120, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##ps                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> in                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> china                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> :                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> (                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> sh                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> )                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> (                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> b                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##j                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> )                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> .                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> [SEP]                    </font></mark></td><tr><tr><td><text style=\"padding-right:2em\"><b>negative</b></text></td><td><text style=\"padding-right:2em\"><b>negative (0.99)</b></text></td><td><text style=\"padding-right:2em\"><b>negative</b></text></td><td><text style=\"padding-right:2em\"><b>4.01</b></text></td><td><mark style=\"background-color: hsl(120, 75%, 83%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> [CLS]                    </font></mark><mark style=\"background-color: hsl(120, 75%, 75%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> recession                    </font></mark><mark style=\"background-color: hsl(120, 75%, 82%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> hit                    </font></mark><mark style=\"background-color: hsl(120, 75%, 94%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ve                    </font></mark><mark style=\"background-color: hsl(120, 75%, 94%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##ron                    </font></mark><mark style=\"background-color: hsl(120, 75%, 91%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##ique                    </font></mark><mark style=\"background-color: hsl(120, 75%, 92%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> bran                    </font></mark><mark style=\"background-color: hsl(120, 75%, 88%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##quin                    </font></mark><mark style=\"background-color: hsl(120, 75%, 92%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##ho                    </font></mark><mark style=\"background-color: hsl(120, 75%, 95%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ,                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> she                    </font></mark><mark style=\"background-color: hsl(120, 75%, 95%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> has                    </font></mark><mark style=\"background-color: hsl(120, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> to                    </font></mark><mark style=\"background-color: hsl(120, 75%, 92%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> quit                    </font></mark><mark style=\"background-color: hsl(120, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> her                    </font></mark><mark style=\"background-color: hsl(120, 75%, 94%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> company                    </font></mark><mark style=\"background-color: hsl(120, 75%, 94%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ,                    </font></mark><mark style=\"background-color: hsl(120, 75%, 93%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> such                    </font></mark><mark style=\"background-color: hsl(120, 75%, 95%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> a                    </font></mark><mark style=\"background-color: hsl(120, 75%, 84%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> shame                    </font></mark><mark style=\"background-color: hsl(120, 75%, 95%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> !                    </font></mark><mark style=\"background-color: hsl(120, 75%, 94%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> [SEP]                    </font></mark></td><tr><tr><td><text style=\"padding-right:2em\"><b>negative</b></text></td><td><text style=\"padding-right:2em\"><b>negative (0.99)</b></text></td><td><text style=\"padding-right:2em\"><b>neutral</b></text></td><td><text style=\"padding-right:2em\"><b>4.01</b></text></td><td><mark style=\"background-color: hsl(120, 75%, 85%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> [CLS]                    </font></mark><mark style=\"background-color: hsl(120, 75%, 75%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> recession                    </font></mark><mark style=\"background-color: hsl(120, 75%, 81%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> hit                    </font></mark><mark style=\"background-color: hsl(120, 75%, 94%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ve                    </font></mark><mark style=\"background-color: hsl(120, 75%, 94%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##ron                    </font></mark><mark style=\"background-color: hsl(120, 75%, 90%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##ique                    </font></mark><mark style=\"background-color: hsl(120, 75%, 92%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> bran                    </font></mark><mark style=\"background-color: hsl(120, 75%, 88%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##quin                    </font></mark><mark style=\"background-color: hsl(120, 75%, 92%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##ho                    </font></mark><mark style=\"background-color: hsl(120, 75%, 95%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ,                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> she                    </font></mark><mark style=\"background-color: hsl(120, 75%, 95%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> has                    </font></mark><mark style=\"background-color: hsl(120, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> to                    </font></mark><mark style=\"background-color: hsl(120, 75%, 91%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> quit                    </font></mark><mark style=\"background-color: hsl(120, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> her                    </font></mark><mark style=\"background-color: hsl(120, 75%, 93%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> company                    </font></mark><mark style=\"background-color: hsl(120, 75%, 94%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ,                    </font></mark><mark style=\"background-color: hsl(120, 75%, 93%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> such                    </font></mark><mark style=\"background-color: hsl(120, 75%, 95%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> a                    </font></mark><mark style=\"background-color: hsl(120, 75%, 82%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> shame                    </font></mark><mark style=\"background-color: hsl(120, 75%, 94%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> !                    </font></mark><mark style=\"background-color: hsl(120, 75%, 95%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> [SEP]                    </font></mark></td><tr><tr><td><text style=\"padding-right:2em\"><b>negative</b></text></td><td><text style=\"padding-right:2em\"><b>negative (0.99)</b></text></td><td><text style=\"padding-right:2em\"><b>positive</b></text></td><td><text style=\"padding-right:2em\"><b>3.98</b></text></td><td><mark style=\"background-color: hsl(120, 75%, 86%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> [CLS]                    </font></mark><mark style=\"background-color: hsl(120, 75%, 75%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> recession                    </font></mark><mark style=\"background-color: hsl(120, 75%, 84%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> hit                    </font></mark><mark style=\"background-color: hsl(120, 75%, 93%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ve                    </font></mark><mark style=\"background-color: hsl(120, 75%, 94%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##ron                    </font></mark><mark style=\"background-color: hsl(120, 75%, 92%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##ique                    </font></mark><mark style=\"background-color: hsl(120, 75%, 95%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> bran                    </font></mark><mark style=\"background-color: hsl(120, 75%, 92%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##quin                    </font></mark><mark style=\"background-color: hsl(120, 75%, 95%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##ho                    </font></mark><mark style=\"background-color: hsl(120, 75%, 95%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ,                    </font></mark><mark style=\"background-color: hsl(120, 75%, 95%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> she                    </font></mark><mark style=\"background-color: hsl(120, 75%, 94%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> has                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> to                    </font></mark><mark style=\"background-color: hsl(120, 75%, 90%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> quit                    </font></mark><mark style=\"background-color: hsl(120, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> her                    </font></mark><mark style=\"background-color: hsl(120, 75%, 93%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> company                    </font></mark><mark style=\"background-color: hsl(120, 75%, 93%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ,                    </font></mark><mark style=\"background-color: hsl(120, 75%, 92%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> such                    </font></mark><mark style=\"background-color: hsl(120, 75%, 94%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> a                    </font></mark><mark style=\"background-color: hsl(120, 75%, 78%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> shame                    </font></mark><mark style=\"background-color: hsl(120, 75%, 94%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> !                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> [SEP]                    </font></mark></td><tr></table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ktk0ULgQ5beJ"
      },
      "source": [
        "# Human-in-the-Loop"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ekiJYxC8T8WZ"
      },
      "source": [
        "## Task 4\n",
        "Observing the saliency maps:\n",
        "\n",
        "\n",
        "*   Can you use the saliency maps to spot patterns which can be used to **re-write an example to an adversarial one**, which fools the model to predict the wrong label?\n",
        "*   Can you use the saliency maps on wrong predictions to find what the model fails to capture?\n",
        "\n",
        "Hint: you can use the visualizer from above."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RRCsZsmvSUNs"
      },
      "source": [
        "# YOUR CODE HERE"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OpRHaeFs4V6m"
      },
      "source": [
        "# Properties\n",
        "\n",
        "As there are a lot of explainability approaches and they can produce quite different saliency maps, it is important to be aware of their properties and be able to perform sanity checks with them. \n",
        "\n",
        "For a full list of propoerties for evaluating explainability techniques, see this [paper](https://www.aclweb.org/anthology/2020.emnlp-main.263.pdf).\n",
        "\n",
        "Here, we will look at two of the most common properties."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TdBhEdLI4YMj"
      },
      "source": [
        "## Faithfulness\n",
        "\n",
        " Since explanation techniques are employed to explain model predictions for a single instance, an essential property is that they are faithful to the model’s inner workings and not based on arbitrary choices. A well-established way of measuring this property is by replacing a number of the most-salient words with a mask token and observing the drop in the model’s performance.\n",
        "\n",
        " Here, for simplicity, we will remove top 20 words from each instance and observe the change in the predictive performance. The bigger the drop in the performance, the more faithful the explainability approach is."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DVfzGIFtmHG1"
      },
      "source": [
        "from sklearn.metrics import f1_score\n",
        "from ALPS_2021.tutorial_src.explainability_utils import attribute_predict"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "75s1qh2d4ZOB"
      },
      "source": [
        "def threshold_predict(dataset, attribution_method, model, words_remove=20, neg_inf=1e-10, target = 0):\n",
        "  predictions_old, predictions_new, true_target = [], [], []\n",
        "  dl = DataLoader(batch_size=model_args.batch_size, dataset=dataset, collate_fn=collate_fn)\n",
        "      \n",
        "  for batch in tqdm(dl):\n",
        "    token_ids = batch[0]\n",
        "    additional_args = None\n",
        "\n",
        "    input_embeds, additional_args = get_embeddings_input_captum(model, \n",
        "                                                                model_args.model, \n",
        "                                                                collate_fn,\n",
        "                                                                None,\n",
        "                                                                tokenizer.pad_token_id, \n",
        "                                                                batch=batch)\n",
        "\n",
        "    if isinstance(attribution_method, LimeBase):\n",
        "      inputs = token_ids\n",
        "      instance_attribution = attribution_method.attribute(token_ids, \n",
        "                                                          additional_forward_args=additional_args,\n",
        "                                                          n_perturb_samples=100, \n",
        "                                                          target=1)\n",
        "    else:\n",
        "      inputs = input_embeds\n",
        "      instance_attribution = attribution_method.attribute(inputs, \n",
        "                                                          additional_forward_args=additional_args,\n",
        "                                                          target=1)\n",
        "      instance_attribution = summarize_attributions(instance_attribution, type='mean').detach().cpu()\n",
        "      \n",
        "    mask = (token_ids != tokenizer.pad_token_id).long()\n",
        "    instance_attribution = torch.tensor(instance_attribution).to(device) + (1 - mask) * neg_inf\n",
        "    \n",
        "    # take first words woth highest scores\n",
        "    words_to_mask = (torch.argsort(torch.tensor(instance_attribution), descending=True) < words_remove).long() * mask\n",
        "    \n",
        "    inputs_masked = token_ids * words_to_mask + (1 - words_to_mask) * tokenizer.mask_token_id\n",
        "    \n",
        "    predictions_old += torch.max(model(inputs, additional_args[0] if additional_args else None), dim=1)[1].detach().cpu().numpy().tolist()\n",
        "    predictions_new += torch.max(model(inputs_masked, additional_args[0]  if additional_args else None), dim=1)[1].detach().cpu().numpy().tolist()\n",
        "    true_target += batch[1].detach().cpu().numpy().tolist()\n",
        "\n",
        "  return (f1_score(true_target, predictions_old, average='macro'), \n",
        "          f1_score(true_target, predictions_new, average='macro'))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 237,
          "referenced_widgets": [
            "37467989caf54aa89936e7c8ca9584e1",
            "bb1bc3cdfa2d4a488fad9227ef4df011",
            "83a508ad828242e68844344077560b21",
            "291ef6aedc6d40b68733dfa231cc928d",
            "d89760a24d7244278856ac405319def1",
            "4c40a712ce874a0a9cace124e048b5fe",
            "01f462b832dc4057b7000fd99faa6699",
            "9e1f76ce613a42bab787a7828ec1e67e"
          ]
        },
        "id": "LtvXD6wDKVGM",
        "outputId": "8dc26ab6-3025-4811-a8a1-cf399991bd4f"
      },
      "source": [
        "dev.dataset = dev.dataset[:100]\n",
        "threshold_predict(dev, ablator, model)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/captum/attr/_models/base.py:189: UserWarning: In order to make embedding layers more interpretable they will be replaced with an interpretable embedding layer which wraps the original embedding layer and takes word embedding vectors as inputs of the forward function. This allows us to generate baselines for word embeddings and compute attributions for each embedding dimension. The original embedding layer must be set back by calling `remove_interpretable_embedding_layer` function after model interpretation is finished. \n",
            "  \"In order to make embedding layers more interpretable they will \"\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "37467989caf54aa89936e7c8ca9584e1",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=13.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/captum/_utils/gradient.py:58: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the gradient for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations.\n",
            "  if input.grad is not None:\n",
            "/usr/local/lib/python3.6/dist-packages/captum/_utils/gradient.py:87: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the gradient for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations.\n",
            "  if input.grad is not None:\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:30: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:33: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(0.8929330065359476, 0.8344035212322379)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sJReBhs046bd"
      },
      "source": [
        "## Stability\n",
        "\n",
        "With this property we test whether instances with similar\n",
        "rationales also receive similar explanations.\n",
        "\n",
        "Here, for simplicity, we will consider two instances to have similar rationales if the input is similar and the produced output is the same. A more consistent approach would be also to measure the similarity between the activation maps in the separate layers, which we won't consider here for computational reasons.\n",
        "\n",
        "To simplify the experiment further, we will add some words in the end of the instances, which don't change their meaning and will measure the correlation between the change in the prediction and the change in the saliency maps."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XTz0qb84mQSN"
      },
      "source": [
        "from scipy.stats import pearsonr, spearmanr\n",
        "from scipy.spatial import distance"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JDLJOeXm4yfu"
      },
      "source": [
        "def add_dataset_sentence(dataset):\n",
        "  for i in range(len(dataset)):\n",
        "    dataset.dataset[i] = (dataset[i][0] + ' Bye.', dataset[i][1])\n",
        "  return dataset\n",
        "\n",
        "def stability_attributions(dataset, attribution_method, model, edit_fn, neg_inf=1e-10, abs=False):\n",
        "  predictions_old, attributions_old, token_ids_old,_ = attribute_predict(collate_fn, \n",
        "                                                                         model_args, \n",
        "                                                                         dataset, \n",
        "                                                                         attribution_method,\n",
        "                                                                         model, \n",
        "                                                                         target=0)\n",
        "\n",
        "  dataset_edited = edit_fn(dataset)\n",
        "  predictions_new, attributions_new, token_ids_new,_ = attribute_predict(collate_fn,\n",
        "                                                                         model_args,\n",
        "                                                                         dataset_edited, \n",
        "                                                                         attribution_method, \n",
        "                                                                         model, \n",
        "                                                                         target=0)\n",
        "  diff_pred, diff_attr = [], []\n",
        "\n",
        "  for p1, p2 in zip(predictions_old, predictions_new):\n",
        "    diff_pred.append(distance.euclidean(p1, p2))\n",
        "\n",
        "  for a1, a2, token_ids in zip(attributions_old, attributions_new, token_ids_old):\n",
        "    a1 = [token_score for i, token_score in enumerate(a1) if token_ids[i]!=tokenizer.pad_token_id]\n",
        "    a2 = a2[:len(a1)]\n",
        "    diff_attr.append(distance.euclidean(a1, a2))\n",
        "\n",
        "  if abs:\n",
        "    diff_pred = np.abs(diff_pred)\n",
        "    diff_attr = np.abs(diff_attr)\n",
        "  \n",
        "  return pearsonr(diff_pred, diff_attr), spearmanr(diff_pred, diff_attr)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 269,
          "referenced_widgets": [
            "3abea2fbddff483fb7d340106ce5b79e",
            "8d5e9e04823c4472b7712a510b1ebd54",
            "f56c6a0905274da0ad94437dc5a6721b",
            "02a4c5fd5a664f7aade917d7d66a1578",
            "a5318d44129546f296d5d036bb952c15",
            "9b854947f2fa42ff83c2f8b1a14a8dba",
            "54f50efa178c45da9f66803b2e73c9d1",
            "a7f923bb32f74ab09b981ec4708bbfc6",
            "8d8158cf9fa84af2a4ac632110e2f883",
            "3cb04339779840c9931e966dc75ef277",
            "5e65464d57a04687ba6e69a57cc8b5fb",
            "7cb94b9b0cc5434eacf005f1af80f76f",
            "5e56debe576a4ab287d8ec88d5a3cae9",
            "52bf7c646aac4afc91d0380d71e2ea48",
            "8027ceca8e0240b998ac585fca7074e6",
            "00cb837ec6244e08a6abe05930632300"
          ]
        },
        "id": "gZCYeqeQ5bLL",
        "outputId": "67a8e571-753a-47d9-8ee3-54fc8bc61496"
      },
      "source": [
        "stability_attributions(dev, ablator, model, add_dataset_sentence)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/captum/attr/_models/base.py:189: UserWarning: In order to make embedding layers more interpretable they will be replaced with an interpretable embedding layer which wraps the original embedding layer and takes word embedding vectors as inputs of the forward function. This allows us to generate baselines for word embeddings and compute attributions for each embedding dimension. The original embedding layer must be set back by calling `remove_interpretable_embedding_layer` function after model interpretation is finished. \n",
            "  \"In order to make embedding layers more interpretable they will \"\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3abea2fbddff483fb7d340106ce5b79e",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=13.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/captum/_utils/gradient.py:58: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the gradient for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations.\n",
            "  if input.grad is not None:\n",
            "/usr/local/lib/python3.6/dist-packages/captum/_utils/gradient.py:87: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the gradient for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations.\n",
            "  if input.grad is not None:\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8d8158cf9fa84af2a4ac632110e2f883",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=13.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((0.5534135545926878, 2.3510616145391444e-09),\n",
              " SpearmanrResult(correlation=0.5269486948694869, pvalue=1.7811362756981797e-08))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TUOeJ72PbNK3"
      },
      "source": [
        "## Task 5\n",
        "\n",
        "Experiment with the model to find what characteristics make it more difficult for interpreting.\n",
        "\n",
        "Guidelines:\n",
        "You can think of several parameters that you could change and train a new model assuming that the resulting model would be harder to interpret. (The CNN model is the fastest to train, you could also decrease the number of epochs in the model_args).\n",
        "\n",
        "Then, you can compare the faithfulness and the stability of the same explainability approach to see what has changed.\n",
        "\n",
        "Why do you think is that?\n",
        "Let us know of your findings!"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4NYPIZ-sq58N"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}