CV_14_NN_compression_LAB3_answers.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "AIMS_NN_compression_LAB3.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "5fad0a3fcbfe405f824f1ff07e7b8227": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_a7a0665c8b114d59b56e207fadc41174",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_a8141df7ba474c0d802e46c4eb058583",
              "IPY_MODEL_bcd08e9deb184c41a3ca634162231ad5"
            ]
          }
        },
        "a7a0665c8b114d59b56e207fadc41174": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a8141df7ba474c0d802e46c4eb058583": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_bae8a5a887654320b172b81d57b881eb",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b17f83f8b54943138c0e8ffe9465a717"
          }
        },
        "bcd08e9deb184c41a3ca634162231ad5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_4f75744c392e4ef68b41a0120e77fb52",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 9920512/? [00:20&lt;00:00, 1698879.38it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a4c15211ebc545e3a944f16d02120785"
          }
        },
        "bae8a5a887654320b172b81d57b881eb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b17f83f8b54943138c0e8ffe9465a717": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4f75744c392e4ef68b41a0120e77fb52": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a4c15211ebc545e3a944f16d02120785": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5b25805c311640aa93055a82527d1ff8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_aa52c52932274c6fa402594886eb4704",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_d170e0d7bf0c424fbe46cf863d999310",
              "IPY_MODEL_bee4bfdb8547498f9f79057bb12a1588"
            ]
          }
        },
        "aa52c52932274c6fa402594886eb4704": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d170e0d7bf0c424fbe46cf863d999310": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_4d7c536167bb45338c596087423176cc",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f5318d2af5a14296b83f043504d94747"
          }
        },
        "bee4bfdb8547498f9f79057bb12a1588": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f8ad7f73f13445f8be345672762f48d0",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 32768/? [00:03&lt;00:00, 10425.20it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_eaaff6db9a7846c6bfb6f5741b5f061c"
          }
        },
        "4d7c536167bb45338c596087423176cc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f5318d2af5a14296b83f043504d94747": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f8ad7f73f13445f8be345672762f48d0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "eaaff6db9a7846c6bfb6f5741b5f061c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0333316b73e6418ca2bd0f5e98037984": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_9fd3c84840d14cbea6ff04e80ca9c72a",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_fe5bba9393e747988bbdcca3efcffa3f",
              "IPY_MODEL_cbe6202bbb524fc4a5a8980eb4940661"
            ]
          }
        },
        "9fd3c84840d14cbea6ff04e80ca9c72a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fe5bba9393e747988bbdcca3efcffa3f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_5ca1f394f7c742a6821939264bfc83c9",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e226cadbf8714852a9d1e8879ceee090"
          }
        },
        "cbe6202bbb524fc4a5a8980eb4940661": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_1b5b701afb0d480a88469fd6b89c327b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1654784/? [00:02&lt;00:00, 684384.32it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c346be83325543f0beaad630278d4e48"
          }
        },
        "5ca1f394f7c742a6821939264bfc83c9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e226cadbf8714852a9d1e8879ceee090": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1b5b701afb0d480a88469fd6b89c327b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c346be83325543f0beaad630278d4e48": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "01d3633acf534050a2a1cfb5e676b81e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_037becfd84c849e983b1361e3378c63e",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_817051434c664b1f9674f6c5204b7003",
              "IPY_MODEL_e0e49d6fb57e42ca9234ae87e85a7f6e"
            ]
          }
        },
        "037becfd84c849e983b1361e3378c63e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "817051434c664b1f9674f6c5204b7003": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_feb7e1c89a464722aaf0a2cc37b9e3f0",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_23675d2e697246c981e3405ef669075d"
          }
        },
        "e0e49d6fb57e42ca9234ae87e85a7f6e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_30406e30af084e4bb104149c0b3e80d9",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 8192/? [00:00&lt;00:00, 12767.71it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_75f3071fd1804732b48545950952e70f"
          }
        },
        "feb7e1c89a464722aaf0a2cc37b9e3f0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "23675d2e697246c981e3405ef669075d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "30406e30af084e4bb104149c0b3e80d9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "75f3071fd1804732b48545950952e70f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Gedeon-m-gedus/Image_Processing/blob/master/CV_14_NN_compression_LAB3_answers.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gtDXorK6UI6p",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 292
        },
        "outputId": "9e82d71d-9866-483d-f9a5-0bff491d3ea1"
      },
      "source": [
        "import argparse\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "from torchvision import datasets, transforms\n",
        "from torch.optim.lr_scheduler import StepLR\n",
        "!pip install tensorly\n",
        "import tensorly as tl\n",
        "#Need to set Tensorly's backend as pytorch\n",
        "tl.set_backend('pytorch')\n",
        "from tensorly import random, tenalg\n",
        "import numpy as np\n",
        "np.random.seed(0)\n",
        "import copy \n",
        "import time\n",
        "def tic():\n",
        "  return time.time()\n",
        "def toc(t):\n",
        "  return time.time()-t\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting tensorly\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/37/d6/8ddeec3d840635b78fdb1ed8f3dae938553b0a38c245b9c59845e89e4c2b/tensorly-0.4.5.tar.gz (70kB)\n",
            "\u001b[K     |████████████████████████████████| 71kB 1.9MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from tensorly) (1.18.5)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from tensorly) (1.4.1)\n",
            "Collecting nose\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/15/d8/dd071918c040f50fa1cf80da16423af51ff8ce4a0f2399b7bf8de45ac3d9/nose-1.3.7-py3-none-any.whl (154kB)\n",
            "\u001b[K     |████████████████████████████████| 163kB 7.5MB/s \n",
            "\u001b[?25hBuilding wheels for collected packages: tensorly\n",
            "  Building wheel for tensorly (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for tensorly: filename=tensorly-0.4.5-cp36-none-any.whl size=100156 sha256=9d0ea97e2e0f37b1a41f3c7602f061ad5cf2a846d762d68f285393fd5e616c5a\n",
            "  Stored in directory: /root/.cache/pip/wheels/44/ae/02/8d00229a4fd0af192b48d24da903f9975c7ac10e706685fc39\n",
            "Successfully built tensorly\n",
            "Installing collected packages: nose, tensorly\n",
            "Successfully installed nose-1.3.7 tensorly-0.4.5\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PlTzu2GM8sxK",
        "colab_type": "text"
      },
      "source": [
        "#Background\n",
        "In recent researches, we have seen numerous successes with neural networks, epseicially deep learning methods. However, for these deep architectures, the number of parameters is often extremely big. This overparameterized structure, although often has state-of-the-art performance, also suffers from  long processing time and the need of expensive hardwares. Therefore, many efforts have been made into deriving deep learning models with less number of parameters but still remains similar performance. One interesting idea is to leverage the structure of matrix product operator (MPO) and represent the weights of the NN using this MPO format and directly do gradient descent w.r.t this structure. In this practice, we will explore how to train a fully connected neural network in its matrix product operator format. \\\\\n",
        "This practice is inspired by the paper: https://arxiv.org/abs/1509.06569"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "I10qf-zR9pmY",
        "colab_type": "text"
      },
      "source": [
        "# Directly learning the MPO represented NN (MPO-NN)\n",
        "## Detour: What is MPO and how to contract a MPO with an input tensor?\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jQM9NFm3xtAs",
        "colab_type": "text"
      },
      "source": [
        "Matrix product operator (MPO) is essentially an MPS with one extra leg on each core. For example, in the following figure, we should a typical MPO structure of the tensor $\\mathcal{G} = [\\mathcal{G}_1, \\mathcal{G}_2, \\mathcal{G}_3, \\mathcal{G}_4]$, which is of size $c_1\\times c_2\\times c_3\\times c_4\\times d_1\\times d_2\\times d_3\\times d_4$. As a similar notion to MPS, the leg connecting each core representing the MPO rank. In this example, the rank of the MPO is a list of $[r_1, r_2, r_3]. $![MPO_strcurtre](https://docs.google.com/uc?export=download&id=1a9GRYOhrC8Zn_rcumigt8gc4EZkKJJuz)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DT2zecGPnjRu",
        "colab_type": "text"
      },
      "source": [
        "### 1. \n",
        "In this first problem, you will need to implement the MPO contraction operation between a MPO and an input tensor $\\mathcal{X}$. Assuming you are given a MPO $\\mathcal{G}$ which is a list of MPO cores, i.e. $[\\mathcal{G}_1, \\cdots \\mathcal{G}_m]$ and the MPO is of the rank $[r_1 \\cdots r_{m-1}]$. We also assume that the i-th core is of the shape $r_{i-1}\\times d_i\\times c_i\\times r_i$, where for the first core we let $r_{0}= 1$ and for the last core $r_m = 1$ (Note a leg with dimension 1 is equivalent to an empty leg), as shown in the following figure. \n",
        "\n",
        "![MPO_core](https://docs.google.com/uc?export=download&id=1OAWiLk3kCBarhzQ6-BhmFyabmZi65eSh) \n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_Y0TN5k08-qS",
        "colab_type": "text"
      },
      "source": [
        "Now, given an input tensor $\\mathcal{X}\\in \\mathbb{R}^{n \\times d_1 \\times \\cdots \\times d_m}$, where n is the batch size, implement the contraction between the corresponding legs such that the contracted tensor will be of size $n \\times c_1\\times \\cdots \\times c_m$. As an example, for an MPO with 4 cores, the contraction between it and an input tensor $\\mathcal{X}$ is shown as the following:\n",
        "\n",
        "![MPO_strcurtre](https://docs.google.com/uc?export=download&id=1pOQn5YeqfwWWD7mIemC3sYjecp59IYue) "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SMTW4PnF_-Ti",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def MPO_contraction(mpo, x, return_vec = True):\n",
        "  '''\n",
        "  mpo: a list of MPO cores (tensorly format), the i-th core is of shape (r_{i-1}, d_i, c_i, r_i), \n",
        "       for the first core r_0 =1, for the last core r_m = 1.\n",
        "  x: input tensor to be contracted (tensorly format), should be of size (n, d_1, d_2, ..., d_m).\n",
        "  return: a tensor of shape (n, c_1, c_2, ..., c_m)\n",
        "  '''\n",
        "  assert len(mpo) == x.ndim - 1, \"dimension mismatch between MPO and input tensor x\"\n",
        "  for i in range(len(mpo)):\n",
        "    assert mpo[i].shape[1] == x.shape[i+1], str(i)+\"th core does not match the input dimension, note the thrid dimension of the mpo core should be the same of the corresponding input tensor dimension\" \n",
        "\n",
        "  output_tmp = tenalg.contract(x, [1], mpo[0], [1])\n",
        "  for i in range(1, x.ndim - 1):\n",
        "    output_tmp = tenalg.contract(output_tmp, [1, output_tmp.ndim - 1], mpo[i], [1, 0])\n",
        "  return output_tmp.squeeze()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_xwmReDKA97n",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "results = [[410.3555, 367.0687, 394.2908, 358.6689, 381.4024, 341.9384, 360.2169,\n",
        "        329.8646, 407.1425, 363.4720, 393.6679, 355.0994, 390.3836, 349.0190,\n",
        "        371.2960, 335.3856, 381.8200, 349.6505, 372.9323, 343.3012, 366.3085,\n",
        "        336.4729, 351.3506, 325.3335], \n",
        "       [356.8043, 418.2853, 361.2211, 410.5959, 316.2948, 370.2491, 312.2206,\n",
        "        352.7069, 382.1384, 444.2850, 380.6153, 436.5235, 355.7083, 413.0649,\n",
        "        344.6199, 393.3047, 396.4348, 455.7986, 393.3049, 447.2321, 361.2296,\n",
        "        414.9194, 348.0826, 394.7057],\n",
        "        [354.9081, 293.2105, 359.8825, 296.4861, 377.5464, 315.0949, 377.2471,\n",
        "        312.9066, 370.3658, 318.1749, 367.7635, 312.0098, 404.4339, 350.1533,\n",
        "        393.8055, 334.7884, 376.3939, 327.9788, 379.5774, 330.4409, 408.0018,\n",
        "        352.9874, 404.4712, 350.2234],\n",
        "        [318.2197, 296.4062, 346.0204, 326.2782, 296.7090, 278.4173, 326.6268,\n",
        "        307.8708, 329.1500, 309.8562, 356.8482, 340.7281, 308.3035, 291.0971,\n",
        "        335.1161, 318.2343, 336.9507, 307.0458, 366.0646, 340.7538, 315.1257,\n",
        "        290.0126, 346.2084, 322.9559],\n",
        "        [318.1615, 349.5595, 331.5495, 369.5155, 328.6678, 364.3328, 342.3642,\n",
        "        382.9929, 330.7670, 360.3952, 347.7114, 379.8224, 354.8564, 389.9683,\n",
        "        370.4451, 407.6407, 355.8427, 385.7378, 372.6829, 404.4416, 379.2988,\n",
        "        412.8916, 393.8806, 429.8946],\n",
        "        [270.0208, 350.8890, 262.4508, 333.1976, 293.0804, 382.3908, 288.4987,\n",
        "        368.0071, 282.7515, 381.6471, 271.3544, 362.4180, 303.4720, 409.4315,\n",
        "        296.4285, 395.9372, 333.2086, 421.1110, 319.1843, 399.1218, 358.9009,\n",
        "        454.4263, 350.2224, 437.3429],\n",
        "        [352.3250, 368.0678, 358.6050, 376.5540, 373.9196, 390.7939, 386.3511,\n",
        "        400.3182, 359.8570, 371.7061, 368.3690, 378.2644, 375.4347, 388.3662,\n",
        "        389.0728, 397.3719, 391.6757, 409.0408, 401.8670, 415.1198, 410.2174,\n",
        "        428.3144, 426.0149, 437.9070],\n",
        "        [389.6984, 366.2637, 408.0897, 381.6472, 333.4066, 313.0393, 353.7299,\n",
        "        331.9791, 415.8656, 384.7394, 432.8951, 405.1320, 356.6425, 330.8488,\n",
        "        379.1675, 357.9731, 456.8054, 414.1855, 482.8067, 437.2590, 393.1959,\n",
        "        357.1537, 421.8253, 384.6655],\n",
        "        [343.7931, 335.9971, 359.5031, 351.5452, 298.8585, 291.3354, 321.7639,\n",
        "        311.5385, 319.9868, 315.0025, 331.4951, 329.4894, 281.8465, 278.9174,\n",
        "        299.7816, 297.7877, 338.9509, 337.0882, 350.7429, 348.8928, 305.9270,\n",
        "        303.5083, 326.6728, 323.9245],\n",
        "        [542.8569, 527.5298, 541.1932, 531.6619, 430.4786, 419.5898, 421.7222,\n",
        "        409.8663, 523.4540, 494.5858, 518.2686, 499.6876, 395.8323, 376.9006,\n",
        "        383.6669, 367.5051, 512.1805, 493.6794, 503.1376, 493.2400, 402.5128,\n",
        "        393.7093, 390.9211, 382.5303] ]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RYEElL8OEd9N",
        "colab_type": "text"
      },
      "source": [
        "You can run the following program to make sure the output of your function is correct. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g1N5vsYOBAfG",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        },
        "outputId": "12fa620e-9bdc-4f33-cf45-1b88d19c4b11"
      },
      "source": [
        "#Test for MPO_contraction\n",
        "np.random.seed(0)\n",
        "for k in range(10):\n",
        "  x = tl.tensor(np.random.rand(3, 4, 5, 6))\n",
        "  mpo = []\n",
        "  rank = 7\n",
        "  out_dim = 2\n",
        "  for i in range(1, x.ndim):\n",
        "    if i == 1:\n",
        "      dim_0 = 1 \n",
        "    else:\n",
        "      dim_0 = rank\n",
        "    if i == x.ndim - 1:\n",
        "      dim_3 = 1\n",
        "    else:\n",
        "      dim_3 = rank\n",
        "    dim_1 = x.shape[i]\n",
        "    dim_2 = out_dim\n",
        "    tmp = tl.tensor(np.random.rand(dim_0, dim_1, dim_2, dim_3))\n",
        "    mpo.append(tmp)\n",
        "  out = tl.to_numpy(tl.tensor_to_vec(MPO_contraction(mpo, x))).round(decimals = 4)\n",
        "  results[k] = np.asarray(results[k])\n",
        "  if ((out - results[k]) < 1e-3).all():\n",
        "    print('Test '+str(k)+' passed!')\n",
        "  else:\n",
        "    print('Test '+str(k)+' failed!')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Test 0 passed!\n",
            "Test 1 passed!\n",
            "Test 2 passed!\n",
            "Test 3 passed!\n",
            "Test 4 passed!\n",
            "Test 5 passed!\n",
            "Test 6 passed!\n",
            "Test 7 passed!\n",
            "Test 8 passed!\n",
            "Test 9 passed!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BcKhP8gYBFtZ",
        "colab_type": "text"
      },
      "source": [
        "## Define the MPO-NN stucture the forward pass\n",
        "### 2. \n",
        "From the structure of MPO we can see that an MPO essentially has two set of legs: using the previous notations, one set is $\\{c_1,\\cdots c_m\\}$, the other one is $\\{d_1,\\cdots d_n\\}$. In the above problem, we implemented the contraction between the input tensor and one set of legs of the MPO. The resulted tensor is of shape $(n, c_1, \\cdots, c_m)$. Note this can be seen as another input tensor and can be contracted with another MPO. One can see that this layer-by-layer structure is very similar to multi-layer perceptron (MLP). The idea of MPO neural networks, or the so-called tensorized neural networks, is to replace the fully-connected (or convolutional layer) weights (i.e. weight matrices) by an MPO structure, then use gradient descent to learn these MPO layers. This replacement will drastically reduce the number of parameters that the neural networks need. In this problem, we will be implementing MPO-NN to replace a fully connected neural networks. The dataset we use here is the classic MNist hand-written dataset, which contains pictures of size $28\\times 28$. \n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "B-f2tNWL-l09",
        "colab_type": "text"
      },
      "source": [
        "# Prepare the data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2NRCWJZW-hKQ",
        "colab_type": "text"
      },
      "source": [
        "In this practice, we will be using the classic Mnist dataset. The first step is of course to load the dataset from Pytorch. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IY4DEnur_ETz",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 352,
          "referenced_widgets": [
            "5fad0a3fcbfe405f824f1ff07e7b8227",
            "a7a0665c8b114d59b56e207fadc41174",
            "a8141df7ba474c0d802e46c4eb058583",
            "bcd08e9deb184c41a3ca634162231ad5",
            "bae8a5a887654320b172b81d57b881eb",
            "b17f83f8b54943138c0e8ffe9465a717",
            "4f75744c392e4ef68b41a0120e77fb52",
            "a4c15211ebc545e3a944f16d02120785",
            "5b25805c311640aa93055a82527d1ff8",
            "aa52c52932274c6fa402594886eb4704",
            "d170e0d7bf0c424fbe46cf863d999310",
            "bee4bfdb8547498f9f79057bb12a1588",
            "4d7c536167bb45338c596087423176cc",
            "f5318d2af5a14296b83f043504d94747",
            "f8ad7f73f13445f8be345672762f48d0",
            "eaaff6db9a7846c6bfb6f5741b5f061c",
            "0333316b73e6418ca2bd0f5e98037984",
            "9fd3c84840d14cbea6ff04e80ca9c72a",
            "fe5bba9393e747988bbdcca3efcffa3f",
            "cbe6202bbb524fc4a5a8980eb4940661",
            "5ca1f394f7c742a6821939264bfc83c9",
            "e226cadbf8714852a9d1e8879ceee090",
            "1b5b701afb0d480a88469fd6b89c327b",
            "c346be83325543f0beaad630278d4e48",
            "01d3633acf534050a2a1cfb5e676b81e",
            "037becfd84c849e983b1361e3378c63e",
            "817051434c664b1f9674f6c5204b7003",
            "e0e49d6fb57e42ca9234ae87e85a7f6e",
            "feb7e1c89a464722aaf0a2cc37b9e3f0",
            "23675d2e697246c981e3405ef669075d",
            "30406e30af084e4bb104149c0b3e80d9",
            "75f3071fd1804732b48545950952e70f"
          ]
        },
        "outputId": "a48aa962-a6de-4cbe-de45-0c3e356ae210"
      },
      "source": [
        "use_cuda = True # we will be using gpu to train the model, therefore setting use_cuda to True\n",
        "batch_size = 512\n",
        "test_batch_size = 2048\n",
        "kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}\n",
        "# Load MNIST dataset from pytorch using dataloader\n",
        "train_loader = torch.utils.data.DataLoader(\n",
        "    datasets.MNIST('../data', train=True, download=True,\n",
        "                    transform=transforms.Compose([\n",
        "                        transforms.ToTensor(),\n",
        "                        transforms.Normalize((0.1307,), (0.3081,))\n",
        "                    ])),\n",
        "    batch_size=batch_size, shuffle=True, **kwargs)\n",
        "test_loader = torch.utils.data.DataLoader(\n",
        "    datasets.MNIST('../data', train=False, transform=transforms.Compose([\n",
        "                        transforms.ToTensor(),\n",
        "                        transforms.Normalize((0.1307,), (0.3081,))\n",
        "                    ])),\n",
        "    batch_size=test_batch_size, shuffle=True, **kwargs)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz to ../data/MNIST/raw/train-images-idx3-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "5fad0a3fcbfe405f824f1ff07e7b8227",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Extracting ../data/MNIST/raw/train-images-idx3-ubyte.gz to ../data/MNIST/raw\n",
            "Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz to ../data/MNIST/raw/train-labels-idx1-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "5b25805c311640aa93055a82527d1ff8",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Extracting ../data/MNIST/raw/train-labels-idx1-ubyte.gz to ../data/MNIST/raw\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz to ../data/MNIST/raw/t10k-images-idx3-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0333316b73e6418ca2bd0f5e98037984",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Extracting ../data/MNIST/raw/t10k-images-idx3-ubyte.gz to ../data/MNIST/raw\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz to ../data/MNIST/raw/t10k-labels-idx1-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "01d3633acf534050a2a1cfb5e676b81e",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Extracting ../data/MNIST/raw/t10k-labels-idx1-ubyte.gz to ../data/MNIST/raw\n",
            "Processing...\n",
            "Done!\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/pytorch/torch/csrc/utils/tensor_numpy.cpp:141: UserWarning: The given NumPy array is not writeable, and PyTorch does not support non-writeable tensors. This means you can write to the underlying (supposedly non-writeable) NumPy array using the tensor. You may want to copy the array to protect its data or make it writeable before converting it to a tensor. This type of warning will be suppressed for the rest of this program.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1t2JGY_R_Lvz",
        "colab_type": "text"
      },
      "source": [
        "Then let us create training_loader, validation_loader and test_loader for our data to facilitate our training process later on. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4ifXboVa_dFN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Use the first 50000 data for training and the rest for validation\n",
        "train_ds  = torch.utils.data.Subset(train_loader.dataset, range(0, 50000-1))\n",
        "train_ds = torch.utils.data.DataLoader(train_ds, batch_size=batch_size, shuffle=True, **kwargs)\n",
        "valid_ds = torch.utils.data.Subset(train_loader.dataset, range(50000, 60000-1))\n",
        "valid_ds = torch.utils.data.DataLoader(valid_ds, batch_size=batch_size, shuffle=True, **kwargs)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hYMwzIg9Garw",
        "colab_type": "text"
      },
      "source": [
        "#### (a)\n",
        "In this problem, you need to implement the function `MPO_layer`, which takes a list of mpo ranks, a list of input tensor dimensions and a list of output tensor dimensions. The function outputs a list of MPO cores with corresponding dimensions determined by the input argument. For example, if `mpo_rank = [2,3]`, `input_dims = [4,5,6]`, `output_dims = [7,8,9]`, then the output of this function should be a list of 3 tensors, which are of shape `(1,4,7,2)`, `(2,5,8,3)` and `(3,6,9,1)`. Each core should be initialized with weights drawn from a uniform distribution $(-1/sqrt(a), 1/sqrt(a))$, where $a$ is the number of parameters for the corresponding core."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fk1pmKjkSzbK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z6Td5Vo4XitE",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        },
        "outputId": "b343960f-05eb-45d7-85f7-28e7ec834429"
      },
      "source": [
        "def get_tensor_size(tensor):\n",
        "  size_tmp = 1.\n",
        "  for dim in tensor.size():\n",
        "    size_tmp *= dim\n",
        "  return size_tmp\n",
        "  \n",
        "def get_tensor_size_given_shape(shape):\n",
        "  size_tmp = 1.\n",
        "  for dim in shape:\n",
        "    size_tmp *= dim\n",
        "  return size_tmp\n",
        "def check_list_and_int(x, var_name):\n",
        "  assert isinstance(x, list) or isinstance(x, int), 'Expecting ' + var_name + ' to be int or a list of int'\n",
        "  if isinstance(x, list):\n",
        "    assert all(isinstance(item, int) for item in x), 'Expecting ' + var_name + ' to be int or a list of int'\n",
        "  return \n",
        "\n",
        "\n",
        "def MPO_layer(mpo_rank, input_dims, output_dims, num_cores = None, seed = 0, device = 'cuda'):\n",
        "  check_list_and_int(mpo_rank, 'mpo_rank')\n",
        "  check_list_and_int(input_dims, 'input_dims')\n",
        "  check_list_and_int(output_dims, 'output_dims')\n",
        "  if isinstance(input_dims, list) and isinstance(output_dims, list):\n",
        "    assert len(input_dims) == len(output_dims), 'if both input_dims and output_dims are list, then len(input_dims) should be the same as len(output_dims)'\n",
        "  if isinstance(input_dims, list) or isinstance(output_dims, list):\n",
        "    if num_cores is None:\n",
        "      num_cores = len(input_dims) if isinstance(input_dims, list) else len(output_dims)\n",
        "    else:\n",
        "      num_cores_dims = len(input_dims) if isinstance(input_dims, list) else len(output_dims)\n",
        "      assert num_cores_dims == num_cores, 'The length of the input_dims/output_dims mismatch num_cores'\n",
        "  else:\n",
        "    assert num_cores is not None, 'If input_dims and output_dims are not list, you need to specify num_cores'\n",
        "  if isinstance(mpo_rank, list):\n",
        "    assert len(mpo_rank) + 1 == num_cores, 'len(mpo_rank) + 1  should be the same as num_cores/len(input_dims)/len(output_dims)'\n",
        "      \n",
        "  np.random.seed(seed)\n",
        "  mpo_layer = []\n",
        "  for i in range(num_cores):\n",
        "    if i == 0:\n",
        "      dim_0 = 1\n",
        "    else:\n",
        "      dim_0 = mpo_rank[i-1] if isinstance(mpo_rank, list) else mpo_rank\n",
        "    if i == num_cores - 1:\n",
        "      dim_3 = 1\n",
        "    else:\n",
        "      dim_3 = mpo_rank[i] if isinstance(mpo_rank, list) else mpo_rank\n",
        "    dim_1 = input_dims[i] if isinstance(input_dims, list) else input_dim\n",
        "    dim_2 = output_dims[i] if isinstance(output_dims, list) else output_dim\n",
        "\n",
        "    tmp = tl.tensor(np.random.rand(dim_0, dim_1, dim_2, dim_3))\n",
        "    bound = 1./np.sqrt(get_tensor_size(tmp))\n",
        "    \n",
        "    tmp = tmp*2*bound - bound\n",
        "    mpo_layer.append(tl.tensor(tmp, device = device, requires_grad=True))\n",
        "  return mpo_layer"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0Qw8OYiNvmNp",
        "colab_type": "text"
      },
      "source": [
        "#### (b)\n",
        "Finish writing the class `Net_MPO`, you will need to implement the `__init__` function and the `forward` function. In `__init__` you will need to define the network structure using the function `MPO_layer` you have just implemented in the previous question. \n",
        "\n",
        "For `__init__`, you will need to write the strcutre of the MPO-NN. For this particular problem, the network you will be implementing has two MPO layers and then a fully  connected layer to connect to the output. After each MPO layers, there are two `ReLu` layer, which takes the intermediate output and pass it through a `ReLu` function. You will need to construct a list of MPO layers, which is called `self.mpo_layers`, this should contain 2 lists, each of the list correspond to the MPO cores for that particular MPO layer. The function takes 5 arguements, `number_layers` indicates how many layers you want for the MPO-NN; `mpo_rank` is a list of MPO ranks for each MPO layer. For example, if there are 2 MPO layers, then `mpo_rank` is a list of two lists, where each list contains the MPO ranks for the respective MPO layer. `hidden_dims`  is also in this format, it is a list of all hidden layer dimensions. These hidden layer dimensions correspond to the output tensor shape of each MPO layer that you have implemented in the `MPO_layer` function. `input_shape` is the input tensor's shape and finally `output_dim` is the output dimension. For this particular dataset, the `output_dim` is 10 (10 classes in total). **Use the default parameters for your network structure.**\n",
        " \n",
        "\n",
        "For `forward` function, it only takes the input tensor of size $(n, 28, 28)$ as argument, where $n$ is the batch size and $28\\times 28$ is the input image's dimension. The output shape of this function should be $n \\times 10$. You will first need to reshpe the input tensor to the input_shape defined in the `__init__` function. Then you will need to write a forward pass of the desginated MPO networks, which contains two contractions with the MPO layers and use `ReLu` function to activate after each MPO contraction and finally multiply by the last fully connected layer. **Use the function you implemented in (1)**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7nbxXQjhd3Y0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Net_MPO(nn.Module):\n",
        "  def __init__(self, number_layers = 2, mpo_ranks = [[5, 5, 5], [5,5,5]], \n",
        "               input_shape = [4, 7, 4, 7], output_dim = 10, \n",
        "               hidden_dims = [[5, 5, 5, 5], [5, 5, 5, 5]], seed = 0, device = 'cuda'):\n",
        "    super(Net_MPO, self).__init__()\n",
        "    assert number_layers == len(mpo_ranks), 'Len(mpo_ranks) should be the same as number of layers'\n",
        "    assert len(hidden_dims) == number_layers, 'Len(hidden_dims) should be number of layers minus 1'\n",
        "\n",
        "    # Construct MPO layers\n",
        "    mpo_layers = []\n",
        "    mpo_layer = MPO_layer(mpo_rank = mpo_ranks[0], input_dims = input_shape, output_dims = hidden_dims[0], num_cores = None, seed = 0, device = 'cuda')\n",
        "    mpo_layers.append(mpo_layer)\n",
        "    for i in range(1, number_layers):\n",
        "      mpo_layer = MPO_layer(mpo_rank = mpo_ranks[i], input_dims = hidden_dims[i-1], output_dims = hidden_dims[i], num_cores = None, seed = 0, device = 'cuda')\n",
        "      mpo_layers.append(mpo_layer)\n",
        "    self.mpo_layers = mpo_layers\n",
        "\n",
        "    #Construct final fully-connected layer\n",
        "    fc_in = get_tensor_size_given_shape(hidden_dims[-1])\n",
        "    self.fc = nn.Linear(int(fc_in), output_dim).to(device)\n",
        "\n",
        "    #Some class properties that might come handy\n",
        "    self.output_dim = output_dim\n",
        "    self.input_shape = input_shape\n",
        "    self.device = device\n",
        "\n",
        "  def forward(self, x):\n",
        "    new_shape = [-1]\n",
        "    for i in self.input_shape:\n",
        "      new_shape.append(i)\n",
        "    inter_out = tl.tensor(x.reshape(new_shape)).to(self.device)\n",
        "\n",
        "    for mpo_layer in self.mpo_layers:\n",
        "      inter_out = MPO_contraction(mpo_layer, inter_out)\n",
        "      inter_out = F.relu(inter_out)\n",
        "    inter_out = tl.unfold(inter_out, 0).to(self.device)\n",
        "    \n",
        "    out = self.fc(inter_out)\n",
        "    out = F.log_softmax(out, dim=1)\n",
        "    return out\n",
        "\n",
        "  def get_number_parameters(self):\n",
        "    size = 0\n",
        "    for mpo_layer in self.mpo_layers:\n",
        "      for core in mpo_layer:\n",
        "        size += get_tensor_size_given_shape(core.shape)\n",
        "    fc_layer_shape = hidden_dims[-1]\n",
        "    fc_layer_shape.append(self.output_dim)\n",
        "    size += get_tensor_size_given_shape(fc_layer_shape)\n",
        "    return size\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "alUW8KV7Dmq0",
        "colab_type": "text"
      },
      "source": [
        "## Training MPO-NN"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gl1PvS6qDEK2",
        "colab_type": "text"
      },
      "source": [
        "The training and testing methods for the MPO-NN"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "soOnKuPZrjSk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#The train and test function of the Tensorized Network\n",
        "def train(model, device, train_loader, optimizer, epoch, log_interval = 10):\n",
        "    model.train()\n",
        "    error = []\n",
        "    for batch_idx, (data, target) in enumerate(train_loader):\n",
        "        data, target = data.to(device), target.to(device)\n",
        "        optimizer.zero_grad()\n",
        "        output = model(data).to(device)\n",
        "        loss = F.nll_loss(output, target)\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "        if batch_idx % log_interval == 0:\n",
        "            print('Train Epoch: {} [{}/{} ({:.0f}%)]\\tLoss: {:.6f}'.format(\n",
        "                epoch, batch_idx * len(data), len(train_loader.dataset),\n",
        "                100. * batch_idx / len(train_loader), loss.item()))\n",
        "          \n",
        "        error.append(loss.item())\n",
        "    return sum(error)/len(error)\n",
        "\n",
        "\n",
        "def test(model, device, test_loader, return_acc = False):\n",
        "    model.eval()\n",
        "    test_loss = 0\n",
        "    correct = 0\n",
        "    with torch.no_grad():\n",
        "        for data, target in test_loader:\n",
        "            data, target = data.to(device), target.to(device)\n",
        "            output = model(data).to(device)\n",
        "            test_loss += F.nll_loss(output, target, reduction='sum').item()  # sum up batch loss\n",
        "            pred = output.argmax(dim=1, keepdim=True)  # get the index of the max log-probability\n",
        "            correct += pred.eq(target.view_as(pred)).sum().item()\n",
        "\n",
        "    test_loss /= len(test_loader.dataset)\n",
        "\n",
        "    print('\\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\\n'.format(\n",
        "        test_loss, correct, len(test_loader.dataset),\n",
        "        100. * correct / len(test_loader.dataset)))\n",
        "    if return_acc:\n",
        "      return 100. * correct / len(test_loader.dataset)\n",
        "    else:\n",
        "      return test_loss\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TwxDLYgfD0ZJ",
        "colab_type": "text"
      },
      "source": [
        "Hyper-parameters setup"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1vZPycJ9Dzm1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "lr = 0.01\n",
        "save_model = True\n",
        "epochs = 10\n",
        "torch.manual_seed(1)\n",
        "gamma = 0.5\n",
        "log_interval = 100\n",
        "step_size = 3\n",
        "device = torch.device(\"cuda\" if use_cuda else \"cpu\")\n",
        "number_layers = 2\n",
        "mpo_ranks = [[5, 5, 5], [5,5,5]] \n",
        "input_shape = [4, 7, 4, 7]\n",
        "output_dim = 10\n",
        "hidden_dims = [[5, 5, 5, 5], [5, 5, 5, 5]]\n",
        "seed = 0"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rXURvdCeD3qA",
        "colab_type": "text"
      },
      "source": [
        "Training process"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TWQKqognD4qC",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 802
        },
        "outputId": "4ff7f5b5-7247-4dd8-9b76-fca6565873e1"
      },
      "source": [
        "\n",
        "model_MPO = Net_MPO(number_layers = number_layers, mpo_ranks =mpo_ranks, \n",
        "               input_shape = input_shape, output_dim = output_dim, \n",
        "               hidden_dims = hidden_dims, seed = seed)\n",
        "params = model_MPO.mpo_layers[0] + model_MPO.mpo_layers[0]\n",
        "#print(model_MPO.get_number_parameters())\n",
        "params.append(model_MPO.fc.weight)\n",
        "\n",
        "optimizer = optim.Adam(params , lr=lr)\n",
        "# scheduler for automatic decaying the learning rate\n",
        "scheduler = StepLR(optimizer, step_size=step_size, gamma=gamma)\n",
        "                   \n",
        "train_loss_tt = []\n",
        "test_loss_tt = []\n",
        "\n",
        "# Training\n",
        "for epoch in range(1, epochs + 1):\n",
        "    train_loss_tt.append(train(model_MPO, device, train_ds, optimizer, epoch, log_interval))\n",
        "    test_loss_tt.append(test(model_MPO, device, valid_ds))\n",
        "    scheduler.step()\n",
        "\n",
        "# Testing\n",
        "t = tic()\n",
        "MPO_acc = test(model_MPO, device, test_loader, return_acc = True)\n",
        "time_MPO = toc(t)\n",
        "if save_model:\n",
        "    torch.save(model_MPO.state_dict(), \"mnist_mpo.pt\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/tensorly/backend/pytorch_backend.py:36: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  requires_grad=requires_grad)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Train Epoch: 1 [0/49999 (0%)]\tLoss: 2.301429\n",
            "\n",
            "Test set: Average loss: 0.2443, Accuracy: 9313/9999 (93%)\n",
            "\n",
            "Train Epoch: 2 [0/49999 (0%)]\tLoss: 0.349698\n",
            "\n",
            "Test set: Average loss: 0.1686, Accuracy: 9518/9999 (95%)\n",
            "\n",
            "Train Epoch: 3 [0/49999 (0%)]\tLoss: 0.215096\n",
            "\n",
            "Test set: Average loss: 0.1413, Accuracy: 9594/9999 (96%)\n",
            "\n",
            "Train Epoch: 4 [0/49999 (0%)]\tLoss: 0.107486\n",
            "\n",
            "Test set: Average loss: 0.1212, Accuracy: 9667/9999 (97%)\n",
            "\n",
            "Train Epoch: 5 [0/49999 (0%)]\tLoss: 0.072934\n",
            "\n",
            "Test set: Average loss: 0.1159, Accuracy: 9675/9999 (97%)\n",
            "\n",
            "Train Epoch: 6 [0/49999 (0%)]\tLoss: 0.082413\n",
            "\n",
            "Test set: Average loss: 0.1135, Accuracy: 9671/9999 (97%)\n",
            "\n",
            "Train Epoch: 7 [0/49999 (0%)]\tLoss: 0.086253\n",
            "\n",
            "Test set: Average loss: 0.1074, Accuracy: 9692/9999 (97%)\n",
            "\n",
            "Train Epoch: 8 [0/49999 (0%)]\tLoss: 0.076102\n",
            "\n",
            "Test set: Average loss: 0.1056, Accuracy: 9693/9999 (97%)\n",
            "\n",
            "Train Epoch: 9 [0/49999 (0%)]\tLoss: 0.100694\n",
            "\n",
            "Test set: Average loss: 0.1031, Accuracy: 9704/9999 (97%)\n",
            "\n",
            "Train Epoch: 10 [0/49999 (0%)]\tLoss: 0.076554\n",
            "\n",
            "Test set: Average loss: 0.1000, Accuracy: 9718/9999 (97%)\n",
            "\n",
            "\n",
            "Test set: Average loss: 0.1046, Accuracy: 9683/10000 (97%)\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YrOdLPw1-CHx",
        "colab_type": "text"
      },
      "source": [
        "## Learning fully connected NN\n",
        "### FCNN structure\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CLDSQCVEUYxJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "class Net(nn.Module):\n",
        "    def __init__(self, hidden_units = 300, fc_weights = None):\n",
        "        super(Net, self).__init__()\n",
        "        self.hidden_units = hidden_units\n",
        "        if fc_weights is None:\n",
        "          self.fc1 = nn.Linear(28*28, hidden_units)\n",
        "          self.fc2 = nn.Linear(hidden_units, hidden_units)\n",
        "          self.fc3 = nn.Linear(hidden_units, 10)\n",
        "        else:\n",
        "          self.fc1 = nn.Linear(28*28, hidden_units)\n",
        "          self.fc2 = nn.Linear(hidden_units, hidden_units)\n",
        "          self.fc3 = nn.Linear(hidden_units, 10)\n",
        "          self.fc1.weight.data = torch.tensor(fc_weights[0])\n",
        "          self.fc2.weight.data = torch.tensor(fc_weights[1])\n",
        "          self.fc3.weight.data = torch.tensor(fc_weights[2])\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = torch.flatten(x, 1)\n",
        "        x = self.fc1(x)\n",
        "        x = F.leaky_relu(x)\n",
        "        x = self.fc2(x)\n",
        "        x = F.leaky_relu(x)\n",
        "        x = self.fc3(x)\n",
        "        output = F.log_softmax(x, dim=1)\n",
        "        #print(output[0])\n",
        "        return output\n",
        "        \n",
        "    def get_number_parameters(self):\n",
        "      return 28*28 * self.hidden_units + self.hidden_units**2 + self.hidden_units * 10"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TJze8E1fFJbl",
        "colab_type": "text"
      },
      "source": [
        "### Hyper-parameters setup"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b2cFJXdkyUzQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "use_cuda = True\n",
        "batch_size = 512\n",
        "test_batch_size = 2048\n",
        "lr = 0.001\n",
        "save_model = True\n",
        "epochs = 10\n",
        "torch.manual_seed(1)\n",
        "gamma = 0.1\n",
        "log_interval = 100\n",
        "hidden_units = 625\n",
        "step_size = 5\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FR2GmmuaFR6d",
        "colab_type": "text"
      },
      "source": [
        "### Training process"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mWhoVp0FFRVg",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 748
        },
        "outputId": "23df4579-486f-45cb-f4a8-600212923ae9"
      },
      "source": [
        "model_fc = Net(hidden_units).to(device)\n",
        "optimizer = optim.Adam(model_fc.parameters(), lr=lr)\n",
        "scheduler = StepLR(optimizer, step_size=step_size, gamma=gamma)\n",
        "\n",
        "train_loss_fc = []\n",
        "test_loss_fc = []\n",
        "\n",
        "for epoch in range(1, epochs + 1):\n",
        "    train_loss_fc.append(train(model_fc, device, train_ds, optimizer, epoch, log_interval))\n",
        "    test_loss_fc.append(test(model_fc, device, valid_ds))\n",
        "    scheduler.step()\n",
        "\n",
        "t = tic()\n",
        "fc_acc = test(model_fc, device, test_loader, return_acc = True)\n",
        "time_fc = toc(t)\n",
        "if save_model:\n",
        "    torch.save(model_fc.state_dict(), \"mnist_fc.pt\")\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train Epoch: 1 [0/49999 (0%)]\tLoss: 2.303605\n",
            "\n",
            "Test set: Average loss: 0.1589, Accuracy: 9535/9999 (95%)\n",
            "\n",
            "Train Epoch: 2 [0/49999 (0%)]\tLoss: 0.159265\n",
            "\n",
            "Test set: Average loss: 0.1054, Accuracy: 9684/9999 (97%)\n",
            "\n",
            "Train Epoch: 3 [0/49999 (0%)]\tLoss: 0.107164\n",
            "\n",
            "Test set: Average loss: 0.0950, Accuracy: 9721/9999 (97%)\n",
            "\n",
            "Train Epoch: 4 [0/49999 (0%)]\tLoss: 0.051664\n",
            "\n",
            "Test set: Average loss: 0.0934, Accuracy: 9715/9999 (97%)\n",
            "\n",
            "Train Epoch: 5 [0/49999 (0%)]\tLoss: 0.040534\n",
            "\n",
            "Test set: Average loss: 0.0789, Accuracy: 9758/9999 (98%)\n",
            "\n",
            "Train Epoch: 6 [0/49999 (0%)]\tLoss: 0.029279\n",
            "\n",
            "Test set: Average loss: 0.0611, Accuracy: 9829/9999 (98%)\n",
            "\n",
            "Train Epoch: 7 [0/49999 (0%)]\tLoss: 0.009920\n",
            "\n",
            "Test set: Average loss: 0.0605, Accuracy: 9830/9999 (98%)\n",
            "\n",
            "Train Epoch: 8 [0/49999 (0%)]\tLoss: 0.007672\n",
            "\n",
            "Test set: Average loss: 0.0608, Accuracy: 9828/9999 (98%)\n",
            "\n",
            "Train Epoch: 9 [0/49999 (0%)]\tLoss: 0.009761\n",
            "\n",
            "Test set: Average loss: 0.0607, Accuracy: 9823/9999 (98%)\n",
            "\n",
            "Train Epoch: 10 [0/49999 (0%)]\tLoss: 0.008310\n",
            "\n",
            "Test set: Average loss: 0.0617, Accuracy: 9828/9999 (98%)\n",
            "\n",
            "\n",
            "Test set: Average loss: 0.0585, Accuracy: 9817/10000 (98%)\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GK4D3T5K-rew",
        "colab_type": "text"
      },
      "source": [
        "## Training and testing loss for TNN and FCNN"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7LDPqX5AqVdE",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 414
        },
        "outputId": "54446495-c153-4faa-b3d6-ac4455e782ec"
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline  \n",
        "plt.plot(train_loss_tt, label = 'Training_loss_MPO')\n",
        "plt.plot(test_loss_tt, label = 'Test_loss_MPO')\n",
        "plt.plot(train_loss_fc, label = 'Training_loss_FC')\n",
        "plt.plot(test_loss_fc, label = 'Test_loss_FC')\n",
        "plt.title('Training Loss and Test Loss for MPO-NN')\n",
        "plt.legend()\n",
        "plt.xlabel('Epoch')\n",
        "plt.ylabel('Cross Entropy')\n",
        "plt.show()\n",
        "MPO_size = int(model_MPO.get_number_parameters())\n",
        "print('Number of Parameters (MPO): ', MPO_size)\n",
        "print('Accuracy (MPO): ', MPO_acc)\n",
        "print('Forward pass time (MPO): ', time_MPO)\n",
        "\n",
        "print('Number of Parameters (FC): ', model_fc.get_number_parameters())\n",
        "print('Accuracy (FC): ', fc_acc)\n",
        "print('Forward pass time (FC): ', time_fc)\n",
        "\n",
        "print('Compression Rate:', MPO_size/model_fc.get_number_parameters())\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeXxU1fn48c8zSxayTSBhCwQB2QWSEDfiWmurWJdSUREVsf1WbRUV17oCLl+p2tpS/FFtBb/aiktdAWutFVFcgSCbKDWGJOwJZCPrzJzfH/cmTPYJZLI+79frvmbunXPPPXcC89xzzj33iDEGpZRSPZejowuglFKqY2kgUEqpHk4DgVJK9XAaCJRSqofTQKCUUj2cBgKllOrhNBB0cSLyjojMbOu0PZGILBWRhzq6HB1NRB4SkXwR2dPRZVHtQwNBBxCR0oDFLyLlAeszWpOXMeZcY8xzbZ22NUTkDBHJa+t8OwsRuTvg71MhIr6A9S1HkF+L31dHBSURSQZuBcYaY/q3UZ5GRPaJiCtgm9veZgK2rbK/31I7EL0mIgMCPp8sIv8RkRIRKRKRt0VkbAvHnmsf/5KAbS572zH2+lJ7/YSANMcGlq2700DQAYwx0TULkAOcH7DtbzXpAv/jqI5jjHkk4O91HfBpwN9rXEeXr40lAwXGmH2t3bGFf68HgXMD1s+1t9V3g/09jwQ8wO/tvE8G/gW8CQwEhgJfAWtEZFgLRTsAzBMRZwtpemxtUANBJ1JzpSgid9rV8iUiEi8iy0Vkv4gctN8PCthnlYj8wn5/tYh8LCKP22m/F5FzjzDtUBFZbV99/VtEFonIC0dwTmPs4xaKyBYRuSDgsykistU+xk4Ruc3enmCfZ6GIHBCRj0Sk0X+rIvIHEckVkWIRWScipwZ8NldEXhaR/7OPsUVE0gM+TxWR9fZnLwERR3B+o0XkPbuc39S78mxwfiISBbwDDAyoVQxs5TH/R0T+ax/zrZr9xfJ7+0q7WEQ2ichxTZWlkXx/CLwXULal9vYL7O+u0P5bjgnYJ9v+97oRONRMMHgeuCpg/Srg/5o6R2PMAeAfwHH2pt8C/2eM+YMxpsQYc8AYcy/wGTC3ha/sn0AVcEUzaZ4DJojI6S3k1S1pIOh8+gO9gSHAL7H+Rkvs9WSgHPhTM/ufCHwDJGD95/mriMgRpP078AXQB+s/2pWtPRERcQNvY13J9QVuBP4mIqPsJH8FrjXGxGD9h/+Pvf1WIA9IBPoBdwNNVdO/BFKwvrO/A6+ISOAP+gXAMqyry7ewvzsRCQPewPqB6g28AvyslecXhfXD+Xf7/C4DnpLDzRUNzs8YcwjranhXQK1iVyuO+QPgf4FLgAHADvv8AH4EnIZ1NR1npyloqiz18zbG/Lte2a4WkZHAi8DNWH+PlcDb9vdXYzpwHuAxxnibKPobwGki4hGReOBUrKv7ps4zAevvkSkivYDJWH+j+l4Gzm4qn5pTA+4DHrD/TTamDHgEeLiFvLolDQSdjx94wBhTaYwpN8YUGGP+YYwpM8aUYP1Dbe6qZYcx5hljjA/rKmcA1o9p0GnFaic+HrjfGFNljPkY60e0tU4CooFH7Xz+AyzH+uEAqAbGikisMeagMWZ9wPYBwBBjTLUx5iPTxEOxjDEv2N+R1xjzBBAOjApI8rExZqV9js8DEwPK5gaetI/xKlZQaY2fANnGmCX28TOxrmKntXB+R2MG8KwxZr0xphL4DXCyWO3d1UAMMBoQY8zXxpjdR1mWS4EVxpj3jDHVwONAJNYPc40/GmNyjTHlzeRTgXVRcKm9vGVvq++PIlKI1eyzG5iDFagd9np9u7EuZJpljHkL2A/8oplkfwaSA2vGPYUGgs5nvzGm9j+IiPQSkT+LyA4RKQZWAx5pur2z9k4PY0yZ/Ta6lWkHAgcCtgHktvI8sPPJNcb4A7btAJLs9z8DpgA7RORDsdqBAR4D/gv8S0SyROSupg5gN7d8LVbnYSHWlXDgD0PgnS9lQITdfDEQ2FkvwOxo5fkNAU60m0wK7ePPwKrVNXd+R2NgYDmNMaVYV/1JdqD9E7AI2CciT4tI7FGWpf7x/Fj/FpIC0gT7b+P/sJqEmmsWmm2M8RhjkowxM4wx+7H6EvxYFwf1DQDyAURkRkBz2zuNpL0XuIcmmgDtwPqgvfQoGgg6n/pXvrdiXeGeaIyJxar6AzTV3NMWdgO97Sp5jcFHkM8uYHC99v1kYCeAMeZLY8yFWM0qb2BV87HbgG81xgzDatqZIyJn1c/c7g+4A6sJJN4Y4wGKCO672Q0k1Ws2S27l+eUCH9o/XDVLtDHm+ubOj6abuYKxCysAAbXNU304/J3+0RgzCRiL1UR0ewtlae3xBOvfws6ANMGez0ccrqF+HOQ+2M1pn3K4phXoEuB9O93fAprbGlzVG2Pew7rA+FUzh1uC1Yw4NdjydQcaCDq/GKx+gUIR6Q08EOoDGmN2AGuBuSISZl89nt/SfiISEbhg9TGUAXeIdbvgGXY+y+x8Z4hInN3kUIx11YeI/ESs2/cE64fdV/NZPTGAF6vK7xKR+4HYRtI15lN739l22aYCJ7SwT33LgZEicqWdh1tEjherg7zJ8wP2An1EJK6F/J31vtMwrPb6WSKSIiLhWO3anxtjsu1jn2i3gx/Canrxt1CWlrwMnCciZ9n53gpUAp8E/zVZ7NrX+cAFTTX1NeMuYKaIzBaRGLFuongIOBmY14p87sG6eGiqjF6s/2N3trJ8XZoGgs7vSaw22XysOyT+2U7HnYH1n6wA67a6l7B+AJqShBWwApfBWP/xz8Uq/1PAVcaYbfY+VwLZdpPXdfYxAUYA/wZKsX6wnzLGfNDIMd/F+j6+xWq+qCDIZgpjTBXWVd/VWLcOXgq8Fsy+AXmUYHXQXoZ15bwHWIDVT9Hk+dnn/yKQZTcpNXXX0F3U/T7/Y3fo3ofVF7EbGG4fH6wg+AxWU8oOrL/dY82VJYhz/AbrbpuFWH/D87Fud64KZv9G8ttijGn12Au7n+rHWH+z3VjnlwqcYozZ3op81mBdoDTnRRrvj+i2pPWBWfVEYt1euc0YE/IaiVKqfWmNQDXKbmYYLiIOETkHuBCrbVkp1c3oyFXVlP5YTSV9sO7pv96+PVIp1c1o05BSSvVw2jSklFI9XJdrGkpISDDHHHNMRxdDKaW6lHXr1uUbYxIb+6zLBYJjjjmGtWvXdnQxlFKqSxGRJkfOa9OQUkr1cBoIlFKqh9NAoJRSPVyX6yNQSgWnurqavLw8Kioae9qz6q4iIiIYNGgQbndTUy80pIFAqW4qLy+PmJgYjjnmGJqem0h1J8YYCgoKyMvLY+jQoUHvp01DSnVTFRUV9OnTR4NADyIi9OnTp9W1QA0ESnVjGgR6niP5m/eYQLA2+wCPvrMNfaSGUkrV1WMCwZZdxSz+8Dt2FWnHmVJKBeoxgSAtOR6A9TsOdnBJlOr+CgoKSElJISUlhf79+5OUlFS7XlXV/Jw2a9euZfbs2S0eY/LkyW1VXACWLl3KDTfc0KZ51iciXHHFFbXrXq+XxMREfvKTn9SWITExkZSUFMaOHcszzzxTm/aNN95gwoQJjBkzhvHjx/PGG233VPgec9fQ6AExRLgdZOYUcv7EpiaEUkq1hT59+rBhwwYA5s6dS3R0NLfddlvt516vF5er8Z+f9PR00tPTWzzGJ5+0erbMDhcVFcXmzZspLy8nMjKS9957j6SkpDppLr30Uv70pz+xb98+xo0bxwUXXMCePXu47bbbeO+99xg6dCjff/89Z599NsOGDWPChAlHXa4eEwjcTgcTkjysz9Eagep55r29ha27its0z7EDY3ng/HFBp7/66quJiIggMzOTjIwMLrvsMm666SYqKiqIjIxkyZIljBo1ilWrVvH444+zfPly5s6dS05ODllZWeTk5HDzzTfX1haio6MpLS1l1apVzJ07l4SEBDZv3sykSZN44YUXEBFWrlzJnDlziIqKIiMjg6ysLJYvX95iWbOzs7nmmmvIz88nMTGRJUuWkJyczCuvvMK8efNwOp3ExcWxevVqtmzZwqxZs6iqqsLv9/OPf/yDESNGNJn3lClTWLFiBRdffDEvvvgi06dP56OPPmqQrm/fvgwfPpwdO3awcOFC7r777tpbQocOHcpvfvMbHnvsMZ5//vmg/wZN6TFNQwCpyR627iqm0uvr6KIo1SPl5eXxySef8Lvf/Y7Ro0fz0UcfkZmZyfz587n77rsb3Wfbtm28++67fPHFF8ybN4/q6uoGaTIzM3nyySfZunUrWVlZrFmzhoqKCq699lreeecd1q1bx/79+4Mu54033sjMmTPZuHEjM2bMqA0+8+fP59133+Wrr77irbfeAmDx4sXcdNNNbNiwgbVr1zJo0KBm877ssstYtmwZFRUVbNy4kRNPPLHRdFlZWWRlZXHssceyZcsWJk2aVOfz9PR0tmxp9fTPjeoxNQKA1OR4/rw6i807i5k0JL6ji6NUu2nNlXsoTZs2DafTCUBRUREzZ85k+/btiEijP/AA5513HuHh4YSHh9O3b1/27t3b4Mf2hBNOqN2WkpJCdnY20dHRDBs2rPYqevr06Tz99NNBlfPTTz/ltddeA+DKK6/kjjvuACAjI4Orr76aSy65hKlTpwJw8skn8/DDD5OXl8fUqVObrQ0ATJgwgezsbF588UWmTJnS4POXXnqJjz/+mPDwcP785z/Tu3fvoMp8NHpUjSAt2QNApjYPKdUhoqKiat/fd999nHnmmWzevJm33367yUFQ4eHhte+dTider/eI0rSFxYsX89BDD5Gbm8ukSZMoKCjg8ssv56233iIyMpIpU6bwn//8p8V8LrjgAm677TamT5/e4LNLL72UDRs28Pnnn/PTn/4UgLFjx7Ju3bo66datW8e4cW0T4HtUIOgbG0GSJ5LMnMKOLopSPV5RUVFtR+nSpUvbPP9Ro0aRlZVFdnY2YF1pB2vy5MksW7YMgL/97W+ceuqpAHz33XeceOKJzJ8/n8TERHJzc8nKymLYsGHMnj2bCy+8kI0bN7aY/zXXXMMDDzzA+PHjgyrPbbfdxv/+7//Wnkt2djaPPPIIt956a9Dn1Jwe1TQEkDYknnXZBzq6GEr1eHfccQczZ87koYce4rzzzmvz/CMjI3nqqac455xziIqK4vjjjw9634ULFzJr1iwee+yx2s5igNtvv53t27djjOGss85i4sSJLFiwgOeffx63203//v2b7OsINGjQoKBuka2RkpLCggULOP/886mursbtdvPb3/6WlJSUoPNoTpebvD49Pd0czQxlz378PfOXb+Wz35xF/7iINiyZUp3L119/zZgxYzq6GB2qtLSU6OhojDH8+te/ZsSIEdxyyy0dXayQa+xvLyLrjDGN3pfbo5qGwKoRAHobqVI9wDPPPENKSgrjxo2jqKiIa6+9tqOL1Cn1uKahsQNiCXM5yMw5yJTxAzq6OEqpELrlllsa1ACWLFnCH/7whzrbMjIyWLRoUZscs6CggLPOOqvB9vfff58+ffq0yTHaWo8LBGEuB+OT4livHcZK9UizZs1i1qxZIcs/cFR1V9HjmoYAUgd72LSziCqvv6OLopRSHa5HBoK0IfFUef1s3d22Q+6VUqor6pGBIFUHlimlVK0eGQgGxEUyIC5C+wmUUooeGgjAmp9AawRKhcbRzEcAsGrVqhYfMz137lwef/zxtipyA9nZ2YgI9957b+22/Px83G537bwFc+fOrT234447rvZBdABPP/00o0ePZvTo0Zxwwgl8/PHHISvr0eqxgSA12UPewXL2leiMZUq1tZo7ZzZs2MB1113HLbfcUrseFhbW4v7BBIL2MHToUFasWFG7/sorrzR4vk/Nub3yyitcc801+P1+li9fzp///Gc+/vhjtm3bxuLFi7n88svZs2dPe59CUHrc7aM1UmtnLCvknOP6d3BplAqxd+6CPZvaNs/+4+HcR4NOvm7dOubMmUNpaSkJCQksXbqUAQMG8Mc//pHFixfjcrkYO3Ysjz76KIsXL8bpdPLCCy+wcOHC2mf9NKUm4JSVlTF8+HCeffZZ4uPjG+S9bNkyPvzwQ2666SbAmjFs9erVxMTENJpvr169GDNmDGvXriU9PZ2XXnqJSy65hF27djVIO2bMGFwuF/n5+SxYsIDHHnuMhIQEANLS0pg5cyaLFi3iwQcfDPo7ay89NhCMGxiL2ylk5h7UQKBUiBljuPHGG3nzzTdJTEzkpZde4p577uHZZ5/l0Ucf5fvvvyc8PJzCwkI8Hg/XXXddg1nNmnPVVVexcOFCTj/9dO6//37mzZvHk08+2SBvgMcff5xFixaRkZFBaWkpERHNP2qmZv6Afv364XQ6GThwYKOB4PPPP8fhcJCYmNjk/AHPPfdckN9Y++qxgSDC7WTcwDgyd2iHseoBWnHlHgqVlZVs3ryZs88+GwCfz8eAAdbI/gkTJjBjxgwuuugiLrroolbnXVRURGFhIaeffjoAM2fOZNq0aU3mnZGRwZw5c5gxYwZTp05tcSKZc845h/vuu49+/fpx6aWXNvj897//PS+88AIxMTG89NJLiEirz6GjhbSPQETOEZFvROS/InJXI58ni8gHIpIpIhtFpOEsDSGUmuxh485Cqn06sEypUDLGMG7cuNp+gk2bNvGvf/0LgBUrVvDrX/+a9evXc/zxx7fpXAKN5X3XXXfxl7/8hfLycjIyMti2bVuzeYSFhTFp0iSeeOIJLr744gaf1/QRfPTRR7VNWKGeP6CthSwQiIgTWAScC4wFpovI2HrJ7gVeNsakApcBT4WqPI1JS46notrPtt0l7XlYpXqc8PBw9u/fz6effgpAdXU1W7Zswe/3k5uby5lnnsmCBQsoKiqitLSUmJgYSkqC+38ZFxdHfHx87by/zz//PKeffnqTeX/33XeMHz+eO++8k+OPP77FQABw6623smDBgqBnC7vjjju48847KSgoAKw+jKVLl/KrX/0qqP3bWyibhk4A/muMyQIQkWXAhcDWgDQGiLXfxwENG95CqOZJpJm5Bxk/KK49D61Uj+JwOHj11VeZPXs2RUVFeL1ebr75ZkaOHMkVV1xBUVERxhhmz56Nx+Ph/PPP5+KLL+bNN98MqrP4ueeeq+0sHjZsGEuWLMHn8zWa93333ccHH3yAw+Fg3LhxnHvuuS2Wf9y4ca26mr/gggvYuXMnkydPRkSIiYnhhRdeqG0O62xCNh+BiFwMnGOM+YW9fiVwojHmhoA0A4B/AfFAFPBDY8y6RvL6JfBLgOTk5Ek7duxokzIaYzjxkfeZPLwPT16W2iZ5KtVZ6HwEPVdXm49gOrDUGDMImAI8LyINymSMedoYk26MSU9MTGyzg4sIacnxOsJYKdWjhbJpaCcwOGB9kL0t0M+BcwCMMZ+KSASQAOwLYbnqSE328M8te8gvrSQhOrzlHZRS7erhhx/mlVdeqbNt2rRp3HPPPW2S/6ZNm7jyyivrbAsPD+fzzz9vk/y7glAGgi+BESIyFCsAXAZcXi9NDnAWsFRExgARwP4QlqmB2n6CnELOHtuvPQ+tlArCPffc02Y/+o0ZP358l5s/oK2FrGnIGOMFbgDeBb7Gujtoi4jMF5EL7GS3Av8jIl8BLwJXm3aeRHl8Uhwuh+hzh5RSPVZIB5QZY1YCK+ttuz/g/VYgI5RlaEmE28nYgbE6h7FSqsfq6M7iTiF1sIeNeUV4dWCZUqoH0kCA1U9QVuXjm706sEwp1fNoIMAaYQxWh7FS6ugdzXwEa9euZfbs2S0eY/LkyW1VXACWLl1aO89AqDidztrvISUlhezsbAC++OILTjvtNEaNGkVqaiq/+MUvKCsrC2lZAvXYh84FGhQfSUJ0GOtzDnLFSUM6ujhKdXk18xGANXlL/SeJer1eXK7Gf37S09NJT2903FMdnWG+gtaKjIxscIfS3r17mTZtGsuWLePkk08G4NVXX6WkpIRevXq1S7k0EGANLEtNjtcageq2FnyxgG0HWn6mTmuM7j2aO0+4M+j0V199NREREWRmZpKRkcFll13GTTfdREVFBZGRkSxZsoRRo0axatUqHn/8cZYvX87cuXPJyckhKyuLnJwcbr755traQnR0NKWlpaxatYq5c+eSkJDA5s2bmTRpEi+88AIiwsqVK5kzZw5RUVFkZGSQlZXF8uXLWyxrdnY211xzDfn5+SQmJrJkyRKSk5N55ZVXmDdvHk6nk7i4OFavXs2WLVuYNWsWVVVV+P1+/vGPfzBixIigv5dFixYxc+bM2iAANPpwu1DSQGBLTfbw3ta9HDxURXxUyzMoKaVaLy8vj08++QSn00lxcTEfffQRLpeLf//739x999384x//aLDPtm3b+OCDDygpKWHUqFFcf/31uN3uOmkyMzPZsmULAwcOJCMjgzVr1pCens61117L6tWrGTp0KNOnTw+6nDfeeCMzZ85k5syZPPvss8yePZs33niD+fPn8+6775KUlFQ7v8HixYu56aabmDFjBlVVVfh8vibzLS8vJyUlBbBmP3v99dfZvHkzM2fODLpsoaCBwFbbT5B7kB+M1oFlqntpzZV7KE2bNg2n0wlY8wjMnDmT7du3IyJUV1c3us95551HeHg44eHh9O3bl7179zaYQ+CEE06o3VbT9h4dHc2wYcMYOnQoANOnT+fpp58Oqpyffvopr732GgBXXnkld9xxB2DNZXD11VdzySWXMHXqVABOPvlkHn74YfLy8pg6dWqztYHGmoY6A+0stk0YFIfTIdo8pFQIRUVF1b6/7777OPPMM9m8eTNvv/02FRWNzx8eHn740S9Op7PR+QqCSdMWFi9ezEMPPURubi6TJk2ioKCAyy+/nLfeeovIyEimTJnCf/7zn1blOW7cuAZzF7Q3DQS2XmEuRveP0YFlSrWToqIikpKSAOuOnbY2atQosrKyau/Meemll4Led/LkySxbtgyAv/3tb7WPwf7uu+848cQTmT9/PomJieTm5pKVlcWwYcOYPXs2F154IRs3bmxVOW+44Qaee+65Os82eu2119i7d2+r8jkaGggCpCXH81VuET5/uz7lQqke6Y477uA3v/kNqampIbmCj4yM5KmnnuKcc85h0qRJxMTEEBcX3LwjCxcuZMmSJUyYMIHnn3+eP/zhDwDcfvvtjB8/nuOOO47JkyczceJEXn75ZY477jhSUlLYvHkzV111VavK2a9fP5YtW8Ztt93GqFGjGDNmDO+++y4xMTGtPucjFbL5CEIlPT3drF27NiR5v7Y+jzkvf8U/bz6V0f1jW95BqU5M5yOA0tJSoqOjMcbw61//mhEjRnDLLbd0dLFCrqvNR9Cp6MAypbqXZ555hpSUFMaNG0dRURHXXnttRxepU9K7hgIM6dOL3lFhrN9xkOknJHd0cZRSR+mWW25pUANYsmRJbVNPjYyMDBYtWtQmxywoKOCss85qsP3999+nT58+bXKMtqaBIICIkDrYox3GSnVjs2bNYtasWSHLP3BUdVehTUP1pCZ7+G7/IYrKGr+nWSmluhsNBPUEDixTSqmeQANBPRMGe3CIdhgrpXoODQT1RIe7GNlPB5YppXoODQSNSBsSz4bcQvw6sEypI3I08xEArFq1qsXHTM+dO5fHH3+8rYrcQHZ2NpGRkXXmD6gp+zvvvEN6ejpjx44lNTWVW2+9NWTlaA9611AjUgd7+PvnOXy3v5QR/dpvdJ9S3UVL8xG0ZNWqVURHR7f55DOtNXz48AZ3AG3evJkbbriBFStWMHr0aHw+X9APs+usNBA0Im2I1WG8PuegBgLVLex55BEqv27b+QjCx4ym/913B51+3bp1zJkzh9LSUhISEli6dCkDBgzgj3/8I4sXL8blcjF27FgeffRRFi9ejNPp5IUXXmDhwoW1z/ppyoYNG7juuusoKytj+PDhPPvss8THxzfIe9myZXz44YfcdNNNgHXL+OrVq1v1OIff/va33HPPPYwePRqwHnJ3/fXXB71/Z6SBoBFD+0QRF+kmM6eQS4/XgWVKHS1jDDfeeCNvvvkmiYmJvPTSS9xzzz08++yzPProo3z//feEh4dTWFiIx+Phuuuua1Ut4qqrrmLhwoWcfvrp3H///cybN48nn3yyQd4Ajz/+OIsWLSIjI4PS0lIiIiKazPe7776rnT+gZtDZ5s2bu3xTUH0aCBrhcAipyTqwTHUfrblyD4XKyko2b97M2WefDYDP52PAgAEATJgwgRkzZnDRRRdx0UUXtTrvoqIiCgsLOf300wGYOXMm06ZNazLvjIwM5syZw4wZM5g6dWqDuQ0CNdY01B1pZ3ETUgfHs31fKcUVOrBMqaNljGHcuHFs2LCBDRs2sGnTJv71r38BsGLFCn7961+zfv16jj/++DZ9Emljed9111385S9/oby8nIyMDLZta12TWWeYP6CtaSBoQtoQD8bAV7k6nkCpoxUeHs7+/fv59NNPAaiurmbLli34/X5yc3M588wzWbBgAUVFRZSWlhITE0NJSUlQecfFxREfH89HH30EwPPPP8/pp5/eZN7fffcd48eP58477+T4449vdSC4/fbbeeSRR/j2228B8Pv9LF68uFV5dDbaNNSEiYM9iD2w7NQRiR1dHKW6NIfDwauvvsrs2bMpKirC6/Vy8803M3LkSK644gqKioowxjB79mw8Hg/nn38+F198MW+++WZQncXPPfdcbWfxsGHDWLJkCT6fr9G877vvPj744AMcDgfjxo3j3HPPbdW5TJgwgSeffJLp06dTVlaGiPCTn/zkaL6eDqfzETTjR7//kIGeSJbOOqFdjqdUW9L5CHounY+gDaUlx5OZU0hXC5ZKKdUa2jTUjNRkD8u+zCUr/xDDE6M7ujhK9UgPP/wwr7zySp1t06ZN45577mmT/Ddt2sSVV15ZZ1t4eHidOYS7Ow0Ezah5Eun6HQc1EKguyRiDiHR0MY7KPffc02Y/+o0ZP358t7pF9EhaMLRpqBnDE6OJiXCRqXcOqS4oIiKCgoICbdrsQYwxFBQUNDtIrjFaI2iGwyGkDPawfocOLFNdz6BBg8jLy5L/YbMAACAASURBVGP//v0dXRTVjiIiIpodJNcYDQQtSE2O50//2U5ppZfocP26VNfhdrsZOnRoRxdDdQHaNNSCtGQPfgMbtXlIKdVNaSBoQergmqkrNRAopbqnFgOBiNwoIvFHkrmInCMi34jIf0XkribSXCIiW0Vki4j8/UiOE0pxvdwMT4zSfgKlVLcVTI2gH/CliLxs/7AHdS+aiDiBRcC5wFhguoiMrZdmBPAbIMMYMw64uVWlbydpyfFk5urAMqVU99RiIDDG3AuMAP4KXA1sF5FHRGR4C7ueAPzXGJNljKkClgEX1kvzP8AiY8xB+1j7Wln+dpGaHM+BQ1XsKCjr6KIopVSbC6qPwFiXwnvsxQvEA6+KyG+b2S0JyA1Yz7O3BRoJjBSRNSLymYic01hGIvJLEVkrIms74la4tCEeAJ2fQCnVLQXTR3CTiKwDfgusAcYbY64HJgE/O8rju7BqG2cA04FnRMRTP5Ex5mljTLoxJj0xsf2fBDqibwzR4S4yc7TDWCnV/QRzY3xvYKoxZkfgRmOMX0Sae/bqTmBwwPoge1ugPOBzY0w18L2IfIsVGL4MolztxukQJg6O0xqBUqpbCqaP4AGgj4jMtu8gSgv47Otmdv0SGCEiQ0UkDLgMeKtemjewagOISAJWU1FW606hfaQOjmfbnhLKqtpu9iSllOoMgmkaug94DugDJABLROTelvYzxniBG4B3ga+Bl40xW0RkvohcYCd7FygQka3AB8DtxpiCIzuV0Eob4sHnN2zMK+rooiilVJsKpmnoCmCiMaYCQEQeBTYAD7W0ozFmJbCy3rb7A94bYI69dGq1A8tyCjlpWJ8OLo1SSrWdYO4a2gUEPsounIZt/d1efFQYQxOitJ9AKdXtBFMjKAK2iMh7gAHOBr4QkT8CGGNmh7B8nUpqsofV3+7vFs94V0qpGsEEgtftpcaq0BSl80tNjue19TvJO1jO4N69Oro4SinVJloMBMaY5+y7fkbam76xb/fscdKSDw8s00CglOougrlr6AxgO9Zzg54CvhWR00Jcrk5pVL8YeoU5dWCZUqpbCaZp6AngR8aYbwBEZCTwItbI4h7F5XQwYZAOLFNKdS/B3DXkrgkCAMaYbwF36IrUuaUlx7N1VzEV1b6OLopSSrWJYALBOhH5i4icYS/PAGtDXbDOKjU5Hq/fsGmnDixTSnUPwQSC64CtwGx72QpcH8pCdWapdodxpjYPKaW6iWb7COzJZb4yxowGftc+RercEqLDSe7di/U7tMNYKdU9NFsjMMb4gG9EJLmdytMlpCV7WJ9zUGcsU0p1C8E0DcVjjSx+X0TeqllCXbDOLDU5nn0llewqqujooiil1FEL5vbR+0Jeii4mLdl6AN36HQdJ8kR2cGmUUuroBFMjmGKM+TBwAaaEumCd2egBMUS4HTqwTCnVLQQTCM5uZNu5bV2QrsTtdDAhyaMDy5RS3UKTgUBErheRTcAoEdkYsHwPbGq/InZOqUM8bN1VTKVXB5Yppbq25moEfwfOx5pe8vyAZZIxZkY7lK1TSx0cT5XPz+adxR1dFKWUOipNBgJjTJExJtsYMx1rkvlqrPkIovV20sNPItWBZUqprq7Fu4ZE5AZgLrAX8NubDTAhdMXq/PrGRpDkidQOY6VUlxfM7aM3A6M666TyHSltSDxrsw90dDGUUuqoBHPXUC7WdJWqntTBHnYXVbC7qLyji6KUUkcsmBpBFrBKRFYAlTUbjTE9/tlDaUOsgWWZOYUMGK8Dy5RSXVMwNYIc4D0gDIgJWHq8sQNiCXM5tMNYKdWlBTNn8bz620QkmJpEtxfmcjA+KY712mGslOrCmhtQ9nHA++frffxFyErUxaQle9i0s4gqr7/lxEop1Qk11zQUFfD+uHqfSQjK0iWlJsdT5fWzdbcOLFNKdU3NBQLTxPvG1nuswCeRKqVUV9RcW79HRH6KFSw8IjLV3i5AXMhL1kX0j4tgQFwEmbnaT6CU6pqaCwQfAhcEvD8/4LPVIStRF5SWHK81AqVUl9VkIDDGzGrPgnRlqckeVmzazb7iCvrGRnR0cZRSqlWCGUegWpBa00+gt5EqpbogDQRt4LikWMKcDjJztXlIKdX1aCBoA+EuJ2MHxpK5Q2sESqmup8VAICLTRCTGfn+viLwmImmhL1rXkpYcz8adhVT7dGCZUqprCaZGcJ8xpkRETgF+CPwV+H/BZC4i54jINyLyXxG5q5l0PxMRIyLpwRW780lN9lBR7Wfb7pKOLopSSrVKMIGgZlLe84CnjTErsB5A1ywRcQKLsCa6HwtMF5GxjaSLAW4CPg+20J1RzZNIdUJ7pVRXE0wg2CkifwYuBVaKSHiQ+50A/NcYk2WMqQKWARc2ku5BYAFQEWSZO6WBcRH0jQnXJ5EqpbqcYH7QLwHeBX5sjCkEegO3B7FfEtakNjXy7G217L6GwXYto0sTEWtgmd5CqpTqYoIJBAOAFcaY7SJyBjCNNnj6qIg4gN8BtwaR9pcislZE1u7fv/9oDx0yaUM85BwoI7+0suXESinVSQQTCP4B+ETkWOBpYDDw9yD222mnrTHI3lYjBuuppqtEJBs4CXirsQ5jY8zTxph0Y0x6YmJiEIfuGDUDy3RCe6VUVxJMIPAbY7zAVGChMeZ2rFpCS74ERojIUBEJAy4D3qr50BhTZIxJMMYcY4w5BvgMuMAYs7bVZ9FJjE+Kw+UQ7SdQSnUpwQSCahGZDlwFLLe3uVvayQ4eN2D1L3wNvGyM2SIi80Xkgub37poi3NbAMr1zSCnVlQQz5eQs4DrgYWPM9yIyFKg/Y1mjjDErgZX1tt3fRNozgsmzs0tLjufltbl4fX5cTh24rZTq/Fr8pTLGbAVuAzaJyHFAnjFmQchL1kWlJnsoq/LxzV4dWKaU6hqCecTEGcB2rMFhTwHfishpIS5Xl5WmTyJVSnUxwbRdPAH8yBhzujHmNODHwO9DW6zQyC/PD/kxBsVHkhAdph3GSqkuI5hA4DbGfFOzYoz5liA6izubv276Kxe+cSH7y0I7DkFESE2O11tIlVJdRjCBYJ2I/EVEzrCXZ4Aud4vnD5J/QIW3goc/fxhjTEiPlZYcz/f5hzh4qCqkx1FKqbYQTCC4DtgKzLaXrcD1oSxUKAyNG8qvUn7F+znv896O90J6rNRkD4BOVKOU6hKaDQT2E0S/Msb8zhgz1V5+b4zpks9QmDluJmN6j+Hhzx+msCJ0TTcTBsXhdIg2DymluoRmA4Exxgd8IyLJ7VSekHI5XDyY8SDFlcX89svfhuw4vcJcjO4fowPLlFJdQjBNQ/HAFhF5X0TeqllCXbBQGdV7FNeMv4a3s97mo7yPQnactOR4NuQU4vOHtj9CKaWOVlAzlAE/AeZj3Upas3RZ1064luFxw5n/2XxKq0pDcozUZA+Hqnxs36cDy5RSnVuTgUBEjhWRDGPMh4EL1oxlee1XxLYX5gxjXsY89h7ay5PrnwzJMWoHlumE9kqpTq65GsGTQHEj24vsz7q0iYkTmTFmBi998xJf7vmyzfMf0qcXvaN0YJlSqvNrLhD0M8Zsqr/R3nZMyErUjm5MvZFB0YOY+8lcKrxtO1OmiJA62KMdxkqpTq+5QOBp5rPIti5IR+jl7sXcyXPJKcnhqQ1PtXn+aUPi+W7/IYrKqts8b6WUaivNBYK1IvI/9TeKyC+AdaErUvs6ccCJ/GzEz3hu63Nszt/cpnmnDtaBZUqpzq+5QHAzMEtEVonIE/byIfBz4Kb2KV77uDX9VhIiErj/k/up9rXd1fuEwR4colNXKqU6tyYDgTFmrzFmMjAPyLaXecaYk40xe9qneO0jJiyG+06+j+0Ht/OXzX9ps3yjw12M7KcDy5RSnVswE9N8YIxZaC//aY9CdYQzBp/Bucecy9Mbn2b7we1tlm/akHg25Bbi14FlSqlOSudSDHDXiXcR447hgU8ewOf3tUmeqYM9lFR4+W5/aAauKaXU0dJAEKB3RG/uOuEuNuVv4oWvX2iTPNOG1MxYps1DSqnOSQNBPecOPZczBp3BnzL/RE5xzlHnNywhirhIt3YYK6U6LQ0E9YgI9550Ly6Hi7mfzj3qSWysGct0YJlSqvPSQNCIflH9uDX9Vr7c8yWvbn/1qPNLS45n+75Siit0YJlSqvPRQNCEn434GSf0P4En1j7BnkNHd7dsarIHY+CrXG0eUkp1PhoImiAizD15Lj6/jwc/e/ComogmDvYgOrBMKdVJaSBoxuDYwdyYeiOr81az8vuVR5xPbISbEX2jtZ9AKdUpaSBowYwxM5iQMIFHv3iUgvKCI84nLTmezBwdWKaU6nx6TiAo3QfZa1q9m9PhZN7keZRWl7LgiwVHfPjUZA9F5dV8X3DoiPNQSqlQ6DmB4PM/w9IpsGQKfPcBtKLN/9j4Y7l2wrW8k/0OH+R8cESHPzxjmTYPKaU6l54TCE67Dc5ZAAey4PmL4K9nw7f/Cjog/Hz8zxkZP5IHP3uQ4qrGJm5r3vDEaGIiXGTqnUNKqU6m5wQCdyScdB3c9BWc9zso2Qt/nwZPnwFfLwe/v/ndHW7mZ8ynoKKAJ9Y+0erDOxxCymCP1giUUp1OzwkENVzhcPzPYfZ6uOBPUFEEL82AxafA5tegmYfNjeszjpnjZvLa9tf4bPdnrT50WnI83+4tobTSezRnoJRSbarnBYIaTjekXQk3rIWfPg3+anh1Fjx1Eny1DHyN/1j/auKvGBI7hLmfzKWsuqxVh0xN9uA3sFGbh5RSnUjPDQQ1nC6YeCn86jO4eAk43PD6tfCndFj/f+CtqpM8whXBvMnz2Fm6k4WZC1t1qNTB+iRSpVTno4GghsMJx02F6z6GS/8GEbHw1o2wMA2+/At4K2uTTuo3iUtHXcrfvv4bG/ZtCPoQcb3cDE+M4uP/5lNWpc1DSqnOIaSBQETOEZFvROS/InJXI5/PEZGtIrJRRN4XkSGhLE9QHA4Y8xP45Ydw+SsQ0x9W3Ap/mAif/T+ospqDbpl0C/2j+vPAJw9Q5atqIdPDfjyuP59lHeD4h/7NnJc38PH2fHw6yEwp1YHkaB+z3GTGIk7gW+BsIA/4EphujNkakOZM4HNjTJmIXA+cYYy5tLl809PTzdq1a0NS5kYZA99/CB8+Bjs+hqhEmHwjpP+cNflfcd2/r+N/xv8Ps9NmB5Wd32/4IvsAb2TuZMWm3ZRUeOkXG85FKUlclJrEmAGxIT4hpVRPJCLrjDHpjX4WwkBwMjDXGPNje/03AMaY/20ifSrwJ2NMRnP5tnsgCJS9BlY/BlkfQGRvOPlX3GP2s3LHe7z4kxcZ3Xt0q7KrqPbx/tf7eD0zj1Xf7MfrN4zuH8PUtCQuTEmiX2xEiE5EKdXTdFQguBg4xxjzC3v9SuBEY8wNTaT/E7DHGPNQI5/9EvglQHJy8qQdO3aEpMxBy/3SCgjb36UowsOFSX3pG5vM389/CZfDdURZFpRWsnzjbl7P3MmG3EJEIGN4Aj9NTeKc4/oTFX5k+SqlFHSBQCAiVwA3AKcbYyrrfx6oQ2sE9e3aAKsf472c95nTL5GbYo/jF2f/EaITjyrbrP2lvJG5k9c37CT3QDmRbic/HtePn6YNImN4H1xO7eNXSrVOp24aEpEfAguxgsC+lvLtVIGgxt4tzHnvej6sPsCrew8yNGUmZMy2OpqPgjGGdTsO8lrmTpZ/tYviCi+JMeFcMHEgP01NYtzAWESkjU5CKdWddVQgcGF1Fp8F7MTqLL7cGLMlIE0q8CpWzWF7MPl2ykAA5Jfnc+Hr53OsX1jy3TYcDhekXQWn3Axxg446/0qvjw+27eO19Tv54Jt9VPsMI/tFc1FqEhelJDHQE9kGZ6GU6q46JBDYB54CPAk4gWeNMQ+LyHxgrTHmLRH5NzAe2G3vkmOMuaC5PDtrIAB4879vcu+ae/nNcb/k8p3fwoa/AwIpl8OpcyD+mDY5zsFDVazYZPUnrNtxEBE4aWgffpqWxLnH9Scmwt0mx1FKdR8dFghCoTMHAmMM1//7etbvW88bF77BQK8P1jxpjVD2+2DCpXDqrZBwbJsdc0fBIV7P3MkbmTvJLigj3OXg7LH9mJqWxKkjEnFrf4JSCg0E7WpX6S5++uZPSembwuIfLrba8It3wyd/hLVLwFcJw38A/SdAv3HQdywkjLCefXQUjDFk5hby+vqdvL1xF4Vl1fSJCuN8uz9hwqA47U9QqgfTQNDOXtz2Io98/ggPZjzIRcdedPiD0n3w6SL49l3I/xaM/aRThxsSR1lBod9Y6DvOeo1NgiP48a7y+ln1zT7e2LCTf2/dR5XPz7DEKKamWuMTBvfu1UZnqpTqKjQQAKVr1lD89nJiz5tC1EknIe7QtaP7jZ9Z/5zF9sLtvHnhmyT2auR2Um+lFQz2boV9W+zXrVC883CaiDgrONQPEBFxQZelqLyalZt28/r6nXyRfQCAE4b25qepSZw9th8J0eFHe7pKqS5AAwFw8OWX2ffY4/hLSnDGxxPz4x8RO2UKvdLTEUfbt6NnF2Xzs7d+xmmDTuP3Z/4++B3LD8K+r2HvFisw1ASIyoBZ0WIHWQGh37jDwaHPCHCFNZt17oEy3tywk9cyd5K135o7OckTSUqyh5RBHiYO9jA+KY7IMOeRnLJSqhPTQGDzV1Vx6KOPKF6xkpIPPsCUl+Pq25fYc88h9rzziBg/vk3b0f+66a88uf5Jnjj9CX50zI+OPCNjoCjPDgybDweH/G/Bbz/F1OGChJENaw9xgxs0Lxlj2LyzmM+yCtiQV8iGnEJ2FpYD4HQII/vFkDI4jomDPKQkexjRNwanQ/sXlOrKNBA0wl9WRumqVRStWMmh1asx1dW4Bw0idsoUYs+bQvjIkUcdFLx+LzNWzmDPoT28eeGbeCI8R13uugeogoLtDZuXinIPpwmPhb5j7AAx7nCgiIyvk9X+kko25hWyIddavsotpLjCCjK9wpwclxRHymAPKYOtmsPAuAjtfFaqC9FA0AJfcTEl/36f4hUrOPTZZ+DzEXbscGKnTCFuyhTCjjnmiPP+5sA3XLb8MqYMm8LDpzzcdoVuTkVRveYlO0hUFh1OE5sEfY6F+CHgGWKNcfAMsdajEjFAdkEZG3IP8lVuERtyC9m6q5gqnzW3c0J0uB0Y4pg42MOEQR7iInX8glKdlQaCVvAWFFDyr39RvGIlZfZxIsaOJfa884g99xzcAwe2Os+FmQt5euPTPHXWU5w66NS2LnJwjLE6ogNrDwe+g4M7oCy/blp3L/AkHw4M9mtVzGC+qexN5j5/bc2hpq8BYFhiVG1fQ8pgD6MHxBDu0v4GpToDDQRHqHrPHorf+SfFK1dSsWkTAJFpaVbz0Tk/xpWQEFQ+Vb4qLnn7Eg55D/H6Ba8THRYdymK3XmUpFOZA4Q4rMNR/rSqpmz7CYwWI+GOojB5MrunL1nIPnx+M5YM94ew6ZP2bCnM6GDMwlpRBcaQke5g4yMMxfaJwaH+DUu1OA0EbqMrJoXjlSopXrKRy+3ZwOIg66URip0wh5uyzccY1f0vnV/u/4sqVV3LJqEu496R726nUbcAY606mg9mNB4rCHKg3Q5svqj9FEQPZRV+2VfYmsziW77wJ5PoTORTRj/GDe1t9DXbtISE6TPsblAoxDQRtrOLbbyl+5x2KV6ykOicH3G6iTznFCgo/OBNHVFSj+y34YgEvfP0CS368hPT+jf49uh6/H0r3NF6TKNxhNUcZf21yH072ORLJ8iaQ408k1yRS5ojGGRaJM6wX7ohehEVEEhEZRXhkFJG9oonqFU1UdDTR0dHExcQQFxODJypMm52UagUNBCFijKFi8xarprByJd69e5GICKLPPIPYKVOIPu00HOGHB2yVVZfxs7d+Rn55Psf3P55Tkk7hlKRTSI5N7sCzCDFvFRTnNQgQvgPZ+A9k464oOKJsK4ybSsKokjC8jnC8jnD8znCMKwLjikDckTjCInGFReIK70VYRC/CIqzg4nRHgDsSXAGvrghwuqzbcGsXZ8B7d731+p/XLPpsJ9U5aSBoB8bvp3z9eiso/PNdfAcO4IiKIuaHPyT2J+fVjmbOLsrm79v+zpqda8gpyQFgcMzg2qCQ3i+dXu4e9AiIqjKoLAFvhbVUl9e+Gm8FVRVlHDpUSkVZKRXlZVRVlFFVcQhvZTm+qjJ8VYf3EW8FTn8FDl8VEVQRThURUkU41URgbXOJv+UyHRVpIVg4redKNRdM3JH20quR9/arK7Lpz2rTRGhgUrU0ELQz4/Vy6LPPKV65kpL33rNGM3s8xPz4x/Zo5kmI00lOcQ4f7/yYNbvW8MXuL6jwVRDmCGNSv0lkJGVwStIpDIsbpu3nrWSMobTSS2FZNYVl1Rwsq6KwvJrCsiqKS8spOVRC+aFDlJWVUlF+iMryMiorDlFZWYUDH078uPDiwo8TX+1rmMNPtJvaJcpeermsJdJpiHQZIpz24jCEOw1h4iNM/IjxWQMAa5dG1n1VVmCrXcrswFh2ZF+GKxLcEc0HlZr3roh6QcQJ4rBfnXVfG9vWVNo6212tSBsQJPX/wFHTQNCBGhvNLG43Dk8cztg4nLGxVkdzbDQFrgqyTQHfeneSbfIpjYSI+ARGD5lE6vBTSR9xBjFR8S0fVB0Rv99QWuWluLyaInsJfH948db5vCaN19/0/yWHQEyEm7jIuktspJvYSFftuicyDE8v+30vN55eYUSFORE4XGOqLmvitV4AafBZWcPgUv8zb0W7fd+tIg67ea6ZmlZr15utmTW27gZn2OH9nG57m9102GA9rN5ngfs0sh7iYKeBoJPwl5VR8sEHVGzdir+4GF9RMb7iYnxFRfiLivAVF+MvLW02j6owByamF+GePkTGJ+IMDCieOByxsdZ6XBzOuFicsbE44qzPxamdq6FijKGsytdI0GgYTIrrBZXi8uragXqNcTmkNih47AARZwcMT0DAsNbtQNLLTUy4q/W1Sb8fvOXWQxH9PusJuXVe/YdrMnU+8zeStrHt3lak9TVSa2qiJhX0enXr9zehbk60OZoKGgHbT7sdjpt6RNk3FwhcR1Vw1SqOXr2IO+884s47r8k0xuvFV1JSGxh8RUVUFR5gx86t5OZ9zb59WVQVHiCq4hDx+3eRsDuSmEoH7tIKTEXzV3OO6GiccXE44qxg4Yiw2pDF6bCuuJwORBzgdFoP4nMEbnPY25x109vbcAjicNZN55AmPwvc5oyNw5WYgCshAVefPkhY8w/P64xEhKhwF1HhrlZPG2qMoaLaT1F5NYXlVRSVVVNYXm2/VtnNW9UU2e93FVbw9e4SCsuqOFTlazJfp0PsWoa7TiCJCwgY9QOMJzKMmIheOMIav/OtR/L7DwcRXxX4at5XW9t91U2vN/mZt4k01c3nHxEbklPUQNDJiMuFKz4e4us2AfXmQlLt9/vL9rNm1xo+2rmGT3Z9QnFVMQ5xMNGTymlxkzgxahzDHIn4i0sO1zyKivAVF+GvfV+Mt6QE4/eDz7rSMz4/+P2124zxQ4Ntxkrf2LY2ql064+JwJibgSkjE1aePFSASE3Am2NsSrYDhjI/vFrUcESEyzElkmJP+cRGt2rfK668TPKx+kSqKyu2+kYCgsq+kgm/3llBUVk1JpbeZ8kB0uItwl4Mwp4MwlwO3/RoWsC283npgmvAG6Z0B62K/OpvMszYvpwO3Uzq2n8zhAEcYEAZ0zwCpTUNdnM/vY1P+JtbsWsOanWvYnL8ZgyE+PJ6TB57MKUmnMHngZPpE9gl5WYwx1tVTQHAwfgN+H6YmUNTZ5gef1wpK+/Px5u/Hm5+PLz/fWi8owJufj3f/fkx5ecMDOhw4+/S2gkOCXaNIsAJH/aDhiI3VTvcA1T67BhJQ0yisDSZVlFR4qfL5qfIGLL56ry28b0tOh+ByCG47MLicDtwO69XlFMLsV5fD/tzhwO2qSXM4vdvpsN4HpHXb+7qdDlx2nmH2PjXHDHc5iAhzEuGyAnaE20Gk20lE7WIFrc78b0z7CHqQgxUH+WTXJ6zZuYY1u9ZwoMKajGZsn7FkDMzg1EGnMj5hPC5H16oM+g8dsoJCfj7e/IKGQSP/cOCgurrB/uJ2H65lBAQNZ+37RNz9++Hq2xdxda3vpjMyxlDtM40EEx+V9vtqn6ndVuX1126v2afadzid1+/H67PyrPb58frt7T4/1X5DtdeP129/Zqevsj/3+gzVtfvXz886TjP9/EFzCES4nQEBwlFvvW4AiQxzEtFIgIlwOZsMOnGRbiLcR1YL1kDQQ/mNn68PfG0FhZ1r+Gr/V/iMjxh3DCcNPImJiRMZEjuE5JhkkmKSCHd2/dnKjDH4i4oOB439+XgL6gUMe/EdONCwOcvhwNW3L+4BA3AP6I9rwADc/QPeDxhgNUl14is/1Xp+/+Fg4bUDWE2wqAlM5dU+KmoXP+VVPiq8PsqrrOBWXmV9Vm5/XpO2vPbVT2WddStdazx44TiuPPmYIzpHDQQKgOKqYj7b9Rlrdq3h450fs69sX+1ngtA/qj/JMckMjh1Mckxy7fvBMYOJdLWuA7QrMF4vvoMHa5ufqnfvoXrPbry791C9e3fte1NV91lKEh6Ou//hwFAnYAwcgLt//yYfM6JUIGMMlV5/gwDSVNA5/pjejOwXc0TH0kCgGlVUWUROcQ45JfZiv88tzuVg5cE6afv26msFh9hkBscMrvM+yt19f/SMMfgOHKB61+6GQWLXbqr37MG7b1+DmoUjNtYKEv374xpYN0i4BgzA3bdvl7w7SnVdGghUqxVXFZNbkktucW6dIJFTnENBvecD9Ynow5DYIVaAiE2uU6uICTuyq5euxFRX4923j+o9ewICxm6rhrHbeu8rKqq7k4jVN1ETJPr3xz1wAK7ERHtkrYBY6UTEXheg3naHw94m9kv9v7Il4QAACjhJREFUtI1sD8in8e32MQBqfh5qfydM3fV6r7W/J7U/K61MX/84baItmvEMGGPdKWewxhbUrluf4ffbN0zY67VpWlr3W/sQkF9AmsA8oyafTMSYMUf2LWggUG3pUPUhcktyD9cgAt4HNjcBxIfH12lqqgkUybHJxIU3/+ju7sRfVmYFit31gsSe3Xbw2NPiOBCl+s99gPjLLjuifTUQqHZTVl1GXmne4ZqE3dSUU5LDnkN7MAFXerFhsSTHJDMweiADogYwIHoA/aP6MzDKWo8Lj+sxnbLGGHyFhfjy860rRszhK01jrKtCg33FHLC95go12PTNbjd186r97qXOS8322r9N/Vfqb+fo0h+FNvl9M1gVJYfDri05Gq47Ampodu2q+XWHdbo1AzcRawBmQJra2lrAuoSFIe4jmxJWRxardtPL3YuR8SMZGT+ywWeVvkrySvIa1CS+PfgtH+Z9SKWvsk76SFck/aP6W0EiakCd9wOiBtAvqh9hzu7Rzi4iuOLjrcGESrUzDQSq3YQ7wxnuGc5wz/AGnxljOFh5kN2HdrOndA+7D+1m16Fd7Dm0h92lu/nmwDcN+iYAEiITGBg18HCQiK4bMDzhnh5Tq1DqSGkgUJ2CiNA7oje9I3ozrs+4RtNU+irZe2gvuw/trl1qAsW3B79ldd5qKnx129kjnBENgkRN01N3q1UodaQ0EKguI9wZbnU2NzGjmzGGwsrCw4GitG7AWJ23mvzy/Ab7JUQm1DY9DYwayIBoK0jU9F3EhunjKVT3poFAdRsiQnxEPPER8YztM7bRNFW+qkZrFbtKd7H94HZW561u0FfRy9XrcIe2XbMYGDWQgdFWk1RiZCJOR9d/+J3quTQQqP/f3t2GyHXVcRz//mZmJ7ub0LRNQqqbagINSqrGapBqwRetQkSxL1TSoiJSEILWKOLjCxHxhYqIrS1CjIoPQV/ECkGKVhqRQKVpbGvaJkpDjO02CdmYNqZJk336++Leu3tndjbu6s6e3b2/D1zmzLkP+5/DDv+599x7TqU0683saemrru+4PiI4e+lslhwunODEyycmEsXJCyc5dOYQ5y63PhPQUIO1y9e2nEWUE8Z1y6+jtzG7UUXN5pMTgVmJJFb1rWJV3ypuXN25r+LiyMWsMztPDkX51IVTHDh1gNMXTzPeNpnJtb3XdrzsVJR9+clSciIwm6X+nv5p734CGBkf4fTF0xN9FOWE8eyLz7J/cP+UTu3+Rv9E53VvvZdljWUsq7cuzXqT3novzXozq5vFNs1a04nGpuVEYDbHemo9DKwYYGDFQMf1E7fKdkgUQxeHODN2hstjl7NlNHsdHhtmNKafTGYmOiaK8tKYTCbNWpOaai1LXfUpdUW9pJb1ddURol7LX1WnVqtR478fs3w8SVk9tcmyaojsAavieMU60fbaYf+JfUvHKspAx/pin/Jxl5KuJgJJW4F7gDqwKyK+2bZ+GfAz4K3Av4BtEXG8mzGZpdZyq+w0l586GR0fZXhseDJJdEgWl8YutbzOZtvzr5yfXD8+nD3tHGMTr+MxPmUZi7GWp8WrpmOSKL0vkhWiJbm07Ftsn2/bsm++TbH99s3b2bph65x/jq4lAkl14H7g3cAg8JikvRFxuLTZXcCLEXGDpDuAbwHbuhWT2WLWqDVo1Br09/SnDqVFRGSJgTw5jGfJ4UpJ5IqJpW2fIDt+REz8jeJvBtHy98tlgqn15X061BfHLsqd9iGY2Leon9i3eF86PtBy/PL78jHa6zptc1Vz8c1Z/DbgaEQcA5D0K+B2oJwIbge+lpf3APdJUiy2AZDMKqy4jFMnv4XWd9IuOrUuHnsAeL70fjCv67hNRIwC54Apk+tK+oSkg5IODg0NdSlcM7Nq6mYimDMRsTMitkTEljVr1qQOx8xsSelmIngBKD+1sy6v67iNpAawkqzT2MzM5kk3E8FjwEZJGyQ1gTuAvW3b7AU+lpc/COxz/4CZ2fzqWmdxRIxK+hTwe7Luox9HxDOSvg4cjIi9wI+An0s6CpwlSxZmZjaPuvocQUQ8CDzYVvfVUvkS8KFuxmBmZle2KDqLzcyse5wIzMwqbtFNXi9pCPjn/7j7amDqzCTV5fZo5faY5LZotRTa47UR0fH++0WXCP4fkg5GxJbUcSwUbo9Wbo9JbotWS709fGnIzKzinAjMzCquaolgZ+oAFhi3Ryu3xyS3Rasl3R6V6iMwM7OpqnZGYGZmbZwIzMwqrjKJQNJWSX+XdFTSl1LHk4qk6yX9UdJhSc9I2pE6poVAUl3SE5J+mzqW1CRdLWmPpL9JOiLp7aljSkXSZ/PvydOSfimpN3VM3VCJRFCaNvM9wCbgTkmb0kaVzCjwuYjYBNwMfLLCbVG2AziSOogF4h7gdxHxemAzFW0XSQPAp4EtEfEGssEzl+TAmJVIBJSmzYyIYaCYNrNyIuJkRDyel8+TfcnbZ46rFEnrgPcCu1LHkpqklcA7yUYGJiKGI+KltFEl1QD68vlS+oETiePpiqokgplMm1k5ktYDNwGPpo0kue8BXwDGUweyAGwAhoCf5JfKdklanjqoFCLiBeA7wHPASeBcRDyUNqruqEoisDaSVgC/Bj4TEf9OHU8qkt4HnI6Iv6SOZYFoAG8BfhARNwEXgEr2qUm6huzKwQbg1cBySR9JG1V3VCURzGTazMqQ1EOWBHZHxAOp40nsFuD9ko6TXTK8VdIv0oaU1CAwGBHFWeIessRQRe8C/hERQxExAjwAvCNxTF1RlUQwk2kzK0GSyK7/HomI76aOJ7WI+HJErIuI9WT/F/siYkn+6puJiDgFPC/pdXnVbcDhhCGl9Bxws6T+/HtzG0u047yrM5QtFNNNm5k4rFRuAT4KPCXpybzuK/lscmYAdwO78x9Nx4CPJ44niYh4VNIe4HGyu+2eYIkONeEhJszMKq4ql4bMzGwaTgRmZhXnRGBmVnFOBGZmFedEYGZWcU4EZm0kjUl6srTM2ZO1ktZLenqujmc2FyrxHIHZLL0SEW9OHYTZfPEZgdkMSTou6duSnpJ0QNINef16SfskHZL0sKTX5PVrJf1G0l/zpRieoC7ph/k49w9J6kv2ocxwIjDrpK/t0tC20rpzEfFG4D6yUUsBvg/8NCLeBOwG7s3r7wX+FBGbycbrKZ5m3wjcHxE3Ai8BH+jy5zG7Ij9ZbNZG0ssRsaJD/XHg1og4lg/cdyoiVkk6A7wqIkby+pMRsVrSELAuIi6XjrEe+ENEbMzffxHoiYhvdP+TmXXmMwKz2YlpyrNxuVQew311lpgTgdnsbCu9/jkvP8LkFIYfBvbn5YeB7TAxJ/LK+QrSbDb8S8Rsqr7SyKyQzd9b3EJ6jaRDZL/q78zr7iab0evzZLN7FaN17gB2SrqL7Jf/drKZrswWFPcRmM1Q3kewJSLOpI7FbC750pCZWcX5jMDMrOJ8RmBmVnFOBGZmFedEYGZWcU4EZmYV50RgZlZx/wEdkfp7QrysngAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Number of Parameters (MPO):  9400\n",
            "Accuracy (MPO):  96.83\n",
            "Forward pass time (MPO):  2.3550949096679688\n",
            "Number of Parameters (FC):  886875\n",
            "Accuracy (FC):  98.17\n",
            "Forward pass time (FC):  2.1748743057250977\n",
            "Compression Rate: 0.010599013389711065\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vqrK4WrU-61R",
        "colab_type": "text"
      },
      "source": [
        "# [Bonus] Decompose a fullly connected NN\n",
        "Now that we have showed that by using MPO format to train the neural networks, it can compress a significant amount of parameters. However, another more straight-forward idea would be to use various tensor decomposition methods to decompose and reconstruct the fully-connected layer. In the following practice we will be exploring if this method is duable and if it is better than MPO-NN. \n",
        "## Functions for decomposing FCNN via CP, Tucker and TT"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rXSFojWfQ9p1",
        "colab_type": "text"
      },
      "source": [
        "#### Data prepration"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LULAkvOnQ9IU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_newshape(weights, tensor_dim):\n",
        "  total_elements = weights.shape[0]*weights.shape[1]\n",
        "  remaining_elements = total_elements\n",
        "  newshape = []\n",
        "  while (remaining_elements/tensor_dim).is_integer():\n",
        "    newshape.append(tensor_dim)\n",
        "    if remaining_elements/tensor_dim - int(remaining_elements/tensor_dim) == 0:\n",
        "      remaining_elements /= tensor_dim\n",
        "    else:\n",
        "      print('Error getting newshape, cannot reshape the matrix into a tensor')\n",
        "      break\n",
        "  if remaining_elements != 1:\n",
        "    newshape.append(int(remaining_elements))\n",
        "  return newshape\n",
        "\n",
        "layers = [model_fc.fc1.weight.data.cpu().numpy(), model_fc.fc2.weight.data.cpu().numpy(), model_fc.fc3.weight.data.cpu().numpy()]\n",
        "\n",
        "layers_ori_shape = [layer.shape for layer in layers]\n",
        "\n",
        "tensor_dim = 5\n",
        "layers_tensor_shape = [get_newshape(layer, tensor_dim) for layer in layers]\n",
        "\n",
        "layers_tensor = [tl.tensor(layer.reshape(layers_tensor_shape[i])) for i, layer in enumerate(layers)]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "o2-IkWhNPIvt",
        "colab_type": "text"
      },
      "source": [
        "(d) Complete the following functions, where the input is the fulled connected layer's weights for FC-NN in its tensor format and the output should be the reconstructed tensor of corresponding tensor decomposition methods and the number of parameters of that decomposition. Note all the `weights` argument is already reshaped into particular tensor shapes and you can directly use tensorly's decomposition functions (e.g. `tensorly.decomposition.parafac` for CP decomposition and  `tensorly.kruskal_to_tensor` to recover the tensor). "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SdJ6QOsN_C2V",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from tensorly.decomposition import parafac\n",
        "from tensorly.decomposition import tucker\n",
        "\n",
        "def get_cp_reconstruction(weights, cp_rank, tol = None):\n",
        "  weights, factors = parafac(weights, rank=cp_rank, init='random', tol=None)\n",
        "  num_params = tl.to_numpy(weights).size + np.sum([tl.to_numpy(f).size for f in factors]) \n",
        "  # Reconstruct the image from the factors\n",
        "  cp_reconstruction = tl.kruskal_to_tensor((weights, factors))\n",
        "  return cp_reconstruction, num_params\n",
        "  \n",
        "def get_tt_reconstruction(weights, rank = 10):\n",
        "  from tensorly.decomposition import matrix_product_state\n",
        "  factors = matrix_product_state(weights, rank)\n",
        "  num_params = np.sum([tl.to_numpy(f).size for f in factors])\n",
        "  from tensorly import mps_to_tensor\n",
        "  return mps_to_tensor(factors), num_params\n",
        "\n",
        "def get_tucker_reconstruction(weights, tucker_ranks, tol = None):\n",
        "  from tensorly.decomposition import tucker\n",
        "  core, factors = tucker(weights, tucker_ranks)\n",
        "  num_params = tl.to_numpy(core).size + np.sum([tl.to_numpy(f).size for f in factors])\n",
        "  return tl.tucker_to_tensor((core, factors)), num_params\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "R4iWuasL_SW5",
        "colab_type": "text"
      },
      "source": [
        "## TT"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SVwIE6ui7CwE",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 479
        },
        "outputId": "0a50a550-a836-46d0-f767-67fd116e2145"
      },
      "source": [
        "#TT\n",
        "tt_ranks = [10, 20, 30, 40, 50]\n",
        "tt_acc = []\n",
        "num_parameters_TT = []\n",
        "for rank_lim in tt_ranks:\n",
        "  recon_layers = []\n",
        "  num_param_total = 0\n",
        "  for layer in layers_tensor:\n",
        "    recon_layer, num_param_layer = get_tt_reconstruction(layer, rank_lim)\n",
        "    recon_layers.append(recon_layer)\n",
        "    num_param_total += num_param_layer\n",
        "  num_parameters_TT.append(num_param_total)\n",
        "  recon_model = Net(hidden_units, [layer.reshape(layers_ori_shape[i]) for i, layer in enumerate(recon_layers)])\n",
        "  recon_model.cuda()\n",
        "  print('Current TT rank is :' +str(rank_lim))\n",
        "  tt_acc.append(test(recon_model, device, test_loader, return_acc = True))\n",
        "\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Current TT rank is :10\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:14: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  \n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:15: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  from ipykernel import kernelapp as app\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:16: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  app.launch_new_instance()\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 2.2117, Accuracy: 1905/10000 (19%)\n",
            "\n",
            "Current TT rank is :20\n",
            "\n",
            "Test set: Average loss: 1.2556, Accuracy: 6061/10000 (61%)\n",
            "\n",
            "Current TT rank is :30\n",
            "\n",
            "Test set: Average loss: 0.4111, Accuracy: 8700/10000 (87%)\n",
            "\n",
            "Current TT rank is :40\n",
            "\n",
            "Test set: Average loss: 0.1553, Accuracy: 9517/10000 (95%)\n",
            "\n",
            "Current TT rank is :50\n",
            "\n",
            "Test set: Average loss: 0.1038, Accuracy: 9676/10000 (97%)\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_A2opE7A_WAv",
        "colab_type": "text"
      },
      "source": [
        "## Tucker"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KeVUrTZer-Gf",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 343
        },
        "outputId": "32bdc0c0-2b35-4449-ecb4-7a817cfc94ac"
      },
      "source": [
        "import gc\n",
        "gc.collect()\n",
        "#Tucker\n",
        "Tucker_rank_list = [2, 3, 4]\n",
        "tucker_acc = []\n",
        "num_parameters_tucker = []\n",
        "for tucker_rank in Tucker_rank_list:\n",
        "  print('Current Tucker rank is :' +str(tucker_rank))\n",
        "  recon_layers = []\n",
        "  num_param_total = 0\n",
        "  for layer in layers_tensor:\n",
        "    tucker_ranks = np.ones(tl.ndim(layer)).astype(int)*tucker_rank\n",
        "    recon_layer, num_param_layer = get_tucker_reconstruction(layer, tucker_ranks)\n",
        "    recon_layers.append(recon_layer)\n",
        "    num_param_total += num_param_layer\n",
        "  num_parameters_tucker.append(num_param_total)\n",
        "  recon_model = Net(hidden_units, [layer.reshape(layers_ori_shape[i]) for i, layer in enumerate(recon_layers)])\n",
        "  recon_model.cuda()\n",
        "  tucker_acc.append(test(recon_model, device, test_loader, return_acc = True))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Current Tucker rank is :2\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:14: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  \n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:15: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  from ipykernel import kernelapp as app\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:16: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  app.launch_new_instance()\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 2.3027, Accuracy: 958/10000 (10%)\n",
            "\n",
            "Current Tucker rank is :3\n",
            "\n",
            "Test set: Average loss: 2.2985, Accuracy: 786/10000 (8%)\n",
            "\n",
            "Current Tucker rank is :4\n",
            "\n",
            "Test set: Average loss: 2.1868, Accuracy: 1520/10000 (15%)\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "C5wDzFO4_UJE",
        "colab_type": "text"
      },
      "source": [
        "## CP"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HsQ9O2_0BQpY",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 615
        },
        "outputId": "f6379d19-e233-4837-cc2e-2d59dbfca55a"
      },
      "source": [
        "import gc\n",
        "gc.collect()\n",
        "#set up CP pararmeters\n",
        "cp_ranks = [10, 50, 100, 200, 300, 400, 500]\n",
        "cp_acc = []\n",
        "num_parameters_cp = []\n",
        "\n",
        "for cp_rank in cp_ranks:\n",
        "  recon_layers = []\n",
        "  num_param_total = 0\n",
        "  print('Current CP rank is :' +str(cp_rank))\n",
        "  for layer in layers_tensor:\n",
        "    recon_layer, num_param_layer = get_cp_reconstruction(layer, cp_rank)\n",
        "    recon_layers.append(recon_layer)\n",
        "    num_param_total += num_param_layer\n",
        "  num_parameters_cp.append(num_param_total)\n",
        "  recon_model = Net(hidden_units, [layer.reshape(layers_ori_shape[i]) for i, layer in enumerate(recon_layers)])\n",
        "  recon_model.cuda()\n",
        "  cp_acc.append(test(recon_model, device, test_loader, return_acc = True))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Current CP rank is :10\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:14: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  \n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:15: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  from ipykernel import kernelapp as app\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:16: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  app.launch_new_instance()\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Test set: Average loss: 2.2926, Accuracy: 933/10000 (9%)\n",
            "\n",
            "Current CP rank is :50\n",
            "\n",
            "Test set: Average loss: 2.4194, Accuracy: 1033/10000 (10%)\n",
            "\n",
            "Current CP rank is :100\n",
            "\n",
            "Test set: Average loss: 2.5244, Accuracy: 1447/10000 (14%)\n",
            "\n",
            "Current CP rank is :200\n",
            "\n",
            "Test set: Average loss: 2.2979, Accuracy: 2860/10000 (29%)\n",
            "\n",
            "Current CP rank is :300\n",
            "\n",
            "Test set: Average loss: 1.7522, Accuracy: 4431/10000 (44%)\n",
            "\n",
            "Current CP rank is :400\n",
            "\n",
            "Test set: Average loss: 1.3265, Accuracy: 4947/10000 (49%)\n",
            "\n",
            "Current CP rank is :500\n",
            "\n",
            "Test set: Average loss: 0.9604, Accuracy: 6002/10000 (60%)\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VZS3deeN_axd",
        "colab_type": "text"
      },
      "source": [
        "## Results"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TER1Li_I4sMJ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 296
        },
        "outputId": "97e18c23-beb3-4982-bbcc-fa6eea477749"
      },
      "source": [
        "plt.plot( num_parameters_cp, cp_acc, label = 'CP')\n",
        "plt.plot( num_parameters_TT, tt_acc, label = 'TT')\n",
        "plt.plot( num_parameters_tucker, tucker_acc, label = 'Tucker')\n",
        "plt.plot( MPO_size, MPO_acc, '-bx', label = 'MPO_NN')\n",
        "plt.legend()\n",
        "plt.xlabel('Number_parameters')\n",
        "plt.ylabel('Accuracy')\n",
        "plt.title('Accuracy on different types of decomposition of the orginal NN')\n",
        "plt.show()\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZsAAAEXCAYAAAB76ulbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3wUdf748debJJCEHhJaAtJ7J4hdQQXFwqmIeuqB56nn935nxe4pd97ZTk8928mdip4V8A7sBYRTLEgR6SUJJaFngVDTP78/Pp/EJWySTcjubJL38/HYx+6UnXnP7My8dz6fmc+IMQallFIqlBp4HYBSSqm6T5ONUkqpkNNko5RSKuQ02SillAo5TTZKKaVCTpONUkqpkNNkU4uIyBkikuXXvVJEznCfRUReFZE9IvKD63ejiOwQkQMi0sqjsOsMEekpIktFZL+I3BTE+JNF5I1wxOYlt311qWB46XYaLoH2hyC+M1VE/hzq2KpKRDq6dRxVA9PybBnDnmxEZJ7bABqFe951jTGmrzFmnus8BTgbSDHGHC8iMcDfgFHGmCbGGF84YxORjSJyVgXDj0ictcSdwFxjTFNjzN+9DiZSuO0rAwIfzMpsp+FyxP5QdqCITBSR+WGOqVqMMZvdOi4K5XzcOjEicmeZ/ll+f2onu3HG+w2Pdv06VTT9sCYbF8ypgAEuDPO8o8M5Pw8cB2w0xhx03W2AWGBldSZWE/+i6qDjqOb6VGFXdn+IWBF2bNoN3CkiTSsZ549VPkYYY8L2Ah4AvsH+4/6wzLAOwH+AXYAPeM5v2HXAamA/sAoY4voboJvfeFOBP7vPZwBZwF3AduDfQEvgQzePPe5zit/3E4BXga1u+EzXfwVwgd94MUA2MLic5bwOSHM/yvtAe79hBvgtsB7YCzwPSDnTiXPLtMct9x1Alt/wjcBZwLVALlAEHADeBg66eR0AvnTj9wK+cHGtBcaXWXcvAh+7754FtAfec+trA3CT3/iTgWnA6+53WQmkumH/BoqBw27+d5ZZrsZuWLEbfsDN6xDQym+8IW7eMcBE7LbzHJADrAHO9Bu3OfAysA3YAvwZiHLDugH/c9/LBt6tYBu90C3LXmAe0Nv1/9Kt31wXb48A3+3s5rPfrefngDf8hp8AfOum/RNwRmXbXpDb0/9ht6f9wENAVzeffe43alhmn7jXrYeNwJVl1uHrbp1vAu4HGlS2Dl0M3YDrgQIg362jD/y3U/e5EfC0W86t7nOjMvHdDux0v+U1FfxW7d362O3Wz3Wuf9n94Y9lvte7zPC9fvvA88BHbl0uALr6fa/c/SfY2Pz2nRnAG+43+g122/nKzXe2i+MNN34nt46jXfc89zt/48b/HEj0m/507DEvx02zb6BjZICYJwLzgQ+AB/36Z+G2VRf7m9jtd4LrF+3i61Th8b86SaO6L7fS/w8Y6jbKNq5/lAv+KeyBKBY4xQ27FHvwGAYIdqM+zn8jD7Qi3YZbCDyG3cDjgFbAJUA80NT9KP479UfAu9ikFAOc7vrfyZE711hgeTnLOBK7Mw5x830W+KrMjvkh0ALoiN2xzylnWo8CX2MPRB2wSe+oZOO/ofgNK7uBNgYygWvcxjHYxdnHb93lACdjz3jjgcXYPwgNgS5ABjDab6PLBca43+8R4PtAsZWzbGf4L4vr9zFwo1/3U8CzfstXCNzqfpvLXLwJbvh/gZfccrYGfgBucMPeBu5zy1W6bQWIqQc20Z7t5nEndpstOVjPA35TwTJ9h/0j1Qg4DXsgKDlgJGP/RI1xcZztupMq2faC2Z5mAc2AvkAeMMf9Xs2xf1Im+K3zQr8YT3fL29MNf91Nqyl2+1kHXFvZOsRvPyTAwYwjt9M/Ad+73ygJmxQfKhPfn9w6GIP9A9KynPX9FfCCi2cQdl8aGWh/CPDdo4a72H3A8dh95E3gnWD2nyrGNhl7/PuFW59x2G3nCey+dgo2CVWUbNKx22uc637Ub96/dr9hSWJfGugYWd46cfHu4ed9q2yyeQP7pyzD/U6RlWzcCizAZWDsP9Nb3ecT3Y8RHeB7nwE3lzPNypJNPhBbQUyDgD3uczvsP+2jNmzsv5T9QDPXPYMy/9b9xn0ZeNyvu4lb7k5+MfvvqNOAu8uZVgZ+iQj7z7G6yeYy4Osy038J9w/GrbvX/YYNBzaXGf8e4FW/jW6237A+wOFAsZWzbGdwdLK5DPjGfY7C/js73m/5tuJ3FohNKFdjiwzzgDi/YVdg61fAHkSn4HcWW05MfwCm+XU3wP7RKdnR5lFOssH+cSgEGvv1e4ufDxh3Af8OsG1PqGTbC2Z7Otlv+GLgLr/uJ4Gn/dZ52RinueWOwu4vffyG3QDMq2wdUrVkkw6M8Rs2GlvcVRLfYfyOA9gznBMCzLMD9sykqV+/R4CpgfaHAN8/ariL/V9+3WOANcHsP1WMbTJH/mEo2Xbi/fq9QcXJ5n6/cf8P+LSc5Wzhvtu8vN8n0Dpx28Vj7vNRycZ9XgDcSJDJJpx1NhOAz40x2a77LdcP7I+zyRhTGOB7HbAbaHXsMsbklnSISLyIvCQim0RkH/bfRwtX9tgB2G2M2VN2IsaYrdhT1ktEpAVwLvZfTyDtsUUQJd89gP23lOw3zna/z4ewB5DyppXp172pnPGCcRwwXET2lryAK4G2fuNklhm/fZnx78Ue2EuUXY7YYyx/ngX0EZHO2H/+OcYY/yuJthi3lTubsOvoOOw/rG1+sb6E/fcM9gxFgB/clVG/Lmf+ZX+7Yuw6SS5n/LLf3WOOrCPw/72OAy4tsz5PwSaacre9ADEF2p52+H0+HKDbf/sKFGN7IBG7DjeVGVYyn2DXYWWOWB6/+ZfwlTkOlLd/tMeus/3lxFtd5e2bwew/VYktM8D4h8oZHnScIhIlIo+KSLo7xm104yRWMr2yHgBuFJE2FYxzP/ZsNzaYCYalYkpE4oDxQJSIlKykRtgD/UDsiu0oItEBEk4mtgw6kEPY4p4SbbFZuIQ5cnRuB3oCw40x20VkEPAjdifKBBJEpIUxZm+Aeb2GLVuNBr4zxmwpJ6at2A0TABFpjC2+K2/8imzDHohKKqU7VmMaJTKB/xljzq5gHP/1lQlsMMZ0r+b8yq77SocbY3JFZBpwFbZ8/N9lRkkWEfFLOB2x5eKZ2DObxEB/WIwx27H1HojIKcBsEfnKGJNWZtStQP+SDhER7PoP5rfbBrQUkcZ+B/OOfsuZiT2zua7sF0WkHeVvezW5PVFOjCuwRUIFbl6r/IZtgSqtw8p+95Ll8d+mt1ZjObZi11lTv4N6abxBqCzOsoLZf6oSm//8t7nx4/0STocqxlfil9hi/rOwiaY5tkhMqjIRY8waEfkPNpmUN84XIlJSNVKpcJ3Z/AJ7WtkHW3Q1CFtJ9zXwK2xxyDbgURFpLCKxInKy++6/gEkiMtRdO99NREp2vqXAL102PwdbBl2Rpth/entFJAF4sGSAMWYb8Anwgoi0FJEYETnN77szseXmN2OLFMrzNnCNiAxyl3c/DCwwxmysJLZApgH3uHhSgN9XYxolPgR6iMjVbtliRGSYiPQuZ/wfgP0icpeIxLl13E9EhgU5vx3YeoOKhrcSkeZl+r+OPZ2/kKOTTWvgJhf7pdht6GP3230OPCkizUSkgYh0FZHTAUTkUrf+wO54BltsVdY04DwROdNdOn47Nol9W9nCGmM2AYuwV+k0dAfkC/xGeQO4QERGu3UZK/by75RKtr2a3J5KlMR4KnA+MN3Yy2qnAX8RkaZuH7vNxV2VdVjZ7/42cL+IJIlIIvYfdJXvRTLGZGJ/l0fcuhyAvTAg2GntAFJEpGGQ4we9/1Q1Nr9tZ7L7XU7kyG2nKppit1kf9o/4w9WcDsAfsXVULSoY5z7sWW+lwpVsJmDL+jcbY7aXvLBX61yJzboXYCv/N2PPTi4DMMZMB/6CLXbbjz3oJ7jp3uy+V3JKO7OSOJ7GVqhlYyspPy0z/Grsv7s12LLiW0oGGGMOY6/M6oy9ai4gY8xsbBn4e9gE2hW4vJK4yvNH7On3BuzBtOzBN2juH9YoF8tW7Gl4ycUTgcYvwh6IBrn5Z2MTf9nkUJ5HsAeVvSIyKcD012APPBlunPau/zfYg9gStxP6WwB0d7H8BRhnfr5/6FfYytVV2IPhDGwRFdiLSxaIyAHsmdDNxt0XUiamtdizqmfdPC7AXoWYH+Qy/xJb17Ub+0em9E+JOwCNxRZF7sL+U76Dn/fBgNteDW9PYH/3Pdht4E3gt+63APtn5iC2rnA+dp97xQ0Lah1i65j6uN800P74Z+yBdRmwHFji+lXHFdj6jK3YC0QedOsrGF9iz662i0h2ZSNXdf+pRmxXYuuufdj18S42aVTV69hjxhbsvvB9NaYBgDFmA/aY07iCcb7B/jGtlBxZBK4qIiIPYC95vcrrWOoyEfkSeMsY8y+/fhOxlfOneBZYLSf2xrw3jDEplY2rvCUi72IvTniw0pFrCW2uJkiu2O1a7BU5KkRcMd0Q7D87peoFVyTX1RUBn4M9C66spKZW0WQTBBG5Dlvs8Ykx5iuv46mrROQ17A1tt5S5kkepuq4t9pLmA8Dfsfeb/ehpRDVMi9GUUkqFnJ7ZKKWUCrmQ3WcjIq9gr2baaYzp5/olYMviO2GvAR9vjNnj7md4hp+bp5hojFlS2TwSExNNp06dQhK/UkrVVYsXL842xiSFc56hvKlzKvbSZv97Uu4G5hhjHhWRu133Xdg78ru713Bsg5DDK5tBp06dWLRoUQ2HrZRSdZuIHEtrJNUSsmI0V5G+u0zvsdg78XHvv/Dr/7qxvse2LNAOpZRSdUK462zauLulwd4UVdLuTjJHtgWURTltHInI9SKySEQW7dq1K3SRKqWUqjGeXSDg2req8qVwxpgpxphUY0xqUlJYixyVUkpVU7ifELdDRNoZY7a5YrKdrv8Wjmx4LoXqNzSolKqDCgoKyMrKIjc3t/KRFQCxsbGkpKQQExPjdShhTzbvY9tJe9S9z/Lr//9E5B3shQE5fsVtSilFVlYWTZs2pVOnTtgLWFVFjDH4fD6ysrLo3Lmz1+GErhhNRN7GPn2up4hkici12CRztoisxzaB/agb/WNs439pwD8JssnqmvD44zB37pH95s61/ZVSkSM3N5dWrVppogmSiNCqVauIORMM2ZmNMeaKcgadGWBcA/wuVLFUZNgwGD8epk2DESNsoinpVkpFFk00VRNJ66vetyAwYoRNLOPHwwMPHJl46rXDe2HjfNinpZlKqWMX7jqbiDRiBNx4Izz0EPzhD/Us0RgDOVmwfbl7LbOvvZvt8PP+BsOu9TZGpSLE9u3bueWWW1i4cCEtWrSgTZs2PP300wwcOJCePXuSn5/PaaedxgsvvECDBvX+v/wRNNlgi85efNEmmhdftMmmTiacogLIXvdzYtn2k33PLXkSsUCrbpCcCkOvgbYDIHmIpyErFSmMMVx00UVMmDCBd955B4CffvqJHTt20LVrV5YuXUphYSEjR45k5syZXHzxxR5HHFnqfbLxr6MpSTJ1oigtdx/sWPnzmcr25bBzNRS5h05Gx0KbvtD3Imjb3yaWNn2gYbkP5VOqXps7dy4xMTH89re/Le03cOBANm7cWNodHR3NSSedRFpamgcRRrZ6n2wWLjwysZTU4SxcWAuTTXERfPEArPkI9mz4uX98IrQbACfcaJNK2/6Q0BWi6v3Pr2qpP36wklVb99XoNPu0b8aDF/Qtd/iKFSsYOnRohdM4dOgQc+bM4U9/+lONxlYX1PujzZ13Ht2vVhajGQMf3QaLp0LPMTD4qp8TS9O2EEFXpShV16SnpzNo0CBEhLFjx3Luued6HVLEqffJps748s820Zx6O5z5gNfRKBVSFZ2BhErfvn2ZMWNGwGEldTaqfHq5RF3w/Yvw9RMwdCKM/IPX0ShVJ40cOZK8vDymTJlS2m/ZsmVkZmZW8C1VQpNNbffTu/Dp3dD7QnuZshaXKRUSIsJ///tfZs+eTdeuXenbty/33HMPbdu29Tq0WkGL0WqzdZ/BzBuh8+lwyb+gQZTXESlVp7Vv355pAZoXWbFihQfR1C56ZlNbbf4epk2wV5ld/iZEN/I6IqWUKpcmm9po+wp4azw0T4ErZ0Cjpl5HpJRSFdJkU9vs3gBvXAwNm8DV/4XGiV5HpJRSldI6m9pk/w7490W2FYAJH0CLDpV/RymlIoAmm9oiNwfeuAQO7IQJ70NST68jUkqpoGmyqQ0KDsPbV8CuNfDLdyEl1euIlFKqSjTZRLqiQpjxa9j0LYx7Bbod9ew5pVSI+Xw+zjzT7nvbt28nKiqK7du3079/f/Lz89m+fTvNmzenefPmJCYmMnv2bI8jjjyabCKZMfDBTbD2YzjvSeinTZYr5YVWrVqVNkczefJkmjRpwqRJk0qHT5w4kfPPP59x48Z5FWLE06vRIpUx8Pn9sPRNGHEfDPuN1xEppVS16ZlNpPrmafjuOTj+BjjtDq+jUSqyfHK3fUZTTWrbH859tGanqUrpmU0kWvI6zJ4M/S+Fcx7V9s6UUrWentlEmtUfwAc3Q7ezYewLoM8xV+poegZS6+iRLJJs/h5mXAvJqTD+NYhu6HVESilVIzTZRJL5T0N8K3svTcPGXkejlFI1RovRIkVhPmz4CgZdAfEJXkejlCrH5MmTj+o3derUsMdR2+iZTaTI/B4KDkJXvWlTKVX3aLKJFGlzoEE0dD7V60iUUqrGabKJFOlzoOOJ+mwapVSdpMkmEuzfYW9Q6zrS60iUUiokNNlEgvQv7bs2sqmUqqM02USC9DnQuDW06e91JEopFRKabLxWXGzPbLqO1NYClIpQPp+PQYMGMWjQINq2bUtycnJpd35+flDTOOOMM1i0aFGII41cep+N17YthUM+LUJTKoJV9oiBUCosLCQ6uvYfqvWvtNfS5wCiFwcoVctMnDiRGTNmlHY3adKk9PNjjz1G//79GThwIHffffcR3ysuLmbixIncf//9FBUVcccddzBs2DAGDBjASy+9BMC8efM49dRTufDCC+nTp094FijEan+6rO3S5kC7gdA40etIlKo1HvvhMdbsXlOj0+yV0Iu7jr/rmKfzySefMGvWLBYsWEB8fDy7d+8uHVZYWMiVV15Jv379uO+++5gyZQrNmzdn4cKF5OXlcfLJJzNq1CgAlixZwooVK+jcufMxxxQJPDmzEZFbRWSliKwQkbdFJFZEOovIAhFJE5F3RaTut0KZmwOZP2gRmlJ1yOzZs7nmmmuIj48HICHh5+anbrjhhtJEA/D555/z+uuvM2jQIIYPH47P52P9+vUAHH/88XUm0YAHZzYikgzcBPQxxhwWkWnA5cAY4CljzDsi8g/gWuDFcMcXVhu+AlME3c7yOhKlapWaOAM5VtHR0RQXFwO2aCyYCwVOOukk5s6dy+23305sbCzGGJ599llGjx59xHjz5s2jceO61RivV3U20UCciEQD8cA2YCRQUgD6GvALj2ILn7TZ0LAppAzzOhKlVBV16tSJxYsXA/D+++9TUFAAwNlnn82rr77KoUOHAI4oRrv22msZM2YM48ePp7CwkNGjR/Piiy+WfnfdunUcPHgwzEsSHmE/szHGbBGRJ4DNwGHgc2AxsNcYU+hGywKSwx1bWBkDaV9Cl9MhKsbraJRSVXTdddcxduxYBg4cyDnnnFN6JnLOOeewdOlSUlNTadiwIWPGjOHhhx8u/d5tt91GTk4OV199NW+++SYbN25kyJAhGGNISkpi5syZXi1SSIkxJrwzFGkJvAdcBuwFpmPPaCYbY7q5cToAnxhj+gX4/vXA9QAdO3YcumnTpnCFXrOy18NzqXD+U5D6a6+jUSrirV69mt69e3sdRq0TaL2JyGJjTGo44/CiGO0sYIMxZpcxpgD4D3Ay0MIVqwGkAFsCfdkYM8UYk2qMSU1KSgpPxKGQNtu+6yMFlFL1gBfJZjNwgojEi4gAZwKrgLnAODfOBGCWB7GFT9ocaNUNWh7ndSRKKRVyYU82xpgF2GKzJcByF8MU4C7gNhFJA1oBL4c7trApyIWN8/WsRilVb3hyU6cx5kHgwTK9M4DjPQgn/DZ/C4WH9ZJnpVS9oc3VeCFtDkQ1hE4nex2JUkqFhSYbL6R/aZ/K2bBu3bSllFLl0WQTbjlbYOcqLUJTStUrmmzCTZ/KqVTIPf44zJ17ZL+5c23/YyEiXHXVVaXdhYWFJCUlcf755wMwdepUkpKSGDRoEH369OGf//xn6bgzZ85kwIAB9O7dm/79+1d68+bEiRNJTk4mLy8PgOzsbDp16gTAxo0bERGeffbZ0vH/3//7f0ydOvXYFjCENNmEW/ocaNoOWteNZsOVikTDhsH48T8nnLlzbfewY2wZqnHjxqxYsYLDhw8D8MUXX5CcfGRjJ5dddhlLly5l3rx53HvvvezYsYOffvqJSZMmMWvWLFavXs3777/PpEmTWLZsWYXzi4qK4pVXXgk4rHXr1jzzzDNBP7zNa/qIgXAqLoL0udDrfBDxOhqlaq1bbgH3LLNytW8Po0dDu3awbRv07g1//KN9BTJoEDz9dOXzHjNmDB999BHjxo3j7bff5oorruDrr78+arzWrVvTtWtXNm3axLPPPsu9995b2opz586dueeee/jrX//Kv//97wqW8xaeeuoprrvuuqOGJSUlcfLJJ/Paa68FHB5p9MwmnLYsgdy90E0flKZUqLVsaRPN5s32vWXLmpnu5ZdfzjvvvENubi7Lli1j+PDhAcfLyMggIyODbt26sXLlSoYOHXrE8NTUVFauXFnhvDp27Mgpp5xSbkK66667eOKJJygqKqrewoSRntmEU8lTObuM8DoSpWq1YM5ASorO/vAHePFFePBBGFEDu96AAQPYuHEjb7/9NmPGjDlq+Lvvvsv8+fNp1KgRL7300hHPs6mOe+65h7Fjx3LeeecdNaxLly4MHz6ct95665jmEQ6abMIpbTYkD4X4Y9v4lFIVK0k006bZBDNixJHdx+rCCy9k0qRJzJs3D5/Pd8Swyy67jOeee+6Ifn369GHx4sUMHDiwtN/ixYvp27dvpfPq3r07gwYNYtq0aQGH33vvvYwbN47TTz+9GksSPlqMFi6H98CWxXoVmlJhsHDhkYllxAjbvXBhzUz/17/+NQ8++CD9+/cPavxJkybxyCOPsHHjRsBeTfbwww9z++23B/X9++67jyeeeCLgsF69etGnTx8++OCDoKblFT2zCZeMeWCKtT00pcLgzjuP7ldyhlMTUlJSuOmmm4Ief9CgQTz22GNccMEFFBQUEBMTw+OPP86gQYOC+n7fvn0ZMmQIS5YsCTj8vvvuY/DgwUHH44WwP8+mJqWmpppFixZ5HUZwZv0OVn8Ad2RAlOZ4papKn2dTPfX5eTb1T+lTOc/QRKOUqpf0yBcOu9bA/q1ahKaUOsrvfvc7vvnmmyP63XzzzVxzzTUeRRQammzCoeSpnHpxgFLHxBiD1LEbop9//vmQTTuSqkm0GC0c0uZAUi9onuJ1JErVWrGxsfh8vog6gEYyYww+n4/Y2FivQwH0zCb08g/Bpm9h2G+8jkSpWi0lJYWsrCx27drldSi1RmxsLCkpkfEnV5NNqG36BorytIkapY5RTExMadtiqvbRYrRQS5sD0bFwnD6VUylVf2myCbX0OTbRxMR5HYlSSnlGk00o7d0M2ev0KjSlVL2nySaU0ubYd30EtFKqntNkE0rpc6BZCiT28DoSpZTylCabUCkqgIz/2avQ6thNaEopVVWabEIlaxHk7dMiNKWUQpNN6KTPAYmCzpH9QCOllAoHTTahkjYHUlIhroXXkSillOc02YTCQR9s/VGL0JRSytFkEwoZcwGjjxRQSilHk00opM2BuJbQPrhHviqlVF2nyaamGWMvDug6EhpEeR2NUkpFBE02NW3HCjiwQ4vQlFLKjyabmlbSRE1XfaSAUkqV0GRT09JmQ+u+0Kyd15EopVTE0GRTk/IOwObvtZVnpZQqQ5NNTdo4H4oLNNkopVQZlSYbEblARGo0KYlICxGZISJrRGS1iJwoIgki8oWIrHfvLWtynmGRNhti4qHjiV5HopRSESWYJHIZsF5EHheRXjU032eAT40xvYCBwGrgbmCOMaY7MMd11y4Zc6HTKRDdyOtIlFIqolSabIwxVwGDgXRgqoh8JyLXi0jT6sxQRJoDpwEvu+nnG2P2AmOB19xorwG/qM70PVNwGHzpkJzqdSRKqTrKGMOCDB+3T/uJpZl7vQ6nSqKDGckYs09EZgBxwC3ARcAdIvJ3Y8yzVZxnZ2AX8KqIDAQWAzcDbYwx29w424E2gb4sItcD1wN07NixirMOod0ZgIFWXb2ORClVx2zPyeW9JVlMX5TJRt8hmjSK5uRurRjUofY09FtpshGRC4FrgG7A68DxxpidIhIPrAKqmmyigSHA740xC0TkGcoUmRljjIiYQF82xkwBpgCkpqYGHMcT2evte2J3b+NQStUJ+YXFzFm9g2mLMvnful0UGxjeOYHfj+zOuf3bEt8wqHOFiBFMtJcATxljvvLvaYw5JCLXVmOeWUCWMWaB656BTTY7RKSdMWabiLQDdlZj2t7xpdn3BD2zUUpV35rt+5i2MIuZS7ew+2A+bZvF8n9ndGPc0BQ6JTb2OrxqCybZTAZKircQkThskddGY8ycqs7QGLNdRDJFpKcxZi1wJvYMaRUwAXjUvc+q6rQ95UuDpu2hUROvI1FK1TI5hwt4/6etTF+UybKsHGKihFF92nJpagqndk8iqkHtf7R8MMlmOnCSX3eR6zfsGOb7e+BNEWkIZGCL6RoA09zZ0iZg/DFMP/x8aVpfo5QKWnGx4fsMH+8uyuTTFdvJKyymV9umPHB+H34xOJmExg29DrFGBZNsoo0x+SUdxph8lySqzRizFAh02VbtvBvSGFtn0+9iryNRSkW4rD2HeG/xFqYvziRrz2GaxkYzPrUD41M70C+5GSK1/ywmkGCSzS4RudAY8z6AiIwFskMbVi1zaDfk7oVW3byORCkVgXILivh81Q6mL8pkflo2xsAp3RK5Y+MNdyUAAB9mSURBVHRPRvdtS2xM3X8cSTDJ5rfYIq/nAAEygV+FNKraxueuRGulV6IppX62YksO0xZlMvPHLezLLSS5RRw3n9mdS4ak0CEh3uvwwqrSZGOMSQdOEJEmrvtAyKOqbUquRNM6G6XqvT0H85m1dAvTFmWxats+GkY34Nx+bRmf2oETu7SiQR2o7K+OoC7UFpHzgL5AbEl5ojHmTyGMq3bJXg8NYqDFcV5HopTyQFGxYX5aNtMWZfLFyh3kFxXTP7k5D43ty4UDk2keH+N1iJ4L5qbOfwDxwAjgX8A44IcQx1W7+NIgoTNE1a6brJRSx2az7xDTF2cyY3EW23JyaRkfw5UndOTSoR3o076Z1+FFlGCOjicZYwaIyDJjzB9F5Engk1AHVqv40rS+Rql64nB+EZ+s2Ma0RZl8n7GbBgKn9UjiD+f34czerWkUXfcr+6sjmGST694PiUh7wAfoYyhLFBfZdtG6j/I6EqVUiBhj+CnLVvZ/sHQr+/MK6ZgQz6RRPbhkaArtmsd5HWLECybZfCAiLYC/AksAA/wzpFHVJjmZUJSvbaIpVQdlH8hj5o9bmLYok3U7DhAb04Ax/dsxPrUDx3dKqLeV/dVRYbJxD02b4x4B8J6IfAjEGmNywhJdbZBdciWa3mOjVF1QWFTM/9btYtqiTOas3klhsWFwxxY8cnF/zh/QjqaxWtlfHRUmG2NMsYg8j32eDcaYPCAvHIHVGqWXPeuZjVK1WfquA0xflMV7S7LYtT+PxCYN+fUpnbl0aArd21Tr8V3KTzDFaHNE5BLgP8aYyGnSP1L41kOj5tA40etIlFJVdDCvkI+W2cr+RZv2ENVAGNEziUtTOzCyV2tiooJ5mLEKRjDJ5gbgNqBQRHKxrQgYY4xe1wf2zCaxG9TR9oyUqmuMMSzetIdpizL5cNk2DuUX0SWpMXef24uLByfTulms1yHWScG0IKDnjxXJToNOJ3sdhVKqEjv35fLeki1MX5RJRvZBGjeM4oIB7Rk/LIUhHVvW2QYwI0UwN3WeFqh/2Yep1Uv5h2BfltbXKBWh8guL+XLNTqYvymTeul0UFRuGdWrJb8/oynn929G4kd6IHS7BrOk7/D7HAscDi4GRIYmoNtmdbt+1TTSlIsq6HfuZviiT/yzZgu9gPq2bNuL607pw6dAUuiTpAw69EEwx2gX+3SLSAXg6ZBHVJiVXouk9Nkp5bl9uAR/+ZCv7l2buJbqBcFbvNowflsJp3ZOI1sp+T1XnHDIL6F3TgdRKJffYJHTxNg6l6qncgiKWbNrDjMVZfLxiG7kFxfRo04T7z+vNRYOTadWkkdchKieYOptnsa0GgH108yBsSwLKlwbNUqBhY68jUarO8x3IY9W2fazauq/0PX3XAYoNNG0UzSVDUhif2oEBKc21sj8CBXNms8jvcyHwtjHmmxDFU7v41mt9jVI1rLjYsGn3IZdUckqTy459P99P3r55LH3aN+Pcfm3pm9yc07onEddQG8CMZMEkmxlArjGmCEBEokQk3hhzKLShRThj7JlN/0u9jkSpWiu3oIi12/cfccayets+DuUXARDdQOjWugknd0ukT7tm9GnfjD7tmtEivqHHkauqCqoFAeAsoOQJnXHA58BJoQqqVjiYDbk52iaaUkGqqBgMbFFY7/bNGJ/aoTSpdG/TRJvsryOCSTax/o+CNsYcEJH69fDsQHzaAKdSgQRTDJbcIo7e7Zpxbv929GnXjL7tm5HSMk7rWuqwYJLNQREZYoxZAiAiQ4HDoQ2rFvCtt++abFQ9psVgKljBJJtbgOkishXbLlpb4LKQRlUb+NIgqiG06Oh1JEqFRfaBPFZrMZiqpmBu6lwoIr2Anq7XWmNMQWjDqgWy0+z9NQ10R1J1ixaDqVAI5j6b3wFvGmNWuO6WInKFMeaFkEcXyXxp2nKAqvW0GEyFSzDFaNcZY54v6TDG7BGR64D6m2yKCmF3BvQ81+tIlAqaFoMpLwWTbKJEREoenCYiUUD9/luTsxmKC/TiABWRiosNG30Hj7rMeOd+LQZT3gkm2XwKvCsiL7nuG4BPQhdSLZCtDXCqyFJcbPjPj1t4+4fNAYvBTumuxWDKW8Ekm7uA64Hfuu5l2CvS6i+9x0ZFkIUbd/PQh6tYlpVDr7ZNtRhMRaRgrkYrFpEFQFdgPJAIvBfqwCKabz3EtoD4Vl5HouqxrD2HeOSTNXy0bBttm8Xy9GWDuHBgexo00KIwFXnKTTYi0gO4wr2ygXcBjDEjwhNaBPOl2bMaLd9WHjiYV8iL89KZ8nUGDQRuOas715/WhfiG+tRJFbkq2jrXAF8D5xtj0gBE5NawRBXpstOgy+leR6HqmeJiw3tLsnj8s7Xs2p/HRYOTufOcnrRrHud1aEpVqqJkczFwOTBXRD4F3sG2IFC/5R2A/Vv10QIqrH7YYOtllm/JYXDHFky5eiiDO7b0OiylglZusjHGzARmikhjYCy22ZrWIvIi8F9jzOdhijGy7E637630SjQVepm7D/HoJ2v4aPk22jWP5ZnLbb2MXqKsaptgLhA4CLwFvCUiLYFLsVeoHVOycffrLAK2GGPOF5HO2LOnVsBi4GpjTP6xzCMk9Eo0FQYH8gp5YW4a/5q/gSgRbj2rB9ef1kUfEKZqrSrVKBpj9gBT3OtY3QysBpq57seAp4wx74jIP4BrgRdrYD41q+QeGy1GUyFQXGyYsSSLv7p6mYsHJ3OH1suoOsCTy1dEJAU4D/gLcJvYMoGRwC/dKK8Bk4nEZONLg+YdIEZ3flWzFmT4+NOHq1i5dR9DOrbgn79KZVCHFl6HpVSN8OpayaeBO4GmrrsVsNcYU+i6s4BkLwKrlG+9FqGpGpW5+xCPfLKaj5dvp33zWP5+xWAuGNBO62VUnRL2ZCMi5wM7jTGLReSManz/emyLBnTsGOZnyRgDvnQYoI/zUcduf24BL8xL5+WvNxDVQLj97B785lStl1F1kxdnNicDF4rIGCAWW2fzDNBCRKLd2U0KsCXQl40xpXVGqampJjwhOwd2Qt4+bRNNHZOiYsOMxZn89bN1ZB/I4+Ihydw5uhdtm8d6HZpSIRP2ZGOMuQe4B8Cd2UwyxlwpItOBcdgr0iYAs8IdW6V8enGAOjbfZ/j40werWLVtH0OPa8nLE1IZqPUyqh6IpPYt7gLeEZE/Az8CL3scz9F86+273mOjqmiz7xAPf7yaT1duJ7lFHM9eMZjztV5G1SOeJhtjzDxgnvucARzvZTyV8qVBVCNonuJ1JKqW2J9bwHNz03h1/kaio4RJo2y9TGyM1suo+iWSzmwiX3aaLUJroAcKVbGiYsP0RZk88flasg/kM25oCneM7kmbZlovo+onTTZV4UuD1r28jkJFuG/Ts3now9Ws3raP1ONa8srEYQxI0XoZVb9psglWUQHs2QC9L/A6EhWhNvkO8vDHq/ls5Q6SW8Tx3C8Hc15/rZdRCjTZBG/vZigu1Bs61VH25Rbw/JdpvPqNrZe5Y3RPrj2ls9bLKOVHk02wst2VaHqPjXKKig3vLszkyc/XsvtQPuOG2HqZ1lovo9RRNNkES1t7Vn6+TcvmTx+uYs32/RzfKYGp5/ehf0pzr8NSKmJpsgmWbz3EJUB8gteRKA9tyLb1Ml+s2kFKyzheuHII5/Zrq/UySlVCk02wfOl6VlOP5Rwu4Lkv1zP12400jGrAnef05Ncna72MUsHSZBOs7PXQ7Uyvo1BhVlhUzLuLMnny83XsOZTPpUNTmDRK62WUqipNNsHI2w8HtmubaPXMN2nZPFRSL9M5gQfO70O/ZK2XUao6NNkEo/TiAL0SrT7YkH2Qv3y0mtmrd9AhIY4XrxzCOVovo9Qx0WQTDF+6fdc6mzot53ABz85Zz2vf2XqZu87pxTUnd9J6GaVqgCabYGSvBwQSungdiQqBwqJi3l6YyVNf2HqZy1I7cNuoHrRuqvUyStUUTTbB8KVBiw4Qowefuubr9bt46MNVrNtxgOGdE3jggj70ba/1MkrVNE02wfCt1/qaOiZj1wEe/ng1s1fvpENCHP+4agij+2q9jFKhosmmMsbYOpsOJ3gdiaoBOYcK+PuX63nt243ExkRx97m2XqZRtNbLKBVKmmwqs3875B/QNtFqucKiYt7+YTN/+2Idew8XcPmwDtx2dk+SmjbyOjSl6gVNNpUpvexZ77Gprb5aZ+tl1u88wAldEvjD+Vovo1S4abKpjM+19qx1NrVO2k5bL/Plmp0c1yqel64eyqg+bbReRikPaLKpjC8douOgWbLXkagg7diXy/Nz03hrwWbiYqK4d0wvJpyk9TJKeUmTTWWy19sitAYNvI5EVSL7QB4vzkvnje83UVRsGD+sA7ed3YPEJlovo5TXNNlUxpcGbft5HYWqwN5D+bz0VQZTv9lIXmERFw9J4aaR3enYKt7r0JRSjiabihTmw56N0PciryNRAezLLeDlrzfwyvwNHMgv5IIB7bn5rO50TWridWhKqTI02VRk7yYwRdomWoQ5mFfI1G83MuWrDHIOF3BO37bcenYPerZt6nVoSqlyaLKpSLa7Ek3vsYkIuQVF/Pu7Tfzjf+n4DuYzsldrbju7hzb7r1QtoMmmInqPTUTIKyzinR8yeX5uGjv353FKt0RuG9WDIR1beh2aUipImmwq4lsP8YkQpwc1LxQUFTNjcRbPzlnP1pxcju+UwLNXDGZ4l1Zeh6aUqiJNNhXxpWt9jQeKig0zf9zCM3PWs3n3IQZ1aMFj4wZwSrdEvSFTqVpKk01FstdDj1FeR1FvFBcbPly+jadnryNj10H6tm/GyxNSGdmrtSYZpWo5TTblyc2Bgzv1zCYMjDF8vmoHT32xjjXb99OjTRP+cdUQRvVpS4MGmmSUqgs02ZSn9OIAvRItVIwxzFu7i799sY7lW3LoktiYZy4fxPkD2hOlSUapOkWTTXl86fZdz2xqnDGGb9N9PPn5WpZs3ktKyzj+Om4AFw1OJjpKmwVSqi7SZFOe7PUgDSChs9eR1Ck/bNjNk5+vZcGG3bRrHstfLurHpUM70DBak4xSdZkmm/L40qBFR4jWRhxrwtLMvTz5+Vq+Xp9NYpNGPHhBH644viOxMdoSs1L1gSab8vjWa31NDVi5NYenvljH7NU7aRkfw71jenH1CZ2Ia6hJRqn6RJNNIMbYOpvjTvE6klpr/Y79PDV7HR8v306z2GgmjerBxJM706SRbnJK1Udh3/NFpAPwOtAGMMAUY8wzIpIAvAt0AjYC440xe8IdHwD7tkLBIUjUiwOqakP2QZ6ZvY5ZP20lPiaKm0Z249pTu9A8Lsbr0JRSHvLib2YhcLsxZomINAUWi8gXwERgjjHmURG5G7gbuMuD+Pwue9ZkE6zM3Yf4+5z1/OfHLcRECdef1oUbTutKQuOGXoemlIoAYU82xphtwDb3eb+IrAaSgbHAGW6014B5eJZsXGvPWmdTqW05h3nuyzSmLcpERJhwYiduPKMrSU31wgql1M88LUAXkU7AYGAB0MYlIoDt2GK2QN+5HrgeoGPHjqEJzJcOMfHQtF1opl8H7Nyfy4vz0nlzwWaMMVw2rAO/G9GNds3jvA5NKRWBPEs2ItIEeA+4xRizz7/tK2OMERET6HvGmCnAFIDU1NSA4xyz7PX2sQIN9N6PsnYfzOelr9J57duNFBQZLhmSzO9HdqdDgj6CWSlVPk+SjYjEYBPNm8aY/7jeO0SknTFmm4i0A3Z6EZtloHUf72YfgXIOF/CvrzN4Zf4GDhUUMXZge24+qwedExt7HZpSqhbw4mo0AV4GVhtj/uY36H1gAvCoe58V7thKXfWeZ7OONAfyCnl1/gb++XUG+3ILGdO/Lbec1YMebfQRzEqp4HlxZnMycDWwXESWun73YpPMNBG5FtgEjPcgNuUczi/i9e828o//pbPnUAFn9W7NrWf3oG97fQSzUqrqvLgabT5QXpO+Z4YzFnW03IIi3v5hM8/PTSf7QB6ndk/k9lE9GdShhdehKaVqMb2dWwGQX1jM9MWZPPdlGttychneOYEXrxrCsE4JXoemlKoDNNnUc4VFxfzXPYI5a89hhnRswROXDuSkrq306ZhKqRqjyaaeKio2fLhsK8/MXk9G9kH6JTfjoV/044weSZpklFI1TpNNLbbj4A7aNA5472u5iosNn63czlOz17FuxwF6tmnKS1cPZVSfNppklFIho8mmltmdu5uPMz5mVvosNu3bxLzx84iPqfyGSmMMX67ZyZOfr2PVtn10SWrMs1cM5rz+7Wigj2BWSoWYJptaoKCogK+2fMWstFl8nfU1haaQfq36cevQWyv97sG8Qr5at4uXvspgaeZeOibE8+SlAxk7qL0+glkpFTaabCLYmt1rmJU2i48yPmJP3h4S4xK5us/VXNj1Qrq1LL9F6q17DzNn9Q5mr97Jdxk+8guLad88lkcv7s8lQ1OI0SSjlAozTTYRxnfYx8cbPmZW2izW7llLTIMYRnQYwdhuYzmp/UlENzj6JysuNizbklOaYFZv2wfAca3iuWr4cZzVuzXDOidoklFKeUaTTQQor5jsvuH3cW7nc2ne6Oi79g/lFzJ/fTZzVu/ky7U72bU/jwYCQ49ryd3n9uKs3q3pmtREK/2VUhFBk42HqlpMti3nMHNW72TO6h18m+4jr7CYJo2iOb1HEmf2bs2Inq1pqQ8rU0pFIE02YeY77OOjjI+YlT6LdXvWVVhMVlxsWLE1h9kuwazcaovHOibE88vhHTmzVxuO75xAw2gtHlNKRTZNNmFQUFTAV1lfMTN9JvOz5lNoCumf2J/7h9/POZ3POaKY7HB+Ed+kZTNnzQ7mrN7JTlc8NqRjS+46xxaPdWutxWNKqdpFk02IGGNsMVm6LSbbm7eXpLgkru57NWO7jqVri66l4+7Yl1taPDY/Lbu0eOy0Homc2asNI3q1JkGLx5RStZgmmxoWqJhsZMeRjO06lhPbn0h0g2iMMazYksPs1fbsZfmWHACSW8RxxfEdObN3a47vnECj6CiPl0YppWqGJpsaUFEx2ehOoxHTmOwDecxb4+PLtTv5cvVOtu/LRQQGdWjBHaN7cmbv1vRs01SLx5RSdZImm2owxpBzuIAfti7now3v8/2O2Rwq2kdcg5YcF30u8fnDObihFX9fns8fDnxHfmFx6XfjG0ZxavdEbuvdg5G9WpPYpJGHS6KUUuFRL5PN+j3rmb5uOrcOvZW46DjAJpB9uYVkH8gje38eu9x79oF8sg/ksWt/HtkH8th5KJu9DRYgTRcRFbsdUxxN4f4+FOQM5eChbpgm8SQ2aURikxi6tm5CUpNGtrtpQ9o2i2NwxxbExmjxmFKqfqmXyeaH7T/w9pq3+W7rdzxy6iP0S+zHeX+fzyp3572/qAZCy8YNaNJyHQVNfuBg0+VEU0zbRj0YlngppyefzXEtW5HUpBEt4xtqo5ZKKRWAGGO8jqHaUlNTzaJFi6r13QXbFnDf/PvwHfZxw8AbaHx4FHkFuLOSRiQ2aciewg3M3fIxn2z8uPRqsgu6XsDYrmPp0qJLDS+NUkqFh4gsNsakhnWe9TXZAOTk5fCXBX/hkw2fMDBpII+c8ghxMXF8lPERM9NmkrY3jYYNGtqrybqN5YR2JwRsm0wppWoTTTZVdKzJpsTHGR/z5+//TEFxAQXFBRSZIgYkDWBs17GM7jQ6YNtkSilVW3mRbPRvOjCmyxiGtBnC80ufJyE2gbHdxtKluRaTKaVUTdFk47Rt3JaHTn7I6zCUUqpO0hYclVJKhZwmG6WUUiGnyUYppVTIabJRSikVcppslFJKhZwmG6WUUiGnyUYppVTIabJRSikVcrW6uRoR2QVsqsZXE4HsGg6nLtD1Epiul8B0vQRWG9bLccaYpHDOsFYnm+oSkUXhbheoNtD1Epiul8B0vQSm6yUwLUZTSikVcppslFJKhVx9TTZTvA4gQul6CUzXS2C6XgLT9RJAvayzUUopFV719cxGKaVUGGmyUUopFXL1KtmIyDkislZE0kTkbq/jqSki8oqI7BSRFX79EkTkCxFZ795buv4iIn9362CZiAzx+84EN/56EZng13+oiCx33/m7iEhF84gUItJBROaKyCoRWSkiN7v+9XrdiEisiPwgIj+59fJH17+ziCxwy/KuiDR0/Ru57jQ3vJPftO5x/deKyGi//gH3tfLmESlEJEpEfhSRD113vV8nNcYYUy9eQBSQDnQBGgI/AX28jquGlu00YAiwwq/f48Dd7vPdwGPu8xjgE0CAE4AFrn8CkOHeW7rPLd2wH9y44r57bkXziJQX0A4Y4j43BdYBfer7unGxNnGfY4AFbhmmAZe7/v8AbnSf/w/4h/t8OfCu+9zH7UeNgM5u/4qqaF8rbx6R8gJuA94CPqwo3vq0Tmps3XodQBg3ohOBz/y67wHu8TquGly+ThyZbNYC7dzndsBa9/kl4Iqy4wFXAC/59X/J9WsHrPHrXzpeefOI1BcwCzhb180R6yQeWAIMx971Hu36l+4vwGfAie5ztBtPyu5DJeOVt6+57wScRyS8gBRgDjAS+LCieOvLOqnJV30qRksGMv26s1y/uqqNMWab+7wdaOM+l7ceKuqfFaB/RfOIOK6YYzD2X3y9XzeuuGgpsBP4Avuve68xptCN4r8spcvvhucAraj6+mpVwTwiwdPAnUCx664o3vqyTmpMfUo29Zaxf5lCeo17OOZRXSLSBHgPuMUYs89/WH1dN8aYImPMIOy/+eOBXh6H5CkROR/YaYxZ7HUsdVV9SjZbgA5+3SmuX121Q0TaAbj3na5/eeuhov4pAfpXNI+IISIx2ETzpjHmP663rhvHGLMXmIstvmkhItFukP+ylC6/G94c8FH19eWrYB5eOxm4UEQ2Au9gi9KeoX6vkxpVn5LNQqC7u/KjIbZS732PYwql94GSq6YmYOsrSvr/yl15dQKQ44p7PgNGiUhLd+XUKGzZ8TZgn4ic4K60+lWZaQWaR0Rw8b4MrDbG/M1vUL1eNyKSJCIt3Oc4bD3WamzSGedGK7teSpZlHPClO1t7H7jcXZnVGeiOvWAi4L7mvlPePDxljLnHGJNijOmEjfdLY8yV1ON1UuO8rjQK5wt7tdE6bPn0fV7HU4PL9TawDSjAlvleiy0LngOsB2YDCW5cAZ5362A5kOo3nV8Dae51jV//VGCF+85z/NzyRMB5RMoLOAVbfLUMWOpeY+r7ugEGAD+69bICeMD174I9MKYB04FGrn+s605zw7v4Tes+t+xrcVfiuf4B97Xy5hFJL+AMfr4aTddJDb20uRqllFIhV5+K0ZRSSnlEk41SSqmQ02SjlFIq5DTZKKWUCjlNNkoppUJOk41SSqmQ02SjIoqIGBF50q97kohMrqFpTxWRcZWPWTeIyEQRae91HEqBJhsVefKAi0Uk0etA/Pk1J1IrputMBKqUbEIcj6rHNNmoSFMITAFuLTug7JmJiBxw72eIyP9EZJaIZIjIoyJypdgHhC0Xka5+kzlLRBaJyDrX+GJJC8h/FZGFYh+adoPfdL8WkfeBVYGCFZFOIrJGRN4UkdUiMkNE4t2wB9w0V4jIFNekDSIyT0SeFpFFwM0icoF7eNaPIjJbRNq48SaLyGsuhk0icrGIPO6W6VPX7lvJA9z+JyKLReQzEWnn1lMq8KaILBWRuEDjlRPPpS7mn0Tkq2P5MZUq5XUTBvrSl/8LOAA0AzZiGzecBEx2w6YC4/zHde9nAHuxz41phG3I8I9u2M3A037f/xT7J6s7tmmfWOB64H43TiNgEfbBV2cAB4HOFcTbCdskzsmu+xVgkvuc4Dfev4EL3Od5wAt+w1ryczM3vwGedJ8nA/OxDzgbCBzi54ez/Rf4hRv2LZDk+l8GvOI3n1T3ubLx/ONZDiS7zy283ib0VTdeesqsIo4xZp+IvA7cBBwO8msLjXt+jIikA5+7/suBEX7jTTPGFAPrRSQD27T+KGCA31lTc2wyygd+MMZsqGTemcaYb9znN1zcTwAjRORO7APKEoCVwAduvHf9vp8CvOvONBoC/vP7xBhTICLLsU97/NRvuToBPYF+wBfuxCkK205eWZWN5x/PN8BUEZkG/AelaoAmGxWpnsY+QfJVv36FuKJfEWmAPTCXyPP7XOzXXcyR23nZxgANtgHO3xtjPvMfICJnYM9sKnPUNEUkFngBe2aR6S5yiPUbx3+6zwJ/M8a87+Y52W9YHoAxplhECowxJfMqWS4BVhpjTqwkxsrGK43HGPNbERkOnAcsFpGhxhhfJdNXqkJaZ6MikjFmN/bZ7Nf69d4IDHWfL8QWDVXVpSLSwNXjdMG2zPsZcKNfHUgPEWlchWl2FJGSg/gvsUVfJYklW+zD2yq6Cq45Pz/DZEIF4wWyFkgqmb+IxIhIXzdsP9A0iPGOICJdjTELjDEPALs48jksSlWLJhsVyZ4E/K9K+ydwuoj8hH3YVzBnHWVtxjbn/gnwW2NMLvAv7AUAS0RkBfASVTvrXwv8TkRWY+tfXjT2oWT/xDbh/xn2eSblmQxMF5HF2OfRB80Yk49NZI+59bIUOMkNngr8Q+zjn6MqGK+sv7qLEFZg63l+qkpMSgWijxhQ6hiISCfss0/6eRyKUhFNz2yUUkqFnJ7ZKBUEESl58mZZZ2rluVKV02SjlFIq5LQYTSmlVMhpslFKKRVymmyUUkqFnCYbpZRSIff/AbPrbnF5N1kLAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    }
  ]
}