diff --git a/README.md b/README.md index aec7b07b..1bf8345c 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,26 @@ run in your browser. + +## Feature Visualization +*Notebooks corresponding to the [Feature Visualization](https://distill.pub/2017/feature-visualization/) article* + + + + + + + + + + + + + + + + + ## Building Blocks *Notebooks corresponding to the [Building Blocks of Interpretability](https://distill.pub/2018/building-blocks/) article* diff --git a/lucid/optvis/style.py b/lucid/optvis/style.py index d0858532..368bc179 100644 --- a/lucid/optvis/style.py +++ b/lucid/optvis/style.py @@ -44,12 +44,13 @@ def __init__(self, style_layers, ema_decay=None, outputs of 'style_func'. """ self.input_grams = [style_func(s) for s in style_layers] + self.ema = None if ema_decay is not None: - ema = tf.train.ExponentialMovingAverage(decay=ema_decay) - update_ema_op = ema.apply(self.input_grams) + self.ema = tf.train.ExponentialMovingAverage(decay=ema_decay) + update_ema_op = self.ema.apply(self.input_grams) with tf.control_dependencies([update_ema_op]): - self.effective_grams = [g + tf.stop_gradient(ema.average(g)-g) - for g in self.input_grams] + self.effective_grams = [g + tf.stop_gradient(self.ema.average(g)-g) + for g in self.input_grams] else: self.effective_grams = self.input_grams diff --git a/lucid/scratch/web/svelte.py b/lucid/scratch/web/svelte.py index ac4b5fe8..0325061e 100644 --- a/lucid/scratch/web/svelte.py +++ b/lucid/scratch/web/svelte.py @@ -68,7 +68,8 @@ def inner(data): @register_cell_magic def html_define_svelte(line, cell): base_name = line.split()[0] - name_str = base_name + "_" + str(uuid.uuid4()) + # name_str will become the name of a javascript variable, and can't contain dashes. + name_str = base_name + "_" + str(uuid.uuid4()).replace('-', '_') html_fname = osp.join(_svelte_temp_dir, name_str + ".html") with open(html_fname, "w") as f: f.write(cell) diff --git a/notebooks/feature-visualization/negative_neurons.ipynb b/notebooks/feature-visualization/negative_neurons.ipynb new file mode 100644 index 00000000..3b58fb4b --- /dev/null +++ b/notebooks/feature-visualization/negative_neurons.ipynb @@ -0,0 +1,244 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "Negative Neurons - Feature Visualization", + "version": "0.3.2", + "views": {}, + "default_view": {}, + "provenance": [] + }, + "kernelspec": { + "name": "python2", + "display_name": "Python 2" + }, + "accelerator": "GPU" + }, + "cells": [ + { + "metadata": { + "id": "JndnmDMp66FL", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "##### Copyright 2018 Google LLC.\n", + "\n", + "Licensed under the Apache License, Version 2.0 (the \"License\");" + ] + }, + { + "metadata": { + "id": "hMqWDc_m6rUC", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "cellView": "both" + }, + "cell_type": "code", + "source": [ + "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "_vAVmphMywZR", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "# Negative Neurons - Feature Visualization\n", + "\n", + "This notebook uses [**Lucid**](https://github.com/tensorflow/lucid) to reproduce the results in [Feature Visualization](https://distill.pub/2017/feature-visualization/).\n", + "\n", + "This notebook doesn't introduce the abstractions behind lucid; you may wish to also read the [Lucid tutorial](https://colab.research.google.com/github/tensorflow/lucid/blob/master/notebooks/tutorial.ipynb).\n", + "\n", + "**Note**: The easiest way to use this tutorial is as a colab notebook, which allows you to dive in with no setup. We recommend you enable a free GPU by going:\n", + "\n", + "> **Runtime**   →   **Change runtime type**   →   **Hardware Accelerator: GPU**" + ] + }, + { + "metadata": { + "id": "FsFc1mE51tCd", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Install, Import, Load Model" + ] + }, + { + "metadata": { + "id": "RBr8QbboRAdU", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "!pip install --quiet lucid==0.0.5\n", + "\n", + "import numpy as np\n", + "import scipy.ndimage as nd\n", + "import tensorflow as tf\n", + "\n", + "import lucid.modelzoo.vision_models as models\n", + "from lucid.misc.io import show\n", + "import lucid.optvis.objectives as objectives\n", + "import lucid.optvis.param as param\n", + "import lucid.optvis.render as render\n", + "import lucid.optvis.transform as transform" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "yNALaA0QRJVT", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "# Let's import a model from the Lucid modelzoo!\n", + "\n", + "model = models.InceptionV1()\n", + "model.load_graphdef()" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "rybqW97zGGTm", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "# Negative Channel Visualizations" + ] + }, + { + "metadata": { + "id": "LwRi0XBvGGTo", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "\n", + "" + ] + }, + { + "metadata": { + "id": "QHfwKTiALXN3", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Unfortunately, constraints on ImageNet mean we can't provide an easy way for you to reproduce the dataset examples. However, we can reproduce the positive / negative optimized visualizations:" + ] + }, + { + "metadata": { + "id": "lYCxFFimLkn0", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "base_uri": "https://localhost:8080/", + "height": 166 + }, + "outputId": "888b052a-5d00-4a66-c4d2-9dbc1588fd62", + "executionInfo": { + "status": "ok", + "timestamp": 1526678025393, + "user_tz": 420, + "elapsed": 15165, + "user": { + "displayName": "Christopher Olah", + "photoUrl": "//lh5.googleusercontent.com/-GhJP0RTFLEs/AAAAAAAAAAI/AAAAAAAAEZ8/wDVK-lwJYfA/s50-c-k-no/photo.jpg", + "userId": "104171973056281402320" + } + } + }, + "cell_type": "code", + "source": [ + "param_f = lambda: param.image(128, batch=2)\n", + "obj = objectives.channel(\"mixed4a_pre_relu\", 492, batch=1) - objectives.channel(\"mixed4a_pre_relu\", 492, batch=0)\n", + "_ = render.render_vis(model, obj, param_f)" + ], + "execution_count": 7, + "outputs": [ + { + "output_type": "stream", + "text": [ + "512 3142.6597\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "" + ] + }, + "metadata": { + "tags": [] + } + } + ] + }, + { + "metadata": { + "id": "TwOurlFpFn56", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "" + ], + "execution_count": 0, + "outputs": [] + } + ] +} \ No newline at end of file diff --git a/notebooks/feature-visualization/neuron_diversity.ipynb b/notebooks/feature-visualization/neuron_diversity.ipynb new file mode 100644 index 00000000..0a015ff2 --- /dev/null +++ b/notebooks/feature-visualization/neuron_diversity.ipynb @@ -0,0 +1,251 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "Diversity - Feature Visualization", + "version": "0.3.2", + "views": {}, + "default_view": {}, + "provenance": [] + }, + "kernelspec": { + "name": "python2", + "display_name": "Python 2" + } + }, + "cells": [ + { + "metadata": { + "id": "JndnmDMp66FL", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "##### Copyright 2018 Google LLC.\n", + "\n", + "Licensed under the Apache License, Version 2.0 (the \"License\");" + ] + }, + { + "metadata": { + "id": "hMqWDc_m6rUC", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "cellView": "both" + }, + "cell_type": "code", + "source": [ + "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "_vAVmphMywZR", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "# Negative Neurons - Feature Visualization\n", + "\n", + "This notebook uses [**Lucid**](https://github.com/tensorflow/lucid) to reproduce some of the results in [Feature Visualization](https://distill.pub/2017/feature-visualization/).\n", + "\n", + "This notebook doesn't introduce the abstractions behind lucid; you may wish to also read the [Lucid tutorial](https://colab.research.google.com/github/tensorflow/lucid/blob/master/notebooks/tutorial.ipynb).\n", + "\n", + "**Note**: The easiest way to use this tutorial is as a colab notebook, which allows you to dive in with no setup. We recommend you enable a free GPU by going:\n", + "\n", + "> **Runtime**   →   **Change runtime type**   →   **Hardware Accelerator: GPU**" + ] + }, + { + "metadata": { + "id": "FsFc1mE51tCd", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Install, Import, Load Model" + ] + }, + { + "metadata": { + "id": "RBr8QbboRAdU", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "!pip install --quiet lucid==0.0.5\n", + "\n", + "import numpy as np\n", + "import scipy.ndimage as nd\n", + "import tensorflow as tf\n", + "\n", + "import lucid.modelzoo.vision_models as models\n", + "from lucid.misc.io import show\n", + "import lucid.optvis.objectives as objectives\n", + "import lucid.optvis.param as param\n", + "import lucid.optvis.render as render\n", + "import lucid.optvis.transform as transform" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "yNALaA0QRJVT", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "# Let's import a model from the Lucid modelzoo!\n", + "\n", + "model = models.InceptionV1()\n", + "model.load_graphdef()" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "ujM5RP7LGGnt", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "# Diversity" + ] + }, + { + "metadata": { + "id": "aXPtRHK0GGnv", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "\n", + "" + ] + }, + { + "metadata": { + "id": "VeRQbcWXZ6cE", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "base_uri": "https://localhost:8080/", + "height": 288 + }, + "outputId": "dc7831bb-e60a-46f2-c040-fc47d48e4a6a", + "executionInfo": { + "status": "ok", + "timestamp": 1526678594901, + "user_tz": 420, + "elapsed": 422578, + "user": { + "displayName": "Christopher Olah", + "photoUrl": "//lh5.googleusercontent.com/-GhJP0RTFLEs/AAAAAAAAAAI/AAAAAAAAEZ8/wDVK-lwJYfA/s50-c-k-no/photo.jpg", + "userId": "104171973056281402320" + } + } + }, + "cell_type": "code", + "source": [ + "param_f = lambda: param.image(128, batch=4)\n", + "obj = objectives.channel(\"mixed5a_pre_relu\", 9) - 1e2*objectives.diversity(\"mixed5a\")\n", + "_ = render.render_vis(model, obj, param_f)" + ], + "execution_count": 4, + "outputs": [ + { + "output_type": "stream", + "text": [ + "WARNING:tensorflow:From /usr/local/lib/python2.7/dist-packages/lucid/optvis/objectives.py:399: calling l2_normalize (from tensorflow.python.ops.nn_impl) with dim is deprecated and will be removed in a future version.\n", + "Instructions for updating:\n", + "dim is deprecated, use axis instead\n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "WARNING:tensorflow:From /usr/local/lib/python2.7/dist-packages/lucid/optvis/objectives.py:399: calling l2_normalize (from tensorflow.python.ops.nn_impl) with dim is deprecated and will be removed in a future version.\n", + "Instructions for updating:\n", + "dim is deprecated, use axis instead\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "512 745.20917\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "" + ] + }, + "metadata": { + "tags": [] + } + } + ] + }, + { + "metadata": { + "id": "tA-tuupQGd-f", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "" + ], + "execution_count": 0, + "outputs": [] + } + ] +} \ No newline at end of file diff --git a/notebooks/feature-visualization/neuron_interaction.ipynb b/notebooks/feature-visualization/neuron_interaction.ipynb new file mode 100644 index 00000000..9b1c8d1d --- /dev/null +++ b/notebooks/feature-visualization/neuron_interaction.ipynb @@ -0,0 +1,496 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "Interaction Between Neurons - Feature Visualization", + "version": "0.3.2", + "views": {}, + "default_view": {}, + "provenance": [] + }, + "kernelspec": { + "name": "python2", + "display_name": "Python 2" + }, + "accelerator": "GPU" + }, + "cells": [ + { + "metadata": { + "id": "JndnmDMp66FL", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "##### Copyright 2018 Google LLC.\n", + "\n", + "Licensed under the Apache License, Version 2.0 (the \"License\");" + ] + }, + { + "metadata": { + "id": "hMqWDc_m6rUC", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "cellView": "both" + }, + "cell_type": "code", + "source": [ + "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "_vAVmphMywZR", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "# Interaction Between Neurons - Feature Visualization\n", + "\n", + "This notebook uses [**Lucid**](https://github.com/tensorflow/lucid) to reproduce some of the results in [Feature Visualization](https://distill.pub/2017/feature-visualization/).\n", + "\n", + "This notebook doesn't introduce the abstractions behind lucid; you may wish to also read the [Lucid tutorial](https://colab.research.google.com/github/tensorflow/lucid/blob/master/notebooks/tutorial.ipynb).\n", + "\n", + "**Note**: The easiest way to use this tutorial is as a colab notebook, which allows you to dive in with no setup. We recommend you enable a free GPU by going:\n", + "\n", + "> **Runtime**   →   **Change runtime type**   →   **Hardware Accelerator: GPU**" + ] + }, + { + "metadata": { + "id": "FsFc1mE51tCd", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Install, Import, Load Model" + ] + }, + { + "metadata": { + "id": "tavMPe3KQ8Cs", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "# Install Lucid\n", + "\n", + "!pip install --quiet lucid==0.0.5\n", + "#!pip install --quiet --upgrade-strategy=only-if-needed git+https://github.com/tensorflow/lucid.git" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "RBr8QbboRAdU", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "# Imports\n", + "\n", + "import numpy as np\n", + "import scipy.ndimage as nd\n", + "import tensorflow as tf\n", + "\n", + "import lucid.modelzoo.vision_models as models\n", + "from lucid.misc.io import show\n", + "import lucid.optvis.objectives as objectives\n", + "import lucid.optvis.param as param\n", + "import lucid.optvis.render as render\n", + "import lucid.optvis.transform as transform" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "yNALaA0QRJVT", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "# Let's import a model from the Lucid modelzoo!\n", + "\n", + "model = models.InceptionV1()\n", + "model.load_graphdef()" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "0x-Zi49jGULb", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "# Combining Objectives" + ] + }, + { + "metadata": { + "id": "YsES8tRfGULc", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "\n", + "" + ] + }, + { + "metadata": { + "id": "hkpAuu-64zfE", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "neuron1 = ('mixed4b_pre_relu', 111) # large fluffy\n", + "# neuron1 = ('mixed3a_pre_relu', 139) # pointilist\n", + "# neuron1 = ('mixed3b_pre_relu', 81) # brush trokes\n", + "# neuron1 = ('mixed4a_pre_relu', 97) # wavy\n", + "# neuron1 = ('mixed4a_pre_relu', 41) # frames\n", + "# neuron1 = ('mixed4a_pre_relu', 479) # B/W\n", + "\n", + "neuron2 = ('mixed4a_pre_relu', 476) # art\n", + "# neuron2 = ('mixed4b_pre_relu', 360) # lattices\n", + "# neuron2 = ('mixed4b_pre_relu', 482) # arcs\n", + "# neuron2 = ('mixed4c_pre_relu', 440) # small fluffy\n", + "# neuron2 = ('mixed4d_pre_relu', 479) # bird beaks\n", + "# neuron2 = ('mixed4e_pre_relu', 718) # shoulders" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "uxfsCgZ-jQ9_", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "base_uri": "https://localhost:8080/", + "height": 464 + }, + "outputId": "24d2ed93-45cc-4333-d3c2-7258965c7358", + "executionInfo": { + "status": "ok", + "timestamp": 1526674752492, + "user_tz": 420, + "elapsed": 200644, + "user": { + "displayName": "Christopher Olah", + "photoUrl": "//lh5.googleusercontent.com/-GhJP0RTFLEs/AAAAAAAAAAI/AAAAAAAAEZ8/wDVK-lwJYfA/s50-c-k-no/photo.jpg", + "userId": "104171973056281402320" + } + } + }, + "cell_type": "code", + "source": [ + "C = lambda neuron: objectives.channel(*neuron)\n", + "\n", + "_ = render.render_vis(model, C(neuron1))\n", + "_ = render.render_vis(model, C(neuron2))\n", + "_ = render.render_vis(model, C(neuron1) + C(neuron2))" + ], + "execution_count": 8, + "outputs": [ + { + "output_type": "stream", + "text": [ + "512 791.29175\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "512 1146.8491\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "512 1272.907\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "" + ] + }, + "metadata": { + "tags": [] + } + } + ] + }, + { + "metadata": { + "id": "nxPbPfXzGWCD", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "# Random Directions" + ] + }, + { + "metadata": { + "id": "YKoI2xj3GWCE", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "\n", + "" + ] + }, + { + "metadata": { + "id": "yvIEwpo0aZGs", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Unfortunately, constraints on ImageNet mean we can't provide an easy way for you to reproduce the dataset examples. However, we can reproduce the random directions, although since they're random, you'll get a different result each time (and they won't match the ones in the article)." + ] + }, + { + "metadata": { + "id": "Oavrm8eJaZ1d", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "base_uri": "https://localhost:8080/", + "height": 166 + }, + "outputId": "8a3bb0e1-65fc-477b-eb52-a71beab73576", + "executionInfo": { + "status": "ok", + "timestamp": 1526666552680, + "user_tz": 420, + "elapsed": 17103, + "user": { + "displayName": "Christopher Olah", + "photoUrl": "//lh5.googleusercontent.com/-GhJP0RTFLEs/AAAAAAAAAAI/AAAAAAAAEZ8/wDVK-lwJYfA/s50-c-k-no/photo.jpg", + "userId": "104171973056281402320" + } + } + }, + "cell_type": "code", + "source": [ + "obj = objectives.direction(\"mixed4d_pre_relu\", np.random.randn(528))\n", + "_ = render.render_vis(model, obj)" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "512 61.127705\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "" + ] + }, + "metadata": { + "tags": [] + } + } + ] + }, + { + "metadata": { + "id": "MxRmLCTlGWSs", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "# Aligned Interpolation" + ] + }, + { + "metadata": { + "id": "WpbF_gDNGWSs", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "\n", + "" + ] + }, + { + "metadata": { + "id": "_HPM3tN5dSmn", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "We hope to explore and explain this trick in more detail in an upcoming article." + ] + }, + { + "metadata": { + "id": "GYujByXLdap7", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "base_uri": "https://localhost:8080/", + "height": 166 + }, + "outputId": "be8fb4d8-3f63-40ad-b7ec-810f97b5610a", + "executionInfo": { + "status": "ok", + "timestamp": 1526667595453, + "user_tz": 420, + "elapsed": 26039, + "user": { + "displayName": "Christopher Olah", + "photoUrl": "//lh5.googleusercontent.com/-GhJP0RTFLEs/AAAAAAAAAAI/AAAAAAAAEZ8/wDVK-lwJYfA/s50-c-k-no/photo.jpg", + "userId": "104171973056281402320" + } + } + }, + "cell_type": "code", + "source": [ + "def interpolate_param_f():\n", + " unique = param.fft_image((6, 128, 128, 3))\n", + " shared = [\n", + " param.lowres_tensor((6, 128, 128, 3), (1, 128//2, 128//2, 3)),\n", + " param.lowres_tensor((6, 128, 128, 3), (1, 128//4, 128//4, 3)),\n", + " param.lowres_tensor((6, 128, 128, 3), (1, 128//8, 128//8, 3)),\n", + " param.lowres_tensor((6, 128, 128, 3), (2, 128//8, 128//8, 3)),\n", + " param.lowres_tensor((6, 128, 128, 3), (1, 128//16, 128//16, 3)),\n", + " param.lowres_tensor((6, 128, 128, 3), (2, 128//16, 128//16, 3)),\n", + " ]\n", + " return param.to_valid_rgb(unique + sum(shared), decorrelate=True)\n", + "\n", + "obj = objectives.channel_interpolate(\"mixed4a_pre_relu\", 476, \"mixed4a_pre_relu\", 460)\n", + "\n", + "_ = render.render_vis(model, obj, interpolate_param_f)" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "512 7413.614\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "" + ] + }, + "metadata": { + "tags": [] + } + } + ] + } + ] +} \ No newline at end of file diff --git a/notebooks/feature-visualization/regularization.ipynb b/notebooks/feature-visualization/regularization.ipynb new file mode 100644 index 00000000..f7accf6c --- /dev/null +++ b/notebooks/feature-visualization/regularization.ipynb @@ -0,0 +1,556 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "Visualization Regularization - Feature Visualization", + "version": "0.3.2", + "views": {}, + "default_view": {}, + "provenance": [] + }, + "kernelspec": { + "name": "python2", + "display_name": "Python 2" + }, + "accelerator": "GPU" + }, + "cells": [ + { + "metadata": { + "id": "JndnmDMp66FL", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "##### Copyright 2018 Google LLC.\n", + "\n", + "Licensed under the Apache License, Version 2.0 (the \"License\");" + ] + }, + { + "metadata": { + "id": "hMqWDc_m6rUC", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "cellView": "both" + }, + "cell_type": "code", + "source": [ + "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "_vAVmphMywZR", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "# Visualization Regularization - Feature Visualization\n", + "\n", + "This notebook uses [**Lucid**](https://github.com/tensorflow/lucid) to reproduce some of the results in the section [\"The Enemy of Feature Visualization\"](https://distill.pub/2017/feature-visualization/#enemy-of-feature-vis) of [Feature Visualization](https://distill.pub/2017/feature-visualization/).\n", + "\n", + "This notebook doesn't introduce the abstractions behind lucid; you may wish to also read the [Lucid tutorial](https://colab.research.google.com/github/tensorflow/lucid/blob/master/notebooks/tutorial.ipynb).\n", + "\n", + "**Note**: The easiest way to use this tutorial is as a colab notebook, which allows you to dive in with no setup. We recommend you enable a free GPU by going:\n", + "\n", + "> **Runtime**   →   **Change runtime type**   →   **Hardware Accelerator: GPU**" + ] + }, + { + "metadata": { + "id": "FsFc1mE51tCd", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Install, Import, Load Model" + ] + }, + { + "metadata": { + "id": "RBr8QbboRAdU", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "!pip install --quiet lucid==0.0.5\n", + "\n", + "import numpy as np\n", + "import scipy.ndimage as nd\n", + "import tensorflow as tf\n", + "\n", + "import lucid.modelzoo.vision_models as models\n", + "from lucid.misc.io import show\n", + "import lucid.optvis.objectives as objectives\n", + "import lucid.optvis.param as param\n", + "import lucid.optvis.render as render\n", + "import lucid.optvis.transform as transform" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "yNALaA0QRJVT", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "# Let's import a model from the Lucid modelzoo!\n", + "\n", + "model = models.InceptionV1()\n", + "model.load_graphdef()" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "d3ZKjNizGWtD", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "# Naive Feature Visualization" + ] + }, + { + "metadata": { + "id": "THlwQw-1GWtD", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "\n", + "" + ] + }, + { + "metadata": { + "id": "3d6xz3CvTLu5", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "base_uri": "https://localhost:8080/", + "height": 170 + }, + "outputId": "d6f236a5-3108-4246-98c5-a4ebef1c3138", + "executionInfo": { + "status": "ok", + "timestamp": 1526677586263, + "user_tz": 420, + "elapsed": 35190, + "user": { + "displayName": "Christopher Olah", + "photoUrl": "//lh5.googleusercontent.com/-GhJP0RTFLEs/AAAAAAAAAAI/AAAAAAAAEZ8/wDVK-lwJYfA/s50-c-k-no/photo.jpg", + "userId": "104171973056281402320" + } + } + }, + "cell_type": "code", + "source": [ + "LEARNING_RATE = 0.05\n", + "\n", + "optimizer = tf.train.AdamOptimizer(LEARNING_RATE)\n", + "\n", + "imgs = render.render_vis(model, \"mixed4b_pre_relu:452\",\n", + " optimizer=optimizer,\n", + " transforms=[],\n", + " param_f=lambda: param.image(64, fft=False, decorrelate=False), \n", + " thresholds=(1, 32, 128, 256, 2048), verbose=False)\n", + "\n", + "\n", + "# Note that we're doubling the image scale to make artifacts more obvious\n", + "show([nd.zoom(img[0], [2,2,1], order=0) for img in imgs])" + ], + "execution_count": 3, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "
\n", + " 0
\n", + " \n", + "
\n", + " 1
\n", + " \n", + "
\n", + " 2
\n", + " \n", + "
\n", + " 3
\n", + " \n", + "
\n", + " 4
\n", + " \n", + "
" + ] + }, + "metadata": { + "tags": [] + } + } + ] + }, + { + "metadata": { + "id": "zlIWBnagGd4q", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "# Frequency Penalization" + ] + }, + { + "metadata": { + "id": "1HdYBKktGd4s", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "\n", + "" + ] + }, + { + "metadata": { + "id": "_dwIwFdmTHgd", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "base_uri": "https://localhost:8080/", + "height": 170 + }, + "outputId": "fa3a94a5-cbab-490d-9aca-869e92c0e3bc", + "executionInfo": { + "status": "ok", + "timestamp": 1526677620923, + "user_tz": 420, + "elapsed": 34616, + "user": { + "displayName": "Christopher Olah", + "photoUrl": "//lh5.googleusercontent.com/-GhJP0RTFLEs/AAAAAAAAAAI/AAAAAAAAEZ8/wDVK-lwJYfA/s50-c-k-no/photo.jpg", + "userId": "104171973056281402320" + } + } + }, + "cell_type": "code", + "source": [ + "L1 = -0.05\n", + "TV = -0.25\n", + "BLUR = -1\n", + "\n", + "obj = objectives.channel(\"mixed4b_pre_relu\", 452)\n", + "obj += L1 * objectives.L1()\n", + "obj += TV * objectives.total_variation()\n", + "obj += BLUR * objectives.blur_input_each_step()\n", + "\n", + "imgs = render.render_vis(model, obj,\n", + " transforms=[],\n", + " param_f=lambda: param.image(64, fft=False, decorrelate=False), \n", + " thresholds=(1, 32, 128, 256, 2048), verbose=False)\n", + "\n", + "\n", + "# Note that we're doubling the image scale to make artifacts more obvious\n", + "show([nd.zoom(img[0], [2,2,1], order=0) for img in imgs])" + ], + "execution_count": 4, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "
\n", + " 0
\n", + " \n", + "
\n", + " 1
\n", + " \n", + "
\n", + " 2
\n", + " \n", + "
\n", + " 3
\n", + " \n", + "
\n", + " 4
\n", + " \n", + "
" + ] + }, + "metadata": { + "tags": [] + } + } + ] + }, + { + "metadata": { + "id": "mSDJKMf5GeZy", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "# Transformation Robustness" + ] + }, + { + "metadata": { + "id": "Za8zeP8HGeZ6", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "\n", + "" + ] + }, + { + "metadata": { + "id": "1rRrTzSBNrmP", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "base_uri": "https://localhost:8080/", + "height": 170 + }, + "outputId": "ae3f89ee-9f08-470a-ad40-04ed2c0d8291", + "executionInfo": { + "status": "ok", + "timestamp": 1526677670210, + "user_tz": 420, + "elapsed": 49229, + "user": { + "displayName": "Christopher Olah", + "photoUrl": "//lh5.googleusercontent.com/-GhJP0RTFLEs/AAAAAAAAAAI/AAAAAAAAEZ8/wDVK-lwJYfA/s50-c-k-no/photo.jpg", + "userId": "104171973056281402320" + } + } + }, + "cell_type": "code", + "source": [ + "JITTER = 1\n", + "ROTATE = 5\n", + "SCALE = 1.1\n", + "\n", + "transforms = [\n", + " transform.pad(2*JITTER),\n", + " transform.jitter(JITTER),\n", + " transform.random_scale([SCALE ** (n/10.) for n in range(-10, 11)]),\n", + " transform.random_rotate(range(-ROTATE, ROTATE+1))\n", + "]\n", + "\n", + "imgs = render.render_vis(model, \"mixed4b_pre_relu:452\", transforms=transforms,\n", + " param_f=lambda: param.image(64), \n", + " thresholds=(1, 32, 128, 256, 2048), verbose=False)\n", + "\n", + "\n", + "# Note that we're doubling the image scale to make artifacts more obvious\n", + "show([nd.zoom(img[0], [2,2,1], order=0) for img in imgs])" + ], + "execution_count": 5, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "
\n", + " 0
\n", + " \n", + "
\n", + " 1
\n", + " \n", + "
\n", + " 2
\n", + " \n", + "
\n", + " 3
\n", + " \n", + "
\n", + " 4
\n", + " \n", + "
" + ] + }, + "metadata": { + "tags": [] + } + } + ] + }, + { + "metadata": { + "id": "gvF6j5h4GkDe", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "# Preconditioning" + ] + }, + { + "metadata": { + "id": "r4o7DzPXGkDg", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "\n", + "" + ] + }, + { + "metadata": { + "id": "1E5O1S3DR8YC", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "base_uri": "https://localhost:8080/", + "height": 170 + }, + "outputId": "2e919ef1-8865-41d6-a303-dd36cb76c60a", + "executionInfo": { + "status": "ok", + "timestamp": 1526677718571, + "user_tz": 420, + "elapsed": 48287, + "user": { + "displayName": "Christopher Olah", + "photoUrl": "//lh5.googleusercontent.com/-GhJP0RTFLEs/AAAAAAAAAAI/AAAAAAAAEZ8/wDVK-lwJYfA/s50-c-k-no/photo.jpg", + "userId": "104171973056281402320" + } + } + }, + "cell_type": "code", + "source": [ + "LEARNING_RATE = 0.05\n", + "DECORRELATE = True\n", + "ROBUSTNESS = True\n", + "\n", + "# Note: fft does spatial decorrelation, decorrelate does channel decorrelation\n", + "param_f = lambda: param.image(64, fft=DECORRELATE, decorrelate=DECORRELATE)\n", + "\n", + "if ROBUSTNESS:\n", + " transforms = transform.standard_transforms\n", + "else:\n", + " transforms = []\n", + "\n", + "optimizer = tf.train.AdamOptimizer(LEARNING_RATE)\n", + "\n", + "imgs = render.render_vis(model, \"mixed4b_pre_relu:452\",\n", + " optimizer=optimizer,\n", + " transforms=transforms,\n", + " param_f=param_f, \n", + " thresholds=(1, 32, 128, 256, 2048), verbose=False)\n", + "\n", + "\n", + "# Note that we're doubling the image scale to make artifacts more obvious\n", + "show([nd.zoom(img[0], [2,2,1], order=0) for img in imgs])" + ], + "execution_count": 6, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "
\n", + " 0
\n", + " \n", + "
\n", + " 1
\n", + " \n", + "
\n", + " 2
\n", + " \n", + "
\n", + " 3
\n", + " \n", + "
\n", + " 4
\n", + " \n", + "
" + ] + }, + "metadata": { + "tags": [] + } + } + ] + }, + { + "metadata": { + "id": "7RKNP3a8Etgv", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "" + ], + "execution_count": 0, + "outputs": [] + } + ] +} \ No newline at end of file