From b501cfc53edaeaf64d8a451b3c55c7bea3dbbd10 Mon Sep 17 00:00:00 2001
From: Andreas Leitherer <leitherer@fhi-berlin.mpg.de>
Date: Fri, 29 Jan 2021 15:23:50 +0100
Subject: [PATCH] Changed notebook to version of AI tutorial
---
convolutional_nn.ipynb | 176 +++++++++++++----------------------------
1 file changed, 55 insertions(+), 121 deletions(-)
diff --git a/convolutional_nn.ipynb b/convolutional_nn.ipynb
index 8a8f9c8..d9494fc 100644
--- a/convolutional_nn.ipynb
+++ b/convolutional_nn.ipynb
@@ -14,7 +14,7 @@
"\n",
" \n",
" <div style=\"text-align:center\">\n",
- " <b><font size=\"6.4\">Convolutional Neural networks</font></b> \n",
+ " <b><font size=\"6.4\">Convolutional neural networks</font></b> \n",
" </div>\n",
" \n",
"<p>\n",
@@ -22,6 +22,7 @@
" created by:\n",
" Angelo Ziletti, <sup>1</sup> \n",
" Andreas Leitherer,<sup>1</sup> \n",
+ " Matthias Scheffler,<sup>1</sup> \n",
" and Luca Ghiringhelli<sup>1</sup> <br><br>\n",
" \n",
" <sup>1</sup> Fritz Haber Institute of the Max Planck Society, Faradayweg 4-6, D-14195 Berlin, Germany <br>\n",
@@ -58,8 +59,8 @@
"execution_count": null,
"metadata": {
"ExecuteTime": {
- "end_time": "2021-01-29T13:41:34.146532Z",
- "start_time": "2021-01-29T13:41:34.131890Z"
+ "end_time": "2020-01-31T16:07:53.077441Z",
+ "start_time": "2020-01-31T16:07:49.017039Z"
}
},
"outputs": [],
@@ -177,7 +178,7 @@
"### 3.1 Summary and intuition\n",
"The convolutional layer's parameters consist of a set of learnable filters. Every filter is small spatially (along width and height), but extends through the full depth of the input volume. For example, a typical filter on a first layer of a ConvNet might have size 5x5x3 (i.e. 5 pixels width and height, and 3 because images have depth 3, the color channels). \n",
"\n",
- "* During the forward pass, we slide (more precisely, convolve) each filter across the width and height of the input volume and compute dot products between the entries of the filter and the input at any position. Intuitively, a convolution can be thought as a sliding (weigthed) average. \n",
+ "* During the forward pass, we slide (more precisely, convolve) each filter across the width and height of the input volume and compute dot products between the entries of the filter and the input at any position. Intuitively, a convolution can be thought as a sliding (weighted) average. \n",
"\n",
"* As we slide the filter over the width and height of the input volume we will produce a 2-dimensional activation map that gives the responses of that filter at every spatial position. Intuitively, the network will learn filters that activate when they see some type of visual feature such as an edge of some orientation or a blotch of some color on the first layer, or eventually entire honeycomb or wheel-like patterns on higher layers of the network. \n",
"\n",
@@ -266,8 +267,8 @@
"execution_count": null,
"metadata": {
"ExecuteTime": {
- "end_time": "2021-01-29T13:41:34.963545Z",
- "start_time": "2021-01-29T13:41:34.149160Z"
+ "end_time": "2020-01-31T16:14:27.840047Z",
+ "start_time": "2020-01-31T16:14:24.991881Z"
}
},
"outputs": [],
@@ -278,7 +279,7 @@
"print(\"Retrieving picture of Max Planck. Saving image to './img_max_planck.jpg'.\")\n",
"urllib.request.urlretrieve(\"https://upload.wikimedia.org/wikipedia/commons/thumb/c/c7/Max_Planck_1933.jpg/220px-Max_Planck_1933.jpg\", \"./img_max_planck.jpg\")\n",
"\n",
- "# retrive a picture of Berlin\n",
+ "# retrieve a picture of Berlin\n",
"print(\"Retrieving picture of Berlin landscape. Saving image to './img_berlin_landscape.jpg'.\")\n",
"urllib.request.urlretrieve(\"http://vivalifestyleandtravel.com/images/cache/c-1509326560-44562570.jpg\", \"./img_berlin_landscape.jpg\")\n",
"\n",
@@ -297,8 +298,8 @@
"execution_count": null,
"metadata": {
"ExecuteTime": {
- "end_time": "2021-01-29T13:41:34.980991Z",
- "start_time": "2021-01-29T13:41:34.967608Z"
+ "end_time": "2020-01-31T16:14:29.172575Z",
+ "start_time": "2020-01-31T16:14:29.153521Z"
},
"code_folding": [
7
@@ -346,8 +347,8 @@
"execution_count": null,
"metadata": {
"ExecuteTime": {
- "end_time": "2021-01-29T13:41:35.014817Z",
- "start_time": "2021-01-29T13:41:34.985684Z"
+ "end_time": "2020-01-31T16:14:30.214968Z",
+ "start_time": "2020-01-31T16:14:30.086647Z"
}
},
"outputs": [],
@@ -434,8 +435,8 @@
"execution_count": null,
"metadata": {
"ExecuteTime": {
- "end_time": "2021-01-29T13:41:35.029178Z",
- "start_time": "2021-01-29T13:41:35.017415Z"
+ "end_time": "2020-01-31T16:14:36.274832Z",
+ "start_time": "2020-01-31T16:14:36.245426Z"
}
},
"outputs": [],
@@ -474,7 +475,7 @@
" [-1., 8., -1.], \n",
" [-1., -1., -1.]])\n",
"\n",
- "#the emboss kernel givens the illusion of depth by emphasizing the differences of pixels in a given direction\n",
+ "# the emboss kernel gives the illusion of depth by emphasizing the differences of pixels in a given direction\n",
"# in this case, in a direction along a line from the top left to the bottom right.\n",
"k_emboss = np.array([[-2., -1., 0.], \n",
" [-1., 1., 1.], \n",
@@ -486,8 +487,8 @@
"execution_count": null,
"metadata": {
"ExecuteTime": {
- "end_time": "2021-01-29T13:41:35.142836Z",
- "start_time": "2021-01-29T13:41:35.031088Z"
+ "end_time": "2020-01-31T16:14:37.151461Z",
+ "start_time": "2020-01-31T16:14:36.906182Z"
}
},
"outputs": [],
@@ -508,8 +509,8 @@
"execution_count": null,
"metadata": {
"ExecuteTime": {
- "end_time": "2021-01-29T13:41:36.038195Z",
- "start_time": "2021-01-29T13:41:35.144383Z"
+ "end_time": "2020-01-31T16:14:38.814111Z",
+ "start_time": "2020-01-31T16:14:37.154074Z"
}
},
"outputs": [],
@@ -522,8 +523,8 @@
"execution_count": null,
"metadata": {
"ExecuteTime": {
- "end_time": "2021-01-29T13:41:37.333228Z",
- "start_time": "2021-01-29T13:41:36.041424Z"
+ "end_time": "2020-01-31T16:14:43.760221Z",
+ "start_time": "2020-01-31T16:14:41.938990Z"
}
},
"outputs": [],
@@ -544,7 +545,7 @@
"\n",
"As you can see above, the effect are similar for both pictures, and it is defined by the kernel with which the image is convolved.\n",
"\n",
- "In the case of **convolutional neural network**, the **kernels** will not be the one reported above, but they are going to be **learned by the network** from the data (by minimizing the classification error). "
+ "In the case of **convolutional neural networks**, the **kernels** will not be the one reported above, but they are going to be **learned by the network** from the data (by minimizing the classification error). "
]
},
{
@@ -588,8 +589,8 @@
"execution_count": null,
"metadata": {
"ExecuteTime": {
- "end_time": "2021-01-29T13:41:37.669098Z",
- "start_time": "2021-01-29T13:41:37.335534Z"
+ "end_time": "2020-01-31T16:15:00.451462Z",
+ "start_time": "2020-01-31T16:14:50.496500Z"
},
"code_folding": []
},
@@ -636,7 +637,8 @@
"\n",
"We now build a convolutional neural network using Keras, a simple and intuitive high-level neural networks API, written in Python and capable of running on top of TensorFlow, CNTK, or Theano; it also runs seamlessly on CPU and GPU.\n",
"\n",
- "For more information on Keras, please visit https://keras.io/\n",
+ "For more information on Keras, please visit https://keras.io/. Note that this link refers to the newest version of Keras (>2.4), which only supports Tensorflow (https://www.tensorflow.org/) as a backend. This tutorial (as well as the one on multilayer perceptrons) is compatible with versions <=2.3 which allows multiple backends (CNTK, Tensorflow, Theano). There are only slight differences in syntax and you can find archived documentations at https://github.com/faroit/keras-docs, e.g., for \n",
+ "version 2.1.5 https://faroit.com/keras-docs/2.1.5/. In both tutorials, we use tensorflow as backend (version <2.0).\n",
"\n",
"We start by defining the architecture (i.e. the shape) of the network. We use two convolutional layers, one max pooling, and one fully connected layer. There is no particular reason behind this choice, and other - better performing - choices are possible."
]
@@ -644,12 +646,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2021-01-29T13:41:37.753694Z",
- "start_time": "2021-01-29T13:41:37.671293Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"model_no_reg = Sequential()\n",
@@ -675,22 +672,17 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "Now, we train the neural network; you can decide the number of epoch you want to use. The more epochs, the more time the network will be able to see the training samples, but this will results in an increase of computational time (proportial to `nb_epochs`).\n",
+ "Now, we train the neural network; you can decide the number of epoch you want to use. The more epochs, the more time the network will be able to see the training samples, but this will result in an increase of computational time (proportional to `nb_epochs`).\n",
"\n",
"An **epoch** is a single step in training a neural network; one epoch is completed when the neural network has seen every training sample once. \n",
"\n",
- "<span style=\"color:red\"> **Run the cell below to start training your first convolutional neural network. The full optimization should take approximately 7 minutes (~1 min per epoch); please take this time to read carefully the materials above, and maybe check out some external references. To save time, you can load a pretrained model via setting \"reload=True\" in the below cell (the standard behaviour) while setting \"reload=False\" allows you to start with a fresh model**</span>."
+ "<span style=\"color:red\"> **Run the cell below to start training your first convolutional neural network. For the current setting for the number of epochs, the full optimization should take approximately 20 minutes (~4 min per epoch); please take this time to read carefully the materials above, and maybe check out some external references. To save time, you can load a pretrained model via setting \"reload=True\" in the below cell (the standard behaviour) while setting \"reload=False\" allows you to start with a fresh model**</span>."
]
},
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2021-01-29T13:41:43.403377Z",
- "start_time": "2021-01-29T13:41:37.755703Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"nb_epochs = 7 \n",
@@ -718,7 +710,7 @@
"score = model_no_reg.evaluate(x_test, y_test, verbose=0)\n",
"\n",
"print('Test loss:', score[0])\n",
- "print('Test accuracy:', score[1])\n"
+ "print('Test accuracy:', score[1])"
]
},
{
@@ -731,12 +723,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2021-01-29T13:41:43.536081Z",
- "start_time": "2021-01-29T13:41:43.404882Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"# Plot training & validation accuracy values\n",
@@ -752,12 +739,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2021-01-29T13:41:43.675909Z",
- "start_time": "2021-01-29T13:41:43.537960Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"# Plot training & validation loss values\n",
@@ -777,9 +759,9 @@
"#### Questions\n",
"\n",
"1. Look at the *Accuracy plot* above. Which information can you gather from it? What is happening to the *training* and *test* accuracy over epochs?\n",
- "2. Is the behaviour of *training* and *test* accuracy that you are observing desirable? Is *overfitting* occuring?\n",
+ "2. Is the behavior of *training* and *test* accuracy that you are observing desirable? Is *overfitting* occurring?\n",
"3. Should you look at the *training* or at the *test* accuracy to have an estimate of the generalization ability of the model?\n",
- "4. Compare the *Accuracy plot* with the *Model loss plot* below. Do they show the same qualititive behaviour?\n"
+ "4. Compare the *Accuracy plot* with the *Model loss plot* below. Do they show the same qualitative behavior?\n"
]
},
{
@@ -793,34 +775,13 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "To have a valuable machine learning model, it is of paramount important to make sure that the model generalizes well to unseen samples.\n",
- "\n",
- "Bengio in Ref. [1] defines regularization as *any modification we make to a learning algorithm that is intended to reduce its generalization error but not its training error.* In practice, additional terms are added to the training optimization objective to prevent overfitting or help the optimization.\n",
- "\n",
- "There are numerous ways to regularize a neural network; we refer the interested reader to Chap 7 of Ref. 1: https://www.deeplearningbook.org/contents/regularization.html\n",
- "\n",
- "Here, we use **dropout layers** to regularize our neural network. To explain in a few word what dropout is, we report below the abstract from the article that introduced dropout (Srivastava et al., J. Mach. Learn. Res. 15 1929 (2014))\n",
- "\n",
- "*Deep neural nets with a large number of parameters are very powerful machine learning systems. However, overfitting is a serious problem in such networks. Large networks are also slow to use, making it difficult to deal with overfitting by combining the predictions of many different large neural nets at test time. Dropout is a technique for addressing this problem.*\n",
- "\n",
- "*The key idea is to randomly drop units (along with their connections) from the neural network during training. This prevents units from co-adapting too much. During training, dropout samples from an exponential number of different “thinned” networks. At test time, it is easy to approximate the effect of averaging the predictions of all these thinned networks by simply using a single unthinned network that has smaller weights.*\n",
- "\n",
- "*This significantly reduces overfitting and gives major improvements over other regularization methods. We show that dropout improves the performance of neural networks on supervised learning tasks in vision, speech recognition, document classification and computational biology, obtaining state-of-the-art results on many benchmark data sets.*\n",
- "\n",
- "For the full article \"Srivastava et al., Dropout: A Simple Way to Prevent Neural Networks from Overfitting\", please visit http://jmlr.org/papers/volume15/srivastava14a.old/srivastava14a.pdf.\n",
- "\n",
- "\n"
+ "As we discussed in the tutorial on multilayer perceptrons, regularization techniques are extremely useful to improve the generalization ability of machine learning models. We will again use dropout, now in context of convolutional neural networks, and investigate its influence on model performance."
]
},
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2021-01-29T13:41:43.778665Z",
- "start_time": "2021-01-29T13:41:43.677715Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"model = Sequential()\n",
@@ -848,18 +809,13 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- " <span style=\"color:red\"> **Run the cell below to start training your *regularized* convolutional neural network. The full optimization should take approximately 7 minutes (~1 min per epoch); please take this time to read carefully the materials above, and maybe check out some external references. To save time, you can load a pretrained model via setting \"reload=True\" in the below cell (the standard behaviour) while setting \"reload=False\" allows you to start with a fresh model**</span>."
+ " <span style=\"color:red\"> **Run the cell below to start training your *regularized* convolutional neural network. For the current setting for the number of epochs, the full optimization should take approximately 20 minutes (~4 min per epoch); please take this time to read carefully the materials above, and maybe check out some external references. To save time, you can load a pretrained model via setting \"reload=True\" in the below cell (the standard behaviour) while setting \"reload=False\" allows you to start with a fresh model**</span>."
]
},
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2021-01-29T13:41:49.788038Z",
- "start_time": "2021-01-29T13:41:43.780338Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"nb_epochs = 7\n",
@@ -894,12 +850,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2021-01-29T13:41:49.930075Z",
- "start_time": "2021-01-29T13:41:49.790072Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"# Plot training & validation accuracy values\n",
@@ -915,12 +866,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2021-01-29T13:41:50.059339Z",
- "start_time": "2021-01-29T13:41:49.932461Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"# Plot training & validation loss values\n",
@@ -954,9 +900,9 @@
"\n",
"To answer this question, in this tutorial we will compute **attentive response maps**.\n",
"\n",
- "The main idea is to invert the data flow of a convolutional neural network, going from the last layers activations until image space. Then, an heatmap is constructed to shows which parts of the input image are most strongly activating when a classification decision is made - and thus are the most discriminative. \n",
+ "The main idea is to invert the data flow of a convolutional neural network, going from the last layers activations until image space. Then, a heatmap is constructed to shows which parts of the input image are most strongly activating when a classification decision is made - and thus are the most discriminative. \n",
"\n",
- "Specifically, in this tutorial we will use **guided back-propagation**, as introduced in J. Springenberg, A. Dosovitskiy, T. Brox, and Riedmiller, *Striving for Simplicity: The All Convolutional Net*, https://arxiv.org/pdf/1412.6806.pdf (2015), and implemented in the Keras-vis package (https://raghakot.github.io/keras-vis/)\n",
+ "Specifically, in this tutorial we will use **guided back-propagation**, as introduced in J. Springenberg, A. Dosovitskiy, T. Brox, and Riedmiller, *Striving for Simplicity: The All Convolutional Net*, https://arxiv.org/pdf/1412.6806.pdf (2015), and implemented in the Keras-vis package (https://raghakot.github.io/keras-vis/, https://github.com/raghakot/keras-vis)\n",
"\n",
"This is not the only technique to explain the classification decisions made by convolutional neural networks; some useful references are listed below:\n",
"\n",
@@ -980,12 +926,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2021-01-29T13:41:50.223058Z",
- "start_time": "2021-01-29T13:41:50.061052Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"class_idx = 0\n",
@@ -1003,12 +944,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2021-01-29T13:41:52.936788Z",
- "start_time": "2021-01-29T13:41:50.224702Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"from vis.visualization import visualize_saliency\n",
@@ -1027,12 +963,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2021-01-29T13:41:56.598164Z",
- "start_time": "2021-01-29T13:41:52.938422Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"for modifier in ['guided']:\n",
@@ -1047,10 +978,6 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "ExecuteTime": {
- "end_time": "2021-01-29T13:42:04.451854Z",
- "start_time": "2021-01-29T13:41:56.600538Z"
- },
"scrolled": false
},
"outputs": [],
@@ -1078,7 +1005,7 @@
"source": [
"#### Questions\n",
"\n",
- "1. Have a look at the attentive response maps. Do they look reasonable? How to you evaluate their quality?\n",
+ "1. Have a look at the attentive response maps. Do they look reasonable? How do you evaluate their quality?\n",
"2. Try to train another neural network - even a bad one - and calculate its attentive response maps. Are they similar to the one of this network? \n",
"3. Are there materials-science-oriented applications of this technique that can be useful in your research?"
]
@@ -1095,6 +1022,13 @@
"4. Code from MNIST example: https://github.com/keras-team/keras/blob/master/examples/mnist_cnn.py\n",
"5. Keras-vis MNIST example: https://github.com/raghakot/keras-vis/blob/master/examples/mnist/attention.ipynb"
]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
}
],
"metadata": {
@@ -1113,7 +1047,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.7.3"
+ "version": "3.8.5"
},
"toc": {
"base_numbering": 1,
--
GitLab