From fb72a9850404581ba18413033d17105a5ade1ac9 Mon Sep 17 00:00:00 2001
From: Adam Fekete <adam@fekete.co.uk>
Date: Mon, 22 May 2023 09:29:24 +0200
Subject: [PATCH] tutorials
---
.../Application_of_pretrained_model.ipynb | 468 +++++++++++
notebook/Unsupervised_analysis_example.ipynb | 790 ++++++++++++++++++
2 files changed, 1258 insertions(+)
create mode 100644 notebook/Application_of_pretrained_model.ipynb
create mode 100644 notebook/Unsupervised_analysis_example.ipynb
diff --git a/notebook/Application_of_pretrained_model.ipynb b/notebook/Application_of_pretrained_model.ipynb
new file mode 100644
index 0000000..9f92f63
--- /dev/null
+++ b/notebook/Application_of_pretrained_model.ipynb
@@ -0,0 +1,468 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "428595f4",
+ "metadata": {},
+ "source": [
+ "This notebook provides a step-by-step guide on how to use AI-STEM for analyzing experimental images. \n",
+ "\n",
+ "The accompanying paper can be found [here](https://doi.org/10.48550/arXiv.2303.12702)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "85483591",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "! pip install 'git+https://github.com/AndreasLeitherer/ai4stem.git'\n",
+ "! pip install tensorflow\n",
+ "! pip install opencv-python"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "7809904c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Import packages\n",
+ "import os\n",
+ "# tensorflow info/warnings switched off\n",
+ "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'\n",
+ "import numpy as np\n",
+ "import cv2\n",
+ "from collections import defaultdict\n",
+ "from copy import deepcopy\n",
+ "\n",
+ "import matplotlib\n",
+ "matplotlib.rcParams.update({'font.size': 10})\n",
+ "import matplotlib.pyplot as plt\n",
+ "import matplotlib.patches as mpatches\n",
+ "\n",
+ "from ai4stem.utils.utils_data import load_pretrained_model, load_example_image, load_class_dicts\n",
+ "from ai4stem.utils.utils_prediction import predict\n",
+ "from ai4stem.utils.utils_nn import predict_with_uncertainty\n",
+ "from ai4stem.utils.utils_fft import calc_fft\n",
+ "from ai4stem.utils.utils_prediction import localwindow\n",
+ "\n",
+ "numerical_to_text_labels, text_to_numerical_labels = load_class_dicts()\n",
+ "\n",
+ "# Set logger\n",
+ "import logging\n",
+ "logger = logging.getLogger()\n",
+ "logger.setLevel(logging.INFO)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "46eea44c",
+ "metadata": {},
+ "source": [
+ "# Quick start\n",
+ "\n",
+ "After specifying an input image (here, Fe bcc [100] alongside the pixel/Angstrom relation), the following code can be used to analyze it via AI-STEM, employing a pretrained model (which is also used in the [the AI-STEM paper](https://doi.org/10.48550/arXiv.2303.12702))."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "dff60803",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Load image\n",
+ "input_image = load_example_image()\n",
+ "pixel_to_angstrom = 0.1245 # 1 pixel = 0.1245 Angstrom\n",
+ "# Load pretrained model\n",
+ "model = load_pretrained_model()\n",
+ "# Define window size\n",
+ "window_size = int(12. / pixel_to_angstrom)\n",
+ "\n",
+ "# Analyze image: \n",
+ "# return slices of images (fragmentation step, 2D array 'sliced_images'),\n",
+ "# calcualted FFT-HAADF descriptors (2D arrray 'fft_descriptors')\n",
+ "# neural-network classification propabilities (2D array 'prediction'),\n",
+ "# and uncertainty quantification (dictionary 'uncertainty', containing mutual information)\n",
+ "sliced_images, fft_descriptors, prediction, uncertainty = predict(input_image, model,\n",
+ " window_size=window_size)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "4e41e625",
+ "metadata": {},
+ "source": [
+ "\n",
+ "***Note 1***: The neural-network classifier is a *Bayesian* neural network, i.e., its predictions are not deterministic but rather probabilistic - modelling the uncertainty in the predictions, which may arise due to experimental noise and/or insufficiency of the model parameters. In particular, we employ [Monte Carlo dropout](https://proceedings.mlr.press/v48/gal16.html?trk=public_post_comment-text) to obtain an uncertainty estimate. In practice, several forward passes (here T=100) are calculated to estimate the uncertainty in the predictions (referred to as 'Performing forward pass n/100' in the cell below). In particular, this suffices to identify the expected bulk symmetry and detect the interface as regions of high uncertainty (as quantified by mutual information). We refer to [the AI-STEM paper](https://doi.org/10.48550/arXiv.2303.12702) for more details (in particular the section 'The Bayesian classification model' and Supplementary Figure S4 for more details on how to choose the number of forward passes).\n",
+ "\n",
+ "***Note 2***: The model is trained on a specific pixel/angstrom relation. Specifically, the model is trained to classify local windows of size 12 Angstrom, where 1 pixel corresponds to 0.12 angstrom. Thus a window size of 12 Angstrom corresponds to 100 pixels in the simulation settings that we employed for creating the training set. If a different resolution is employed, we recommend to adapt the window size (and this is also done in the above code): given the pixel-to-Angstrom relation, calculate how much pixels correspond to 12 Angstrom and use this as window size. Alternatively, you may rescale the whole image (up/downsampling, e.g., via cv2.resize) such that the resolutions of your input image match the training resolution and then simply use a 100 pixels window size. "
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "8dd3a391",
+ "metadata": {},
+ "source": [
+ "Now we can visualize the results:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b17acafd",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "fig, axs = plt.subplots(1, 3, figsize=(15, 15))\n",
+ "# Input image\n",
+ "im1 = axs[0].imshow(input_image, cmap='gray')\n",
+ "fig.colorbar(im1, ax=axs[0], orientation='vertical', fraction=0.05)\n",
+ "axs[0].set_title('Input image')\n",
+ "# Assignments: \n",
+ "# Get assigned label which corresponds to\n",
+ "# most likely class\n",
+ "assignments = prediction.argmax(axis=-1)\n",
+ "im2 = axs[1].imshow(assignments, cmap='tab10')\n",
+ "axs[1].set_title('Assigned label')\n",
+ "# Mutual informatoin\n",
+ "im3 = axs[2].imshow(uncertainty, cmap='hot', vmin=0.0)\n",
+ "fig.colorbar(im3, ax=axs[2], orientation='vertical', fraction=0.05)\n",
+ "axs[2].set_title('Bayesian uncertainty \\n (mutual information)')\n",
+ "\n",
+ "# add nice legend for assignments\n",
+ "all_colors = plt.cm.tab10.colors\n",
+ "unique_assignments = np.unique(assignments.flatten())\n",
+ "my_colors = [all_colors[idx] for idx in unique_assignments]\n",
+ "patches = [mpatches.Patch(facecolor=c, edgecolor=c) for c in my_colors]\n",
+ "axs[1].legend(patches, sorted(text_to_numerical_labels.keys()), handlelength=0.8, loc='lower right')\n",
+ "\n",
+ "plt.show()\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "1cd43799",
+ "metadata": {},
+ "source": [
+ "The above calculations provide a quickstart, hiding the individual steps performed in the function 'predict'in from ai4stem.utils.utils_prediction. More detailed explanations are provided in the following."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "f6acc9fa",
+ "metadata": {},
+ "source": [
+ "# Step-by-step explanations"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "6b6793cb",
+ "metadata": {},
+ "source": [
+ "First we load the image:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "6c93d963",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "input_image = load_example_image()\n",
+ "image_name = 'Fe_bcc'\n",
+ "plt.imshow(input_image, cmap='gray')\n",
+ "plt.show()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "826aebaa",
+ "metadata": {},
+ "source": [
+ "Next, we need the pixel/Anstrom relation. Moreover, the window size and stride employed in the AI-STEM algorithm has to be specified - here the stride is set to a rather coarse value, while more detailed resolution of structural transitions across defects (such as interfaces) may be achieved by decreasing the stride:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "e33deed8",
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2023-02-15T09:12:03.793233Z",
+ "start_time": "2023-02-15T09:12:03.775703Z"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "pixel_to_angstrom = 0.1245\n",
+ "window_size = 12.\n",
+ "stride_size = [36, 36]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "f114b1f2",
+ "metadata": {},
+ "source": [
+ "As remarked in the quickstart section, the model is trained on a specific pixel/angstrom relation and window size, so we need to take this into account - where here we do not rescale the image but adapt the window size such that a 12 Angstrom size is obtained:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "4c0bfdb5",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "adapted_window_size = int(window_size * (1. / pixel_to_angstrom))\n",
+ "print(adapted_window_size)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "cd79b07f",
+ "metadata": {},
+ "source": [
+ "Now we can proceed to the first step in the AI-STEM workflow, the fragmentation. "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "3650b396",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "sliced_images, _, ni, nj = localwindow(input_image, stride_size=stride_size, \n",
+ " pixel_max=adapted_window_size)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "8bd14c20",
+ "metadata": {},
+ "source": [
+ "We may visualize some of the fragments:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "dffe0fad",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "selection_size = 10\n",
+ "fig, axs = plt.subplots(1, 10, figsize=(25, 10))\n",
+ "for idx, local_image in enumerate(sliced_images[:selection_size]):\n",
+ " axs[idx].imshow(local_image, cmap='gray')"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "55aeef16",
+ "metadata": {},
+ "source": [
+ "Next, we calculate the FFT-HAADF descriptor for each of the local images, where one particular setting, the application of a threshold to the calculated FFT is discussed:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "2122b419",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# FFT HAADF descriptor settings\n",
+ "# Threshold parameter; given FFT spectrum normalized\n",
+ "# to [0, 1], cut off at 0.1 to reduce low-frequency \n",
+ "# contributions; default is is to use this setting.\n",
+ "thresholding = True # very important\n",
+ "\n",
+ "fft_descriptors = []\n",
+ "for im in sliced_images:\n",
+ " fft_desc = calc_fft(im, thresholding=thresholding)\n",
+ " fft_descriptors.append(fft_desc)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "dc9b3c2d",
+ "metadata": {},
+ "source": [
+ "We may again visualize local fragments and their corresponding 64x64 FFT-HAADF descriptor:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "d668c9e8",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "selection_size = 10\n",
+ "fig, axs = plt.subplots(2, 10, figsize=(25, 5))\n",
+ "for idx, local_image in enumerate(sliced_images[:selection_size]):\n",
+ " axs[0, idx].imshow(local_image, cmap='gray')\n",
+ " axs[1, idx].imshow(fft_descriptors[idx], cmap='gray', vmax=0.5) \n",
+ " # vmax=0.5, to enhance visibility of high-freq. peaks"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "827bb9ff",
+ "metadata": {},
+ "source": [
+ "Next we load the pretrained neural-network model and visualize its architecture:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "3c9f141c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "model = load_pretrained_model()\n",
+ "model_name = 'pretrained_model'\n",
+ "model.summary()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "db70ad50",
+ "metadata": {},
+ "source": [
+ "The above FFT-HAADF descriptors serve as input to the neural-network classifier. We first adapt the \n",
+ "above calculated descriptors such that they fit the model input:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "9dc71ca2",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "input_shape_from_model = model.layers[0].get_input_at(0).get_shape().as_list()[1:]\n",
+ "target_shape = tuple([-1] + input_shape_from_model)\n",
+ "nn_input = np.reshape(fft_descriptors, target_shape)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "8531f30a",
+ "metadata": {},
+ "source": [
+ "Then, we calculate the neural-network predictions and uncertainty:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "8110067f",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "prediction, uncertainty = predict_with_uncertainty(nn_input, model)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "a6a01ae0",
+ "metadata": {},
+ "source": [
+ "In the function 'predict_with_uncertainty', other uncertainty quantifiers are calculated as well, while in this work, we explored the use of mutual information.\n",
+ "\n",
+ "Predictions and uncertainty estimates are not yet in the right shape, they are simply a 1D array and thus we reshape:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "ed50c238",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "prediction = np.reshape(prediction, (ni, nj, prediction.shape[-1]))\n",
+ "mutual_information = np.reshape(uncertainty['mutual_information'], (ni, nj))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "6e5aa756",
+ "metadata": {},
+ "source": [
+ "Finally, we can visualize the results:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "ed0b8aac",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "fig, axs = plt.subplots(1, 3, figsize=(15, 15))\n",
+ "# Input image\n",
+ "im1 = axs[0].imshow(input_image, cmap='gray')\n",
+ "fig.colorbar(im1, ax=axs[0], orientation='vertical', fraction=0.05)\n",
+ "axs[0].set_title('Input image')\n",
+ "# Assignments: \n",
+ "# Get assigned label which corresponds to\n",
+ "# most likely class\n",
+ "assignments = prediction.argmax(axis=-1)\n",
+ "im2 = axs[1].imshow(assignments, cmap='tab10')\n",
+ "axs[1].set_title('Assigned label')\n",
+ "# Mutual information\n",
+ "im3 = axs[2].imshow(mutual_information, cmap='hot', vmin=0.0)\n",
+ "fig.colorbar(im3, ax=axs[2], orientation='vertical', fraction=0.05)\n",
+ "axs[2].set_title('Bayesian uncertainty \\n (mutual information)')\n",
+ "\n",
+ "# add nice legend for assignments\n",
+ "all_colors = plt.cm.tab10.colors\n",
+ "unique_assignments = np.unique(assignments.flatten())\n",
+ "my_colors = [all_colors[idx] for idx in unique_assignments]\n",
+ "patches = [mpatches.Patch(facecolor=c, edgecolor=c) for c in my_colors]\n",
+ "axs[1].legend(patches, sorted(text_to_numerical_labels.keys()), handlelength=0.8, loc='lower right')\n",
+ "\n",
+ "plt.show()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "c01a8bc1",
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.7.13"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/notebook/Unsupervised_analysis_example.ipynb b/notebook/Unsupervised_analysis_example.ipynb
new file mode 100644
index 0000000..ed53298
--- /dev/null
+++ b/notebook/Unsupervised_analysis_example.ipynb
@@ -0,0 +1,790 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "7bd00845",
+ "metadata": {},
+ "source": [
+ "This notebook demonstrates how to use unsupervised learning to inspect internal neural-network representations of AI-STEM that are learned during training. Specifically, the application of UMAP (Uniform Manifold Approximation and Projection) to AI-STEM's representation is demonstrated. The unsupervised analysis is conducted for individual images (with one specific interface and bulk symmetry) as well as multiple images (with several, distinct interface patterns and bulk symmetry).\n",
+ "\n",
+ "The accompanying paper can be found [here](https://doi.org/10.48550/arXiv.2303.12702)."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "bc2ccd7e",
+ "metadata": {},
+ "source": [
+ "# Neural-network representation of a single image"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "bc390fe9",
+ "metadata": {},
+ "source": [
+ "First we import the required packages."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "2364a376",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "! pip install 'git+https://github.com/AndreasLeitherer/ai4stem.git'\n",
+ "! pip install tensorflow\n",
+ "! pip install opencv-python\n",
+ "! pip install umap-learn\n",
+ "! pip install bokeh"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "453e8b8c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "# tensorflow info/warnings switched off\n",
+ "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'\n",
+ "from tensorflow.keras.models import Model\n",
+ "\n",
+ "from ai4stem.utils.utils_data import load_pretrained_model, load_example_image, load_class_dicts\n",
+ "from ai4stem.utils.utils_prediction import predict, localwindow\n",
+ "from ai4stem.utils.utils_fft import calc_fft\n",
+ "from ai4stem.utils.utils_nn import decode_preds, predict_with_uncertainty\n",
+ "from ai4stem.utils.utils_unsupervised import embeddable_image\n",
+ "\n",
+ "import numpy as np\n",
+ "import umap\n",
+ "import pandas as pd\n",
+ "import seaborn as sns\n",
+ "\n",
+ "import matplotlib\n",
+ "matplotlib.rcParams.update({'font.size': 10})\n",
+ "import matplotlib.pyplot as plt\n",
+ "\n",
+ "from bokeh.plotting import figure, show, output_notebook\n",
+ "from bokeh.models import HoverTool, ColumnDataSource, CategoricalColorMapper\n",
+ "from bokeh.palettes import Spectral10\n",
+ "\n",
+ "import logging\n",
+ "logger = logging.getLogger()\n",
+ "logger.setLevel(logging.INFO)\n",
+ "\n",
+ "numerical_to_text_labels, text_to_numerical_labels = load_class_dicts()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "9db4e2f9",
+ "metadata": {},
+ "source": [
+ "Next, we load the example image: Fe bcc in [100] orientation."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "8b07b5a3",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# load image\n",
+ "image = load_example_image()\n",
+ "image_name = 'Fe_bcc'\n",
+ "plt.imshow(image, cmap='gray')\n",
+ "plt.show()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "977527d8",
+ "metadata": {},
+ "source": [
+ "When analyzing an image with AI-STEM, a fragmentation into local images is performed. \n",
+ "The numerical representation of these local images is performed in two steps: first a FFT descriptor is calculated and then a Bayesian convolutional neural network is employed, which is the final, data-driven descriptor for the local windows. \n",
+ "\n",
+ "Fragmentation and FFT- desciptor-calculation steps are performed in the following cell (as done in the quickstart [notebook](https://colab.research.google.com/github/AndreasLeitherer/ai4stem/blob/main/notebooks/Application_of_pretrained_model.ipynb)):"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b2a9f2e2",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# pixel/Angstrom relation\n",
+ "pixel_to_angstrom = 0.1245\n",
+ "# AI-STEM parameters\n",
+ "window_size = 12.\n",
+ "stride_size = [36, 36]\n",
+ "# convert window [Angstrom] to window [pixels]\n",
+ "adapted_window_size = int(window_size * (1. / pixel_to_angstrom))\n",
+ "\n",
+ "logger.info('Fragmentation.')\n",
+ "# calc fft\n",
+ "sliced_images, spm_pos, ni, nj = localwindow(image, \n",
+ " stride_size=stride_size, \n",
+ " pixel_max=adapted_window_size)\n",
+ "\n",
+ "logger.info('Calculate FFT-HAADF descriptor.')\n",
+ "fft_descriptors = []\n",
+ "for im in sliced_images:\n",
+ " fft_desc = calc_fft(im, sigma=None, thresholding=True)\n",
+ " fft_descriptors.append(fft_desc)\n",
+ " \n",
+ "# reshape such that matches model input shape\n",
+ "data = np.array([np.stack([_]) for _ in fft_descriptors])\n",
+ "data = np.moveaxis(data, 1, -1)\n",
+ "logger.info('Finished.')"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "34c37bdb",
+ "metadata": {},
+ "source": [
+ "Now we want to extract the neural-network representations, where we inspect the last layer before classification is performed (this layer's name is 'Dense_1'). We first load the pretrained model and then truncate it:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "0bad0827",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Load pretrained model\n",
+ "model = load_pretrained_model()\n",
+ "\n",
+ "# Define model, where remove last classification layer\n",
+ "inputs = model.input\n",
+ "# select layer before last classification layer\n",
+ "# as new final layer:\n",
+ "outpout_layer_name = 'Dense_1' \n",
+ "outputs = model.get_layer(outpout_layer_name).output\n",
+ "intermediate_layer_model = Model(inputs=inputs,\n",
+ " outputs=outputs)\n",
+ "intermediate_layer_model.summary()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "fefd3eb1",
+ "metadata": {},
+ "source": [
+ "Using this truncated model, we can calculate the hidden representations:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "ab63dcb8",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Compute representations\n",
+ "nn_representations = decode_preds(data, intermediate_layer_model, n_iter=100)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "fca0bf44",
+ "metadata": {},
+ "source": [
+ "Now we can apply the Uniform Manifold Approximation and Projection (UMAP) algorithm to visualize the hidden space. We recommend the excellent [documentation](https://umap-learn.readthedocs.io/en/latest/index.html) for more details on UMAP. We explore some of the most important parameters in UMAP and visualize them in the following cell:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "72aceb64",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Apply UMAP\n",
+ "\n",
+ "# most important parameter:\n",
+ "# number of neighbors employed\n",
+ "# for calculating low-dimensional (here, 2D)\n",
+ "# embeddin;, we try a range of values, where\n",
+ "# we use 1%, 5% and 10% of the data set size as # neighbors\n",
+ "n_neighbors_list = nn_representations.shape[0] * np.array([0.02, 0.05, 0.1]) \n",
+ "n_neighbors_list = n_neighbors_list.astype(int)\n",
+ "\n",
+ "# Choose minimum distance (0<min_dist<1.0) \n",
+ "# which controls the spread of the points\n",
+ "# in the low-dimensional embedding (only for improving visualization)\n",
+ "min_dist_list = [0.1, 0.5, 0.9]\n",
+ "\n",
+ "# choose Euclidean metric\n",
+ "# for measuring distance between data points\n",
+ "metric = 'euclidean'\n",
+ "\n",
+ "# Choose 2 as embedding dimension\n",
+ "n_components = 2\n",
+ "\n",
+ "# plotting parameter\n",
+ "s = 2.5\n",
+ "\n",
+ "data_for_fitting = nn_representations\n",
+ "\n",
+ "results = dict()\n",
+ "\n",
+ "for i, n_neighbors in enumerate(n_neighbors_list):\n",
+ " for j, min_dist in enumerate(min_dist_list):\n",
+ " logger.info('Calculate UMAP embedding for # neighbors = {}, min. distance = {}'.format(n_neighbors, min_dist))\n",
+ " mapper = umap.UMAP(n_neighbors=n_neighbors, \n",
+ " min_dist=min_dist,\n",
+ " metric=metric,\n",
+ " n_components=n_components).fit(data_for_fitting)\n",
+ " embedding = mapper.transform(data_for_fitting)\n",
+ " \n",
+ " results[(n_neighbors, min_dist)] = embedding"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "da2e9c6d",
+ "metadata": {},
+ "source": [
+ "Now let us visualize the results:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "f1088ae1",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "fig, axs = plt.subplots(len(min_dist_list), len(n_neighbors_list), figsize=(15, 15))\n",
+ "for i, n_neighbors in enumerate(n_neighbors_list):\n",
+ " for j, min_dist in enumerate(min_dist_list):\n",
+ " embedding = results[(n_neighbors, min_dist)]\n",
+ " im = axs[i, j].scatter(embedding[:, 0], embedding[:, 1],\n",
+ " s=s)\n",
+ " axs[i, j].set_aspect('equal')\n",
+ " axs[i, j].axis('off')\n",
+ " axs[i, j].set_title('# Neighbors = {},\\n min_dist = {}'.format(n_neighbors, min_dist))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "46658220",
+ "metadata": {},
+ "source": [
+ "We can see that for small # neighbors, no patterns can be observed, while for larger values, two main clusters emerge. The minimium distance controls the spread of the points in both clusters.\n",
+ "\n",
+ "Now we would like to know why these clusters arise. For that, we calculate the AI-STEM predictions (assignments and uncertainty estimates):"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "4b2da4fd",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "prediction, uncertainty = predict_with_uncertainty(data, model, \n",
+ " model_type='classification', \n",
+ " n_iter=100)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "36d14cd8",
+ "metadata": {},
+ "source": [
+ "Let us first check which symmetry is assigned to the main clusters - by choosing the most likely label as the color scale:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "c4535206",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import seaborn as sns\n",
+ "import pandas as pd\n",
+ "color_scale = prediction.argmax(axis=-1)\n",
+ "\n",
+ "fig, axs = plt.subplots(len(min_dist_list), len(n_neighbors_list), figsize=(15, 15))\n",
+ "for i, n_neighbors in enumerate(n_neighbors_list):\n",
+ " for j, min_dist in enumerate(min_dist_list):\n",
+ " embedding = results[(n_neighbors, min_dist)]\n",
+ " df = pd.DataFrame({'e1': embedding[:, 0], 'e2': embedding[:, 1], \n",
+ " 'target': [numerical_to_text_labels[str(_)] for _ in color_scale]})\n",
+ " im = sns.scatterplot(x=\"e1\", y=\"e2\", hue=\"target\", \n",
+ " data=df, ax=axs[i, j], palette='tab10', s=s)\n",
+ " im.set(xticks=[])\n",
+ " im.set(yticks=[])\n",
+ " im.set(xlabel=None)\n",
+ " im.set(ylabel=None)\n",
+ " \n",
+ " axs[i, j].set_aspect('equal')\n",
+ " axs[i, j].legend(loc='lower right')\n",
+ " axs[i, j].set_title('# Neighbors = {},\\n min_dist = {}'.format(n_neighbors, min_dist))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "4469fd3e",
+ "metadata": {},
+ "source": [
+ "We can see that the two clusters are assigned the same label (Bcc Fe [100]) while there seems to be more substructure with smaller, separate or sub-clusters with different assignments. These correspond to the interface regions which can be made more visible using the mutual information as color scale:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "aafaec25",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import seaborn as sns\n",
+ "import pandas as pd\n",
+ "color_scale = uncertainty['mutual_information']\n",
+ "\n",
+ "fig, axs = plt.subplots(len(min_dist_list), len(n_neighbors_list), figsize=(15, 15))\n",
+ "for i, n_neighbors in enumerate(n_neighbors_list):\n",
+ " for j, min_dist in enumerate(min_dist_list):\n",
+ " embedding = results[(n_neighbors, min_dist)]\n",
+ " im = axs[i, j].scatter(embedding[:, 0], embedding[:, 1],\n",
+ " s=s, c=color_scale, cmap='hot')\n",
+ "\n",
+ " axs[i, j].set_xticks([])\n",
+ " axs[i, j].set_yticks([])\n",
+ " \n",
+ " axs[i, j].set_aspect('equal')\n",
+ " axs[i, j].set_title('# Neighbors = {},\\n min_dist = {}'.format(n_neighbors, min_dist))\n",
+ " fig.colorbar(im, ax=axs[i, j])"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "847dba58",
+ "metadata": {},
+ "source": [
+ "One can also see that interface regions (points with high uncertainty) are either pushed into separate clusters or located at the borders of the two main clusters. As we will see below (section 'Neural-network representation of multiple interfaces'), if we decrease the stride, one can establish a connection between the two main clusters (essentially, we increase the sampling of the bulk-interface region and this way enable UMAP to capture the manifold structure better and represent the transition between bulk and interface region by a string of connected points)."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "2f5f6239",
+ "metadata": {},
+ "source": [
+ "The above detection of bulk and interface is fully automatic, which helps at analyzing and explaining the UMAP plots. Otherweise one would have to inspect the real-space local fragments by hand. For such analysis, the following cell provides an interactive visualization tool (hover over the images to see the real-space images that correspond to the points in the scatter plot; labels correspond to the neural-network assignments):"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "99f086d1",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "output_notebook()\n",
+ "\n",
+ "data_dict = {}\n",
+ "data_dict['target'] = prediction.argmax(axis=-1)\n",
+ "data_dict['target_names'] = [numerical_to_text_labels[str(_)] for _ in prediction.argmax(axis=-1)]\n",
+ "data_dict['images'] = sliced_images\n",
+ "\n",
+ "#####################################\n",
+ "data_dict_df = pd.DataFrame(embedding, columns=('x', 'y'))\n",
+ "data_dict_df['digit'] = [str(x) for x in data_dict['target_names']]\n",
+ "data_dict_df['image'] = list(map(embeddable_image, data_dict['images']))\n",
+ "\n",
+ "datasource = ColumnDataSource(data_dict_df)\n",
+ "color_mapping = CategoricalColorMapper(factors=[x for x in np.unique(data_dict['target_names'])],\n",
+ " palette=Spectral10)\n",
+ "\n",
+ "plot_figure = figure(\n",
+ " title='UMAP projection of neural-network representations of local image patches\\n (Fe bcc [100] HAADF STEM image)',\n",
+ " plot_width=600,\n",
+ " plot_height=600,\n",
+ " tools=('pan, wheel_zoom, reset')\n",
+ ")\n",
+ "\n",
+ "plot_figure.add_tools(HoverTool(tooltips=\"\"\"\n",
+ "<div>\n",
+ " <div>\n",
+ " <img src='@image' style='float: left; margin: 5px 5px 5px 5px'/>\n",
+ " </div>\n",
+ " <div>\n",
+ " <span style='font-size: 16px; color: #224499'>Label:</span>\n",
+ " <span style='font-size: 18px'>@digit</span>\n",
+ " </div>\n",
+ "</div>\n",
+ "\"\"\"))\n",
+ "\n",
+ "plot_figure.circle(\n",
+ " 'x',\n",
+ " 'y',\n",
+ " source=datasource,\n",
+ " color=dict(field='digit', transform=color_mapping),\n",
+ " line_alpha=0.6,\n",
+ " fill_alpha=0.6,\n",
+ " size=10\n",
+ ")\n",
+ "show(plot_figure)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "edabf6f5",
+ "metadata": {},
+ "source": [
+ "To conclude, we have visualized how the network separaters bulk and interface regions."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "b02a9798",
+ "metadata": {},
+ "source": [
+ "# Neural-network representation of multiple interfaces"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "f8dcb2b4",
+ "metadata": {},
+ "source": [
+ "The question is now: what happens if we consider different interfaces? Will they be assigned the same cluster? Does our choice of training set and optimization routine make it impossible to distinguish different interface types?\n",
+ "\n",
+ "The answer is no - we can distinguish different interfaces, and we will demonstrate that in the following - by considering three experimental images:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "e47b69c3",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "download_link_fcc = 'https://www.dropbox.com/s/flfy5qe1qxv47t6/Cu_fcc_111.npy?dl=0'\n",
+ "download_link_bcc = 'https://www.dropbox.com/s/ukab367rktmddse/Fe_bcc_100.npy?dl=0'\n",
+ "download_link_hcp = 'https://www.dropbox.com/s/q4rvqcy87u3ath9/Ti_hcp_0001.npy?dl=0'\n",
+ "\n",
+ "!wget -q $download_link_fcc -O 'Cu_fcc_100.npy'\n",
+ "!wget -q $download_link_bcc -O 'Fe_bcc_100.npy'\n",
+ "!wget -q $download_link_hcp -O 'Ti_hcp_0001.npy'\n",
+ "\n",
+ "images = [np.load('Cu_fcc_100.npy'),\n",
+ " np.load('Fe_bcc_100.npy'),\n",
+ " np.load('Ti_hcp_0001.npy')]\n",
+ "\n",
+ "image_names = ['Cu_fcc_100',\n",
+ " 'Fe_bcc_100',\n",
+ " 'Ti_hcp_0001']\n",
+ "\n",
+ "pixel_to_angstrom= [0.08805239,\n",
+ " 0.12452489,\n",
+ " 0.12452489]\n",
+ "\n",
+ "adapted_window_sizes = [int(window_size * (1. / ratio)) for ratio in pixel_to_angstrom]\n",
+ "\n",
+ "\n",
+ "fig, axs = plt.subplots(1, 3, figsize=(20,20))\n",
+ "axs[0].imshow(images[0], cmap='gray')\n",
+ "axs[0].set_title('Cu fcc [100]')\n",
+ "axs[1].imshow(images[1], cmap='gray')\n",
+ "axs[1].set_title('Fe bcc [100]')\n",
+ "axs[2].imshow(images[2], cmap='gray')\n",
+ "axs[2].set_title('Ti hcp [0001]')\n",
+ "plt.show()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "30eeb59d",
+ "metadata": {},
+ "source": [
+ "Next, we load precalculated neural-network representations:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "4f1f03d2",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "url_nn_rep_fcc = 'https://www.dropbox.com/s/e4ny6a1ird1v7u8/nn_representations_Cu_fcc_100.npy?dl=0'\n",
+ "url_nn_rep_bcc = 'https://www.dropbox.com/s/wbpjgiwyd0iozgm/nn_representations_Fe_bcc_100.npy?dl=0'\n",
+ "url_nn_rep_hcp = 'https://www.dropbox.com/s/l59chdveknm4mq5/nn_representations_Ti_hcp_0001.npy?dl=0'\n",
+ " \n",
+ "!wget -q $url_nn_rep_fcc -O 'nn_rep_Cu_fcc_100.npy'\n",
+ "!wget -q $url_nn_rep_bcc -O 'nn_rep_Fe_bcc_100.npy'\n",
+ "!wget -q $url_nn_rep_hcp -O 'nn_rep_Ti_hcp_0001.npy'"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "a8bbd598",
+ "metadata": {},
+ "source": [
+ "To calculate these representations, we employed a 12x12 pixels stride for Cu and Ti, while reducing the stride to 6x6 pixels for Fe. A window size of 12 Angstrom is selected (corresponding to 96 pixels for Fe, Ti and 136 for Cu).\n",
+ "\n",
+ "Next, we concatenate these representations in order to calculate the embedding into 2D via UMAP:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "fac51be8",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "nn_representations = [np.load('nn_rep_{}.npy'.format(_)) for _ in image_names]\n",
+ "nn_representations_combined = np.concatenate(nn_representations, axis=0)\n",
+ "print(nn_representations_combined.shape)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "8b7b1cad",
+ "metadata": {},
+ "source": [
+ "To be able to conduct a similar analysis as done for the Fe bcc [100] image before, we also load precalculated assignments and uncertainty estimates:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "d87f7f6c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "url_assignments_fcc = 'https://www.dropbox.com/s/e0hav8kkufat57w/assignments_Cu_fcc_100.npy?dl=0'\n",
+ "url_assignments_bcc = 'https://www.dropbox.com/s/ovtadac1whxkm2g/assignments_Fe_bcc_100.npy?dl=0'\n",
+ "url_assignments_hcp = 'https://www.dropbox.com/s/ksrayk5dopizdqa/assignments_Ti_hcp_0001.npy?dl=0'\n",
+ "\n",
+ "!wget -q $url_assignments_fcc -O 'assignments_Cu_fcc_100.npy'\n",
+ "!wget -q $url_assignments_bcc -O 'assignments_Fe_bcc_100.npy'\n",
+ "!wget -q $url_assignments_hcp -O 'assignments_Ti_hcp_0001.npy'\n",
+ "\n",
+ "assignments = [np.load('assignments_{}.npy'.format(_)) for _ in image_names]\n",
+ "\n",
+ "url_uncertainty_fcc = 'https://www.dropbox.com/s/hy1hrr4rq22cqgu/uncertainty_Cu_fcc_100.npy?dl=0'\n",
+ "url_uncertainty_bcc = 'https://www.dropbox.com/s/y9g5r1u0k3h7vvs/uncertainty_Fe_bcc_100.npy?dl=0'\n",
+ "url_uncertainty_hcp = 'https://www.dropbox.com/s/e1mx9rjeyadg4m9/uncertainty_Ti_hcp_0001.npy?dl=0'\n",
+ "\n",
+ "!wget -q $url_uncertainty_fcc -O 'uncertainty_Cu_fcc_100.npy'\n",
+ "!wget -q $url_uncertainty_bcc -O 'uncertainty_Fe_bcc_100.npy'\n",
+ "!wget -q $url_uncertainty_hcp -O 'uncertainty_Ti_hcp_0001.npy'\n",
+ "\n",
+ "uncertainty = [np.load('uncertainty_{}.npy'.format(_)) for _ in image_names]\n",
+ "\n",
+ "assignments_combined = np.concatenate(assignments, axis=0)\n",
+ "uncertainty_combined = np.concatenate(uncertainty, axis=0)\n",
+ "print(assignments_combined.shape, uncertainty_combined.shape)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "22f201ad",
+ "metadata": {},
+ "source": [
+ "Now we calculated the UMAP embedding, where we choose a specific number of neighbors and minimum distance value (other values may be easily tested, see above for the code to test different settings of, for instance, number of neighbors and minimum distance):"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "55b297a5",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "n_neighbors = 500\n",
+ "min_dist = 0.9\n",
+ "metric = 'euclidean'\n",
+ "n_components = 2\n",
+ "s = 1\n",
+ "\n",
+ "data_for_fitting = nn_representations_combined\n",
+ "\n",
+ "mapper = umap.UMAP(n_neighbors=n_neighbors, \n",
+ " min_dist=min_dist,\n",
+ " metric=metric,\n",
+ " n_components=n_components).fit(data_for_fitting)\n",
+ "embedding = mapper.transform(data_for_fitting)\n",
+ "\n",
+ "fig, axs = plt.subplots(figsize=(10, 10))\n",
+ "axs.scatter(embedding[:, 0], embedding[:, 1], s=s)\n",
+ "axs.set_xticks([])\n",
+ "axs.set_yticks([])\n",
+ "\n",
+ "axs.set_aspect('equal')"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "629fd06a",
+ "metadata": {},
+ "source": [
+ "Employing assignments and mutual information as color scales, respectively, supports the above claim of AI-STEM being able to separate not only different bulk symmetries or bulk from interface regions - but also different interface types, despite never being explicitly instructed to do so:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "fc664860",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "fig, axs = plt.subplots(1, 2, figsize=(20, 20), gridspec_kw={'width_ratios': [0.91, 1]})\n",
+ "\n",
+ "df = pd.DataFrame({'e1': embedding[:, 0], 'e2': embedding[:, 1], \n",
+ " 'target': assignments_combined})\n",
+ "im = sns.scatterplot(x=\"e1\", y=\"e2\", hue=\"target\", \n",
+ " data=df, ax=axs[0], palette='tab10', s=s)\n",
+ "im.set(xticks=[])\n",
+ "im.set(yticks=[])\n",
+ "im.set(xlabel=None)\n",
+ "im.set(ylabel=None)\n",
+ "axs[0].set_aspect('equal')\n",
+ "axs[0].legend(loc='lower right')\n",
+ "axs[0].set_title('Color scale: most likely class')\n",
+ "\n",
+ "\n",
+ "im = axs[1].scatter(embedding[:, 0], embedding[:, 1],\n",
+ " s=s, c=uncertainty_combined, cmap='hot')\n",
+ "axs[1].set_xticks([])\n",
+ "axs[1].set_yticks([])\n",
+ "axs[1].set_aspect('equal')\n",
+ "axs[1].set_title('Color scale: Bayesian uncertainty (mutual information)')\n",
+ "fig.colorbar(im, ax=axs[1], fraction=0.05)\n",
+ "plt.show()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "cc8a814d",
+ "metadata": {},
+ "source": [
+ "One may again inspect the real-space local images. For this we first extract the local fragments:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "86aea729",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "pixel_to_angstrom= [0.08805239, 0.12452489, 0.12452489]\n",
+ "strides = [[12, 12], [6, 6], [12, 12]]\n",
+ "adapted_window_sizes = [int(window_size * (1. / ratio)) for ratio in pixel_to_angstrom]\n",
+ "\n",
+ "sliced_images_combined = []\n",
+ "for idx, input_image in enumerate(images):\n",
+ " \n",
+ " stride_size = strides[idx]\n",
+ " adapted_window_size = adapted_window_sizes[idx]\n",
+ " image_name = image_names[idx]\n",
+ " \n",
+ " logger.info('Extract local fragments for image {}.'.format(image_name))\n",
+ " sliced_images, spm_pos, ni, nj = localwindow(input_image, \n",
+ " stride_size=stride_size, \n",
+ " pixel_max=adapted_window_size)\n",
+ " sliced_images_combined.extend(sliced_images)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "da737ace",
+ "metadata": {},
+ "source": [
+ "Now we can create an interactive plot as done in the previous chapter:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "e20ca12b",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "output_notebook()\n",
+ "\n",
+ "# select only every 10th point to enable faster creation of interactive map\n",
+ "subselection = 10\n",
+ "\n",
+ "data_dict = {}\n",
+ "data_dict['target'] = assignments_combined[::subselection]\n",
+ "data_dict['target_names'] = assignments_combined[::subselection]\n",
+ "data_dict['images'] = sliced_images_combined[::subselection]\n",
+ "\n",
+ "#####################################\n",
+ "data_dict_df = pd.DataFrame(embedding[::subselection], columns=('x', 'y'))\n",
+ "data_dict_df['digit'] = [str(x) for x in data_dict['target_names']]\n",
+ "data_dict_df['image'] = list(map(embeddable_image, data_dict['images']))\n",
+ "\n",
+ "datasource = ColumnDataSource(data_dict_df)\n",
+ "color_mapping = CategoricalColorMapper(factors=[x for x in np.unique(data_dict['target_names'])],\n",
+ " palette=Spectral10)\n",
+ "\n",
+ "plot_figure = figure(\n",
+ " title='UMAP projection of neural-network representations of local image patches\\n\\\n",
+ " (Fe bcc [100], Cu fcc [100], and Ti hcp [0001] HAADF STEM images)',\n",
+ " plot_width=600,\n",
+ " plot_height=600,\n",
+ " tools=('pan, wheel_zoom, reset')\n",
+ ")\n",
+ "\n",
+ "plot_figure.add_tools(HoverTool(tooltips=\"\"\"\n",
+ "<div>\n",
+ " <div>\n",
+ " <img src='@image' style='float: left; margin: 5px 5px 5px 5px'/>\n",
+ " </div>\n",
+ " <div>\n",
+ " <span style='font-size: 16px; color: #224499'>Label:</span>\n",
+ " <span style='font-size: 18px'>@digit</span>\n",
+ " </div>\n",
+ "</div>\n",
+ "\"\"\"))\n",
+ "\n",
+ "plot_figure.circle(\n",
+ " 'x',\n",
+ " 'y',\n",
+ " source=datasource,\n",
+ " color=dict(field='digit', transform=color_mapping),\n",
+ " line_alpha=0.6,\n",
+ " fill_alpha=0.6,\n",
+ " size=3\n",
+ ")\n",
+ "show(plot_figure)"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.7.13"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
--
GitLab