Added notebook for dataset and model evaluation

supplementary_notebooks/latent_space_sampling.ipynb → supplementary_notebooks/deepof_data_exploration.ipynb

@@ -6,22 +6,27 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%load_ext autoreload\n",
-    "%autoreload 2"
+    "import os\n",
+    "os.chdir(os.path.dirname(\"../\"))"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "### Latent space sampling"
+    "# deepOF data exploration"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "Given a dataset and a trained model, this notebook allows the user to sample from the latent space distributions and generate video clips showcasing the results"
+    "Given a dataset, this notebook allows the user to \n",
+    "\n",
+    "* Load and process the dataset using deepof.data\n",
+    "* Visualize data quality with interactive plots\n",
+    "* Visualize training instances as multi-timepoint scatter plots with interactive configurations\n",
+    "* Visualize training instances as video clips with interactive configurations"
    ]
   },
   {
@@ -30,9 +35,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import os\n",
-    "\n",
-    "os.chdir(os.path.dirname(\"../\"))"
+    "import warnings\n",
+    "warnings.filterwarnings(\"ignore\")"
    ]
   },
   {
@@ -41,25 +45,17 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import cv2\n",
     "import deepof.data\n",
-    "import deepof.models\n",
-    "import matplotlib.pyplot as plt\n",
-    "from mpl_toolkits.mplot3d import Axes3D\n",
+    "import deepof.utils\n",
     "import numpy as np\n",
     "import pandas as pd\n",
-    "import re\n",
-    "import seaborn as sns\n",
-    "from sklearn.preprocessing import StandardScaler, MinMaxScaler\n",
     "import tensorflow as tf\n",
-    "import tqdm.notebook as tqdm\n",
-    "from datetime import datetime\n",
-    "from sklearn.manifold import TSNE\n",
-    "from sklearn.decomposition import PCA\n",
-    "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n",
-    "import umap\n",
     "\n",
-    "from ipywidgets import interact"
+    "from ipywidgets import interact\n",
+    "from IPython import display\n",
+    "from matplotlib.animation import FuncAnimation\n",
+    "import matplotlib.pyplot as plt\n",
+    "import seaborn as sns"
    ]
   },
   {
@@ -72,68 +68,44 @@
   {
    "cell_type": "code",
    "execution_count": 4,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "path = os.path.join(\"..\", \"..\", \"Desktop\", \"deepoftesttemp\")\n",
-    "trained_network = os.path.join(\"..\", \"..\", \"Desktop\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "metadata": {},
+   "metadata": {
+    "scrolled": true
+   },
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "CPU times: user 394 ms, sys: 34.1 ms, total: 428 ms\n",
-      "Wall time: 351 ms\n"
+      "Loading trajectories...\n",
+      "Smoothing trajectories...\n",
+      "Interpolating outliers...\n",
+      "Iterative imputation of ocluded bodyparts...\n",
+      "Computing distances...\n",
+      "Computing angles...\n",
+      "Done!\n"
      ]
     }
    ],
    "source": [
-    "%%time\n",
+    "# Use deepof to load a project\n",
     "proj = deepof.data.project(\n",
-    "    path=path,\n",
-    "    smooth_alpha=0.99,\n",
-    "    exclude_bodyparts=[\"Tail_1\", \"Tail_2\", \"Tail_tip\", \"Tail_base\"],\n",
+    "    path=\"../../Desktop/deepoftesttemp/\",\n",
     "    arena_dims=[380],\n",
-    ")"
+    "    arena_detection=\"rule-based\",\n",
+    "    interpolate_outliers=True,\n",
+    ").run()"
    ]
   },
   {
-   "cell_type": "code",
-   "execution_count": 6,
+   "cell_type": "markdown",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Loading trajectories...\n",
-      "Smoothing trajectories...\n",
-      "Interpolating outliers...\n",
-      "Iterative imputation of ocluded bodyparts...\n",
-      "Computing distances...\n",
-      "Computing angles...\n",
-      "Done!\n",
-      "deepof analysis of 2 videos\n",
-      "CPU times: user 2.89 s, sys: 196 ms, total: 3.09 s\n",
-      "Wall time: 887 ms\n"
-     ]
-    }
-   ],
    "source": [
-    "%%time\n",
-    "proj = proj.run(verbose=True)\n",
-    "print(proj)"
+    "### 2. Inspect dataset quality"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -142,7 +114,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 6,
    "metadata": {
     "scrolled": true
    },
@@ -168,13 +140,15 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
-   "metadata": {},
+   "execution_count": 7,
+   "metadata": {
+    "scrolled": true
+   },
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "426727c53d7e482685d6d7c16119a9c9",
+       "model_id": "a6c156fc66944efb8482b42eab3a7c38",
        "version_major": 2,
        "version_minor": 0
       },
@@ -206,122 +180,267 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "### 2. Load pretrained deepof model"
+    "In the cell above, you see the percentage of labels per body part which have a quality lower than the selected value (0.50 by default) **before** preprocessing. The values are taken directly from DeepLabCut."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### 3. Get coordinates, distances and angles"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "And get speed, acceleration and jerk for each"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [],
    "source": [
-    "coords = proj.get_coords(center=\"Center\", align=\"Spine_1\", align_inplace=True)\n",
-    "preprocessed_data,_,_,_ = coords.preprocess(test_videos=0, window_step=5, window_size=24)"
+    "# Get coordinates, speeds, accelerations and jerks for positions\n",
+    "position_coords = proj.get_coords(center=\"Center\", align=\"Spine_1\", align_inplace=True)\n",
+    "position_speeds = proj.get_coords(center=\"Center\", speed=1)\n",
+    "position_accels = proj.get_coords(center=\"Center\", speed=2)\n",
+    "position_jerks = proj.get_coords(center=\"Center\", speed=3)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Set model parameters\n",
-    "encoding=6\n",
-    "loss=\"ELBO\"\n",
-    "k=25\n",
-    "pheno=0\n",
-    "predictor=0"
+    "# Get coordinates, speeds, accelerations and jerks for distances\n",
+    "distance_coords = proj.get_distances()\n",
+    "distance_speeds = proj.get_distances(speed=1)\n",
+    "distance_accels = proj.get_distances(speed=2)\n",
+    "distance_jerks = proj.get_distances(speed=3)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 10,
    "metadata": {},
    "outputs": [],
    "source": [
-    "encoder, decoder, grouper, gmvaep = deepof.models.SEQ_2_SEQ_GMVAE(\n",
-    "    loss=loss,\n",
-    "    number_of_components=k,\n",
-    "    compile_model=True,\n",
-    "    encoding=encoding,\n",
-    "    predictor=predictor,\n",
-    "    phenotype_prediction=pheno,\n",
-    ").build(preprocessed_data.shape)[:4]\n",
+    "# Get coordinates, speeds, accelerations and jerks for angles\n",
+    "angle_coords = proj.get_angles()\n",
+    "angle_speeds = proj.get_angles(speed=1)\n",
+    "angle_accels = proj.get_angles(speed=2)\n",
+    "angle_jerks = proj.get_angles(speed=3)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### 4. Display training instances"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "befe8b832e2d40d2b581aea2a570073e",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "interactive(children=(FloatSlider(value=7500.0, description='time_slider', max=15000.0, step=25.0), IntSlider(…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "random_exp = np.random.choice(list(position_coords.keys()), 1)[0]\n",
     "\n",
-    "gmvaep.load_weights(\n",
-    "    os.path.join(\n",
-    "        trained_network, [i for i in os.listdir(trained_network) if i.endswith(\"h5\")][0]\n",
+    "@interact(time_slider=(0.0, 15000, 25), length_slider=(10, 100, 5))\n",
+    "def plot_mice_across_time(time_slider, length_slider):\n",
+    "\n",
+    "    plt.figure(figsize=(10, 10))\n",
+    "\n",
+    "    for bpart in position_coords[random_exp].columns.levels[0]:\n",
+    "        if bpart != \"Center\":\n",
+    "            sns.scatterplot(\n",
+    "                data=position_coords[random_exp].loc[\n",
+    "                    time_slider : time_slider + length_slider - 1, bpart\n",
+    "                ],\n",
+    "                x=\"x\",\n",
+    "                y=\"y\",\n",
+    "                label=bpart,\n",
+    "                palette=sns.color_palette(\"tab10\"),\n",
+    "            )\n",
+    "\n",
+    "    plt.title(\"Positions across time for centered data\")\n",
+    "    plt.legend(\n",
+    "        fontsize=15,\n",
+    "        bbox_to_anchor=(1.5, 1),\n",
+    "        title=\"Body part\",\n",
+    "        title_fontsize=18,\n",
+    "        shadow=False,\n",
+    "        facecolor=\"white\",\n",
     "    )\n",
-    ")"
+    "\n",
+    "    plt.ylim(-100, 60)\n",
+    "    plt.xlim(-60, 60)\n",
+    "    plt.show()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "The figure above is a multi time-point scatter plot. The time_slider allows you to scroll across the video, and the length_slider selects the number of time-points to include. The idea is to intuitively visualize the data that goes into a training instance for a given preprocessing setting."
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
-   "metadata": {
-    "scrolled": true
-   },
+   "execution_count": 12,
+   "metadata": {},
    "outputs": [],
    "source": [
-    "# Uncomment to see model summaries\n",
-    "# encoder.summary()\n",
-    "# decoder.summary()\n",
-    "# grouper.summary()\n",
-    "# gmvaep.summary()"
+    "def plot_mouse_graph(instant_x, instant_y, ax, edges):\n",
+    "    \"\"\"Generates a graph plot of the mouse\"\"\"\n",
+    "    plots = []\n",
+    "    for edge in edges:\n",
+    "        (temp_plot,) = ax.plot(\n",
+    "            [float(instant_x[edge[0]]), float(instant_x[edge[1]])],\n",
+    "            [float(instant_y[edge[0]]), float(instant_y[edge[1]])],\n",
+    "            color=\"#006699\",\n",
+    "        )\n",
+    "        plots.append(temp_plot)\n",
+    "    return plots\n",
+    "\n",
+    "\n",
+    "def update_mouse_graph(x, y, plots, edges):\n",
+    "    \"\"\"Updates the graph plot to enable animation\"\"\"\n",
+    "\n",
+    "    for plot, edge in zip(plots, edges):\n",
+    "        plot.set_data(\n",
+    "            [float(x[edge[0]]), float(x[edge[1]])],\n",
+    "            [float(y[edge[0]]), float(y[edge[1]])],\n",
+    "        )"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": 13,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "f5a8a2dbd0074f33af0f87dfcc279a28",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "interactive(children=(FloatSlider(value=7500.0, description='time_slider', max=15000.0, step=25.0), IntSlider(…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
    "source": [
-    "# Uncomment to plot model structure\n",
-    "def plot_model(model, name):\n",
-    "    tf.keras.utils.plot_model(\n",
-    "        model,\n",
-    "        to_file=os.path.join(\n",
-    "            path,\n",
-    "            \"deepof_{}_{}.png\".format(name, datetime.now().strftime(\"%Y%m%d-%H%M%S\")),\n",
-    "        ),\n",
-    "        show_shapes=True,\n",
-    "        show_dtype=False,\n",
-    "        show_layer_names=True,\n",
-    "        rankdir=\"TB\",\n",
-    "        expand_nested=True,\n",
-    "        dpi=200,\n",
+    "random_exp = np.random.choice(list(position_coords.keys()), 1)[0]\n",
+    "\n",
+    "\n",
+    "@interact(time_slider=(0.0, 15000, 25), length_slider=(10, 100, 5))\n",
+    "def animate_mice_across_time(time_slider, length_slider):\n",
+    "\n",
+    "    fig, ax = plt.subplots(1, 1, figsize=(10, 10))\n",
+    "\n",
+    "    edges = deepof.utils.connect_mouse_topview()\n",
+    "    for limb in [\"Left_fhip\", \"Right_fhip\", \"Left_bhip\", \"Right_bhip\"]:\n",
+    "        edges.remove_edge(\"Center\", limb)\n",
+    "    edges = edges.edges()\n",
+    "\n",
+    "    data = position_coords[random_exp].loc[\n",
+    "        time_slider : time_slider + length_slider - 1, :\n",
+    "    ]\n",
+    "\n",
+    "    data[\"Center\", \"x\"] = 0\n",
+    "    data[\"Center\", \"y\"] = 0\n",
+    "\n",
+    "    init_x = data.xs(\"x\", level=\"coords\", axis=1, drop_level=False).iloc[0, :]\n",
+    "    init_y = data.xs(\"y\", level=\"coords\", axis=1, drop_level=False).iloc[0, :]\n",
+    "\n",
+    "    plots = plot_mouse_graph(init_x, init_y, ax, edges)\n",
+    "    scatter = ax.scatter(x=np.array(init_x), y=np.array(init_y), color=\"#006699\",)\n",
+    "\n",
+    "    # Update data in main plot\n",
+    "    def animation_frame(i):\n",
+    "        # Update scatter plot\n",
+    "        x = data.xs(\"x\", level=\"coords\", axis=1, drop_level=False).iloc[i, :]\n",
+    "        y = data.xs(\"y\", level=\"coords\", axis=1, drop_level=False).iloc[i, :]\n",
+    "\n",
+    "        scatter.set_offsets(np.c_[np.array(x), np.array(y)])\n",
+    "        update_mouse_graph(x, y, plots, edges)\n",
+    "\n",
+    "        return scatter\n",
+    "\n",
+    "    animation = FuncAnimation(\n",
+    "        fig, func=animation_frame, frames=length_slider, interval=100,\n",
     "    )\n",
     "\n",
+    "    ax.set_title(\"Positions across time for centered data\")\n",
+    "    ax.set_ylim(-100, 60)\n",
+    "    ax.set_xlim(-60, 60)\n",
+    "    ax.set_xlabel(\"x\")\n",
+    "    ax.set_ylabel(\"y\")\n",
     "\n",
-    "# plot_model(encoder, \"encoder\")\n",
-    "# plot_model(decoder, \"decoder\")\n",
-    "# plot_model(grouper, \"grouper\")\n",
-    "# plot_model(gmvaep, \"gmvaep\")"
+    "    video = animation.to_html5_video()\n",
+    "    html = display.HTML(video)\n",
+    "    display.display(html)\n",
+    "    plt.close()"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "### 3. Pass data through all models"
+    "The figure above displays exactly the same data as the multi time-point scatter plot, but in the form of a video (one training instance at the time)."
    ]
   },
   {
-   "cell_type": "code",
-   "execution_count": 15,
+   "cell_type": "markdown",
    "metadata": {},
-   "outputs": [],
    "source": [
-    "encodings = encoder.predict(preprocessed_data)\n",
-    "groupings = grouper.predict(preprocessed_data)\n",
-    "reconstrs = gmvaep.predict(preprocessed_data)"
+    "### 5. Visualize speeds"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "### 4. Evaluate reconstruction"
+    "### 6. Visualize acceleration"
    ]
   },
   {
@@ -342,7 +461,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "### 5. Evaluate latent space"
+    "### 7. Visualize jerk"
    ]
   },
   {