diff --git a/deepof/models.py b/deepof/models.py
index 2dfcccb92249281e76dda3bb73e2a915c4f462d2..11f294654115ac4f4128860b5d52d9662fdfec8a 100644
--- a/deepof/models.py
+++ b/deepof/models.py
@@ -658,7 +658,9 @@ class SEQ_2_SEQ_GMVAE:
generator = Model_B3(generator)
generator = Model_D5(generator)
generator = Model_B4(generator)
- generator = Dense(tfpl.IndependentNormal.params_size(input_shape[2:]))(generator)
+ generator = Dense(tfpl.IndependentNormal.params_size(input_shape[2:]))(
+ generator
+ )
x_decoded_mean = tfpl.IndependentNormal(
event_shape=input_shape[2:],
convert_to_tensor_fn=tfp.distributions.Distribution.mean,
@@ -675,15 +677,6 @@ class SEQ_2_SEQ_GMVAE:
model_metrics = {"vae_reconstruction": ["mae", "mse"]}
loss_weights = [1.0]
- # x_decoded_mean = TimeDistributed(Dense(input_shape[2]), name="vae_prediction")(
- # generator
- # )
- #
- # model_outs = [x_decoded_mean]
- # model_losses = [Huber(delta=self.delta, reduction="sum_over_batch_size")]
- # model_metrics = {"vae_reconstruction": ["mae", "mse"]}
- # loss_weights = [1.0]
-
if self.predictor > 0:
# Define and instantiate predictor
predictor = Dense(
diff --git a/deepof/train_utils.py b/deepof/train_utils.py
index dfc960c2f129d883e3c3c2757224c095a5f74df3..a8ac8d9dbdb046f957f373cf54095676681ac151 100644
--- a/deepof/train_utils.py
+++ b/deepof/train_utils.py
@@ -104,8 +104,8 @@ def get_callbacks(
("P" if predictor > 0 and variational else ""),
("_Pheno" if phenotype_class > 0 else ""),
("_loss={}".format(loss) if variational else ""),
- ("_encoding={}".format(logparam["encoding"])),
- ("_k={}".format(logparam["k"])),
+ ("_encoding={}".format(logparam["encoding"]) if logparam is not None else ""),
+ ("_k={}".format(logparam["k"]) if logparam is not None else ""),
(datetime.now().strftime("%Y%m%d-%H%M%S")),
)
diff --git a/examples/main.ipynb b/examples/main.ipynb
index fc4463750c74a2de2bd168d762d92b4aea5b5c61..0579d8bbecf2e4bd9a3d5a199e4720e3e02754f2 100644
--- a/examples/main.ipynb
+++ b/examples/main.ipynb
@@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
- "execution_count": 3,
+ "execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
@@ -12,7 +12,7 @@
},
{
"cell_type": "code",
- "execution_count": 4,
+ "execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
@@ -22,7 +22,7 @@
},
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
@@ -48,7 +48,7 @@
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
@@ -57,7 +57,7 @@
},
{
"cell_type": "code",
- "execution_count": 7,
+ "execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
@@ -91,7 +91,7 @@
},
{
"cell_type": "code",
- "execution_count": 8,
+ "execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
@@ -122,7 +122,7 @@
},
{
"cell_type": "code",
- "execution_count": 9,
+ "execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
@@ -153,7 +153,7 @@
},
{
"cell_type": "code",
- "execution_count": 10,
+ "execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
@@ -178,7 +178,7 @@
},
{
"cell_type": "code",
- "execution_count": 11,
+ "execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
@@ -188,7 +188,7 @@
},
{
"cell_type": "code",
- "execution_count": 12,
+ "execution_count": 10,
"metadata": {},
"outputs": [
{
@@ -215,31 +215,31 @@
},
{
"cell_type": "code",
- "execution_count": 13,
+ "execution_count": 20,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
- "CPU times: user 28.4 s, sys: 5 s, total: 33.4 s\n",
- "Wall time: 10.8 s\n"
+ "CPU times: user 28.1 s, sys: 4.99 s, total: 33.1 s\n",
+ "Wall time: 7.5 s\n"
]
}
],
"source": [
"%%time\n",
- "deepof_main = deepof.data.project(path=os.path.join(\"..\",\"..\",\"Desktop\",\"deepof_single_topview\"),\n",
+ "deepof_main = deepof.data.project(path=os.path.join(\"..\",\"..\",\"Desktop\",\"deepof-data\",\"deepof_single_topview\"),\n",
" smooth_alpha=0.99, \n",
" arena_dims=[380],\n",
- " exclude_bodyparts=[\"Tail_1\", \"Tail_2\", \"Tail_tip\", \"Tail_base\", \"Spine_2\"]\n",
+ " #exclude_bodyparts=[\"Tail_1\", \"Tail_2\", \"Tail_tip\", \"Tail_base\", \"Spine_2\"]\n",
" #exp_conditions=dset2inv\n",
" )"
]
},
{
"cell_type": "code",
- "execution_count": 14,
+ "execution_count": 21,
"metadata": {},
"outputs": [
{
@@ -252,8 +252,8 @@
"Computing angles...\n",
"Done!\n",
"deepof analysis of 167 videos\n",
- "CPU times: user 23.5 s, sys: 1.76 s, total: 25.3 s\n",
- "Wall time: 29.7 s\n"
+ "CPU times: user 46.6 s, sys: 5.09 s, total: 51.7 s\n",
+ "Wall time: 53 s\n"
]
}
],
@@ -265,7 +265,7 @@
},
{
"cell_type": "code",
- "execution_count": 15,
+ "execution_count": 22,
"metadata": {},
"outputs": [],
"source": [
@@ -274,7 +274,7 @@
},
{
"cell_type": "code",
- "execution_count": 16,
+ "execution_count": 23,
"metadata": {},
"outputs": [
{
@@ -298,13 +298,13 @@
},
{
"cell_type": "code",
- "execution_count": 17,
+ "execution_count": 24,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
- "model_id": "e0c0e9768256411980ab36f6714d441a",
+ "model_id": "9a3430d72cb64caba2c1679487fdb94b",
"version_major": 2,
"version_minor": 0
},
@@ -339,7 +339,7 @@
},
{
"cell_type": "code",
- "execution_count": 18,
+ "execution_count": 25,
"metadata": {
"scrolled": true
},
@@ -348,8 +348,8 @@
"name": "stdout",
"output_type": "stream",
"text": [
- "CPU times: user 49.3 s, sys: 510 ms, total: 49.8 s\n",
- "Wall time: 49.5 s\n"
+ "CPU times: user 54.3 s, sys: 475 ms, total: 54.8 s\n",
+ "Wall time: 54.8 s\n"
]
}
],
@@ -369,7 +369,7 @@
},
{
"cell_type": "code",
- "execution_count": 55,
+ "execution_count": 29,
"metadata": {},
"outputs": [
{
@@ -378,20 +378,9 @@
"text": [
"Preprocessing training set...\n",
"Loading pre-trained model...\n",
- "(227686, 11, 16)\n"
- ]
- },
- {
- "ename": "ValueError",
- "evalue": "Layer #12 (named \"kl_divergence_layer\" in the current model) was found to correspond to layer kl_divergence_layer in the save file. However the new layer kl_divergence_layer expects 3 weights, but the saved weights have 1 elements.",
- "output_type": "error",
- "traceback": [
- "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
- "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)",
- "\u001b[0;32m<timed exec>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n",
- "\u001b[0;32m~/opt/anaconda3/envs/Machine_Learning/lib/python3.6/site-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36mload_weights\u001b[0;34m(self, filepath, by_name, skip_mismatch, options)\u001b[0m\n\u001b[1;32m 2209\u001b[0m f, self.layers, skip_mismatch=skip_mismatch)\n\u001b[1;32m 2210\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2211\u001b[0;31m \u001b[0mhdf5_format\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_weights_from_hdf5_group\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2212\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2213\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_updated_config\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
- "\u001b[0;32m~/opt/anaconda3/envs/Machine_Learning/lib/python3.6/site-packages/tensorflow/python/keras/saving/hdf5_format.py\u001b[0m in \u001b[0;36mload_weights_from_hdf5_group\u001b[0;34m(f, layers)\u001b[0m\n\u001b[1;32m 704\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msymbolic_weights\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 705\u001b[0m \u001b[0;34m' weights, but the saved weights have '\u001b[0m \u001b[0;34m+\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 706\u001b[0;31m str(len(weight_values)) + ' elements.')\n\u001b[0m\u001b[1;32m 707\u001b[0m \u001b[0mweight_value_tuples\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msymbolic_weights\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight_values\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 708\u001b[0m \u001b[0mK\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatch_set_value\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mweight_value_tuples\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
- "\u001b[0;31mValueError\u001b[0m: Layer #12 (named \"kl_divergence_layer\" in the current model) was found to correspond to layer kl_divergence_layer in the save file. However the new layer kl_divergence_layer expects 3 weights, but the saved weights have 1 elements."
+ "(227686, 11, 26)\n",
+ "CPU times: user 3.35 s, sys: 656 ms, total: 4.01 s\n",
+ "Wall time: 4.07 s\n"
]
}
],
@@ -411,58 +400,24 @@
")[0]\n",
"\n",
"print(\"Loading pre-trained model...\")\n",
- "encoder, _, grouper, gmvaep, = deepof.models.SEQ_2_SEQ_GMVAE(\n",
+ "encoder, decoder, grouper, gmvaep, = deepof.models.SEQ_2_SEQ_GMVAE(\n",
" loss=\"ELBO\",\n",
" number_of_components=10,\n",
" compile_model=True,\n",
" kl_warmup_epochs=20,\n",
- " montecarlo_kl=333,\n",
- " neuron_control=True,\n",
+ " montecarlo_kl=10,\n",
" encoding=6,\n",
- " mmd_warmup_epochs=0,\n",
+ " mmd_warmup_epochs=20,\n",
" predictor=0,\n",
" phenotype_prediction=0,\n",
").build(deepof_train.shape)[:4]\n",
"\n",
- "gmvaep.load_weights(\"../../Desktop/deepof-data/trained_weights/GMVAE_loss=ELBO_encoding=6_run_1_final_weights.h5\")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 27,
- "metadata": {},
- "outputs": [],
- "source": [
- "samples = 10000\n",
- "\n",
- "all_clusters = grouper.predict(deepof_train[:samples])\n",
- "all_encodings = encoder.predict(deepof_train[:samples])"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 44,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "'Test 42_s12'"
- ]
- },
- "execution_count": 44,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "video_key = np.random.choice(list(deepof_coords.keys()), 1)[0]\n",
- "video_key"
+ "gmvaep.load_weights(\"../../Desktop/GMVAE_loss=ELBO_encoding=6_run_0_final_weights.h5\")"
]
},
{
"cell_type": "code",
- "execution_count": 45,
+ "execution_count": 46,
"metadata": {},
"outputs": [],
"source": [
@@ -471,19 +426,19 @@
" window_size=11,\n",
" window_step=1,\n",
" conv_filter=None,\n",
- " scale=\"minmax\",\n",
+ " scale=\"standard\",\n",
" shuffle=False,\n",
" test_videos=0,\n",
" )[0]\n",
- "scaler = MinMaxScaler()\n",
- "scaler.fit(deepof_coords[video_key])\n",
+ "scaler = StandardScaler()\n",
+ "scaler.fit(np.array(pd.concat(list(deepof_coords.values()))))\n",
"\n",
"# Get reconstruction\n",
"video_pred = gmvaep.predict(video_input)[:, 6, :]\n",
"\n",
"# Get encodings\n",
- "video_clusters = grouper.predict(video_input)\n",
- "video_encodings = encoder.predict(video_input)\n",
+ "# video_clusters = grouper.predict(video_input)\n",
+ "# video_encodings = encoder.predict(video_input)\n",
"\n",
"scaled_video_pred = scaler.inverse_transform(video_pred)\n",
"scaled_video_input = scaler.inverse_transform(video_input[:, 6, :])\n",
@@ -494,16 +449,16 @@
},
{
"cell_type": "code",
- "execution_count": 29,
+ "execution_count": 47,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
- "<tf.Tensor: shape=(), dtype=float64, numpy=0.6589403550173625>"
+ "<tf.Tensor: shape=(), dtype=float64, numpy=0.8289839462658529>"
]
},
- "execution_count": 29,
+ "execution_count": 47,
"metadata": {},
"output_type": "execute_result"
}
@@ -514,18 +469,18 @@
},
{
"cell_type": "code",
- "execution_count": 30,
+ "execution_count": 48,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
- "model_id": "c990199778514e4c9b630d8bb3d473bc",
+ "model_id": "c442eba8214949e19aee3eea4d99a736",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
- "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=500.0), HTML(value='')))"
+ "HBox(children=(HTML(value=''), FloatProgress(value=0.0), HTML(value='')))"
]
},
"metadata": {},
@@ -557,7 +512,7 @@
" True,\n",
")\n",
"\n",
- "for frame in tqdm.tqdm(range(500)):\n",
+ "for frame in tqdm.tqdm(range(100)):\n",
"\n",
" image = np.zeros((h, w, 3), np.uint8) + 30\n",
" for bpart in scaled_video_input.columns.levels[0]:\n",
@@ -593,9 +548,9 @@
" for df, col in zip([scaled_video_input, scaled_video_pred], [(0,0,255),(0,255,0)]):\n",
" draw_line(\"Nose\", [\"Left_ear\", \"Right_ear\"], df, col)\n",
" draw_line(\"Spine_1\", [\"Left_ear\", \"Right_ear\", \"Left_fhip\", \"Right_fhip\"], df, col)\n",
- " #draw_line(\"Spine_2\", [\"Spine_1\", \"Tail_base\", \"Left_bhip\", \"Right_bhip\"], df, col)\n",
- " #draw_line(\"Tail_1\", [\"Tail_base\", \"Tail_2\"], df, col)\n",
- " #draw_line(\"Tail_tip\", [\"Tail_2\"], df, col)\n",
+ " draw_line(\"Spine_2\", [\"Spine_1\", \"Tail_base\", \"Left_bhip\", \"Right_bhip\"], df, col)\n",
+ " draw_line(\"Tail_1\", [\"Tail_base\", \"Tail_2\"], df, col)\n",
+ " draw_line(\"Tail_tip\", [\"Tail_2\"], df, col)\n",
"\n",
" image = cv2.resize(image, (0, 0), fx=factor, fy=factor)\n",
" writer.write(image)\n",
@@ -606,7 +561,37 @@
},
{
"cell_type": "code",
- "execution_count": 24,
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "samples = 100000\n",
+ "\n",
+ "all_clusters = grouper.predict(deepof_train[:samples])\n",
+ "all_encodings = encoder.predict(deepof_train[:samples])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "all_clusters"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "sns.scatterplot(all_encodings[:,0], all_encodings[:,1], hue=np.argmax(all_clusters,axis=1))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@@ -619,7 +604,7 @@
},
{
"cell_type": "code",
- "execution_count": 52,
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@@ -656,22 +641,9 @@
},
{
"cell_type": "code",
- "execution_count": 53,
+ "execution_count": null,
"metadata": {},
- "outputs": [
- {
- "ename": "NameError",
- "evalue": "name 'LinearDiscriminantAnalysis' is not defined",
- "output_type": "error",
- "traceback": [
- "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
- "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)",
- "\u001b[0;32m<ipython-input-53-299d75cbfe33>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mplot_encodings\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mall_encodings\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m10000\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mall_clusters\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m10\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
- "\u001b[0;32m<ipython-input-52-f274dd41a1f9>\u001b[0m in \u001b[0;36mplot_encodings\u001b[0;34m(data, samples, n, clusters, threshold, highlight)\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mplot_encodings\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msamples\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclusters\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mthreshold\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhighlight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mreducer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mLinearDiscriminantAnalysis\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_components\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 4\u001b[0m \u001b[0mclusters\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclusters\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0msamples\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
- "\u001b[0;31mNameError\u001b[0m: name 'LinearDiscriminantAnalysis' is not defined"
- ]
- }
- ],
+ "outputs": [],
"source": [
"plot_encodings(all_encodings, 10000, 2, all_clusters, 1, 10)"
]