Commit 3234508f authored by lucas_miranda's avatar lucas_miranda
Browse files

Reformatted files using last version of Black; fixed seaborn update issues

parent 9db84390
Pipeline #88235 passed with stage
in 24 minutes and 22 seconds
...@@ -138,7 +138,7 @@ class project: ...@@ -138,7 +138,7 @@ class project:
@property @property
def angles(self): def angles(self):
"""Bool. Toggles angle computation. True by default. If turned off, """Bool. Toggles angle computation. True by default. If turned off,
enhances performance for big datasets""" enhances performance for big datasets"""
return self._angles return self._angles
@property @property
...@@ -274,7 +274,7 @@ class project: ...@@ -274,7 +274,7 @@ class project:
def get_distances(self, tab_dict: dict, verbose: bool = False) -> dict: def get_distances(self, tab_dict: dict, verbose: bool = False) -> dict:
"""Computes the distances between all selected body parts over time. """Computes the distances between all selected body parts over time.
If ego is provided, it only returns distances to a specified bodypart""" If ego is provided, it only returns distances to a specified bodypart"""
if verbose: if verbose:
print("Computing distances...") print("Computing distances...")
...@@ -290,7 +290,11 @@ class project: ...@@ -290,7 +290,11 @@ class project:
scales = self.scales[:, 2:] scales = self.scales[:, 2:]
distance_dict = { distance_dict = {
key: deepof.utils.bpart_distance(tab, scales[i, 1], scales[i, 0],) key: deepof.utils.bpart_distance(
tab,
scales[i, 1],
scales[i, 0],
)
for i, (key, tab) in enumerate(tab_dict.items()) for i, (key, tab) in enumerate(tab_dict.items())
} }
...@@ -782,7 +786,7 @@ class table_dict(dict): ...@@ -782,7 +786,7 @@ class table_dict(dict):
def filter_videos(self, keys: list) -> Table_dict: def filter_videos(self, keys: list) -> Table_dict:
"""Returns a subset of the original table_dict object, containing only the specified keys. Useful, for example, """Returns a subset of the original table_dict object, containing only the specified keys. Useful, for example,
for selecting data coming from videos of a specified condition.""" for selecting data coming from videos of a specified condition."""
assert np.all([k in self.keys() for k in keys]), "Invalid keys selected" assert np.all([k in self.keys() for k in keys]), "Invalid keys selected"
...@@ -825,7 +829,9 @@ class table_dict(dict): ...@@ -825,7 +829,9 @@ class table_dict(dict):
return heatmaps return heatmaps
def get_training_set( def get_training_set(
self, test_videos: int = 0, encode_labels: bool = True, self,
test_videos: int = 0,
encode_labels: bool = True,
) -> Tuple[np.ndarray, list, Union[np.ndarray, list], list]: ) -> Tuple[np.ndarray, list, Union[np.ndarray, list], list]:
"""Generates training and test sets as numpy.array objects for model training""" """Generates training and test sets as numpy.array objects for model training"""
......
...@@ -30,18 +30,40 @@ class SEQ_2_SEQ_AE(HyperModel): ...@@ -30,18 +30,40 @@ class SEQ_2_SEQ_AE(HyperModel):
"""Retrieve hyperparameters to tune""" """Retrieve hyperparameters to tune"""
conv_filters = hp.Int( conv_filters = hp.Int(
"units_conv", min_value=32, max_value=256, step=32, default=256, "units_conv",
min_value=32,
max_value=256,
step=32,
default=256,
) )
lstm_units_1 = hp.Int( lstm_units_1 = hp.Int(
"units_lstm", min_value=128, max_value=512, step=32, default=256, "units_lstm",
min_value=128,
max_value=512,
step=32,
default=256,
) )
dense_2 = hp.Int( dense_2 = hp.Int(
"units_dense2", min_value=32, max_value=256, step=32, default=64, "units_dense2",
min_value=32,
max_value=256,
step=32,
default=64,
) )
dropout_rate = hp.Float( dropout_rate = hp.Float(
"dropout_rate", min_value=0.0, max_value=0.5, default=0.25, step=0.05, "dropout_rate",
min_value=0.0,
max_value=0.5,
default=0.25,
step=0.05,
)
encoding = hp.Int(
"encoding",
min_value=16,
max_value=64,
step=8,
default=24,
) )
encoding = hp.Int("encoding", min_value=16, max_value=64, step=8, default=24,)
return conv_filters, lstm_units_1, dense_2, dropout_rate, encoding return conv_filters, lstm_units_1, dense_2, dropout_rate, encoding
......
...@@ -141,16 +141,16 @@ def compute_kernel(x: tf.Tensor, y: tf.Tensor) -> tf.Tensor: ...@@ -141,16 +141,16 @@ def compute_kernel(x: tf.Tensor, y: tf.Tensor) -> tf.Tensor:
def compute_mmd(tensors: Tuple[Any, Any]) -> tf.Tensor: def compute_mmd(tensors: Tuple[Any, Any]) -> tf.Tensor:
""" """
Computes the MMD between the two specified vectors using a gaussian kernel. Computes the MMD between the two specified vectors using a gaussian kernel.
Parameters: Parameters:
- tensors (tuple): tuple containing two tf.Tensor objects - tensors (tuple): tuple containing two tf.Tensor objects
Returns Returns
- mmd (tf.Tensor): returns the maximum mean discrepancy for each - mmd (tf.Tensor): returns the maximum mean discrepancy for each
training instance training instance
""" """
x = tensors[0] x = tensors[0]
y = tensors[1] y = tensors[1]
...@@ -339,8 +339,8 @@ class DenseTranspose(Layer): ...@@ -339,8 +339,8 @@ class DenseTranspose(Layer):
class KLDivergenceLayer(tfpl.KLDivergenceAddLoss): class KLDivergenceLayer(tfpl.KLDivergenceAddLoss):
""" """
Identity transform layer that adds KL Divergence Identity transform layer that adds KL Divergence
to the final model loss. to the final model loss.
""" """
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
...@@ -360,7 +360,9 @@ class KLDivergenceLayer(tfpl.KLDivergenceAddLoss): ...@@ -360,7 +360,9 @@ class KLDivergenceLayer(tfpl.KLDivergenceAddLoss):
kl_batch = self._regularizer(distribution_a) kl_batch = self._regularizer(distribution_a)
self.add_loss(kl_batch, inputs=[distribution_a]) self.add_loss(kl_batch, inputs=[distribution_a])
self.add_metric( self.add_metric(
kl_batch, aggregation="mean", name="kl_divergence", kl_batch,
aggregation="mean",
name="kl_divergence",
) )
# noinspection PyProtectedMember # noinspection PyProtectedMember
self.add_metric(self._regularizer._weight, aggregation="mean", name="kl_rate") self.add_metric(self._regularizer._weight, aggregation="mean", name="kl_rate")
......
...@@ -33,7 +33,9 @@ class SEQ_2_SEQ_AE: ...@@ -33,7 +33,9 @@ class SEQ_2_SEQ_AE:
""" Simple sequence to sequence autoencoder implemented with tf.keras """ """ Simple sequence to sequence autoencoder implemented with tf.keras """
def __init__( def __init__(
self, architecture_hparams: Dict = {}, huber_delta: float = 1.0, self,
architecture_hparams: Dict = {},
huber_delta: float = 1.0,
): ):
self.hparams = self.get_hparams(architecture_hparams) self.hparams = self.get_hparams(architecture_hparams)
self.CONV_filters = self.hparams["units_conv"] self.CONV_filters = self.hparams["units_conv"]
...@@ -118,13 +120,19 @@ class SEQ_2_SEQ_AE: ...@@ -118,13 +120,19 @@ class SEQ_2_SEQ_AE:
# Decoder layers # Decoder layers
Model_D0 = deepof.model_utils.DenseTranspose( Model_D0 = deepof.model_utils.DenseTranspose(
Model_E5, activation="elu", output_dim=self.ENCODING, Model_E5,
activation="elu",
output_dim=self.ENCODING,
) )
Model_D1 = deepof.model_utils.DenseTranspose( Model_D1 = deepof.model_utils.DenseTranspose(
Model_E4, activation="elu", output_dim=self.DENSE_2, Model_E4,
activation="elu",
output_dim=self.DENSE_2,
) )
Model_D2 = deepof.model_utils.DenseTranspose( Model_D2 = deepof.model_utils.DenseTranspose(
Model_E3, activation="elu", output_dim=self.DENSE_1, Model_E3,
activation="elu",
output_dim=self.DENSE_1,
) )
Model_D3 = RepeatVector(input_shape[1]) Model_D3 = RepeatVector(input_shape[1])
Model_D4 = Bidirectional( Model_D4 = Bidirectional(
...@@ -161,7 +169,10 @@ class SEQ_2_SEQ_AE: ...@@ -161,7 +169,10 @@ class SEQ_2_SEQ_AE:
Model_D5, Model_D5,
) )
def build(self, input_shape: tuple,) -> Tuple[Any, Any, Any]: def build(
self,
input_shape: tuple,
) -> Tuple[Any, Any, Any]:
"""Builds the tf.keras model""" """Builds the tf.keras model"""
( (
...@@ -213,7 +224,10 @@ class SEQ_2_SEQ_AE: ...@@ -213,7 +224,10 @@ class SEQ_2_SEQ_AE:
model.compile( model.compile(
loss=Huber(delta=self.delta), loss=Huber(delta=self.delta),
optimizer=Nadam(lr=self.learn_rate, clipvalue=0.5,), optimizer=Nadam(
lr=self.learn_rate,
clipvalue=0.5,
),
metrics=["mae"], metrics=["mae"],
) )
...@@ -298,7 +312,10 @@ class SEQ_2_SEQ_GMVAE: ...@@ -298,7 +312,10 @@ class SEQ_2_SEQ_GMVAE:
), ),
components=[ components=[
tfd.Independent( tfd.Independent(
tfd.Normal(loc=init_means[k], scale=1,), tfd.Normal(
loc=init_means[k],
scale=1,
),
reinterpreted_batch_ndims=1, reinterpreted_batch_ndims=1,
) )
for k in range(self.number_of_components) for k in range(self.number_of_components)
...@@ -537,7 +554,10 @@ class SEQ_2_SEQ_GMVAE: ...@@ -537,7 +554,10 @@ class SEQ_2_SEQ_GMVAE:
encoder = BatchNormalization()(encoder) encoder = BatchNormalization()(encoder)
# encoding_shuffle = deepof.model_utils.MCDropout(self.DROPOUT_RATE)(encoder) # encoding_shuffle = deepof.model_utils.MCDropout(self.DROPOUT_RATE)(encoder)
z_cat = Dense(self.number_of_components, activation="softmax",)(encoder) z_cat = Dense(
self.number_of_components,
activation="softmax",
)(encoder)
z_cat = deepof.model_utils.Entropy_regulariser(self.entropy_reg_weight)(z_cat) z_cat = deepof.model_utils.Entropy_regulariser(self.entropy_reg_weight)(z_cat)
z_gauss = Dense( z_gauss = Dense(
deepof.model_utils.tfpl.IndependentNormal.params_size( deepof.model_utils.tfpl.IndependentNormal.params_size(
...@@ -553,12 +573,16 @@ class SEQ_2_SEQ_GMVAE: ...@@ -553,12 +573,16 @@ class SEQ_2_SEQ_GMVAE:
if self.overlap_loss: if self.overlap_loss:
z_gauss = deepof.model_utils.Gaussian_mixture_overlap( z_gauss = deepof.model_utils.Gaussian_mixture_overlap(
self.ENCODING, self.number_of_components, loss=self.overlap_loss, self.ENCODING,
self.number_of_components,
loss=self.overlap_loss,
)(z_gauss) )(z_gauss)
z = deepof.model_utils.tfpl.DistributionLambda( z = deepof.model_utils.tfpl.DistributionLambda(
lambda gauss: tfd.mixture.Mixture( lambda gauss: tfd.mixture.Mixture(
cat=tfd.categorical.Categorical(probs=gauss[0],), cat=tfd.categorical.Categorical(
probs=gauss[0],
),
components=[ components=[
tfd.Independent( tfd.Independent(
tfd.Normal( tfd.Normal(
...@@ -663,7 +687,11 @@ class SEQ_2_SEQ_GMVAE: ...@@ -663,7 +687,11 @@ class SEQ_2_SEQ_GMVAE:
grouper = Model(x, z_cat, name="Deep_Gaussian_Mixture_clustering") grouper = Model(x, z_cat, name="Deep_Gaussian_Mixture_clustering")
# noinspection PyUnboundLocalVariable # noinspection PyUnboundLocalVariable
gmvaep = Model(inputs=x, outputs=model_outs, name="SEQ_2_SEQ_GMVAE",) gmvaep = Model(
inputs=x,
outputs=model_outs,
name="SEQ_2_SEQ_GMVAE",
)
# Build generator as a separate entity # Build generator as a separate entity
g = Input(shape=self.ENCODING) g = Input(shape=self.ENCODING)
...@@ -682,7 +710,10 @@ class SEQ_2_SEQ_GMVAE: ...@@ -682,7 +710,10 @@ class SEQ_2_SEQ_GMVAE:
if self.compile: if self.compile:
gmvaep.compile( gmvaep.compile(
loss=model_losses, loss=model_losses,
optimizer=Nadam(lr=self.learn_rate, clipvalue=self.clipvalue,), optimizer=Nadam(
lr=self.learn_rate,
clipvalue=self.clipvalue,
),
metrics=model_metrics, metrics=model_metrics,
loss_weights=loss_weights, loss_weights=loss_weights,
) )
......
...@@ -33,18 +33,18 @@ def close_single_contact( ...@@ -33,18 +33,18 @@ def close_single_contact(
) -> np.array: ) -> np.array:
"""Returns a boolean array that's True if the specified body parts are closer than tol. """Returns a boolean array that's True if the specified body parts are closer than tol.
Parameters: Parameters:
- pos_dframe (pandas.DataFrame): DLC output as pandas.DataFrame; only applicable - pos_dframe (pandas.DataFrame): DLC output as pandas.DataFrame; only applicable
to two-animal experiments. to two-animal experiments.
- left (string): First member of the potential contact - left (string): First member of the potential contact
- right (string): Second member of the potential contact - right (string): Second member of the potential contact
- tol (float): maximum distance for which a contact is reported - tol (float): maximum distance for which a contact is reported
- arena_abs (int): length in mm of the diameter of the real arena - arena_abs (int): length in mm of the diameter of the real arena
- arena_rel (int): length in pixels of the diameter of the arena in the video - arena_rel (int): length in pixels of the diameter of the arena in the video
Returns: Returns:
- contact_array (np.array): True if the distance between the two specified points - contact_array (np.array): True if the distance between the two specified points
is less than tol, False otherwise""" is less than tol, False otherwise"""
close_contact = ( close_contact = (
np.linalg.norm(pos_dframe[left] - pos_dframe[right], axis=1) * arena_abs np.linalg.norm(pos_dframe[left] - pos_dframe[right], axis=1) * arena_abs
...@@ -66,21 +66,21 @@ def close_double_contact( ...@@ -66,21 +66,21 @@ def close_double_contact(
) -> np.array: ) -> np.array:
"""Returns a boolean array that's True if the specified body parts are closer than tol. """Returns a boolean array that's True if the specified body parts are closer than tol.
Parameters: Parameters:
- pos_dframe (pandas.DataFrame): DLC output as pandas.DataFrame; only applicable - pos_dframe (pandas.DataFrame): DLC output as pandas.DataFrame; only applicable
to two-animal experiments. to two-animal experiments.
- left1 (string): First contact point of animal 1 - left1 (string): First contact point of animal 1
- left2 (string): Second contact point of animal 1 - left2 (string): Second contact point of animal 1
- right1 (string): First contact point of animal 2 - right1 (string): First contact point of animal 2
- right2 (string): Second contact point of animal 2 - right2 (string): Second contact point of animal 2
- tol (float): maximum distance for which a contact is reported - tol (float): maximum distance for which a contact is reported
- arena_abs (int): length in mm of the diameter of the real arena - arena_abs (int): length in mm of the diameter of the real arena
- arena_rel (int): length in pixels of the diameter of the arena in the video - arena_rel (int): length in pixels of the diameter of the arena in the video
- rev (bool): reverses the default behaviour (nose2tail contact for both mice) - rev (bool): reverses the default behaviour (nose2tail contact for both mice)
Returns: Returns:
- double_contact (np.array): True if the distance between the two specified points - double_contact (np.array): True if the distance between the two specified points
is less than tol, False otherwise""" is less than tol, False otherwise"""
if rev: if rev:
double_contact = ( double_contact = (
...@@ -117,19 +117,19 @@ def climb_wall( ...@@ -117,19 +117,19 @@ def climb_wall(
) -> np.array: ) -> np.array:
"""Returns True if the specified mouse is climbing the wall """Returns True if the specified mouse is climbing the wall
Parameters: Parameters:
- arena_type (str): arena type; must be one of ['circular'] - arena_type (str): arena type; must be one of ['circular']
- arena (np.array): contains arena location and shape details - arena (np.array): contains arena location and shape details
- pos_dict (table_dict): position over time for all videos in a project - pos_dict (table_dict): position over time for all videos in a project
- tol (float): minimum tolerance to report a hit - tol (float): minimum tolerance to report a hit
- nose (str): indicates the name of the body part representing the nose of - nose (str): indicates the name of the body part representing the nose of
the selected animal the selected animal
- arena_dims (int): indicates radius of the real arena in mm - arena_dims (int): indicates radius of the real arena in mm
- centered_data (bool): indicates whether the input data is centered - centered_data (bool): indicates whether the input data is centered
Returns: Returns:
- climbing (np.array): boolean array. True if selected animal - climbing (np.array): boolean array. True if selected animal
is climbing the walls of the arena""" is climbing the walls of the arena"""
nose = pos_dict[nose] nose = pos_dict[nose]
...@@ -166,7 +166,7 @@ def huddle( ...@@ -166,7 +166,7 @@ def huddle(
Returns: Returns:
hudd (np.array): True if the animal is huddling, False otherwise hudd (np.array): True if the animal is huddling, False otherwise
""" """
if animal_id != "": if animal_id != "":
animal_id += "_" animal_id += "_"
...@@ -254,7 +254,8 @@ def following_path( ...@@ -254,7 +254,8 @@ def following_path(
) )
follow = np.all( follow = np.all(
np.array([(dist_df.min(axis=1) < tol), right_orient1, right_orient2]), axis=0, np.array([(dist_df.min(axis=1) < tol), right_orient1, right_orient2]),
axis=0,
) )
return follow return follow
...@@ -270,28 +271,27 @@ def single_behaviour_analysis( ...@@ -270,28 +271,27 @@ def single_behaviour_analysis(
ylim: float = None, ylim: float = None,
) -> list: ) -> list:
"""Given the name of the behaviour, a dictionary with the names of the groups to compare, and a dictionary """Given the name of the behaviour, a dictionary with the names of the groups to compare, and a dictionary
with the actual tags, outputs a box plot and a series of significance tests amongst the groups with the actual tags, outputs a box plot and a series of significance tests amongst the groups
Parameters: Parameters:
- behaviour_name (str): name of the behavioural trait to analize - behaviour_name (str): name of the behavioural trait to analize
- treatment_dict (dict): dictionary containing video names as keys and experimental conditions as values - treatment_dict (dict): dictionary containing video names as keys and experimental conditions as values
- behavioural_dict (dict): tagged dictionary containing video names as keys and annotations as values - behavioural_dict (dict): tagged dictionary containing video names as keys and annotations as values
- plot (int): Silent if 0; otherwise, indicates the dpi of the figure to plot - plot (int): Silent if 0; otherwise, indicates the dpi of the figure to plot
- stat_tests (bool): performs FDR corrected Mann-U non-parametric tests among all groups if True - stat_tests (bool): performs FDR corrected Mann-U non-parametric tests among all groups if True
- save (str): Saves the produced figure to the specified file - save (str): Saves the produced figure to the specified file
- ylim (float): y-limit for the boxplot. Ignored if plot == False - ylim (float): y-limit for the boxplot. Ignored if plot == False
Returns: Returns:
- beh_dict (dict): dictionary containing experimental conditions as keys and video names as values - beh_dict (dict): dictionary containing experimental conditions as keys and video names as values
- stat_dict (dict): dictionary containing condition pairs as keys and stat results as values""" - stat_dict (dict): dictionary containing condition pairs as keys and stat results as values"""
beh_dict = {condition: [] for condition in treatment_dict.keys()} beh_dict = {condition: [] for condition in treatment_dict.keys()}
for condition in beh_dict.keys(): for condition in beh_dict.keys():
for ind in treatment_dict[condition]: for ind in treatment_dict[condition]:
beh_dict[condition].append( beh_dict[condition] += np.sum(behavioural_dict[ind][behaviour_name]) / len(
np.sum(behavioural_dict[ind][behaviour_name]) behavioural_dict[ind][behaviour_name]
/ len(behavioural_dict[ind][behaviour_name])
) )
return_list = [beh_dict] return_list = [beh_dict]
...@@ -301,7 +301,10 @@ def single_behaviour_analysis( ...@@ -301,7 +301,10 @@ def single_behaviour_analysis(
fig, ax = plt.subplots(dpi=plot) fig, ax = plt.subplots(dpi=plot)
sns.boxplot( sns.boxplot(
list(beh_dict.keys()), list(beh_dict.values()), orient="vertical", ax=ax x=list(beh_dict.keys()),
y=list(beh_dict.values()),
orient="vertical",
ax=ax,
) )
ax.set_title("{} across groups".format(behaviour_name)) ax.set_title("{} across groups".format(behaviour_name))
...@@ -343,16 +346,16 @@ def max_behaviour( ...@@ -343,16 +346,16 @@ def max_behaviour(
) -> np.array: ) -> np.array:
"""Returns the most frequent behaviour in a window of window_size frames """Returns the most frequent behaviour in a window of window_size frames
Parameters: Parameters:
- behaviour_dframe (pd.DataFrame): boolean matrix containing occurrence - behaviour_dframe (pd.DataFrame): boolean matrix containing occurrence
of tagged behaviours per frame in the video of tagged behaviours per frame in the video
- window_size (int): size of the window to use when computing - window_size (int): size of the window to use when computing
the maximum behaviour per time slot the maximum behaviour per time slot
- stepped (bool): sliding windows don't overlap if True. False by default - stepped (bool): sliding windows don't overlap if True. False by default
Returns: Returns:
- max_array (np.array): string array with the most common behaviour per instance - max_array (np.array): string array with the most common behaviour per instance
of the sliding window""" of the sliding window"""
speeds = [col for col in behaviour_dframe.columns if "speed" in col.lower()] speeds = [col for col in behaviour_dframe.columns if "speed" in col.lower()]
...@@ -369,12 +372,12 @@ def max_behaviour( ...@@ -369,12 +372,12 @@ def max_behaviour(
def get_hparameters(hparams: dict = {}) -> dict: def get_hparameters(hparams: dict = {}) -> dict:
"""Returns the most frequent behaviour in a window of window_size frames """Returns the most frequent behaviour in a window of window_size frames
Parameters: Parameters:
- hparams (dict): dictionary containing hyperparameters to overwrite - hparams (dict): dictionary containing hyperparameters to overwrite
Returns: Returns:
- defaults (dict): dictionary with overwriten parameters. Those not - defaults (dict): dictionary with overwriten parameters. Those not
specified in the input retain their default values""" specified in the input retain their default values"""
defaults = { defaults = {
"speed_pause": 3, "speed_pause": 3,
...@@ -398,14 +401,14 @@ def get_hparameters(hparams: dict = {}) -> dict: ...@@ -398,14 +401,14 @@ def get_hparameters(hparams: dict = {}) -> dict:
def frame_corners(w, h, corners: dict = {}): def frame_corners(w, h, corners: dict = {}):
"""Returns a dictionary with the corner positions of the video frame """Returns a dictionary with the corner positions of the video frame
Parameters: Parameters:
- w (int): width of the frame in pixels - w (int): width of the frame in pixels
- h (int): height of the frame in pixels