Removed outdated non-variational autoencoder model

deepof/data.py

@@ -900,7 +900,6 @@ class coordinates:
         pretrained: str = False,
         save_checkpoints: bool = False,
         save_weights: bool = True,
-        variational: bool = True,
         reg_cat_clusters: bool = False,
         reg_cluster_variance: bool = False,
         entropy_samples: int = 10000,
@@ -938,8 +937,6 @@ class coordinates:
             is appended to the latent space,
             aiming to predict what happens immediately next in the sequence, which can help with regularization.
             - pretrained (bool): If True, a pretrained set of weights is expected.
-            - variational (bool): If True (default) a variational autoencoder is used. If False,
-            a simple autoencoder is used for dimensionality reduction
 
         Returns:
             - return_list (tuple): List containing all relevant trained models for unsupervised prediction.
@@ -968,7 +965,6 @@ class coordinates:
             pretrained=pretrained,
             save_checkpoints=save_checkpoints,
             save_weights=save_weights,
-            variational=variational,
             reg_cat_clusters=reg_cat_clusters,
             reg_cluster_variance=reg_cluster_variance,
             entropy_samples=entropy_samples,

deepof/hypermodels.py

@@ -18,78 +18,7 @@ tfd = tfp.distributions
 tfpl = tfp.layers
 
 
-class SEQ_2_SEQ_AE(HyperModel):
-    """Hyperparameter tuning pipeline for deepof.models.SEQ_2_SEQ_AE"""
-
-    def __init__(self, input_shape):
-        super().__init__()
-        self.input_shape = input_shape
-
-    @staticmethod
-    def get_hparams(hp):
-        """Retrieve hyperparameters to tune"""
-
-        conv_filters = hp.Int(
-            "units_conv",
-            min_value=32,
-            max_value=256,
-            step=32,
-            default=256,
-        )
-        lstm_units_1 = hp.Int(
-            "units_lstm",
-            min_value=128,
-            max_value=512,
-            step=32,
-            default=256,
-        )
-        dense_2 = hp.Int(
-            "units_dense2",
-            min_value=32,
-            max_value=256,
-            step=32,
-            default=64,
-        )
-        dropout_rate = hp.Float(
-            "dropout_rate",
-            min_value=0.0,
-            max_value=0.5,
-            default=0.25,
-            step=0.05,
-        )
-        encoding = hp.Int(
-            "encoding",
-            min_value=16,
-            max_value=64,
-            step=8,
-            default=24,
-        )
-
-        return conv_filters, lstm_units_1, dense_2, dropout_rate, encoding
-
-    def build(self, hp):
-        """Overrides Hypermodel's build method"""
-
-        # HYPERPARAMETERS TO TUNE
-        conv_filters, lstm_units_1, dense_2, dropout_rate, encoding = self.get_hparams(
-            hp
-        )
-
-        # INSTANCIATED MODEL
-        model = deepof.models.SEQ_2_SEQ_AE(
-            architecture_hparams={
-                "units_conv": conv_filters,
-                "units_lstm": lstm_units_1,
-                "units_dense_2": dense_2,
-                "dropout_rate": dropout_rate,
-                "encoding": encoding,
-            }
-        ).build(self.input_shape)[2]
-
-        return model
-
-
-class SEQ_2_SEQ_GMVAE(HyperModel):
+class GMVAE(HyperModel):
     """Hyperparameter tuning pipeline for deepof.models.SEQ_2_SEQ_GMVAE"""
 
     def __init__(
@@ -173,7 +102,7 @@ class SEQ_2_SEQ_GMVAE(HyperModel):
             lstm_units_1,
         ) = self.get_hparams(hp)
 
-        gmvaep = deepof.models.SEQ_2_SEQ_GMVAE(
+        gmvaep = deepof.models.GMVAE(
             architecture_hparams={
                 "bidirectional_merge": "ave",
                 "clipvalue": clipvalue,

deepof/model_utils.py

@@ -215,7 +215,6 @@ class neighbor_latent_entropy(tf.keras.callbacks.Callback):
     def __init__(
         self,
         encoding_dim: int,
-        variational: bool = True,
         validation_data: np.ndarray = None,
         k: int = 100,
         samples: int = 10000,
@@ -223,7 +222,6 @@ class neighbor_latent_entropy(tf.keras.callbacks.Callback):
     ):
         super().__init__()
         self.enc = encoding_dim
-        self.variational = variational
         self.validation_data = validation_data
         self.k = k
         self.samples = samples
@@ -233,7 +231,7 @@ class neighbor_latent_entropy(tf.keras.callbacks.Callback):
     def on_epoch_end(self, epoch, logs=None):
         """ Passes samples through the encoder and computes cluster purity on the latent embedding """
 
-        if self.validation_data is not None and self.variational:
+        if self.validation_data is not None:
 
             # Get encoer and grouper from full model
             latent_distribution = [
@@ -536,10 +534,18 @@ class Cluster_overlap(Layer):
     using the average inter-cluster MMD as a metric
     """
 
-    def __init__(self, lat_dims, n_components, loss=False, samples=10, *args, **kwargs):
-        self.lat_dims = lat_dims
-        self.n_components = n_components
-        self.loss = loss
+    def __init__(
+        self,
+        encoding_dim: int,
+        k: int = 100,
+        loss_weight: float = False,
+        samples: int = 512,
+        *args,
+        **kwargs
+    ):
+        self.enc = encoding_dim
+        self.k = k
+        self.loss_weight = loss_weight
         self.samples = samples
         super(Cluster_overlap, self).__init__(*args, **kwargs)
 
@@ -547,9 +553,9 @@ class Cluster_overlap(Layer):
         """Updates Constraint metadata"""
 
         config = super().get_config().copy()
-        config.update({"lat_dims": self.lat_dims})
-        config.update({"n_components": self.n_components})
-        config.update({"loss": self.loss})
+        config.update({"enc": self.enc})
+        config.update({"k": self.k})
+        config.update({"loss_weight": self.loss_weight})
         config.update({"samples": self.samples})
         return config
 

deepof/models.py

@@ -28,217 +28,8 @@ tfb = tfp.bijectors
 tfd = tfp.distributions
 tfpl = tfp.layers
 
-
-# noinspection PyDefaultArgument
-class SEQ_2_SEQ_AE:
-    """  Simple sequence to sequence autoencoder implemented with tf.keras """
-
-    def __init__(
-        self,
-        architecture_hparams: Dict = {},
-        huber_delta: float = 1.0,
-    ):
-        self.hparams = self.get_hparams(architecture_hparams)
-        self.CONV_filters = self.hparams["units_conv"]
-        self.LSTM_units_1 = self.hparams["units_lstm"]
-        self.LSTM_units_2 = int(self.hparams["units_lstm"] / 2)
-        self.DENSE_1 = int(self.hparams["units_lstm"] / 2)
-        self.DENSE_2 = self.hparams["units_dense2"]
-        self.DROPOUT_RATE = self.hparams["dropout_rate"]
-        self.ENCODING = self.hparams["encoding"]
-        self.learn_rate = self.hparams["learning_rate"]
-        self.delta = huber_delta
-
-    @staticmethod
-    def get_hparams(hparams):
-        """Sets the default parameters for the model. Overwritable with a dictionary"""
-
-        defaults = {
-            "units_conv": 256,
-            "units_lstm": 256,
-            "units_dense2": 64,
-            "dropout_rate": 0.25,
-            "encoding": 16,
-            "learning_rate": 1e-3,
-        }
-
-        for k, v in hparams.items():
-            defaults[k] = v
-
-        return defaults
-
-    def get_layers(self, input_shape):
-        """Instanciate all layers in the model"""
-
-        # Encoder Layers
-        Model_E0 = tf.keras.layers.Conv1D(
-            filters=self.CONV_filters,
-            kernel_size=5,
-            strides=1,
-            padding="causal",
-            activation="elu",
-            kernel_initializer=he_uniform(),
-        )
-        Model_E1 = Bidirectional(
-            LSTM(
-                self.LSTM_units_1,
-                activation="tanh",
-                recurrent_activation="sigmoid",
-                return_sequences=True,
-                kernel_constraint=UnitNorm(axis=0),
-            )
-        )
-        Model_E2 = Bidirectional(
-            LSTM(
-                self.LSTM_units_2,
-                activation="tanh",
-                recurrent_activation="sigmoid",
-                return_sequences=False,
-                kernel_constraint=UnitNorm(axis=0),
-            )
-        )
-        Model_E3 = Dense(
-            self.DENSE_1,
-            activation="elu",
-            kernel_constraint=UnitNorm(axis=0),
-            kernel_initializer=he_uniform(),
-        )
-        Model_E4 = Dense(
-            self.DENSE_2,
-            activation="elu",
-            kernel_constraint=UnitNorm(axis=0),
-            kernel_initializer=he_uniform(),
-        )
-        Model_E5 = Dense(
-            self.ENCODING,
-            activation="elu",
-            kernel_constraint=UnitNorm(axis=1),
-            activity_regularizer=deepof.model_utils.uncorrelated_features_constraint(
-                2, weightage=1.0
-            ),
-            kernel_initializer=Orthogonal(),
-        )
-
-        # Decoder layers
-        Model_D0 = deepof.model_utils.DenseTranspose(
-            Model_E5,
-            activation="elu",
-            output_dim=self.ENCODING,
-        )
-        Model_D1 = deepof.model_utils.DenseTranspose(
-            Model_E4,
-            activation="elu",
-            output_dim=self.DENSE_2,
-        )
-        Model_D2 = deepof.model_utils.DenseTranspose(
-            Model_E3,
-            activation="elu",
-            output_dim=self.DENSE_1,
-        )
-        Model_D3 = RepeatVector(input_shape[1])
-        Model_D4 = Bidirectional(
-            LSTM(
-                self.LSTM_units_1,
-                activation="tanh",
-                recurrent_activation="sigmoid",
-                return_sequences=True,
-                # kernel_constraint=UnitNorm(axis=1),
-            )
-        )
-        Model_D5 = Bidirectional(
-            LSTM(
-                self.LSTM_units_1,
-                activation="sigmoid",
-                recurrent_activation="sigmoid",
-                return_sequences=True,
-                # kernel_constraint=UnitNorm(axis=1),
-            )
-        )
-
-        return (
-            Model_E0,
-            Model_E1,
-            Model_E2,
-            Model_E3,
-            Model_E4,
-            Model_E5,
-            Model_D0,
-            Model_D1,
-            Model_D2,
-            Model_D3,
-            Model_D4,
-            Model_D5,
-        )
-
-    def build(
-        self,
-        input_shape: tuple,
-    ) -> Tuple[Any, Any, Any]:
-        """Builds the tf.keras model"""
-
-        (
-            Model_E0,
-            Model_E1,
-            Model_E2,
-            Model_E3,
-            Model_E4,
-            Model_E5,
-            Model_D0,
-            Model_D1,
-            Model_D2,
-            Model_D3,
-            Model_D4,
-            Model_D5,
-        ) = self.get_layers(input_shape)
-
-        # Define and instantiate encoder
-        encoder = Sequential(name="SEQ_2_SEQ_Encoder")
-        encoder.add(Input(shape=input_shape[1:]))
-        encoder.add(Model_E0)
-        encoder.add(BatchNormalization())
-        encoder.add(Model_E1)
-        encoder.add(BatchNormalization())
-        encoder.add(Model_E2)
-        encoder.add(BatchNormalization())
-        encoder.add(Model_E3)
-        encoder.add(BatchNormalization())
-        encoder.add(Dropout(self.DROPOUT_RATE))
-        encoder.add(Model_E4)
-        encoder.add(BatchNormalization())
-        encoder.add(Model_E5)
-
-        # Define and instantiate decoder
-        decoder = Sequential(name="SEQ_2_SEQ_Decoder")
-        decoder.add(Model_D0)
-        decoder.add(BatchNormalization())
-        decoder.add(Model_D1)
-        decoder.add(BatchNormalization())
-        decoder.add(Model_D2)
-        decoder.add(BatchNormalization())
-        decoder.add(Model_D3)
-        decoder.add(Model_D4)
-        decoder.add(BatchNormalization())
-        decoder.add(Model_D5)
-        decoder.add(TimeDistributed(Dense(input_shape[2])))
-
-        model = Sequential([encoder, decoder], name="SEQ_2_SEQ_AE")
-
-        model.compile(
-            loss=Huber(delta=self.delta),
-            optimizer=Nadam(
-                lr=self.learn_rate,
-                clipvalue=0.5,
-            ),
-            metrics=["mae"],
-        )
-
-        model.build(input_shape)
-
-        return encoder, decoder, model
-
-
 # noinspection PyDefaultArgument
-class SEQ_2_SEQ_GMVAE:
+class GMVAE:
     """  Gaussian Mixture Variational Autoencoder for pose motif elucidation.  """
 
     def __init__(
@@ -630,7 +421,7 @@ class SEQ_2_SEQ_GMVAE:
             z_gauss = deepof.model_utils.Cluster_overlap(
                 self.ENCODING,
                 self.number_of_components,
-                loss=self.overlap_loss,
+                loss_weight=self.overlap_loss,
             )(z_gauss)
 
         z = tfpl.DistributionLambda(

deepof/train_model.py

@@ -217,13 +217,6 @@ parser.add_argument(
     type=int,
     default=1,
 )
-parser.add_argument(
-    "--variational",
-    "-v",
-    help="Sets the model to train to a variational Bayesian autoencoder. Defaults to True",
-    default=True,
-    type=str2bool,
-)
 parser.add_argument(
     "--window-size",
     "-ws",
@@ -276,7 +269,6 @@ smooth_alpha = args.smooth_alpha
 train_path = os.path.abspath(args.train_path)
 tune = args.hyperparameter_tuning
 val_num = args.val_num
-variational = bool(args.variational)
 window_size = args.window_size
 window_step = args.window_step
 run = args.run
@@ -408,7 +400,6 @@ if not tune:
         rule_based_prediction=rule_based_prediction,
         save_checkpoints=False,
         save_weights=True,
-        variational=variational,
         reg_cat_clusters=("categorical" in latent_reg),
         reg_cluster_variance=("variance" in latent_reg),
         entropy_samples=entropy_samples,
@@ -419,13 +410,9 @@ if not tune:
 
 else:
     # Runs hyperparameter tuning with the specified parameters and saves the results
-
-    hyp = "S2SGMVAE" if variational else "S2SAE"
-
     run_ID, tensorboard_callback, entropy, onecycle = get_callbacks(
         X_train=X_train,
         batch_size=batch_size,
-        variational=variational,
         phenotype_prediction=phenotype_prediction,
         next_sequence_prediction=next_sequence_prediction,
         rule_based_prediction=rule_base_prediction,
@@ -456,7 +443,7 @@ else:
         next_sequence_prediction=next_sequence_prediction,
         phenotype_prediction=phenotype_prediction,
         rule_based_prediction=rule_base_prediction,
-        project_name="{}-based_{}_{}".format(input_type, hyp, tune.capitalize()),
+        project_name="{}-based_GMVAE_{}".format(input_type, tune.capitalize()),
         callbacks=[
             tensorboard_callback,
             onecycle,
@@ -477,7 +464,7 @@ else:
     with open(
         os.path.join(
             output_path,
-            "{}-based_{}_{}_params.pickle".format(input_type, hyp, tune.capitalize()),
+            "{}-based_GMVAE_{}_params.pickle".format(input_type, tune.capitalize()),
         ),
         "wb",
     ) as handle:

deepof/train_utils.py

@@ -68,7 +68,6 @@ def load_treatments(train_path):
 def get_callbacks(
     X_train: np.array,
     batch_size: int,
-    variational: bool,
     phenotype_prediction: float,
     next_sequence_prediction: float,
     rule_based_prediction: float,
@@ -103,13 +102,13 @@ def get_callbacks(
         latreg = "categorical+variance"
 
     run_ID = "{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}".format(
-        ("GMVAE" if variational else "AE"),
+        ("deepof_GMVAE"),
         ("_input_type={}".format(input_type) if input_type else "coords"),
         ("_window_size={}".format(X_train.shape[1])),
-        ("_NextSeqPred={}".format(next_sequence_prediction) if variational else ""),
-        ("_PhenoPred={}".format(phenotype_prediction) if variational else ""),
-        ("_RuleBasedPred={}".format(rule_based_prediction) if variational else ""),
-        ("_loss={}".format(loss) if variational else ""),
+        ("_NextSeqPred={}".format(next_sequence_prediction)),
+        ("_PhenoPred={}".format(phenotype_prediction)),
+        ("_RuleBasedPred={}".format(rule_based_prediction)),
+        ("_loss={}".format(loss)),
         ("_loss_warmup={}".format(loss_warmup)),
         ("_warmup_mode={}".format(warmup_mode)),
         ("_encoding={}".format(logparam["encoding"]) if logparam is not None else ""),
@@ -133,7 +132,6 @@ def get_callbacks(
         samples=entropy_samples,
         validation_data=X_val,
         log_dir=os.path.join(outpath, "metrics", run_ID),
-        variational=variational,
     )
 
     onecycle = deepof.model_utils.one_cycle_scheduler(
@@ -299,7 +297,6 @@ def autoencoder_fitting(
     pretrained: str,
     save_checkpoints: bool,
     save_weights: bool,
-    variational: bool,
     reg_cat_clusters: bool,
     reg_cluster_variance: bool,
     entropy_samples: int,
@@ -342,7 +339,6 @@ def autoencoder_fitting(
     run_ID, *cbacks = get_callbacks(
         X_train=X_train,
         batch_size=batch_size,
-        variational=variational,
         phenotype_prediction=phenotype_prediction,
         next_sequence_prediction=next_sequence_prediction,
         rule_based_prediction=rule_based_prediction,
@@ -384,159 +380,109 @@ def autoencoder_fitting(
     except IndexError:
         rule_based_features = 0
 
-    # Build models
-    if not variational:
-        encoder, decoder, ae = deepof.models.SEQ_2_SEQ_AE(
-            ({} if hparams is None else hparams)
+    # Build model
+    with strategy.scope():
+        (encoder, generator, grouper, ae, prior, posterior,) = deepof.models.GMVAE(
+            architecture_hparams=({} if hparams is None else hparams),
+            batch_size=batch_size * strategy.num_replicas_in_sync,
+            compile_model=True,
+            encoding=encoding_size,
+            kl_annealing_mode=kl_annealing_mode,
+            kl_warmup_epochs=kl_warmup,
+            loss=loss,
+            mmd_annealing_mode=mmd_annealing_mode,
+            mmd_warmup_epochs=mmd_warmup,
+            montecarlo_kl=montecarlo_kl,
+            number_of_components=n_components,
+            overlap_loss=False,
+            next_sequence_prediction=next_sequence_prediction,
+            phenotype_prediction=phenotype_prediction,
+            rule_based_prediction=rule_based_prediction,
+            rule_based_features=rule_based_features,
+            reg_cat_clusters=reg_cat_clusters,
+            reg_cluster_variance=reg_cluster_variance,
         ).build(X_train.shape)
-        return_list = (encoder, decoder, ae)
-
-    else:
-        with strategy.scope():
-            (
-                encoder,
-                generator,
-                grouper,
-                ae,
-                prior,
-                posterior,
-            ) = deepof.models.SEQ_2_SEQ_GMVAE(
-                architecture_hparams=({} if hparams is None else hparams),
-                batch_size=batch_size * strategy.num_replicas_in_sync,
-                compile_model=True,
-                encoding=encoding_size,
-                kl_annealing_mode=kl_annealing_mode,
-                kl_warmup_epochs=kl_warmup,
-                loss=loss,
-                mmd_annealing_mode=mmd_annealing_mode,
-                mmd_warmup_epochs=mmd_warmup,
-                montecarlo_kl=montecarlo_kl,
-                number_of_components=n_components,
-                overlap_loss=False,
-                next_sequence_prediction=next_sequence_prediction,
-                phenotype_prediction=phenotype_prediction,
-                rule_based_prediction=rule_based_prediction,
-                rule_based_features=rule_based_features,
-                reg_cat_clusters=reg_cat_clusters,
-                reg_cluster_variance=reg_cluster_variance,
-            ).build(
-                X_train.shape
-            )
-            return_list = (encoder, generator, grouper, ae)
+        return_list = (encoder, generator, grouper, ae)
 
     if pretrained:
         # If pretrained models are specified, load weights and return
         ae.load_weights(pretrained)
         return return_list
 
-    else:
-        if not variational:
-
-            ae.fit(
-                x=X_train,
-                y=X_train,
-                epochs=epochs,
-                verbose=1,
-                validation_data=(X_val, X_val),
-                callbacks=cbacks
-                + [
-                    CustomStopper(
-                        monitor="val_loss",
-                        patience=15,
-                        restore_best_weights=True,
-                        start_epoch=max(kl_warmup, mmd_warmup),
-                    ),
-                ],
-            )
+    callbacks_ = cbacks + [
+        CustomStopper(
+            monitor="val_loss",
+            patience=15,
+            restore_best_weights=True,
+            start_epoch=max(kl_warmup, mmd_warmup),
+        ),
+    ]
 
-            if not os.path.exists(os.path.join(output_path, "trained_weights")):
-                os.makedirs(os.path.join(output_path, "trained_weights"))
-
-            if save_weights:
-                ae.save_weights(
-                    os.path.join(
-                        "{}".format(output_path),
-                        "trained_weights",
-                        "{}_final_weights.h5".format(run_ID),
-                    )
-                )
-
-        else:
-