deepof_experiments.smk 4.96 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
# @authors lucasmiranda42
# encoding: utf-8
# deepof_experiments

"""

Snakefile for data and imputation.
Execution: sbatch snakemake
Plot DAG: snakemake --snakefile deepof_experiments.smk --forceall --dag | dot -Tpdf > deepof_experiments_DAG.pdf
Plot rule graph: snakemake --snakefile deepof_experiments.smk --forceall --rulegraph | dot -Tpdf > deepof_experiments_RULEGRAPH.pdf

"""

14
import os
15

lucas_miranda's avatar
lucas_miranda committed
16
outpath = "/psycl/g/mpsstatgen/lucas/DLC/DLC_autoencoders/DeepOF/deepof/logs/"
17
losses = ["ELBO"]  # , "MMD", "ELBO+MMD"]
18
encodings = [6]  # [2, 4, 6, 8, 10, 12, 14, 16]
19
20
cluster_numbers = [25]  # [1, 5, 10, 15, 20, 25]
latent_reg = ["none", "categorical", "variance", "categorical+variance"]
21
entropy_min_n = [2, 5, 8, 12]
22
pheno_weights = [0.01, 0.1, 0.25, 0.5, 1.0, 2.0, 4.0, 10.0, 100.0]
23

24

25
26
rule deepof_experiments:
    input:
27
28
29
30
31
32
33
34
35
        # expand(
        #     os.path.join(
        #         outpath,
        #         "coarse_hyperparameter_tuning/trained_weights/GMVAE_loss={loss}_k={k}_encoding={enc}_final_weights.h5",
        #     ),
        #     loss=losses,
        #     k=cluster_numbers,
        #     enc=encodings,
        # ),
Lucas Miranda's avatar
Lucas Miranda committed
36
        expand(
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
            "/psycl/g/mpsstatgen/lucas/DLC/DLC_autoencoders/DeepOF/deepof/logs/latent_regularization_experiments/trained_weights/"
            "GMVAE_loss={loss}_encoding={encs}_k={k}_latreg={latreg}_entropymink={entmin}_final_weights.h5",
            loss=losses,
            encs=encodings,
            k=cluster_numbers,
            latreg=latent_reg,
            entmin=entropy_min_n,
        ),
        # expand(
        #     "/psycl/g/mpsstatgen/lucas/DLC/DLC_autoencoders/DeepOF/deepof/logs/pheno_classification_experiments/trained_weights/"
        #     "GMVAE_loss={loss}_encoding={encs}_k={k}_pheno={phenos}_run_1_final_weights.h5",
        #     loss=losses,
        #     encs=encodings,
        #     k=cluster_numbers,
        #     phenos=pheno_weights,
        # ),
53
54


55
rule coarse_hyperparameter_tuning:
56
    input:
57
        data_path="/psycl/g/mpsstatgen/lucas/DLC/DLC_models/deepof_single_topview/",
58
59
60
    output:
        trained_models=os.path.join(
            outpath,
61
            "coarse_hyperparameter_tuning/trained_weights/GMVAE_loss={loss}_k={k}_encoding={enc}_final_weights.h5",
62
63
64
65
        ),
    shell:
        "pipenv run python -m deepof.train_model "
        "--train-path {input.data_path} "
66
        "--val-num 25 "
67
        "--components {wildcards.k} "
68
69
70
71
72
73
        "--input-type coords "
        "--predictor 0 "
        "--phenotype-classifier 0 "
        "--variational True "
        "--loss {wildcards.loss} "
        "--kl-warmup 20 "
74
        "--mmd-warmup 0 "
75
        "--encoding-size {wildcards.enc} "
76
77
        "--batch-size 256 "
        "--window-size 24 "
78
        "--window-step 12 "
79
80
        "--output-path {outpath}coarse_hyperparameter_tuning "
        "--hyperparameter-tuning hyperband "
81
        "--hpt-trials 1"
82
83


84
85
rule latent_regularization_experiments:
    input:
86
87
88
        data_path=ancient(
            "/psycl/g/mpsstatgen/lucas/DLC/DLC_models/deepof_single_topview/"
        ),
89
90
91
    output:
        trained_models=os.path.join(
            outpath,
92
            "latent_regularization_experiments/trained_weights/GMVAE_loss={loss}_encoding={encs}_k={k}_latreg={latreg}_entropymink={entmin}_final_weights.h5",
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
        ),
    shell:
        "pipenv run python -m deepof.train_model "
        "--train-path {input.data_path} "
        "--val-num 5 "
        "--components {wildcards.k} "
        "--input-type coords "
        "--predictor 0 "
        "--phenotype-classifier 0 "
        "--variational True "
        "--latent-reg {wildcards.latreg} "
        "--loss {wildcards.loss} "
        "--kl-warmup 20 "
        "--mmd-warmup 20 "
        "--montecarlo-kl 10 "
        "--encoding-size {wildcards.encs} "
109
        "--entropy-min-n {wildcards.entmin} "
110
111
112
        "--batch-size 256 "
        "--window-size 24 "
        "--window-step 12 "
113
        # "--exclude-bodyparts Tail_base,Tail_1,Tail_2,Tail_tip "
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
        "--output-path {outpath}latent_regularization_experiments"


rule explore_phenotype_classification:
    input:
        data_path="/psycl/g/mpsstatgen/lucas/DLC/DLC_models/deepof_single_topview/",
    output:
        trained_models=os.path.join(
            outpath,
            "pheno_classification_experiments/trained_weights/GMVAE_loss={loss}_encoding={encs}_k={k}_pheno={phenos}_run_1_final_weights.h5",
        ),
    shell:
        "pipenv run python -m deepof.train_model "
        "--train-path {input.data_path} "
        "--val-num 15 "
        "--components {wildcards.k} "
        "--input-type coords "
        "--predictor 0 "
        "--phenotype-classifier {wildcards.phenos} "
        "--variational True "
        "--loss {wildcards.loss} "
        "--kl-warmup 20 "
        "--mmd-warmup 20 "
        "--montecarlo-kl 10 "
        "--encoding-size {wildcards.encs} "
        "--batch-size 256 "
        "--window-size 11 "
        "--window-step 11 "
        "--stability-check 3  "
        "--output-path {outpath}pheno_classification_experiments"