deepof_experiments.smk 5.7 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
# @authors lucasmiranda42
# encoding: utf-8
# deepof_experiments

"""

Snakefile for data and imputation.
Execution: sbatch snakemake
Plot DAG: snakemake --snakefile deepof_experiments.smk --forceall --dag | dot -Tpdf > deepof_experiments_DAG.pdf
Plot rule graph: snakemake --snakefile deepof_experiments.smk --forceall --rulegraph | dot -Tpdf > deepof_experiments_RULEGRAPH.pdf

"""

14
import os
15

lucas_miranda's avatar
lucas_miranda committed
16
outpath = "/psycl/g/mpsstatgen/lucas/DLC/DLC_autoencoders/DeepOF/deepof/logs/"
17

18
19
warmup_epochs = [15]
warmup_mode = ["sigmoid"]
20
losses = ["ELBO"]  # , "MMD", "ELBO+MMD"]
21
encodings = [6]  # [2, 4, 6, 8, 10, 12, 14, 16]
22
cluster_numbers = [15]  # [1, 5, 10, 15, 20, 25]
23
latent_reg = ["variance"]  # ["none", "categorical", "variance", "categorical+variance"]
24
entropy_knn = [100]
25
next_sequence_pred_weights = [0.15]
26
phenotype_pred_weights = [0.0]
27
rule_based_pred_weights = [0.0]
28
window_lengths = [22]  # range(11,56,11)
29
input_types = ["coords"]
30
run = list(range(1, 11))
31

32

33
34
rule deepof_experiments:
    input:
35
        # Elliptical arena detection
lucas_miranda's avatar
lucas_miranda committed
36
        # "/psycl/g/mpsstatgen/lucas/DLC/DLC_autoencoders/DeepOF/deepof/supplementary_notebooks/recognise_elliptical_arena.ipynb",
37
        #
38
        # Hyperparameter tuning
39
40
41
42
43
44
45
46
47
        # expand(
        #     os.path.join(
        #         outpath,
        #         "coarse_hyperparameter_tuning/trained_weights/GMVAE_loss={loss}_k={k}_encoding={enc}_final_weights.h5",
        #     ),
        #     loss=losses,
        #     k=cluster_numbers,
        #     enc=encodings,
        # ),
48
        #
49
        # Train a variety of models
lucas_miranda's avatar
lucas_miranda committed
50
        expand(
51
            outpath + "train_models/trained_weights/"
52
            "GMVAE_input_type={input_type}_"
53
            "window_size={window_size}_"
54
            "NextSeqPred={nspredweight}_"
55
56
57
            "PhenoPred={phenpredweight}_"
            "RuleBasedPred={rulesweight}_"
            "loss={loss}_"
58
59
            "loss_warmup={warmup}_"
            "warmup_mode={warmup_mode}_"
60
61
62
            "encoding={encs}_"
            "k={k}_"
            "latreg={latreg}_"
63
            "entknn={entknn}_"
64
65
            "run={run}_"
            "final_weights.h5",
66
            input_type=input_types,
67
            window_size=window_lengths,
lucas_miranda's avatar
lucas_miranda committed
68
            loss=losses,
69
70
            warmup=warmup_epochs,
            warmup_mode=warmup_mode,
lucas_miranda's avatar
lucas_miranda committed
71
72
73
74
            encs=encodings,
            k=cluster_numbers,
            latreg=latent_reg,
            entknn=entropy_knn,
75
76
77
78
            nspredweight=next_sequence_pred_weights,
            phenpredweight=phenotype_pred_weights,
            rulesweight=rule_based_pred_weights,
            run=run,
lucas_miranda's avatar
lucas_miranda committed
79
        ),
80

81

82
83
rule elliptical_arena_detector:
    input:
84
        to_exec="/psycl/g/mpsstatgen/lucas/DLC/DLC_autoencoders/DeepOF/deepof/supplementary_notebooks/recognise_elliptical_arena_blank.ipynb",
85
    output:
86
        exec="/psycl/g/mpsstatgen/lucas/DLC/DLC_autoencoders/DeepOF/deepof/supplementary_notebooks/recognise_elliptical_arena.ipynb",
87
    shell:
88
        "papermill {input.to_exec} "
89
        "-p vid_path './supplementary_notebooks/' "
90
        "-p log_path './logs/' "
91
        "-p out_path './deepof/trained_models/' "
92
93
94
        "{output.exec}"


95
rule coarse_hyperparameter_tuning:
96
    input:
97
        data_path="/psycl/g/mpsstatgen/lucas/DLC/DLC_models/deepof_single_topview/",
98
99
100
    output:
        trained_models=os.path.join(
            outpath,
101
            "coarse_hyperparameter_tuning/trained_weights/GMVAE_loss={loss}_k={k}_encoding={enc}_final_weights.h5",
102
103
104
105
        ),
    shell:
        "pipenv run python -m deepof.train_model "
        "--train-path {input.data_path} "
106
        "--val-num 25 "
107
        "--components {wildcards.k} "
108
        "--input-type coords "
109
110
111
        "--next-sequence-prediction {wildcards.nspredweight} "
        "--phenotype-prediction {wildcards.phenpredweight} "
        "--rule-based-prediction {wildcards.rulesweight} "
112
        "--loss {wildcards.loss} "
113
114
        "--kl-warmup 30 "
        "--mmd-warmup 30 "
115
        "--encoding-size {wildcards.enc} "
116
        "--batch-size 512 "
117
        "--window-size 24 "
118
        "--window-step 12 "
119
120
        "--output-path {outpath}coarse_hyperparameter_tuning "
        "--hyperparameter-tuning hyperband "
121
        "--hpt-trials 1"
122
123


124
rule train_models:
125
    input:
126
127
128
        data_path=ancient(
            "/psycl/g/mpsstatgen/lucas/DLC/DLC_models/deepof_single_topview/"
        ),
129
    output:
130
        trained_models=outpath + "train_models/trained_weights/"
131
        "GMVAE_input_type={input_type}_"
132
        "window_size={window_size}_"
133
        "NextSeqPred={nspredweight}_"
134
135
136
        "PhenoPred={phenpredweight}_"
        "RuleBasedPred={rulesweight}_"
        "loss={loss}_"
137
138
        "loss_warmup={warmup}_"
        "warmup_mode={warmup_mode}_"
139
140
141
        "encoding={encs}_"
        "k={k}_"
        "latreg={latreg}_"
142
        "entknn={entknn}_"
143
144
        "run={run}_"
        "final_weights.h5",
145
146
147
    shell:
        "pipenv run python -m deepof.train_model "
        "--train-path {input.data_path} "
148
        "--val-num 15 "
149
        "--components {wildcards.k} "
150
151
152
153
        "--input-type {wildcards.input_type} "
        "--next-sequence-prediction {wildcards.nspredweight} "
        "--phenotype-prediction {wildcards.phenpredweight} "
        "--rule-based-prediction {wildcards.rulesweight} "
154
155
        "--latent-reg {wildcards.latreg} "
        "--loss {wildcards.loss} "
156
157
        "--kl-annealing-mode {wildcards.warmup_mode} "
        "--kl-warmup {wildcards.warmup} "
158
        "--mmd-annealing-mode {wildcards.warmup_mode} "
159
        "--mmd-warmup {wildcards.warmup} "
160
161
        "--montecarlo-kl 10 "
        "--encoding-size {wildcards.encs} "
162
        "--entropy-knn {wildcards.entknn} "
163
        "--batch-size 256 "
164
        "--window-size {wildcards.window_size} "
165
        "--window-step 11 "
166
        "--run {wildcards.run} "
167
        "--output-path {outpath}train_models"