deepof_experiments.smk 4.9 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
# @authors lucasmiranda42
# encoding: utf-8
# deepof_experiments

"""

Snakefile for data and imputation.
Execution: sbatch snakemake
Plot DAG: snakemake --snakefile deepof_experiments.smk --forceall --dag | dot -Tpdf > deepof_experiments_DAG.pdf
Plot rule graph: snakemake --snakefile deepof_experiments.smk --forceall --rulegraph | dot -Tpdf > deepof_experiments_RULEGRAPH.pdf

"""

14
import os
15

lucas_miranda's avatar
lucas_miranda committed
16
outpath = "/psycl/g/mpsstatgen/lucas/DLC/DLC_autoencoders/DeepOF/deepof/logs/"
17

18
19
20
21
22
23
24
25
26
27
losses = ["ELBO" , "MMD", "ELBO+MMD"]
encodings =  [6] # [2, 4, 6, 8, 10, 12, 14, 16]
cluster_numbers = [25] # [1, 5, 10, 15, 20, 25]
latent_reg = ["none"] # ["none", "categorical", "variance", "categorical+variance"]
entropy_knn = [100]
next_sequence_pred_weights = [0.0, 0.15, 0.30]
phenotype_pred_weights = [0.0, 0.15]
rule_based_pred_weights = [0.0, 0.15, 0.30]
input_types = ["coords"]
run = list(range(1, 11))
28

29
30
rule deepof_experiments:
    input:
31
        # Elliptical arena detection
lucas_miranda's avatar
lucas_miranda committed
32
        # "/psycl/g/mpsstatgen/lucas/DLC/DLC_autoencoders/DeepOF/deepof/supplementary_notebooks/recognise_elliptical_arena.ipynb",
33
34

        # Hyperparameter tuning
35
36
37
38
39
40
41
42
43
        # expand(
        #     os.path.join(
        #         outpath,
        #         "coarse_hyperparameter_tuning/trained_weights/GMVAE_loss={loss}_k={k}_encoding={enc}_final_weights.h5",
        #     ),
        #     loss=losses,
        #     k=cluster_numbers,
        #     enc=encodings,
        # ),
44
45

        # Train a variety of models
lucas_miranda's avatar
lucas_miranda committed
46
        expand(
47
            "/psycl/g/mpsstatgen/lucas/DLC/DLC_autoencoders/DeepOF/deepof/logs/train_models/trained_weights/"
lucas_miranda's avatar
lucas_miranda committed
48
            "GMVAE_loss={loss}_encoding={encs}_k={k}_latreg={latreg}_entropyknn={entknn}_final_weights.h5",
49
            input_type=input_types,
lucas_miranda's avatar
lucas_miranda committed
50
51
52
53
54
            loss=losses,
            encs=encodings,
            k=cluster_numbers,
            latreg=latent_reg,
            entknn=entropy_knn,
55
56
57
58
            nspredweight=next_sequence_pred_weights,
            phenpredweight=phenotype_pred_weights,
            rulesweight=rule_based_pred_weights,
            run=run,
lucas_miranda's avatar
lucas_miranda committed
59
        ),
60

61
62
rule elliptical_arena_detector:
    input:
63
        to_exec="/psycl/g/mpsstatgen/lucas/DLC/DLC_autoencoders/DeepOF/deepof/supplementary_notebooks/recognise_elliptical_arena_blank.ipynb",
64
    output:
65
        exec="/psycl/g/mpsstatgen/lucas/DLC/DLC_autoencoders/DeepOF/deepof/supplementary_notebooks/recognise_elliptical_arena.ipynb",
66
    shell:
67
        "papermill {input.to_exec} "
68
        "-p vid_path './supplementary_notebooks/' "
69
        "-p log_path './logs/' "
70
        "-p out_path './deepof/trained_models/' "
71
72
73
        "{output.exec}"


74
rule coarse_hyperparameter_tuning:
75
    input:
76
        data_path="/psycl/g/mpsstatgen/lucas/DLC/DLC_models/deepof_single_topview/",
77
78
79
    output:
        trained_models=os.path.join(
            outpath,
80
            "coarse_hyperparameter_tuning/trained_weights/GMVAE_loss={loss}_k={k}_encoding={enc}_final_weights.h5",
81
82
83
84
        ),
    shell:
        "pipenv run python -m deepof.train_model "
        "--train-path {input.data_path} "
85
        "--val-num 25 "
86
        "--components {wildcards.k} "
87
        "--input-type coords "
88
89
90
        "--next-sequence-prediction {wildcards.nspredweight} "
        "--phenotype-prediction {wildcards.phenpredweight} "
        "--rule-based-prediction {wildcards.rulesweight} "
91
92
        "--variational True "
        "--loss {wildcards.loss} "
93
94
        "--kl-warmup 5 "
        "--mmd-warmup 5 "
95
        "--encoding-size {wildcards.enc} "
96
97
        "--batch-size 256 "
        "--window-size 24 "
98
        "--window-step 12 "
99
100
        "--output-path {outpath}coarse_hyperparameter_tuning "
        "--hyperparameter-tuning hyperband "
101
        "--hpt-trials 1"
102
103


104
rule train_models:
105
    input:
106
107
108
        data_path=ancient(
            "/psycl/g/mpsstatgen/lucas/DLC/DLC_models/deepof_single_topview/"
        ),
109
110
111
    output:
        trained_models=os.path.join(
            outpath,
112
            "latent_regularization_experiments/trained_weights/GMVAE_loss={loss}_encoding={encs}_k={k}_latreg={latreg}_entropyknn={entknn}_final_weights.h5",
113
114
115
116
        ),
    shell:
        "pipenv run python -m deepof.train_model "
        "--train-path {input.data_path} "
117
        "--val-num 15 "
118
        "--components {wildcards.k} "
119
120
121
122
        "--input-type {wildcards.input_type} "
        "--next-sequence-prediction {wildcards.nspredweight} "
        "--phenotype-prediction {wildcards.phenpredweight} "
        "--rule-based-prediction {wildcards.rulesweight} "
123
124
125
        "--variational True "
        "--latent-reg {wildcards.latreg} "
        "--loss {wildcards.loss} "
126
127
        "--kl-warmup 5 "
        "--mmd-warmup 5 "
128
129
        "--montecarlo-kl 10 "
        "--encoding-size {wildcards.encs} "
130
        "--entropy-knn {wildcards.entknn} "
131
132
133
        "--batch-size 256 "
        "--window-size 24 "
        "--window-step 12 "
134
        "--run {wildcards.run}"
135

136
        "--output-path {outpath}latent_regularization_experiments"
137
        # "--exclude-bodyparts Tail_base,Tail_1,Tail_2,Tail_tip "