Commit 4d1510c0 authored by Lucas Miranda's avatar Lucas Miranda
Browse files

added snakemake pipeline for autoencoder experiments

parent 7aaa0341
Pipeline #88518 failed with stage
in 60 minutes and 30 seconds
{
"__default__":
{
"mem": "100G",
"excl": "psycl[02-03]",
"time": "10-0",
"stderr": "snakemake_logs/{rule}.err",
"stdout": "snakemake_logs/{rule}.out",
"jobname": "{rule}",
"cpus": "10"
}
}
# @authors lucasmiranda42
# encoding: utf-8
# deepof_experiments
"""
Snakefile for data and imputation.
Execution: sbatch snakemake
Plot DAG: snakemake --snakefile deepof_experiments.smk --forceall --dag | dot -Tpdf > deepof_experiments_DAG.pdf
Plot rule graph: snakemake --snakefile deepof_experiments.smk --forceall --rulegraph | dot -Tpdf > deepof_experiments_RULEGRAPH.pdf
"""
import numpy as np
import pandas as pd
import subprocess
import re
losses = ["ELBO", "MMD", "ELBO+MMD"]
encodings = [2, 4, 6, 8, 12, 16]
rule deepof_experiments:
input:
expand(
"/u/lucasmir/DLC/DLC_autoencoders/DeepOF/deepof/GMVAE_loss={loss}_encoding={encs}_final_weights.h5",
loss=losses,
encs=encodings,
),
rule explore_encoding_dimension_and_loss_function:
input:
data_path="/u/lucasmir/DLC/DLC_models/deepof_single_topview/",
output:
trained_models="/u/lucasmir/DLC/DLC_autoencoders/DeepOF/deepof/GMVAE_loss={loss}_encoding={encs}_final_weights.h5",
shell:
"pipenv run python -m deepof.train_model --train-path {input.data_path} --val-num 10 --components 10 --input-type coords --predictor 0 --variational True --loss {wildcards.loss} --kl-warmup 20 --mmd-warmup 20 --encoding-size {wildcards.encs} --batch-size 256 --window-size 11 --window-step 11 --exclude-bodyparts Tail_base,Tail_1,Tail_2,Tail_tip,Spine_2 --stability-check 3"
# rule explore_number_of_components:
# input:
# output:
# shell:
# rule explore_phenotype_prediction:
# input:
# output:
# shell:
# rule explore_predictor_branch:
# input:
# output:
# shell:
mkdir snakemake_logs
snakemake --snakefile deepof_experiments.smk --forceall --dag | dot -Tpdf > deepof_experiments_DAG.pdf
snakemake --snakefile deepof_experiments.smk --forceall --rulegraph | dot -Tpdf > deepof_experiments_RULEGRAPH.pdf
# source /mpcdf/soft/distribution/obs_modules.sh
# module load anaconda/3
# module load intel/18.0.5
# module load cuda/10.2
# module load cudnn/8.0.4
# module load nccl/2.7.8
# module load tensorrt/7.1.3
# module load tensorflow/gpu/2.3.0
# module load tensorboard/2.3.0
snakemake --snakefile deepof_experiments.smk deepof_experiments -j 8 --latency-wait 15 --cluster-config cluster.json --cluster "sbatch --time={cluster.time} --mem={cluster.mem} --exclude={cluster.excl} -o {cluster.stdout} -e {cluster.stderr} --job-name={cluster.jobname} --cpus-per-task={cluster.cpus} --constraint="gpu" --gres=gpu:gtx980:1" > deepof_experiments.out 2> deepof_experiments.err
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment