From 353c8b85d51f8e7b6afd86b6d5dc2f91cc4ac73c Mon Sep 17 00:00:00 2001
From: Philipp Arras <c@philipp-arras.de>
Date: Thu, 17 Mar 2022 15:37:54 +0100
Subject: [PATCH] output directory is always were the config file is located
---
cfg/cygnusa.cfg | 1 -
cfg/cygnusa_mf.cfg | 1 -
cfg/cygnusa_mf_cfm.cfg | 1 -
cfg/cygnusa_polarization.cfg | 1 -
cfg/gauss_001almacycle53noisy.cfg | 1 -
cfg/mf.cfg | 1 -
.../cfg.txt} | 0
misc/mpa_cluster/total_script.sh | 9 ++++----
resolve/command_line/resolve.py | 22 ++++++++++++++-----
resolve/config_utils.py | 1 -
10 files changed, 21 insertions(+), 17 deletions(-)
rename misc/mpa_cluster/{cfgs/cygnusa_polarization_13360.cfg => sample_reconstruction/cfg.txt} (100%)
diff --git a/cfg/cygnusa.cfg b/cfg/cygnusa.cfg
index 94093854..ffafd0f0 100644
--- a/cfg/cygnusa.cfg
+++ b/cfg/cygnusa.cfg
@@ -5,7 +5,6 @@ science target = CYG-ALL-2052-2MHZ.ms:(Cygnus-N$0$DATA)
number of randomly sampled rows = 10000
[optimization]
-output folder = output_cygnusa_stokesi
total iterations = 23
likelihood = points,data weights,full,*
constants = None,None,4*sky,None,*
diff --git a/cfg/cygnusa_mf.cfg b/cfg/cygnusa_mf.cfg
index 5945386a..4651a089 100644
--- a/cfg/cygnusa_mf.cfg
+++ b/cfg/cygnusa_mf.cfg
@@ -5,7 +5,6 @@ science target = CYG-ALL-2052-2MHZ.ms:(Cygnus-N$0$DATA),CYG-ALL-4811-8MHZ.ms:(Cy
number of randomly sampled rows = 10000
[optimization]
-output folder = output
total iterations = 30
likelihood = points,data weights,full,*
constants = None,None,4*sky,5*weights,None,*
diff --git a/cfg/cygnusa_mf_cfm.cfg b/cfg/cygnusa_mf_cfm.cfg
index 84bb5f19..18055b65 100644
--- a/cfg/cygnusa_mf_cfm.cfg
+++ b/cfg/cygnusa_mf_cfm.cfg
@@ -5,7 +5,6 @@ science target = CYG-ALL-2052-2MHZ.ms:(Cygnus-N$0$DATA),CYG-ALL-4811-8MHZ.ms:(Cy
number of randomly sampled rows = 10000
[optimization]
-output folder = output
total iterations = 20
likelihood = data weights
constants = None
diff --git a/cfg/cygnusa_polarization.cfg b/cfg/cygnusa_polarization.cfg
index 1b2701d8..d642d183 100644
--- a/cfg/cygnusa_polarization.cfg
+++ b/cfg/cygnusa_polarization.cfg
@@ -5,7 +5,6 @@ science target = CYG-ALL-2052-2MHZ.ms:(Cygnus-N$0$DATA)
number of randomly sampled rows = 10000
[optimization]
-output folder = output_cygnusa_polarization
total iterations = 23
likelihood = points,data weights,full,*
constants = None,None,4*sky,5*weights,None,*
diff --git a/cfg/gauss_001almacycle53noisy.cfg b/cfg/gauss_001almacycle53noisy.cfg
index 82e6d0e2..c3ee9493 100644
--- a/cfg/gauss_001almacycle53noisy.cfg
+++ b/cfg/gauss_001almacycle53noisy.cfg
@@ -5,7 +5,6 @@ science target = /data/gauss_001almacycle53noisy.ms:(gauss_001.alma.cycle5.3_0$0
#number of randomly sampled rows = 10000
[optimization]
-output folder = output
total iterations = 23
likelihood = data weights
constants = None
diff --git a/cfg/mf.cfg b/cfg/mf.cfg
index 4942663b..146b5e68 100644
--- a/cfg/mf.cfg
+++ b/cfg/mf.cfg
@@ -5,7 +5,6 @@ science target = mf_test_data.npz
#number of randomly sampled rows = 10000
[optimization]
-output folder = output_mf
total iterations = 1
likelihood = data weights
constants = None
diff --git a/misc/mpa_cluster/cfgs/cygnusa_polarization_13360.cfg b/misc/mpa_cluster/sample_reconstruction/cfg.txt
similarity index 100%
rename from misc/mpa_cluster/cfgs/cygnusa_polarization_13360.cfg
rename to misc/mpa_cluster/sample_reconstruction/cfg.txt
diff --git a/misc/mpa_cluster/total_script.sh b/misc/mpa_cluster/total_script.sh
index 7ecc87ef..7085f31c 100644
--- a/misc/mpa_cluster/total_script.sh
+++ b/misc/mpa_cluster/total_script.sh
@@ -1,8 +1,9 @@
set -ex
-INSTALL_DIR=~/temp4/cluster_playground
+CONFIG_FILE=`realpath cfgs/cygnusa_polarization_13360.cfg`
+INSTALL_DIR=`dirname $CONFIG_FILE`
QUEUE=pascal
-RESOLVE_BRANCH=mpa_cluster
+RESOLVE_BRANCH=devel
NIFTY_BRANCH=NIFTy_8
# Possible values for pe
@@ -15,7 +16,7 @@ TOTAL_THREADS=100 # Total number of threads across all MPI tasks
MEM=40G # Memory per Task
MPI_NP=5 # Number of MPI processes
-#./prepare_environment.sh $INSTALL_DIR $QUEUE $RESOLVE_BRANCH $NIFTY_BRANCH
+./prepare_environment.sh $INSTALL_DIR $QUEUE $RESOLVE_BRANCH $NIFTY_BRANCH
python3 generate_cluster_files.py \
--qname $QUEUE \
--venv-dir $INSTALL_DIR \
@@ -24,4 +25,4 @@ python3 generate_cluster_files.py \
--mpi-np $MPI_NP \
--total-threads $TOTAL_THREADS \
--mem $MEM \
- cfgs/cygnusa_polarization_13360.cfg --max-iteration 2
+ $CONFIG_FILE # --max-iteration 2
diff --git a/resolve/command_line/resolve.py b/resolve/command_line/resolve.py
index 74546b23..2b94aaba 100644
--- a/resolve/command_line/resolve.py
+++ b/resolve/command_line/resolve.py
@@ -47,14 +47,23 @@ def main():
nthreads = args.j
ift.set_nthreads(nthreads)
+ # Read config file
+ if not os.path.isfile(args.config_file):
+ raise RuntimeError(f"Config file {args.config_file} not found")
+ output_directory = os.path.split(args.config_file)[0]
+ if output_directory == "":
+ output_directory = "."
cfg = ConfigParser()
cfg.read(args.config_file)
+ # /Read config file
+ # Read data
obs_calib_flux, obs_calib_phase, obs_science = parse_data_config(cfg)
if cfg["sky"]["polarization"] == "I":
obs_science = [oo.restrict_to_stokesi().average_stokesi() for oo in obs_science]
assert len(obs_calib_flux) == len(obs_calib_phase) == 0
+ # /Read data
# Model operators
diffuse, additional_diffuse = sky_model_diffuse(cfg["sky"], obs_science, nthreads=nthreads)
@@ -90,13 +99,11 @@ def main():
lhs["data weights"] = ImagingLikelihood(obs_science, sky, epsilon=epsilon, do_wgridding=do_wgridding, nthreads=nthreads)
# /Likelihoods
- outdir = parse_optimize_kl_config(cfg["optimization"], lhs, domains)["output_directory"]
-
# Profiling
position = 0.1 * ift.from_random(lhs["full"].domain)
barrier(comm)
if master:
- os.makedirs(outdir, exist_ok=True)
+ os.makedirs(output_directory, exist_ok=True)
with ift.random.Context(12):
ift.exec_time(lhs["full"], verbose=args.verbose)
if args.profile_only:
@@ -105,10 +112,11 @@ def main():
barrier(comm)
# /Profiling
+ # Inference
def inspect_callback(sl, iglobal, position):
- visualize_weighted_residuals(obs_science, sl, iglobal, sky, weights, outdir, io=master,
+ visualize_weighted_residuals(obs_science, sl, iglobal, sky, weights, output_directory, io=master,
do_wgridding=do_wgridding, epsilon=epsilon, nthreads=nthreads)
- plot_sky(sl.average(sky), os.path.join(outdir, f"sky/sky_{iglobal}.pdf"))
+ plot_sky(sl.average(sky), os.path.join(output_directory, f"sky/sky_{iglobal}.pdf"))
if args.terminate is None:
terminate_callback = lambda iglobal: False
@@ -119,7 +127,9 @@ def main():
get_comm = comm
ift.optimize_kl(**parse_optimize_kl_config(cfg["optimization"], lhs, domains, inspect_callback),
plottable_operators=operators, comm=get_comm, overwrite=True,
- plot_latent=True, resume=args.resume, terminate_callback=terminate_callback)
+ plot_latent=True, resume=args.resume, terminate_callback=terminate_callback,
+ output_directory=output_directory)
+ # /Inference
if __name__ == "__main__":
diff --git a/resolve/config_utils.py b/resolve/config_utils.py
index b2d599c6..b55d6cc6 100644
--- a/resolve/config_utils.py
+++ b/resolve/config_utils.py
@@ -79,7 +79,6 @@ def parse_optimize_kl_config(cfg, likelihood_dct, constants_dct={}, inspect_call
sampling_iterations = f_int(cfg["sampling iteration limit"])
res["n_samples"] = lambda ii: f_int(cfg["n samples"])[ii]
res["sampling_iteration_controller"] = lambda ii: ift.AbsDeltaEnergyController(0.05, iteration_limit=sampling_iterations[ii], convergence_level=3, name="Sampling")
- res["output_directory"] = os.path.expanduser(cfg["output folder"])
def optimizer(ii):
--
GitLab