Skip to content
Snippets Groups Projects
Commit 0cd05395 authored by Simon Ding's avatar Simon Ding
Browse files

better default params

parent 081fbfd8
No related branches found
No related tags found
1 merge request!26Observation slicing
Pipeline #109078 passed
......@@ -355,22 +355,22 @@ class Observation(BaseObservation):
)
@staticmethod
def split_data_file(data_path, ntask, target_folder, base_name, nwork):
def split_data_file(data_path, ntask, target_folder, base_name, nwork, compress):
from os import makedirs
makedirs(target_folder, exist_ok=True)
for rank in range(ntask):
lo, hi = ift.utilities.shareRange(nwork, ntask, rank)
obs = Observation.load(data_path, (lo, hi))
obs.save(f"{target_folder}/{base_name}_{rank}.npz", compress=False)
obs.save(f"{target_folder}/{base_name}_{rank}.npz", compress=compress)
@staticmethod
def mpi_load(data_folder, base_name, full_data_set, nwork, comm=None):
def mpi_load(data_folder, base_name, full_data_set, nwork, comm=None, compress=False):
if master:
from os.path import isdir
if not isdir(data_folder):
Observation.split_data_file(full_data_set, comm.Get_size(), data_folder, base_name, nwork)
Observation.split_data_file(full_data_set, comm.Get_size(), data_folder, base_name, nwork, compress)
if comm is None:
return Observation.load(full_data_set)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment