diff --git a/syconn/extraction/object_extraction_steps.py b/syconn/extraction/object_extraction_steps.py index 207bd7a00064073fa6a2e0a1156f11a4547cb380..21b757f8d96f31066740896b04c4587ac1fa7854 100644 --- a/syconn/extraction/object_extraction_steps.py +++ b/syconn/extraction/object_extraction_steps.py @@ -918,7 +918,7 @@ def _extract_voxels_thread(args): return map_dict -def combine_voxels(workfolder, hdf5names, +def combine_voxels(workfolder, hdf5names, version_sd=0, version_dict_sd=None, n_folders_fs=10000, stride=10, nb_cpus=1, qsub_pe=None, qsub_queue=None, n_max_co_processes=None): """ @@ -954,12 +954,13 @@ def combine_voxels(workfolder, hdf5names, segdataset = segmentation.SegmentationDataset(obj_type=hdf5_name, working_dir=workfolder, - version="new", - create=True, + version=version_sd, + create=True, version_dict=version_dict_sd, n_folders_fs=n_folders_fs) for p in voxel_rel_paths_2stage: - os.makedirs(segdataset.so_storage_path + p) + if not os.path.isdir(segdataset.so_storage_path + p): + os.makedirs(segdataset.so_storage_path + p) multi_params = [] path_blocks = np.array_split(np.array(voxel_rel_paths), @@ -992,7 +993,7 @@ def combine_voxels(workfolder, hdf5names, multi_params.append([workfolder, hdf5_name, path_block, path_block_dicts, segdataset.version, - n_folders_fs]) + n_folders_fs, version_dict_sd]) if qsub_pe is None and qsub_queue is None: results = sm.start_multiprocess_imap(_combine_voxels_thread, @@ -1017,12 +1018,13 @@ def _combine_voxels_thread(args): path_block_dicts = args[3] dataset_version = args[4] n_folders_fs = args[5] + version_dict_sd = args[6] dataset_temp_path = workfolder + "/%s_temp/" % hdf5_name segdataset = segmentation.SegmentationDataset( obj_type=hdf5_name, working_dir=workfolder, version=dataset_version, - n_folders_fs=n_folders_fs) + n_folders_fs=n_folders_fs, version_dict=version_dict_sd) for i_voxel_rel_path, voxel_rel_path in enumerate(voxel_rel_paths): diff --git a/syconn/extraction/object_extraction_wrapper.py b/syconn/extraction/object_extraction_wrapper.py index e2d309ea80415e54d11b3ba2f8b23ab6614f1f48..1e75240a5ced8bceb466d4547be8ea33d1eb9ee8 100644 --- a/syconn/extraction/object_extraction_wrapper.py +++ b/syconn/extraction/object_extraction_wrapper.py @@ -432,7 +432,7 @@ def from_probabilities_to_objects_parameter_sweeping(cset, def from_ids_to_objects(cset, filename, hdf5names=None, n_folders_fs=10000, overlaydataset_path=None, chunk_list=None, offset=None, size=None, suffix="", qsub_pe=None, qsub_queue=None, qsub_slots=None, - n_max_co_processes=None, n_chunk_jobs=5000): + n_max_co_processes=None, n_chunk_jobs=5000, version_dict_sd=None): """ Main function for the object extraction step; combines all needed steps Parameters @@ -494,13 +494,13 @@ def from_ids_to_objects(cset, filename, hdf5names=None, n_folders_fs=10000, all_times.append(time.time() - time_start) step_names.append("voxel extraction") print("\nTime needed for extracting voxels: %.3fs" % all_times[-1]) - # + # # -------------------------------------------------------------------------- # time_start = time.time() oes.combine_voxels(os.path.dirname(cset.path_head_folder.rstrip("/")), hdf5names, qsub_pe=qsub_pe, qsub_queue=qsub_queue, - n_folders_fs=n_folders_fs, + n_folders_fs=n_folders_fs, version_dict_sd=version_dict_sd, n_max_co_processes=n_max_co_processes) all_times.append(time.time() - time_start) step_names.append("combine voxels") diff --git a/syconn/handler/basics.py b/syconn/handler/basics.py index 8c962a046849f38cb82690e2af9dc1ece2b38df8..3eaf2aa037889a6329e5450618cad9a4debd7deb 100644 --- a/syconn/handler/basics.py +++ b/syconn/handler/basics.py @@ -380,8 +380,8 @@ def data2kzip(kzip_path, fpaths, fnames_in_zip=None, force_overwrite=True, verbose : bool force_overwrite : bool """ - if not force_overwrite: - raise NotImplementedError('Currently modification of data in existing kzip is not implemented.') + # if not force_overwrite: + # raise NotImplementedError('Currently modification of data in existing kzip is not implemented.') nb_files = len(fpaths) if verbose: log_handler.info('Writing {} files to .zip.'.format(nb_files)) diff --git a/syconn/handler/prediction.py b/syconn/handler/prediction.py index d51104889e091b78808525bfd1f0c82c44800d02..02de96c7f612664181c0eb740fd2b35f351f3a9e 100644 --- a/syconn/handler/prediction.py +++ b/syconn/handler/prediction.py @@ -327,7 +327,7 @@ def create_h5_gt_file(fname, raw, label, foreground_ids=None, debug=False): raw = xyz2zxy(raw) print("Raw:", raw.shape, raw.dtype, raw.min(), raw.max()) print("Label:", label.shape, label.dtype, label.min(), label.max()) - print("-----------------\nGT Summary:\n%s\n" %str(Counter(label.flatten()).items())) + print("-----------------\nGT Summary:\n%s\n" % str(Counter(label.flatten()).items())) if not fname[-2:] == "h5": fname = fname + ".h5" if debug: @@ -433,11 +433,11 @@ def predict_dataset(kd_p, kd_pred_p, cd_p, model_p, imposed_patch_size=None, """ if isinstance(gpu_ids, int) or len(gpu_ids) == 1: _pred_dataset(kd_p, kd_pred_p, cd_p, model_p, imposed_patch_size, - mfp_active, gpu_ids, overwrite) + mfp_active, gpu_ids, overwrite) else: print("Starting multi-gpu prediction with GPUs:", gpu_ids) - _multi_gpu_ds_pred(kd_p, kd_pred_p, cd_p, model_p,imposed_patch_size, gpu_ids) + _multi_gpu_ds_pred(kd_p, kd_pred_p, cd_p, model_p, imposed_patch_size, gpu_ids) def _pred_dataset(kd_p, kd_pred_p, cd_p, model_p, imposed_patch_size=None, @@ -464,7 +464,7 @@ def _pred_dataset(kd_p, kd_pred_p, cd_p, model_p, imposed_patch_size=None, the GPU used overwrite : bool True: fresh predictions ; False: earlier prediction continues - + Returns ------- @@ -515,7 +515,7 @@ def _pred_dataset(kd_p, kd_pred_p, cd_p, model_p, imposed_patch_size=None, if n is None: kd_pred = KnossosDataset() kd_pred.initialize_without_conf(kd_pred_p, kd.boundary, kd.scale, - kd.experiment_name, mags=[1,2,4,8]) + kd.experiment_name, mags=[1, 2, 4, 8]) cd.export_cset_to_kd(kd_pred, "pred", ["pred"], [4, 4], as_raw=True, stride=[256, 256, 256]) @@ -537,7 +537,7 @@ def to_knossos_dataset(kd_p, kd_pred_p, cd_p, model_p, imposed_patch_size,mfp_ac cd.initialize(kd, kd.boundary, [512, 512, 256], cd_p, overlap=offset, box_coords=np.zeros(3), fit_box_size=True) kd_pred.initialize_without_conf(kd_pred_p, kd.boundary, kd.scale, - kd.experiment_name, mags=[1,2,4,8]) + kd.experiment_name, mags=[1, 2, 4, 8]) cd.export_cset_to_kd(kd_pred, "pred", ["pred"], [4, 4], as_raw=True, stride=[256, 256, 256]) @@ -607,6 +607,7 @@ class NeuralNetworkInterface(object): Inference class for elektronn2 models, support will end at some point. Switching to 'InferenceModel' in elektronn3.model.base in the long run. """ + def __init__(self, model_path, arch='marvin', imposed_batch_size=1, channels_to_load=(0, 1, 2, 3), normal=False, nb_labels=2, normalize_data=False, normalize_func=None, init_gpu=None): @@ -666,7 +667,7 @@ class NeuralNetworkInterface(object): cnt += 1 pbar.update() x_b = x[b] - proba[b] = self.model.predict(x_b)[None, ] + proba[b] = self.model.predict(x_b)[None,] overhead = len(x) % bs # TODO: add proper axis handling, maybe introduce axistags if overhead != 0: @@ -680,8 +681,8 @@ class NeuralNetworkInterface(object): end = time.time() sys.stdout.write("\r%0.2f\n" % 1.0) sys.stdout.flush() - print("Prediction of %d samples took %0.2fs; %0.4fs/sample." %\ - (len(x), end-start, (end-start)/len(x))) + print("Prediction of %d samples took %0.2fs; %0.4fs/sample." % \ + (len(x), end - start, (end - start) / len(x))) pbar.close() if self.arch == "triplet": return proba[..., 0] @@ -865,7 +866,6 @@ def _multi_gpu_ds_pred(kd_p, kd_pred_p, cd_p, model_p, def start_partial_pred(kd_p, kd_pred_p, cd_p, model_p, imposed_patch_size, gpuid, i, n): - fpath = os.path.dirname(os.path.abspath(__file__)) path, file = os.path.split(os.path.dirname(fpath)) cmd = "python {0}/syconn/handler/partial_ds_pred.py {1} {2} {3} {4}" \