diff --git a/nomad/client/migration.py b/nomad/client/migration.py index 0bfb2053d844c2ec69bbeec9d73bd18488e1fd40..cd240c940fc617fa9457f89738e7653388ad3ad4 100644 --- a/nomad/client/migration.py +++ b/nomad/client/migration.py @@ -118,12 +118,13 @@ def determine_upload_paths(paths, pattern=None): @migration.command(help='Add an upload folder to the package index.') @click.argument('upload-paths', nargs=-1) @click.option('--pattern', default=None, type=str, help='Interpret the paths as directory and migrate those subdirectory that match the given regexp') -@click.option('--parallel', default=1, type=int, help='Use the given amount of parallel processes. Default is 1.') -def package(upload_paths, pattern, parallel): +@click.option('--parallel', default=1, type=int, help='Use the given amount of parallel processes to process uploads. Default is 1.') +@click.option('--parallel-zip', default=1, type=int, help='Use the given amount of parallel processes to pack packages. Default is 1.') +def package(upload_paths, pattern, parallel, parallel_zip): upload_paths = determine_upload_paths(upload_paths, pattern) upload_path_queue = multiprocessing.Queue(len(upload_paths)) - print('Package %d uploads with %d processes.' % (len(upload_paths), parallel)) + print('Package %d uploads with %d/%d processes.' % (len(upload_paths), parallel, parallel_zip)) for upload_path in upload_paths: upload_path_queue.put(upload_path) @@ -137,7 +138,7 @@ def package(upload_paths, pattern, parallel): try: while True: upload_path = upload_path_queue.get() - migration.package_index(upload_path, parallel=parallel) + migration.package_index(upload_path, parallel=parallel_zip) except queue.Empty: pass diff --git a/nomad/migration.py b/nomad/migration.py index c68b01aec6d67421232f5eb4843861d4aa204c9a..a1ec61b350f72482a45f70102634b5aa5db4eb91 100644 --- a/nomad/migration.py +++ b/nomad/migration.py @@ -47,7 +47,7 @@ from nomad.processing import FAILURE default_pid_prefix = 7000000 """ The default pid prefix for new non migrated calculations """ -max_package_size = 32 * 1024 * 1024 * 1024 # 16 GB +max_package_size = 32 * 1024 * 1024 * 1024 # 32 GB """ The maximum size of a package that will be used as an upload on nomad@FAIRDI """ use_stats_for_filestats_threshold = 1024