Commit e8f4318e authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Added progress dots to CLI processes. Fixes #670

parent ede0e416
......@@ -24,8 +24,9 @@ from nomad import config
from .admin import admin
def _run_parallel(uploads, parallel: int, callable, label: str):
def _run_parallel(uploads, parallel: int, callable, label: str, print_progress: int = 0):
import threading
import time
from nomad import utils, processing as proc
......@@ -82,13 +83,23 @@ def _run_parallel(uploads, parallel: int, callable, label: str):
threads.append(thread)
thread.start()
def print_progress_lines():
while True:
time.sleep(print_progress)
print('.', flush=True)
if print_progress > 0:
progress_thread = threading.Thread(target=print_progress_lines)
progress_thread.daemon = True
progress_thread.start()
for thread in threads:
thread.join()
def _run_processing(
uploads, parallel: int, process, label: str, process_running: bool = False,
wait_until_complete: bool = True, reset_first: bool = False):
wait_until_complete: bool = True, reset_first: bool = False, **kwargs):
from nomad import processing as proc
......@@ -121,7 +132,7 @@ def _run_processing(
logger.info('%s complete' % label, upload_id=upload.upload_id)
return True
_run_parallel(uploads, parallel=parallel, callable=run_process, label=label)
_run_parallel(uploads, parallel=parallel, callable=run_process, label=label, **kwargs)
@admin.group(help='Upload related commands')
......@@ -435,8 +446,9 @@ def rm(ctx, uploads, skip_es, skip_mongo, skip_files):
@click.option('--parallel', default=1, type=int, help='Use the given amount of parallel processes. Default is 1.')
@click.option('--process-running', is_flag=True, help='Also reprocess already running processes.')
@click.option('--setting', type=str, multiple=True, help='key=value to overwrite a default reprocess config setting.')
@click.option('--print-progress', default=0, type=int, help='Prints a dot every given seconds. Can be used to keep terminal open that have an i/o-based timeout.')
@click.pass_context
def process(ctx, uploads, parallel: int, process_running: bool, setting=typing.List[str]):
def process(ctx, uploads, parallel: int, process_running: bool, setting: typing.List[str], print_progress: int):
_, uploads = _query_uploads(uploads, **ctx.obj.uploads_kwargs)
settings: typing.Dict[str, bool] = {}
for settings_str in setting:
......@@ -444,7 +456,7 @@ def process(ctx, uploads, parallel: int, process_running: bool, setting=typing.L
settings[key] = bool(value)
_run_processing(
uploads, parallel, lambda upload: upload.process_upload(reprocess_settings=settings),
'processing', process_running=process_running, reset_first=True)
'processing', process_running=process_running, reset_first=True, print_progress=print_progress)
@uploads.command(help='Repack selected uploads.')
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment