Commit d5ab8f51 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Added wait parameter to migration upload. [skip ci]

parent 1af7d996
Pipeline #47462 skipped
......@@ -215,6 +215,7 @@ def pid_prefix(prefix: int):
@click.option('--parallel', default=1, type=int, help='Use the given amount of parallel processes. Default is 1.')
@click.option('--create-packages', is_flag=True, help='Indicate that packages should be created, if they do not already exist.')
@click.option('--republish', is_flag=True, help='Will only republish already published packages.')
@click.option('--wait', default=0, type=int, help='Wait for a random (upto given) number of seconds before each upload to scatter io and compute heavy processing tasks.')
def upload(
upload_paths: list, pattern: str, parallel: int, delete_failed: str,
create_packages: bool, republish: bool):
......
......@@ -39,6 +39,7 @@ import threading
from contextlib import contextmanager
import shutil
import json
import random
from nomad import utils, infrastructure, files, config
from nomad.coe_repo import User, Calc, LoginException
......@@ -728,7 +729,8 @@ class NomadCOEMigration:
def migrate(
self, *args, delete_failed: str = '',
create_packages: bool = False, only_republish: bool = False) -> utils.POPO:
create_packages: bool = False, only_republish: bool = False,
wait: int = 0) -> utils.POPO:
"""
Migrate the given uploads.
......@@ -753,6 +755,8 @@ class NomadCOEMigration:
do not exists.
only_republish: If the package exists and is published, it will be republished.
Nothing else. Useful to reindex/recreate coe repo, etc.
offset: Will add a random sleep before migrating each package between 0 and
``wait`` seconds.
Returns: Dictionary with statistics on the migration.
"""
......@@ -769,6 +773,10 @@ class NomadCOEMigration:
logger = self.logger.bind(
package_id=package.package_id, source_upload_id=package.upload_id)
if wait > 0:
self.logger.info('wait for a random amount of time')
time.sleep(random.randint(0, wait))
if package.migration_version is not None and package.migration_version >= self.migration_version:
if only_republish:
self.republish_package(package)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment