Commit dcc2291a authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Added mirroring of staging files to mirror CLI.

parent 44cd0656
Pipeline #67612 passed with stages
in 13 minutes and 59 seconds
......@@ -156,9 +156,17 @@ class Mapping:
'--migration', type=str, default=None,
help='The name of a migration script used to transform the metadata.'
)
@click.option(
'--staging', is_flag=True, default=False,
help='Mirror non published uploads. Only works with --move or --link.'
)
def mirror(
query, move: bool, link: bool, dry: bool, files_only: bool, source_mapping: str,
target_mapping: str, migration: str):
target_mapping: str, migration: str, staging: bool):
if staging and not (move or link):
print('--with-staging requires either --move or --link')
sys.exit(1)
migration_func = None
if migration is not None:
......@@ -181,7 +189,10 @@ def mirror(
print('Cannot parse the given query %s: %s' % (query, str(e)))
sys.exit(1)
else:
query = dict(published=True)
if staging:
query = dict(published=False)
else:
query = dict(published=True)
utils.configure_logging()
......@@ -249,7 +260,8 @@ def mirror(
upload_files_path = source_mapping_obj.apply(upload_files_path)
target_upload_files_path = files.PathObject(
config.fs.public, upload_id, create_prefix=False, prefix=True).os_path
config.fs.public if not staging else config.fs.staging,
upload_id, create_prefix=False, prefix=True).os_path
target_upload_files_path = target_mapping_obj.apply(target_upload_files_path)
if not os.path.exists(target_upload_files_path):
......
......@@ -251,6 +251,32 @@ class TestClient:
assert json.dumps(new_search_results[key]) == json.dumps(ref_search_results[key])
published.upload_files.exists
proc.Upload.objects(upload_id=published.upload_id).first().upload_files.exists
def test_mirror_staging(self, non_empty_processed, admin_user_bravado_client, monkeypatch):
ref_search_results = search.SearchRequest().search_parameters(upload_id=non_empty_processed.upload_id).execute_paginated()['results'][0]
monkeypatch.setattr('nomad.cli.client.mirror.__in_test', True)
result = click.testing.CliRunner().invoke(
cli, ['client', 'mirror', '--staging', '--link'], catch_exceptions=False, obj=utils.POPO())
assert result.exit_code == 0
assert non_empty_processed.upload_id in result.output
assert non_empty_processed.upload_files.os_path in result.output
assert proc.Upload.objects(upload_id=non_empty_processed.upload_id).count() == 1
assert proc.Calc.objects(upload_id=non_empty_processed.upload_id).count() == 1
new_search = search.SearchRequest().search_parameters(upload_id=non_empty_processed.upload_id).execute_paginated()
calcs_in_search = new_search['pagination']['total']
assert calcs_in_search == 1
new_search_results = new_search['results'][0]
for key in new_search_results.keys():
if key not in ['upload_time', 'last_processing']: # There is a sub second change due to date conversions (?)
assert json.dumps(new_search_results[key]) == json.dumps(ref_search_results[key])
non_empty_processed.upload_files.exists
proc.Upload.objects(upload_id=non_empty_processed.upload_id).first().upload_files.exists
def test_mirror_files_only(self, published, admin_user_bravado_client, monkeypatch):
monkeypatch.setattr('nomad.cli.client.mirror.__in_test', True)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment