Commit df4515a1 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Merge branch 'v0.10.11_hotfixes' into 'v0.10.11'

V0.10.11 hotfixes

See merge request !482
parents b91b2e5c 56a75c7b
Pipeline #117732 passed with stages
in 26 minutes and 40 seconds
......@@ -405,8 +405,11 @@ def re_pack(ctx, uploads, parallel: int):
@click.argument('UPLOADS', nargs=-1)
@click.option('--dry', is_flag=True, help='Just check, do nothing.')
@click.option('-f', '--force', is_flag=True, help='Ignore warnings and perform the operation regardless.')
@click.option('-q', '--quiet', is_flag=True, help='No output (only logs).')
@click.option('--upload-ids', is_flag=True, help='Print uploads with errors.')
@click.option('--label', type=str, help='A label to label log entries with.')
@click.pass_context
def prepare_migration(ctx, uploads, dry, force):
def prepare_migration(ctx, uploads, dry, force, label, quiet, upload_ids):
'''
Removes one of the raw files, either public or restricted depending on the embargo.
Files that need to be removed are saved as `quarantined` in the upload folder.
......@@ -415,12 +418,24 @@ def prepare_migration(ctx, uploads, dry, force):
import os.path
import os
logger = utils.get_logger(__name__)
if label:
logger = logger.bind(label=label)
_, uploads = query_uploads(ctx, uploads)
for upload in uploads:
print(f'Preparing {upload.upload_id} for migration ...')
def log_event(event: str, error: bool = False, **kwargs):
if not quiet:
print(f' {"!!! " if error else ""}{event}', *[value for value in kwargs.values()])
method = getattr(logger, 'error' if error else 'info')
method(event, upload_id=upload.upload_id, **kwargs)
if not quiet:
print(f'Preparing {upload.upload_id} for migration ...')
if not upload.published:
print(' upload is not published, nothing to do')
log_event('upload is not published, nothing to do')
break
with_embargo_values: typing.List[bool] = []
......@@ -431,7 +446,15 @@ def prepare_migration(ctx, uploads, dry, force):
with_embargo_values.append(with_embargo_value)
if len(with_embargo_values) > 1:
print(' !!! inconsistent upload !!!')
if upload_ids:
print(upload.upload_id)
log_event('inconsistent upload', error=True)
break
if len(with_embargo_values) == 0:
if upload_ids:
print(upload.upload_id)
log_event('upload with no indexed entries', error=True)
break
with_embargo = with_embargo_values[0]
......@@ -445,25 +468,29 @@ def prepare_migration(ctx, uploads, dry, force):
to_stay = upload_files._raw_file_object(access)
if not to_move.exists():
print(' obsolute raw.zip was already removed', upload.upload_id, to_move.os_path)
log_event('obsolute raw.zip was already removed', path=to_move.os_path)
elif to_stay.size < to_move.size and not force:
print(' !!! likely inconsistent pack !!!')
if upload_ids:
print(upload.upload_id)
log_event('likely inconsistent pack', error=True)
elif to_move.size == 22:
if not dry:
to_move.delete()
print(' removed empty zip', upload.upload_id, to_move.os_path)
log_event('removed empty zip', path=to_move.os_path)
elif with_embargo and not force:
print(' !!! embargo upload with non empty public file !!!')
if upload_ids:
print(upload.upload_id)
log_event('embargo upload with non empty public file', error=True)
else:
if not dry:
target = upload_files._raw_file_object('quarantined')
assert not target.exists()
os.rename(to_move.os_path, target.os_path)
print(' quarantined', upload.upload_id, to_move.os_path)
log_event('quarantined file', path=to_move.os_path)
@uploads.command(help='Moves certain files from public or restricted to quarantine in published uploads.')
......
......@@ -151,7 +151,7 @@ class LogstashFormatter(logstash.formatter.LogstashFormatterBase):
if record.name.startswith('nomad'):
for key, value in structlog.items():
if key in ('event', 'stack_info', 'id', 'timestamp'):
if key in ('event', 'stack_info', 'id', 'timestamp', 'path'):
continue
elif key == 'exception':
exception_trace = value.strip('\n')
......
......@@ -137,14 +137,14 @@ def transform_for_index_test(calc):
return calc
@pytest.mark.usefixtures('reset_config', 'no_warn')
@pytest.mark.usefixtures('reset_config')
class TestAdminUploads:
@pytest.mark.parametrize('codes, count', [
(['VASP'], 1),
(['doesNotExist'], 0),
(['VASP', 'doesNotExist'], 1)])
def test_uploads_code(self, published, codes, count):
def test_uploads_code(self, published, codes, count, no_warn):
codes_args = []
for code in codes:
codes_args.append('--code')
......@@ -155,7 +155,7 @@ class TestAdminUploads:
assert result.exit_code == 0
assert '%d uploads selected' % count in result.stdout
def test_query_mongo(self, published):
def test_query_mongo(self, published, no_warn):
upload_id = published.upload_id
query = dict(upload_id=upload_id)
......@@ -166,7 +166,7 @@ class TestAdminUploads:
assert result.exit_code == 0
assert '1 uploads selected' in result.stdout
def test_ls(self, published):
def test_ls(self, published, no_warn):
upload_id = published.upload_id
result = click.testing.CliRunner().invoke(
......@@ -175,7 +175,7 @@ class TestAdminUploads:
assert result.exit_code == 0
assert '1 uploads selected' in result.stdout
def test_ls_query(self, published):
def test_ls_query(self, published, no_warn):
upload_id = published.upload_id
result = click.testing.CliRunner().invoke(
......@@ -184,7 +184,7 @@ class TestAdminUploads:
assert result.exit_code == 0
assert '1 uploads selected' in result.stdout
def test_rm(self, published):
def test_rm(self, published, no_warn):
upload_id = published.upload_id
result = click.testing.CliRunner().invoke(
......@@ -195,7 +195,7 @@ class TestAdminUploads:
assert Upload.objects(upload_id=upload_id).first() is None
assert Calc.objects(upload_id=upload_id).first() is None
def test_index(self, published):
def test_index(self, published, no_warn):
upload_id = published.upload_id
calc = Calc.objects(upload_id=upload_id).first()
calc.metadata['comment'] = 'specific'
......@@ -210,7 +210,7 @@ class TestAdminUploads:
assert search.SearchRequest().search_parameters(comment='specific').execute()['total'] == 1
def test_index_with_transform(self, published):
def test_index_with_transform(self, published, no_warn):
upload_id = published.upload_id
assert search.SearchRequest().search_parameters(comment='specific').execute()['total'] == 0
......@@ -225,7 +225,7 @@ class TestAdminUploads:
assert search.SearchRequest().search_parameters(comment='specific').execute()['total'] == 1
def test_re_process(self, published, monkeypatch):
def test_re_process(self, published, monkeypatch, no_warn):
monkeypatch.setattr('nomad.config.meta.version', 'test_version')
upload_id = published.upload_id
calc = Calc.objects(upload_id=upload_id).first()
......@@ -239,7 +239,7 @@ class TestAdminUploads:
calc.reload()
assert calc.metadata['nomad_version'] == 'test_version'
def test_re_pack(self, published, monkeypatch):
def test_re_pack(self, published, monkeypatch, no_warn):
upload_id = published.upload_id
calc = Calc.objects(upload_id=upload_id).first()
assert calc.metadata['with_embargo']
......@@ -263,7 +263,7 @@ class TestAdminUploads:
published.reload()
assert published.tasks_status == SUCCESS
def test_chown(self, published, test_user, other_test_user):
def test_chown(self, published, test_user, other_test_user, no_warn):
upload_id = published.upload_id
calc = Calc.objects(upload_id=upload_id).first()
assert calc.metadata['uploader'] == other_test_user.user_id
......@@ -280,7 +280,7 @@ class TestAdminUploads:
assert upload.user_id == test_user.user_id
assert calc.metadata['uploader'] == test_user.user_id
def test_edit(self, published):
def test_edit(self, published, no_warn):
upload_id = published.upload_id
def assert_calcs(publish, with_embargo):
......@@ -318,7 +318,7 @@ class TestAdminUploads:
(False, False, False),
(True, True, False),
(False, False, True)])
def test_reset(self, non_empty_processed, with_calcs, success, failure):
def test_reset(self, non_empty_processed, with_calcs, success, failure, no_warn):
upload_id = non_empty_processed.upload_id
upload = Upload.objects(upload_id=upload_id).first()
......@@ -353,7 +353,7 @@ class TestAdminUploads:
pytest.param(['0/POTCAR', 'POTCAR.xz'], ['public'], id='root'),
pytest.param(['0/POTCAR'], ['embargo'], id='embargo')
])
def test_quarantine_raw_files(self, paths, entries, test_user, proc_infra):
def test_quarantine_raw_files(self, paths, entries, test_user, proc_infra, no_warn):
upload_path = os.path.join(config.fs.tmp, 'upload.zip')
nomad_json = {}
for i, entry in enumerate(entries):
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment