Commit 63b8306f authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Merge branch 'bugfixes' into 'v0.10.1'

Fixed broken nomad.yaml processing.

See merge request !303
parents 688c242b 92cb73d9
Pipeline #97842 passed with stages
in 25 minutes and 39 seconds
......@@ -578,21 +578,14 @@ class Calc(Proc):
# metadata file name defined in nomad.config nomad_metadata.yaml/json
# which can be placed in the directory containing the mainfile or somewhere up
# highest priority is directory with mainfile
metadata_file = config.metadata_file_name
metadata_dir = os.path.dirname(self.mainfile_file.os_path)
upload_raw_dir = self.upload_files._raw_dir.os_path
metadata = {}
metadata_part = None
# apply the nomad files of the current directory and parent directories
while True:
# top-level nomad file can also contain an entries dict with entry
# metadata per mainfile as key
if metadata_dir == self.upload_files.os_path:
entries = metadata_part.get('entries', {})
metadata_part = entries.get(self.mainfile, {})
for key, val in metadata_part.items():
metadata.setdefault(key, val)
# consider the nomad file of the current directory
metadata_part = self.upload.metadata_file_cached(
os.path.join(metadata_dir, metadata_file))
for key, val in metadata_part.items():
......@@ -600,11 +593,18 @@ class Calc(Proc):
continue
metadata.setdefault(key, val)
if metadata_dir == self.upload_files.os_path:
if metadata_dir == upload_raw_dir:
break
metadata_dir = os.path.dirname(metadata_dir)
# Top-level nomad file can also contain an entries dict with entry
# metadata per mainfile as key. This takes precedence of the other files.
entries = metadata_part.get('entries', {})
metadata_part = entries.get(self.mainfile, {})
for key, val in metadata_part.items():
metadata[key] = val
if len(metadata) > 0:
logger.info('Apply user metadata from nomad.yaml/json file')
......@@ -644,7 +644,10 @@ class Calc(Proc):
if self.upload.publish_directly:
self._entry_metadata.published |= True
self._read_metadata_from_file(logger)
try:
self._read_metadata_from_file(logger)
except Exception as e:
logger.error('could not process user metadata in nomad.yaml/json file', exc_info=e)
# persist the calc metadata
with utils.timer(logger, 'calc metadata saved'):
......@@ -1108,7 +1111,7 @@ class Upload(Proc):
if self.from_oasis:
# we might need to add datasets from the oasis before processing and
# adding the entries
oasis_metadata_file = os.path.join(self.upload_files.os_path, 'raw', config.metadata_file_name + '.json')
oasis_metadata_file = os.path.join(self.upload_files._raw_dir.os_path, config.metadata_file_name + '.json')
with open(oasis_metadata_file, 'rt') as f:
oasis_metadata = json.load(f)
oasis_datasets = oasis_metadata.get('oasis_datasets', {})
......
......@@ -594,16 +594,18 @@ def test_read_metadata_from_file(proc_infra, test_user, other_test_user):
calcs = Calc.objects(upload_id=upload.upload_id)
calcs = sorted(calcs, key=lambda calc: calc.mainfile)
comment = ['Calculation 1 of 2', 'Calculation 2 of 2', None]
with_embargo = [True, False, True]
references = [['http://test'], ['http://ttest'], None]
coauthors = [[other_test_user], [], []]
comment = ['Calculation 1 of 3', 'Calculation 2 of 3', 'Calculation 3 of 3', None]
external_ids = ['external_id_1', 'external_id_2', 'external_id_3', None]
with_embargo = [True, False, False, True]
references = [['http://test'], ['http://ttest'], ['http://ttest'], None]
coauthors = [[other_test_user], [], [], []]
for i in range(len(calcs)):
entry_metadata = calcs[i].entry_metadata(upload.upload_files)
assert entry_metadata.comment == comment[i]
assert entry_metadata.with_embargo == with_embargo[i]
assert entry_metadata.references == references[i]
assert entry_metadata.external_id == external_ids[i]
entry_coauthors = [a.m_proxy_resolve() for a in entry_metadata.coauthors]
for j in range(len(entry_coauthors)):
assert entry_coauthors[j].user_id == coauthors[i][j].user_id
......@@ -631,5 +633,6 @@ def test_read_adminmetadata_from_file(proc_infra, test_user, other_test_user, ad
upload = run_processing(
('test_upload', 'tests/data/proc/examples_with_adminmetadata.zip'), user)
calc = Calc.objects(upload_id=upload.upload_id).first()
assert calc.metadata['uploader'] == uploader.user_id
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment