Commit 012ed967 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Merge branch 'v0.9.11-hotfix' into v0.9.11

parents 0384e472 f5fe1aaa
Pipeline #99793 skipped with stage
......@@ -548,21 +548,14 @@ class Calc(Proc):
# metadata file name defined in nomad.config nomad_metadata.yaml/json
# which can be placed in the directory containing the mainfile or somewhere up
# highest priority is directory with mainfile
metadata_file = config.metadata_file_name
metadata_dir = os.path.dirname(self.mainfile_file.os_path)
upload_raw_dir = self.upload_files._raw_dir.os_path
metadata = {}
metadata_part = None
# apply the nomad files of the current directory and parent directories
while True:
# top-level nomad file can also contain an entries dict with entry
# metadata per mainfile as key
if metadata_dir == self.upload_files.os_path:
entries = metadata_part.get('entries', {})
metadata_part = entries.get(self.mainfile, {})
for key, val in metadata_part.items():
metadata.setdefault(key, val)
# consider the nomad file of the current directory
metadata_part = self.upload.metadata_file_cached(
os.path.join(metadata_dir, metadata_file))
for key, val in metadata_part.items():
......@@ -570,11 +563,18 @@ class Calc(Proc):
continue
metadata.setdefault(key, val)
if metadata_dir == self.upload_files.os_path:
if metadata_dir == upload_raw_dir:
break
metadata_dir = os.path.dirname(metadata_dir)
# Top-level nomad file can also contain an entries dict with entry
# metadata per mainfile as key. This takes precedence of the other files.
entries = metadata_part.get('entries', {})
metadata_part = entries.get(self.mainfile, {})
for key, val in metadata_part.items():
metadata[key] = val
if len(metadata) > 0:
logger.info('Apply user metadata from nomad.yaml/json file')
......@@ -614,7 +614,10 @@ class Calc(Proc):
if self.upload.publish_directly:
self._entry_metadata.published |= True
self._read_metadata_from_file(logger)
try:
self._read_metadata_from_file(logger)
except Exception as e:
logger.error('could not process user metadata in nomad.yaml/json file', exc_info=e)
# persist the calc metadata
with utils.timer(logger, 'saved calc metadata', step='metadata'):
......@@ -1089,7 +1092,7 @@ class Upload(Proc):
if self.from_oasis:
# we might need to add datasets from the oasis before processing and
# adding the entries
oasis_metadata_file = os.path.join(self.upload_files.os_path, 'raw', config.metadata_file_name + '.json')
oasis_metadata_file = os.path.join(self.upload_files._raw_dir.os_path, config.metadata_file_name + '.json')
with open(oasis_metadata_file, 'rt') as f:
oasis_metadata = json.load(f)
oasis_datasets = oasis_metadata.get('oasis_datasets', {})
......
......@@ -589,16 +589,18 @@ def test_read_metadata_from_file(proc_infra, test_user, other_test_user):
calcs = Calc.objects(upload_id=upload.upload_id)
calcs = sorted(calcs, key=lambda calc: calc.mainfile)
comment = ['Calculation 1 of 2', 'Calculation 2 of 2', None]
with_embargo = [True, False, True]
references = [['http://test'], ['http://ttest'], None]
coauthors = [[other_test_user], [], []]
comment = ['Calculation 1 of 3', 'Calculation 2 of 3', 'Calculation 3 of 3', None]
external_ids = ['external_id_1', 'external_id_2', 'external_id_3', None]
with_embargo = [True, False, False, True]
references = [['http://test'], ['http://ttest'], ['http://ttest'], None]
coauthors = [[other_test_user], [], [], []]
for i in range(len(calcs)):
entry_metadata = calcs[i].entry_metadata(upload.upload_files)
assert entry_metadata.comment == comment[i]
assert entry_metadata.with_embargo == with_embargo[i]
assert entry_metadata.references == references[i]
assert entry_metadata.external_id == external_ids[i]
entry_coauthors = [a.m_proxy_resolve() for a in entry_metadata.coauthors]
for j in range(len(entry_coauthors)):
assert entry_coauthors[j].user_id == coauthors[i][j].user_id
......@@ -626,5 +628,6 @@ def test_read_adminmetadata_from_file(proc_infra, test_user, other_test_user, ad
upload = run_processing(
('test_upload', 'tests/data/proc/examples_with_adminmetadata.zip'), user)
calc = Calc.objects(upload_id=upload.upload_id).first()
assert calc.metadata['uploader'] == uploader.user_id
......@@ -337,23 +337,3 @@ def assert_search_upload(
for coauthor in hit.get('coauthors', []):
assert coauthor.get('name', None) is not None
if __name__ == '__main__':
from .test_datamodel import generate_calc
from elasticsearch.helpers import bulk
import sys
print('Generate index with random example calculation data. First arg is number of items')
infrastructure.setup_mongo()
infrastructure.setup_elastic()
n = 100
if len(sys.argv) > 1:
n = int(sys.argv[1])
def gen_data():
for pid in range(0, n):
calc = generate_calc(pid)
calc = entry_document.from_entry_metadata(calc)
yield calc.to_dict(include_meta=True)
bulk(infrastructure.elastic_client, gen_data())
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment