Merge branch 'v0.9.11-hotfix' into v0.9.11

nomad/processing/data.py

@@ -548,21 +548,14 @@ class Calc(Proc):
         # metadata file name defined in nomad.config nomad_metadata.yaml/json
         # which can be placed in the directory containing the mainfile or somewhere up
         # highest priority is directory with mainfile
-
         metadata_file = config.metadata_file_name
         metadata_dir = os.path.dirname(self.mainfile_file.os_path)
+        upload_raw_dir = self.upload_files._raw_dir.os_path
 
         metadata = {}
+        metadata_part = None
+        # apply the nomad files of the current directory and parent directories
         while True:
-            # top-level nomad file can also contain an entries dict with entry
-            # metadata per mainfile as key
-            if metadata_dir == self.upload_files.os_path:
-                entries = metadata_part.get('entries', {})
-                metadata_part = entries.get(self.mainfile, {})
-                for key, val in metadata_part.items():
-                    metadata.setdefault(key, val)
-
-            # consider the nomad file of the current directory
             metadata_part = self.upload.metadata_file_cached(
                 os.path.join(metadata_dir, metadata_file))
             for key, val in metadata_part.items():
@@ -570,11 +563,18 @@ class Calc(Proc):
                     continue
                 metadata.setdefault(key, val)
 
-            if metadata_dir == self.upload_files.os_path:
+            if metadata_dir == upload_raw_dir:
                 break
 
             metadata_dir = os.path.dirname(metadata_dir)
 
+        # Top-level nomad file can also contain an entries dict with entry
+        # metadata per mainfile as key. This takes precedence of the other files.
+        entries = metadata_part.get('entries', {})
+        metadata_part = entries.get(self.mainfile, {})
+        for key, val in metadata_part.items():
+            metadata[key] = val
+
         if len(metadata) > 0:
             logger.info('Apply user metadata from nomad.yaml/json file')
 
@@ -614,7 +614,10 @@ class Calc(Proc):
         if self.upload.publish_directly:
             self._entry_metadata.published |= True
 
-        self._read_metadata_from_file(logger)
+        try:
+            self._read_metadata_from_file(logger)
+        except Exception as e:
+            logger.error('could not process user metadata in nomad.yaml/json file', exc_info=e)
 
         # persist the calc metadata
         with utils.timer(logger, 'saved calc metadata', step='metadata'):
@@ -1089,7 +1092,7 @@ class Upload(Proc):
         if self.from_oasis:
             # we might need to add datasets from the oasis before processing and
             # adding the entries
-            oasis_metadata_file = os.path.join(self.upload_files.os_path, 'raw', config.metadata_file_name + '.json')
+            oasis_metadata_file = os.path.join(self.upload_files._raw_dir.os_path, config.metadata_file_name + '.json')
             with open(oasis_metadata_file, 'rt') as f:
                 oasis_metadata = json.load(f)
             oasis_datasets = oasis_metadata.get('oasis_datasets', {})

tests/processing/test_data.py

@@ -589,16 +589,18 @@ def test_read_metadata_from_file(proc_infra, test_user, other_test_user):
     calcs = Calc.objects(upload_id=upload.upload_id)
     calcs = sorted(calcs, key=lambda calc: calc.mainfile)
 
-    comment = ['Calculation 1 of 2', 'Calculation 2 of 2', None]
-    with_embargo = [True, False, True]
-    references = [['http://test'], ['http://ttest'], None]
-    coauthors = [[other_test_user], [], []]
+    comment = ['Calculation 1 of 3', 'Calculation 2 of 3', 'Calculation 3 of 3', None]
+    external_ids = ['external_id_1', 'external_id_2', 'external_id_3', None]
+    with_embargo = [True, False, False, True]
+    references = [['http://test'], ['http://ttest'], ['http://ttest'], None]
+    coauthors = [[other_test_user], [], [], []]
 
     for i in range(len(calcs)):
         entry_metadata = calcs[i].entry_metadata(upload.upload_files)
         assert entry_metadata.comment == comment[i]
         assert entry_metadata.with_embargo == with_embargo[i]
         assert entry_metadata.references == references[i]
+        assert entry_metadata.external_id == external_ids[i]
         entry_coauthors = [a.m_proxy_resolve() for a in entry_metadata.coauthors]
         for j in range(len(entry_coauthors)):
             assert entry_coauthors[j].user_id == coauthors[i][j].user_id
@@ -626,5 +628,6 @@ def test_read_adminmetadata_from_file(proc_infra, test_user, other_test_user, ad
 
     upload = run_processing(
         ('test_upload', 'tests/data/proc/examples_with_adminmetadata.zip'), user)
+
     calc = Calc.objects(upload_id=upload.upload_id).first()
     assert calc.metadata['uploader'] == uploader.user_id

tests/test_search.py

@@ -337,23 +337,3 @@ def assert_search_upload(
 
             for coauthor in hit.get('coauthors', []):
                 assert coauthor.get('name', None) is not None
-
-
-if __name__ == '__main__':
-    from .test_datamodel import generate_calc
-    from elasticsearch.helpers import bulk
-    import sys
-    print('Generate index with random example calculation data. First arg is number of items')
-    infrastructure.setup_mongo()
-    infrastructure.setup_elastic()
-    n = 100
-    if len(sys.argv) > 1:
-        n = int(sys.argv[1])
-
-    def gen_data():
-        for pid in range(0, n):
-            calc = generate_calc(pid)
-            calc = entry_document.from_entry_metadata(calc)
-            yield calc.to_dict(include_meta=True)
-
-    bulk(infrastructure.elastic_client, gen_data())