Store backend if available on calc proc failure. More tests.

nomad/datamodel/datamodel.py

@@ -463,7 +463,8 @@ class EntryMetadata(metainfo.MSection):
         if domain_section is None:
             domain_section = self.m_create(domain_section_def.section_cls)
 
-        domain_section.apply_domain_metadata(backend)
+        if backend is not None:
+            domain_section.apply_domain_metadata(backend)
 
 
 class EntryArchive(metainfo.MSection):

nomad/datamodel/dft.py

@@ -169,7 +169,7 @@ class DFTMetadata(MSection):
         a_search=Search(default_statistic=True))
 
     spacegroup = Quantity(
-        type=int, default='not processed',
+        type=int, default=-1,
         description='The spacegroup of the simulated system as number.',
         a_search=Search())
 

nomad/processing/data.py

@@ -328,6 +328,11 @@ class Calc(Proc):
 
             self._entry_metadata.processed = False
             self.apply_entry_metadata(self._entry_metadata)
+            if self._parser_backend and self._parser_backend.resource:
+                backend = self._parser_backend
+            else:
+                backend = None
+            self._entry_metadata.apply_domain_metadata(backend)
             self._entry_metadata.a_elastic.index()
         except Exception as e:
             self.get_logger().error(

tests/processing/test_data.py

@@ -123,6 +123,12 @@ def test_processing(processed, no_warn, mails, monkeypatch):
     assert re.search(r'Processing completed', mails.messages[0].data.decode('utf-8')) is not None
 
 
+def test_processing_two_runs(test_user, proc_infra):
+    processed = run_processing(
+        ('test_upload_id', 'tests/data/proc/examples_template_tworuns.zip',), test_user)
+    assert_processing(processed)
+
+
 def test_processing_with_large_dir(test_user, proc_infra):
     upload_path = 'tests/data/proc/examples_large_dir.zip'
     upload_id = upload_path[:-4]
@@ -207,6 +213,7 @@ def test_process_non_existing(proc_infra, test_user, with_error):
     assert upload.tasks_status == FAILURE
     assert len(upload.errors) > 0
 
+
 @pytest.mark.timeout(config.tests.default_timeout)
 @pytest.mark.parametrize('with_failure', [None, 'before', 'after', 'not-matched'])
 def test_re_processing(published: Upload, internal_example_user_metadata, monkeypatch, with_failure):
@@ -369,6 +376,7 @@ def mock_failure(cls, task, monkeypatch):
 @pytest.mark.parametrize('task', ['extracting', 'parse_all', 'cleanup', 'parsing'])
 @pytest.mark.timeout(config.tests.default_timeout)
 def test_task_failure(monkeypatch, uploaded, task, proc_infra, test_user, with_error):
+    upload_id, _ = uploaded
     # mock the task method to through exceptions
     if hasattr(Upload, task):
         cls = Upload
@@ -400,6 +408,15 @@ def test_task_failure(monkeypatch, uploaded, task, proc_infra, test_user, with_e
                 assert calc.current_task == 'parsing'
                 assert len(calc.errors) > 0
 
+    calc = Calc.objects(upload_id=upload_id).first()
+    if calc is not None:
+        with upload.upload_files.read_archive(calc.calc_id) as archive:
+            calc_archive = archive[calc.calc_id]
+            assert 'section_metadata' in calc_archive
+            assert 'processing_logs' in calc_archive
+            if task != 'parsing':
+                assert 'section_run' in calc_archive
+
 # TODO timeout
 # consume_ram, segfault, and exit are not testable with the celery test worker
 @pytest.mark.parametrize('failure', ['exception'])