From 377c55a4a15a5b36c2f3cfa44bb4c860aa8a10ca Mon Sep 17 00:00:00 2001
From: Markus Scheidgen <markus.scheidgen@gmail.com>
Date: Wed, 27 Jan 2021 15:52:44 +0100
Subject: [PATCH] Added processing errors to elastic search.
---
nomad/datamodel/datamodel.py | 4 ++++
nomad/processing/data.py | 7 +++++++
tests/processing/test_data.py | 3 +++
3 files changed, 14 insertions(+)
diff --git a/nomad/datamodel/datamodel.py b/nomad/datamodel/datamodel.py
index 4ae56e688c..85fd2a51cd 100644
--- a/nomad/datamodel/datamodel.py
+++ b/nomad/datamodel/datamodel.py
@@ -415,6 +415,10 @@ class EntryMetadata(metainfo.MSection):
categories=[MongoMetadata],
a_search=Search())
+ processing_errors = metainfo.Quantity(
+ type=str, shape=['*'], description='Errors that occured during processing',
+ a_search=Search(many_and='append'))
+
nomad_version = metainfo.Quantity(
type=str,
description='The NOMAD version used for the last processing',
diff --git a/nomad/processing/data.py b/nomad/processing/data.py
index 8e552937ee..c3e93155e5 100644
--- a/nomad/processing/data.py
+++ b/nomad/processing/data.py
@@ -268,6 +268,11 @@ class Calc(Proc):
dump_dict.update(level=method_name.upper())
self._calc_proc_logs.append(dump_dict)
+ if method_name == 'error':
+ error = event_dict.get('event', None)
+ if error is not None:
+ self._entry_metadata.processing_errors.append(error)
+
except Exception:
# Exceptions here will cause indefinite loop
pass
@@ -333,6 +338,7 @@ class Calc(Proc):
self._entry_metadata.nomad_version = config.meta.version
self._entry_metadata.nomad_commit = config.meta.commit
self._entry_metadata.last_processing = datetime.utcnow()
+ self._entry_metadata.processing_errors = []
self._entry_metadata.files = self.upload_files.calc_files(self.mainfile)
self.parsing()
@@ -351,6 +357,7 @@ class Calc(Proc):
self._entry_metadata = self.create_metadata()
self._entry_metadata.calc_hash = self.upload_files.calc_hash(self.mainfile)
self._entry_metadata.last_processing = datetime.utcnow()
+ self._entry_metadata.processing_errors = []
self._entry_metadata.files = self.upload_files.calc_files(self.mainfile)
self._entry_metadata.parser_name = self.parser
diff --git a/tests/processing/test_data.py b/tests/processing/test_data.py
index 6b7e4b59bc..6c84433955 100644
--- a/tests/processing/test_data.py
+++ b/tests/processing/test_data.py
@@ -121,6 +121,7 @@ def assert_processing(upload: Upload, published: bool = False):
# check some domain metadata
assert entry_metadata.n_atoms > 0
assert len(entry_metadata.atoms) > 0
+ assert len(entry_metadata.processing_errors) == 0
assert upload.get_calc(calc.calc_id) is not None
@@ -532,6 +533,8 @@ def test_task_failure(monkeypatch, uploaded, task, proc_infra, test_user, with_e
assert 'section_metadata' in calc_archive
assert calc_archive['section_metadata']['dft']['code_name'] not in [
config.services.unavailable_value, config.services.not_processed_value]
+ if task != 'cleanup':
+ assert len(calc_archive['section_metadata']['processing_errors']) > 0
assert 'processing_logs' in calc_archive
if task != 'parsing':
assert 'section_run' in calc_archive
--
GitLab