Configurable reprocess behaviour on non matching parsers.

nomad/config.py

@@ -219,6 +219,7 @@ console_log_level = logging.WARNING
 max_upload_size = 32 * (1024 ** 3)
 raw_file_strip_cutoff = 1000
 use_empty_parsers = False
+reprocess_unmatched = True
 
 
 springer_db_relative_path = 'normalizing/data/SM_all08.db'

nomad/processing/data.py

@@ -189,7 +189,8 @@ class Calc(Proc):
         records.
         """
         parser = match_parser(self.mainfile, self.upload_files, strict=False)
-        if parser is None:
+
+        if parser is None and not config.reprocess_unmatched:
             # Remove the logsfile and set a fake logwriter to avoid creating a log file,
             # because we will probably remove this calc and don't want to have ghost logfiles.
             if self._calc_proc_logwriter_ctx is not None:
@@ -199,7 +200,9 @@ class Calc(Proc):
             self._calc_proc_logwriter_ctx = open('/dev/null', 'wt')
             self._calc_proc_logwriter = self._calc_proc_logwriter_ctx.__enter__()
             self.get_logger().error(
-                'no parser matches during re-process, use the old parser')
+                'no parser matches during re-process, will not re-process this calc')
+
+            self.errors = ['no parser matches during re-process, will not re-process this calc']
 
             # mock the steps of actual processing
             self._continue_with('parsing')
@@ -208,11 +211,15 @@ class Calc(Proc):
             self._complete()
             return
 
-        logger = self.get_logger(parser=parser.name)
+        logger = self.get_logger()
+        if parser is None:
+            self.get_logger().error('no parser matches during re-process, use the old parser')
+            self.errors = ['no matching parser found during re-processing']
         if self.parser != parser.name:
             self.parser = parser.name
             logger.info(
-                'different parser matches during re-process, use new parser')
+                'different parser matches during re-process, use new parser',
+                parser=parser.name)
 
         try:
             calc_with_metadata = datamodel.CalcWithMetadata(**self.metadata)

ops/helm/nomad/templates/nomad-configmap.yml

@@ -11,6 +11,7 @@ data:
   nomad.yaml: |
     release: "{{ .Release.Name }}"
     domain: "{{ .Values.domain }}"
+    reprocess_unmatched: {{ .Values.reprocess_unmatched }}
     fs:
       tmp: "{{ .Values.volumes.tmp }}"
       prefix_size: {{ .Values.volumes.prefixSize }}

ops/helm/nomad/values.yaml

@@ -154,6 +154,10 @@ domain: DFT
 
 springerDbPath: /nomad/fairdi/db/data/springer.db
 
+# Will reprocess calculations with their old matched parser, even if they do not
+# match this parser anymore
+reprocess_unmatched: true
+
 datacite:
   enabled: false
   prefix: "10.17172"

tests/processing/test_data.py

@@ -208,6 +208,9 @@ def test_process_non_existing(proc_infra, test_user, with_error):
 @pytest.mark.timeout(config.tests.default_timeout)
 @pytest.mark.parametrize('with_failure', [None, 'before', 'after', 'not-matched'])
 def test_re_processing(published: Upload, example_user_metadata, monkeypatch, with_failure):
+    if with_failure == 'not-matched':
+        monkeypatch.setattr('nomad.config.reprocess_unmatched', False)
+
     if with_failure == 'before':
         calc = published.all_calcs(0, 1).first()
         calc.tasks_status = FAILURE