Commit 21a458f1 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Configurable reprocess behaviour on non matching parsers.

parent cf2dc1e7
......@@ -219,6 +219,7 @@ console_log_level = logging.WARNING
max_upload_size = 32 * (1024 ** 3)
raw_file_strip_cutoff = 1000
use_empty_parsers = False
reprocess_unmatched = True
springer_db_relative_path = 'normalizing/data/SM_all08.db'
......
......@@ -189,7 +189,8 @@ class Calc(Proc):
records.
"""
parser = match_parser(self.mainfile, self.upload_files, strict=False)
if parser is None:
if parser is None and not config.reprocess_unmatched:
# Remove the logsfile and set a fake logwriter to avoid creating a log file,
# because we will probably remove this calc and don't want to have ghost logfiles.
if self._calc_proc_logwriter_ctx is not None:
......@@ -199,7 +200,9 @@ class Calc(Proc):
self._calc_proc_logwriter_ctx = open('/dev/null', 'wt')
self._calc_proc_logwriter = self._calc_proc_logwriter_ctx.__enter__()
self.get_logger().error(
'no parser matches during re-process, use the old parser')
'no parser matches during re-process, will not re-process this calc')
self.errors = ['no parser matches during re-process, will not re-process this calc']
# mock the steps of actual processing
self._continue_with('parsing')
......@@ -208,11 +211,15 @@ class Calc(Proc):
self._complete()
return
logger = self.get_logger(parser=parser.name)
logger = self.get_logger()
if parser is None:
self.get_logger().error('no parser matches during re-process, use the old parser')
self.errors = ['no matching parser found during re-processing']
if self.parser != parser.name:
self.parser = parser.name
logger.info(
'different parser matches during re-process, use new parser')
'different parser matches during re-process, use new parser',
parser=parser.name)
try:
calc_with_metadata = datamodel.CalcWithMetadata(**self.metadata)
......
......@@ -11,6 +11,7 @@ data:
nomad.yaml: |
release: "{{ .Release.Name }}"
domain: "{{ .Values.domain }}"
reprocess_unmatched: {{ .Values.reprocess_unmatched }}
fs:
tmp: "{{ .Values.volumes.tmp }}"
prefix_size: {{ .Values.volumes.prefixSize }}
......
......@@ -154,6 +154,10 @@ domain: DFT
springerDbPath: /nomad/fairdi/db/data/springer.db
# Will reprocess calculations with their old matched parser, even if they do not
# match this parser anymore
reprocess_unmatched: true
datacite:
enabled: false
prefix: "10.17172"
......@@ -208,6 +208,9 @@ def test_process_non_existing(proc_infra, test_user, with_error):
@pytest.mark.timeout(config.tests.default_timeout)
@pytest.mark.parametrize('with_failure', [None, 'before', 'after', 'not-matched'])
def test_re_processing(published: Upload, example_user_metadata, monkeypatch, with_failure):
if with_failure == 'not-matched':
monkeypatch.setattr('nomad.config.reprocess_unmatched', False)
if with_failure == 'before':
calc = published.all_calcs(0, 1).first()
calc.tasks_status = FAILURE
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment