diff --git a/common/python/nomadcore/baseclasses.py b/common/python/nomadcore/baseclasses.py index 8702fe08bf1eb60fd2c825a6f548ec8243debefd..d6ab46c33aa26e01fb1f5f0ce2fff0607cc16b10 100644 --- a/common/python/nomadcore/baseclasses.py +++ b/common/python/nomadcore/baseclasses.py @@ -1,5 +1,5 @@ """ -This module contains the base classes that help in building parsers for the +This module contains base classes that might help in building parsers for the NoMaD project. """ from builtins import str @@ -435,7 +435,7 @@ class MainHierarchicalParser(HierarchicalParser): definitions inside the root_matcher attribute of this class. """ self.parser_context.caching_backend = parser.backend - self.parser_context.cache_service.backend = parser.backend + self.parser_context.cache_service.parser_context = self.parser_context self.parser_context.super_backend = parser.backend.superBackend self.backend = self.parser_context.caching_backend self.super_backend = self.parser_context.super_backend @@ -530,9 +530,9 @@ class CacheService(object): no metadata associated with them. Also you can control how many times the data can be read and written from the cache. """ - def __init__(self): + def __init__(self, parser_context=None): self._cache = {} - self.backend = None + self.parser_context = parser_context def __getitem__(self, name): """Get the value identified by name. If the cachemode does not support @@ -562,9 +562,18 @@ class CacheService(object): cache_object.value = value cache_object._updated = True + def clear(self): + """Frees all object from the cache. + """ + self._cache.clear() + def add(self, name, value=None, single=True, update=True): """Used to add a cache object. Two cache objects with the same name are not allowed. + + Args: + single (bool): If the value is allowed to be pushed only once. + update (bool): If the value should be update before pushing. """ if name in self._cache: raise LookupError("There already exists a cached value with the name '{}'. All keys in the CacheService should be unique.".format(name)) @@ -605,7 +614,7 @@ class CacheService(object): return if self.check_push_allowed(cache_object): - self.backend.addValue(metaname, cache_object.value) + self.parser_context.caching_backend.addValue(metaname, cache_object.value) cache_object._pushed = True cache_object._updated = False @@ -632,7 +641,7 @@ class CacheService(object): if self.check_push_allowed(cache_object): if cache_object.value is not None: - self.backend.addRealValue(metaname, cache_object.value, unit=unit) + self.parser_context.caching_backend.addRealValue(metaname, cache_object.value, unit=unit) cache_object._pushed = True cache_object._updated = False else: @@ -661,7 +670,7 @@ class CacheService(object): if self.check_push_allowed(cache_object): if cache_object.value is not None: - self.backend.addArrayValues(metaname, np.array(cache_object.value), unit=unit) + self.parser_context.caching_backend.addArrayValues(metaname, np.array(cache_object.value), unit=unit) cache_object._pushed = True cache_object._updated = False else: diff --git a/common/python/nomadcore/simple_parser.py b/common/python/nomadcore/simple_parser.py index ba0dd5ec15a2fa351887b78056f25ed94ac70182..307475e89919e228eec40f4d3c6acc2ce9e673fe 100644 --- a/common/python/nomadcore/simple_parser.py +++ b/common/python/nomadcore/simple_parser.py @@ -76,7 +76,8 @@ class SimpleMatcher(object): coverageIgnore=False, # mark line as ignored in coverage analysis endAction = None, # A function that is called when this SimpleMatcher finishes onClose = None, # A dictionary of onClose callbacks that are specific to this SimpleMatcher - onOpen = None, # A dictionary of onClose callbacks that are specific to this SimpleMatcher + onOpen = None, # A dictionary of onOpen callbacks that are specific to this SimpleMatcher + startReTransform = None, # A callback function that is called with the groups that were matched from the startReStr. ): self.index = -1 self.startReStr = startReStr @@ -99,6 +100,7 @@ class SimpleMatcher(object): self.endAction = endAction self.onClose = onClose self.onOpen = onOpen + self.startReTransform = startReTransform self.keep = False # Boolean flag used by the ParserOptimizer to determine which SimpleMatchers to keep # boolean flag to signal that this SimpleMatcher does not have any # effect (besides progressing input file): @@ -106,11 +108,14 @@ class SimpleMatcher(object): # - no submatchers # - no adHoc # - no sections + # - no endActions self.does_nothing = (len(subMatchers) == 0 and len(sections) == 0 and fixedStartValues is None and fixedEndValues is None and - adHoc is None) + adHoc is None and + endAction is None and + startReTransform is None) if self.does_nothing: if startReStr is not None and len(extractGroupNames(startReStr))>0: self.does_nothing = False @@ -164,6 +169,10 @@ class SimpleMatcher(object): dependencies = self.dependencies, defLine = self.defLine, defFile = self.defFile, + endAction = self.endAction, + onClose = self.onClose, + onOpen = self.onOpen, + startReTransform = self.startReTransform, ) simpleMatcher.keep = self.keep return simpleMatcher @@ -593,6 +602,8 @@ class CompiledMatcher(object): def addValue(self, backend, metaName, value): """adds a value with unit conversions (only for the groups in start and endRe)""" + if metaName == "x_nwchem_qmd_step_dipole": + print(value) converter = self.converters.get(metaName, None) if converter: value = converter(value) @@ -618,17 +629,24 @@ class CompiledMatcher(object): raise Exception("Expected to match %s on %r" % (self.startRe.pattern, line)) self.handleMatchTelemetry(parser, m, line, 0) result_dict = {} - for k,v in sorted(m.groupdict().items()): - if v is None: - # a group may be optional (subexpression of ? or | in regex) - continue - k_converted, v_converted = self.addStrValue(parser.backend, k, v) - result_dict[k_converted] = v_converted - if self.matcher.fixedStartValues: - for k,v in sorted(self.matcher.fixedStartValues.items()): - if k not in result_dict: - v_converted = self.addValue(parser.backend, k, v) - result_dict[k] = v_converted + + # If a separate transform function was defined in the matcher, call it + # and do nothing else. + if self.matcher.startReTransform is not None: + self.matcher.startReTransform(parser.backend, m.groups()) + else: + + for k,v in sorted(m.groupdict().items()): + if v is None: + # a group may be optional (subexpression of ? or | in regex) + continue + k_converted, v_converted = self.addStrValue(parser.backend, k, v) + result_dict[k_converted] = v_converted + if self.matcher.fixedStartValues: + for k,v in sorted(self.matcher.fixedStartValues.items()): + if k not in result_dict: + v_converted = self.addValue(parser.backend, k, v) + result_dict[k] = v_converted if self.matcher.forwardMatch: logger.debug("handleStartMatch of %s on (%s) pushing back line", self.matcher.desc(),line) parser.fIn.pushbackLine(line) @@ -1108,11 +1126,6 @@ class SimpleParser(object): def contextClose(self, cNow): - # Handle end action before closing sections - # Call the end action - if cNow.compiledMatcher.matcher.endAction: - cNow.compiledMatcher.matcher.endAction() - if cNow.startEnd == ParsingContext.Start: for s, gIndex in reversed(list(cNow.sections.items())): @@ -1134,6 +1147,12 @@ class SimpleParser(object): def contextPop(self): cNow = self.context.pop() + + # Handle end action before closing sections + # Call the end action + if cNow.compiledMatcher.matcher.endAction: + cNow.compiledMatcher.matcher.endAction() + self.contextClose(cNow) def contextDesc(self):