Classes for intermediate storing of results, better error messages in the local backend.

common/python/nomadcore/baseclasses.py

@@ -4,6 +4,7 @@ NoMaD project.
 """
 
 import os
+import numpy as np
 import logging
 from abc import ABCMeta, abstractmethod
 from nomadcore.unit_conversion import unit_conversion
@@ -53,6 +54,7 @@ class ParserInterface(object):
         self.parser_context.metainfo_to_keep = metainfo_to_keep
         self.parser_context.main_file = main_file
         self.parser_context.file_service = FileService()
+        self.parser_context.cache_service = CacheService()
         self.parser_context.parser_info = self.get_parser_info()
         self.main_parser = None
 
@@ -258,6 +260,7 @@ class BasicParser(object):
         self.backend = parser_context.caching_backend
         self.super_backend = parser_context.super_backend
         self.file_service = parser_context.file_service
+        self.cache_service = parser_context.cache_service
 
     @abstractmethod
     def parse(self):
@@ -296,6 +299,16 @@ class HierarchicalParser(BasicParser):
         self.default_data_caching_level = CachingLevel.ForwardAndCache
         self.default_section_caching_level = CachingLevel.Forward
         self.onClose = {}
+        self.cm = None
+
+    def setup_common_matcher(self, common_matcher):
+        """ Used to setup the CommonMatcher object. This object will contain
+        SimpleMatchers, onClose functions, caching levels, etc. that are common
+        for many different HierarchicalParsers.
+        """
+        self.cm = common_matcher
+        self.onClose.update(common_matcher.getOnCloseTriggers())
+        self.caching_level_for_metaname.update(common_matcher.caching_levels)
 
 
 #===============================================================================
@@ -341,6 +354,7 @@ class MainHierarchicalParser(HierarchicalParser):
                 definitions inside the root_matcher attribute of this class.
         """
         self.parser_context.caching_backend = parser.backend
+        self.parser_context.cache_service.backend = parser.backend
         self.parser_context.super_backend = parser.backend.superBackend
         self.backend = self.parser_context.caching_backend
         self.super_backend = self.parser_context.super_backend
@@ -363,6 +377,131 @@ class ParserContext(object):
         self.file_service = None
         self.metainfo_env = None
         self.parser_info = None
+        self.cache_service = None
+
+
+#===============================================================================
+class CacheMode(object):
+    """Enumeration for different types of cache modes for CacheObjects.
+    """
+    MULTI_IN_MULTI_OUT = 1
+    MULTI_OUT_SINGLE_IN = 2
+    SINGLE_IN_MULTI_OUT = 3
+    SINGLE_IN_SINGLE_OUT = 4
+
+
+#===============================================================================
+class CacheObject(object):
+    """Wraps an intermediate value stored during parsing.
+    """
+    def __init__(self, name, mode, value):
+        self.name = name
+        self.cache_mode = mode
+        self._value = value
+        # self._reset = False
+        self._in = False
+        self._out = False
+
+    @property
+    def value(self):
+
+        if self.cache_mode == CacheMode.SINGLE_IN_SINGLE_OUT:
+            if self._out:
+                raise Exception("Cannot get the value for '{}' multiple times because of the cache mode.")
+        self._out = True
+        return self._value
+
+    @value.setter
+    def value(self, value):
+        if self.cache_mode == CacheMode.SINGLE_IN_MULTI_OUT or self.cache_mode == CacheMode.SINGLE_IN_SINGLE_OUT:
+            if self._in:
+                raise Exception("Cannot set the value for '{}' multiple times because of the cache mode.")
+        self._value = value
+        self._in = True
+
+
+#===============================================================================
+class CacheService(object):
+    """A class that can be used to store intermediate results in the parsing
+    process. This is quite similar to the caching system that is used by the
+    ActiveBackend object but this version also allows caching values that have
+    no metadata associated with them. Also you can control how many times the
+    data can be read and written from the cache.
+    """
+    def __init__(self):
+        self._cache = {}
+        self.backend = None
+
+    def __getitem__(self, name):
+        """Get the value identified by name. If the cachemode does not support
+        getting the value, an exception is raised.
+
+        returns:
+
+        raises:
+        """
+        cache_object = self._cache.get(name)
+        if cache_object is None:
+            logger.warning("The cache object for '{}' could not be found".format(name))
+            return None
+        value = cache_object.value
+        return value
+
+    def __setitem__(self, name, value):
+        """Used to set the value for an item. The CacheObject corresponding to
+        the name has to be first created by using the function
+        add_cache_object().
+        """
+        cache_object = self._cache[name]
+        cache_object.value = value
+
+    def add_cache_object(self, name, mode, value=None):
+        """Used to add a cache object. Two cache objects with the same name are
+        not allowed.
+        """
+        if name in self._cache:
+            raise LookupError("There already exists a cached value with the name '{}'. All keys in the CacheService should be unique.".format(name))
+        cache_object = CacheObject(name, mode, value)
+        self._cache[name] = cache_object
+
+    def push_value(self, name, metaname=None):
+        """Pushes the scalar value stored with the given name to the backend.
+        If the name cannot be found, nothing is pushed. If the metaname
+        property is not defined the value is pushed with the name that was used
+        as key.
+        """
+        if metaname is None:
+            metaname = name
+        value = self[name]
+        if value is not None:
+            self.backend.addValue(metaname, value)
+            # del self._cache[name]
+
+    def push_real_value(self, name, unit=None, metaname=None):
+        """Pushes the scalar value stored with the given name to the backend.
+        If the name cannot be found, nothing is pushed. If the metaname
+        property is not defined the value is pushed with the name that was used
+        as key.
+        """
+        if metaname is None:
+            metaname = name
+        value = self[name]
+        if value is not None:
+            self.backend.addRealValue(metaname, value, unit=unit)
+            # del self._cache[name]
+
+    def push_array_values(self, name, unit=None, metaname=None):
+        """Pushes the scalar value stored with the given name to the backend.
+        If the name cannot be found, nothing is pushed. If the metaname
+        property is not defined the value is pushed with the name that was used
+        as key.
+        """
+        if metaname is None:
+            metaname = name
+        value = self[name]
+        if value is not None:
+            self.backend.addArrayValues(metaname, np.array(value), unit=unit)
+            # del self._cache[name]
 
 
 #===============================================================================

common/python/nomadcore/local_backend.py

@@ -57,7 +57,7 @@ class LocalBackend(object):
             # Check that the value is actually of scalar type
             value_type = type(value)
             if value_type not in [float, int, bool, str, unicode]:
-                raise TypeError("The value for metainfo '{}' is not of scalar nature.".format(metaName))
+                raise TypeError("The value '{}' for metainfo '{}' is not of scalar nature.".format(value, metaName))
 
             # Check that the metainfo should be scalar
             metainfo_shape = dataManager.metaInfo.shape
@@ -412,7 +412,10 @@ class SectionManager(object):
         parents = []
         for parentName in self.parentSectionNames:
             pSect = backend.sectionManagers.get(parentName, None)
+            try:
                 parentSection = pSect.openSections[pSect.lastSectionGIndex]
+            except KeyError:
+                raise LookupError("Could not open section '{}' because it's parent section could not be found".format(self.metaInfo.name))
             parents.append(parentSection)
             if pSect:
                 references.append(pSect.lastSectionGIndex)