Fixed issues with the JSON archive, added a function for getting scalar values...

Fixed issues with the JSON archive, added a function for getting scalar values more easily (HDF5 stores everything as arrays, so there is no way to automatically distinguish between a scalar and an array with one element).

common/python/nomadcore/archive.py

@@ -87,6 +87,29 @@ class ArchiveSection(metaclass=ABCMeta):
                 value = default
         return value
 
+    def get_scalar(self, key):
+        """Get a single value from within this section. If the requested value
+        is not scalar, this will raise a KeyError.
+
+        Args:
+            key (string): The key to look for
+
+        Returns:
+            A scalar value correspondin to the given key.
+
+        Raises:
+            KeyError if the found value is not sized like a scalar.
+        """
+        value = self[key]
+        shape = value.shape
+        if shape != (1,):
+            raise KeyError(
+                "The value for '{}/{}' is not shaped like a scalar value. "
+                "Instead it has a shape of '{}'."
+                .format(self._path, key, shape)
+            )
+        return value[0]
+
     def __setitem__(self, path, value):
         """Used to override or set invalid or missing values in the original
         archive.
@@ -96,7 +119,14 @@ class ArchiveSection(metaclass=ABCMeta):
         """
         if self._archive.use_write_cache:
             path = "{}/{}".format(self._path, path)
-            self._archive.overrides[path] = value
+
+            # If the value is not already a numpy array, make it so
+            if not isinstance(value, np.ndarray):
+                if isinstance(value, (list, tuple)):
+                    value = np.array(value)
+                else:
+                    value = np.array((value))
+            self._archive._overrides[path] = value
         else:
             raise ValueError(
                 "Writing to the source file is currently disabled. If you want "
@@ -186,7 +216,7 @@ class ArchiveSection(metaclass=ABCMeta):
             The value found from the cache, is it is set.
         """
         key = "{}/{}".format(self._path, key)
-        value = self._archive.overrides[key]
+        value = self._archive._overrides[key]
         return value
 
     def get_path_parts(self, path):
@@ -251,24 +281,26 @@ class Archive(object):
 
     Attributes:
         filepath (string): Path to the archive file
-        root_section (ArchiveSection): The root section of the file.
-        overrides (dictionary): Contains the values that are set during the
+        repositories (dict): Contains all the repositories within this Archive.
+        calculations (dict): Contains all the calculations within this Archive.
+        _overrides (dictionary): Contains the values that are set during the
             lifetime of this object. These values will not persists on the
             original file.
+        use_write_cache (boolean): Whether to enable writing to a cache that
+            will not persist to the original source file.
     """
     def __init__(self, filepath, use_write_cache=False):
         """
         Args:
             filepath (string): Filepath to an archive file.
+            use_write_cache (boolean): Whether to enable writing to a cache
+                that will not persist to the original source file.
         """
         self.filepath = filepath
-        self.root_section = None
-        self.overrides = {}
-        self._path = ""
-        self._archive = self
         self.repositories = {}
         self.calculations = {}
         self.use_write_cache = use_write_cache
+        self._overrides = {}
 
     @staticmethod
     def factory(archive_path, use_write_cache=False):
@@ -293,26 +325,11 @@ class Archive(object):
                 .format(extension)
             )
 
+    @abstractmethod
     def setup(self, root):
         """Used to setup the dictionaries that contain the repositories and
         calculations.
         """
-        for repo_name, repo in root.items():
-            self.repositories[repo_name] = ArchiveSectionHDF5(
-                repo,
-                "{}".format(repo_name),
-                self._archive,
-                [[0]],
-                0
-            )
-            for calc_name, calc in repo.items():
-                self.calculations[calc_name] = ArchiveSectionHDF5(
-                    calc,
-                    "{}/{}".format(repo_name, calc_name),
-                    self._archive,
-                    [[0]],
-                    0
-                )
 
 
 class ArchiveHDF5(Archive):
@@ -333,6 +350,27 @@ class ArchiveHDF5(Archive):
         self.index_cache = {}
         self.setup(h5_root)
 
+    def setup(self, root):
+        """Used to setup the dictionaries that contain the repositories and
+        calculations.
+        """
+        for repo_name, repo in root.items():
+            self.repositories[repo_name] = ArchiveSectionHDF5(
+                repo,
+                "{}".format(repo_name),
+                self,
+                [[0]],
+                0
+            )
+            for calc_name, calc in repo.items():
+                self.calculations[calc_name] = ArchiveSectionHDF5(
+                    calc,
+                    "{}/{}".format(repo_name, calc_name),
+                    self,
+                    [[0]],
+                    0
+                )
+
 
 class ArchiveSectionHDF5(ArchiveSection):
     """A section from a HDF5 file. Roughly corresponds to the concept of a
@@ -618,23 +656,26 @@ class ArchiveJSON(Archive):
             repository_name = mainfile_uri[6:]
             repository_name = repository_name.split("/", 1)[0]
 
-            self.root_section = ArchiveSectionJSON(
-                {
-                    repository_name: ArchiveSectionJSON(
-                        {
-                            calculation_name: ArchiveSectionJSON(
-                                json_root,
-                                "{}/{}".format(repository_name, calculation_name),
-                                self
-                            )
-                        },
-                        repository_name,
-                        self
-                    )
-                },
-                "",
+            root_section = {
+                repository_name: {
+                    calculation_name: json_root
+                }
+            }
+            self.setup(root_section)
+
+    def setup(self, root):
+        for repo_name, repo in root.items():
+            self.repositories[repo_name] = ArchiveSectionJSON(
+                repo,
+                "{}".format(repo_name),
                 self
             )
+            for calc_name, calc in repo.items():
+                self.calculations[calc_name] = ArchiveSectionJSON(
+                    calc,
+                    "{}/{}".format(repo_name, calc_name),
+                    self
+                )
 
 
 class ArchiveSectionJSON(ArchiveSection):
@@ -653,6 +694,12 @@ class ArchiveSectionJSON(ArchiveSection):
             if isinstance(value, dict):
                 yield (key, ArchiveSectionJSON(value, "{}/{}".format(self._path, key), self._archive))
             else:
+                # Wrap scalar values inside a numpy array to be consistent with the HDF5
+                # Archive.
+                if not isinstance(value, list):
+                    value = np.array([value])
+                else:
+                    value = np.array(value)
                 yield (key, value)
 
     def keys(self):
@@ -663,6 +710,12 @@ class ArchiveSectionJSON(ArchiveSection):
             if isinstance(value, dict):
                 yield ArchiveSectionJSON(value, "{}/{}".format(self._path, key), self._archive)
             else:
+                # Wrap scalar values inside a numpy array to be consistent with the HDF5
+                # Archive.
+                if not isinstance(value, list):
+                    value = np.array([value])
+                else:
+                    value = np.array(value)
                 yield value
 
     @property
@@ -670,14 +723,14 @@ class ArchiveSectionJSON(ArchiveSection):
         """Used to the the mainfile uri from a section corresponding to a
         calculation.
         """
-        return self["mainFileUri"]
+        return self._data["mainFileUri"]
 
     @property
     def parser_info(self):
         """Used to the the parser info from a section corresponding to a
         calculation.
         """
-        return self["parserInfo"]
+        return self._data["parserInfo"]
 
     def get_child(self, path):
 
@@ -718,6 +771,12 @@ class ArchiveSectionJSON(ArchiveSection):
         if index is None:
             if not is_section:
                 data = self._data[name]
+                # Wrap scalar values inside a numpy array to be consistent with the HDF5
+                # Archive.
+                if not isinstance(data, list):
+                    data = np.array([data])
+                else:
+                    data = np.array(data)
             else:
                 try:
                     sections = self._data[name]