Continued to work on optimade and metainfo.

nomad/datamodel/base.py

@@ -17,6 +17,7 @@ import datetime
 from elasticsearch_dsl import Keyword
 
 from nomad import utils, config
+from nomad.metainfo import MObject
 
 
 class UploadWithMetadata():
@@ -106,10 +107,17 @@ class CalcWithMetadata():
         self.update(**kwargs)
 
     def to_dict(self):
-        return {
-            key: value for key, value in self.__dict__.items()
-            if value is not None and key not in ['backend']
-        }
+        def items():
+            for key, value in self.__dict__.items():
+                if value is None or key in ['backend']:
+                    continue
+
+                if isinstance(value, MObject):
+                    value = value.m_to_dict()
+
+                yield key, value
+
+        return {key: value for key, value in items()}
 
     def update(self, **kwargs):
         for key, value in kwargs.items():

nomad/datamodel/dft.py

@@ -92,7 +92,7 @@ class DFTCalcWithMetadata(CalcWithMetadata):
         self.geometries = []
         self.group_hash: str = None
 
-        self.optimade: optimade.StructureEntry = None
+        self.optimade: optimade.OptimadeStructureEntry = None
 
         super().__init__(**kwargs)
 
@@ -176,6 +176,8 @@ class DFTCalcWithMetadata(CalcWithMetadata):
         self.n_total_energies = n_total_energies
         self.n_geometries = n_geometries
 
+        self.optimade = backend.get_mi2_section(optimade.OptimadeStructureEntry.m_section)
+
 
 def only_atoms(atoms):
     numbers = [ase.data.atomic_numbers[atom] for atom in atoms]
@@ -230,8 +232,8 @@ Domain(
             elastic_mapping=Integer()),
         optimade=DomainQuantity(
             'Data for the optimade API',
-            elastic_mapping=Object(optimade.ESStructureEntry),
-            elastic_value=lambda entry: optimade.elastic_obj(entry, optimade.ESStructureEntry)
+            elastic_mapping=Object(optimade.ESOptimadeEntry),
+            elastic_value=lambda entry: optimade.elastic_obj(entry, optimade.ESOptimadeEntry)
         )),
     metrics=dict(
         total_energies=('n_total_energies', 'sum'),

nomad/metainfo/metainfo.py

@@ -145,6 +145,7 @@ import sys
 import inspect
 import re
 
+import numpy as np
 from pint.unit import _Unit
 from pint import UnitRegistry
 import inflection
@@ -364,17 +365,22 @@ class MObject(metaclass=MObjectMeta):
         if shape is None or len(shape) == 0 or check_item:
             check_value(value)
 
-        elif len(shape) == 1:
-            if not isinstance(value, list):
-                raise TypeError('Wrong shape')
+        else:
+            if type(definition.type) == np.dtype:
+                if len(shape) != len(value.shape):
+                    raise TypeError('Wrong shape')
+            else:
+                if len(shape) == 1:
+                    if not isinstance(value, list):
+                        raise TypeError('Wrong shape')
 
-            for item in value:
-                check_value(item)
+                    for item in value:
+                        check_value(item)
 
-        else:
-            # TODO
-            # raise Exception('Higher shapes not implemented')
-            pass
+                else:
+                    # TODO
+                    # raise Exception('Higher shapes not implemented')
+                    pass
 
         # TODO check dimension
 
@@ -525,7 +531,29 @@ class MObject(metaclass=MObjectMeta):
 
     def m_to_dict(self) -> Dict[str, Any]:
         """Returns the data of this section as a json serializeable dictionary. """
-        pass
+
+        def items() -> Iterable[Tuple[str, Any]]:
+            yield 'm_section', self.m_section.name
+            if self.m_parent_index != -1:
+                yield 'm_parnet_index', self.m_parent_index
+
+            for name, sub_section in self.m_section.sub_sections.items():
+                if name not in self.m_data:
+                    continue
+
+                if sub_section.repeats:
+                    yield name, [item.m_to_dict() for item in self.m_data[name]]
+                else:
+                    yield name, self.m_data[name].m_to_dict()
+
+            for name in self.m_section.quantities:
+                if name in self.m_data:
+                    value = getattr(self, name)
+                    if hasattr(value, 'tolist'):
+                        value = value.tolist()
+                    yield name, value
+
+        return {key: value for key, value in items()}
 
     def m_to_json(self):
         """Returns the data of this section as a json string. """
@@ -620,6 +648,15 @@ class Quantity(Definition):
             raise KeyError('Cannot overwrite quantity definition. Only values can be set.')
 
         # object (instance) case
+        if type(self.type) == np.dtype:
+            if type(value) != np.ndarray:
+                value = np.array(value, dtype=self.type)
+            elif self.type != value.dtype:
+                value = np.array(value, dtype=self.type)
+
+        elif type(value) == np.ndarray:
+            value = value.tolist()
+
         MObject.m_type_check(self, value)
         obj.m_data[self.__name] = value
 
@@ -792,7 +829,7 @@ Section.parent = Quantity(
 
 Quantity.m_section.section_cls = Quantity
 Quantity.type = Quantity(
-    type=Union[type, Enum, Section], name='type', _bs=True, description='''
+    type=Union[type, Enum, Section, np.dtype], name='type', _bs=True, description='''
     The type of the quantity.
 
     Can be one of the following:
@@ -802,6 +839,12 @@ Quantity.type = Quantity(
     - an instance of :class:`Enum`, e.g. ``Enum(['one', 'two', 'three'])
     - a instance of Section, i.e. a section definition. This will define a reference
     - a custom meta-info DataType
+    - a numpy dtype,
+
+    If set to a dtype, this quantity will use a numpy array to store values. It will use
+    the given dtype. If not set, this quantity will use (nested) Python lists to store values.
+    If values are set to the property, they will be converted to the respective
+    representation.
 
     In the NOMAD CoE meta-info this was basically the ``dTypeStr``.
     ''')

nomad/metainfo/optimade.py

 from ase.data import chemical_symbols
 from elasticsearch_dsl import Keyword, Integer, Float, Text, InnerDoc, Nested
+import numpy as np
 
 from nomad.metainfo import MObject, Section, Quantity, Enum, units
 
@@ -15,7 +16,7 @@ class ElementRatio(InnerDoc):
     ratio = Float()
 
     @staticmethod
-    def from_structure_entry(entry: 'StructureEntry'):
+    def from_structure_entry(entry: 'OptimadeStructureEntry'):
         return [
             ElementRatio(element=entry.elements[i], ratio=entry.elements_ratios[i])
             for i in range(0, entry.nelements)]
@@ -26,7 +27,7 @@ class Optimade():
         pass
 
 
-class StructureEntry(MObject):
+class OptimadeStructureEntry(MObject):
     m_section = Section(
         links=optimade_links('h.6.2'),
         a_flask=dict(skip_none=True),
@@ -115,7 +116,7 @@ class StructureEntry(MObject):
         ''')
 
     lattice_vectors = Quantity(
-        type=float, shape=[3, 3], unit=units.angstrom,
+        type=np.dtype('f8'), shape=[3, 3], unit=units.angstrom,
         links=optimade_links('h.6.2.9'),
         a_optimade=Optimade(query=False, entry=True),
         description='''
@@ -123,7 +124,7 @@ class StructureEntry(MObject):
         ''')
 
     cartesian_site_positions = Quantity(
-        type=float, shape=['nsites', 3], unit=units.angstrom,
+        type=np.dtype('f8'), shape=['nsites', 3], unit=units.angstrom,
         links=optimade_links('h.6.2.10'),
         a_optimade=Optimade(query=False, entry=True), description='''
             Cartesian positions of each site. A site is an atom, a site potentially occupied by
@@ -173,7 +174,7 @@ class Species(MObject):
     """
 
     m_section = Section(
-        repeats=True, parent=StructureEntry.m_section,
+        repeats=True, parent=OptimadeStructureEntry.m_section,
         links=optimade_links('h.6.2.13'))
 
     name = Quantity(
@@ -264,4 +265,4 @@ def elastic_obj(source: MObject, target_cls: type):
     return target
 
 
-ESStructureEntry = elastic_mapping(StructureEntry.m_section, InnerDoc)
+ESOptimadeEntry = elastic_mapping(OptimadeStructureEntry.m_section, InnerDoc)

nomad/normalizing/optimade.py

@@ -18,7 +18,8 @@ import numpy as np
 from nomad import config
 from nomad.parsing import LocalBackend
 from nomad.normalizing.normalizer import SystemBasedNormalizer
-from nomad.metainfo.optimade import StructureEntry as Optimade
+from nomad.metainfo import units
+from nomad.metainfo.optimade import OptimadeStructureEntry
 
 
 class OptimadeNormalizer(SystemBasedNormalizer):
@@ -30,26 +31,29 @@ class OptimadeNormalizer(SystemBasedNormalizer):
     def __init__(self, backend):
         super().__init__(backend, all_sections=config.normalize.all_systems)
 
-    def get_optimade_data(self, index) -> Optimade:
+    def get_optimade_data(self, index) -> OptimadeStructureEntry:
         """
         The 'main' method of this :class:`SystemBasedNormalizer`.
         Normalizes the section with the given `index`.
         Normalizes geometry, classifies, system_type, and runs symmetry analysis.
         """
-        optimade = Optimade()
+        optimade = OptimadeStructureEntry()
 
-        def get_value(key: str, default: Any = None, nonp: bool = False) -> Any:
+        def get_value(key: str, default: Any = None, numpy: bool = False) -> Any:
             try:
                 value = self._backend.get_value(key, index)
-                if nonp and type(value).__module__ == np.__name__:
-                    value = value.tolist()
+                if type(value) == np.ndarray and not numpy:
+                    return value.tolist()
+                if isinstance(value, list) and numpy:
+                    return np.array(value)
+
                 return value
             except KeyError:
                 return default
 
         from nomad.normalizing.system import normalized_atom_labels
 
-        nomad_species = get_value('atom_labels', nonp=True)
+        nomad_species = get_value('atom_labels')
 
         # elements
         atoms = normalized_atom_labels(nomad_species)
@@ -63,7 +67,7 @@ class OptimadeNormalizer(SystemBasedNormalizer):
         optimade.elements.sort()
         optimade.nelements = len(optimade.elements)
         optimade.elements_ratios = [
-            optimade.nelements / atom_counts[element]
+            atom_counts[element] / optimade.nelements
             for element in optimade.elements]
 
         # formulas
@@ -77,11 +81,11 @@ class OptimadeNormalizer(SystemBasedNormalizer):
         # sites
         optimade.nsites = len(nomad_species)
         optimade.species_at_sites = nomad_species
-        optimade.lattice_vectors = get_value('lattice_vectors', nonp=True)
-        optimade.cartesian_site_positions = get_value('atom_positions', nonp=True)
+        optimade.lattice_vectors = (get_value('lattice_vectors', numpy=True) * units.m).to(units.angstrom).magnitude
+        optimade.cartesian_site_positions = (get_value('atom_positions', numpy=True) * units.m).to(units.angstrom).magnitude
         optimade.dimension_types = [
             1 if value else 0
-            for value in get_value('configuration_periodic_dimensions', nonp=True)]
+            for value in get_value('configuration_periodic_dimensions')]
 
         # TODO subsections with species def
         # TODO optimade.structure_features
@@ -93,4 +97,6 @@ class OptimadeNormalizer(SystemBasedNormalizer):
             optimade = self.get_optimade_data(index)
             self._backend.add_mi2_section(optimade)
         except Exception as e:
+            import traceback
+            traceback.print_exc()
             self.logger.warn('could not acquire optimade data', exc_info=e)

nomad/parsing/backend.py

@@ -355,7 +355,7 @@ class LocalBackend(LegacyParserBackend):
 
     def get_mi2_section(self, section_def: MI2Section):
         """ Allows to mix a metainfo2 style section into backend. """
-        self.mi2_data.get(section_def.name, None)
+        return self.mi2_data.get(section_def.name, None)
 
     def __getattr__(self, name):
         """ Support for unimplemented and unexpected methods. """

tests/app/test_optimade.py

+import json
+
+from nomad.processing import Upload
+from nomad.search import SearchRequest
+
+
+def test_get_entry(published: Upload):
+    calc_id = list(published.calcs)[0].calc_id
+
+    with published.upload_files.archive_file(calc_id) as f:
+        data = json.load(f)
+
+    assert 'OptimadeStructureEntry' in data
+    search_result = SearchRequest().search_parameter('calc_id', calc_id).execute_paginated()['results'][0]
+    assert 'optimade' in search_result

tests/test_metainfo.py

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import numpy as np
 
 from nomad.metainfo.metainfo import MObject, Section, Quantity, Definition, sub_section
 
@@ -92,6 +93,7 @@ class System(MObject):
     m_section = Section(repeats=True, parent=Run.m_section)
     n_atoms = Quantity(type=int, default=0)
     atom_label = Quantity(type=str, shape=['n_atoms'])
+    atom_positions = Quantity(type=np.dtype('f8'), shape=['n_atoms', 3])
 
 
 class Parsing(MObject):
@@ -251,3 +253,8 @@ class TestM1:
             pass
         else:
             assert False, 'Expected TypeError'
+
+    def test_np(self):
+        system = System()
+        system.atom_positions = [[1, 2, 3]]
+        assert type(system.atom_positions) == np.ndarray