Continued to add optimade to nomad.

nomad/datamodel/base.py

@@ -181,7 +181,7 @@ class DomainQuantity:
             self, description: str = None, multi: bool = False, aggregations: int = 0,
             order_default: bool = False, metric: Tuple[str, str] = None,
             zero_aggs: bool = True, metadata_field: str = None,
-            elastic_mapping: str = None,
+            elastic_mapping: type = None,
             elastic_search_type: str = 'term', elastic_field: str = None,
             elastic_value: Callable[[Any], Any] = None):
 

nomad/datamodel/dft.py

@@ -18,12 +18,13 @@ DFT specific metadata
 
 from typing import List
 import re
-from elasticsearch_dsl import Integer
+from elasticsearch_dsl import Integer, Object
 import ase.data
 
 from nomadcore.local_backend import ParserEvent
 
 from nomad import utils, config
+from nomad.metainfo import optimade
 
 from .base import CalcWithMetadata, DomainQuantity, Domain, get_optional_backend_value
 
@@ -91,6 +92,8 @@ class DFTCalcWithMetadata(CalcWithMetadata):
         self.geometries = []
         self.group_hash: str = None
 
+        self.optimade: optimade.StructureEntry = None
+
         super().__init__(**kwargs)
 
     def apply_domain_metadata(self, backend):
@@ -222,7 +225,14 @@ Domain(
         n_geometries=DomainQuantity(
             'Number of unique geometries',
             elastic_mapping=Integer()),
-        n_atoms=DomainQuantity('Number of atoms in the simulated system', elastic_mapping=Integer())),
+        n_atoms=DomainQuantity(
+            'Number of atoms in the simulated system',
+            elastic_mapping=Integer()),
+        optimade=DomainQuantity(
+            'Data for the optimade API',
+            elastic_mapping=Object(optimade.ESStructureEntry),
+            elastic_value=lambda entry: optimade.elastic_obj(entry, optimade.ESStructureEntry)
+        )),
     metrics=dict(
         total_energies=('n_total_energies', 'sum'),
         calculations=('n_calculations', 'sum'),

nomad/metainfo/metainfo.py

@@ -274,7 +274,14 @@ class MObject(metaclass=MObjectMeta):
             assert self.m_section == cls.m_section, \
                 'Section class and section definition must match'
 
-        self.m_data = dict(**kwargs)
+        self.m_annotations: Dict[str, Any] = {}
+        self.m_data: Dict[str, Any] = {}
+        for key, value in kwargs.items():
+            if key.startswith('a_'):
+                self.m_annotations[key[2:]] = value
+            else:
+                self.m_data[key] = value
+
         # TODO
         # self.m_data = {}
         # if _bs:
@@ -366,7 +373,8 @@ class MObject(metaclass=MObjectMeta):
 
         else:
             # TODO
-            raise Exception('Higher shapes not implemented')
+            # raise Exception('Higher shapes not implemented')
+            pass
 
         # TODO check dimension
 

nomad/app/optimade/data.py → nomad/metainfo/optimade.py

@@ -230,3 +230,38 @@ class Species(MObject):
         databases that use species names, containing characters that are not allowed (see
         description of the species_at_sites list).
         ''')
+
+
+def elastic_mapping(section: Section, base_cls: type) -> type:
+    """ Creates an elasticsearch_dsl document class from a section definition. """
+
+    dct = {
+        name: quantity.m_annotations['elastic']['type']()
+        for name, quantity in section.quantities.items()
+        if 'elastic' in quantity.m_annotations}
+
+    return type(section.name, (base_cls,), dct)
+
+
+def elastic_obj(source: MObject, target_cls: type):
+    if source is None:
+        return None
+
+    target = target_cls()
+
+    for name, quantity in source.m_section.quantities.items():
+        elastic_annotation = quantity.m_annotations.get('elastic')
+        if elastic_annotation is None:
+            continue
+
+        if 'mapping' in elastic_annotation:
+            value = elastic_annotation['mapping'](source)
+        else:
+            value = getattr(source, name)
+
+        setattr(target, name, value)
+
+    return target
+
+
+ESStructureEntry = elastic_mapping(StructureEntry.m_section, InnerDoc)

nomad/normalizing/__init__.py

@@ -36,9 +36,11 @@ from typing import List, Any, Iterable, Type
 from .normalizer import Normalizer
 from .system import SystemNormalizer
 from .fhiaims import FhiAimsBaseNormalizer
+from .optimade import OptimadeNormalizer
 
 
 normalizers: Iterable[Type[Normalizer]] = [
     SystemNormalizer,
+    OptimadeNormalizer,
     FhiAimsBaseNormalizer
 ]

nomad/normalizing/optimade.py

+# Copyright 2018 Markus Scheidgen
+#
+# Licensed under the Apache License, Version 2.0 (the 'License');
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an'AS IS' BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Any, Dict, cast
+import numpy as np
+
+from nomad import config
+from nomad.parsing import LocalBackend
+from nomad.normalizing.normalizer import SystemBasedNormalizer
+from nomad.metainfo.optimade import StructureEntry as Optimade
+
+
+class OptimadeNormalizer(SystemBasedNormalizer):
+
+    """
+    This normalizer performs all produces a section all data necessary for the Optimade API.
+    It assumes that the :class:`SystemNormalizer` was run before.
+    """
+    def __init__(self, backend):
+        super().__init__(backend, all_sections=config.normalize.all_systems)
+
+    def get_optimade_data(self, index) -> Optimade:
+        """
+        The 'main' method of this :class:`SystemBasedNormalizer`.
+        Normalizes the section with the given `index`.
+        Normalizes geometry, classifies, system_type, and runs symmetry analysis.
+        """
+        optimade = Optimade()
+
+        def get_value(key: str, default: Any = None, nonp: bool = False) -> Any:
+            try:
+                value = self._backend.get_value(key, index)
+                if nonp and type(value).__module__ == np.__name__:
+                    value = value.tolist()
+                return value
+            except KeyError:
+                return default
+
+        from nomad.normalizing.system import normalized_atom_labels
+
+        nomad_species = get_value('atom_labels', nonp=True)
+
+        # elements
+        atoms = normalized_atom_labels(nomad_species)
+        atom_counts: Dict[str, int] = {}
+        for atom in atoms:
+            current = atom_counts.setdefault(atom, 0)
+            current += 1
+            atom_counts[atom] = current
+
+        optimade.elements = list(set(atoms))
+        optimade.elements.sort()
+        optimade.nelements = len(optimade.elements)
+        optimade.elements_ratios = [
+            optimade.nelements / atom_counts[element]
+            for element in optimade.elements]
+
+        # formulas
+        optimade.chemical_formula_reduced = get_value('chemical_composition_reduced')
+        optimade.chemical_formula_hill = get_value('chemical_composition_bulk_reduced')
+        optimade.chemical_formula_descriptive = optimade.chemical_formula_hill
+        optimade.chemical_formula_anonymous = ''.join([
+            '%s' % element + (str(atom_counts[element]) if atom_counts[element] > 1 else '')
+            for element in optimade.elements])
+
+        # sites
+        optimade.nsites = len(nomad_species)
+        optimade.species_at_sites = nomad_species
+        optimade.lattice_vectors = get_value('lattice_vectors', nonp=True)
+        optimade.cartesian_site_positions = get_value('atom_positions', nonp=True)
+        optimade.dimension_types = [
+            1 if value else 0
+            for value in get_value('configuration_periodic_dimensions', nonp=True)]
+
+        # TODO subsections with species def
+        # TODO optimade.structure_features
+
+        return optimade
+
+    def normalize_system(self, index):
+        try:
+            optimade = self.get_optimade_data(index)
+            self._backend.add_mi2_section(optimade)
+        except Exception as e:
+            self.logger.warn('could not acquire optimade data', exc_info=e)

nomad/parsing/backend.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import TextIO, Tuple, List, Any, Callable
+from typing import TextIO, Tuple, List, Any, Callable, Dict
 from abc import ABCMeta, abstractmethod
 from io import StringIO
 import json
@@ -23,6 +23,7 @@ from nomadcore.local_backend import LocalBackend as LegacyLocalBackend
 from nomadcore.local_backend import Section, Results
 
 from nomad.utils import get_logger
+from nomad.metainfo import MObject, Section as MI2Section
 
 logger = get_logger(__name__)
 
@@ -336,6 +337,8 @@ class LocalBackend(LegacyParserBackend):
         delegate = LegacyLocalBackend(*args, **kwargs)
         super().__init__(delegate)
 
+        self.mi2_data: Dict[str, MObject] = {}
+
         self.reset_status()
 
         self._open_context: Tuple[str, int] = None
@@ -346,6 +349,14 @@ class LocalBackend(LegacyParserBackend):
         self._known_attributes = ['results']
         self.fileOut = io.StringIO()
 
+    def add_mi2_section(self, section: MObject):
+        """ Allows to mix a metainfo2 style section into backend. """
+        self.mi2_data[section.m_section.name] = section
+
+    def get_mi2_section(self, section_def: MI2Section):
+        """ Allows to mix a metainfo2 style section into backend. """
+        self.mi2_data.get(section_def.name, None)
+
     def __getattr__(self, name):
         """ Support for unimplemented and unexpected methods. """
         if name not in self._known_attributes and self._unknown_attributes.get(name) is None:
@@ -558,6 +569,9 @@ class LocalBackend(LegacyParserBackend):
             json_writer.key(root_section)
             self._write(json_writer, self._delegate.results[root_section], filter=filter)
 
+        for name, section in self.mi2_data.items():
+            json_writer.key_value(name, section.m_to_dict())
+
         json_writer.close_object()
         json_writer.close()