Commit be915c0a authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Continued to work on optimade and metainfo.

parent 5e8b67c9
......@@ -17,6 +17,7 @@ import datetime
from elasticsearch_dsl import Keyword
from nomad import utils, config
from nomad.metainfo import MObject
class UploadWithMetadata():
......@@ -106,10 +107,17 @@ class CalcWithMetadata():
def to_dict(self):
return {
key: value for key, value in self.__dict__.items()
if value is not None and key not in ['backend']
def items():
for key, value in self.__dict__.items():
if value is None or key in ['backend']:
if isinstance(value, MObject):
value = value.m_to_dict()
yield key, value
return {key: value for key, value in items()}
def update(self, **kwargs):
for key, value in kwargs.items():
......@@ -92,7 +92,7 @@ class DFTCalcWithMetadata(CalcWithMetadata):
self.geometries = []
self.group_hash: str = None
self.optimade: optimade.StructureEntry = None
self.optimade: optimade.OptimadeStructureEntry = None
......@@ -176,6 +176,8 @@ class DFTCalcWithMetadata(CalcWithMetadata):
self.n_total_energies = n_total_energies
self.n_geometries = n_geometries
self.optimade = backend.get_mi2_section(optimade.OptimadeStructureEntry.m_section)
def only_atoms(atoms):
numbers = [[atom] for atom in atoms]
......@@ -230,8 +232,8 @@ Domain(
'Data for the optimade API',
elastic_value=lambda entry: optimade.elastic_obj(entry, optimade.ESStructureEntry)
elastic_value=lambda entry: optimade.elastic_obj(entry, optimade.ESOptimadeEntry)
total_energies=('n_total_energies', 'sum'),
......@@ -145,6 +145,7 @@ import sys
import inspect
import re
import numpy as np
from pint.unit import _Unit
from pint import UnitRegistry
import inflection
......@@ -364,17 +365,22 @@ class MObject(metaclass=MObjectMeta):
if shape is None or len(shape) == 0 or check_item:
elif len(shape) == 1:
if not isinstance(value, list):
raise TypeError('Wrong shape')
if type(definition.type) == np.dtype:
if len(shape) != len(value.shape):
raise TypeError('Wrong shape')
if len(shape) == 1:
if not isinstance(value, list):
raise TypeError('Wrong shape')
for item in value:
for item in value:
# raise Exception('Higher shapes not implemented')
# raise Exception('Higher shapes not implemented')
# TODO check dimension
......@@ -525,7 +531,29 @@ class MObject(metaclass=MObjectMeta):
def m_to_dict(self) -> Dict[str, Any]:
"""Returns the data of this section as a json serializeable dictionary. """
def items() -> Iterable[Tuple[str, Any]]:
yield 'm_section',
if self.m_parent_index != -1:
yield 'm_parnet_index', self.m_parent_index
for name, sub_section in self.m_section.sub_sections.items():
if name not in self.m_data:
if sub_section.repeats:
yield name, [item.m_to_dict() for item in self.m_data[name]]
yield name, self.m_data[name].m_to_dict()
for name in self.m_section.quantities:
if name in self.m_data:
value = getattr(self, name)
if hasattr(value, 'tolist'):
value = value.tolist()
yield name, value
return {key: value for key, value in items()}
def m_to_json(self):
"""Returns the data of this section as a json string. """
......@@ -620,6 +648,15 @@ class Quantity(Definition):
raise KeyError('Cannot overwrite quantity definition. Only values can be set.')
# object (instance) case
if type(self.type) == np.dtype:
if type(value) != np.ndarray:
value = np.array(value, dtype=self.type)
elif self.type != value.dtype:
value = np.array(value, dtype=self.type)
elif type(value) == np.ndarray:
value = value.tolist()
MObject.m_type_check(self, value)
obj.m_data[self.__name] = value
......@@ -792,7 +829,7 @@ Section.parent = Quantity(
Quantity.m_section.section_cls = Quantity
Quantity.type = Quantity(
type=Union[type, Enum, Section], name='type', _bs=True, description='''
type=Union[type, Enum, Section, np.dtype], name='type', _bs=True, description='''
The type of the quantity.
Can be one of the following:
......@@ -802,6 +839,12 @@ Quantity.type = Quantity(
- an instance of :class:`Enum`, e.g. ``Enum(['one', 'two', 'three'])
- a instance of Section, i.e. a section definition. This will define a reference
- a custom meta-info DataType
- a numpy dtype,
If set to a dtype, this quantity will use a numpy array to store values. It will use
the given dtype. If not set, this quantity will use (nested) Python lists to store values.
If values are set to the property, they will be converted to the respective
In the NOMAD CoE meta-info this was basically the ``dTypeStr``.
from import chemical_symbols
from elasticsearch_dsl import Keyword, Integer, Float, Text, InnerDoc, Nested
import numpy as np
from nomad.metainfo import MObject, Section, Quantity, Enum, units
......@@ -15,7 +16,7 @@ class ElementRatio(InnerDoc):
ratio = Float()
def from_structure_entry(entry: 'StructureEntry'):
def from_structure_entry(entry: 'OptimadeStructureEntry'):
return [
ElementRatio(element=entry.elements[i], ratio=entry.elements_ratios[i])
for i in range(0, entry.nelements)]
......@@ -26,7 +27,7 @@ class Optimade():
class StructureEntry(MObject):
class OptimadeStructureEntry(MObject):
m_section = Section(
......@@ -115,7 +116,7 @@ class StructureEntry(MObject):
lattice_vectors = Quantity(
type=float, shape=[3, 3], unit=units.angstrom,
type=np.dtype('f8'), shape=[3, 3], unit=units.angstrom,
a_optimade=Optimade(query=False, entry=True),
......@@ -123,7 +124,7 @@ class StructureEntry(MObject):
cartesian_site_positions = Quantity(
type=float, shape=['nsites', 3], unit=units.angstrom,
type=np.dtype('f8'), shape=['nsites', 3], unit=units.angstrom,
a_optimade=Optimade(query=False, entry=True), description='''
Cartesian positions of each site. A site is an atom, a site potentially occupied by
......@@ -173,7 +174,7 @@ class Species(MObject):
m_section = Section(
repeats=True, parent=StructureEntry.m_section,
repeats=True, parent=OptimadeStructureEntry.m_section,
name = Quantity(
......@@ -264,4 +265,4 @@ def elastic_obj(source: MObject, target_cls: type):
return target
ESStructureEntry = elastic_mapping(StructureEntry.m_section, InnerDoc)
ESOptimadeEntry = elastic_mapping(OptimadeStructureEntry.m_section, InnerDoc)
......@@ -18,7 +18,8 @@ import numpy as np
from nomad import config
from nomad.parsing import LocalBackend
from nomad.normalizing.normalizer import SystemBasedNormalizer
from nomad.metainfo.optimade import StructureEntry as Optimade
from nomad.metainfo import units
from nomad.metainfo.optimade import OptimadeStructureEntry
class OptimadeNormalizer(SystemBasedNormalizer):
......@@ -30,26 +31,29 @@ class OptimadeNormalizer(SystemBasedNormalizer):
def __init__(self, backend):
super().__init__(backend, all_sections=config.normalize.all_systems)
def get_optimade_data(self, index) -> Optimade:
def get_optimade_data(self, index) -> OptimadeStructureEntry:
The 'main' method of this :class:`SystemBasedNormalizer`.
Normalizes the section with the given `index`.
Normalizes geometry, classifies, system_type, and runs symmetry analysis.
optimade = Optimade()
optimade = OptimadeStructureEntry()
def get_value(key: str, default: Any = None, nonp: bool = False) -> Any:
def get_value(key: str, default: Any = None, numpy: bool = False) -> Any:
value = self._backend.get_value(key, index)
if nonp and type(value).__module__ == np.__name__:
value = value.tolist()
if type(value) == np.ndarray and not numpy:
return value.tolist()
if isinstance(value, list) and numpy:
return np.array(value)
return value
except KeyError:
return default
from nomad.normalizing.system import normalized_atom_labels
nomad_species = get_value('atom_labels', nonp=True)
nomad_species = get_value('atom_labels')
# elements
atoms = normalized_atom_labels(nomad_species)
......@@ -63,7 +67,7 @@ class OptimadeNormalizer(SystemBasedNormalizer):
optimade.nelements = len(optimade.elements)
optimade.elements_ratios = [
optimade.nelements / atom_counts[element]
atom_counts[element] / optimade.nelements
for element in optimade.elements]
# formulas
......@@ -77,11 +81,11 @@ class OptimadeNormalizer(SystemBasedNormalizer):
# sites
optimade.nsites = len(nomad_species)
optimade.species_at_sites = nomad_species
optimade.lattice_vectors = get_value('lattice_vectors', nonp=True)
optimade.cartesian_site_positions = get_value('atom_positions', nonp=True)
optimade.lattice_vectors = (get_value('lattice_vectors', numpy=True) * units.m).to(units.angstrom).magnitude
optimade.cartesian_site_positions = (get_value('atom_positions', numpy=True) * units.m).to(units.angstrom).magnitude
optimade.dimension_types = [
1 if value else 0
for value in get_value('configuration_periodic_dimensions', nonp=True)]
for value in get_value('configuration_periodic_dimensions')]
# TODO subsections with species def
# TODO optimade.structure_features
......@@ -93,4 +97,6 @@ class OptimadeNormalizer(SystemBasedNormalizer):
optimade = self.get_optimade_data(index)
except Exception as e:
import traceback
self.logger.warn('could not acquire optimade data', exc_info=e)
......@@ -355,7 +355,7 @@ class LocalBackend(LegacyParserBackend):
def get_mi2_section(self, section_def: MI2Section):
""" Allows to mix a metainfo2 style section into backend. """
self.mi2_data.get(, None)
return self.mi2_data.get(, None)
def __getattr__(self, name):
""" Support for unimplemented and unexpected methods. """
import json
from nomad.processing import Upload
from import SearchRequest
def test_get_entry(published: Upload):
calc_id = list(published.calcs)[0].calc_id
with published.upload_files.archive_file(calc_id) as f:
data = json.load(f)
assert 'OptimadeStructureEntry' in data
search_result = SearchRequest().search_parameter('calc_id', calc_id).execute_paginated()['results'][0]
assert 'optimade' in search_result
......@@ -11,6 +11,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import numpy as np
from nomad.metainfo.metainfo import MObject, Section, Quantity, Definition, sub_section
......@@ -92,6 +93,7 @@ class System(MObject):
m_section = Section(repeats=True, parent=Run.m_section)
n_atoms = Quantity(type=int, default=0)
atom_label = Quantity(type=str, shape=['n_atoms'])
atom_positions = Quantity(type=np.dtype('f8'), shape=['n_atoms', 3])
class Parsing(MObject):
......@@ -251,3 +253,8 @@ class TestM1:
assert False, 'Expected TypeError'
def test_np(self):
system = System()
system.atom_positions = [[1, 2, 3]]
assert type(system.atom_positions) == np.ndarray
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment