Commit 2130cfcd authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Added basic reference serialization and resolution.

parent bd6de886
......@@ -46,14 +46,22 @@ class System(MSection):
description='A vector of booleans indicating in which dimensions the unit cell is repeated.')
class SCC(MSection):
energy_total = Quantity(type=float, default=0.0, unit=units.J)
system = Quantity(type=System.m_def, description='The system that this calculation is based on.')
class Run(MSection):
""" All data that belongs to a single code run. """
code_name = Quantity(type=str, description='The name of the code that was run.')
code_version = Quantity(type=str, description='The version of the code that was run.')
systems = SubSection(sub_section=System.m_def, repeats=True)
parsing = SubSection(sub_section=Parsing.m_def)
systems = SubSection(sub_section=System.m_def, repeats=True)
sccs = SubSection(sub_section=SCC.m_def, repeats=True)
class VaspRun(Run):
......@@ -92,11 +100,21 @@ if __name__ == '__main__':
system = run.m_create(System)
system.atom_labels = ['H', 'H', 'O']
calc = run.m_create(SCC)
calc.energy_total = 1.23e-10
calc.system = system
# Or to read data from existing metainfo data:
# To serialize the data:
serializable = run.m_to_dict()
# or
# To deserialize data
run = Run.m_from_dict(serializable)
# print(m_package.m_to_json(indent=2)) # type: ignore, pylint: disable=undefined-variable
......@@ -172,6 +172,13 @@ class Enum(list):
class MProxy():
""" A placeholder object that acts as reference to a value that is not yet resolved. """
def __init__(self, url: str):
self.url = url
class DataType:
Allows to define custom data types that can be used in the meta-info.
......@@ -219,6 +226,7 @@ class Reference(DataType):
is actually a reference (or references, depending on shape) to a section of the
given definition.
# TODO not used yet
def __init__(self, section: 'Section'):
self.section = section
......@@ -464,6 +472,9 @@ class MDataTypeAndShapeChecks(MDataDelegating):
return quantity_def.type.type_check(section, value)
elif isinstance(quantity_def.type, Section):
if isinstance(value, MProxy):
return value
if not isinstance(value, MSection):
raise TypeError(
'The value %s for reference quantity %s is not a section instance.' %
......@@ -705,7 +716,14 @@ class MSection(metaclass=MObjectMeta):
except Exception as e:
raise DeriveError('Could not derive value for %s: %s' % (quantity_def, str(e)))
return self.m_data.m_get(self, quantity_def)
value = self.m_data.m_get(self, quantity_def)
if isinstance(quantity_def.type, Section):
if isinstance(value, MProxy):
value = self.m_resolve(value.url)
self.m_data.m_set(self, quantity_def, value)
return value
def m_is_set(self, quantity_def: 'Quantity') -> bool:
quantity_def = self.__resolve_synonym(quantity_def)
......@@ -750,32 +768,36 @@ class MSection(metaclass=MObjectMeta):
section_def = section_cls.m_def
sub_section_def = self.m_def.all_sub_sections_by_section.get(section_def, None)
if sub_section_def is None:
raise TypeError('There is not sub section for %s in %s.' % (section_def, self))
raise TypeError('There is no sub section to hold a %s in %s.' % (section_def, self.m_def))
sub_section = section_cls(**kwargs)
self.m_add_sub_section(sub_section_def, sub_section)
return cast(MSectionBound, sub_section)
def m_update(self, **kwargs):
def m_update(self, safe: bool = True, **kwargs):
""" Updates all quantities and sub-sections with the given arguments. """
for name, value in kwargs.items():
prop = self.m_def.all_properties.get(name, None)
if prop is None:
raise KeyError('%s is not an attribute of this section %s' % (name, self))
if isinstance(prop, SubSection):
if prop.repeats:
if isinstance(value, List):
for item in value:
self.m_add_sub_section(prop, item)
if safe:
for name, value in kwargs.items():
prop = self.m_def.all_properties.get(name, None)
if prop is None:
raise KeyError('%s is not an attribute of this section %s' % (name, self))
if isinstance(prop, SubSection):
if prop.repeats:
if isinstance(value, List):
for item in value:
self.m_add_sub_section(prop, item)
raise TypeError('Sub section %s repeats, but no list was given' %
raise TypeError('Sub section %s repeats, but no list was given' %
self.m_add_sub_section(prop, item)
self.m_add_sub_section(prop, item)
self.m_set(prop, value)
self.m_set(prop, value)
self.m_data.m_data.dct.update(**kwargs) # type: ignore
def m_as(self, section_cls: Type[MSectionBound]) -> MSectionBound:
""" 'Casts' this section to the given extending sections. """
......@@ -806,19 +828,15 @@ class MSection(metaclass=MObjectMeta):
for name, quantity in self.m_def.all_quantities.items():
if self.m_is_set(quantity) and quantity.derived is None:
to_json_serializable: Callable[[Any], Any] = str
serialize: Callable[[Any], Any] = str
if isinstance(quantity.type, DataType):
def serialize(v):
quantity.type.to_json_serializable(self, v)
to_json_serializable = serialize
serialize = lambda v: quantity.type.to_json_serializable(self, v)
elif isinstance(quantity.type, Section):
to_json_serializable = str
serialize = lambda s: s.m_path()
elif quantity.type in [str, int, float, bool]:
to_json_serializable = quantity.type
serialize = quantity.type
......@@ -831,9 +849,9 @@ class MSection(metaclass=MObjectMeta):
if len(quantity.shape) == 0:
serializable_value = to_json_serializable(value)
serializable_value = serialize(value)
elif len(quantity.shape) == 1:
serializable_value = [to_json_serializable(i) for i in value]
serializable_value = [serialize(i) for i in value]
raise NotImplementedError('Higher shapes (%s) not supported: %s' % (quantity.shape, quantity))
......@@ -845,7 +863,7 @@ class MSection(metaclass=MObjectMeta):
def m_from_dict(cls: Type[MSectionBound], dct: Dict[str, Any]) -> MSectionBound:
""" Creates a section from the given data dictionary.
This is the 'oposite' of :func:`m_to_dict`. It takes a deserialized dict, e.g
This is the 'opposite' of :func:`m_to_dict`. It takes a deserialised dict, e.g
loaded from JSON, and turns it into a proper section, i.e. instance of the given
section class.
......@@ -867,8 +885,14 @@ class MSection(metaclass=MObjectMeta):
yield name, sub_section_def.sub_section.section_cls.m_from_dict(sub_section_value)
for key, value in dct.items():
yield key, value
for name, quantity_def in section_def.all_quantities.items():
if name in dct:
quantity_value = dct[name]
if isinstance(quantity_def.type, Section):
quantity_value = MProxy(quantity_value)
yield name, quantity_value
dct = {key: value for key, value in items()}
section_instance = cast(MSectionBound, section_def.section_cls())
......@@ -900,6 +924,78 @@ class MSection(metaclass=MObjectMeta):
sub_section = self.m_get_sub_section(sub_section_def, -1)
yield sub_section, -1, sub_section_def, self
def m_path(self, quantity_def: 'Quantity' = None) -> str:
""" Returns the path of this section or the given quantity within the section hierarchy. """
if self.m_parent is None:
return '/'
if self.m_parent_index == -1:
segment =
segment = '%s/%d' % (, self.m_parent_index)
if quantity_def is not None:
segment = '%s/%s' % (segment,
return '%s/%s' % (self.m_parent.m_path().rstrip('/'), segment)
def m_root(self, cls: Type[MSectionBound] = None) -> MSectionBound:
if self.m_parent is None:
return cast(MSectionBound, self)
return self.m_parent.m_root(cls)
def m_resolve(self, path: str, cls: Type[MSectionBound] = None) -> MSectionBound:
""" Resolves the given path using this section as context. """
if path.startswith('/'):
context: 'MSection' = self.m_root()
context = self
path_stack = path.strip('/').split('/')
while len(path_stack) > 1:
prop_name = path_stack.pop()
prop_def = context.m_def.all_properties.get(prop_name, None)
if prop_def is None:
raise ReferenceError(
'Could not resolve %s, property %s does not exist in %s' %
(path, prop_name, context.m_def))
if isinstance(prop_def, SubSection):
if prop_def.repeats:
index = int(path_stack.pop())
except ValueError:
raise ReferenceError(
'Could not resolve %s, %s repeats but there is no index in the path' %
(path, prop_name))
context = context.m_get_sub_section(prop_def, index)
except Exception:
raise ReferenceError(
'Could not resolve %s, there is no sub section for %s at %d' %
(path, prop_name, index))
context = context.m_get_sub_section(prop_def, -1)
if context is None:
raise ReferenceError(
'Could not resolve %s, there is no sub section for %s' %
(path, prop_name))
elif isinstance(prop_def, Quantity):
if len(path_stack) > 0:
raise ReferenceError(
'Could not resolve %s, %s is not a sub section' % (path, prop_name))
return context.m_get(prop_def)
return cast(MSectionBound, context)
def __repr__(self):
m_section_name =
# name_quantity_def = self.m_def.all_quantities.get('name', None)
......@@ -1288,7 +1384,7 @@ SubSection.sub_section = Quantity(
Quantity.m_def.section_cls = Quantity
Quantity.type = Quantity(
Quantity.type = DirectQuantity(
type=Union[type, Enum, Section, np.dtype], name='type', description='''
The type of the quantity.
......@@ -107,13 +107,13 @@ class TestM2:
assert Run.m_def.all_quantities['code_name'] == Run.__dict__['code_name']
def test_sub_sections(self):
assert len(Run.m_def.sub_sections) == 2
assert len(Run.m_def.sub_sections) == 3
assert Run.m_def.all_sub_sections['systems'] in Run.m_def.sub_sections
assert Run.m_def.all_sub_sections['systems'].sub_section == System.m_def
assert Run.m_def.all_sub_sections_by_section[System.m_def].sub_section == System.m_def
def test_properties(self):
assert len(Run.m_def.all_properties) == 5
assert len(Run.m_def.all_properties) == 6
def test_get_quantity_def(self):
assert System.n_atoms == System.m_def.all_properties['n_atoms']
......@@ -141,7 +141,7 @@ class TestM2:
def test_package(self):
assert == 'nomad.metainfo.example'
assert example_package.description == 'An example metainfo package.'
assert example_package.m_sub_section_count(Package.section_definitions) == 4
assert example_package.m_sub_section_count(Package.section_definitions) == 5
assert example_package.m_sub_section_count(Package.category_definitions) == 1
def test_base_sections(self):
......@@ -302,3 +302,10 @@ class TestM1:
run = Run()
run.m_as(VaspRun).x_vasp_raw_format = 'outcar'
assert run.m_as(VaspRun).x_vasp_raw_format == 'outcar'
def test_resolve(self):
run = Run()
system = run.m_create(System)
assert run.m_resolve('/systems/0') == system
assert system.m_resolve('/systems/0') == system
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment