Refactored metainfo to separate section and sub section.

nomad/app/optimade/endpoints.py

@@ -128,11 +128,11 @@ class CalculationInfo(Resource):
                 # TODO non optimade, nomad specific properties
                 'properties': {
                     attr.name: dict(description=attr.description)
-                    for attr in OptimadeStructureEntry.m_def.attributes.values()
+                    for attr in OptimadeStructureEntry.m_def.all_properties.values()
                 },
                 'formats': ['json'],
                 'output_fields_by_format': {
-                    'json': OptimadeStructureEntry.m_def.attributes.keys()
+                    'json': OptimadeStructureEntry.m_def.all_properties.keys()
                 }
             }
         }

nomad/app/optimade/filterparser.py

@@ -29,7 +29,7 @@ quantities: Dict[str, Quantity] = {
         q.name, es_field='optimade.%s' % q.name,
         elastic_mapping_type=q.m_annotations['elastic']['type'])
 
-    for q in OptimadeStructureEntry.m_def.quantities.values()
+    for q in OptimadeStructureEntry.m_def.all_quantities.values()
     if 'elastic' in q.m_annotations}
 
 quantities['elements'].length_quantity = quantities['nelements']

nomad/metainfo/CONCEPT.md

@@ -79,6 +79,8 @@ the *backend*s for data were designed for creating data only and not very *pytho
 - `categories`, a list of references to category definitions
 - `annotations`, a list of `Annotations`
 
+- *derived*: `qualified_name`
+
 
 ### Property
 
@@ -123,6 +125,10 @@ A `Section` is a special and concrete `Definition`.
 - `repeats`, a boolean
 - `extends`, list of reference to other section definitions. This section automatically inherits all quantities of the other sections. (Might not be necessary)
 
+- *derived*: `all_sub_sections`, all sub sections, included added and inherited ones, by name
+- *derived*: `all_quantities`, all quantities, included added and inherited ones, by name
+- *derived*: `all_properties`, all properties, included added and inherited ones, by name
+
 - *constraint*: `extends` is not circular
 - *constraint*: `adds_to` is not circular
 - *constraint*: all quantities that have *this* (or an `extends`) section as `section` have unique names
@@ -142,12 +148,30 @@ A `Category` is a special `Definition`.
 A `Package` is a special `Definition` that contains definitions. `Packages` are mapped
 to Python modules.
 
+- *derived*: `definitions`, all definitions in this package
+- *derived*: `sections`, all sections in this package
+- *derived*: `categories`, all categories in this package
+
 
 ### Annotations
 
 Arbitrary serializable objects that can contain additional information.
 
 
+### MSection
+
+`MSection` is a Python base-class for all sections and provides additional reflection.
+
+- `m_def`: Python variable with the definition of this section
+- `m_data`: container for all the section data
+- `m_parent`: Python variable with the parent section instance
+- `m_parent_index`: Python variable with the index in the parent's repeatable sub section
+- `m_contents()`: all sub section instances
+- `m_all_contents()`: traverse all sub and sub sub section instances
+- `m_to_dict()`: serializable dict form
+- `m_to_json()`
+
+
 ## Examples (of the Python interface)
 
 ### Definitions

nomad/metainfo/__init__.py

-from .metainfo import MSection, MCategory, Definition, Section, Quantity, Category, Package, Enum, units
+from .metainfo import MSection, MCategory, Definition, Property, Quantity, SubSection, \
+    Section, Category, Package, Enum, units

nomad/metainfo/example.py

@@ -2,7 +2,7 @@
 
 import numpy as np
 
-from nomad.metainfo import MSection, MCategory, Section, Quantity, Enum, Package, units
+from nomad.metainfo import MSection, MCategory, Section, Quantity, Enum, Package, SubSection, units
 
 m_package = Package(links=['http://metainfo.nomad-coe.eu'])
 
@@ -11,25 +11,8 @@ class SystemHash(MCategory):
     """ All quantities that contribute to what makes a system unique. """
 
 
-class Run(MSection):
-    """ All data that belongs to a single code run. """
-
-    code_name = Quantity(type=str, description='The name of the code that was run.')
-    code_version = Quantity(type=str, description='The version of the code that was run.')
-
-
-class VaspRun(MSection):
-    """ All VASP specific quantities for section Run. """
-    m_def = Section(extends=Run.m_def)
-
-    x_vasp_raw_format = Quantity(
-        type=Enum(['xml', 'outcar']),
-        description='The file format of the parsed VASP mainfile.')
-
-
 class Parsing(MSection):
     """ All data that describes the NOMAD parsing of this run. """
-    m_def = Section(parent=Run.m_def)
 
     parser_name = Quantity(type=str)
     parser_version = Quantity(type=str)
@@ -39,7 +22,6 @@ class Parsing(MSection):
 
 class System(MSection):
     """ All data that describes a simulated system. """
-    m_def = Section(repeats=True, parent=Run.m_def)
 
     n_atoms = Quantity(
         type=int, default=0,
@@ -62,6 +44,25 @@ class System(MSection):
         description='A vector of booleans indicating in which dimensions the unit cell is repeated.')
 
 
+class Run(MSection):
+    """ All data that belongs to a single code run. """
+
+    code_name = Quantity(type=str, description='The name of the code that was run.')
+    code_version = Quantity(type=str, description='The version of the code that was run.')
+
+    systems = SubSection(sub_section=System.m_def, repeats=True)
+    parsing = SubSection(sub_section=Parsing.m_def)
+
+
+class VaspRun(MSection):
+    """ All VASP specific quantities for section Run. """
+    m_def = Section(extends=Run.m_def)
+
+    x_vasp_raw_format = Quantity(
+        type=Enum(['xml', 'outcar']),
+        description='The file format of the parsed VASP mainfile.')
+
+
 if __name__ == '__main__':
     # Demonstration of how to reflect on the definitions
 
@@ -76,7 +77,7 @@ if __name__ == '__main__':
 
     # There are also some definition specific helper methods.
     # For example to get all attributes (Quantities and possible sub-sections) of a section.
-    print(Run.m_def.attributes)
+    print(Run.m_def.all_properties)
 
     # Demonstration on how to use the definitions, e.g. to create a run with system:
     run = Run()

nomad/metainfo/optimade.py

@@ -238,7 +238,7 @@ def elastic_mapping(section: Section, base_cls: type) -> type:
 
     dct = {
         name: quantity.m_annotations['elastic']['type']()
-        for name, quantity in section.quantities.items()
+        for name, quantity in section.all_quantities.items()
         if 'elastic' in quantity.m_annotations}
 
     return type(section.name, (base_cls,), dct)
@@ -252,7 +252,7 @@ def elastic_obj(source: MSection, target_cls: type):
 
     target = target_cls()
 
-    for name, quantity in source.m_def.quantities.items():
+    for name, quantity in source.m_def.all_quantities.items():
         elastic_annotation = quantity.m_annotations.get('elastic')
         if elastic_annotation is None:
             continue

tests/test_metainfo.py

@@ -15,7 +15,7 @@
 import pytest
 import numpy as np
 
-from nomad.metainfo.metainfo import MSection, MCategory, Section, Quantity, Definition, Category, sub_section
+from nomad.metainfo.metainfo import MSection, MCategory, Section, Quantity, Definition, Category, SubSection
 from nomad.metainfo.example import Run, System, SystemHash, Parsing, m_package as example_package
 
 
@@ -28,13 +28,6 @@ def assert_section_def(section_def: Section):
 
     assert section_def.name is not None
 
-    if section_def.parent is not None:
-        if section_def.parent != section_def:
-            assert_section_def(section_def.parent)
-
-    if section_def.repeats:
-        assert section_def.parent is not None
-
 
 def assert_section_instance(section: MSection):
     assert_section_def(section.m_def)
@@ -54,14 +47,25 @@ class TestM3:
         assert Section.name.m_def == Quantity.m_def
         assert Section.description.description is not None
 
-        assert Section.m_def.m_sub_section(Quantity, 0).name in Section.m_def.attributes
+        for quantity in Section.m_def.quantities:
+            assert quantity.name in Section.m_def.all_properties
+            assert quantity.name in Section.m_def.all_quantities
+            assert quantity.m_parent == Section.m_def
+
+        for sub_section in Section.m_def.sub_sections:
+            assert sub_section.name in Section.m_def.all_properties
+            assert sub_section.name in Section.m_def.all_sub_sections
+            assert sub_section.sub_section in Section.m_def.all_sub_sections_by_section
+            assert sub_section.m_parent == Section.m_def
+
+        assert 'quantities' in Section.m_def.all_sub_sections
+        assert 'sub_sections' in Section.m_def.all_sub_sections
 
         assert_section_instance(Section.m_def)
 
     def test_quantity(self):
         assert Quantity.m_def.m_def == Section.m_def
         assert Quantity.m_def.name == 'Quantity'
-        assert Quantity.m_def.parent == Section.m_def
 
         assert_section_instance(Quantity.m_def)
 
@@ -96,34 +100,23 @@ class TestM2:
         """ A section class without an explicit section def must set a default section def. """
         assert Run.m_def is not None
         assert Run.m_def.name == 'Run'
-        assert not Run.m_def.repeats
-        assert Run.m_def.parent is None
 
     def test_quantities(self):
         assert len(Run.m_def.quantities) == 2
-        assert Run.m_def.quantities['code_name'] == Run.__dict__['code_name']
+        assert Run.m_def.all_quantities['code_name'] in Run.m_def.quantities
+        assert Run.m_def.all_quantities['code_name'] == Run.__dict__['code_name']
 
     def test_sub_sections(self):
         assert len(Run.m_def.sub_sections) == 2
-        assert Run.m_def.sub_sections['System'] == System.m_def
+        assert Run.m_def.all_sub_sections['systems'] in Run.m_def.sub_sections
+        assert Run.m_def.all_sub_sections['systems'].sub_section == System.m_def
+        assert Run.m_def.all_sub_sections_by_section[System.m_def].sub_section == System.m_def
 
-    def test_attributes(self):
-        assert len(Run.m_def.attributes) == 4
-        assert Run.m_def.attributes['System'] == System.m_def
-        assert Run.m_def.attributes['code_name'] == Run.__dict__['code_name']
+    def test_properties(self):
+        assert len(Run.m_def.all_properties) == 4
 
     def test_get_quantity_def(self):
-        assert System.n_atoms == System.m_def.attributes['n_atoms']
-
-    def test_add_quantity(self):
-        System.m_def.add_quantity(Quantity(name='test', type=str))
-
-        system = System()
-        system.test = 'test_value'
-
-        assert 'test' in system.m_data
-        assert system.test == 'test_value'
-        assert getattr(System, 'test') == System.m_def.quantities['test']
+        assert System.n_atoms == System.m_def.all_properties['n_atoms']
 
     def test_section_name(self):
         assert Run.m_def.name == 'Run'
@@ -197,16 +190,7 @@ class TestM1:
         run = Run()
         system = run.m_create(System)
 
-        assert run.system[0] == system  # pylint: disable=E1101
-        assert run.m_sub_section(System, 0) == system
-
-    def test_children_sub_section(self):
-        setattr(Run, 'a_system_sub_section', sub_section(System))
-        run = Run()
-        system = run.m_create(System)
-
-        assert run.a_system_sub_section[0] == system  # pylint: disable=E1101
-        assert run.system[0] == system  # pylint: disable=E1101
+        assert run.systems[0] == system  # pylint: disable=E1101
         assert run.m_sub_section(System, 0) == system
 
     def test_parent_repeats(self):