Refactored legacy metainfo, metainfo derived, references, proxies.

.gitlab-ci.yml

@@ -63,7 +63,7 @@ linting:
   script:
     - cd /app
     - python -m pycodestyle --ignore=E501,E701,E731 nomad tests
-    - python -m pylint --load-plugins=pylint_mongoengine nomad tests
+    - python -m pylint --load-plugins=pylint_mongoengine,nomad/metainfo/pylint_plugin nomad tests
     - python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional nomad tests
   except:
     refs:

nomad-meta-info @ ad28fc1d

-Subproject commit 43c28c292079f819f06383af123e2e933f5f55ba
+Subproject commit ad28fc1d4787f31999b2be6ebec7ffd98bf0a3bb

abinit @ 142ba39f

-Subproject commit df889f4873be90d4a0a3d0b0d3b5fca907e9561c
+Subproject commit 142ba39f70ec298818775028631db4037494dbb0

eels @ ed3d77fd

-Subproject commit 3b9b3c802ed5c59687e863c0b5487c373a6649a9
+Subproject commit ed3d77fdb5c16ebc11c9a8b509fe5cdbc57674fd

exciting @ 528f355e

-Subproject commit d73611bc1b16ea71daa3d0fd24ee275c78853557
+Subproject commit 528f355e6a7835f630db41fbfa8a9dae4be4991a

onetep @ 40c58ba1

-Subproject commit bd5b5c6f947ec9b7172ef7970a92825c737e1e60
+Subproject commit 40c58ba15fe6434aa8522770dec3650385ad22f9

vasp @ 6e4654fa

-Subproject commit 5f07d80f9d1838b3f6b95e39266221002061e0d1
+Subproject commit 6e4654fa3b016fa099104d3f33078c510c980434

nomad/app/api/dataset.py

@@ -95,7 +95,7 @@ class DatasetListResource(Resource):
                 abort(400, 'The dataset contains information you are not allowed to set.')
 
         # no other keys
-        if any(key not in Dataset.m_def.all_quantities for key in data):
+        if any(key not in Dataset.m_def.all_quantities for key in data):  # pylint: disable=all
             abort(400, 'The dataset contains unknown keys.')
 
         if 'user_id' not in data:

nomad/cli/admin/admin.py

@@ -327,7 +327,7 @@ RewriteCond %{QUERY_STRING} ^pid=([^&]+)$
 RewriteRule ^/NomadRepository-1.1/views/calculation.zul$ /{0}/gui/entry/pid/%1? [R=301]
 
 AllowEncodedSlashes On
-'''.format(prefix, host, port))
+'''.format(prefix, host, port))  # type: ignore
 
 
 def write_prototype_data_file(aflow_prototypes: dict, filepath) -> None:

nomad/cli/dev.py

@@ -37,7 +37,7 @@ def qa(skip_tests: bool, exitfirst: bool):
     click.echo('Run code style checks ...')
     ret_code += os.system('python -m pycodestyle --ignore=E501,E701,E731 nomad tests')
     click.echo('Run linter ...')
-    ret_code += os.system('python -m pylint --load-plugins=pylint_mongoengine nomad tests')
+    ret_code += os.system('python -m pylint --load-plugins=pylint_mongoengine,nomad/metainfo/pylint_plugin nomad tests')
     click.echo('Run static type checks ...')
     ret_code += os.system('python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional nomad tests')
 

nomad/cli/parse.py

@@ -6,7 +6,7 @@ import sys
 
 from nomad import config, utils, files
 from nomad.parsing import LocalBackend, parser_dict, match_parser, MatchingParser, MetainfoBackend
-from nomad.metainfo.legacy import LegacyMetainfoEnvironment
+from nomad.metainfo.legacy import convert
 from nomad.normalizing import normalizers
 from nomad.datamodel import EntryMetadata
 
@@ -119,7 +119,7 @@ def _parse(
     if metainfo:
 
         def backend_factory(env, logger):
-            return MetainfoBackend(LegacyMetainfoEnvironment(env), logger=logger)
+            return MetainfoBackend(convert(env), logger=logger)
 
         kwargs.update(backend_factory=backend_factory)
 

nomad/datamodel/datamodel.py

@@ -108,14 +108,8 @@ class UserReference(metainfo.Reference):
     def __init__(self):
         super().__init__(User.m_def)
 
-    def set_normalize(self, section: metainfo.MSection, quantity_def: metainfo.Quantity, value: Any) -> Any:
-        if isinstance(value, str):
-            return metainfo.MProxy(value)
-        else:
-            return super().set_normalize(section, quantity_def, value)
-
-    def resolve(self, section: metainfo.MSection, quantity_def: metainfo.Quantity, value: Any) -> metainfo.MSection:
-        return User.get(user_id=value.url)
+    def resolve(self, proxy: metainfo.MProxy) -> metainfo.MSection:
+        return User.get(user_id=proxy.m_proxy_url)
 
     def serialize(self, section: metainfo.MSection, quantity_def: metainfo.Quantity, value: Any) -> Any:
         return value.user_id
@@ -178,24 +172,15 @@ class DatasetReference(metainfo.Reference):
     def __init__(self):
         super().__init__(Dataset.m_def)
 
-    def set_normalize(self, section: metainfo.MSection, quantity_def: metainfo.Quantity, value: Any) -> Any:
-        if isinstance(value, str):
-            return metainfo.MProxy(value)
-        else:
-            return super().set_normalize(section, quantity_def, value)
-
-    def resolve(self, section: metainfo.MSection, quantity_def: metainfo.Quantity, value: Any) -> metainfo.MSection:
-        return Dataset.m_def.a_mongo.get(dataset_id=value.url)
+    def resolve(self, proxy: metainfo.MProxy) -> metainfo.MSection:
+        return Dataset.m_def.a_mongo.get(dataset_id=proxy.m_proxy_url)
 
     def serialize(self, section: metainfo.MSection, quantity_def: metainfo.Quantity, value: Any) -> Any:
         if isinstance(value, metainfo.MProxy):
-            return value.url
+            return value.m_proxy_url
         else:
             return value.user_id
 
-    def deserialize(self, section: metainfo.MSection, quantity_def: metainfo.Quantity, value: Any) -> Any:
-        return metainfo.MProxy(value)
-
 
 dataset_reference = DatasetReference()
 

nomad/metainfo/__init__.py

@@ -299,4 +299,7 @@ from .metainfo import (
     units,
     Annotation,
     DefinitionAnnotation,
-    SectionAnnotation)
+    SectionAnnotation,
+    SectionProxy,
+    derived,
+    constraint)

nomad/metainfo/example.py

+# Copyright 2018 Markus Scheidgen
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an"AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 ''' An example metainfo package. '''
 
 import numpy as np
 from datetime import datetime
 
-from nomad.metainfo import MSection, MCategory, Section, Quantity, Package, SubSection, MEnum, Datetime, units
+from nomad.metainfo import (
+    MSection, MCategory, Section, Quantity, Package, SubSection, MEnum, Datetime, units,
+    constraint)
 
 m_package = Package(links=['http://metainfo.nomad-coe.eu'])
 
@@ -58,6 +74,8 @@ class System(MSection):
 class SCC(MSection):
 
     energy_total = Quantity(type=float, default=0.0, unit=units.J)
+    energy_total_0 = Quantity(type=np.dtype(np.float32), default=0.0, unit=units.J)
+    an_int = Quantity(type=np.dtype(np.int32))
 
     system = Quantity(type=System, description='The system that this calculation is based on.')
 
@@ -72,7 +90,8 @@ class Run(MSection):
     systems = SubSection(sub_section=System, repeats=True)
     sccs = SubSection(sub_section=SCC, repeats=True)
 
-    def c_one_scc_per_system(self):
+    @constraint
+    def one_scc_per_system(self):
         assert self.m_sub_section_count(Run.systems) == self.m_sub_section_count(Run.sccs),\
             'Numbers of system does not match numbers of calculations.'
 
@@ -96,7 +115,7 @@ if __name__ == '__main__':
     print(Run.m_def.m_get_sub_sections(Section.quantities))
 
     # Or all Sections in the package
-    print(m_package.m_get_sub_sections(Package.section_definitions))  # type: ignore, pylint: disable=undefined-variable
+    print(m_package.m_get_sub_sections(Package.section_definitions))
 
     # There are also some definition specific helper methods.
     # For example to get all attributes (Quantities and possible sub-sections) of a section.

nomad/metainfo/flask_extension.py

+# Copyright 2018 Markus Scheidgen
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an"AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 from flask_restplus import fields
 
 from nomad.app.common import RFC3339DateTime

nomad/metainfo/pylint_plugin.py

@@ -24,7 +24,8 @@ annotation_names = {
         'a_test': '*'
     },
     'Section': {
-        'a_elastic': 'nomad.metainfo.elastic_extension.ElasticDocument'
+        'a_elastic': 'nomad.metainfo.elastic_extension.ElasticDocument',
+        'a_mongo': 'nomad.metainfo.mongoengine_extension.MongoDocument'
     },
     'Quantity': {
         'a_elastic': 'nomad.metainfo.elastic_extension.Elastic'
@@ -38,6 +39,7 @@ def register(linter):
 
 
 def transform(cls):
+    ''' Transforms annotation fields for known annotation classes. '''
     for cls_name, annotations in annotation_names.items():
         if cls.name == cls_name:
             for name, type_spec in annotations.items():
@@ -51,4 +53,26 @@ def transform(cls):
                     cls.locals[name] = [cls.instantiate_class() for cls in module.lookup(type_name)[1]]
 
 
+def is_derived(func):
+    ''' Check if this is call to the derived decorator. '''
+    decorators = func.decorators.nodes if func.decorators else []
+    for decorator_node in decorators:
+        if isinstance(decorator_node, astroid.Call):
+            if decorator_node.func and isinstance(decorator_node.func, astroid.Name):
+                return decorator_node.func.name == 'derived'
+
+    return False
+
+
+def derived_transform(node, context=None):
+    '''
+    The derived decorator produces a Quantity. Pylint does not infer this on its own.
+    We change the inferred type of a @derived call to a Quantity instance here.
+    '''
+    module = MANAGER.ast_from_module_name('nomad.metainfo.metainfo')
+    class_defs = [cls.instantiate_class() for cls in module.lookup('Quantity')[1]]
+    return iter([class_defs[0].instantiate_class()])
+
+
 MANAGER.register_transform(astroid.ClassDef, transform)
+MANAGER.register_transform(astroid.FunctionDef, astroid.inference_tip(derived_transform), is_derived)

nomad/metainfo/templates/package.j2

-import numpy as np
-import typing
-from nomad.metainfo import MSection, Package, Quantity, SubSection, MProxy
+import numpy as np            # pylint: disable=unused-import
+import typing                 # pylint: disable=unused-import
+from nomad.metainfo import (  # pylint: disable=unused-import
+    MSection, MCategory, Category, Package, Quantity, Section, SubSection, SectionProxy
+)
+{% for dependency in pkg.dependencies %}
+from . import {{dependency.name}}
+{%- endfor %}
 
 m_package = Package(name='{{ pkg.name }}', description='{{ pkg.description }}')
+{% for category in pkg.category_definitions %}
+
+class {{ category.name }}(MCategory):
+    {% if category.description is not none -%}
+    '''
+    {{ format_description(category.description, indent=1) }}
+    '''
+    {% elif category.categories | length > 0 %}
+    m_def = Category(categories=[{{ format_definition_refs(pkg, category.categories) }}])
+    {% else %}
+    pass
+    {% endif %}
+{%- endfor -%}
+
 {% for section in pkg.section_definitions %}
 
-class {{ section.name }}(MSection):
+class {{ section.name }}({%- if section.extends_base_section -%}{{ format_definition_refs(pkg, section.base_sections)}}{%- else -%}MSection{%- endif -%}):
     {% if section.description is not none -%}
     '''
     {{ format_description(section.description, indent=1) }}
     '''
     {% endif %}
-
-    {%- for quantity in section.quantities %}
+    m_def = Section(validate=False{%- if section.extends_base_section -%}, extends_base_section=True{%- endif -%})
+    {% for quantity in section.quantities %}
     {{ quantity.name }} = Quantity(
         type={{ format_type(quantity.type) }},
         shape={{ quantity.shape }}
@@ -22,12 +41,15 @@ class {{ section.name }}(MSection):
         {%- if quantity.description is not none -%},
         description='''
         {{ format_description(quantity.description, indent=2) }}
-        '''{%- endif -%})
+        '''{%- endif -%}
+        {%- if quantity.categories | length > 0 -%},
+        categories=[{{ format_definition_refs(pkg, quantity.categories) }}]
+        {%- endif -%})
     {% endfor -%}
 
     {%- for sub_section in section.sub_sections %}
     {{ sub_section.name }} = SubSection(
-        sub_section=MProxy('{{ sub_section.sub_section.name }}'),
+        sub_section=SectionProxy('{{ sub_section.sub_section.name }}'),
         repeats={{ sub_section.repeats }}
         {%- if sub_section.description is not none -%},
         description='''
@@ -35,3 +57,6 @@ class {{ section.name }}(MSection):
         '''{%- endif -%})
     {% endfor -%}
 {%- endfor %}
+
+m_package.__init_metainfo__()
+

nomad/normalizing/dos.py

@@ -35,11 +35,21 @@ class DosNormalizer(Normalizer):
                     # section dos without doc_values
                     continue
 
-                system_index = self._backend.get_value(
-                    'single_configuration_calculation_to_system_ref', scc_index)
+                try:
+                    system_index = self._backend.get_value(
+                        'single_configuration_calculation_to_system_ref', scc_index)
+                except KeyError:
+                    system_index = scc_index
 
-                atom_positions = self._backend.get_value('atom_positions', system_index)
-                lattice_vectors = self._backend.get_value('lattice_vectors', system_index)
+                try:
+                    atom_positions = self._backend.get_value('atom_positions', system_index)
+                    lattice_vectors = self._backend.get_value('lattice_vectors', system_index)
+                except IndexError:
+                    self.logger.error('referenced system for dos calculation could not be found')
+                    return
+                except KeyError as e:
+                    self.logger.error('required quantity %s is not available' % e.args[0])
+                    return
 
                 number_of_atoms = np.shape(atom_positions)[0]
                 unit_cell_volume = np.linalg.det(lattice_vectors)
@@ -48,4 +58,7 @@ class DosNormalizer(Normalizer):
                 dos_normed = dos / (number_of_atoms * unit_cell_volume)
 
                 # Add quantities to NOMAD's Metainfo
+                scc_url = '/section_run/0/section_single_configuration_calculation/%d/section_dos/0' % scc_index
+                self._backend.openContext(scc_url)
                 self._backend.addArrayValues('dos_values_normalized', dos_normed, dos_index)
+                self._backend.closeContext(scc_url)