Commit 3518271f authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Refactored legacy metainfo, metainfo derived, references, proxies.

parent 0c5188ef
......@@ -63,7 +63,7 @@ linting:
script:
- cd /app
- python -m pycodestyle --ignore=E501,E701,E731 nomad tests
- python -m pylint --load-plugins=pylint_mongoengine nomad tests
- python -m pylint --load-plugins=pylint_mongoengine,nomad/metainfo/pylint_plugin nomad tests
- python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional nomad tests
except:
refs:
......
Subproject commit 43c28c292079f819f06383af123e2e933f5f55ba
Subproject commit ad28fc1d4787f31999b2be6ebec7ffd98bf0a3bb
Subproject commit df889f4873be90d4a0a3d0b0d3b5fca907e9561c
Subproject commit 142ba39f70ec298818775028631db4037494dbb0
Subproject commit 3b9b3c802ed5c59687e863c0b5487c373a6649a9
Subproject commit ed3d77fdb5c16ebc11c9a8b509fe5cdbc57674fd
Subproject commit d73611bc1b16ea71daa3d0fd24ee275c78853557
Subproject commit 528f355e6a7835f630db41fbfa8a9dae4be4991a
Subproject commit bd5b5c6f947ec9b7172ef7970a92825c737e1e60
Subproject commit 40c58ba15fe6434aa8522770dec3650385ad22f9
Subproject commit 5f07d80f9d1838b3f6b95e39266221002061e0d1
Subproject commit 6e4654fa3b016fa099104d3f33078c510c980434
......@@ -95,7 +95,7 @@ class DatasetListResource(Resource):
abort(400, 'The dataset contains information you are not allowed to set.')
# no other keys
if any(key not in Dataset.m_def.all_quantities for key in data):
if any(key not in Dataset.m_def.all_quantities for key in data): # pylint: disable=all
abort(400, 'The dataset contains unknown keys.')
if 'user_id' not in data:
......
......@@ -327,7 +327,7 @@ RewriteCond %{QUERY_STRING} ^pid=([^&]+)$
RewriteRule ^/NomadRepository-1.1/views/calculation.zul$ /{0}/gui/entry/pid/%1? [R=301]
AllowEncodedSlashes On
'''.format(prefix, host, port))
'''.format(prefix, host, port)) # type: ignore
def write_prototype_data_file(aflow_prototypes: dict, filepath) -> None:
......
......@@ -37,7 +37,7 @@ def qa(skip_tests: bool, exitfirst: bool):
click.echo('Run code style checks ...')
ret_code += os.system('python -m pycodestyle --ignore=E501,E701,E731 nomad tests')
click.echo('Run linter ...')
ret_code += os.system('python -m pylint --load-plugins=pylint_mongoengine nomad tests')
ret_code += os.system('python -m pylint --load-plugins=pylint_mongoengine,nomad/metainfo/pylint_plugin nomad tests')
click.echo('Run static type checks ...')
ret_code += os.system('python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional nomad tests')
......
......@@ -6,7 +6,7 @@ import sys
from nomad import config, utils, files
from nomad.parsing import LocalBackend, parser_dict, match_parser, MatchingParser, MetainfoBackend
from nomad.metainfo.legacy import LegacyMetainfoEnvironment
from nomad.metainfo.legacy import convert
from nomad.normalizing import normalizers
from nomad.datamodel import EntryMetadata
......@@ -119,7 +119,7 @@ def _parse(
if metainfo:
def backend_factory(env, logger):
return MetainfoBackend(LegacyMetainfoEnvironment(env), logger=logger)
return MetainfoBackend(convert(env), logger=logger)
kwargs.update(backend_factory=backend_factory)
......
......@@ -108,14 +108,8 @@ class UserReference(metainfo.Reference):
def __init__(self):
super().__init__(User.m_def)
def set_normalize(self, section: metainfo.MSection, quantity_def: metainfo.Quantity, value: Any) -> Any:
if isinstance(value, str):
return metainfo.MProxy(value)
else:
return super().set_normalize(section, quantity_def, value)
def resolve(self, section: metainfo.MSection, quantity_def: metainfo.Quantity, value: Any) -> metainfo.MSection:
return User.get(user_id=value.url)
def resolve(self, proxy: metainfo.MProxy) -> metainfo.MSection:
return User.get(user_id=proxy.m_proxy_url)
def serialize(self, section: metainfo.MSection, quantity_def: metainfo.Quantity, value: Any) -> Any:
return value.user_id
......@@ -178,24 +172,15 @@ class DatasetReference(metainfo.Reference):
def __init__(self):
super().__init__(Dataset.m_def)
def set_normalize(self, section: metainfo.MSection, quantity_def: metainfo.Quantity, value: Any) -> Any:
if isinstance(value, str):
return metainfo.MProxy(value)
else:
return super().set_normalize(section, quantity_def, value)
def resolve(self, section: metainfo.MSection, quantity_def: metainfo.Quantity, value: Any) -> metainfo.MSection:
return Dataset.m_def.a_mongo.get(dataset_id=value.url)
def resolve(self, proxy: metainfo.MProxy) -> metainfo.MSection:
return Dataset.m_def.a_mongo.get(dataset_id=proxy.m_proxy_url)
def serialize(self, section: metainfo.MSection, quantity_def: metainfo.Quantity, value: Any) -> Any:
if isinstance(value, metainfo.MProxy):
return value.url
return value.m_proxy_url
else:
return value.user_id
def deserialize(self, section: metainfo.MSection, quantity_def: metainfo.Quantity, value: Any) -> Any:
return metainfo.MProxy(value)
dataset_reference = DatasetReference()
......
......@@ -299,4 +299,7 @@ from .metainfo import (
units,
Annotation,
DefinitionAnnotation,
SectionAnnotation)
SectionAnnotation,
SectionProxy,
derived,
constraint)
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
''' An example metainfo package. '''
import numpy as np
from datetime import datetime
from nomad.metainfo import MSection, MCategory, Section, Quantity, Package, SubSection, MEnum, Datetime, units
from nomad.metainfo import (
MSection, MCategory, Section, Quantity, Package, SubSection, MEnum, Datetime, units,
constraint)
m_package = Package(links=['http://metainfo.nomad-coe.eu'])
......@@ -58,6 +74,8 @@ class System(MSection):
class SCC(MSection):
energy_total = Quantity(type=float, default=0.0, unit=units.J)
energy_total_0 = Quantity(type=np.dtype(np.float32), default=0.0, unit=units.J)
an_int = Quantity(type=np.dtype(np.int32))
system = Quantity(type=System, description='The system that this calculation is based on.')
......@@ -72,7 +90,8 @@ class Run(MSection):
systems = SubSection(sub_section=System, repeats=True)
sccs = SubSection(sub_section=SCC, repeats=True)
def c_one_scc_per_system(self):
@constraint
def one_scc_per_system(self):
assert self.m_sub_section_count(Run.systems) == self.m_sub_section_count(Run.sccs),\
'Numbers of system does not match numbers of calculations.'
......@@ -96,7 +115,7 @@ if __name__ == '__main__':
print(Run.m_def.m_get_sub_sections(Section.quantities))
# Or all Sections in the package
print(m_package.m_get_sub_sections(Package.section_definitions)) # type: ignore, pylint: disable=undefined-variable
print(m_package.m_get_sub_sections(Package.section_definitions))
# There are also some definition specific helper methods.
# For example to get all attributes (Quantities and possible sub-sections) of a section.
......
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from flask_restplus import fields
from nomad.app.common import RFC3339DateTime
......
This diff is collapsed.
This diff is collapsed.
......@@ -24,7 +24,8 @@ annotation_names = {
'a_test': '*'
},
'Section': {
'a_elastic': 'nomad.metainfo.elastic_extension.ElasticDocument'
'a_elastic': 'nomad.metainfo.elastic_extension.ElasticDocument',
'a_mongo': 'nomad.metainfo.mongoengine_extension.MongoDocument'
},
'Quantity': {
'a_elastic': 'nomad.metainfo.elastic_extension.Elastic'
......@@ -38,6 +39,7 @@ def register(linter):
def transform(cls):
''' Transforms annotation fields for known annotation classes. '''
for cls_name, annotations in annotation_names.items():
if cls.name == cls_name:
for name, type_spec in annotations.items():
......@@ -51,4 +53,26 @@ def transform(cls):
cls.locals[name] = [cls.instantiate_class() for cls in module.lookup(type_name)[1]]
def is_derived(func):
''' Check if this is call to the derived decorator. '''
decorators = func.decorators.nodes if func.decorators else []
for decorator_node in decorators:
if isinstance(decorator_node, astroid.Call):
if decorator_node.func and isinstance(decorator_node.func, astroid.Name):
return decorator_node.func.name == 'derived'
return False
def derived_transform(node, context=None):
'''
The derived decorator produces a Quantity. Pylint does not infer this on its own.
We change the inferred type of a @derived call to a Quantity instance here.
'''
module = MANAGER.ast_from_module_name('nomad.metainfo.metainfo')
class_defs = [cls.instantiate_class() for cls in module.lookup('Quantity')[1]]
return iter([class_defs[0].instantiate_class()])
MANAGER.register_transform(astroid.ClassDef, transform)
MANAGER.register_transform(astroid.FunctionDef, astroid.inference_tip(derived_transform), is_derived)
import numpy as np
import typing
from nomad.metainfo import MSection, Package, Quantity, SubSection, MProxy
import numpy as np # pylint: disable=unused-import
import typing # pylint: disable=unused-import
from nomad.metainfo import ( # pylint: disable=unused-import
MSection, MCategory, Category, Package, Quantity, Section, SubSection, SectionProxy
)
{% for dependency in pkg.dependencies %}
from . import {{dependency.name}}
{%- endfor %}
m_package = Package(name='{{ pkg.name }}', description='{{ pkg.description }}')
{% for category in pkg.category_definitions %}
class {{ category.name }}(MCategory):
{% if category.description is not none -%}
'''
{{ format_description(category.description, indent=1) }}
'''
{% elif category.categories | length > 0 %}
m_def = Category(categories=[{{ format_definition_refs(pkg, category.categories) }}])
{% else %}
pass
{% endif %}
{%- endfor -%}
{% for section in pkg.section_definitions %}
class {{ section.name }}(MSection):
class {{ section.name }}({%- if section.extends_base_section -%}{{ format_definition_refs(pkg, section.base_sections)}}{%- else -%}MSection{%- endif -%}):
{% if section.description is not none -%}
'''
{{ format_description(section.description, indent=1) }}
'''
{% endif %}
{%- for quantity in section.quantities %}
m_def = Section(validate=False{%- if section.extends_base_section -%}, extends_base_section=True{%- endif -%})
{% for quantity in section.quantities %}
{{ quantity.name }} = Quantity(
type={{ format_type(quantity.type) }},
shape={{ quantity.shape }}
......@@ -22,12 +41,15 @@ class {{ section.name }}(MSection):
{%- if quantity.description is not none -%},
description='''
{{ format_description(quantity.description, indent=2) }}
'''{%- endif -%})
'''{%- endif -%}
{%- if quantity.categories | length > 0 -%},
categories=[{{ format_definition_refs(pkg, quantity.categories) }}]
{%- endif -%})
{% endfor -%}
{%- for sub_section in section.sub_sections %}
{{ sub_section.name }} = SubSection(
sub_section=MProxy('{{ sub_section.sub_section.name }}'),
sub_section=SectionProxy('{{ sub_section.sub_section.name }}'),
repeats={{ sub_section.repeats }}
{%- if sub_section.description is not none -%},
description='''
......@@ -35,3 +57,6 @@ class {{ section.name }}(MSection):
'''{%- endif -%})
{% endfor -%}
{%- endfor %}
m_package.__init_metainfo__()
......@@ -35,11 +35,21 @@ class DosNormalizer(Normalizer):
# section dos without doc_values
continue
system_index = self._backend.get_value(
'single_configuration_calculation_to_system_ref', scc_index)
try:
system_index = self._backend.get_value(
'single_configuration_calculation_to_system_ref', scc_index)
except KeyError:
system_index = scc_index
atom_positions = self._backend.get_value('atom_positions', system_index)
lattice_vectors = self._backend.get_value('lattice_vectors', system_index)
try:
atom_positions = self._backend.get_value('atom_positions', system_index)
lattice_vectors = self._backend.get_value('lattice_vectors', system_index)
except IndexError:
self.logger.error('referenced system for dos calculation could not be found')
return
except KeyError as e:
self.logger.error('required quantity %s is not available' % e.args[0])
return
number_of_atoms = np.shape(atom_positions)[0]
unit_cell_volume = np.linalg.det(lattice_vectors)
......@@ -48,4 +58,7 @@ class DosNormalizer(Normalizer):
dos_normed = dos / (number_of_atoms * unit_cell_volume)
# Add quantities to NOMAD's Metainfo
scc_url = '/section_run/0/section_single_configuration_calculation/%d/section_dos/0' % scc_index
self._backend.openContext(scc_url)
self._backend.addArrayValues('dos_values_normalized', dos_normed, dos_index)
self._backend.closeContext(scc_url)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment