Commit 335008d7 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Merge branch 'v0.10.4' into 'master'

Merge for release

See merge request !357
parents 0bde4980 6919ccf6
Pipeline #103569 passed with stage
in 4 minutes and 55 seconds
...@@ -117,6 +117,12 @@ class User(Author): ...@@ -117,6 +117,12 @@ class User(Author):
from nomad import infrastructure from nomad import infrastructure
return infrastructure.keycloak.get_user(*args, **kwargs) # type: ignore return infrastructure.keycloak.get_user(*args, **kwargs) # type: ignore
def full_user(self) -> 'User':
''' Returns a User object with all attributes loaded from the user management system. '''
from nomad import infrastructure
assert self.user_id is not None
return infrastructure.keycloak.get_user(user_id=self.user_id) # type: ignore
class UserReference(metainfo.Reference): class UserReference(metainfo.Reference):
''' '''
......
...@@ -18,8 +18,7 @@ ...@@ -18,8 +18,7 @@
import sys import sys
from nomad.metainfo import Environment from nomad.metainfo import Environment
from nomad.metainfo.legacy import LegacyMetainfoEnvironment
import nomad.datamodel.metainfo.common_dft import nomad.datamodel.metainfo.common_dft
m_env = LegacyMetainfoEnvironment() m_env = Environment()
m_env.m_add_sub_section(Environment.packages, sys.modules['nomad.datamodel.metainfo.common_dft'].m_package) # type: ignore m_env.m_add_sub_section(Environment.packages, sys.modules['nomad.datamodel.metainfo.common_dft'].m_package) # type: ignore
...@@ -9,7 +9,7 @@ from nomad.metainfo.search_extension import Search ...@@ -9,7 +9,7 @@ from nomad.metainfo.search_extension import Search
m_package = Package( m_package = Package(
name='nomad.datamodel.metainfo.public_old', name='nomad.datamodel.metainfo.public',
description='None', description='None',
a_legacy=LegacyDefinition(name='public.nomadmetainfo.json')) a_legacy=LegacyDefinition(name='public.nomadmetainfo.json'))
...@@ -1118,8 +1118,8 @@ class BasisSet(MSection): ...@@ -1118,8 +1118,8 @@ class BasisSet(MSection):
dependent to the simulated cell as a whole). dependent to the simulated cell as a whole).
Basis sets used in this section_single_configuration_calculation, belonging to either Basis sets used in this section_single_configuration_calculation, belonging to either
class, are defined in the dedicated section: [section_basis_set_cell_dependent class, are defined in the dedicated section: section_basis_set_cell_dependent or
](section_basis_set_cell_dependent) or section_basis_set_atom_centered. The section_basis_set_atom_centered. The
correspondence between the basis sets listed in this section and the definition given correspondence between the basis sets listed in this section and the definition given
in the dedicated sessions is given by the two concrete metadata: in the dedicated sessions is given by the two concrete metadata:
mapping_section_basis_set_cell_dependent and mapping_section_basis_set_atom_centered. mapping_section_basis_set_cell_dependent and mapping_section_basis_set_atom_centered.
...@@ -1961,9 +1961,8 @@ class FrameSequenceUserQuantity(MSection): ...@@ -1961,9 +1961,8 @@ class FrameSequenceUserQuantity(MSection):
Dedicated metadata monitored along a sequence of frames are created for the Dedicated metadata monitored along a sequence of frames are created for the
conserved energy-like quantity (frame_sequence_conserved_quantity), the kinetic conserved energy-like quantity (frame_sequence_conserved_quantity), the kinetic
and potential energies ([frame_sequence_kinetic_energy and and potential energies (frame_sequence_kinetic_energy and
frame_sequence_potential_energy](frame_sequence_kinetic_energy and frame_sequence_potential_energy), the instantaneous temperature
frame_sequence_potential_energy)), the instantaneous temperature
(frame_sequence_temperature) and the pressure (frame_sequence_pressure). (frame_sequence_temperature) and the pressure (frame_sequence_pressure).
''', ''',
categories=[Unused], categories=[Unused],
...@@ -3821,8 +3820,7 @@ class Run(MSection): ...@@ -3821,8 +3820,7 @@ class Run(MSection):
''' '''
Every section_run represents a single call of a program. What exactly is contained in Every section_run represents a single call of a program. What exactly is contained in
a run depends on the run type (see for example section_method and a run depends on the run type (see for example section_method and
section_single_configuration_calculation) and the program (see [program_info section_single_configuration_calculation) and the program (see ProgramInfo).
](program_info)).
''' '''
m_def = Section( m_def = Section(
...@@ -6658,6 +6656,16 @@ class GeometryOptimization(MSection): ...@@ -6658,6 +6656,16 @@ class GeometryOptimization(MSection):
categories=[Unused], categories=[Unused],
a_legacy=LegacyDefinition(name='input_force_maximum_tolerance')) a_legacy=LegacyDefinition(name='input_force_maximum_tolerance'))
input_displacement_maximum_tolerance = Quantity(
type=np.dtype(np.float64),
shape=[],
unit='meter',
description='''
The input maximum displacement tolerance criterion.
''',
categories=[Unused],
a_legacy=LegacyDefinition(name='input_displacement_maximum_tolerance'))
final_energy_difference = Quantity( final_energy_difference = Quantity(
type=np.dtype(np.float64), type=np.dtype(np.float64),
shape=[], shape=[],
...@@ -6677,6 +6685,15 @@ class GeometryOptimization(MSection): ...@@ -6677,6 +6685,15 @@ class GeometryOptimization(MSection):
''', ''',
a_legacy=LegacyDefinition(name='final_force_maximum')) a_legacy=LegacyDefinition(name='final_force_maximum'))
final_displacement_maximum = Quantity(
type=np.dtype(np.float64),
shape=[],
unit='meter',
description='''
The maximum displacement in the last optimization step with respect to previous.
''',
a_legacy=LegacyDefinition(name='final_displacement_maximum'))
optimization_steps = Quantity( optimization_steps = Quantity(
type=int, type=int,
shape=[], shape=[],
...@@ -6685,6 +6702,14 @@ class GeometryOptimization(MSection): ...@@ -6685,6 +6702,14 @@ class GeometryOptimization(MSection):
''', ''',
a_legacy=LegacyDefinition(name='optimization_steps')) a_legacy=LegacyDefinition(name='optimization_steps'))
is_converged_geometry = Quantity(
type=bool,
shape=[],
description='''
Indicates if the geometry convergence criteria were fulfilled.
''',
a_legacy=LegacyDefinition(name='is_converged_geometry'))
class Phonon(MSection): class Phonon(MSection):
''' '''
...@@ -7832,7 +7857,7 @@ class Topology(MSection): ...@@ -7832,7 +7857,7 @@ class Topology(MSection):
description=''' description='''
A unique string idenfiying the force field defined in this section. Strategies to A unique string idenfiying the force field defined in this section. Strategies to
define it are discussed in the define it are discussed in the
[topology\\_force\\_field\\_name](https://gitlab.mpcdf.mpg.de/nomad-lab/nomad-meta- [topology_force_field_name](https://gitlab.mpcdf.mpg.de/nomad-lab/nomad-meta-
info/wikis/metainfo/topology-force-field-name). info/wikis/metainfo/topology-force-field-name).
''', ''',
a_legacy=LegacyDefinition(name='topology_force_field_name')) a_legacy=LegacyDefinition(name='topology_force_field_name'))
......
This diff is collapsed.
This diff is collapsed.
...@@ -24,7 +24,7 @@ definitions. ...@@ -24,7 +24,7 @@ definitions.
import numpy as np import numpy as np
from nomad import utils from nomad import utils
from nomad.metainfo import Definition, SubSection, Package, Quantity, Section, Reference, MEnum from nomad.metainfo import Definition, Package, Reference, MEnum
logger = utils.get_logger(__name__) logger = utils.get_logger(__name__)
...@@ -120,7 +120,7 @@ def generate_metainfo_code(metainfo_pkg: Package, python_package_path: str): ...@@ -120,7 +120,7 @@ def generate_metainfo_code(metainfo_pkg: Package, python_package_path: str):
format_aliases=format_aliases) format_aliases=format_aliases)
with open(python_package_path, 'wt') as f: with open(python_package_path, 'wt') as f:
code = env.get_template('package_new.j2').render(pkg=metainfo_pkg) code = env.get_template('package.j2').render(pkg=metainfo_pkg)
code = '\n'.join([ code = '\n'.join([
line.rstrip() if line.strip() != '' else '' line.rstrip() if line.strip() != '' else ''
for line in code.split('\n')]) for line in code.split('\n')])
...@@ -128,51 +128,7 @@ def generate_metainfo_code(metainfo_pkg: Package, python_package_path: str): ...@@ -128,51 +128,7 @@ def generate_metainfo_code(metainfo_pkg: Package, python_package_path: str):
if __name__ == '__main__': if __name__ == '__main__':
# Simple use case that merges old common/public defs # Simple use case that re-generates the common_dft package
import json from nomad.datamodel.metainfo.common_dft import m_package
from nomad.metainfo import Category
from nomad.datamodel.metainfo.public_old import m_package
from nomad.datamodel.metainfo.common_old import m_package as common_pkg
for section in common_pkg.section_definitions: # pylint: disable=not-an-iterable
if section.extends_base_section:
base_section = section.base_sections[0]
for name, attr in section.section_cls.__dict__.items():
if isinstance(attr, Quantity):
base_section.m_add_sub_section(Section.quantities, attr.m_copy(deep=True))
elif isinstance(attr, SubSection):
base_section.m_add_sub_section(Section.sub_sections, attr.m_copy(deep=True))
else:
m_package.m_add_sub_section(Package.section_definitions, section)
for category in common_pkg.category_definitions: # pylint: disable=not-an-iterable
m_package.m_add_sub_section(Package.category_definitions, category)
for definition in m_package.section_definitions + m_package.category_definitions:
old_name = definition.name
new_name = ''.join([item[0].title() + item[1:] for item in old_name.split('_')])
if new_name.startswith('Section'):
new_name = new_name[7:]
if new_name != old_name:
definition.aliases = [old_name]
definition.name = new_name
unused_category = m_package.m_create(Category)
unused_category.name = 'Unused'
unused_category.description = 'This metainfo definition is not used by NOMAD data.'
with open('local/metainfostats.json', 'rt') as f:
stats = json.load(f)
unused = []
for (definition, _, _) in m_package.m_traverse():
if isinstance(definition, (SubSection, Quantity)):
if definition.name not in stats:
unused.append(definition)
for definition in unused:
if unused_category not in definition.categories:
definition.categories += [unused_category]
generate_metainfo_code(m_package, 'nomad/datamodel/metainfo/common_dft.py') generate_metainfo_code(m_package, 'nomad/datamodel/metainfo/common_dft.py')
...@@ -22,25 +22,7 @@ new nomad@fairdi infrastructure. This covers aspects like the new metainfo, a un ...@@ -22,25 +22,7 @@ new nomad@fairdi infrastructure. This covers aspects like the new metainfo, a un
wrapper for parsers, parser logging, and a parser backend. wrapper for parsers, parser logging, and a parser backend.
''' '''
from typing import cast, Dict, List, Any, Tuple, Type from nomad.metainfo import DefinitionAnnotation, Environment
import numpy as np
import os.path
import importlib
from nomadcore.local_meta_info import InfoKindEl, InfoKindEnv
from nomad import utils
from nomad.metainfo import (
Definition, SubSection, Package, Quantity, Category, Section, Reference,
Environment, MEnum, MSection, DefinitionAnnotation, MetainfoError, MSectionBound)
logger = utils.get_logger(__name__)
_ignored_packages = [
'meta_types.nomadmetainfo.json',
'repository.nomadmetainfo.json']
class LegacyDefinition(DefinitionAnnotation): class LegacyDefinition(DefinitionAnnotation):
...@@ -49,229 +31,4 @@ class LegacyDefinition(DefinitionAnnotation): ...@@ -49,229 +31,4 @@ class LegacyDefinition(DefinitionAnnotation):
self.name = name self.name = name
def def_name(definition): LegacyMetainfoEnvironment = Environment
try:
return definition.a_legacy.name
except AttributeError:
return definition.name
def normalize_name(name: str):
return name.replace('.', '_').replace('-', '_')
def normalized_package_name(name: str):
'''
Transforms legacy metainfo '.nomadmetainfo.json' filenames into proper (python)
identifier.
'''
name = name.replace('.nomadmetainfo.json', '')
return normalize_name(name)
def python_package_mapping(metainfo_package_name: str) -> Tuple[str, str]:
'''
Compute the python package for the given metainfo package name. It returns
a tuple containing a package name and a file path. The filepath denotes the file
for this package within the nomad git project.
'''
prefix = metainfo_package_name.replace('.nomadmetainfo.json', '').split('.')[0]
metainfo_package_name = normalized_package_name(metainfo_package_name)
if prefix in ['common', 'general', 'public', 'dft', 'ems']:
directory = 'nomad/datamodel/metainfo'
python_package_name = 'nomad.datamodel.metainfo.%s' % metainfo_package_name
else:
parser_dir = prefix.replace('_', '-')
prefix = prefix.replace('_', '')
directory = 'dependencies/parsers/%s/%sparser/metainfo' % (parser_dir, prefix)
python_package_name = '%sparser.metainfo.%s' % (prefix, metainfo_package_name)
path = '%s/%s.py' % (directory, metainfo_package_name)
return python_package_name, path
class LegacyMetainfoEnvironment(Environment):
'''
A metainfo environment with functions to create a legacy metainfo version of
the environment.
'''
@staticmethod
def from_legacy_package_path(path):
metainfo_package_name = os.path.basename(path)
package = metainfo_package_name
if package.endswith('.nomadmetainfo.json'):
package = package[:-19]
if package.endswith('.json'):
package = package[:-5]
python_package_name, _ = python_package_mapping(package)
python_package_name = '.'.join(python_package_name.split('.')[:-1])
python_module = importlib.import_module(python_package_name)
metainfo = getattr(python_module, 'm_env')
return metainfo
legacy_package_name = Quantity(type=str)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.__section_to_sub_section_name = None
self.__legacy_names = None
def from_legacy_name(self, name: str, section_cls: Type[MSectionBound]) -> MSectionBound:
''' Returns the definition of the given globally unique legacy metainfo name. '''
if self.__legacy_names is None:
self.__legacy_names = dict()
for definition in self.m_all_contents():
try:
if isinstance(definition, Section):
if definition.extends_base_section:
continue
legacy = definition.a_legacy
key = (legacy.name, definition.m_def.section_cls)
if key in self.__legacy_names:
raise MetainfoError('Legacy name %s is not globally unique' % legacy.name)
self.__legacy_names[key] = definition
except AttributeError:
pass
return self.__legacy_names.get((name, section_cls))
@property
def section_to_sub_section_name(self) -> Dict[str, str]:
if self.__section_to_sub_section_name is not None:
return self.__section_to_sub_section_name
self.__section_to_sub_section_name = dict()
for definition in self.m_all_contents():
if definition.m_def == SubSection.m_def:
self.__section_to_sub_section_name[definition.sub_section.name] = definition.name
return self.__section_to_sub_section_name
def legacy_info(self, definition: Definition, *args, **kwargs) -> InfoKindEl:
''' Creates a legacy metainfo object for the given definition. '''
super_names: List[str] = list()
result: Dict[str, Any] = dict(
name=def_name(definition),
description=definition.description,
superNames=super_names)
for category in definition.categories:
super_names.append(def_name(category))
if isinstance(definition, Section):
sub_section_name = self.section_to_sub_section_name.get(definition.name, definition.name)
result['kindStr'] = 'type_section'
result['repeats'] = any(
sub_section.repeats
for sub_section in self.resolve_definitions(sub_section_name, SubSection))
for sub_section in self.resolve_definitions(sub_section_name, SubSection):
super_names.append(def_name(sub_section.m_parent_as(Definition)))
elif isinstance(definition, Quantity):
result['kindStr'] = 'type_document_content'
result['shape'] = definition.shape
dtype_str = None
if definition.type == int:
dtype_str = 'i'
elif definition.type == float:
dtype_str = 'f'
elif definition.type == bool:
dtype_str = 'b'
elif definition.type == str:
dtype_str = 'C'
elif isinstance(definition.type, Reference):
dtype_str = 'r'
result['referencedSections'] = [
def_name(definition.type.target_section_def.m_resolved())]
elif isinstance(definition.type, MEnum):
dtype_str = 'C'
elif isinstance(definition.type, np.dtype):
dtype_str = definition.type.name[0]
elif definition.type == Any:
dtype_str = 'D'
else:
dtype_str = str(definition.type)
# raise TypeError(
# 'Unsupported quantity type %s in %s.' % (definition.type, definition))
result['dtypeStr'] = dtype_str
if definition.unit is not None:
result['units'] = str(definition.unit)
super_names.append(def_name(definition.m_parent_as(Definition)))
elif isinstance(definition, Category):
result['kindStr'] = 'type_abstract_document_content'
package = cast(MSection, definition)
while not isinstance(package, Package):
package = package.m_parent
result['package'] = package.name
return InfoKindEl(*args, **result, **kwargs)
def legacy_info_env(self, packages: List[Package] = None, *args, **kwargs) -> InfoKindEnv:
''' Creates a legacy metainfo environment with all definitions from the given packages. '''
if packages is None:
packages = self.packages
env = InfoKindEnv(*args, **kwargs)
for package in packages:
for definition in package.all_definitions.values():
if not (isinstance(definition, Section) and definition.extends_base_section):
env.addInfoKindEl(self.legacy_info(definition))
if isinstance(definition, Section):
for quantity in definition.quantities:
env.addInfoKindEl(self.legacy_info(quantity))
return env
def to_legacy_dict(
self, packages: List[Package] = None, description: str = None,
*args, **kwargs) -> Dict[str, Any]:
'''
Creates a dictionary that can be serialized to a legacy metainfo definition file
(*.nomadmetainfo.json).
Arguments:
package: Will add all definitions of these packages as actual definitions,
all other packages will be added by import.
description: The description for the legacy file. If None the description of
the firs package will be used.
'''
if packages is None:
packages = []
definitions = []
dependencies = []
for package in self.packages:
if package in packages:
if description is None:
description = package.description
for definition in package.all_definitions.values():
if not (isinstance(definition, Section) and definition.extends_base_section):
definitions.append(self.legacy_info(definition).toDict())
if isinstance(definition, Section):
for quantity in definition.quantities:
definitions.append(self.legacy_info(quantity).toDict())
else:
dependencies.append(package)
return {
'type': 'nomad_meta_info_1_0',
'description': description,
'dependencies': [
{'relativePath': def_name(dependency)}
for dependency in dependencies],
'metaInfos': definitions
}
...@@ -796,7 +796,13 @@ class MSection(metaclass=MObjectMeta): # TODO find a way to make this a subclas ...@@ -796,7 +796,13 @@ class MSection(metaclass=MObjectMeta): # TODO find a way to make this a subclas
if isinstance(attr, Property): if isinstance(attr, Property):
attr.name = name attr.name = name
if attr.description is not None: if attr.description is not None:
attr.description = inspect.cleandoc(attr.description).strip() description = inspect.cleandoc(attr.description)
description = description.strip()
description = re.sub(
r'\(https?://[^\)]*\)',
lambda m: re.sub(r'\n', '', m.group(0)),
description)
attr.description = description
attr.__doc__ = attr.description attr.__doc__ = attr.description
if isinstance(attr, Quantity): if isinstance(attr, Quantity):
......
import sys
from nomad.metainfo import Environment
from nomad.metainfo.legacy import LegacyMetainfoEnvironment
{%- for package in env.packages %}
import {{ package.a_legacy.python_module }}
{%- endfor %}
m_env = LegacyMetainfoEnvironment()
{%- for package in env.packages %}
m_env.m_add_sub_section(Environment.packages, sys.modules['{{ package.a_legacy.python_module }}'].m_package) # type: ignore
{%- endfor %}
...@@ -2,19 +2,19 @@ import numpy as np # pylint: disable=unused-import ...@@ -2,19 +2,19 @@ import numpy as np # pylint: disable=unused-import
import typing # pylint: disable=unused-import import typing # pylint: disable=unused-import
from nomad.metainfo import ( # pylint: disable=unused-import from nomad.metainfo import ( # pylint: disable=unused-import
MSection, MCategory, Category, Package, Quantity, Section, SubSection, SectionProxy, MSection, MCategory, Category, Package, Quantity, Section, SubSection, SectionProxy,
Reference Reference, MEnum)
)
from nomad.metainfo.legacy import LegacyDefinition from nomad.metainfo.legacy import LegacyDefinition
from nomad.metainfo.search_extension import Search
{% for dependency in pkg.dependencies %} {% for dependency in pkg.dependencies %}
{{ fromat_package_import(dependency) }} {{ fromat_package_import(dependency) }}
{%- endfor %} {%- endfor %}
m_package = Package( m_package = Package(
name='{{ pkg.name }}', name='{{ pkg.name }}',
description='{{ pkg.description }}', description='{{ pkg.description }}'
{% if pkg.a_legacy is defined -%} {%- if pkg.a_legacy is defined %},
a_legacy=LegacyDefinition(name='{{pkg.a_legacy.name}}') a_legacy=LegacyDefinition(name='{{pkg.a_legacy.name}}')
{% endif -%}) {%- endif %})
{% for category in order_categories(pkg.category_definitions) %} {% for category in order_categories(pkg.category_definitions) %}
class {{ category.name }}(MCategory): class {{ category.name }}(MCategory):
...@@ -25,12 +25,15 @@ class {{ category.name }}(MCategory): ...@@ -25,12 +25,15 @@ class {{ category.name }}(MCategory):
{%- endif %} {%- endif %}
m_def = Category( m_def = Category(
{% if category.categories | length > 0 -%} {%- if category.aliases | length > 0 %}
aliases=['{{ category.aliases[0] }}'],
{%- endif -%}
{%- if category.categories | length > 0 %}
categories=[{{ format_definition_refs(pkg, category.categories) }}], categories=[{{ format_definition_refs(pkg, category.categories) }}],
{% endif -%} {%- endif -%}
{% if category.a_legacy is defined -%} {%- if category.a_legacy is defined %}
a_legacy=LegacyDefinition(name='{{category.a_legacy.name}}')