Merge branch 'v0.10.4' into 'master'

Merge for release

See merge request !357

nomad/datamodel/datamodel.py

@@ -117,6 +117,12 @@ class User(Author):
         from nomad import infrastructure
         return infrastructure.keycloak.get_user(*args, **kwargs)  # type: ignore
 
+    def full_user(self) -> 'User':
+        ''' Returns a User object with all attributes loaded from the user management system. '''
+        from nomad import infrastructure
+        assert self.user_id is not None
+        return infrastructure.keycloak.get_user(user_id=self.user_id)  # type: ignore
+
 
 class UserReference(metainfo.Reference):
     '''

nomad/datamodel/metainfo/__init__.py

@@ -18,8 +18,7 @@
 
 import sys
 from nomad.metainfo import Environment
-from nomad.metainfo.legacy import LegacyMetainfoEnvironment
 import nomad.datamodel.metainfo.common_dft
 
-m_env = LegacyMetainfoEnvironment()
+m_env = Environment()
 m_env.m_add_sub_section(Environment.packages, sys.modules['nomad.datamodel.metainfo.common_dft'].m_package)  # type: ignore

nomad/datamodel/metainfo/common_dft.py

@@ -9,7 +9,7 @@ from nomad.metainfo.search_extension import Search
 
 
 m_package = Package(
-    name='nomad.datamodel.metainfo.public_old',
+    name='nomad.datamodel.metainfo.public',
     description='None',
     a_legacy=LegacyDefinition(name='public.nomadmetainfo.json'))
 
@@ -1118,8 +1118,8 @@ class BasisSet(MSection):
     dependent to the simulated cell as a whole).
 
     Basis sets used in this section_single_configuration_calculation, belonging to either
-    class, are defined in the dedicated section: [section_basis_set_cell_dependent
-    ](section_basis_set_cell_dependent) or section_basis_set_atom_centered. The
+    class, are defined in the dedicated section: section_basis_set_cell_dependent or
+    section_basis_set_atom_centered. The
     correspondence between the basis sets listed in this section and the definition given
     in the dedicated sessions is given by the two concrete metadata:
     mapping_section_basis_set_cell_dependent and mapping_section_basis_set_atom_centered.
@@ -1961,9 +1961,8 @@ class FrameSequenceUserQuantity(MSection):
 
         Dedicated metadata monitored along a sequence of frames are created for the
         conserved energy-like quantity (frame_sequence_conserved_quantity), the kinetic
-        and potential energies ([frame_sequence_kinetic_energy and
-        frame_sequence_potential_energy](frame_sequence_kinetic_energy and
-        frame_sequence_potential_energy)), the instantaneous temperature
+        and potential energies (frame_sequence_kinetic_energy and
+        frame_sequence_potential_energy), the instantaneous temperature
         (frame_sequence_temperature) and the pressure (frame_sequence_pressure).
         ''',
         categories=[Unused],
@@ -3821,8 +3820,7 @@ class Run(MSection):
     '''
     Every section_run represents a single call of a program. What exactly is contained in
     a run depends on the run type (see for example section_method and
-    section_single_configuration_calculation) and the program (see [program_info
-    ](program_info)).
+    section_single_configuration_calculation) and the program (see ProgramInfo).
     '''
 
     m_def = Section(
@@ -6658,6 +6656,16 @@ class GeometryOptimization(MSection):
         categories=[Unused],
         a_legacy=LegacyDefinition(name='input_force_maximum_tolerance'))
 
+    input_displacement_maximum_tolerance = Quantity(
+        type=np.dtype(np.float64),
+        shape=[],
+        unit='meter',
+        description='''
+        The input maximum displacement tolerance criterion.
+        ''',
+        categories=[Unused],
+        a_legacy=LegacyDefinition(name='input_displacement_maximum_tolerance'))
+
     final_energy_difference = Quantity(
         type=np.dtype(np.float64),
         shape=[],
@@ -6677,6 +6685,15 @@ class GeometryOptimization(MSection):
         ''',
         a_legacy=LegacyDefinition(name='final_force_maximum'))
 
+    final_displacement_maximum = Quantity(
+        type=np.dtype(np.float64),
+        shape=[],
+        unit='meter',
+        description='''
+        The maximum displacement in the last optimization step with respect to previous.
+        ''',
+        a_legacy=LegacyDefinition(name='final_displacement_maximum'))
+
     optimization_steps = Quantity(
         type=int,
         shape=[],
@@ -6685,6 +6702,14 @@ class GeometryOptimization(MSection):
         ''',
         a_legacy=LegacyDefinition(name='optimization_steps'))
 
+    is_converged_geometry = Quantity(
+        type=bool,
+        shape=[],
+        description='''
+        Indicates if the geometry convergence criteria were fulfilled.
+        ''',
+        a_legacy=LegacyDefinition(name='is_converged_geometry'))
+
 
 class Phonon(MSection):
     '''
@@ -7832,7 +7857,7 @@ class Topology(MSection):
         description='''
         A unique string idenfiying the force field defined in this section. Strategies to
         define it are discussed in the
-        [topology\\_force\\_field\\_name](https://gitlab.mpcdf.mpg.de/nomad-lab/nomad-meta-
+        [topology_force_field_name](https://gitlab.mpcdf.mpg.de/nomad-lab/nomad-meta-
         info/wikis/metainfo/topology-force-field-name).
         ''',
         a_legacy=LegacyDefinition(name='topology_force_field_name'))

nomad/metainfo/generate.py

@@ -24,7 +24,7 @@ definitions.
 import numpy as np
 
 from nomad import utils
-from nomad.metainfo import Definition, SubSection, Package, Quantity, Section, Reference, MEnum
+from nomad.metainfo import Definition, Package, Reference, MEnum
 
 logger = utils.get_logger(__name__)
 
@@ -120,7 +120,7 @@ def generate_metainfo_code(metainfo_pkg: Package, python_package_path: str):
         format_aliases=format_aliases)
 
     with open(python_package_path, 'wt') as f:
-        code = env.get_template('package_new.j2').render(pkg=metainfo_pkg)
+        code = env.get_template('package.j2').render(pkg=metainfo_pkg)
         code = '\n'.join([
             line.rstrip() if line.strip() != '' else ''
             for line in code.split('\n')])
@@ -128,51 +128,7 @@ def generate_metainfo_code(metainfo_pkg: Package, python_package_path: str):
 
 
 if __name__ == '__main__':
-    # Simple use case that merges old common/public defs
-    import json
-
-    from nomad.metainfo import Category
-    from nomad.datamodel.metainfo.public_old import m_package
-    from nomad.datamodel.metainfo.common_old import m_package as common_pkg
-
-    for section in common_pkg.section_definitions:  # pylint: disable=not-an-iterable
-        if section.extends_base_section:
-            base_section = section.base_sections[0]
-            for name, attr in section.section_cls.__dict__.items():
-                if isinstance(attr, Quantity):
-                    base_section.m_add_sub_section(Section.quantities, attr.m_copy(deep=True))
-                elif isinstance(attr, SubSection):
-                    base_section.m_add_sub_section(Section.sub_sections, attr.m_copy(deep=True))
-        else:
-            m_package.m_add_sub_section(Package.section_definitions, section)
-
-    for category in common_pkg.category_definitions:  # pylint: disable=not-an-iterable
-        m_package.m_add_sub_section(Package.category_definitions, category)
-
-    for definition in m_package.section_definitions + m_package.category_definitions:
-        old_name = definition.name
-        new_name = ''.join([item[0].title() + item[1:] for item in old_name.split('_')])
-        if new_name.startswith('Section'):
-            new_name = new_name[7:]
-        if new_name != old_name:
-            definition.aliases = [old_name]
-            definition.name = new_name
-
-    unused_category = m_package.m_create(Category)
-    unused_category.name = 'Unused'
-    unused_category.description = 'This metainfo definition is not used by NOMAD data.'
-
-    with open('local/metainfostats.json', 'rt') as f:
-        stats = json.load(f)
-
-    unused = []
-    for (definition, _, _) in m_package.m_traverse():
-        if isinstance(definition, (SubSection, Quantity)):
-            if definition.name not in stats:
-                unused.append(definition)
-
-    for definition in unused:
-        if unused_category not in definition.categories:
-            definition.categories += [unused_category]
+    # Simple use case that re-generates the common_dft package
+    from nomad.datamodel.metainfo.common_dft import m_package
 
     generate_metainfo_code(m_package, 'nomad/datamodel/metainfo/common_dft.py')

nomad/metainfo/legacy.py

@@ -22,25 +22,7 @@ new nomad@fairdi infrastructure. This covers aspects like the new metainfo, a un
 wrapper for parsers, parser logging, and a parser backend.
 '''
 
-from typing import cast, Dict, List, Any, Tuple, Type
-import numpy as np
-import os.path
-import importlib
-
-
-from nomadcore.local_meta_info import InfoKindEl, InfoKindEnv
-
-from nomad import utils
-from nomad.metainfo import (
-    Definition, SubSection, Package, Quantity, Category, Section, Reference,
-    Environment, MEnum, MSection, DefinitionAnnotation, MetainfoError, MSectionBound)
-
-logger = utils.get_logger(__name__)
-
-
-_ignored_packages = [
-    'meta_types.nomadmetainfo.json',
-    'repository.nomadmetainfo.json']
+from nomad.metainfo import DefinitionAnnotation, Environment
 
 
 class LegacyDefinition(DefinitionAnnotation):
@@ -49,229 +31,4 @@ class LegacyDefinition(DefinitionAnnotation):
         self.name = name
 
 
-def def_name(definition):
-    try:
-        return definition.a_legacy.name
-    except AttributeError:
-        return definition.name
-
-
-def normalize_name(name: str):
-    return name.replace('.', '_').replace('-', '_')
-
-
-def normalized_package_name(name: str):
-    '''
-    Transforms legacy metainfo '.nomadmetainfo.json' filenames into proper (python)
-    identifier.
-    '''
-    name = name.replace('.nomadmetainfo.json', '')
-    return normalize_name(name)
-
-
-def python_package_mapping(metainfo_package_name: str) -> Tuple[str, str]:
-    '''
-    Compute the python package for the given metainfo package name. It returns
-    a tuple containing a package name and a file path. The filepath denotes the file
-    for this package within the nomad git project.
-    '''
-    prefix = metainfo_package_name.replace('.nomadmetainfo.json', '').split('.')[0]
-    metainfo_package_name = normalized_package_name(metainfo_package_name)
-
-    if prefix in ['common', 'general', 'public', 'dft', 'ems']:
-        directory = 'nomad/datamodel/metainfo'
-        python_package_name = 'nomad.datamodel.metainfo.%s' % metainfo_package_name
-
-    else:
-        parser_dir = prefix.replace('_', '-')
-        prefix = prefix.replace('_', '')
-
-        directory = 'dependencies/parsers/%s/%sparser/metainfo' % (parser_dir, prefix)
-        python_package_name = '%sparser.metainfo.%s' % (prefix, metainfo_package_name)
-
-    path = '%s/%s.py' % (directory, metainfo_package_name)
-
-    return python_package_name, path
-
-
-class LegacyMetainfoEnvironment(Environment):
-    '''
-    A metainfo environment with functions to create a legacy metainfo version of
-    the environment.
-    '''
-
-    @staticmethod
-    def from_legacy_package_path(path):
-        metainfo_package_name = os.path.basename(path)
-        package = metainfo_package_name
-        if package.endswith('.nomadmetainfo.json'):
-            package = package[:-19]
-        if package.endswith('.json'):
-            package = package[:-5]
-
-        python_package_name, _ = python_package_mapping(package)
-        python_package_name = '.'.join(python_package_name.split('.')[:-1])
-        python_module = importlib.import_module(python_package_name)
-        metainfo = getattr(python_module, 'm_env')
-
-        return metainfo
-
-    legacy_package_name = Quantity(type=str)
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.__section_to_sub_section_name = None
-        self.__legacy_names = None
-
-    def from_legacy_name(self, name: str, section_cls: Type[MSectionBound]) -> MSectionBound:
-        ''' Returns the definition of the given globally unique legacy metainfo name. '''
-        if self.__legacy_names is None:
-            self.__legacy_names = dict()
-            for definition in self.m_all_contents():
-                try:
-                    if isinstance(definition, Section):
-                        if definition.extends_base_section:
-                            continue
-                    legacy = definition.a_legacy
-                    key = (legacy.name, definition.m_def.section_cls)
-                    if key in self.__legacy_names:
-                        raise MetainfoError('Legacy name %s is not globally unique' % legacy.name)
-                    self.__legacy_names[key] = definition
-                except AttributeError:
-                    pass
-
-        return self.__legacy_names.get((name, section_cls))
-
-    @property
-    def section_to_sub_section_name(self) -> Dict[str, str]:
-        if self.__section_to_sub_section_name is not None:
-            return self.__section_to_sub_section_name
-
-        self.__section_to_sub_section_name = dict()
-        for definition in self.m_all_contents():
-            if definition.m_def == SubSection.m_def:
-                self.__section_to_sub_section_name[definition.sub_section.name] = definition.name
-
-        return self.__section_to_sub_section_name
-
-    def legacy_info(self, definition: Definition, *args, **kwargs) -> InfoKindEl:
-        ''' Creates a legacy metainfo object for the given definition. '''
-        super_names: List[str] = list()
-        result: Dict[str, Any] = dict(
-            name=def_name(definition),
-            description=definition.description,
-            superNames=super_names)
-
-        for category in definition.categories:
-            super_names.append(def_name(category))
-
-        if isinstance(definition, Section):
-            sub_section_name = self.section_to_sub_section_name.get(definition.name, definition.name)
-            result['kindStr'] = 'type_section'
-            result['repeats'] = any(
-                sub_section.repeats
-                for sub_section in self.resolve_definitions(sub_section_name, SubSection))
-
-            for sub_section in self.resolve_definitions(sub_section_name, SubSection):
-                super_names.append(def_name(sub_section.m_parent_as(Definition)))
-
-        elif isinstance(definition, Quantity):
-            result['kindStr'] = 'type_document_content'
-            result['shape'] = definition.shape
-            dtype_str = None
-            if definition.type == int:
-                dtype_str = 'i'
-            elif definition.type == float:
-                dtype_str = 'f'
-            elif definition.type == bool:
-                dtype_str = 'b'
-            elif definition.type == str:
-                dtype_str = 'C'
-            elif isinstance(definition.type, Reference):
-                dtype_str = 'r'
-                result['referencedSections'] = [
-                    def_name(definition.type.target_section_def.m_resolved())]
-            elif isinstance(definition.type, MEnum):
-                dtype_str = 'C'
-            elif isinstance(definition.type, np.dtype):
-                dtype_str = definition.type.name[0]
-            elif definition.type == Any:
-                dtype_str = 'D'
-            else:
-                dtype_str = str(definition.type)
-                # raise TypeError(
-                #     'Unsupported quantity type %s in %s.' % (definition.type, definition))
-            result['dtypeStr'] = dtype_str
-            if definition.unit is not None:
-                result['units'] = str(definition.unit)
-            super_names.append(def_name(definition.m_parent_as(Definition)))
-
-        elif isinstance(definition, Category):
-            result['kindStr'] = 'type_abstract_document_content'
-
-        package = cast(MSection, definition)
-        while not isinstance(package, Package):
-            package = package.m_parent
-
-        result['package'] = package.name
-
-        return InfoKindEl(*args, **result, **kwargs)
-
-    def legacy_info_env(self, packages: List[Package] = None, *args, **kwargs) -> InfoKindEnv:
-        ''' Creates a legacy metainfo environment with all definitions from the given packages. '''
-        if packages is None:
-            packages = self.packages
-
-        env = InfoKindEnv(*args, **kwargs)
-        for package in packages:
-            for definition in package.all_definitions.values():
-                if not (isinstance(definition, Section) and definition.extends_base_section):
-                    env.addInfoKindEl(self.legacy_info(definition))
-
-                if isinstance(definition, Section):
-                    for quantity in definition.quantities:
-                        env.addInfoKindEl(self.legacy_info(quantity))
-
-        return env
-
-    def to_legacy_dict(
-            self, packages: List[Package] = None, description: str = None,
-            *args, **kwargs) -> Dict[str, Any]:
-        '''
-        Creates a dictionary that can be serialized to a legacy metainfo definition file
-        (*.nomadmetainfo.json).
-
-        Arguments:
-            package: Will add all definitions of these packages as actual definitions,
-                all other packages will be added by import.
-            description: The description for the legacy file. If None the description of
-                the firs package will be used.
-        '''
-        if packages is None:
-            packages = []
-
-        definitions = []
-        dependencies = []
-        for package in self.packages:
-            if package in packages:
-                if description is None:
-                    description = package.description
-
-                for definition in package.all_definitions.values():
-                    if not (isinstance(definition, Section) and definition.extends_base_section):
-                        definitions.append(self.legacy_info(definition).toDict())
-
-                    if isinstance(definition, Section):
-                        for quantity in definition.quantities:
-                            definitions.append(self.legacy_info(quantity).toDict())
-            else:
-                dependencies.append(package)
-
-        return {
-            'type': 'nomad_meta_info_1_0',
-            'description': description,
-            'dependencies': [
-                {'relativePath': def_name(dependency)}
-                for dependency in dependencies],
-            'metaInfos': definitions
-        }
+LegacyMetainfoEnvironment = Environment

nomad/metainfo/metainfo.py

@@ -796,7 +796,13 @@ class MSection(metaclass=MObjectMeta):  # TODO find a way to make this a subclas
             if isinstance(attr, Property):
                 attr.name = name
                 if attr.description is not None:
-                    attr.description = inspect.cleandoc(attr.description).strip()
+                    description = inspect.cleandoc(attr.description)
+                    description = description.strip()
+                    description = re.sub(
+                        r'\(https?://[^\)]*\)',
+                        lambda m: re.sub(r'\n', '', m.group(0)),
+                        description)
+                    attr.description = description
                     attr.__doc__ = attr.description
 
                 if isinstance(attr, Quantity):

nomad/metainfo/templates/environment.j2

-import sys
-from nomad.metainfo import Environment
-from nomad.metainfo.legacy import LegacyMetainfoEnvironment
-
-{%- for package in env.packages %}
-import {{ package.a_legacy.python_module }}
-{%- endfor %}
-
-m_env = LegacyMetainfoEnvironment()
-
-{%- for package in env.packages %}
-m_env.m_add_sub_section(Environment.packages, sys.modules['{{ package.a_legacy.python_module }}'].m_package)  # type: ignore
-{%- endfor %}
-

nomad/metainfo/templates/package.j2

@@ -2,19 +2,19 @@ import numpy as np            # pylint: disable=unused-import
 import typing                 # pylint: disable=unused-import
 from nomad.metainfo import (  # pylint: disable=unused-import
     MSection, MCategory, Category, Package, Quantity, Section, SubSection, SectionProxy,
-    Reference
-)
+    Reference, MEnum)
 from nomad.metainfo.legacy import LegacyDefinition
+from nomad.metainfo.search_extension import Search
 {% for dependency in pkg.dependencies %}
 {{ fromat_package_import(dependency) }}
 {%- endfor %}
 
 m_package = Package(
     name='{{ pkg.name }}',
-    description='{{ pkg.description }}',
-    {% if pkg.a_legacy is defined -%}
+    description='{{ pkg.description }}'
+    {%- if pkg.a_legacy is defined %},
     a_legacy=LegacyDefinition(name='{{pkg.a_legacy.name}}')
-    {% endif -%})
+    {%- endif %})
 {% for category in order_categories(pkg.category_definitions) %}
 
 class {{ category.name }}(MCategory):
@@ -25,12 +25,15 @@ class {{ category.name }}(MCategory):
     {%- endif %}
 
     m_def = Category(
-        {% if category.categories | length > 0 -%}
+        {%- if category.aliases | length > 0 %}
+        aliases=['{{ category.aliases[0] }}'],
+        {%- endif -%}
+        {%- if category.categories | length > 0 %}
         categories=[{{ format_definition_refs(pkg, category.categories) }}],
-        {% endif -%}
-        {% if category.a_legacy is defined -%}
+        {%- endif -%}
+        {%- if category.a_legacy is defined %}
         a_legacy=LegacyDefinition(name='{{category.a_legacy.name}}')
-        {% endif -%})
+        {%- endif %})
 {% endfor -%}
 
 {% for section in pkg.section_definitions %}
@@ -42,11 +45,14 @@ class {{ section.name }}({%- if section.extends_base_section -%}{{ format_defini
     '''
     {% endif %}
     m_def = Section(
+        {%- if section.aliases | length > 0 %}
+        aliases=['{{ section.aliases[0] }}'],
+        {%- endif %}
         validate=False{%- if section.extends_base_section -%},
         extends_base_section=True{%- endif -%},
         {% if section.a_legacy is defined -%}
         a_legacy=LegacyDefinition(name='{{section.a_legacy.name}}')
-        {% endif -%})
+        {%- endif -%})
     {% for quantity in section.quantities %}
     {{ quantity.name }} = Quantity(
         type={{ format_type(pkg, quantity.type) }},
@@ -61,9 +67,12 @@ class {{ section.name }}({%- if section.extends_base_section -%}{{ format_defini
         {%- if quantity.categories | length > 0 -%},
         categories=[{{ format_definition_refs(pkg, quantity.categories) }}]
         {%- endif -%},
-        {% if quantity.a_legacy is defined -%}
+        {%- if quantity.a_search is defined %}
+        a_search=Search(),
+        {%- endif -%}
+        {%- if quantity.a_legacy is defined %}
         a_legacy=LegacyDefinition(name='{{quantity.a_legacy.name}}')
-        {% endif -%})
+        {%- endif -%})
     {% endfor -%}
 
     {%- for sub_section in section.sub_sections %}
@@ -74,11 +83,14 @@ class {{ section.name }}({%- if section.extends_base_section -%}{{ format_defini
         description='''
         {{ format_description(sub_section.description, indent=2) }}
         '''{%- endif -%},
+        {%- if sub_section.categories | length > 0 %}
+        categories=[{{ format_definition_refs(pkg, sub_section.categories) }}],
+        {%- endif %}
         {% if sub_section.a_legacy is defined -%}
         a_legacy=LegacyDefinition(name='{{sub_section.a_legacy.name}}')
-        {% endif -%})
+        {%- endif -%})
     {% endfor -%}
 {%- endfor %}
 
 m_package.__init_metainfo__()
-
+{{- format_aliases(pkg) -}}

nomad/metainfo/templates/package_new.j2

-import numpy as np            # pylint: disable=unused-import
-import typing                 # pylint: disable=unused-import
-from nomad.metainfo import (  # pylint: disable=unused-import
-    MSection, MCategory, Category, Package, Quantity, Section, SubSection, SectionProxy,
-    Reference, MEnum)
-from nomad.metainfo.legacy import LegacyDefinition
-from nomad.metainfo.search_extension import Search
-{% for dependency in pkg.dependencies %}
-{{ fromat_package_import(dependency) }}
-{%- endfor %}
-
-m_package = Package(
-    name='{{ pkg.name }}',
-    description='{{ pkg.description }}'
-    {%- if pkg.a_legacy is defined %},
-    a_legacy=LegacyDefinition(name='{{pkg.a_legacy.name}}')
-    {%- endif %})
-{% for category in order_categories(pkg.category_definitions) %}
-
-class {{ category.name }}(MCategory):
-    {% if category.description is not none -%}
-    '''
-    {{ format_description(category.description, indent=1) }}
-    '''
-    {%- endif %}
-
-    m_def = Category(
-        {%- if category.aliases | length > 0 %}
-        aliases=['{{ category.aliases[0] }}'],
-        {%- endif -%}
-        {%- if category.categories | length > 0 %}
-        categories=[{{ format_definition_refs(pkg, category.categories) }}],
-        {%- endif -%}
-        {%- if category.a_legacy is defined %}