Commit e888e730 authored by Lauri Himanen's avatar Lauri Himanen
Browse files

Fixes in the hash calculation and in what information is required.

parent 74799526
Pipeline #70562 canceled with stages
in 3 minutes and 31 seconds
......@@ -1015,11 +1015,11 @@ class MethodNormalizer():
def method_hash(self, method: Method, settings_basis_set: RestrictedDict, repr_method: Section):
method_dict = RestrictedDict(
mandatory=(
mandatory_keys=[
"program_name",
"subsettings",
),
forbidden_values=(None)
],
forbidden_values=[None]
)
method_dict['program_name'] = method.code_name
......@@ -1031,8 +1031,8 @@ class MethodNormalizer():
# If all required information is present, safe the hash
try:
method_dict.check(recursive=True)
except (KeyError, ValueError):
self.logger.info("Could not create method hash, missing required information.")
except (KeyError, ValueError) as e:
self.logger.info("Could not create method hash, missing required information.", exc_info=e)
else:
method.method_hash = method_dict.hash()
......@@ -1042,12 +1042,12 @@ class MethodNormalizer():
def group_eos_hash(self, method: Method, material: Material, repr_method: Section):
eos_dict = RestrictedDict(
mandatory=(
mandatory_keys=(
"upload_id",
"method_hash",
"formula",
),
forbidden_values=(None)
forbidden_values=[None]
)
# Only calculations from the same upload are grouped
......@@ -1062,22 +1062,22 @@ class MethodNormalizer():
# Form a hash from the dictionary
try:
eos_dict.check(recursive=True)
except (KeyError, ValueError):
self.logger.info("Could not create EOS hash, missing required information.")
except (KeyError, ValueError) as e:
self.logger.info("Could not create EOS hash, missing required information.", exc_info=e)
else:
method.group_eos_hash = eos_dict.hash()
def group_parametervariation_hash(self, method: Method, settings_basis_set: RestrictedDict, repr_system: Section, repr_method: Section):
# Create ordered dictionary with the values. Order is important for
param_dict = RestrictedDict(
mandatory=(
mandatory_keys=[
"upload_id",
"program_name",
"program_version",
"settings_geometry",
"subsettings",
),
forbidden_values=(None)
],
forbidden_values=[None]
)
# Only calculations from the same upload are grouped
......@@ -1115,8 +1115,8 @@ class MethodNormalizer():
# Form a hash from the dictionary
try:
param_dict.check(recursive=True)
except (KeyError, ValueError):
self.logger.info("Could not create parameter variation hash, missing required information.")
except (KeyError, ValueError) as e:
self.logger.info("Could not create parameter variation hash, missing required information.", exc_info=e)
else:
method.group_parametervariation_hash = param_dict.hash()
......@@ -1262,18 +1262,17 @@ class MethodDFTNormalizer(MethodNormalizer):
def method_hash_dict(self, method: Method, settings_basis_set: RestrictedDict, repr_method: Section) -> RestrictedDict:
# Extend by DFT settings.
hash_dict = RestrictedDict(
mandatory=(
mandatory_keys=(
"functional_long_name",
"settings_basis_set",
"scf_threshold_energy_change",
),
optional=(
"settings_k_point_sampling",
optional_keys=(
"smearing_kind",
"smearing_parameter",
"number_of_eigenvalues_kpoints",
),
forbidden_values=(None)
forbidden_values=[None]
)
# Functional settings
hash_dict['functional_long_name'] = method.functional_long_name
......@@ -1286,11 +1285,11 @@ class MethodDFTNormalizer(MethodNormalizer):
# _reducible_ k-point-mesh:
# - grid dimensions (e.g. [ 4, 4, 8 ])
# - or list of reducible k-points
hash_dict['smearing_kind'] = method.smearing_kind
smearing_parameter = method.smearing_parameter
if smearing_parameter is not None:
smearing_parameter = '%.4f' % (smearing_parameter * J_to_Ry)
hash_dict['smearing_parameter'] = smearing_parameter
if method.smearing_kind is not None:
hash_dict['smearing_kind'] = method.smearing_kind
if method.smearing_parameter is not None:
smearing_parameter = '%.4f' % (method.smearing_parameter * J_to_Ry)
hash_dict['smearing_parameter'] = smearing_parameter
try:
scc = self.backend[s_scc][-1]
eigenvalues = scc['eigenvalues']
......@@ -1304,7 +1303,7 @@ class MethodDFTNormalizer(MethodNormalizer):
conv_thr = repr_method.get('scf_threshold_energy_change', None)
if conv_thr is not None:
conv_thr = '%.13f' % (conv_thr * J_to_Ry)
hash_dict['scf_threshold_energy_change'] = conv_thr
hash_dict['scf_threshold_energy_change'] = conv_thr
return hash_dict
......@@ -1313,14 +1312,14 @@ class MethodDFTNormalizer(MethodNormalizer):
grouping
This is the source for generating the related hash."""
param_dict = RestrictedDict(
mandatory=(
mandatory_keys=(
"functional_long_name",
"scf_threshold_energy_change",
),
optional=(
optional_keys=(
"atoms_pseudopotentials",
),
forbidden_values=(None)
forbidden_values=[None]
)
# TODO: Add other DFT-specific properties
......@@ -1337,7 +1336,9 @@ class MethodDFTNormalizer(MethodNormalizer):
# Pseudopotentials are kept constant, if applicable
if settings_basis_set is not None:
param_dict['atoms_pseudopotentials'] = settings_basis_set.get('atoms_pseudopotentials', None)
pseudos = settings_basis_set.get('atoms_pseudopotentials', None)
if pseudos is not None:
param_dict['atoms_pseudopotentials'] = pseudos
return param_dict
......@@ -1670,16 +1671,6 @@ class PropertiesNormalizer():
kpoints.append(seg_k_points)
energies.append(seg_energies)
# A full copy of the band structure is currently not store
# within the encyclopedia data, although the metainfo is
# there to support it.
# segment = BandSegment()
# segment.k_points = seg_k_points
# segment.energies = seg_energies
# segment.labels = seg_labels
# properties.m_add_sub_section(Properties.electronic_band_structure, band_structure)
# band_structure.m_add_sub_section(ElectronicBandStructure.segments, segment)
# Continue to calculate band gaps and other properties.
kpoints = np.concatenate(kpoints, axis=0)
energies = np.concatenate(energies, axis=2)
......
......@@ -3,28 +3,41 @@ from collections import OrderedDict
import numpy as np
from typing import Tuple, List
from nomad.parsing.backend import LocalBackend
from nomad.parsing.backend import Section
from nomad.utils import RestrictedDict
def get_basis_set_settings(context, backend, logger):
"""Decide which type of basis set settings are applicable to the entry
and return a corresponding SettingsBasisSet class.
def get_basis_set_settings(context, backend: LocalBackend, logger) -> RestrictedDict:
"""Decide which type of basis set settings are applicable to the entry and
return a corresponding settings as a RestrictedDict.
Args:
context: The calculation context.
backend: Backend from which values are extracted.
logger: Shared logger.
Returns:
RestrictedDict or None: Returns the extracted settings as a
RestrictedDict. If no suitable basis set settings could be identified,
returns None.
"""
settings = None
settings: SettingsBasisSet = None
program_name = backend.get('program_name')
if program_name == "exciting":
settings = SettingsBasisSetExciting(context, backend, logger).settings
settings = SettingsBasisSetExciting(context, backend, logger)
elif program_name == "FHI-aims":
settings = SettingsBasisSetFHIAims(context, backend, logger).settings
return settings
settings = SettingsBasisSetFHIAims(context, backend, logger)
else:
return None
return settings.to_dict()
class SettingsBasisSet(ABC):
"""Abstract base class for basis set settings
Provides a factory() static method for delegating to the concrete
implementation applicable for a given calculation.
class SettingsBasisSet(ABC):
"""Abstract base class for basis set settings. The idea is to create
subclasses that inherit this class and hierarchically add new mandatory and
optional settings with the setup()-function.
"""
def __init__(self, context, backend, logger):
"""
......@@ -33,19 +46,19 @@ class SettingsBasisSet(ABC):
self._backend = backend
self._logger = logger
mandatory, optional = self.setup()
self.settings = RestrictedDict(mandatory, optional)
self.fill()
self.settings = RestrictedDict(mandatory, optional, forbidden_values=[None])
@abstractmethod
def fill(self):
"""Used to fill the settings.
def to_dict(self) -> RestrictedDict:
"""Used to extract basis set settings from the backend and returning
them as a RestrictedDict.
"""
pass
@abstractmethod
def setup(self) -> Tuple:
"""Used to define a list of mandatory and optional settings for a
subclass in the form of a RestrictedDict object.
subclass.
Returns:
Should return a tuple of two lists: the first one defining
......@@ -60,13 +73,15 @@ class SettingsBasisSetFHIAims(SettingsBasisSet):
"""Basis set settings for 'FHI-Aims' (code-dependent).
"""
def setup(self) -> Tuple:
# Get previously defined values from superclass
mandatory, optional = super().setup()
mandatory += ["FhiAims_basis"]
# Add new values
mandatory += ["fhiaims_basis"]
return mandatory, optional
def fill(self):
"""Special case of basis set settings for FHI-Aims code.
"""
def to_dict(self):
# Get basis set settings for each species
aims_bs = self._backend.get('x_fhi_aims_section_controlIn_basis_set')
if aims_bs is not None:
......@@ -81,7 +96,9 @@ class SettingsBasisSetFHIAims(SettingsBasisSet):
basis = OrderedDict()
for k in sorted(bs_by_species.keys()):
basis[k] = bs_by_species[k]
self.settings["FhiAims_basis"] = basis
self.settings["fhiaims_basis"] = basis
return self.settings
@classmethod
def _values_to_dict(cls, data, level=0):
......@@ -121,7 +138,10 @@ class SettingsBasisSetExciting(SettingsBasisSet):
"""Basis set settings for 'Exciting' (code-dependent).
"""
def setup(self) -> Tuple:
# Get previously defined values from superclass
mandatory, optional = super().setup()
# Add new values
mandatory += [
"muffin_tin_settings",
"rgkmax",
......@@ -129,13 +149,13 @@ class SettingsBasisSetExciting(SettingsBasisSet):
"lo",
"lmaxapw",
]
return mandatory, optional
def fill(self):
def to_dict(self):
"""Special case of basis set settings for Exciting code. See list at:
https://gitlab.mpcdf.mpg.de/nomad-lab/encyclopedia-general/wikis/FHI-visit-preparation
"""
# Add the muffin-tin settings for each species ordered alphabetically by atom label
try:
groups = self._backend["x_exciting_section_atoms_group"]
......@@ -173,3 +193,5 @@ class SettingsBasisSetExciting(SettingsBasisSet):
self.settings['lmaxapw'] = "%d" % (system['x_exciting_lmaxapw'])
except KeyError:
pass
return self.settings
......@@ -555,55 +555,78 @@ class RestrictedDict(OrderedDict):
"""Dictionary-like container with predefined set of mandatory and optional
keys and a set of forbidden values.
"""
def __init__(self, mandatory: Iterable = None, optional: Iterable = None, forbidden_values: Iterable = None, lazy: bool = True):
def __init__(self, mandatory_keys: Iterable = None, optional_keys: Iterable = None, forbidden_values: Iterable = None, lazy: bool = True):
"""
Args:
mandatory_keys: Keys that have to be present.
optional_keys: Keys that are optional.
forbidden_values: Values that are forbidden.
forbidden_values: Values that are forbidden. Only supports hashable values.
lazy: If false, the values are checked already when inserting. If
true, the values are only checked manually by calling the
check()-function.
"""
super().__init__()
if mandatory:
self._mandatory = set(mandatory)
if isinstance(mandatory_keys, (list, tuple, set)):
self._mandatory_keys = set(mandatory_keys)
elif mandatory_keys is None:
self._mandatory_keys = set()
else:
self._mandatory = set()
if optional:
self._optional = set(optional)
raise ValueError("Please provide the mandatory_keys as a list, tuple or set.")
if isinstance(optional_keys, (list, tuple, set)):
self._optional_keys = set(optional_keys)
elif optional_keys is None:
self._optional_keys = set()
else:
self._optional = set()
if forbidden_values:
raise ValueError("Please provide the optional_keys as a list, tuple or set.")
if isinstance(forbidden_values, (list, tuple, set)):
self._forbidden_values = set(forbidden_values)
else:
elif forbidden_values is None:
self._forbidden_values = set()
else:
raise ValueError("Please provide the forbidden_values as a list or tuple of values.")
self._lazy = lazy
def __setitem__(self, key, value):
if not self._lazy:
if key not in self._mandatory and key not in self._optional:
# Check that only the defined keys are used
if key not in self._mandatory_keys and key not in self._optional_keys:
raise KeyError("The key {} is not allowed.".format(key))
for forbidden_value in self._forbidden_values:
if value == forbidden_value:
# Check that forbidden values are not used.
try:
match = value in self._forbidden_values
except TypeError:
pass # Unhashable value will not match
else:
if match:
raise ValueError("The value {} is not allowed.".format(key))
super().__setitem__(key, value)
def check(self, recursive=False):
# Check that only the defined keys are used
for key in self.keys():
if key not in self._mandatory and key not in self._optional:
if key not in self._mandatory_keys and key not in self._optional_keys:
raise KeyError("The key {} is not allowed.".format(key))
# Check that all mandatory values are all defined
for key in self._mandatory:
for key in self._mandatory_keys:
if key not in self:
raise KeyError("The mandatory key {} is not present.".format(key))
# Check that forbidden values are not used.
for value in self.values():
for forbidden_value in self._forbidden_values:
if value == forbidden_value:
try:
match = value in self._forbidden_values
except TypeError:
pass # Unhashable value will not match
else:
if match:
raise ValueError("The value {} is not allowed.".format(key))
# Check recursively
......
......@@ -218,3 +218,8 @@ def hash_exciting() -> LocalBackend:
backend = parse_file((parser_name, filepath))
backend = run_normalize(backend)
return backend
@pytest.fixture(scope='session')
def hash_vasp(bands_unpolarized_gap_indirect) -> LocalBackend:
return bands_unpolarized_gap_indirect
......@@ -37,6 +37,7 @@ from tests.normalizing.conftest import ( # pylint: disable=unused-import
dos_unpolarized_vasp,
dos_polarized_vasp,
hash_exciting,
hash_vasp,
)
ureg = UnitRegistry()
......@@ -521,7 +522,7 @@ def test_band_structure(bands_unpolarized_no_gap, bands_polarized_no_gap, bands_
def test_hashes_exciting(hash_exciting):
"""Tests that the hashes has been successfully calculated for calculations
"""Tests that the hashes has been successfully created for calculations
from exciting.
"""
enc = hash_exciting.get_mi2_section(Encyclopedia.m_def)
......@@ -531,3 +532,17 @@ def test_hashes_exciting(hash_exciting):
assert method_hash is not None
assert group_eos_hash is not None
assert group_parametervariation_hash is not None
def test_hashes_undefined(hash_vasp):
"""Tests that the hashes are not present when the method settings cannot be
determined at a sufficient accuracy.
"""
# VASP
enc = hash_vasp.get_mi2_section(Encyclopedia.m_def)
method_hash = enc.method.method_hash
group_eos_hash = enc.method.group_eos_hash
group_parametervariation_hash = enc.method.group_parametervariation_hash
assert method_hash is None
assert group_eos_hash is None
assert group_parametervariation_hash is None
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment