Commit c8df24de authored by Lauri Himanen's avatar Lauri Himanen
Browse files

Added a package level conftest.py for normalizer tests, fixed issue in...

Added a package level conftest.py for normalizer tests, fixed issue in fcc_crystal_structure.json (cell was given in angstroms, not SI units), added first test configuration for EncyclopediaNormaliser.
parent 4b3b9bd1
Pipeline #66762 failed with stages
in 13 minutes and 29 seconds
......@@ -30,7 +30,7 @@ class Calculation(MSection):
"""
)
run_type = Quantity(
type=Enum("single point", "geometry optimization", "molecular dynamics", "phonon calculation", "QHA calculation", "GW calculation", "equation of state", "parameter variation"),
type=Enum("single point", "geometry optimization", "molecular dynamics", "phonon calculation", "QHA calculation", "GW calculation", "equation of state", "parameter variation", "unavailable"),
description="""
Defines the type of run identified for this entry.
"""
......
......@@ -18,8 +18,10 @@ In NOMAD-coe those were programmed in python (we'll reuse) and scala (we'll rewr
Currently the normalizers are:
- system.py (contains aspects of format stats, system, system type, and symmetry normalizer)
- optimade.py
- fhiaims.py
- repository.py
- dos.py
- encyclopedia.py (used to create the data in NOMAD-coe Encyclopedia)
The normalizers are available via
......
......@@ -30,7 +30,6 @@ class EncyclopediaNormalizer(Normalizer):
"""
def __init__(self, backend):
super().__init__(backend)
self.sec_enc: Encyclopedia = None
# NOTE: Enc specific visualization
def get_atom_labels(self) -> None:
......@@ -324,9 +323,9 @@ class EncyclopediaNormalizer(Normalizer):
# unsupported.
elif n_frame_seq == 1:
frame_seq = frame_sequences[0]
i_sampling_method = frame_seq[r_frame_sequence_to_sampling][0]
i_sampling_method = frame_seq[r_frame_sequence_to_sampling]
section_sampling_method = self._backend[s_sampling_method][i_sampling_method]
sampling_method = section_sampling_method["sampling_method"][0]
sampling_method = section_sampling_method["sampling_method"]
if sampling_method == "molecular_dynamics":
run_type = "molecular dynamics"
......@@ -408,9 +407,9 @@ class EncyclopediaNormalizer(Normalizer):
super().normalize(logger)
# Initialise metainfo structure
self.sec_enc = Encyclopedia()
material = self.sec_enc.m_create(Material)
calculation = self.sec_enc.m_create(Calculation)
sec_enc = Encyclopedia()
material = sec_enc.m_create(Material)
calculation = sec_enc.m_create(Calculation)
# Determine run type, stop if unknown
run_type = self.get_run_type(calculation)
......@@ -439,4 +438,4 @@ class EncyclopediaNormalizer(Normalizer):
self.get_material_hash(material, system)
# Put the encyclopedia section into backend
self._backend.add_mi2_section(self.sec_enc)
self._backend.add_mi2_section(sec_enc)
......@@ -36,11 +36,11 @@ from nomad import config, infrastructure, parsing, processing, app, search
from nomad.datamodel import User, CalcWithMetadata
from nomad.parsing import LocalBackend
from tests import test_parsing, test_normalizing
from tests import test_parsing
from tests.normalizing.conftest import run_normalize
from tests.processing import test_data as test_processing
from tests.test_files import example_file, empty_file
from tests.bravado_flask import FlaskTestHttpClient
from tests.test_normalizing import run_normalize
test_log_level = logging.CRITICAL
example_files = [empty_file, example_file]
......@@ -523,7 +523,7 @@ def parsed(example_mainfile: Tuple[str, str]) -> parsing.LocalBackend:
@pytest.fixture(scope='session')
def normalized(parsed: parsing.LocalBackend) -> parsing.LocalBackend:
""" Provides a normalized calculation in the form of a LocalBackend. """
return test_normalizing.run_normalize(parsed)
return run_normalize(parsed)
@pytest.fixture(scope='function')
......
......@@ -40,7 +40,7 @@
"Na"
],
"lattice_vectors": [
[0, 0.5, 0.5],[0.5, 0 , 0.5],[0.5, 0.5, 0]]
[0, 0.5E-10, 0.5E-10],[0.5E-10, 0 , 0.5E-10],[0.5E-10, 0.5E-10, 0]]
}
],
"section_single_configuration_calculation": [
......@@ -73,4 +73,4 @@
]
}
]
}
\ No newline at end of file
}
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import pytest
from nomad.parsing import LocalBackend
from nomad.normalizing import normalizers
from nomad.metainfo.encyclopedia import Encyclopedia
from tests.test_parsing import parsed_vasp_example # pylint: disable=unused-import
from tests.test_parsing import parsed_template_example # pylint: disable=unused-import
from tests.test_parsing import parsed_example # pylint: disable=unused-import
from tests.test_parsing import parse_file
def run_normalize(backend: LocalBackend) -> LocalBackend:
status, _ = backend.status
assert status == 'ParseSuccess'
for normalizer_class in normalizers:
normalizer = normalizer_class(backend)
normalizer.normalize()
return backend
@pytest.fixture
def normalized_vasp_example(parsed_vasp_example: LocalBackend) -> LocalBackend:
return run_normalize(parsed_vasp_example)
@pytest.fixture
def normalized_example(parsed_example: LocalBackend) -> LocalBackend:
return run_normalize(parsed_example)
@pytest.fixture
def normalized_template_example(parsed_template_example) -> LocalBackend:
return run_normalize(parsed_template_example)
@pytest.fixture
def geometry_optimization(parsed_template_example) -> Encyclopedia:
parser_name = "parsers/template"
filepath = "tests/data/normalizers/fcc_crystal_structure.json"
backend = parse_file((parser_name, filepath))
backend = run_normalize(backend)
enc = backend.get_mi2_section(Encyclopedia.m_def)
return enc
......@@ -14,27 +14,14 @@
import numpy as np
from nomad.parsing import LocalBackend
from nomad.normalizing import normalizers
from tests.test_parsing import parse_file
from tests.normalizing.conftest import run_normalize
vasp_parser_dos = (
'parsers/vasp', 'tests/data/parsers/vasp/vasp_dos.xml')
def run_normalize(backend: LocalBackend) -> LocalBackend:
status, _ = backend.status
assert status == 'ParseSuccess'
for normalizer_class in normalizers:
normalizer = normalizer_class(backend)
normalizer.normalize()
return backend
def test_dos_normalizer():
"""
Ensure the DOS normalizer acted on the DOS values. We take a VASP example.
......
......@@ -12,35 +12,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import pytest
from nomad.metainfo.encyclopedia import Encyclopedia
from tests.normalizing.conftest import geometry_optimization # pylint: disable=unused-import
from nomad.parsing import LocalBackend
from nomad.normalizing import normalizers
from tests.test_parsing import parsed_vasp_example # pylint: disable=unused-import
from tests.test_parsing import parsed_template_example # pylint: disable=unused-import
from tests.test_parsing import parsed_example # pylint: disable=unused-import
from tests.test_parsing import parse_file
from tests.utils import assert_log
def run_normalize(backend: LocalBackend) -> LocalBackend:
status, _ = backend.status
assert status == 'ParseSuccess'
for normalizer_class in normalizers:
normalizer = normalizer_class(backend)
normalizer.normalize()
return backend
@pytest.fixture
def single_point(parsed_template_example) -> LocalBackend:
return run_normalize(parsed_template_example)
def test_single_point(single_point: LocalBackend):
"""Tests that single point calculations are correctly processed."
def test_geometry_optimization(geometry_optimization: Encyclopedia):
"""Tests that geometry optimizations are correctly processed."
"""
pass
run_type = geometry_optimization.calculation.run_type
assert run_type == "geometry optimization"
......@@ -12,16 +12,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import pytest
from nomad import datamodel, config
from nomad.parsing import LocalBackend
from nomad.normalizing import normalizers
from tests.test_parsing import parsed_vasp_example # pylint: disable=unused-import
from tests.test_parsing import parsed_template_example # pylint: disable=unused-import
from tests.test_parsing import parsed_example # pylint: disable=unused-import
from tests.test_parsing import parse_file
from tests.normalizing.conftest import run_normalize
from tests.utils import assert_log
......@@ -75,32 +70,6 @@ map would be empty.
"""
def run_normalize(backend: LocalBackend) -> LocalBackend:
status, _ = backend.status
assert status == 'ParseSuccess'
for normalizer_class in normalizers:
normalizer = normalizer_class(backend)
normalizer.normalize()
return backend
@pytest.fixture
def normalized_vasp_example(parsed_vasp_example: LocalBackend) -> LocalBackend:
return run_normalize(parsed_vasp_example)
@pytest.fixture
def normalized_example(parsed_example: LocalBackend) -> LocalBackend:
return run_normalize(parsed_example)
@pytest.fixture
def normalized_template_example(parsed_template_example) -> LocalBackend:
return run_normalize(parsed_template_example)
def test_template_example_normalizer(parsed_template_example, no_warn, caplog):
run_normalize(parsed_template_example)
......@@ -179,13 +148,12 @@ def test_symmetry_classification_fcc():
def test_system_classification():
"Ensure the classification of fcc Na is atom"
# TODO: @dts - This is a bit strange that a system with only
# one atom is automatically classified as atom. It could be
# an elemental solid.
"""Tests that the system classification is correct for different kind of systems
"""
# Bulk system
backend = parse_file(fcc_symmetry)
backend = run_normalize(backend)
expected_system_type = 'atom'
expected_system_type = 'bulk'
system_type = backend.get_value('system_type')
assert expected_system_type == system_type
......
# Copyright 2018 Markus Scheidgen
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an"AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import pytest
import numpy as np
from nomad import datamodel, config
from nomad.parsing import LocalBackend
from nomad.normalizing import normalizers
from tests.test_parsing import parsed_vasp_example # pylint: disable=unused-import
from tests.test_parsing import parsed_template_example # pylint: disable=unused-import
from tests.test_parsing import parsed_example # pylint: disable=unused-import
from tests.test_parsing import parse_file
from tests.utils import assert_log
boolean_positions = (
'parsers/template', 'tests/data/normalizers/no_sim_cell_boolean_positions.json')
single_string_atom_labels = (
'parsers/template', 'tests/data/normalizers/single_string_atom_labels.json')
unknown_atom_label = (
'parsers/template', 'tests/data/normalizers/unknown_atom_label_test.json')
fcc_symmetry = (
'parsers/template', 'tests/data/normalizers/fcc_crystal_structure.json')
vasp_parser = (
'parsers/vasp', 'tests/data/parsers/vasp/vasp.xml')
vasp_parser_dos = (
'parsers/vasp', 'tests/data/parsers/vasp/vasp_dos.xml')
glucose_atom_labels = (
'parsers/template', 'tests/data/normalizers/glucose_atom_labels.json')
symmetry_keys = ['spacegroup', 'spacegroup_symbol', 'crystal_system']
calc_metadata_keys = [
'code_name', 'code_version', 'basis_set', 'xc_functional', 'system', 'formula'] + symmetry_keys
parser_exceptions = {
'parsers/wien2k': ['xc_functional'],
'parsers/nwchem': symmetry_keys,
'parsers/bigdft': symmetry_keys,
'parsers/gaussian': symmetry_keys,
'parsers/abinit': ['formula', 'system'] + symmetry_keys,
'parsers/dl-poly': ['formula', 'basis_set', 'xc_functional', 'system'] + symmetry_keys,
'parsers/lib-atoms': ['basis_set', 'xc_functional'],
'parsers/orca': symmetry_keys,
'parsers/octopus': symmetry_keys,
'parsers/phonopy': ['basis_set', 'xc_functional'],
'parsers/gpaw2': symmetry_keys,
'parsers/gamess': ['formula', 'system'] + symmetry_keys,
'parsers/gulp': ['formula', 'xc_functional', 'system'] + symmetry_keys,
'parsers/turbomole': symmetry_keys,
'parsers/elastic': ['basis_set', 'xc_functional', 'system'] + symmetry_keys,
'parsers/dmol': ['system'] + symmetry_keys,
'parser/molcas': symmetry_keys,
'parsers/band': ['system'] + symmetry_keys,
'parsers/qbox': ['xc_functional'],
'parser/onetep': ['formula', 'basis_set', 'xc_functional', 'system'] + symmetry_keys
}
"""
Keys that the normalizer for certain parsers might not produce. In an ideal world this
map would be empty.
"""
def run_normalize(backend: LocalBackend) -> LocalBackend:
status, _ = backend.status
assert status == 'ParseSuccess'
for normalizer_class in normalizers:
normalizer = normalizer_class(backend)
normalizer.normalize()
return backend
@pytest.fixture
def normalized_vasp_example(parsed_vasp_example: LocalBackend) -> LocalBackend:
return run_normalize(parsed_vasp_example)
@pytest.fixture
def normalized_example(parsed_example: LocalBackend) -> LocalBackend:
return run_normalize(parsed_example)
@pytest.fixture
def normalized_template_example(parsed_template_example) -> LocalBackend:
return run_normalize(parsed_template_example)
def test_template_example_normalizer(parsed_template_example, no_warn, caplog):
run_normalize(parsed_template_example)
def assert_normalized(backend: LocalBackend):
metadata = datamodel.DFTCalcWithMetadata()
metadata.apply_domain_metadata(backend)
assert metadata.formula is not None
assert metadata.code_name is not None
assert metadata.code_version is not None
assert metadata.basis_set is not None
assert metadata.xc_functional is not None
assert metadata.system is not None
assert metadata.crystal_system is not None
assert len(metadata.atoms) is not None
assert metadata.spacegroup is not None
exceptions = parser_exceptions.get(backend.get_value('parser_name'), [])
if metadata.formula != config.services.unavailable_value:
assert len(metadata.atoms) > 0
for key in calc_metadata_keys:
if key not in exceptions:
assert getattr(metadata, key) != config.services.unavailable_value
def test_normalizer(normalized_example: LocalBackend):
assert_normalized(normalized_example)
def test_normalizer_faulty_matid(caplog):
""" Runs normalizer on an example w/ bools for atom pos. Should force matid error."""
# assert isinstance(backend, LocalBackend)
backend = parse_file(boolean_positions)
run_normalize(backend)
assert_log(caplog, 'ERROR', 'matid project system classification failed')
assert_log(caplog, 'ERROR', 'no lattice vectors but periodicity')
def test_normalizer_single_string_atom_labels(caplog):
"""
Runs normalizer on ['Br1SiSiK'] expects error. Should replace the label with 'X' and
the numbers of postitions should not match the labels.
"""
backend = parse_file(single_string_atom_labels)
run_normalize(backend)
assert_log(caplog, 'ERROR', 'len of atom position does not match number of atoms')
def test_normalizer_unknown_atom_label(caplog, no_warn):
""" Runs normalizer on ['Br','Si','Si','Za'], for normalizeation Za will be replaced,
but stays int the labels.
"""
backend = parse_file(unknown_atom_label)
run_normalize(backend)
assert backend.get_value('atom_labels')[3] == 'Za'
def test_symmetry_classification_fcc():
"""Runs normalizer where lattice vectors should give fcc symmetry."""
backend = parse_file(fcc_symmetry)
backend = run_normalize(backend)
expected_crystal_system = 'cubic'
expected_bravais_lattice = 'cF'
expected_point_group = 'm-3m'
expected_origin_shift = [0, 0, 0]
cyrstal_system = backend.get_value('crystal_system')
assert cyrstal_system == expected_crystal_system
bravais_lattice = backend.get_value('bravais_lattice')
assert bravais_lattice == expected_bravais_lattice
point_group = backend.get_value('point_group')
assert point_group == expected_point_group
origin_shift = backend.get_value('origin_shift')
assert all(origin_shift == expected_origin_shift)
def test_system_classification():
"Ensure the classification of fcc Na is atom"
# TODO: @dts - This is a bit strange that a system with only
# one atom is automatically classified as atom. It could be
# an elemental solid.
backend = parse_file(fcc_symmetry)
backend = run_normalize(backend)
expected_system_type = 'atom'
system_type = backend.get_value('system_type')
assert expected_system_type == system_type
def test_reduced_chemical_formula():
"Ensure we get the right reduced chemical formula for glucose atom labels"
backend = parse_file(glucose_atom_labels)
backend = run_normalize(backend)
expected_red_chem_formula = 'C6H12O6'
reduced_chemical_formula = backend.get_value('chemical_composition_bulk_reduced')
assert expected_red_chem_formula == reduced_chemical_formula
def test_vasp_incar_system():
"""
Ensure we can test an incar value in the VASP example
"""
backend = parse_file(vasp_parser)
backend = run_normalize(backend)
expected_value = 'SrTiO3' # material's formula in vasp.xml
# backend_value = backend.get_value('x_vasp_unknown_incars') # OK
# backend_value = backend.get_value('x_vasp_atom_kind_refs') # OK
backend_value = backend.get_value('x_vasp_incar_SYSTEM') # OK
print("backend_value: ", backend_value)
assert expected_value == backend_value
def test_springer_normalizer():
"""
Ensure the Springer normalizer works well with the VASP example.
"""
backend = parse_file(vasp_parser)
backend = run_normalize(backend)
backend_value = backend.get_value('springer_id', 89)
expected_value = 'sd_1932539'
assert expected_value == backend_value
backend_value = backend.get_value('springer_alphabetical_formula', 89)
expected_value = 'O3SrTi'
assert expected_value == backend_value
backend_value = backend.get_value('springer_url', 89)
expected_value = 'http://materials.springer.com/isp/crystallographic/docs/sd_1932539'
assert expected_value == backend_value
def test_dos_normalizer():
"""
Ensure the DOS normalizer acted on the DOS values. We take a VASP example.
"""
backend = parse_file(vasp_parser_dos)
backend = run_normalize(backend)
# Check if 'dos_values' were indeed normalized
# 'dvn' stands for 'dos_values_normalized'
backend_dvn = backend.get_value('dos_values_normalized', 0)
last_value = backend_dvn[0, -1]
expected = 1.7362195274239454e+47
# Compare floats properly with numpy (delta tolerance involved)
assert np.allclose(last_value, expected)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment