Fixed up normalizer test for faulty matid classification.

.vscode/.gitignore

 .ropeproject/
-*.sql
+*.sql
\ No newline at end of file

.vscode/launch.json

@@ -44,7 +44,27 @@
       "cwd": "${workspaceFolder}",
       "program": "${workspaceFolder}/.pyenv/bin/pytest",
       "args": [
-        "-sv", "tests/test_parsing.py::test_parser[parsers/random-test/data/parsers/random_0]"
+        "-sv", "tests/test_normalizing.py::test_normalizer[parsers/cpmd-tests/data/parsers/cpmd/geo_output.out]"
+      ]
+    },
+    {
+      "name": "Python: crystal normalizer test",
+      "type": "python",
+      "request": "launch",
+      "cwd": "${workspaceFolder}",
+      "program": "${workspaceFolder}/.pyenv/bin/pytest",
+      "args": [
+        "-sv", "tests/test_normalizing.py::test_normalizer[parsers/crystal-tests/data/parsers/crystal/si.out]"
+      ]
+    },
+    {
+      "name": "Python: nwchem normalizer test h2o sp test",
+      "type": "python",
+      "request": "launch",
+      "cwd": "${workspaceFolder}",
+      "program": "${workspaceFolder}/.pyenv/bin/pytest",
+      "args": [
+        "-sv", "tests/test_normalizing.py::test_normalizer[parsers/nwchem-tests/data/parsers/nwchem/sp_output.out]"
       ]
     },
     {

nomad/normalizing/__init__.py

@@ -15,8 +15,6 @@ from typing import List, Any
 
 from .normalizer import Normalizer
 from .system import SystemNormalizer
-from .symmetry import SymmetryAndTypeNormalizer
-from .systemtype import SystemTypeNormalizer
 from .fhiaims import FhiAimsBaseNormalizer
 
 """

nomad/normalizing/symmetry.py

-# Copyright 2018 Markus Scheidgen
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an"AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-""" This file seeks to find the symmetry and classfication of the material.
-
-We use Matid to analyze the symmetry and classify the material. If we don't have
-basic parameters such as lattice vectors or cell structure we simply skip this
-normalizer. In the past (pre Jan 2019), we had a seperate normalizer for system type
-to classify the material. We have since fused the system type and symmetry normalizers
-into one file."""
-
-import numpy as np
-# import os.path
-import logging
-import sys
-from ase import Atoms
-from matid import SymmetryAnalyzer, Classifier
-
-from nomad.normalizing.normalizer import SystemBasedNormalizer
-# from nomadcore.local_meta_info import loadJsonFile, InfoKindEl
-# from nomadcore.parser_backend import JsonParseEventsWriterBackend
-# from nomadcore.parse_streamed_dicts import ParseStreamedDicts
-
-# TODO, one normalier, called system normalizer.
-
-
-class SymmetryAndTypeNormalizer(SystemBasedNormalizer):
-    """ This is basically a copy of the legace NOMAD-coe symmetry normalizer."""
-    def __init__(self, backend):
-        super().__init__(backend, all_sections=True)
-
-    def normalize_system(self, section_system) -> None:
-        print('symmetry normalizer.')
-        self.atom_labels = section_system["atom_labels"]
-        self.atom_pos = section_system["atom_positions"]
-
-        # Try to first read the cell information from the renamed metainfo
-        # lattice_vectors, if this doesn't work try the depreciated name
-        # simulation_cell. Otherwise, if neither are present, assign None.
-        self.cell = section_system.get(
-            'lattice_vectors', section_system.get('simulation_cell', None)
-        )
-
-        if self.cell is None:
-            # Then the parser hasn't parsed any information about periodicity.
-            # We therefore assume we're simulating a single cell without
-            # periodicity and don't try to ascertain symmetry information.
-            return None
-        # TODO: dts@, should we assign pbc = [False, False, False]
-        # when we can't find the pbc? Either skip symmetry without pbc or continue
-        # with False, False, False values. Let's ask around.
-        self.pbc = section_system.get('configuration_periodic_dimensions', None)
-        if self.pbc is None:
-            # Without a pbc value we cannot build an ASE atoms object.
-            return None
-
-        # The pbc should be defined as a single-dimensional list.
-        if len(np.asarray(self.pbc).shape) == 2:
-            self.pbc = self.pbc[0, :]
-
-        # Build an ASE atoms object to feed into Matid.
-        try:
-            self.atoms = Atoms(
-                positions=1e10 * np.asarray(self.atom_pos),
-                symbols=np.asarray(self.atom_labels),
-                cell=1e10 * np.asarray(self.cell),
-                pbc=self.pbc
-            )
-        except Exception:
-            self.logger.error(
-                'The ASE library is unable to build an object from the member'
-                'variables.'
-            )
-
-        # Classify the material's system type.
-        self.system_type_classification()
-        # Analyze the symmetry of the material.
-        self.symmetry_analysis()
-
-    def symmetry_analysis(self) -> None:
-        """Analyze the symmetry of the material bein simulated.
-
-        We feed in the parsed values in section_system to the
-        the symmetry analyzer. We then use the Matid library
-        to classify the system as 0D, 1D, 2D or 3D and more specific
-        when possible. When lattice vectors or simulation cells are
-        not present we skip this analysis.
-
-        Args:
-            None: We feed in the bakend and atoms object from the
-            SymmetryAndType normalizer.
-
-        Returns:
-            None: The method should write symmetry variables
-            to the backend which is member of this class.
-        """
-        # Try to use Matid's symmetry analyzer to anlyze the ASE object.
-        # TODO: dts, find out what the symmetry_tol does.
-        try:
-            symm = SymmetryAnalyzer(self.atoms, symmetry_tol=0.1)
-
-            space_group_number = symm.get_space_group_number()
-
-            hall_number = symm.get_hall_number()
-            hall_symbol = symm.get_hall_symbol()
-
-            crystal_system = symm.get_crystal_system()
-            print("crystal_system is: %s" % crystal_system)
-            bravais_lattice = symm.get_bravais_lattice()
-            point_group = symm.get_point_group()
-
-            orig_wyckoff = symm.get_wyckoff_letters_original()
-            prim_wyckoff = symm.get_wyckoff_letters_primitive()
-            conv_wyckoff = symm.get_wyckoff_letters_conventional()
-
-            orig_equivalent_atoms = symm.get_equivalent_atoms_original()
-            prim_equivalent_atoms = symm.get_equivalent_atoms_primitive()
-            conv_equivalent_atoms = symm.get_equivalent_atoms_conventional()
-            international_short = symm.get_space_group_international_short()
-            point_group = symm.get_point_group()
-
-            conv_sys = symm.get_conventional_system()
-            conv_pos = conv_sys.get_scaled_positions()
-            conv_cell = conv_sys.get_cell()
-            conv_num = conv_sys.get_atomic_numbers()
-
-            prim_sys = symm.get_primitive_system()
-            prim_pos = prim_sys.get_scaled_positions()
-            prim_cell = prim_sys.get_cell()
-            prim_num = prim_sys.get_atomic_numbers()
-
-            transform = symm._get_spglib_transformation_matrix()
-            origin_shift = symm._get_spglib_origin_shift()
-
-        except Exception:
-            self.logger.error(
-                'The matid project symmetry analyzer fails on the ASE'
-                ' object from this section.'
-            )
-            return None # Without trying to write any symmetry data.
-
-        # Write data extracted from Matid symmetry analysis to the backend.
-        symGid = self._backend.openSection("section_symmetry")
-        # TODO: @dts, should we change the symmetry_method to MATID?
-        self._backend.addValue("symmetry_method", "spg_normalized")
-        self._backend.addValue("space_group_number", space_group_number)
-        self._backend.addValue("hall_number", hall_number)
-        self._backend.addValue("hall_symbol", hall_symbol)
-        self._backend.addValue("international_short_symbol", international_short)
-        self._backend.addValue("point_group", point_group)
-        self._backend.addValue("crystal_system", crystal_system)
-        self._backend.addValue("bravais_lattice", bravais_lattice)
-        self._backend.addArrayValues("origin_shift", origin_shift)
-        self._backend.addArrayValues("transformation_matrix", transform)
-
-        stdGid = self._backend.openSection("section_std_system")
-        self._backend.addArrayValues("lattice_vectors_std", conv_cell)
-        self._backend.addArrayValues("atom_positions_std", conv_pos)
-        self._backend.addArrayValues("atomic_numbers_std", conv_num)
-        self._backend.addArrayValues("wyckoff_letters_std", conv_wyckoff)
-        self._backend.addArrayValues("equivalent_atoms_std", conv_equivalent_atoms)
-        self._backend.closeSection("section_std_system", stdGid)
-
-        primGid = self._backend.openSection("section_primitive_system")
-        self._backend.addArrayValues("lattice_vectors_primitive", prim_cell)
-        self._backend.addArrayValues("atom_positions_primitive", prim_pos)
-        self._backend.addArrayValues("atomic_numbers_primitive", prim_num)
-        self._backend.addArrayValues("wyckoff_letters_primitive", prim_wyckoff)
-        self._backend.addArrayValues("equivalent_atoms_primitive", prim_equivalent_atoms)
-        self._backend.closeSection("section_primitive_system", primGid)
-
-        origGid = self._backend.openSection("section_original_system")
-        self._backend.addArrayValues("wyckoff_letters_original", orig_wyckoff)
-        self._backend.addArrayValues("equivalent_atoms_original", orig_equivalent_atoms)
-        self._backend.closeSection("section_original_system", origGid)
-
-        self._backend.closeSection("section_symmetry", symGid)
-        # nomad-xt: context already closed in nomad-xt.
-        # backend.closeContext(context)
-        sys.stdout.flush()
-
-    def system_type_classification(self):
-        # Try to classify the ASE materials object using Matid's classification.
-        # TODO: add mapping of matid to Nomad classifications.
-        try:
-            # Define the classifier as Matid's Classifier that we've imported.
-            classifier = Classifier()
-            # Perform classification on the atoms ASE object.
-            system_type = classifier.classify(self.atoms)
-            # For debug:
-            print(" The classification from Matid is: %s" % system_type)
-        except Exception:
-            self.logger.error(
-                'The matid project clsasification fails on the ASE'
-                ' object from this section.'
-            )
-            return None  # Without saving any system type value.
-        self._backend.addValue('system_type', system_type)

nomad/normalizing/system.py

@@ -62,17 +62,21 @@ class SystemNormalizer(SystemBasedNormalizer):
             # Then the parser hasn't parsed any information about periodicity.
             # We therefore assume we're simulating a single cell without
             # periodicity and don't try to ascertain symmetry information.
-            self.atoms = ase.Atoms(
-                positions=1e10 * np.asarray(self.atom_positions),
-                symbols=np.asarray(self.atom_labels)
-            )
+            try:
+                self.atoms = ase.Atoms(
+                    positions=1e10 * np.asarray(self.atom_positions),
+                    symbols=np.asarray(self.atom_labels)
+                )
+            except Exception:
+                self.logger.error(
+                    'The ASE library is unable to build an object from the parsed vars.'
+                )
             # Classify the material's system type.
             self.system_type_classification()
             if self.nomad_system_type not in ['Atom', 'Molecule / Cluster']:
                 self.logger.error(
-                    'Matid has classified the sim as more than 1D despite having'
-                    'no simulation_cell/lattice_vectors'
-                )
+                    'Matid classified more than 1D despite having no simulation_cell')
+
             # Return w/out symmetry analysis since we don't have a sim_cell.
             return None
 
@@ -95,7 +99,7 @@ class SystemNormalizer(SystemBasedNormalizer):
         except Exception:
             self.logger.error(
                 'The ASE library is unable to build an object from the member'
-                'variables.'
+                'variables: atom_positions, atom_labels, simulation_cell and pbc.'
             )
 
         # Classify the material's system type.
@@ -298,8 +302,8 @@ class SystemNormalizer(SystemBasedNormalizer):
         # Check to make sure a match was found in translating classes.
         if (nomad_classification is None) or (nomad_classification == 'Unknown'):
             # Then something unexpected has happened with our system_type.
-            self.logger.error('Matid classfication has given us an unexpected type.'
-                              ' No match was found for this system type: %s' % system_type)
+            self.logger.error(
+                'Matid classfication has given us an unexpected type: %s' % system_type)
 
         if nomad_classification == 'Atom' and (len(self.atom_labels) > 1):
             nomad_classification = 'Molecule / Cluster'

nomad/normalizing/systemtype.py

-# Copyright 2018 Markus Scheidgen
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an"AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from nomad.normalizing.normalizer import SystemBasedNormalizer
-from systemtypenormalizer import classify_structure
-
-
-class SystemTypeNormalizer(SystemBasedNormalizer):
-    def __init__(self, backend):
-        super().__init__(backend, all_sections=True)
-
-    def normalize_system(self, section_system) -> None:
-        classify_structure.logger = self.logger
-
-        structure = classify_structure.ClassifyStructure(section_system)
-        structure.classify()
-        structure_type = structure.classification
-        print("Structure type form the system type normalizer is:")
-        print(structure_type)
-        self._backend.addValue('system_type', structure_type)

requirements-dep.txt

@@ -9,7 +9,6 @@ scipy
 spglib
 matid
 ase==3.15.0
-systax==0.1.2
 Pint==0.7.2
 mdtraj==1.9.1
 panedr==0.2

requirements-dev.txt

@@ -11,6 +11,5 @@ pytest-timeout
 pytest-cov
 rope
 mongomock
-
 numpy
 cython>=0.19
\ No newline at end of file

tests/data/normalizers/no_sim_cell_boolean_positions.json

+{
+    "section_run": [
+      {
+        "_name": "section_run",
+        "_gIndex": 0,
+        "program_name": "VASP",
+        "program_version": "4.6.35  3Apr08 complex  parallel LinuxIFC",
+        "program_basis_set_type": "plane waves",
+        "section_method": [
+          {
+            "_name": "section_method",
+            "_gIndex": 0,
+
+            "electronic_structure_method": "DFT",
+            "section_XC_functionals": [
+              {
+                "_name": "section_XC_functionals",
+                "_gIndex": 0,
+                "XC_functional_name": "GGA_X_PBE"
+              }
+            ]
+          }
+        ],
+        "section_system": [
+          {
+            "_name": "section_system",
+            "_gIndex": 0,
+            "configuration_periodic_dimensions": [
+              true,
+              true,
+              true
+            ],
+            "atom_positions": [
+              [
+                true,
+                false,
+                true
+              ],
+              [
+                true,
+                false,
+                true
+              ],
+              [
+                true,
+                false,
+                true
+              ],
+              [
+                true,
+                true,
+                false
+              ]
+            ],
+            "atom_labels": [
+              "Br",
+              "K",
+              "Si",
+              "Si"
+            ]
+          }
+        ],
+        "section_single_configuration_calculation": [
+          {
+            "_name": "section_single_configuration_calculation",
+            "_gIndex": 0,
+            "single_configuration_calculation_to_system_ref": 0,
+            "single_configuration_to_calculation_method_ref": 0,
+            "energy_free": -1.5936767191492225e-18,
+            "energy_total": -1.5935696296699573e-18,
+            "energy_total_T0": -3.2126683561907e-22
+          }
+        ],
+        "section_sampling_method": [
+          {
+            "_name": "section_sampling_method",
+            "_gIndex": 0,
+            "sampling_method": "geometry_optimization"
+          }
+        ],
+        "section_frame_sequence": [
+          {
+            "_name": "section_frame_sequence",
+            "_gIndex": 0,
+            "frame_sequence_to_sampling_ref": 0,
+            "frame_sequence_local_frames_ref": [
+              0
+            ]
+          }
+        ]
+      }
+    ]
+  }
\ No newline at end of file

tests/test_normalizing.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import json
 import pytest
 
 from nomad.parsing import LocalBackend
@@ -20,6 +21,7 @@ from nomad.normalizing import normalizers
 from tests.test_parsing import parsed_vasp_example  # pylint: disable=unused-import
 from tests.test_parsing import parsed_template_example  # pylint: disable=unused-import
 from tests.test_parsing import parsed_example  # pylint: disable=unused-import
+from tests.test_parsing import parsed_faulty_unknown_matid_example  # pylint: disable=unused-import
 
 
 def run_normalize(backend: LocalBackend) -> LocalBackend:
@@ -50,8 +52,6 @@ def normalized_template_example(parsed_template_example) -> LocalBackend:
 
 
 def assert_normalized(backend):
-    with open("test_file_name.json", "wt") as file:
-        backend.write_json(file)
     # The assertions are based on the quanitites need for the repository.
     assert backend.get_value('atom_species', 0) is not None
     assert backend.get_value('system_type', 0) is not None
@@ -69,3 +69,26 @@ def assert_normalized(backend):
 
 def test_normalizer(normalized_example: LocalBackend, no_warn):
     assert_normalized(normalized_example)
+
+
+def test_normalizer_faulty_matid(
+        parsed_faulty_unknown_matid_example: LocalBackend, caplog):
+    """ Runs normalizer on an example w/ bools for atom pos. Should force matid error."""
+    run_normalize(parsed_faulty_unknown_matid_example)
+    unknown_class_error = (
+        'Matid classfication has given us an unexpected type')
+
+    wrong_class_for_no_sim_cell = (
+        'Matid classified more than 1D despite having no simulation_cell')
+
+    assert_log(caplog, 'ERROR', unknown_class_error)
+    assert_log(caplog, 'ERROR', wrong_class_for_no_sim_cell)
+
+
+def assert_log(caplog, level, event_part):
+    # TODO: @dts, find a new home for this fxn, sadly it can't go in conftest.py
+    record_receieved = False
+    for record in caplog.get_records(when='call'):
+        if record.levelname == level:
+            record_receieved |= event_part in json.loads(record.msg)['event']
+    assert record_receieved

tests/test_parsing.py

@@ -38,6 +38,10 @@ parser_examples = [
     ('parsers/bigdft', 'tests/data/parsers/bigdft/n2_output.out')
 ]
 
+faulty_unknown_one_d_matid_example = [
+    ('parsers/template', 'tests/data/normalizers/no_sim_cell_boolean_positions.json')
+]
+
 
 class TestLocalBackend(object):
 
@@ -237,6 +241,14 @@ def parsed_template_example() -> LocalBackend:
         'parsers/template', 'tests/data/parsers/template.json')
 
 
+@pytest.fixture(
+    params=faulty_unknown_one_d_matid_example, ids=lambda spec: '%s-%s' % spec)
+
+def parsed_faulty_unknown_matid_example(caplog, request) -> LocalBackend:
+    parser_name, mainfile = request.param
+    return run_parser(parser_name, mainfile)
+
+
 @pytest.fixture(params=parser_examples, ids=lambda spec: '%s-%s' % spec)
 def parsed_example(caplog, request) -> LocalBackend:
     parser_name, mainfile = request.param