Commit 9a5517bd authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Added infrastructure logger to normalizers.

parent 133e089c
Pipeline #37127 failed with stages
in 5 minutes and 45 seconds
...@@ -88,7 +88,9 @@ class FhiAimsBaseNormalizer(Normalizer): ...@@ -88,7 +88,9 @@ class FhiAimsBaseNormalizer(Normalizer):
return [false_hits_integration_grid, false_hits_basis] return [false_hits_integration_grid, false_hits_basis]
def normalize(self) -> None: def normalize(self, logger=None) -> None:
super().normalize(logger)
for index in self._backend.get_sections('section_method'): for index in self._backend.get_sections('section_method'):
to_compare = self._backend.get_value(controlIn_basis_set, index) to_compare = self._backend.get_value(controlIn_basis_set, index)
if to_compare is None: if to_compare is None:
......
...@@ -18,8 +18,6 @@ from typing import List, Dict, Any ...@@ -18,8 +18,6 @@ from typing import List, Dict, Any
from nomad.parsing import AbstractParserBackend from nomad.parsing import AbstractParserBackend
from nomad.utils import get_logger from nomad.utils import get_logger
logger = get_logger(__name__)
s_system = 'section_system' s_system = 'section_system'
s_scc = 'section_single_configuration_calculation' s_scc = 'section_single_configuration_calculation'
s_frame_sequence = 'section_frame_sequence' s_frame_sequence = 'section_frame_sequence'
...@@ -38,10 +36,12 @@ class Normalizer(metaclass=ABCMeta): ...@@ -38,10 +36,12 @@ class Normalizer(metaclass=ABCMeta):
def __init__(self, backend: AbstractParserBackend) -> None: def __init__(self, backend: AbstractParserBackend) -> None:
self._backend = backend self._backend = backend
self.logger = get_logger(__name__)
@abstractmethod @abstractmethod
def normalize(self) -> None: def normalize(self, logger=None) -> None:
pass if logger is not None:
self.logger = logger.bind(normalizer=self.__class__.__name__)
class SystemBasedNormalizer(Normalizer, metaclass=ABCMeta): class SystemBasedNormalizer(Normalizer, metaclass=ABCMeta):
...@@ -93,7 +93,9 @@ class SystemBasedNormalizer(Normalizer, metaclass=ABCMeta): ...@@ -93,7 +93,9 @@ class SystemBasedNormalizer(Normalizer, metaclass=ABCMeta):
def normalize_system(self, section_system: Dict[str, Any]) -> None: def normalize_system(self, section_system: Dict[str, Any]) -> None:
pass pass
def normalize(self) -> None: def normalize(self, logger=None) -> None:
super().normalize(logger)
if self._all_sections: if self._all_sections:
systems = self._backend.get_sections(s_system) systems = self._backend.get_sections(s_system)
else: else:
...@@ -121,11 +123,11 @@ class SystemBasedNormalizer(Normalizer, metaclass=ABCMeta): ...@@ -121,11 +123,11 @@ class SystemBasedNormalizer(Normalizer, metaclass=ABCMeta):
try: try:
self._normalize_system(g_index) self._normalize_system(g_index)
except KeyError as e: except KeyError as e:
logger.error( self.logger.error(
'Could not read all input data', normalizer=self.__class__.__name__, 'Could not read all input data', normalizer=self.__class__.__name__,
section='section_system', g_index=g_index, key_error=str(e)) section='section_system', g_index=g_index, key_error=str(e))
except Exception as e: except Exception as e:
logger.error( self.logger.error(
'Unexpected error during normalizing', normalizer=self.__class__.__name__, 'Unexpected error during normalizing', normalizer=self.__class__.__name__,
section='section_system', g_index=g_index, exc_info=e) section='section_system', g_index=g_index, exc_info=e)
raise e raise e
...@@ -13,7 +13,7 @@ ...@@ -13,7 +13,7 @@
# limitations under the License. # limitations under the License.
from nomad.normalizing.normalizer import SystemBasedNormalizer from nomad.normalizing.normalizer import SystemBasedNormalizer
from symmetrynormalizer.symmetry_analysis import normalize from symmetrynormalizer import symmetry_analysis
class SymmetryNormalizer(SystemBasedNormalizer): class SymmetryNormalizer(SystemBasedNormalizer):
...@@ -24,4 +24,5 @@ class SymmetryNormalizer(SystemBasedNormalizer): ...@@ -24,4 +24,5 @@ class SymmetryNormalizer(SystemBasedNormalizer):
super().__init__(backend, all_sections=True) super().__init__(backend, all_sections=True)
def normalize_system(self, section_system) -> None: def normalize_system(self, section_system) -> None:
normalize(self._backend, section_system) symmetry_analysis.logging = self.logger
symmetry_analysis.normalize(self._backend, section_system)
...@@ -17,8 +17,8 @@ import numpy ...@@ -17,8 +17,8 @@ import numpy
import spglib import spglib
from nomadcore.json_support import addShasOfJson from nomadcore.json_support import addShasOfJson
from statsnormalizer.stats import crystalSystem from statsnormalizer import stats
from statsnormalizer.classify_structure import ClassifyStructure from statsnormalizer import classify_structure
from nomad.normalizing.normalizer import SystemBasedNormalizer from nomad.normalizing.normalizer import SystemBasedNormalizer
...@@ -44,6 +44,9 @@ class SystemNormalizer(SystemBasedNormalizer): ...@@ -44,6 +44,9 @@ class SystemNormalizer(SystemBasedNormalizer):
return 0 return 0
def normalize_system(self, section_system) -> None: def normalize_system(self, section_system) -> None:
stats.logging = self.logger
classify_structure.logger = self.logger
results = dict() results = dict()
atom_labels = section_system['atom_labels'] atom_labels = section_system['atom_labels']
...@@ -87,7 +90,7 @@ class SystemNormalizer(SystemBasedNormalizer): ...@@ -87,7 +90,7 @@ class SystemNormalizer(SystemBasedNormalizer):
results['gIndex'] = section_system['gIndex'] results['gIndex'] = section_system['gIndex']
results['name'] = 'section_system' results['name'] = 'section_system'
structure = ClassifyStructure(None, jsonValue={ structure = classify_structure.ClassifyStructure(None, jsonValue={
"sections": [{ "sections": [{
"name": "section_run", "name": "section_run",
"gIndex": 1, "gIndex": 1,
...@@ -132,7 +135,7 @@ class SystemNormalizer(SystemBasedNormalizer): ...@@ -132,7 +135,7 @@ class SystemNormalizer(SystemBasedNormalizer):
n = symm.get("number") n = symm.get("number")
if n: if n:
self._backend.openNonOverlappingSection('section_symmetry') self._backend.openNonOverlappingSection('section_symmetry')
self._backend.addValue("bravais_lattice", crystalSystem(n)) self._backend.addValue("bravais_lattice", stats.crystalSystem(n))
self._backend.closeNonOverlappingSection('section_symmetry') self._backend.closeNonOverlappingSection('section_symmetry')
# for quantity in ["origin_shift", "std_lattice"]: # for quantity in ["origin_shift", "std_lattice"]:
......
...@@ -13,7 +13,7 @@ ...@@ -13,7 +13,7 @@
# limitations under the License. # limitations under the License.
from nomad.normalizing.normalizer import SystemBasedNormalizer from nomad.normalizing.normalizer import SystemBasedNormalizer
from systemtypenormalizer.classify_structure import ClassifyStructure from systemtypenormalizer import classify_structure
class SystemTypeNormalizer(SystemBasedNormalizer): class SystemTypeNormalizer(SystemBasedNormalizer):
...@@ -21,7 +21,9 @@ class SystemTypeNormalizer(SystemBasedNormalizer): ...@@ -21,7 +21,9 @@ class SystemTypeNormalizer(SystemBasedNormalizer):
super().__init__(backend, all_sections=True) super().__init__(backend, all_sections=True)
def normalize_system(self, section_system) -> None: def normalize_system(self, section_system) -> None:
structure = ClassifyStructure(section_system) classify_structure.logger = self.logger
structure = classify_structure.ClassifyStructure(section_system)
structure.classify() structure.classify()
structure_type = structure.classification structure_type = structure.classification
self._backend.addValue('system_type', structure_type) self._backend.addValue('system_type', structure_type)
...@@ -139,22 +139,24 @@ class Calc(Proc): ...@@ -139,22 +139,24 @@ class Calc(Proc):
@task @task
def parsing(self): def parsing(self):
logger = self.get_logger()
parser = parser_dict[self.parser] parser = parser_dict[self.parser]
self._parser_backend = parser.run(self.mainfile_tmp_path, logger=self.get_logger()) self._parser_backend = parser.run(self.mainfile_tmp_path, logger=logger)
if self._parser_backend.status[0] != 'ParseSuccess': if self._parser_backend.status[0] != 'ParseSuccess':
error = self._parser_backend.status[1] error = self._parser_backend.status[1]
self.fail(error, level=logging.DEBUG) self.fail(error, level=logging.DEBUG)
@task @task
def normalizing(self): def normalizing(self):
logger = self.get_logger()
for normalizer in normalizers: for normalizer in normalizers:
normalizer_name = normalizer.__name__ normalizer_name = normalizer.__name__
normalizer(self._parser_backend).normalize() normalizer(self._parser_backend).normalize(logger=logger)
if self._parser_backend.status[0] != 'ParseSuccess': if self._parser_backend.status[0] != 'ParseSuccess':
error = self._parser_backend.status[1] error = self._parser_backend.status[1]
self.fail(error, normalizer=normalizer_name, level=logging.WARNING) self.fail(error, normalizer=normalizer_name, level=logging.WARNING)
return return
self.get_logger().debug( logger.debug(
'completed normalizer successfully', normalizer=normalizer_name) 'completed normalizer successfully', normalizer=normalizer_name)
@task @task
......
import pytest import pytest
from mongoengine import connect from mongoengine import connect
from mongoengine.connection import disconnect from mongoengine.connection import disconnect
import logging
from nomad import config from nomad import config
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment