Commit c8b07ccd authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Minor optimizations #124.

parent d8588540
Pipeline #44469 passed with stages
in 25 minutes and 41 seconds
......@@ -54,6 +54,9 @@ NomadServicesConfig = namedtuple('NomadServicesConfig', ['api_host', 'api_port',
MailConfig = namedtuple('MailConfig', ['host', 'port', 'user', 'password', 'from_address'])
""" Used to configure how nomad can send email """
NormalizeConfig = namedtuple('NormalizeConfig', ['all_systems'])
""" Used to configure the normalizers """
files = FilesConfig(
uploads_bucket='uploads',
raw_bucket=os.environ.get('NOMAD_FILES_RAW_BUCKET', 'raw'),
......@@ -137,6 +140,9 @@ mail = MailConfig(
password=os.environ.get('NOMAD_SMTP_PASSWORD', None),
from_address=os.environ.get('NOMAD_MAIL_FROM', 'webmaster@nomad-coe.eu')
)
normalize = NormalizeConfig(
all_systems=False
)
console_log_level = get_loglevel_from_env('NOMAD_CONSOLE_LOGLEVEL', default_level=logging.WARNING)
service = os.environ.get('NOMAD_SERVICE', 'unknown nomad service')
......
......@@ -435,7 +435,8 @@ class NomadCOEMigration:
logger.info(
'source target missmatch', quantity=key,
source_value=source_value, target_value=target_value)
source_value=source_value, target_value=target_value,
value_diff='%s->%s' % (str(source_value), str(target_value)))
return False
if source_value is None and target_value is not None:
......
......@@ -89,27 +89,51 @@ class SystemBasedNormalizer(Normalizer, metaclass=ABCMeta):
super().normalize(logger)
if self._all_sections:
systems = self._backend.get_sections(s_system)
try:
systems = self._backend.get_sections(s_system)
except Exception:
systems = []
else:
# look for sccs in last frames
sccs = []
for frame_seq in self._backend.get_sections(s_frame_sequence):
frames = self._backend.get_value(r_frame_sequence_local_frames, frame_seq)
try:
frame_seqs = self._backend.get_sections(s_frame_sequence)
except Exception:
frame_seqs = []
for frame_seq in frame_seqs:
try:
frames = self._backend.get_value(r_frame_sequence_local_frames, frame_seq)
except Exception:
frames = []
if len(frames) > 0:
sccs.append(frames[-1])
# no sccs from frames -> consider all sccs
if len(sccs) == 0:
sccs = self._backend.get_sections(s_scc)
try:
sccs = self._backend.get_sections(s_scc)
except Exception:
sccs = []
# no sccs -> consider all systems
systems = [self._backend.get_value(r_scc_to_system, scc) for scc in sccs]
try:
systems = [self._backend.get_value(r_scc_to_system, scc) for scc in sccs]
except Exception:
systems = []
# only take the first, and last two systems
if len(systems) == 0:
systems = self._backend.get_sections(s_system)
if len(systems) > 2:
systems = [systems[0], systems[-2], systems[-1]]
try:
systems = self._backend.get_sections(s_system)
except Exception:
systems = []
if len(systems) > 2:
systems = [systems[0], systems[-2], systems[-1]]
if len(systems) == 0:
self.logger.error('no section system found')
for g_index in systems:
try:
......
......@@ -20,7 +20,7 @@ import json
from matid import SymmetryAnalyzer, Classifier
from nomad import utils
from nomad import utils, config
from nomad.normalizing.normalizer import SystemBasedNormalizer
......@@ -31,7 +31,7 @@ class SystemNormalizer(SystemBasedNormalizer):
of the legacy NOMAD-coe *stats* normalizer.
"""
def __init__(self, backend):
super().__init__(backend, all_sections=True)
super().__init__(backend, all_sections=config.normalize.all_systems)
@staticmethod
def atom_label_to_num(atom_label):
......@@ -175,7 +175,7 @@ class SystemNormalizer(SystemBasedNormalizer):
self.symmetry_analysis(atoms)
def symmetry_analysis(self, atoms) -> None:
"""Analyze the symmetry of the material bein simulated.
"""Analyze the symmetry of the material being simulated.
We feed in the parsed values in section_system to the
the symmetry analyzer. We then use the Matid library
......
......@@ -186,7 +186,7 @@ mirgation_test_specs = [
('new_calc', 'new_calc', dict(migrated=2, source=2, new=1), {}),
('missing_calc', 'missing_calc', dict(migrated=1, source=2, missing=1), {}),
('missmatch', 'missmatch', dict(migrated=2, source=2, diffs=1), {}),
('failed_calc', 'failed_calc', dict(migrated=1, source=2, diffs=0, missing=1, failed=1, errors=1), {}),
('failed_calc', 'failed_calc', dict(migrated=1, source=2, diffs=0, missing=1, failed=1), {}),
('failed_upload', 'baseline', dict(migrated=0, source=2, missing=2, errors=1), {}),
('failed_publish', 'baseline', dict(migrated=0, source=2, missing=2, failed=2, errors=1), {})
]
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment