Commit 90611fbe authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Fixes to error processing error handling and not processed values.

parent 75486af0
Subproject commit 394b45037a8a4a120a9c492785aa8f1ff96d0d4d Subproject commit 10973cb74a154c5896633cb50f6e576fddd72852
...@@ -102,6 +102,7 @@ def _parse( ...@@ -102,6 +102,7 @@ def _parse(
if not skip_normalizers: if not skip_normalizers:
normalize_all(entry_archive) normalize_all(entry_archive)
entry_archive.section_metadata.apply_domain_metadata(entry_archive)
if show_archive: if show_archive:
json.dump(entry_archive.m_to_dict(), sys.stdout, indent=2) json.dump(entry_archive.m_to_dict(), sys.stdout, indent=2)
......
...@@ -288,8 +288,9 @@ class DFTMetadata(MSection): ...@@ -288,8 +288,9 @@ class DFTMetadata(MSection):
logger = utils.get_logger(__name__).bind( logger = utils.get_logger(__name__).bind(
upload_id=entry.upload_id, calc_id=entry.calc_id, mainfile=entry.mainfile) upload_id=entry.upload_id, calc_id=entry.calc_id, mainfile=entry.mainfile)
self.code_name = self.code_name_from_parser()
if entry_archive is None: if entry_archive is None:
self.code_name = self.code_name_from_parser()
return return
section_run = entry_archive.section_run section_run = entry_archive.section_run
...@@ -298,7 +299,14 @@ class DFTMetadata(MSection): ...@@ -298,7 +299,14 @@ class DFTMetadata(MSection):
return return
section_run = section_run[0] section_run = section_run[0]
section_system = [] # default values
self.system = config.services.unavailable_value
self.crystal_system = config.services.unavailable_value
self.spacegroup_symbol = config.services.unavailable_value
self.basis_set = config.services.unavailable_value
self.xc_functional = config.services.unavailable_value
section_system = None
for section in section_run.section_system: for section in section_run.section_system:
if section.is_representative: if section.is_representative:
section_system = section section_system = section
...@@ -313,7 +321,6 @@ class DFTMetadata(MSection): ...@@ -313,7 +321,6 @@ class DFTMetadata(MSection):
raise KeyError raise KeyError
except KeyError as e: except KeyError as e:
logger.warn('archive without program_name', exc_info=e) logger.warn('archive without program_name', exc_info=e)
self.code_name = self.code_name_from_parser()
try: try:
version = section_run.program_version version = section_run.program_version
...@@ -340,9 +347,13 @@ class DFTMetadata(MSection): ...@@ -340,9 +347,13 @@ class DFTMetadata(MSection):
entry.atoms = atoms entry.atoms = atoms
self.compound_type = compound_types[len(atoms) - 1] if len(atoms) <= 10 else '>decinary' self.compound_type = compound_types[len(atoms) - 1] if len(atoms) <= 10 else '>decinary'
section_symmetry = section_system.section_symmetry if section_system else [] self.system = config.services.unavailable_value
if section_symmetry: self.crystal_system = config.services.unavailable_value
section_symmetry = section_symmetry[0] self.spacegroup_symbol = config.services.unavailable_value
section_symmetry = None
if section_system and len(section_system.section_symmetry) > 0:
section_symmetry = section_system.section_symmetry[0]
self.crystal_system = get_value(section_symmetry.crystal_system) self.crystal_system = get_value(section_symmetry.crystal_system)
spacegroup = section_symmetry.space_group_number spacegroup = section_symmetry.space_group_number
self.spacegroup = 0 if not spacegroup else int(spacegroup) self.spacegroup = 0 if not spacegroup else int(spacegroup)
......
...@@ -130,10 +130,6 @@ class SystemBasedNormalizer(Normalizer, metaclass=ABCMeta): ...@@ -130,10 +130,6 @@ class SystemBasedNormalizer(Normalizer, metaclass=ABCMeta):
if system is None: if system is None:
self.logger.error('no "representative" section system found') self.logger.error('no "representative" section system found')
else:
self.logger.info(
'chose "representative" system for normalization',
)
# If the found system is referencing a subsystem, then we choose it as # If the found system is referencing a subsystem, then we choose it as
# the representative one. These smaller subsystems are much easier to # the representative one. These smaller subsystems are much easier to
......
...@@ -67,6 +67,7 @@ class TemplateParser(ArtificalParser): ...@@ -67,6 +67,7 @@ class TemplateParser(ArtificalParser):
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
from nomad.datamodel.metainfo import m_env as metainfo_env from nomad.datamodel.metainfo import m_env as metainfo_env
self._metainfo_env = metainfo_env self._metainfo_env = metainfo_env
self.code_name = 'Template'
def is_mainfile( def is_mainfile(
self, filename: str, mime: str, buffer: bytes, decoded_buffer: str, self, filename: str, mime: str, buffer: bytes, decoded_buffer: str,
......
...@@ -345,6 +345,7 @@ class Calc(Proc): ...@@ -345,6 +345,7 @@ class Calc(Proc):
self._entry_metadata.calc_hash = self.upload_files.calc_hash(self.mainfile) self._entry_metadata.calc_hash = self.upload_files.calc_hash(self.mainfile)
self._entry_metadata.last_processing = datetime.utcnow() self._entry_metadata.last_processing = datetime.utcnow()
self._entry_metadata.files = self.upload_files.calc_files(self.mainfile) self._entry_metadata.files = self.upload_files.calc_files(self.mainfile)
self._entry_metadata.parser_name = self.parser
@process @process
def process_calc(self): def process_calc(self):
...@@ -399,17 +400,13 @@ class Calc(Proc): ...@@ -399,17 +400,13 @@ class Calc(Proc):
self.get_logger().error( self.get_logger().error(
'could not apply domain metadata to entry', exc_info=e) 'could not apply domain metadata to entry', exc_info=e)
if self._parser_results and self._parser_results.m_resource:
self._parser_results.section_metadata = None
self._parser_results.m_resource.unload()
self._entry_metadata.a_elastic.index() self._entry_metadata.a_elastic.index()
except Exception as e: except Exception as e:
self.get_logger().error( self.get_logger().error(
'could not index after processing failure', exc_info=e) 'could not index after processing failure', exc_info=e)
try: try:
self.write_archive(None) self.write_archive(self._parser_results)
except Exception as e: except Exception as e:
self.get_logger().error( self.get_logger().error(
'could not write archive after processing failure', exc_info=e) 'could not write archive after processing failure', exc_info=e)
...@@ -608,7 +605,8 @@ class Calc(Proc): ...@@ -608,7 +605,8 @@ class Calc(Proc):
def write_archive(self, archive: EntryArchive): def write_archive(self, archive: EntryArchive):
# save the archive mongo entry # save the archive mongo entry
try: try:
write_partial_archive_to_mongo(archive) if self._entry_metadata.processed:
write_partial_archive_to_mongo(archive)
except Exception as e: except Exception as e:
self.get_logger().error('could not write mongodb archive entry', exc_info=e) self.get_logger().error('could not write mongodb archive entry', exc_info=e)
......
...@@ -418,6 +418,8 @@ def test_task_failure(monkeypatch, uploaded, task, proc_infra, test_user, with_e ...@@ -418,6 +418,8 @@ def test_task_failure(monkeypatch, uploaded, task, proc_infra, test_user, with_e
with upload.upload_files.read_archive(calc.calc_id) as archive: with upload.upload_files.read_archive(calc.calc_id) as archive:
calc_archive = archive[calc.calc_id] calc_archive = archive[calc.calc_id]
assert 'section_metadata' in calc_archive assert 'section_metadata' in calc_archive
assert calc_archive['section_metadata']['dft']['code_name'] not in [
config.services.unavailable_value, config.services.not_processed_value]
assert 'processing_logs' in calc_archive assert 'processing_logs' in calc_archive
if task != 'parsing': if task != 'parsing':
assert 'section_run' in calc_archive assert 'section_run' in calc_archive
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment