Improved error logging for parsing/normalizing.

nomad/coe_repo/calc.py

@@ -29,6 +29,9 @@ from .base import Base, calc_citation_association, ownership, co_authorship, sha
     CodeVersion, StructRatio, UserMetaData
 
 
+class IllegalCalcMetadata(Exception): pass
+
+
 class Calc(Base):
     __tablename__ = 'calculations'
 
@@ -194,6 +197,9 @@ class Calc(Base):
         def add_users_to_relation(source_users, relation):
             for source_user in source_users:
                 coe_user = repo_db.query(User).get(source_user.id)
+                if coe_user is None:
+                    raise IllegalCalcMetadata(
+                        'User with user_id %s does not exist.' % source_user.id)
                 source_user.update(coe_user.to_popo())
                 relation.append(coe_user)
 

nomad/processing/data.py

@@ -157,8 +157,13 @@ class Calc(Proc):
         parser = parser_dict[self.parser]
 
         with utils.timer(logger, 'parser executed', input_size=self.mainfile_file.size):
-            self._parser_backend = parser.run(
-                self.upload_files.raw_file_object(self.mainfile).os_path, logger=logger)
+            try:
+                self._parser_backend = parser.run(
+                    self.upload_files.raw_file_object(self.mainfile).os_path, logger=logger)
+            except Exception as e:
+                self.fail(
+                    'parser failed with exception', level=logging.ERROR,
+                    exc_info=e, error=str(e), **context)
 
         self._parser_backend.openNonOverlappingSection('section_calculation_info')
         self._parser_backend.addValue('upload_id', self.upload_id)
@@ -171,7 +176,7 @@ class Calc(Proc):
             logger.error(self._parser_backend.status[1])
             error = self._parser_backend.status[1]
             self._parser_backend.addValue('parse_status', 'ParseFailure')
-            self.fail(error, level=logging.DEBUG, **context)
+            self.fail(error, level=logging.INFO, **context)
         else:
             self._parser_backend.addValue('parse_status', 'ParseSuccess')
 
@@ -221,13 +226,19 @@ class Calc(Proc):
             with utils.timer(
                     logger, 'normalizer executed', input_size=self.mainfile_file.size):
                 with self.use_parser_backend(normalizer_name) as backend:
-                    normalizer(backend).normalize(logger=logger)
+                    try:
+                        normalizer(backend).normalize(logger=logger)
+                    except Exception as e:
+                        self.fail(
+                            'normalizer failed with exception', level=logging.ERROR,
+                            exc_info=e, error=str(e), **context)
+                        self._parser_backend.status = ['ParseFailure', str(e)]
 
             failed = self._parser_backend.status[0] != 'ParseSuccess'
             if failed:
                 logger.error(self._parser_backend.status[1])
                 error = self._parser_backend.status[1]
-                self.fail(error, level=logging.WARNING, **context)
+                self.fail(error, level=logging.WARNING, error=error, **context)
                 break
             else:
                 logger.debug(