Fixed issues about failed uploads/calcs. Added chasos parser.

gui/src/components/ArchiveLogView.js

@@ -61,8 +61,8 @@ class ArchiveLogView extends React.Component {
           <DownloadIcon />
         </Download>
         {
-          data
-            ? <pre>{data}</pre>
+          data !== null
+            ? <pre>{data || 'empty log'}</pre>
             : <LinearProgress variant="query" />
         }
       </div>

gui/src/components/Upload.js

@@ -12,6 +12,7 @@ import { compose } from 'recompose'
 import { withErrors } from './errors'
 import { debug } from '../config'
 import CalcDialog from './CalcDialog'
+import ArchiveLogDialog from './ArchiveLogDialog'
 
 class Upload extends React.Component {
   static propTypes = {
@@ -187,7 +188,7 @@ class Upload extends React.Component {
   renderStepper() {
     const { classes } = this.props
     const { upload } = this.state
-    const { calcs, tasks, current_task, tasks_running, tasks_status, process_running, current_process, errors } = upload
+    const { calcs, tasks, current_task, tasks_running, tasks_status, process_running, current_process } = upload
 
     // map tasks [ uploading, extracting, parse_all, cleanup ] to steps
     const steps = [ 'upload', 'process', 'publish' ]
@@ -273,7 +274,7 @@ class Upload extends React.Component {
         if (tasks_status === 'FAILURE') {
           props.optional = (
             <Typography variant="caption" color="error">
-              {errors.join(' ')}
+              processing failed
             </Typography>
           )
         }
@@ -353,7 +354,7 @@ class Upload extends React.Component {
       const processed = tasks_status === 'FAILURE' || tasks_status === 'SUCCESS'
       const row = (
         <TableRow key={index} hover={processed}
-          onClick={() => this.setState({openCalc: processed ? {uploadId: upload_id, calcId: calc_id} : null})}
+          onClick={() => this.setState({openCalc: processed ? calc : null})}
           className={processed ? classes.clickableRow : null} >
 
           <TableCell>
@@ -387,8 +388,9 @@ class Upload extends React.Component {
       )
 
       if (tasks_status === 'FAILURE') {
+        const error_html = `Calculation processing failed with errors: ${errors.join(', ')}`
         return (
-          <Tooltip key={calc_id} title={errors.map((error, index) => (<p key={`${calc_id}-${index}`}>{error}</p>))}>
+          <Tooltip key={calc_id} title={error_html}>
             {row}
           </Tooltip>
         )
@@ -455,14 +457,30 @@ class Upload extends React.Component {
     )
   }
 
+  renderOpenCalc() {
+    const { openCalc } = this.state
+    if (openCalc) {
+      if (openCalc.errors && openCalc.errors.length > 0) {
+        return <ArchiveLogDialog calcId={openCalc.calc_id} uploadId={openCalc.upload_id}
+          onClose={() => this.setState({openCalc: null})} />
+      } else {
+        return <CalcDialog calcId={openCalc.calc_id} uploadId={openCalc.upload_id}
+          onClose={() => this.setState({openCalc: null})} />
+      }
+    }
+
+    return ''
+  }
+
   render() {
-    const { classes, raiseError } = this.props
-    const { upload, openCalc } = this.state
+    const { classes } = this.props
+    const { upload } = this.state
+    const { errors } = upload
 
     if (this.state.upload) {
       return (
         <div className={classes.root}>
-          { openCalc ? <CalcDialog raiseError={raiseError} {...openCalc} onClose={() => this.setState({openCalc: null})} /> : ''}
+          { this.renderOpenCalc() }
 
           <ExpansionPanel>
             <ExpansionPanelSummary
@@ -483,6 +501,11 @@ class Upload extends React.Component {
               {this.renderTitle()} {this.renderStepper()}
             </ExpansionPanelSummary>
             <ExpansionPanelDetails style={{width: '100%'}} classes={{root: classes.details}}>
+              {errors && errors.length > 0
+                ? <Typography className={classes.detailsContent} color="error">
+                  Upload processing has errors: {errors.join(', ')}
+                </Typography> : ''
+              }
               {upload.calcs ? this.renderCalcTable() : ''}
               {debug
                 ? <div className={classes.detailsContent}>

gui/src/components/Uploads.js

@@ -128,9 +128,12 @@ class Uploads extends React.Component {
       })
   }
 
-  sortedUploads() {
+  sortedUploads(order) {
+    order = order || -1
     return this.state.uploads.concat()
-      .sort((a, b) => (a.gui_upload_id === b.gui_upload_id) ? 0 : ((a.gui_upload_id < b.gui_upload_id) ? -1 : 1))
+      .sort((a, b) => (a.gui_upload_id === b.gui_upload_id)
+        ? 0
+        : ((a.gui_upload_id < b.gui_upload_id) ? -1 : 1) * order)
   }
 
   handleDoesNotExist(nonExistingUupload) {

nomad/config.py

@@ -51,7 +51,7 @@ LogstashConfig = namedtuple('LogstashConfig', ['enabled', 'host', 'tcp_port', 'l
 NomadServicesConfig = namedtuple('NomadServicesConfig', ['api_host', 'api_port', 'api_base_path', 'api_secret', 'admin_password', 'upload_url', 'disable_reset'])
 """ Used to configure nomad services: worker, handler, api """
 
-MailConfig = namedtuple('MailConfig', ['enabled', 'host', 'port', 'user', 'password', 'from_address'])
+MailConfig = namedtuple('MailConfig', ['host', 'port', 'user', 'password', 'from_address'])
 """ Used to configure how nomad can send email """
 
 files = FilesConfig(
@@ -129,8 +129,7 @@ migration_source_db = RepositoryDBConfig(
     password=os.environ.get('NOMAD_MIGRATION_SOURCE_PASSWORD', '*')
 )
 mail = MailConfig(
-    enabled=True,
-    host=os.environ.get('NOMAD_SMTP_HOST', 'localhost'),
+    host=os.environ.get('NOMAD_SMTP_HOST', ''),  # empty or None host disables email
     port=int(os.environ.get('NOMAD_SMTP_PORT', 8995)),
     user=os.environ.get('NOMAD_SMTP_USER', None),
     password=os.environ.get('NOMAD_SMTP_PASSWORD', None),

nomad/parsing/__init__.py

@@ -58,12 +58,13 @@ based on NOMAD-coe's *python-common* module.
 
 from nomad.parsing.backend import AbstractParserBackend, LocalBackend, LegacyLocalBackend, JSONStreamWriter, BadContextURI, WrongContextState
 from nomad.parsing.parser import Parser, LegacyParser
-from nomad.parsing.artificial import TemplateParser, GenerateRandomParser
+from nomad.parsing.artificial import TemplateParser, GenerateRandomParser, ChaosParser
 from nomad.dependencies import dependencies_dict as dependencies
 
 parsers = [
     GenerateRandomParser(),
     TemplateParser(),
+    ChaosParser(),
     LegacyParser(
         python_git=dependencies['parsers/vasp'],
         parser_class_name='vaspparser.VASPRunParserInterface',

nomad/parsing/artificial.py

@@ -23,6 +23,8 @@ import numpy as np
 import random
 from ase.data import chemical_symbols
 import numpy
+import sys
+import time
 
 from nomadcore.local_meta_info import loadJsonFile, InfoKindEl
 import nomad_meta_info
@@ -111,6 +113,48 @@ class TemplateParser(ArtificalParser):
         return self.backend
 
 
+class ChaosParser(ArtificalParser):
+    """
+    Parser that emulates typical error situations. Files can contain a json string (or
+    object with key `chaos`) with one of the following string values:
+    - exit
+    - deadlock
+    - consume_ram
+    - exception
+    - random
+    """
+    name = 'parsers/chaos'
+
+    def is_mainfile(self, filename: str, open: Callable[[str], IO[Any]]) -> bool:
+        return filename.endswith('chaos.json')
+
+    def run(self, mainfile: str, logger=None) -> LocalBackend:
+        self.init_backend()
+
+        chaos_json = json.load(open(mainfile, 'r'))
+        if isinstance(chaos_json, str):
+            chaos = chaos_json
+        elif isinstance(chaos_json, dict):
+            chaos = chaos_json.get('chaos', None)
+        else:
+            chaos = None
+
+        if chaos == 'random':
+            chaos = random.choice(['exit', 'deadlock', 'consume_ram', 'exception'])
+
+        if chaos == 'exit':
+            sys.exit(1)
+        elif chaos == 'deadlock':
+            while True:
+                time.sleep(1)
+        elif chaos == 'consume_ram':
+            pass
+        elif chaos == 'exception':
+            raise Exception('Some chaos happened, muhuha...')
+
+        raise Exception('Unknown chaos')
+
+
 class GenerateRandomParser(TemplateParser):
     name = 'parsers/random'
 

nomad/processing/data.py

@@ -93,33 +93,30 @@ class Calc(Proc, datamodel.Calc):
         return self._upload_files
 
     def get_logger(self, **kwargs):
-        logger = super().get_logger()
-        logger = logger.bind(
-            upload_id=self.upload_id, mainfile=self.mainfile, calc_id=self.calc_id, **kwargs)
-
-        return logger
-
-    def get_calc_logger(self, **kwargs):
         """
         Returns a wrapped logger that additionally saves all entries to the calculation
         processing log in the archive.
         """
-        logger = self.get_logger(**kwargs)
+        logger = super().get_logger()
+        logger = logger.bind(
+            upload_id=self.upload_id, mainfile=self.mainfile, calc_id=self.calc_id, **kwargs)
 
-        if self._calc_proc_logwriter is None:
+        if self._calc_proc_logwriter_ctx is None:
             self._calc_proc_logwriter_ctx = self.upload_files.archive_log_file(self.calc_id, 'wt')
             self._calc_proc_logwriter = self._calc_proc_logwriter_ctx.__enter__()  # pylint: disable=E1101
 
         def save_to_calc_log(logger, method_name, event_dict):
-            program = event_dict.get('normalizer', 'parser')
-            event = event_dict.get('event', '')
-            entry = '[%s] %s: %s' % (method_name, program, event)
-            if len(entry) > 120:
-                self._calc_proc_logwriter.write(entry[:120])
-                self._calc_proc_logwriter.write('...')
-            else:
-                self._calc_proc_logwriter.write(entry)
-            self._calc_proc_logwriter.write('\n')
+            if self._calc_proc_logwriter is not None:
+                program = event_dict.get('normalizer', 'parser')
+                event = event_dict.get('event', '')
+                entry = '[%s] %s: %s' % (method_name, program, event)
+                if len(entry) > 120:
+                    self._calc_proc_logwriter.write(entry[:120])
+                    self._calc_proc_logwriter.write('...')
+                else:
+                    self._calc_proc_logwriter.write(entry)
+                self._calc_proc_logwriter.write('\n')
+
             return event_dict
 
         return wrap_logger(logger, processors=[save_to_calc_log])
@@ -149,7 +146,7 @@ class Calc(Proc, datamodel.Calc):
     @task
     def parsing(self):
         context = dict(parser=self.parser, step=self.parser)
-        logger = self.get_calc_logger(**context)
+        logger = self.get_logger(**context)
         parser = parser_dict[self.parser]
 
         with utils.timer(logger, 'parser executed', input_size=self.mainfile_file.size):
@@ -212,7 +209,7 @@ class Calc(Proc, datamodel.Calc):
         for normalizer in normalizers:
             normalizer_name = normalizer.__name__
             context = dict(normalizer=normalizer_name, step=normalizer_name)
-            logger = self.get_calc_logger(**context)
+            logger = self.get_logger(**context)
 
             with utils.timer(
                     logger, 'normalizer executed', input_size=self.mainfile_file.size):
@@ -481,8 +478,13 @@ class Upload(Chord, datamodel.Upload):
                 self.name if self.name else '', self.upload_time.isoformat()),
             'You can review your data on your upload page: %s' % config.services.upload_url
         ])
-        infrastructure.send_mail(
-            name=name, email=user.email, message=message, subject='Processing completed')
+        try:
+            infrastructure.send_mail(
+                name=name, email=user.email, message=message, subject='Processing completed')
+        except Exception as e:
+            # probably due to email configuration problems
+            # don't fail or present this error to clients
+            self.logger.error('could not send after processing email', exc_info=e)
 
     @property
     def processed_calcs(self):

tests/conftest.py

@@ -318,6 +318,16 @@ def smtpd(request):
 
 
 @pytest.fixture(scope='function', autouse=True)
-def mails(smtpd):
+def mails(smtpd, monkeypatch):
     smtpd.clear()
+
+    old_config = config.mail
+    new_config = config.MailConfig(
+        'localhost',
+        old_config.port,
+        old_config.user,
+        old_config.password,
+        old_config.from_address)
+
+    monkeypatch.setattr('nomad.config.mail', new_config)
     yield smtpd