From 2f8d4e90f6fc2579e2a419c01f29c315f45d972a Mon Sep 17 00:00:00 2001
From: Markus Scheidgen <markus.scheidgen@gmail.com>
Date: Sat, 2 Feb 2019 15:54:07 +0100
Subject: [PATCH] Fixed issues about failed uploads/calcs. Added chasos parser.
---
gui/src/components/ArchiveLogView.js | 4 +--
gui/src/components/Upload.js | 37 +++++++++++++++++-----
gui/src/components/Uploads.js | 7 +++--
nomad/config.py | 5 ++-
nomad/parsing/__init__.py | 3 +-
nomad/parsing/artificial.py | 44 ++++++++++++++++++++++++++
nomad/processing/data.py | 46 +++++++++++++++-------------
tests/conftest.py | 12 +++++++-
8 files changed, 120 insertions(+), 38 deletions(-)
diff --git a/gui/src/components/ArchiveLogView.js b/gui/src/components/ArchiveLogView.js
index 9132437ee0..11404c2857 100644
--- a/gui/src/components/ArchiveLogView.js
+++ b/gui/src/components/ArchiveLogView.js
@@ -61,8 +61,8 @@ class ArchiveLogView extends React.Component {
<DownloadIcon />
</Download>
{
- data
- ? <pre>{data}</pre>
+ data !== null
+ ? <pre>{data || 'empty log'}</pre>
: <LinearProgress variant="query" />
}
</div>
diff --git a/gui/src/components/Upload.js b/gui/src/components/Upload.js
index fedca63335..cc514c6cd8 100644
--- a/gui/src/components/Upload.js
+++ b/gui/src/components/Upload.js
@@ -12,6 +12,7 @@ import { compose } from 'recompose'
import { withErrors } from './errors'
import { debug } from '../config'
import CalcDialog from './CalcDialog'
+import ArchiveLogDialog from './ArchiveLogDialog'
class Upload extends React.Component {
static propTypes = {
@@ -187,7 +188,7 @@ class Upload extends React.Component {
renderStepper() {
const { classes } = this.props
const { upload } = this.state
- const { calcs, tasks, current_task, tasks_running, tasks_status, process_running, current_process, errors } = upload
+ const { calcs, tasks, current_task, tasks_running, tasks_status, process_running, current_process } = upload
// map tasks [ uploading, extracting, parse_all, cleanup ] to steps
const steps = [ 'upload', 'process', 'publish' ]
@@ -273,7 +274,7 @@ class Upload extends React.Component {
if (tasks_status === 'FAILURE') {
props.optional = (
<Typography variant="caption" color="error">
- {errors.join(' ')}
+ processing failed
</Typography>
)
}
@@ -353,7 +354,7 @@ class Upload extends React.Component {
const processed = tasks_status === 'FAILURE' || tasks_status === 'SUCCESS'
const row = (
<TableRow key={index} hover={processed}
- onClick={() => this.setState({openCalc: processed ? {uploadId: upload_id, calcId: calc_id} : null})}
+ onClick={() => this.setState({openCalc: processed ? calc : null})}
className={processed ? classes.clickableRow : null} >
<TableCell>
@@ -387,8 +388,9 @@ class Upload extends React.Component {
)
if (tasks_status === 'FAILURE') {
+ const error_html = `Calculation processing failed with errors: ${errors.join(', ')}`
return (
- <Tooltip key={calc_id} title={errors.map((error, index) => (<p key={`${calc_id}-${index}`}>{error}</p>))}>
+ <Tooltip key={calc_id} title={error_html}>
{row}
</Tooltip>
)
@@ -455,14 +457,30 @@ class Upload extends React.Component {
)
}
+ renderOpenCalc() {
+ const { openCalc } = this.state
+ if (openCalc) {
+ if (openCalc.errors && openCalc.errors.length > 0) {
+ return <ArchiveLogDialog calcId={openCalc.calc_id} uploadId={openCalc.upload_id}
+ onClose={() => this.setState({openCalc: null})} />
+ } else {
+ return <CalcDialog calcId={openCalc.calc_id} uploadId={openCalc.upload_id}
+ onClose={() => this.setState({openCalc: null})} />
+ }
+ }
+
+ return ''
+ }
+
render() {
- const { classes, raiseError } = this.props
- const { upload, openCalc } = this.state
+ const { classes } = this.props
+ const { upload } = this.state
+ const { errors } = upload
if (this.state.upload) {
return (
<div className={classes.root}>
- { openCalc ? <CalcDialog raiseError={raiseError} {...openCalc} onClose={() => this.setState({openCalc: null})} /> : ''}
+ { this.renderOpenCalc() }
<ExpansionPanel>
<ExpansionPanelSummary
@@ -483,6 +501,11 @@ class Upload extends React.Component {
{this.renderTitle()} {this.renderStepper()}
</ExpansionPanelSummary>
<ExpansionPanelDetails style={{width: '100%'}} classes={{root: classes.details}}>
+ {errors && errors.length > 0
+ ? <Typography className={classes.detailsContent} color="error">
+ Upload processing has errors: {errors.join(', ')}
+ </Typography> : ''
+ }
{upload.calcs ? this.renderCalcTable() : ''}
{debug
? <div className={classes.detailsContent}>
diff --git a/gui/src/components/Uploads.js b/gui/src/components/Uploads.js
index a3ebc24d67..ac1489ccb4 100644
--- a/gui/src/components/Uploads.js
+++ b/gui/src/components/Uploads.js
@@ -128,9 +128,12 @@ class Uploads extends React.Component {
})
}
- sortedUploads() {
+ sortedUploads(order) {
+ order = order || -1
return this.state.uploads.concat()
- .sort((a, b) => (a.gui_upload_id === b.gui_upload_id) ? 0 : ((a.gui_upload_id < b.gui_upload_id) ? -1 : 1))
+ .sort((a, b) => (a.gui_upload_id === b.gui_upload_id)
+ ? 0
+ : ((a.gui_upload_id < b.gui_upload_id) ? -1 : 1) * order)
}
handleDoesNotExist(nonExistingUupload) {
diff --git a/nomad/config.py b/nomad/config.py
index 1451776950..27df4eabd7 100644
--- a/nomad/config.py
+++ b/nomad/config.py
@@ -51,7 +51,7 @@ LogstashConfig = namedtuple('LogstashConfig', ['enabled', 'host', 'tcp_port', 'l
NomadServicesConfig = namedtuple('NomadServicesConfig', ['api_host', 'api_port', 'api_base_path', 'api_secret', 'admin_password', 'upload_url', 'disable_reset'])
""" Used to configure nomad services: worker, handler, api """
-MailConfig = namedtuple('MailConfig', ['enabled', 'host', 'port', 'user', 'password', 'from_address'])
+MailConfig = namedtuple('MailConfig', ['host', 'port', 'user', 'password', 'from_address'])
""" Used to configure how nomad can send email """
files = FilesConfig(
@@ -129,8 +129,7 @@ migration_source_db = RepositoryDBConfig(
password=os.environ.get('NOMAD_MIGRATION_SOURCE_PASSWORD', '*')
)
mail = MailConfig(
- enabled=True,
- host=os.environ.get('NOMAD_SMTP_HOST', 'localhost'),
+ host=os.environ.get('NOMAD_SMTP_HOST', ''), # empty or None host disables email
port=int(os.environ.get('NOMAD_SMTP_PORT', 8995)),
user=os.environ.get('NOMAD_SMTP_USER', None),
password=os.environ.get('NOMAD_SMTP_PASSWORD', None),
diff --git a/nomad/parsing/__init__.py b/nomad/parsing/__init__.py
index 6c1915c6f4..826dbda17b 100644
--- a/nomad/parsing/__init__.py
+++ b/nomad/parsing/__init__.py
@@ -58,12 +58,13 @@ based on NOMAD-coe's *python-common* module.
from nomad.parsing.backend import AbstractParserBackend, LocalBackend, LegacyLocalBackend, JSONStreamWriter, BadContextURI, WrongContextState
from nomad.parsing.parser import Parser, LegacyParser
-from nomad.parsing.artificial import TemplateParser, GenerateRandomParser
+from nomad.parsing.artificial import TemplateParser, GenerateRandomParser, ChaosParser
from nomad.dependencies import dependencies_dict as dependencies
parsers = [
GenerateRandomParser(),
TemplateParser(),
+ ChaosParser(),
LegacyParser(
python_git=dependencies['parsers/vasp'],
parser_class_name='vaspparser.VASPRunParserInterface',
diff --git a/nomad/parsing/artificial.py b/nomad/parsing/artificial.py
index a74ebe4e50..d874940a3e 100644
--- a/nomad/parsing/artificial.py
+++ b/nomad/parsing/artificial.py
@@ -23,6 +23,8 @@ import numpy as np
import random
from ase.data import chemical_symbols
import numpy
+import sys
+import time
from nomadcore.local_meta_info import loadJsonFile, InfoKindEl
import nomad_meta_info
@@ -111,6 +113,48 @@ class TemplateParser(ArtificalParser):
return self.backend
+class ChaosParser(ArtificalParser):
+ """
+ Parser that emulates typical error situations. Files can contain a json string (or
+ object with key `chaos`) with one of the following string values:
+ - exit
+ - deadlock
+ - consume_ram
+ - exception
+ - random
+ """
+ name = 'parsers/chaos'
+
+ def is_mainfile(self, filename: str, open: Callable[[str], IO[Any]]) -> bool:
+ return filename.endswith('chaos.json')
+
+ def run(self, mainfile: str, logger=None) -> LocalBackend:
+ self.init_backend()
+
+ chaos_json = json.load(open(mainfile, 'r'))
+ if isinstance(chaos_json, str):
+ chaos = chaos_json
+ elif isinstance(chaos_json, dict):
+ chaos = chaos_json.get('chaos', None)
+ else:
+ chaos = None
+
+ if chaos == 'random':
+ chaos = random.choice(['exit', 'deadlock', 'consume_ram', 'exception'])
+
+ if chaos == 'exit':
+ sys.exit(1)
+ elif chaos == 'deadlock':
+ while True:
+ time.sleep(1)
+ elif chaos == 'consume_ram':
+ pass
+ elif chaos == 'exception':
+ raise Exception('Some chaos happened, muhuha...')
+
+ raise Exception('Unknown chaos')
+
+
class GenerateRandomParser(TemplateParser):
name = 'parsers/random'
diff --git a/nomad/processing/data.py b/nomad/processing/data.py
index 5c1058cfbc..a71937a1db 100644
--- a/nomad/processing/data.py
+++ b/nomad/processing/data.py
@@ -93,33 +93,30 @@ class Calc(Proc, datamodel.Calc):
return self._upload_files
def get_logger(self, **kwargs):
- logger = super().get_logger()
- logger = logger.bind(
- upload_id=self.upload_id, mainfile=self.mainfile, calc_id=self.calc_id, **kwargs)
-
- return logger
-
- def get_calc_logger(self, **kwargs):
"""
Returns a wrapped logger that additionally saves all entries to the calculation
processing log in the archive.
"""
- logger = self.get_logger(**kwargs)
+ logger = super().get_logger()
+ logger = logger.bind(
+ upload_id=self.upload_id, mainfile=self.mainfile, calc_id=self.calc_id, **kwargs)
- if self._calc_proc_logwriter is None:
+ if self._calc_proc_logwriter_ctx is None:
self._calc_proc_logwriter_ctx = self.upload_files.archive_log_file(self.calc_id, 'wt')
self._calc_proc_logwriter = self._calc_proc_logwriter_ctx.__enter__() # pylint: disable=E1101
def save_to_calc_log(logger, method_name, event_dict):
- program = event_dict.get('normalizer', 'parser')
- event = event_dict.get('event', '')
- entry = '[%s] %s: %s' % (method_name, program, event)
- if len(entry) > 120:
- self._calc_proc_logwriter.write(entry[:120])
- self._calc_proc_logwriter.write('...')
- else:
- self._calc_proc_logwriter.write(entry)
- self._calc_proc_logwriter.write('\n')
+ if self._calc_proc_logwriter is not None:
+ program = event_dict.get('normalizer', 'parser')
+ event = event_dict.get('event', '')
+ entry = '[%s] %s: %s' % (method_name, program, event)
+ if len(entry) > 120:
+ self._calc_proc_logwriter.write(entry[:120])
+ self._calc_proc_logwriter.write('...')
+ else:
+ self._calc_proc_logwriter.write(entry)
+ self._calc_proc_logwriter.write('\n')
+
return event_dict
return wrap_logger(logger, processors=[save_to_calc_log])
@@ -149,7 +146,7 @@ class Calc(Proc, datamodel.Calc):
@task
def parsing(self):
context = dict(parser=self.parser, step=self.parser)
- logger = self.get_calc_logger(**context)
+ logger = self.get_logger(**context)
parser = parser_dict[self.parser]
with utils.timer(logger, 'parser executed', input_size=self.mainfile_file.size):
@@ -212,7 +209,7 @@ class Calc(Proc, datamodel.Calc):
for normalizer in normalizers:
normalizer_name = normalizer.__name__
context = dict(normalizer=normalizer_name, step=normalizer_name)
- logger = self.get_calc_logger(**context)
+ logger = self.get_logger(**context)
with utils.timer(
logger, 'normalizer executed', input_size=self.mainfile_file.size):
@@ -481,8 +478,13 @@ class Upload(Chord, datamodel.Upload):
self.name if self.name else '', self.upload_time.isoformat()),
'You can review your data on your upload page: %s' % config.services.upload_url
])
- infrastructure.send_mail(
- name=name, email=user.email, message=message, subject='Processing completed')
+ try:
+ infrastructure.send_mail(
+ name=name, email=user.email, message=message, subject='Processing completed')
+ except Exception as e:
+ # probably due to email configuration problems
+ # don't fail or present this error to clients
+ self.logger.error('could not send after processing email', exc_info=e)
@property
def processed_calcs(self):
diff --git a/tests/conftest.py b/tests/conftest.py
index 7fdc116144..df468fa6ac 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -318,6 +318,16 @@ def smtpd(request):
@pytest.fixture(scope='function', autouse=True)
-def mails(smtpd):
+def mails(smtpd, monkeypatch):
smtpd.clear()
+
+ old_config = config.mail
+ new_config = config.MailConfig(
+ 'localhost',
+ old_config.port,
+ old_config.user,
+ old_config.password,
+ old_config.from_address)
+
+ monkeypatch.setattr('nomad.config.mail', new_config)
yield smtpd
--
GitLab