Commit cf7e1110 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Fixes and improvements to (processing) logging.

parent 6cfc608f
Pipeline #68751 failed with stages
in 4 minutes and 13 seconds
...@@ -21,13 +21,14 @@ class LogEntryUnstyled extends React.Component { ...@@ -21,13 +21,14 @@ class LogEntryUnstyled extends React.Component {
color: amber[700] color: amber[700]
}, },
exception: { exception: {
overflowX: 'scroll',
margin: 0 margin: 0
} }
}) })
render() { render() {
const { classes, entry } = this.props const { classes, entry } = this.props
let data = undefined let data
try { try {
data = JSON.parse(entry) data = JSON.parse(entry)
} catch (e) { } catch (e) {
...@@ -86,9 +87,9 @@ class ArchiveLogView extends React.Component { ...@@ -86,9 +87,9 @@ class ArchiveLogView extends React.Component {
}); });
static defaultState = { static defaultState = {
data: null, data: null,
doesNotExist: false doesNotExist: false
} }
state = {...ArchiveLogView.defaultState} state = {...ArchiveLogView.defaultState}
......
...@@ -33,7 +33,8 @@ import os.path ...@@ -33,7 +33,8 @@ import os.path
from datetime import datetime from datetime import datetime
from pymongo import UpdateOne from pymongo import UpdateOne
import hashlib import hashlib
from structlog.processors import StackInfoRenderer, format_exc_info, TimeStamper, JSONRenderer from structlog.processors import StackInfoRenderer, format_exc_info, TimeStamper
import json
from nomad import utils, config, infrastructure, search, datamodel from nomad import utils, config, infrastructure, search, datamodel
from nomad.files import PathObject, UploadFiles, ExtractError, ArchiveBasedStagingUploadFiles, PublicUploadFiles, StagingUploadFiles from nomad.files import PathObject, UploadFiles, ExtractError, ArchiveBasedStagingUploadFiles, PublicUploadFiles, StagingUploadFiles
...@@ -62,8 +63,7 @@ _log_processors = [ ...@@ -62,8 +63,7 @@ _log_processors = [
StackInfoRenderer(), StackInfoRenderer(),
_pack_log_event, _pack_log_event,
format_exc_info, format_exc_info,
TimeStamper(fmt="%Y-%m-%d %H:%M.%S", utc=False), TimeStamper(fmt="%Y-%m-%d %H:%M.%S", utc=False)]
JSONRenderer(sort_keys=True)]
class Calc(Proc): class Calc(Proc):
...@@ -168,7 +168,9 @@ class Calc(Proc): ...@@ -168,7 +168,9 @@ class Calc(Proc):
def save_to_calc_log(logger, method_name, event_dict): def save_to_calc_log(logger, method_name, event_dict):
if self._calc_proc_logwriter is not None: if self._calc_proc_logwriter is not None:
try: try:
self._calc_proc_logwriter.write(event_dict) dump_dict = dict(event_dict)
dump_dict.update(level=method_name.upper())
json.dump(dump_dict, self._calc_proc_logwriter, sort_keys=True)
self._calc_proc_logwriter.write('\n') self._calc_proc_logwriter.write('\n')
except Exception: except Exception:
......
...@@ -203,6 +203,10 @@ class LogstashFormatter(logstash.formatter.LogstashFormatterBase): ...@@ -203,6 +203,10 @@ class LogstashFormatter(logstash.formatter.LogstashFormatterBase):
# Extra Fields # Extra Fields
'level': record.levelname, 'level': record.levelname,
'logger_name': record.name, 'logger_name': record.name,
# Nomad specific
'nomad.service': config.service,
'nomad.release': config.release
} }
if record.name.startswith('nomad'): if record.name.startswith('nomad'):
...@@ -211,17 +215,13 @@ class LogstashFormatter(logstash.formatter.LogstashFormatterBase): ...@@ -211,17 +215,13 @@ class LogstashFormatter(logstash.formatter.LogstashFormatterBase):
continue continue
elif key == 'exception': elif key == 'exception':
message['digest'] = str(value)[-256:] message['digest'] = str(value)[-256:]
elif key in ( elif key in ['upload_id', 'calc_id', 'mainfile']:
'upload_id', 'calc_id', 'mainfile',
'service', 'release'):
key = 'nomad.%s' % key key = 'nomad.%s' % key
else: else:
key = '%s.%s' % (record.name, key) key = '%s.%s' % (record.name, key)
message[key] = value message[key] = value
else: else:
structlog['nomad.service'] = config.service
structlog['nomad.release'] = config.release
message.update(structlog) message.update(structlog)
# Handle gunicorn access events # Handle gunicorn access events
...@@ -352,7 +352,7 @@ def get_logger(name, **kwargs): ...@@ -352,7 +352,7 @@ def get_logger(name, **kwargs):
if name.startswith('nomad.'): if name.startswith('nomad.'):
name = '.'.join(name.split('.')[:2]) name = '.'.join(name.split('.')[:2])
logger = structlog.get_logger(name, service=config.service, release=config.release, **kwargs) logger = structlog.get_logger(name, **kwargs)
return logger return logger
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment