Commit cf7e1110 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Fixes and improvements to (processing) logging.

parent 6cfc608f
Pipeline #68751 failed with stages
in 4 minutes and 13 seconds
......@@ -21,13 +21,14 @@ class LogEntryUnstyled extends React.Component {
color: amber[700]
},
exception: {
overflowX: 'scroll',
margin: 0
}
})
render() {
const { classes, entry } = this.props
let data = undefined
let data
try {
data = JSON.parse(entry)
} catch (e) {
......@@ -86,9 +87,9 @@ class ArchiveLogView extends React.Component {
});
static defaultState = {
data: null,
doesNotExist: false
}
data: null,
doesNotExist: false
}
state = {...ArchiveLogView.defaultState}
......
......@@ -33,7 +33,8 @@ import os.path
from datetime import datetime
from pymongo import UpdateOne
import hashlib
from structlog.processors import StackInfoRenderer, format_exc_info, TimeStamper, JSONRenderer
from structlog.processors import StackInfoRenderer, format_exc_info, TimeStamper
import json
from nomad import utils, config, infrastructure, search, datamodel
from nomad.files import PathObject, UploadFiles, ExtractError, ArchiveBasedStagingUploadFiles, PublicUploadFiles, StagingUploadFiles
......@@ -62,8 +63,7 @@ _log_processors = [
StackInfoRenderer(),
_pack_log_event,
format_exc_info,
TimeStamper(fmt="%Y-%m-%d %H:%M.%S", utc=False),
JSONRenderer(sort_keys=True)]
TimeStamper(fmt="%Y-%m-%d %H:%M.%S", utc=False)]
class Calc(Proc):
......@@ -168,7 +168,9 @@ class Calc(Proc):
def save_to_calc_log(logger, method_name, event_dict):
if self._calc_proc_logwriter is not None:
try:
self._calc_proc_logwriter.write(event_dict)
dump_dict = dict(event_dict)
dump_dict.update(level=method_name.upper())
json.dump(dump_dict, self._calc_proc_logwriter, sort_keys=True)
self._calc_proc_logwriter.write('\n')
except Exception:
......
......@@ -203,6 +203,10 @@ class LogstashFormatter(logstash.formatter.LogstashFormatterBase):
# Extra Fields
'level': record.levelname,
'logger_name': record.name,
# Nomad specific
'nomad.service': config.service,
'nomad.release': config.release
}
if record.name.startswith('nomad'):
......@@ -211,17 +215,13 @@ class LogstashFormatter(logstash.formatter.LogstashFormatterBase):
continue
elif key == 'exception':
message['digest'] = str(value)[-256:]
elif key in (
'upload_id', 'calc_id', 'mainfile',
'service', 'release'):
elif key in ['upload_id', 'calc_id', 'mainfile']:
key = 'nomad.%s' % key
else:
key = '%s.%s' % (record.name, key)
message[key] = value
else:
structlog['nomad.service'] = config.service
structlog['nomad.release'] = config.release
message.update(structlog)
# Handle gunicorn access events
......@@ -352,7 +352,7 @@ def get_logger(name, **kwargs):
if name.startswith('nomad.'):
name = '.'.join(name.split('.')[:2])
logger = structlog.get_logger(name, service=config.service, release=config.release, **kwargs)
logger = structlog.get_logger(name, **kwargs)
return logger
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment