Fixes and improvements to (processing) logging.

gui/src/components/entry/ArchiveLogView.js

@@ -21,13 +21,14 @@ class LogEntryUnstyled extends React.Component {
       color: amber[700]
     },
     exception: {
+      overflowX: 'scroll',
       margin: 0
     }
   })
 
   render() {
     const { classes, entry } = this.props
-    let data = undefined
+    let data
     try {
       data = JSON.parse(entry)
     } catch (e) {
@@ -86,9 +87,9 @@ class ArchiveLogView extends React.Component {
   });
 
   static defaultState = {
-      data: null,
-      doesNotExist: false
-    }
+    data: null,
+    doesNotExist: false
+  }
 
   state = {...ArchiveLogView.defaultState}
 

nomad/processing/data.py

@@ -33,7 +33,8 @@ import os.path
 from datetime import datetime
 from pymongo import UpdateOne
 import hashlib
-from structlog.processors import StackInfoRenderer, format_exc_info, TimeStamper, JSONRenderer
+from structlog.processors import StackInfoRenderer, format_exc_info, TimeStamper
+import json
 
 from nomad import utils, config, infrastructure, search, datamodel
 from nomad.files import PathObject, UploadFiles, ExtractError, ArchiveBasedStagingUploadFiles, PublicUploadFiles, StagingUploadFiles
@@ -62,8 +63,7 @@ _log_processors = [
     StackInfoRenderer(),
     _pack_log_event,
     format_exc_info,
-    TimeStamper(fmt="%Y-%m-%d %H:%M.%S", utc=False),
-    JSONRenderer(sort_keys=True)]
+    TimeStamper(fmt="%Y-%m-%d %H:%M.%S", utc=False)]
 
 
 class Calc(Proc):
@@ -168,7 +168,9 @@ class Calc(Proc):
             def save_to_calc_log(logger, method_name, event_dict):
                 if self._calc_proc_logwriter is not None:
                     try:
-                        self._calc_proc_logwriter.write(event_dict)
+                        dump_dict = dict(event_dict)
+                        dump_dict.update(level=method_name.upper())
+                        json.dump(dump_dict, self._calc_proc_logwriter, sort_keys=True)
                         self._calc_proc_logwriter.write('\n')
 
                     except Exception:

nomad/utils.py

@@ -203,6 +203,10 @@ class LogstashFormatter(logstash.formatter.LogstashFormatterBase):
             # Extra Fields
             'level': record.levelname,
             'logger_name': record.name,
+
+            # Nomad specific
+            'nomad.service': config.service,
+            'nomad.release': config.release
         }
 
         if record.name.startswith('nomad'):
@@ -211,17 +215,13 @@ class LogstashFormatter(logstash.formatter.LogstashFormatterBase):
                     continue
                 elif key == 'exception':
                     message['digest'] = str(value)[-256:]
-                elif key in (
-                        'upload_id', 'calc_id', 'mainfile',
-                        'service', 'release'):
+                elif key in ['upload_id', 'calc_id', 'mainfile']:
                     key = 'nomad.%s' % key
                 else:
                     key = '%s.%s' % (record.name, key)
 
                 message[key] = value
         else:
-            structlog['nomad.service'] = config.service
-            structlog['nomad.release'] = config.release
             message.update(structlog)
 
         # Handle gunicorn access events
@@ -352,7 +352,7 @@ def get_logger(name, **kwargs):
     if name.startswith('nomad.'):
         name = '.'.join(name.split('.')[:2])
 
-    logger = structlog.get_logger(name, service=config.service, release=config.release, **kwargs)
+    logger = structlog.get_logger(name, **kwargs)
     return logger