Refactored logging setup. Dev elk.

infrastructure/nomad/docker-compose.develk.yml

+# Copyright 2018 Markus Scheidgen
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an"AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: '3.4'
+
+services:
+    # broker for celery
+    rabbitmq:
+        ports:
+            - 5672:5672
+
+    # the search engine
+    elastic:
+        ports:
+            - 9200:9200
+
+    # the user data db
+    mongo:
+        ports:
+            - 27017:27017
+
+    # used for centralized logging
+    elk:
+        restart: always
+        build: ../elk/
+        image: nomad/elk
+        container_name: nomad_elk
+        expose:
+            - 5000 # logstash beats
+            - 5044 # logstash tcp
+        volumes:
+            - nomad_elk:/var/lib/elasticsearch
+        ports:
+            - 5601:5601  # kibana web
+            - 5044:5044   # expose to host, allows metricbeat on host to send data
+            - 15000:5000   # allow nomad services to logstash outside of docker
+            - 29200:9200  # allows metricbeat config to access es
+
+    # nomad processing worker
+    worker:
+        restart: 'no'
+        build: ../../
+        image: nomad/backend
+        environment:
+            NOMAD_LOGSTASH_LEVEL: DEBUG
+            NOMAD_CONSOLE_LOGLEVEL: INFO
+        links:
+            - elk
+
+    # nomad api
+    api:
+        restart: 'no'
+        image: nomad/backend
+        environment:
+            NOMAD_LOGSTASH_LEVEL: DEBUG
+            NOMAD_CONSOLE_LOGLEVEL: INFO
+        depends_on:
+            - worker
+        ports:
+            - 8000:8000
+        links:
+            - elk
+
+    # nomad gui
+    gui:
+        restart: 'no'
+        build: ../../gui
+        image: nomad/frontend
+        ports:
+            - 8080:8000

nomad/client.py

@@ -242,7 +242,6 @@ def cli(host: str, port: int, verbose: bool):
         config.console_log_level = logging.DEBUG
     else:
         config.console_log_level = logging.WARNING
-    utils.configure_logging()
 
     global api_base
     api_base = 'http://%s:%d/nomad/api' % (host, port)
@@ -260,6 +259,7 @@ def cli(host: str, port: int, verbose: bool):
     help='Upload files "offline": files will not be uploaded, but processed were they are. '
     'Only works when run on the nomad host.')
 def upload(path, name: str, offline: bool):
+    utils.configure_logging()
     paths = path
     click.echo('uploading files from %s paths' % len(paths))
     for path in paths:
@@ -289,6 +289,7 @@ def reset():
 @cli.command(help='Run processing locally.')
 @click.argument('ARCHIVE_ID', nargs=1, required=True, type=str)
 def local(archive_id):
+    utils.configure_logging()
     with CalcProcReproduction(archive_id) as local:
         backend = local.parse()
         local.normalize_all(parser_backend=backend)
@@ -302,12 +303,14 @@ def run():
 
 @run.command(help='Run the nomad development worker.')
 def worker():
+    config.service = 'nomad_worker'
     from nomad import processing
     processing.app.worker_main(['worker', '--loglevel=INFO'])
 
 
 @run.command(help='Run the nomad development api.')
 def api():
+    config.service = 'nomad_api'
     from nomad import infrastructure, api
     infrastructure.setup()
     api.app.run(debug=True, port=8000)

nomad/config.py

@@ -99,3 +99,4 @@ services = NomadServicesConfig(
 )
 
 console_log_level = get_loglevel_from_env('NOMAD_CONSOLE_LOGLEVEL', default_level=logging.ERROR)
+service = os.environ.get('NOMAD_SERVICE', 'unknown nomad service')

nomad/infrastructure.py

@@ -46,6 +46,12 @@ def setup():
 
 def setup_logging():
     utils.configure_logging()
+    logger.info(
+        'setup logging',
+        logstash=config.logstash.enabled,
+        logstash_host=config.logstash.host,
+        logstash_port=config.logstash.tcp_port,
+        logstash_level=config.logstash.level)
 
 
 def setup_mongo():

nomad/processing/data.py

@@ -472,7 +472,7 @@ class Upload(Chord):
                         if parser.is_mainfile(filename, lambda fn: mainfile_f):
                             yield potential_mainfile, filename, parser
                 except Exception as e:
-                    self.error(
+                    self.get_logger().error(
                         'exception while matching pot. mainfile',
                         mainfile=filename, exc_info=e)
 

nomad/utils.py

@@ -39,15 +39,12 @@ from structlog.stdlib import LoggerFactory
 import logstash
 from contextlib import contextmanager
 import json
-import os
 import sys
 import uuid
 import time
 
 from nomad import config
 
-_service = os.environ.get('NOMAD_SERVICE', 'nomad service')
-
 
 class LogstashFormatter(logstash.formatter.LogstashFormatterBase):
 
@@ -106,7 +103,7 @@ def add_logstash_handler(logger):
         logstash_handler = logstash.TCPLogstashHandler(
             config.logstash.host,
             config.logstash.tcp_port, version=1)
-        logstash_handler.formatter = LogstashFormatter(tags=['nomad', _service])
+        logstash_handler.formatter = LogstashFormatter(tags=['nomad', config.service])
         logstash_handler.setLevel(config.logstash.level)
         logger.addHandler(logstash_handler)
 
@@ -172,7 +169,7 @@ def get_logger(name, **kwargs):
     if name.startswith('nomad.'):
         name = '.'.join(name.split('.')[:2])
 
-    logger = structlog.get_logger(name, service=_service, **kwargs)
+    logger = structlog.get_logger(name, service=config.service, **kwargs)
     return logger