Commit d476cb66 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Merge branch 'logging' into 'master'

Logging

See merge request !8
parents 35e041f3 c38c884d
Pipeline #37331 passed with stages
in 4 minutes and 59 seconds
......@@ -38,53 +38,13 @@
]
},
{
"name": "Python: tests/test_parsing.py",
"name": "Python: some test",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_parsing.py::test_parser[parsers/template-tests/data/template.json]"
]
},
{
"name": "Python: tests/test_normalizing.py::test_normalizer_reproduce",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer_reproduce"
]
},
{
"name": "Python: tests/processing/test_base.py",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/processing/test_base.py"
]
},
{
"name": "Python: tests/test_search.py",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_search.py"
]
},
{
"name": "Python: tests/processing/test_base.py",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/processing/test_base.py::test_fail"
"-sv", "tests/test_parsing.py::test_parser[parsers/vasp-tests/data/parsers/vasp.xml]"
]
},
{
......
output {
elasticsearch {
hosts => ["localhost"]
# manage_template => false
# index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
# document_type => "%{[@metadata][type]}"
if [type] == "Logstash" {
elasticsearch {
hosts => ["localhost"]
}
} else {
elasticsearch {
hosts => ["localhost"]
manage_template => false
index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
document_type => "%{[@metadata][type]}"
}
}
}
\ No newline at end of file
[
{
"_id": "ce75d080-c2ef-11e8-b9e8-d74f7e7408ed",
"_type": "dashboard",
"_source": {
"title": "Processing and debugging",
"hits": 0,
"description": "",
"panelsJSON": "[{\"panelIndex\":\"1\",\"gridData\":{\"x\":0,\"y\":0,\"w\":24,\"h\":15,\"i\":\"1\"},\"embeddableConfig\":{},\"id\":\"b16c30b0-c2ef-11e8-b9e8-d74f7e7408ed\",\"type\":\"visualization\",\"version\":\"6.3.2\"},{\"panelIndex\":\"2\",\"gridData\":{\"x\":24,\"y\":0,\"w\":24,\"h\":15,\"i\":\"2\"},\"embeddableConfig\":{},\"id\":\"be1b7450-c2f0-11e8-b9e8-d74f7e7408ed\",\"type\":\"visualization\",\"version\":\"6.3.2\"},{\"panelIndex\":\"3\",\"gridData\":{\"x\":0,\"y\":15,\"w\":37,\"h\":8,\"i\":\"3\"},\"embeddableConfig\":{\"spy\":null},\"id\":\"1103dc60-c2f2-11e8-b9e8-d74f7e7408ed\",\"type\":\"visualization\",\"version\":\"6.3.2\"},{\"panelIndex\":\"4\",\"gridData\":{\"x\":37,\"y\":15,\"w\":11,\"h\":8,\"i\":\"4\"},\"embeddableConfig\":{},\"id\":\"b06a49b0-c2f2-11e8-b9e8-d74f7e7408ed\",\"type\":\"visualization\",\"version\":\"6.3.2\"}]",
"optionsJSON": "{\"darkTheme\":false,\"hidePanelTitles\":false,\"useMargins\":true}",
"version": 1,
"timeRestore": false,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[],\"highlightAll\":true,\"version\":true}"
}
},
"_meta": {
"savedObjectVersion": 2
}
},
{
"_id": "b16c30b0-c2ef-11e8-b9e8-d74f7e7408ed",
"_type": "visualization",
"_source": {
"title": "Error events per hour",
"visState": "{\"title\":\"Error events per hour\",\"type\":\"histogram\",\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"event.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"<unknown>\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"h\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}}]}",
"uiStateJSON": "{}",
"description": "",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"index\":\"e8ef6780-c03a-11e8-b9e8-d74f7e7408ed\",\"filter\":[{\"meta\":{\"index\":\"e8ef6780-c03a-11e8-b9e8-d74f7e7408ed\",\"type\":\"phrases\",\"key\":\"level.keyword\",\"value\":\"ERROR, CRITICAL\",\"params\":[\"ERROR\",\"CRITICAL\"],\"negate\":false,\"disabled\":false,\"alias\":null},\"query\":{\"bool\":{\"should\":[{\"match_phrase\":{\"level.keyword\":\"ERROR\"}},{\"match_phrase\":{\"level.keyword\":\"CRITICAL\"}}],\"minimum_should_match\":1}},\"$state\":{\"store\":\"appState\"}}],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}"
}
},
"_meta": {
"savedObjectVersion": 2
}
},
{
"_id": "be1b7450-c2f0-11e8-b9e8-d74f7e7408ed",
"_type": "visualization",
"_source": {
"title": "Error events per logger",
"visState": "{\"title\":\"Error events per logger\",\"type\":\"pie\",\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":false,\"labels\":{\"show\":false,\"values\":true,\"last_level\":true,\"truncate\":200}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"logger_name.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"no name\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"logger\",\"row\":false}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"event.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"no event\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}]}",
"uiStateJSON": "{}",
"description": "",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"index\":\"e8ef6780-c03a-11e8-b9e8-d74f7e7408ed\",\"filter\":[{\"meta\":{\"index\":\"e8ef6780-c03a-11e8-b9e8-d74f7e7408ed\",\"type\":\"phrases\",\"key\":\"level.keyword\",\"value\":\"ERROR, CRITICAL\",\"params\":[\"ERROR\",\"CRITICAL\"],\"negate\":false,\"disabled\":false,\"alias\":null},\"query\":{\"bool\":{\"should\":[{\"match_phrase\":{\"level.keyword\":\"ERROR\"}},{\"match_phrase\":{\"level.keyword\":\"CRITICAL\"}}],\"minimum_should_match\":1}},\"$state\":{\"store\":\"appState\"}}],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}"
}
},
"_meta": {
"savedObjectVersion": 2
}
},
{
"_id": "b06a49b0-c2f2-11e8-b9e8-d74f7e7408ed",
"_type": "visualization",
"_source": {
"title": "Uploads",
"visState": "{\"title\":\"Uploads\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"metric\",\"metric\":{\"percentageMode\":false,\"useRanges\":false,\"colorSchema\":\"Green to Red\",\"metricColorMode\":\"None\",\"colorsRange\":[{\"from\":0,\"to\":10000}],\"labels\":{\"show\":true},\"invertColors\":false,\"style\":{\"bgFill\":\"#000\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\",\"fontSize\":60}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"upload_hash.keyword\",\"customLabel\":\"Uploads\"}}]}",
"uiStateJSON": "{}",
"description": "",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"index\":\"e8ef6780-c03a-11e8-b9e8-d74f7e7408ed\",\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}"
}
},
"_meta": {
"savedObjectVersion": 2
}
},
{
"_id": "1103dc60-c2f2-11e8-b9e8-d74f7e7408ed",
"_type": "visualization",
"_source": {
"title": "Calculations",
"visState": "{\"title\":\"Calculations\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"metric\",\"metric\":{\"percentageMode\":false,\"useRanges\":false,\"colorSchema\":\"Green to Red\",\"metricColorMode\":\"None\",\"colorsRange\":[{\"from\":0,\"to\":10000}],\"labels\":{\"show\":true},\"invertColors\":false,\"style\":{\"bgFill\":\"#000\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\",\"fontSize\":60}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"calc_hash.keyword\",\"customLabel\":\"Calculations\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"level.keyword\",\"otherBucket\":true,\"otherBucketLabel\":\"SUCCESS\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":2,\"order\":\"desc\",\"orderBy\":\"_term\",\"customLabel\":\"\"}}]}",
"uiStateJSON": "{}",
"description": "",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"index\":\"e8ef6780-c03a-11e8-b9e8-d74f7e7408ed\",\"filter\":[{\"meta\":{\"index\":\"e8ef6780-c03a-11e8-b9e8-d74f7e7408ed\",\"negate\":false,\"disabled\":false,\"alias\":null,\"type\":\"exists\",\"key\":\"calc_hash\",\"value\":\"exists\"},\"exists\":{\"field\":\"calc_hash\"},\"$state\":{\"store\":\"appState\"}}],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}"
}
},
"_meta": {
"savedObjectVersion": 2
}
}
]
\ No newline at end of file
......@@ -38,6 +38,8 @@ services:
- nomad_elk:/var/lib/elasticsearch
ports:
- 15601:5601 # kibana web
- 5044:5044 # expose to host, allows metricbeat on host to send data
- 29200:9200 # allows metricbeat config to access es
api:
links:
......@@ -46,6 +48,8 @@ services:
- /nomad:/nomad
worker:
environment:
NOMAD_LOGSTASH_LEVEL: INFO
links:
- elk
volumes:
......
......@@ -205,10 +205,10 @@ class UploadsRes(Resource):
if upload.local_path is not None:
logger = get_logger(
__name__, endpoint='uploads', action='post', upload_id=upload.upload_id)
logger.debug('file uploaded offline')
logger.info('file uploaded offline')
upload.upload_time = datetime.now()
upload.process()
logger.debug('initiated processing')
logger.info('initiated processing')
return upload.json_dict, 200
......@@ -467,10 +467,10 @@ class UploadFileRes(Resource):
uploadFile.delete()
abort(400, message='Bad file format, excpected %s.' % ", ".join(UploadFile.formats))
logger.debug('received uploaded file')
logger.info('received uploaded file')
upload.upload_time = datetime.now()
upload.process()
logger.debug('initiated processing')
logger.info('initiated processing')
return upload.json_dict, 200
......
......@@ -322,7 +322,7 @@ class LocalBackend(LegacyParserBackend):
after parsing via :func:`write_json`.
"""
def __init__(self, *args, **kwargs):
self._logger = kwargs.pop('logger', logger)
self.logger = logger
delegate = LegacyLocalBackend(*args, **kwargs)
super().__init__(delegate)
......@@ -340,7 +340,7 @@ class LocalBackend(LegacyParserBackend):
def __getattr__(self, name):
""" Support for unimplemented and unexpected methods. """
if self._unknown_attributes.get(name) is None:
self._logger.debug('Access of unexpected backend attribute/method', attribute=name)
self.logger.debug('Access of unexpected backend attribute/method', attribute=name)
self._unknown_attributes[name] = name
return lambda *args, **kwargs: None
......@@ -350,7 +350,7 @@ class LocalBackend(LegacyParserBackend):
self._errors = parserErrors
def pwarn(self, msg):
self._logger.warn(msg)
self.logger.warn(msg)
def _parse_context_uri(self, context_uri: str) -> Tuple[str, int]:
"""
......
......@@ -19,6 +19,7 @@ import re
import importlib
import inspect
from unittest.mock import patch
import logging
from nomad.parsing.backend import LocalBackend
from nomad.dependencies import PythonGit
......@@ -94,14 +95,13 @@ class LegacyParser(Parser):
return False
def create_backend(self, meta_info):
return LocalBackend(meta_info, debug=False)
def run(self, mainfile: str, logger=None) -> LocalBackend:
# TODO we need a homogeneous interface to parsers, but we dont have it right now.
# There are some hacks to distringuish between ParserInterface parser and simple_parser
# using hasattr, kwargs, etc.
def create_backend(meta_info):
return LocalBackend(meta_info, debug=False, logger=logger)
module_name = self.parser_class_name.split('.')[:-1]
parser_class = self.parser_class_name.split('.')[1]
module = importlib.import_module('.'.join(module_name))
......@@ -109,23 +109,22 @@ class LegacyParser(Parser):
init_signature = inspect.getargspec(Parser.__init__)
kwargs = dict(
backend=create_backend,
mainfile=mainfile, main_file=mainfile,
debug=True)
backend=lambda meta_info: self.create_backend(meta_info),
log_level=logging.DEBUG, debug=True)
kwargs = {key: value for key, value in kwargs.items() if key in init_signature.args}
parser = Parser(**kwargs)
self.parser = Parser(**kwargs)
if logger is not None:
if hasattr(parser, 'setup_logger'):
parser.setup_logger(logger.bind(parser=self.name))
if hasattr(self.parser, 'setup_logger'):
self.parser.setup_logger(logger.bind(parser=self.name))
else:
logger.warning('could not setup parser logger')
with patch.object(sys, 'argv', []):
backend = parser.parse(mainfile)
backend = self.parser.parse(mainfile)
if backend is None or not hasattr(backend, 'status'):
backend = parser.parser_context.super_backend
backend = self.parser.parser_context.super_backend
return backend
......
......@@ -39,7 +39,6 @@ if config.logstash.enabled:
@worker_process_init.connect
def setup(**kwargs):
utils.get_logger(__name__).debug('test debug log entry')
infrastructure.setup()
......@@ -197,7 +196,7 @@ class Proc(Document, metaclass=ProcMetaclass):
errors_str = "; ".join([str(error) for error in errors])
Proc.log(logger, log_level, 'task failed', errors=errors_str, **kwargs)
logger.debug('process failed')
logger.info('process failed')
self.save()
......@@ -229,10 +228,10 @@ class Proc(Document, metaclass=ProcMetaclass):
assert task == tasks[0] # pylint: disable=E1136
self.status = RUNNING
self.current_task = task
self.get_logger().debug('started process')
self.get_logger().info('started process')
else:
self.current_task = task
self.get_logger().debug('successfully completed task')
self.get_logger().info('successfully completed task')
self.save()
return True
......@@ -242,7 +241,7 @@ class Proc(Document, metaclass=ProcMetaclass):
assert self.status == RUNNING, 'Can only complete a running process.'
self.status = SUCCESS
self.save()
self.get_logger().debug('completed process')
self.get_logger().info('completed process')
def block_until_complete(self, interval=0.01):
"""
......@@ -419,7 +418,7 @@ def proc_task(task, cls_name, self_id, func_attr):
might happen in sharded, distributed mongo setups where the object might not
have yet been propagated and therefore apear missing.
"""
logger = utils.get_logger(__name__, cls=cls_name, id=self_id, func=func_attr)
logger = utils.get_logger('__name__', cls=cls_name, id=self_id, func=func_attr)
# get the process class
logger.debug('received process function call')
......
......@@ -119,7 +119,7 @@ if not _logging_is_configured:
logging.basicConfig(stream=sys.stdout)
root = logging.getLogger()
for handler in root.handlers:
handler.setLevel(config.console_log_level if 'pytest' not in sys.modules else logging.CRITICAL)
handler.setLevel(config.console_log_level if 'pytest' not in sys.modules else logging.DEBUG)
# configure logstash
if config.logstash.enabled and 'pytest' not in sys.modules:
......
......@@ -215,6 +215,7 @@ def parsed_template_example() -> LocalBackend:
@pytest.fixture(params=parser_examples, ids=lambda spec: '%s-%s' % spec)
def parsed_example(request) -> LocalBackend:
parser_name, mainfile = request.param
run_parser(parser_name, mainfile)
return run_parser(parser_name, mainfile)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment