Commit 45600e77 authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Revised some aspects of logging.

parent e0b3a324
Pipeline #37324 passed with stages
in 13 minutes and 28 seconds
......@@ -38,53 +38,13 @@
]
},
{
"name": "Python: tests/test_parsing.py",
"name": "Python: some test",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_parsing.py::test_parser[parsers/template-tests/data/template.json]"
]
},
{
"name": "Python: tests/test_normalizing.py::test_normalizer_reproduce",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_normalizing.py::test_normalizer_reproduce"
]
},
{
"name": "Python: tests/processing/test_base.py",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/processing/test_base.py"
]
},
{
"name": "Python: tests/test_search.py",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/test_search.py"
]
},
{
"name": "Python: tests/processing/test_base.py",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}",
"program": "${workspaceFolder}/.pyenv/bin/pytest",
"args": [
"-sv", "tests/processing/test_base.py::test_fail"
"-sv", "tests/test_parsing.py::test_parser[parsers/vasp-tests/data/parsers/vasp.xml]"
]
},
{
......
......@@ -205,10 +205,10 @@ class UploadsRes(Resource):
if upload.local_path is not None:
logger = get_logger(
__name__, endpoint='uploads', action='post', upload_id=upload.upload_id)
logger.debug('file uploaded offline')
logger.info('file uploaded offline')
upload.upload_time = datetime.now()
upload.process()
logger.debug('initiated processing')
logger.info('initiated processing')
return upload.json_dict, 200
......@@ -467,10 +467,10 @@ class UploadFileRes(Resource):
uploadFile.delete()
abort(400, message='Bad file format, excpected %s.' % ", ".join(UploadFile.formats))
logger.debug('received uploaded file')
logger.info('received uploaded file')
upload.upload_time = datetime.now()
upload.process()
logger.debug('initiated processing')
logger.info('initiated processing')
return upload.json_dict, 200
......
......@@ -322,7 +322,7 @@ class LocalBackend(LegacyParserBackend):
after parsing via :func:`write_json`.
"""
def __init__(self, *args, **kwargs):
self._logger = kwargs.pop('logger', logger)
self.logger = logger
delegate = LegacyLocalBackend(*args, **kwargs)
super().__init__(delegate)
......@@ -340,7 +340,7 @@ class LocalBackend(LegacyParserBackend):
def __getattr__(self, name):
""" Support for unimplemented and unexpected methods. """
if self._unknown_attributes.get(name) is None:
self._logger.debug('Access of unexpected backend attribute/method', attribute=name)
self.logger.debug('Access of unexpected backend attribute/method', attribute=name)
self._unknown_attributes[name] = name
return lambda *args, **kwargs: None
......@@ -350,7 +350,7 @@ class LocalBackend(LegacyParserBackend):
self._errors = parserErrors
def pwarn(self, msg):
self._logger.warn(msg)
self.logger.warn(msg)
def _parse_context_uri(self, context_uri: str) -> Tuple[str, int]:
"""
......
......@@ -19,6 +19,7 @@ import re
import importlib
import inspect
from unittest.mock import patch
import logging
from nomad.parsing.backend import LocalBackend
from nomad.dependencies import PythonGit
......@@ -94,14 +95,13 @@ class LegacyParser(Parser):
return False
def create_backend(self, meta_info):
return LocalBackend(meta_info, debug=False)
def run(self, mainfile: str, logger=None) -> LocalBackend:
# TODO we need a homogeneous interface to parsers, but we dont have it right now.
# There are some hacks to distringuish between ParserInterface parser and simple_parser
# using hasattr, kwargs, etc.
def create_backend(meta_info):
return LocalBackend(meta_info, debug=False, logger=logger)
module_name = self.parser_class_name.split('.')[:-1]
parser_class = self.parser_class_name.split('.')[1]
module = importlib.import_module('.'.join(module_name))
......@@ -109,23 +109,22 @@ class LegacyParser(Parser):
init_signature = inspect.getargspec(Parser.__init__)
kwargs = dict(
backend=create_backend,
mainfile=mainfile, main_file=mainfile,
debug=True)
backend=lambda meta_info: self.create_backend(meta_info),
log_level=logging.DEBUG, debug=True)
kwargs = {key: value for key, value in kwargs.items() if key in init_signature.args}
parser = Parser(**kwargs)
self.parser = Parser(**kwargs)
if logger is not None:
if hasattr(parser, 'setup_logger'):
parser.setup_logger(logger.bind(parser=self.name))
if hasattr(self.parser, 'setup_logger'):
self.parser.setup_logger(logger.bind(parser=self.name))
else:
logger.warning('could not setup parser logger')
with patch.object(sys, 'argv', []):
backend = parser.parse(mainfile)
backend = self.parser.parse(mainfile)
if backend is None or not hasattr(backend, 'status'):
backend = parser.parser_context.super_backend
backend = self.parser.parser_context.super_backend
return backend
......
......@@ -39,7 +39,6 @@ if config.logstash.enabled:
@worker_process_init.connect
def setup(**kwargs):
utils.get_logger(__name__).debug('test debug log entry')
infrastructure.setup()
......@@ -197,7 +196,7 @@ class Proc(Document, metaclass=ProcMetaclass):
errors_str = "; ".join([str(error) for error in errors])
Proc.log(logger, log_level, 'task failed', errors=errors_str, **kwargs)
logger.debug('process failed')
logger.info('process failed')
self.save()
......@@ -229,10 +228,10 @@ class Proc(Document, metaclass=ProcMetaclass):
assert task == tasks[0] # pylint: disable=E1136
self.status = RUNNING
self.current_task = task
self.get_logger().debug('started process')
self.get_logger().info('started process')
else:
self.current_task = task
self.get_logger().debug('successfully completed task')
self.get_logger().info('successfully completed task')
self.save()
return True
......@@ -242,7 +241,7 @@ class Proc(Document, metaclass=ProcMetaclass):
assert self.status == RUNNING, 'Can only complete a running process.'
self.status = SUCCESS
self.save()
self.get_logger().debug('completed process')
self.get_logger().info('completed process')
def block_until_complete(self, interval=0.01):
"""
......@@ -419,7 +418,7 @@ def proc_task(task, cls_name, self_id, func_attr):
might happen in sharded, distributed mongo setups where the object might not
have yet been propagated and therefore apear missing.
"""
logger = utils.get_logger(__name__, cls=cls_name, id=self_id, func=func_attr)
logger = utils.get_logger('__name__', cls=cls_name, id=self_id, func=func_attr)
# get the process class
logger.debug('received process function call')
......
......@@ -119,7 +119,7 @@ if not _logging_is_configured:
logging.basicConfig(stream=sys.stdout)
root = logging.getLogger()
for handler in root.handlers:
handler.setLevel(config.console_log_level if 'pytest' not in sys.modules else logging.CRITICAL)
handler.setLevel(config.console_log_level if 'pytest' not in sys.modules else logging.DEBUG)
# configure logstash
if config.logstash.enabled and 'pytest' not in sys.modules:
......
......@@ -215,6 +215,7 @@ def parsed_template_example() -> LocalBackend:
@pytest.fixture(params=parser_examples, ids=lambda spec: '%s-%s' % spec)
def parsed_example(request) -> LocalBackend:
parser_name, mainfile = request.param
run_parser(parser_name, mainfile)
return run_parser(parser_name, mainfile)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment