replace logging with formatted prints

nomad/cli/admin/run.py

@@ -56,10 +56,6 @@ def logtransfer():
     from nomad.logtransfer import start_logtransfer_service
     from nomad.statistics_logger import start_statistics_logger_process
 
-    # TODO: check why this was necessary to properly apply changes
-    #  made a nomad.yaml file
-    config.load_config()
-
     if not config.logstash.enabled:
         raise RuntimeError(
             'To run the logtransfer service it is required that '

nomad/logtransfer.py

@@ -29,19 +29,16 @@ from datetime import datetime, timedelta
 
 from nomad import config
 
-# It is important to make sure that logstash itself is disabled (otherwise it
-# can happen that logs from the logtransfer server are also included to the
-# logs
-config.logstash.enabled = False
+# For logtransfer no Python logging is used, but instead prints. The reason is that logging can interfere with
+# Nomad logging and may be send to
+enable_log_print = True
 
 
-# logger only for the logstash server (none of the logs should be transferred
-# to the central Nomad instance)
-server_logger = logging.getLogger(name='logtransfer_server')
-server_logger.setLevel(logging.INFO)
-server_handler = logging.StreamHandler()
-server_handler.setFormatter(logging.Formatter('%(levelname)s: %(asctime)s %(message)s', '%Y-%m-%d %H:%M:%S'))
-server_logger.addHandler(server_handler)
+def log_print(message, level='INFO'):
+    if enable_log_print:
+        current_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+        formatted_message = f'{level}: {current_time} {message}'
+        print(formatted_message)
 
 
 def get_log_filepath():
@@ -191,7 +188,7 @@ class Logtransfer:
         all_rotated_logfiles = get_all_rotated_logfiles()
 
         if len(all_rotated_logfiles) > 0:
-            server_logger.info(
+            log_print(
                 f'collected files: '
                 f'{[os.path.basename(f) for f in all_rotated_logfiles]} in '
                 f'directory {config.fs.tmp}'
@@ -203,7 +200,7 @@ class Logtransfer:
 
                 yield file_content, path
         else:
-            server_logger.info('No logfiles to submit.')
+            log_print('No logfiles to submit.')
 
     def _submit_logfile_to_central(self, logfile_content: bytes):
 
@@ -222,12 +219,12 @@ class Logtransfer:
 
             if is_successful:
                 submitted_bytes = ret_request.json()['filesize']
-                server_logger.info(
+                log_print(
                     f'Successfully submitted logfile ({submitted_bytes} bytes) with HTTP status code '
                     f'{ret_request.status_code} to central Oasis at {ret_request.url}.'
                 )
             else:
-                server_logger.info(
+                log_print(
                     f'Submission of logfiles to {central_post_federation_url} failed with HTTP '
                     f'status code {ret_request.status_code}. \n '
                     'logfiles will be included again in next submission.'
@@ -235,7 +232,7 @@ class Logtransfer:
 
         except requests.exceptions.ConnectionError:
             is_successful = False
-            server_logger.info(f'HTTP connection to {central_post_federation_url} could not be established')
+            log_print(f'HTTP connection to {central_post_federation_url} could not be established')
 
         return is_successful
 
@@ -247,7 +244,7 @@ class Logtransfer:
 
     def transfer_logs(self) -> None:
         # (mypy does not recognize the superclass attribute)
-        server_logger.info(
+        log_print(
             'Start logtransfer thread with '  # type: ignore
             f'{self.submit_interval=} sec | '
             f'{self.rotating_file.maxBytes=} bytes'
@@ -269,24 +266,24 @@ class Logtransfer:
             except Exception as e:
                 if not self.raise_unexpected_exceptions:
                     # do not kill the thread and hope it will fix for the next iteration
-                    server_logger.info(f'unexpected exception was raised \n{e}')
+                    log_print(f'unexpected exception was raised \n{e}')
                 else:
                     raise e
 
             next_submission = datetime.now() + timedelta(seconds=self.submit_interval)
             next_submission = next_submission.strftime('%Y-%m-%dT%H:%M:%S')  # type: ignore
-            server_logger.info(f'The next planned submission is at {next_submission}.')
+            log_print(f'The next planned submission is at {next_submission}.')
 
             self.reached_max_bytes_event.clear()
             self.reached_max_bytes_event.wait(self.submit_interval)
             is_notified = self.reached_max_bytes_event.is_set()
 
-            server_logger.info(f'Thread is waking up (is_notified={is_notified}).')
+            log_print(f'Thread is waking up (is_notified={is_notified}).')
 
             with self.rotating_file.rollover_lock:
                 # mypy does not recognize the superclass attribute
                 if self.rotating_file.stream.tell() != 0:  # type: ignore
-                    server_logger.info(
+                    log_print(
                         'Perform rollover on logfile. stream position in '  # type: ignore
                         f'bytes={self.rotating_file.stream.tell()}')
 
@@ -295,7 +292,7 @@ class Logtransfer:
                     # where the logfiles are submitted and removed
                     self.rotating_file.doRollover()
                 else:
-                    server_logger.info(
+                    log_print(
                         'No rollover on logfile performed because '
                         'rotating_file.stream.tell() is at position 0.'
                     )
@@ -366,7 +363,7 @@ def _start_logstash_proxy_server(logger, host, port):
     logstash_proxy_server.timeout = 30
 
     with logstash_proxy_server as server:
-        server_logger.info(f'Start logstash proxy server on host={host} port={port}.')
+        log_print(f'Start logstash proxy server on host={host} port={port}.')
 
         try:
             # NOTE: don't use the serve_forever() method as it does not account
@@ -374,12 +371,12 @@ def _start_logstash_proxy_server(logger, host, port):
             while True:
                 server.handle_request()
         except Exception as e:
-            server_logger.info('logstash proxy server shutdown unexpectedly')
+            log_print('logstash proxy server shutdown unexpectedly')
             raise e
 
 
 def _initialize_logger_and_handler():
-    server_logger.info(
+    log_print(
         f'Initialize logger and handler. Location of intermediate logfiles before '
         f'submission:\n{get_log_filepath()}'
     )
@@ -410,7 +407,7 @@ def start_logtransfer_service(host=None, port=None):
     left None, Nomad config is taken.
     '''
 
-    server_logger.info('Start logtransfer service')
+    log_print('Start logtransfer service')
 
     logstash_logger, rotating_logfile_handler = _initialize_logger_and_handler()
 

tests/logtransfer/test_logtransfer.py

@@ -9,6 +9,8 @@ import logging
 from nomad.utils.structlogging import LogstashFormatter
 from nomad import config
 
+logtransfer.enable_log_print = False
+
 
 def standard_test_log_record(msg='testmsg') -> bytes:
     record = LogRecord(name='test', msg=msg, args=(), exc_info=None, lineno=1, level=logging.INFO,