diff --git a/.gitignore b/.gitignore index 29da07ff6287285e573c1db33a07d537567aa166..271b38436fbdaef4c1934e4886937dd7f2b542b7 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,5 @@ target/ .vscode/ vscode/ nomad.yaml +gunicorn.log.conf +gunicorn.conf \ No newline at end of file diff --git a/nomad/utils.py b/nomad/utils.py index 91df086bf2c1ba44fead10d2320a546b998bcd09..8e911a14956a5e84f8c32d4cbc4715f3c15313d3 100644 --- a/nomad/utils.py +++ b/nomad/utils.py @@ -165,6 +165,20 @@ class LogstashHandler(logstash.TCPLogstashHandler): return False +_gunicorn_pattern_parts = [ + r'(?P<host>\S+)', # host %h + r'\S+', # indent %l (unused) + r'(?P<user>\S+)', # user %u + r'\[(?P<time>.+)\]', # time %t + r'"(?P<request>.+)"', # request "%r" + r'(?P<status>[0-9]+)', # status %>s + r'(?P<size>\S+)', # size %b (careful, can be '-') + r'"(?P<referer>.*)"', # referer "%{Referer}i" + r'"(?P<agent>.*)"', # user agent "%{User-agent}i" +] +_gunicorn_pattern = re.compile(r'\s+'.join(_gunicorn_pattern_parts) + r'\s*\Z') + + class LogstashFormatter(logstash.formatter.LogstashFormatterBase): def format(self, record): @@ -178,7 +192,6 @@ class LogstashFormatter(logstash.formatter.LogstashFormatterBase): '@timestamp': self.format_timestamp(record.created), '@version': '1', 'event': structlog['event'], - 'message': structlog['event'], 'host': self.host, 'path': record.pathname, 'tags': self.tags, @@ -208,6 +221,27 @@ class LogstashFormatter(logstash.formatter.LogstashFormatterBase): structlog['nomad.release'] = config.release message.update(structlog) + # Handle gunicorn access events + if record.name == 'gunicorn.access': + gunicorn_message = structlog['event'] + gunicorn_record = _gunicorn_pattern.match(gunicorn_message).groupdict() + + if gunicorn_record['user'] == '-': + gunicorn_record['user'] = None + + gunicorn_record['status'] = int(gunicorn_record['status']) + + if gunicorn_record['size'] == '-': + gunicorn_record['size'] = 0 + else: + gunicorn_record['size'] = int(gunicorn_record['size']) + + if gunicorn_record['referer'] == '-': + gunicorn_record['referer'] = None + + message.update({'gunicorn.%s' % key: value for key, value in gunicorn_record.items()}) + message['event'] = gunicorn_record['request'] + # Add extra fields message.update(self.get_extra_fields(record))