From 80201db7ee6c245b106e6598c3a0f5dba807fe3a Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Thu, 4 Jan 2024 16:30:09 +0100 Subject: [PATCH] Logfile rotation and configurable log format (#1438) * #1389 added 1st draft * fixed flake8 issues * #1388 added formatable date and time * 1st draf of docu * #1389 added documentation for logfile rotation * #1389 adjusted documentation for rotation * Fix import order * Add advanced logging config to schema * Remove .vscode folder from .gitignore * Move references to bottom of file in docs * Use single quotes * Restore original .gitignore * Update configuration.rst * Update log.py * Update pygeoapi-config-0.x.yml --------- Co-authored-by: L K Co-authored-by: Tom Kralidis --- docs/source/configuration.rst | 26 +++++++ pygeoapi/log.py | 72 ++++++++++++++++--- .../schemas/config/pygeoapi-config-0.x.yml | 37 ++++++++++ 3 files changed, 126 insertions(+), 9 deletions(-) diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst index feca6ab40..b9422d828 100644 --- a/docs/source/configuration.rst +++ b/docs/source/configuration.rst @@ -83,11 +83,35 @@ The ``logging`` section provides directives for logging messages which are usefu logging: level: ERROR # the logging level (see https://docs.python.org/3/library/logging.html#logging-levels) logfile: /path/to/pygeoapi.log # the full file path to the logfile + logformat: # example for miliseconds:'[%(asctime)s.%(msecs)03d] {%(pathname)s:%(lineno)d} %(levelname)s - %(message)s' + dateformat: # example for miliseconds:'%Y-%m-%dT%H:%M:%S' .. note:: If ``level`` is defined and ``logfile`` is undefined, logging messages are output to the server's ``stdout``. +``logging.rotation`` +^^^^^^^^^^^^^^^^^^^^ + +The ``rotation`` supports rotation of disk log files. The ``logfile`` file is opened and used as the stream for logging. + +.. code-block:: yaml + + logging: + logfile: /path/to/pygeoapi.log # the full file path to the logfile + rotation: + mode: # [time|size] + when: # [s|m|h|d|w0-w6|midnight] + interval: + max_bytes: + backup_count: +.. note:: + Rotation block is not mandatory and defined only when needed. The ``mode`` can be defined by size or time. + For RotatingFileHandler_ set mode size and parameters max_bytes and backup_count. + + For TimedRotatingFileHandler_ set mode time and parameters when, interval and backup_count. + + ``metadata`` ^^^^^^^^^^^^ @@ -601,3 +625,5 @@ At this point, you have the configuration ready to administer the server. .. _`JSON-LD`: https://json-ld.org .. _`Google Structured Data Testing Tool`: https://search.google.com/structured-data/testing-tool#url=https%3A%2F%2Fdemo.pygeoapi.io%2Fmaster .. _`Google Dataset Search`: https://developers.google.com/search/docs/appearance/structured-data/dataset +.. _RotatingFileHandler: http://docs.python.org/3/library/logging.handlers.html#rotatingfilehandler +.. _TimedRotatingFileHandler: http://docs.python.org/3/library/logging.handlers.html#timedrotatingfilehandler diff --git a/pygeoapi/log.py b/pygeoapi/log.py index b8e991291..b04cdf4f6 100644 --- a/pygeoapi/log.py +++ b/pygeoapi/log.py @@ -2,7 +2,7 @@ # # Authors: Tom Kralidis # -# Copyright (c) 2018 Tom Kralidis +# Copyright (c) 2023 Tom Kralidis # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -30,6 +30,8 @@ """Logging system""" import logging +from logging.handlers import RotatingFileHandler +from logging.handlers import TimedRotatingFileHandler import sys LOGGER = logging.getLogger(__name__) @@ -44,9 +46,13 @@ def setup_logger(logging_config): :returns: void (creates logging instance) """ - log_format = \ + default_log_format = ( '[%(asctime)s] {%(pathname)s:%(lineno)d} %(levelname)s - %(message)s' - date_format = '%Y-%m-%dT%H:%M:%SZ' + ) + default_date_format = '%Y-%m-%dT%H:%M:%SZ' + + log_format = logging_config.get('logformat', default_log_format) + date_format = logging_config.get('dateformat', default_date_format) loglevels = { 'CRITICAL': logging.CRITICAL, @@ -54,18 +60,66 @@ def setup_logger(logging_config): 'WARNING': logging.WARNING, 'INFO': logging.INFO, 'DEBUG': logging.DEBUG, - 'NOTSET': logging.NOTSET, + 'NOTSET': logging.NOTSET } loglevel = loglevels[logging_config['level']] if 'logfile' in logging_config: - logging.basicConfig(level=loglevel, datefmt=date_format, - format=log_format, - filename=logging_config['logfile']) + rotation = logging_config.get('rotation') + if rotation: + rotate_mode = rotation.get('mode') + + rotate_backup_count = rotation.get('backup_count', 0) + + if rotate_mode == 'size': + rotate_max_bytes = rotation.get('max_bytes', 0) + + logging.basicConfig( + handlers=[ + RotatingFileHandler( + filename=logging_config['logfile'], + maxBytes=rotate_max_bytes, + backupCount=rotate_backup_count + ) + ], + level=loglevel, + datefmt=date_format, + format=log_format, + ) + elif rotate_mode == 'time': + rotate_when = rotation.get('when', 'h') + rotate_interval = rotation.get('interval', 1) + + logging.basicConfig( + handlers=[ + TimedRotatingFileHandler( + filename=logging_config['logfile'], + when=rotate_when, + interval=rotate_interval, + backupCount=rotate_backup_count + ) + ], + level=loglevel, + datefmt=date_format, + format=log_format + ) + else: + raise Exception(f'Invalid rotation mode:{rotate_mode}') + else: + logging.basicConfig( + level=loglevel, + datefmt=date_format, + format=log_format, + filename=logging_config['logfile'] + ) else: - logging.basicConfig(level=loglevel, datefmt=date_format, - format=log_format, stream=sys.stdout) + logging.basicConfig( + level=loglevel, + datefmt=date_format, + format=log_format, + stream=sys.stdout + ) LOGGER.debug('Logging initialized') return diff --git a/pygeoapi/schemas/config/pygeoapi-config-0.x.yml b/pygeoapi/schemas/config/pygeoapi-config-0.x.yml index 5ba8ea8be..5d567c93a 100644 --- a/pygeoapi/schemas/config/pygeoapi-config-0.x.yml +++ b/pygeoapi/schemas/config/pygeoapi-config-0.x.yml @@ -148,6 +148,43 @@ properties: logfile: type: string description: the full file path to the logfile. + logformat: + type: string + description: custom logging format + dateformat: + type: string + description: custom date format to use in logs + rotation: + type: object + description: log rotation settings + properties: + mode: + type: string + description: whether to rotate based on size or time + enum: + - size + - time + when: + type: string + description: type of interval + enum: + - s + - m + - h + - d + - w0-w6 + - midnight + interval: + type: integer + description: how often to rotate in time mode + max_bytes: + type: integer + description: when to rotate in size mode + backup_count: + type: integer + description: how many backups to keep + required: + - mode required: - level metadata: