Logfile rotation and configurable log format (#1438)

* #1389 added 1st draft

* fixed flake8 issues

* #1388 added formatable  date and time

* 1st draf of docu

* #1389 added documentation for logfile rotation

* #1389 adjusted documentation for rotation

* Fix import order

* Add advanced logging config to schema

* Remove .vscode folder from .gitignore

* Move references to bottom of file in docs

* Use single quotes

* Restore original .gitignore

* Update configuration.rst

* Update log.py

* Update pygeoapi-config-0.x.yml

---------

Co-authored-by: L K <[email protected]>
Co-authored-by: Tom Kralidis <[email protected]>

docs/source/configuration.rst

@@ -83,11 +83,35 @@ The ``logging`` section provides directives for logging messages which are usefu
   logging:
       level: ERROR  # the logging level (see https://docs.python.org/3/library/logging.html#logging-levels)
       logfile: /path/to/pygeoapi.log  # the full file path to the logfile
+      logformat: # example for miliseconds:'[%(asctime)s.%(msecs)03d] {%(pathname)s:%(lineno)d} %(levelname)s - %(message)s'
+      dateformat: # example for miliseconds:'%Y-%m-%dT%H:%M:%S'
 
 .. note::
    If ``level`` is defined and ``logfile`` is undefined, logging messages are output to the server's ``stdout``.
 
 
+``logging.rotation``
+^^^^^^^^^^^^^^^^^^^^
+
+The ``rotation`` supports rotation of disk log files. The ``logfile`` file is opened and used as the stream for logging.
+
+.. code-block:: yaml
+
+  logging:
+      logfile: /path/to/pygeoapi.log  # the full file path to the logfile
+      rotation:
+          mode: # [time|size]
+          when: # [s|m|h|d|w0-w6|midnight]
+          interval: 
+          max_bytes: 
+          backup_count: 
+.. note::
+  Rotation block is not mandatory and defined only when needed. The ``mode`` can be defined by size or time.
+  For RotatingFileHandler_ set mode size and parameters max_bytes and backup_count.
+
+  For TimedRotatingFileHandler_ set mode time and parameters when, interval and backup_count.
+
+
 ``metadata``
 ^^^^^^^^^^^^
 
@@ -601,3 +625,5 @@ At this point, you have the configuration ready to administer the server.
 .. _`JSON-LD`: https://json-ld.org
 .. _`Google Structured Data Testing Tool`: https://search.google.com/structured-data/testing-tool#url=https%3A%2F%2Fdemo.pygeoapi.io%2Fmaster
 .. _`Google Dataset Search`: https://developers.google.com/search/docs/appearance/structured-data/dataset
+.. _RotatingFileHandler: http://docs.python.org/3/library/logging.handlers.html#rotatingfilehandler
+.. _TimedRotatingFileHandler: http://docs.python.org/3/library/logging.handlers.html#timedrotatingfilehandler

pygeoapi/log.py

@@ -2,7 +2,7 @@
 #
 # Authors: Tom Kralidis <[email protected]>
 #
-# Copyright (c) 2018 Tom Kralidis
+# Copyright (c) 2023 Tom Kralidis
 #
 # Permission is hereby granted, free of charge, to any person
 # obtaining a copy of this software and associated documentation
@@ -30,6 +30,8 @@
 """Logging system"""
 
 import logging
+from logging.handlers import RotatingFileHandler
+from logging.handlers import TimedRotatingFileHandler
 import sys
 
 LOGGER = logging.getLogger(__name__)
@@ -44,28 +46,80 @@ def setup_logger(logging_config):
     :returns: void (creates logging instance)
     """
 
-    log_format = \
+    default_log_format = (
         '[%(asctime)s] {%(pathname)s:%(lineno)d} %(levelname)s - %(message)s'
-    date_format = '%Y-%m-%dT%H:%M:%SZ'
+    )
+    default_date_format = '%Y-%m-%dT%H:%M:%SZ'
+
+    log_format = logging_config.get('logformat', default_log_format)
+    date_format = logging_config.get('dateformat', default_date_format)
 
     loglevels = {
         'CRITICAL': logging.CRITICAL,
         'ERROR': logging.ERROR,
         'WARNING': logging.WARNING,
         'INFO': logging.INFO,
         'DEBUG': logging.DEBUG,
-        'NOTSET': logging.NOTSET,
+        'NOTSET': logging.NOTSET
     }
 
     loglevel = loglevels[logging_config['level']]
 
     if 'logfile' in logging_config:
-        logging.basicConfig(level=loglevel, datefmt=date_format,
-                            format=log_format,
-                            filename=logging_config['logfile'])
+        rotation = logging_config.get('rotation')
+        if rotation:
+            rotate_mode = rotation.get('mode')
+
+            rotate_backup_count = rotation.get('backup_count', 0)
+
+            if rotate_mode == 'size':
+                rotate_max_bytes = rotation.get('max_bytes', 0)
+
+                logging.basicConfig(
+                    handlers=[
+                        RotatingFileHandler(
+                            filename=logging_config['logfile'],
+                            maxBytes=rotate_max_bytes,
+                            backupCount=rotate_backup_count
+                        )
+                    ],
+                    level=loglevel,
+                    datefmt=date_format,
+                    format=log_format,
+                )
+            elif rotate_mode == 'time':
+                rotate_when = rotation.get('when', 'h')
+                rotate_interval = rotation.get('interval', 1)
+
+                logging.basicConfig(
+                    handlers=[
+                        TimedRotatingFileHandler(
+                            filename=logging_config['logfile'],
+                            when=rotate_when,
+                            interval=rotate_interval,
+                            backupCount=rotate_backup_count
+                        )
+                    ],
+                    level=loglevel,
+                    datefmt=date_format,
+                    format=log_format
+                )
+            else:
+                raise Exception(f'Invalid rotation mode:{rotate_mode}')
+        else:
+            logging.basicConfig(
+                level=loglevel,
+                datefmt=date_format,
+                format=log_format,
+                filename=logging_config['logfile']
+            )
     else:
-        logging.basicConfig(level=loglevel, datefmt=date_format,
-                            format=log_format, stream=sys.stdout)
+        logging.basicConfig(
+            level=loglevel,
+            datefmt=date_format,
+            format=log_format,
+            stream=sys.stdout
+        )
 
     LOGGER.debug('Logging initialized')
     return

pygeoapi/schemas/config/pygeoapi-config-0.x.yml

@@ -148,6 +148,43 @@ properties:
             logfile:
                 type: string
                 description: the full file path to the logfile.
+            logformat:
+                type: string
+                description: custom logging format
+            dateformat:
+                type: string
+                description: custom date format to use in logs
+            rotation:
+                type: object
+                description: log rotation settings
+                properties:
+                    mode:
+                        type: string
+                        description: whether to rotate based on size or time
+                        enum:
+                            - size
+                            - time
+                    when:
+                        type: string
+                        description: type of interval
+                        enum:
+                            - s
+                            - m
+                            - h
+                            - d
+                            - w0-w6
+                            - midnight
+                    interval:
+                        type: integer
+                        description: how often to rotate in time mode
+                    max_bytes:
+                        type: integer
+                        description: when to rotate in size mode
+                    backup_count:
+                        type: integer
+                        description: how many backups to keep
+                required:
+                    - mode
         required:
             - level
     metadata: