From 797fb75d34ce713966d2a69e452a0e4e8283546b Mon Sep 17 00:00:00 2001 From: Mathieu Leplatre Date: Wed, 16 Oct 2024 17:21:36 +0200 Subject: [PATCH] Add prometheus support (Fixes #3407) (#3453) * Introduce the Prometheus lib * Start plugin * Unit testing * Adapt usage of unique parameter to groups * Skip test when no prometheus installed * Add test for events * Enable Prometheus in functional test * Add section about Prometheus * Move plugins section above monitoring * Fix link format in docs * Remove TODO * nit docs --- constraints.in | 1 + constraints.txt | 16 +-- docs/configuration/production.rst | 9 +- docs/configuration/settings.rst | 171 ++++++++++++++++------------- kinto/core/testing.py | 5 +- kinto/plugins/prometheus.py | 150 +++++++++++++++++++++++++ pyproject.toml | 1 + tests/core/resource/test_events.py | 21 +++- tests/functional.ini | 1 + tests/plugins/test_prometheus.py | 120 ++++++++++++++++++++ 10 files changed, 399 insertions(+), 96 deletions(-) create mode 100644 kinto/plugins/prometheus.py create mode 100644 tests/plugins/test_prometheus.py diff --git a/constraints.in b/constraints.in index 1a2c98377..4ee0747dc 100644 --- a/constraints.in +++ b/constraints.in @@ -25,6 +25,7 @@ psycopg2 zope.sqlalchemy # monitoring newrelic +prometheus-client sentry-sdk[sqlalchemy] statsd werkzeug diff --git a/constraints.txt b/constraints.txt index b75189c8e..0fb387ffb 100644 --- a/constraints.txt +++ b/constraints.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --output-file=constraints.txt --strip-extras constraints.in @@ -40,16 +40,12 @@ coverage==7.4.0 # via pytest-cov dockerflow==2024.4.2 # via -r constraints.in -exceptiongroup==1.2.0 - # via pytest execnet==2.0.2 # via pytest-cache fqdn==1.5.1 # via jsonschema greenlet==3.0.3 - # via - # playwright - # sqlalchemy + # via playwright hupper==1.12 # via pyramid idna==3.7 @@ -102,6 +98,8 @@ playwright==1.47.0 # via -r constraints.in pluggy==1.5.0 # via pytest +prometheus-client==0.21.0 + # via -r constraints.in psycopg2==2.9.9 # via -r constraints.in pyee==12.0.0 @@ -186,12 +184,6 @@ statsd==4.0.1 # via -r constraints.in swagger-spec-validator==3.0.3 # via bravado-core -tomli==2.0.1 - # via - # build - # coverage - # pyproject-hooks - # pytest transaction==5.0 # via # -r constraints.in diff --git a/docs/configuration/production.rst b/docs/configuration/production.rst index e26ab4c67..e1be4ea17 100644 --- a/docs/configuration/production.rst +++ b/docs/configuration/production.rst @@ -220,19 +220,14 @@ In the configuration of the CDN service, you should also: Monitoring ---------- -In order to enable monitoring features like *statsd*, install +In order to enable monitoring features (eg. *Prometheus* or *StatsD*), install extra requirements: :: make install-monitoring -And configure its URL: - -.. code-block :: ini - - # StatsD - kinto.statsd_url = udp://carbon.server:8125 +See :ref:`settings section ` for the configuration aspects. Counters :::::::: diff --git a/docs/configuration/settings.rst b/docs/configuration/settings.rst index f23196ff7..6b91b4d29 100644 --- a/docs/configuration/settings.rst +++ b/docs/configuration/settings.rst @@ -301,6 +301,83 @@ hello view `. # kinto.http_host = production.server.com:7777 +.. _configuration-plugins: + +Plugins +======= + +It is possible to extend the default Kinto behaviors by using "plugins". + +The list of plugins to load at startup can be specified in the settings, as a +list of Python modules: + +.. code-block:: ini + + kinto.includes = kinto.plugins.default_bucket + kinto.plugins.history + kinto.plugins.admin + kinto-attachment + custom-myplugin + ++---------------------------------------+--------------------------------------------------------------------------+ +| Built-in plugins | What does it do? | ++=======================================+==========================================================================+ +| ``kinto.plugins.accounts`` | It allows users to sign-up and authenticate using username and password | +| | (:ref:`more details `). | ++---------------------------------------+--------------------------------------------------------------------------+ +| ``kinto.plugins.admin`` | It is a Web admin UI to manage data from a Kinto server. | +| | (:ref:`more details `). | ++---------------------------------------+--------------------------------------------------------------------------+ +| ``kinto.plugins.default_bucket`` | It enables a personal bucket ``default``, where collections are created | +| | implicitly (:ref:`more details `). | ++---------------------------------------+--------------------------------------------------------------------------+ +| ``kinto.plugins.flush`` | Adds an endpoint to completely remove all data from the database backend | +| | for testing/staging purposes. (:ref:`more details `). | ++---------------------------------------+--------------------------------------------------------------------------+ +| ``kinto.plugins.history`` | It tracks every action performed on objects within a bucket | +| | (:ref:`more details `). | ++---------------------------------------+--------------------------------------------------------------------------+ +| ``kinto.plugins.openid`` | It allows to authenticate users using OpenID Connect from Google, | +| | Microsoft, Auth0, etc. (:ref:`more details `). | ++---------------------------------------+--------------------------------------------------------------------------+ +| ``kinto.plugins.quotas`` | It allows to limit storage per collection size, number of records, etc. | +| | (:ref:`more details `). | ++---------------------------------------+--------------------------------------------------------------------------+ +| ``kinto.plugins.prometheus`` | Send metrics about backend duration, authentication, endpoints hits, .. | +| | (:ref:`more details `). | ++---------------------------------------+--------------------------------------------------------------------------+ +| ``kinto.plugins.statsd`` | Send metrics about backend duration, authentication, endpoints hits, .. | +| | (:ref:`more details `). | ++---------------------------------------+--------------------------------------------------------------------------+ + + +There are `many available packages`_ in the Pyramid ecosystem, and it is straightforward to build one, +since the specified module must just define an ``includeme(config)`` function. + +.. _many available packages: https://github.com/ITCase/awesome-pyramid + +See `our list of community plugins `_. + +See also: :ref:`tutorial-write-plugin` for more in-depth informations on how +to create your own plugin. + + +Pluggable components +:::::::::::::::::::: + +:term:`Pluggable ` components can be substituted from configuration files, +as long as the replacement follows the original component API. + +.. code-block:: ini + + kinto.logging_renderer = your_log_renderer.CustomRenderer + +This is the simplest way to extend *Kinto*, but will be limited to its +existing components (cache, storage, log renderer, ...). + +In order to add extra features, including external packages is the way to go! + + Logging and Monitoring ====================== @@ -413,6 +490,22 @@ Or the equivalent environment variables: The application sends an event on startup (mainly for setup check). +.. _monitoring-with-prometheus: + +Monitoring with Prometheus +:::::::::::::::::::::::::: + +Requires the ``prometheus-client`` package (installed with ``kinto[monitoring]``). + +Prometheus metrics can be enabled with (disabled by default): + +.. code-block:: ini + + kinto.includes = kinto.plugins.prometheus + +Metrics can then be crawled from the ``/__metrics__`` endpoint. + + .. _monitoring-with-statsd: Monitoring with StatsD @@ -420,6 +513,10 @@ Monitoring with StatsD Requires the ``statsd`` package. +.. note:: + + Only one of *Prometheus* and *StatsD* can be enabled. It will take precedence and the other one will be ignored. + +------------------------+----------------------------------------+--------------------------------------------------------------------------+ | Setting name | Default | What does it do? | +========================+========================================+==========================================================================+ @@ -475,80 +572,6 @@ New Relic can be enabled (disabled by default): kinto.newrelic_env = prod -.. _configuration-plugins: - -Plugins -======= - -It is possible to extend the default Kinto behaviors by using "plugins". - -The list of plugins to load at startup can be specified in the settings, as a -list of Python modules: - -.. code-block:: ini - - kinto.includes = kinto.plugins.default_bucket - kinto.plugins.history - kinto.plugins.admin - kinto-attachment - custom-myplugin - -+---------------------------------------+--------------------------------------------------------------------------+ -| Built-in plugins | What does it do? | -+=======================================+==========================================================================+ -| ``kinto.plugins.accounts`` | It allows users to sign-up and authenticate using username and password | -| | (:ref:`more details `). | -+---------------------------------------+--------------------------------------------------------------------------+ -| ``kinto.plugins.admin`` | It is a Web admin UI to manage data from a Kinto server. | -| | (:ref:`more details `). | -+---------------------------------------+--------------------------------------------------------------------------+ -| ``kinto.plugins.default_bucket`` | It enables a personal bucket ``default``, where collections are created | -| | implicitly (:ref:`more details `). | -+---------------------------------------+--------------------------------------------------------------------------+ -| ``kinto.plugins.flush`` | Adds an endpoint to completely remove all data from the database backend | -| | for testing/staging purposes. (:ref:`more details `). | -+---------------------------------------+--------------------------------------------------------------------------+ -| ``kinto.plugins.history`` | It tracks every action performed on objects within a bucket | -| | (:ref:`more details `). | -+---------------------------------------+--------------------------------------------------------------------------+ -| ``kinto.plugins.openid`` | It allows to authenticate users using OpenID Connect from Google, | -| | Microsoft, Auth0, etc. (:ref:`more details `). | -+---------------------------------------+--------------------------------------------------------------------------+ -| ``kinto.plugins.quotas`` | It allows to limit storage per collection size, number of records, etc. | -| | (:ref:`more details `). | -+---------------------------------------+--------------------------------------------------------------------------+ -| ``kinto.plugins.statsd`` | Send metrics about backend duration, authentication, endpoints hits, .. | -| | (:ref:`more details `). | -+---------------------------------------+--------------------------------------------------------------------------+ - - -There are `many available packages`_ in Pyramid ecosystem, and it is straightforward to build one, -since the specified module must just define an ``includeme(config)`` function. - -.. _many available packages: https://github.com/ITCase/awesome-pyramid - -See `our list of community plugins `_. - -See also: :ref:`tutorial-write-plugin` for more in-depth informations on how -to create your own plugin. - - -Pluggable components -:::::::::::::::::::: - -:term:`Pluggable ` components can be substituted from configuration files, -as long as the replacement follows the original component API. - -.. code-block:: ini - - kinto.logging_renderer = your_log_renderer.CustomRenderer - -This is the simplest way to extend *Kinto*, but will be limited to its -existing components (cache, storage, log renderer, ...). - -In order to add extra features, including external packages is the way to go! - - .. _configuration-authentication: Authentication diff --git a/kinto/core/testing.py b/kinto/core/testing.py index 9a779fb08..d714b4abb 100644 --- a/kinto/core/testing.py +++ b/kinto/core/testing.py @@ -11,13 +11,16 @@ from kinto.core import DEFAULT_SETTINGS from kinto.core.storage import generators from kinto.core.utils import encode64, follow_subrequest, memcache, sqlalchemy -from kinto.plugins import statsd +from kinto.plugins import prometheus, statsd skip_if_ci = unittest.skipIf("CI" in os.environ, "ci") skip_if_no_postgresql = unittest.skipIf(sqlalchemy is None, "postgresql is not installed.") skip_if_no_memcached = unittest.skipIf(memcache is None, "memcached is not installed.") skip_if_no_statsd = unittest.skipIf(not statsd.statsd_module, "statsd is not installed.") +skip_if_no_prometheus = unittest.skipIf( + not prometheus.prometheus_module, "prometheus is not installed." +) class DummyRequest(mock.MagicMock): diff --git a/kinto/plugins/prometheus.py b/kinto/plugins/prometheus.py new file mode 100644 index 000000000..ae064c96d --- /dev/null +++ b/kinto/plugins/prometheus.py @@ -0,0 +1,150 @@ +import functools +from time import perf_counter as time_now + +from pyramid.exceptions import ConfigurationError +from pyramid.response import Response +from zope.interface import implementer + +from kinto.core import metrics + + +try: + import prometheus_client as prometheus_module +except ImportError: # pragma: no cover + prometheus_module = None + + +_METRICS = {} +_REGISTRY = None + + +def get_registry(): + global _REGISTRY + + if _REGISTRY is None: + _REGISTRY = prometheus_module.CollectorRegistry() + return _REGISTRY + + +def _fix_metric_name(s): + return s.replace("-", "_").replace(".", "_") + + +def safe_wraps(wrapper, *args, **kwargs): + """Safely wraps partial functions.""" + while isinstance(wrapper, functools.partial): + wrapper = wrapper.func + return functools.wraps(wrapper, *args, **kwargs) + + +class Timer: + def __init__(self, summary): + self.summary = summary + self._start_time = None + + def __call__(self, f): + @safe_wraps(f) + def _wrapped(*args, **kwargs): + start_time = time_now() + try: + return f(*args, **kwargs) + finally: + dt_ms = 1000.0 * (time_now() - start_time) + self.summary.observe(dt_ms) + + return _wrapped + + def __enter__(self): + return self.start() + + def __exit__(self, typ, value, tb): + self.stop() + + def start(self): + self._start_time = time_now() + return self + + def stop(self): + if self._start_time is None: # pragma: nocover + raise RuntimeError("Timer has not started.") + dt_ms = 1000.0 * (time_now() - self._start_time) + self.summary.observe(dt_ms) + return self + + +@implementer(metrics.IMetricsService) +class PrometheusService: + def timer(self, key): + global _METRICS + if key not in _METRICS: + _METRICS[key] = prometheus_module.Summary( + _fix_metric_name(key), f"Summary of {key}", registry=get_registry() + ) + + if not isinstance(_METRICS[key], prometheus_module.Summary): + raise RuntimeError( + f"Metric {key} already exists with different type ({_METRICS[key]})" + ) + + return Timer(_METRICS[key]) + + def count(self, key, count=1, unique=None): + global _METRICS + + # Turn `unique` into a group and a value: + # eg. `method.basicauth.mat` -> `method_basicauth="mat"` + label_value = None + if unique: + if "." not in unique: + unique = f"group.{unique}" + label_name, label_value = unique.rsplit(".", 1) + label_names = (_fix_metric_name(label_name),) + else: + label_names = tuple() + + if key not in _METRICS: + _METRICS[key] = prometheus_module.Counter( + _fix_metric_name(key), + f"Counter of {key}", + labelnames=label_names, + registry=get_registry(), + ) + + if not isinstance(_METRICS[key], prometheus_module.Counter): + raise RuntimeError( + f"Metric {key} already exists with different type ({_METRICS[key]})" + ) + + m = _METRICS[key] + if label_value is not None: + m = m.labels(label_value) + + m.inc(count) + + +def metrics_view(request): + registry = get_registry() + data = prometheus_module.generate_latest(registry) + resp = Response(body=data) + resp.headers["Content-Type"] = prometheus_module.CONTENT_TYPE_LATEST + resp.headers["Content-Length"] = str(len(data)) + return resp + + +def includeme(config): + if prometheus_module is None: + error_msg = ( + "Please install Kinto with monitoring dependencies (e.g. prometheus-client package)" + ) + raise ConfigurationError(error_msg) + + config.add_api_capability( + "prometheus", + description="Prometheus metrics.", + url="https://github.com/Kinto/kinto/", + ) + + config.add_route("prometheus_metrics", "/__metrics__") + config.add_view(metrics_view, route_name="prometheus_metrics") + + config.registry.registerUtility(PrometheusService(), metrics.IMetricsService) diff --git a/pyproject.toml b/pyproject.toml index 3385307c1..e7d51a28d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,6 +81,7 @@ monitoring = [ "sentry-sdk[sqlalchemy]", "statsd", "werkzeug", + "prometheus-client", ] test = [ "bravado", diff --git a/tests/core/resource/test_events.py b/tests/core/resource/test_events.py index 6de66c201..4ab0fb6e5 100644 --- a/tests/core/resource/test_events.py +++ b/tests/core/resource/test_events.py @@ -13,7 +13,7 @@ notify_resource_event, ) from kinto.core.storage.exceptions import BackendError -from kinto.core.testing import skip_if_no_statsd, unittest +from kinto.core.testing import skip_if_no_prometheus, skip_if_no_statsd, unittest from kinto.plugins import statsd from ..support import BaseWebTest @@ -506,9 +506,26 @@ def get_app_settings(cls, *args, **kwargs): return settings def test_statds_tracks_listeners_execution_duration(self): - # This test may break when introducing a generic interface for Prometheus. metrics_client = self.app.app.registry.metrics._client with mock.patch.object(metrics_client, "timing") as mocked: self.app.post_json(self.plural_url, {"data": {"name": "pouet"}}, headers=self.headers) timers = set(c[0][0] for c in mocked.call_args_list) self.assertIn("listeners.test", timers) + + +@skip_if_no_prometheus +class PrometheusTest(BaseWebTest, unittest.TestCase): + @classmethod + def get_app_settings(cls, *args, **kwargs): + settings = super().get_app_settings(*args, **kwargs) + settings["includes"] = "kinto.plugins.prometheus" + settings["event_listeners"] = "test" + this_module = "tests.core.resource.test_events" + settings["event_listeners.test.use"] = this_module + return settings + + def test_prometheus_tracks_listeners_execution_duration(self): + self.app.post_json(self.plural_url, {"data": {"name": "pouet"}}, headers=self.headers) + + resp = self.app.get("/__metrics__") + self.assertIn("listeners_test_count 1.0", resp.text) diff --git a/tests/functional.ini b/tests/functional.ini index 72b92d32b..edcc6794a 100644 --- a/tests/functional.ini +++ b/tests/functional.ini @@ -34,6 +34,7 @@ kinto.account_create_principals = system.Everyone # Plugins # kinto.includes = kinto.plugins.default_bucket + kinto.plugins.prometheus kinto.plugins.admin kinto.plugins.accounts kinto.plugins.flush diff --git a/tests/plugins/test_prometheus.py b/tests/plugins/test_prometheus.py new file mode 100644 index 000000000..97ffd0d52 --- /dev/null +++ b/tests/plugins/test_prometheus.py @@ -0,0 +1,120 @@ +import functools +import unittest +from unittest import mock + +from pyramid.exceptions import ConfigurationError + +from kinto.core.testing import get_user_headers, skip_if_no_prometheus +from kinto.plugins import prometheus + +from .. import support + + +DATETIME_REGEX = r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}\+\d{2}:\d{2}$" + + +class PrometheusMissing(unittest.TestCase): + def setUp(self): + self.previous = prometheus.prometheus_module + prometheus.prometheus_module = None + + def tearDown(self): + prometheus.prometheus_module = self.previous + + def test_client_instantiation_raises_properly(self): + with self.assertRaises(ConfigurationError): + prometheus.includeme(mock.MagicMock()) + + +class PrometheusWebTest(support.BaseWebTest, unittest.TestCase): + @classmethod + def get_app_settings(cls, extras=None): + settings = super().get_app_settings(extras) + settings["includes"] = "kinto.plugins.prometheus" + return settings + + +@skip_if_no_prometheus +class ViewsTest(PrometheusWebTest): + def test_prometheus_capability_if_enabled(self): + resp = self.app.get("/") + capabilities = resp.json["capabilities"] + self.assertIn("prometheus", capabilities) + + def test_endpoint_with_metrics(self): + self.app.put("/buckets/test", headers=get_user_headers("aaa")) + + resp = self.app.get("/__metrics__") + self.assertIn("Summary", resp.text) + + +def my_func(a, b): + return a + b + + +@skip_if_no_prometheus +class ServiceTest(PrometheusWebTest): + def test_timer_can_be_used_as_context_manager(self): + with self.app.app.registry.metrics.timer("func.latency.context"): + self.assertEqual(my_func(1, 1), 2) + + resp = self.app.get("/__metrics__") + self.assertIn("TYPE func_latency_context summary", resp.text) + + def test_timer_can_be_used_as_decorator(self): + decorated = self.app.app.registry.metrics.timer("func.latency.decorator")(my_func) + + self.assertEqual(decorated(1, 1), 2) + + resp = self.app.get("/__metrics__") + self.assertIn("TYPE func_latency_decorator summary", resp.text) + + def test_timer_can_be_used_as_decorator_on_partial_function(self): + partial = functools.partial(my_func, 3) + decorated = self.app.app.registry.metrics.timer("func.latency.partial")(partial) + + self.assertEqual(decorated(3), 6) + + resp = self.app.get("/__metrics__") + self.assertIn("TYPE func_latency_partial summary", resp.text) + + def test_count_by_key(self): + self.app.app.registry.metrics.count("key") + + resp = self.app.get("/__metrics__") + self.assertIn("key_total 1.0", resp.text) + + def test_count_by_key_value(self): + self.app.app.registry.metrics.count("bigstep", count=2) + + resp = self.app.get("/__metrics__") + self.assertIn("bigstep_total 2.0", resp.text) + + def test_count_by_key_grouped(self): + self.app.app.registry.metrics.count("http", unique="status.500") + self.app.app.registry.metrics.count("http", unique="status.200") + + resp = self.app.get("/__metrics__") + self.assertIn('http_total{status="500"} 1.0', resp.text) + self.assertIn('http_total{status="200"} 1.0', resp.text) + + def test_count_with_generic_group(self): + self.app.app.registry.metrics.count("mushrooms", unique="boletus") + + resp = self.app.get("/__metrics__") + self.assertIn('mushrooms_total{group="boletus"} 1.0', resp.text) + + def test_metrics_cant_be_mixed(self): + self.app.app.registry.metrics.count("counter") + with self.assertRaises(RuntimeError): + self.app.app.registry.metrics.timer("counter") + + self.app.app.registry.metrics.timer("timer") + with self.assertRaises(RuntimeError): + self.app.app.registry.metrics.count("timer") + + def test_metrics_names_and_labels_are_transformed(self): + self.app.app.registry.metrics.count("http.home.status", unique="code.get.200") + + resp = self.app.get("/__metrics__") + self.assertIn('http_home_status_total{code_get="200"} 1.0', resp.text)