From f859e5195697a2d0d40060b926cad011507cfcbc Mon Sep 17 00:00:00 2001 From: Raphael Vieira Rossi Date: Wed, 27 Apr 2022 19:41:53 -0300 Subject: [PATCH] added redis sentinel support --- .github/workflows/unittest.yaml | 2 +- .pre-commit-config.yaml | 1 + Makefile | 2 +- README.md | 29 ++ docker-compose.yaml | 11 + tc_redis/base_storage.py | 182 ++++++++++ .../result_storages/redis_result_storage.py | 59 +--- tc_redis/storages/redis_storage.py | 53 +-- tests/fixtures/redis-sentinel/Dockerfile | 21 ++ .../fixtures/redis-sentinel/redis-secure.conf | 3 + tests/fixtures/redis-sentinel/redis.conf | 2 + .../redis-sentinel/sentinel-entrypoint.sh | 12 + .../redis-sentinel/sentinel-secure.conf | 13 + tests/fixtures/redis-sentinel/sentinel.conf | 10 + tests/test_redis_result_storage.py | 61 ++++ tests/test_redis_sentinel_result_storage.py | 287 ++++++++++++++++ tests/test_redis_sentinel_storage.py | 320 ++++++++++++++++++ tests/test_redis_storage.py | 60 ++++ 18 files changed, 1033 insertions(+), 95 deletions(-) create mode 100644 tc_redis/base_storage.py create mode 100644 tests/fixtures/redis-sentinel/Dockerfile create mode 100644 tests/fixtures/redis-sentinel/redis-secure.conf create mode 100644 tests/fixtures/redis-sentinel/redis.conf create mode 100644 tests/fixtures/redis-sentinel/sentinel-entrypoint.sh create mode 100644 tests/fixtures/redis-sentinel/sentinel-secure.conf create mode 100644 tests/fixtures/redis-sentinel/sentinel.conf create mode 100644 tests/test_redis_sentinel_result_storage.py create mode 100644 tests/test_redis_sentinel_storage.py diff --git a/.github/workflows/unittest.yaml b/.github/workflows/unittest.yaml index a15b98d..600398c 100644 --- a/.github/workflows/unittest.yaml +++ b/.github/workflows/unittest.yaml @@ -17,7 +17,7 @@ jobs: python-version: ${{ matrix.python-version }} architecture: x64 - name: Install apt dependencies - run: sudo apt install -y libcurl4-openssl-dev libssl-dev libjpeg-dev + run: sudo apt update && sudo apt install -y libcurl4-openssl-dev libssl-dev libjpeg-dev - name: Install pip dependencies run: make setup - run: make test diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0e4cb97..13a1b41 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,3 +10,4 @@ repos: rev: 22.1.0 hooks: - id: black + additional_dependencies: ['click==8.0.4'] diff --git a/Makefile b/Makefile index e8d69e5..01b75e0 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -REDIS_CONTAINER := redis-test +REDIS_CONTAINER := redis-test redis-sentinel-test test: run-redis unit stop-redis diff --git a/README.md b/README.md index ed544df..b932544 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ To use redis as a storage or result storage some values must be configured in `t ##### Redis Storage +###### Single Node ```python STORAGE = "tc_redis.storages.redis_storage" @@ -24,10 +25,25 @@ REDIS_STORAGE_SERVER_PORT = 6379 REDIS_STORAGE_SERVER_HOST = "localhost" REDIS_STORAGE_SERVER_DB = 0 REDIS_STORAGE_SERVER_PASSWORD = None +REDIS_STORAGE_MODE = "single_node" +``` + +###### Sentinel +```python +STORAGE = "tc_redis.storages.redis_storage" + +REDIS_STORAGE_IGNORE_ERRORS = True +REDIS_SENTINEL_STORAGE_INSTANCES = "localhost:26379,localhost:26380" +REDIS_SENTINEL_STORAGE_MASTER_INSTANCE = "redismaster" +REDIS_SENTINEL_STORAGE_MASTER_PASSWORD = "dummy" +REDIS_SENTINEL_STORAGE_PASSWORD = "dummy" +REDIS_SENTINEL_STORAGE_SOCKET_TIMEOUT = 1.0 +REDIS_STORAGE_MODE = "sentinel" ``` ##### Redis Result Storage +###### Single Node ```python RESULT_STORAGE = "tc_redis.result_storages.redis_result_storage" @@ -36,8 +52,21 @@ REDIS_RESULT_STORAGE_SERVER_PORT = 6379 REDIS_RESULT_STORAGE_SERVER_HOST = "localhost" REDIS_RESULT_STORAGE_SERVER_DB = 0 REDIS_RESULT_STORAGE_SERVER_PASSWORD = None +REDIS_RESULT_STORAGE_MODE = "single_node" ``` +###### Sentinel +```python +RESULT_STORAGE = "tc_redis.result_storages.redis_result_storage" + +REDIS_RESULT_STORAGE_IGNORE_ERRORS = True +REDIS_SENTINEL_RESULT_STORAGE_INSTANCES = "localhost:26379,localhost:26380" +REDIS_SENTINEL_RESULT_STORAGE_MASTER_INSTANCE = "redismaster" +REDIS_SENTINEL_RESULT_STORAGE_MASTER_PASSWORD = "dummy" +REDIS_SENTINEL_RESULT_STORAGE_PASSWORD = "dummy" +REDIS_SENTINEL_RESULT_STORAGE_SOCKET_TIMEOUT = 1.0 +REDIS_RESULT_STORAGE_MODE = "sentinel" +``` ## Contribute To build tc_redis locally use diff --git a/docker-compose.yaml b/docker-compose.yaml index 718a574..1c349cf 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -5,3 +5,14 @@ services: image: "redis:6.2-alpine" ports: - 6379:6379 + + redis-sentinel-test: + build: + context: ./tests/fixtures/redis-sentinel + ports: + - 6380:6380 + - 6381:6381 + - 6382:6382 + - 6383:6383 + - 26379:26379 + - 26380:26380 diff --git a/tc_redis/base_storage.py b/tc_redis/base_storage.py new file mode 100644 index 0000000..61dea05 --- /dev/null +++ b/tc_redis/base_storage.py @@ -0,0 +1,182 @@ +# -*- coding: utf-8 -*- + +# Licensed under the MIT license: +# http://www.opensource.org/licenses/mit-license +# Copyright (c) 2014 PopKey +# Copyright (c) 2022 Raphael Rossi + +from redis import Redis, Sentinel + +SINGLE_NODE = "single_node" +SENTINEL = "sentinel" + + +class RedisBaseStorage: + + single_node_storage = None + sentinel_storage = None + + def __init__(self, context, storage_type): + self.storage = None + self.context = context + self.storage_type = storage_type + + self.storage_values = { + "storage": { + "db": self.context.config.get("REDIS_STORAGE_SERVER_DB"), + "host": self.context.config.get("REDIS_STORAGE_SERVER_HOST"), + "instances": self.context.config.get( + "REDIS_SENTINEL_STORAGE_INSTANCES" + ), + "master_instance": self.context.config.get( + "REDIS_SENTINEL_STORAGE_MASTER_INSTANCE" + ), + "master_password": self.context.config.get( + "REDIS_SENTINEL_STORAGE_MASTER_PASSWORD" + ), + "password": self.context.config.get("REDIS_STORAGE_SERVER_PASSWORD"), + "port": self.context.config.get("REDIS_STORAGE_SERVER_PORT"), + "sentinel_password": self.context.config.get( + "REDIS_SENTINEL_STORAGE_PASSWORD" + ), + "socket_timeout": self.context.config.get( + "REDIS_SENTINEL_STORAGE_SOCKET_TIMEOUT", 2.0 + ), + "mode": self.context.config.get( + "REDIS_STORAGE_MODE", SINGLE_NODE + ).lower(), + }, + "result_storage": { + "db": self.context.config.get("REDIS_RESULT_STORAGE_SERVER_DB"), + "host": self.context.config.get("REDIS_RESULT_STORAGE_SERVER_HOST"), + "instances": self.context.config.get( + "REDIS_SENTINEL_RESULT_STORAGE_INSTANCES" + ), + "master_instance": self.context.config.get( + "REDIS_SENTINEL_RESULT_STORAGE_MASTER_INSTANCE" + ), + "master_password": self.context.config.get( + "REDIS_SENTINEL_RESULT_STORAGE_MASTER_PASSWORD" + ), + "password": self.context.config.get( + "REDIS_RESULT_STORAGE_SERVER_PASSWORD" + ), + "port": self.context.config.get("REDIS_RESULT_STORAGE_SERVER_PORT"), + "sentinel_password": self.context.config.get( + "REDIS_SENTINEL_RESULT_STORAGE_PASSWORD" + ), + "socket_timeout": self.context.config.get( + "REDIS_SENTINEL_RESULT_STORAGE_SOCKET_TIMEOUT", 2.0 + ), + "mode": self.context.config.get( + "REDIS_RESULT_STORAGE_MODE", SINGLE_NODE + ).lower(), + }, + } + + def get_storage(self): + """Get the storage instance. + + :return Redis: Redis instance + """ + + if self.storage: + return self.storage + self.storage = self.reconnect_redis() + + return self.storage + + def connect_redis_sentinel(self): + instances_split = self.storage_values[self.storage_type]["instances"].split(",") + instances = [tuple(instance.split(":")) for instance in instances_split] + + if self.storage_values[self.storage_type]["sentinel_password"]: + sentinel_instance = Sentinel( + instances, + socket_timeout=self.storage_values[self.storage_type]["socket_timeout"], + sentinel_kwargs={ + "password": self.storage_values[self.storage_type][ + "sentinel_password" + ] + }, + ) + else: + sentinel_instance = Sentinel( + instances, + socket_timeout=self.storage_values[self.storage_type]["socket_timeout"], + ) + + return sentinel_instance.master_for( + self.storage_values[self.storage_type]["master_instance"], + socket_timeout=self.storage_values[self.storage_type]["socket_timeout"], + password=self.storage_values[self.storage_type]["master_password"], + ) + + def connect_redis_single_node(self): + if self.storage_values[self.storage_type]["password"] is None: + return Redis( + port=self.storage_values[self.storage_type]["port"], + host=self.storage_values[self.storage_type]["host"], + db=self.storage_values[self.storage_type]["db"], + ) + + return Redis( + port=self.storage_values[self.storage_type]["port"], + host=self.storage_values[self.storage_type]["host"], + db=self.storage_values[self.storage_type]["db"], + password=self.storage_values[self.storage_type]["password"], + ) + + def reconnect_redis(self): + shared_client = self.get_shared_storage() + if shared_client: + return shared_client + + storage = self.get_storage_redis() + self.set_shared_storage(storage) + + return storage + + def get_storage_redis(self): + redis_mode = self.storage_values[self.storage_type]["mode"] + if redis_mode == SINGLE_NODE: + return self.connect_redis_single_node() + if redis_mode == SENTINEL: + return self.connect_redis_sentinel() + + if self.storage_type == "storage": + redis_mode_var = "REDIS_STORAGE_MODE" + else: + redis_mode_var = "REDIS_RESULT_STORAGE_MODE" + + raise AttributeError( + f"Unknow value for {redis_mode_var} {redis_mode}. See README for more information." + ) + + def get_shared_storage(self): + redis_mode = self.storage_values[self.storage_type]["mode"] + + if not self.shared_client: + return None + + if redis_mode == SENTINEL and RedisBaseStorage.sentinel_storage: + return RedisBaseStorage.sentinel_storage + + if redis_mode == SINGLE_NODE and RedisBaseStorage.single_node_storage: + return RedisBaseStorage.single_node_storage + + return None + + def set_shared_storage(self, storage): + redis_mode = self.storage_values[self.storage_type]["mode"] + + if not self.shared_client: + return None + + if redis_mode == SENTINEL: + RedisBaseStorage.sentinel_storage = storage + + if redis_mode == SINGLE_NODE: + RedisBaseStorage.single_node_storage = storage + + return None diff --git a/tc_redis/result_storages/redis_result_storage.py b/tc_redis/result_storages/redis_result_storage.py index 95d1c5e..31f328c 100644 --- a/tc_redis/result_storages/redis_result_storage.py +++ b/tc_redis/result_storages/redis_result_storage.py @@ -6,19 +6,21 @@ import time from datetime import datetime, timedelta -from redis import Redis, RedisError -from tc_redis.utils import on_exception + +from redis import RedisError from thumbor.result_storages import BaseStorage from thumbor.utils import logger +from tc_redis.base_storage import RedisBaseStorage +from tc_redis.utils import on_exception -class Storage(BaseStorage): - storage = None +class Storage(BaseStorage, RedisBaseStorage): """start_time is used to calculate the last modified value when an item has no expiration date. """ + start_time = None def __init__(self, context, shared_client=True): @@ -30,52 +32,13 @@ def __init__(self, context, shared_client=True): """ BaseStorage.__init__(self, context) + RedisBaseStorage.__init__(self, context, "result_storage") self.shared_client = shared_client - self.storage = self.reconnect_redis() + self.storage = self.get_storage() if not Storage.start_time: Storage.start_time = time.time() - def get_storage(self): - """Get the storage instance. - - :return Redis: Redis instance - """ - - if self.storage: - return self.storage - self.storage = self.reconnect_redis() - - return self.storage - - def reconnect_redis(self): - """Reconnect to redis. - - :return: Redis client instance - :rettype: redis.Redis - """ - - if self.shared_client and Storage.storage: - return Storage.storage - - if self.context.config.REDIS_RESULT_STORAGE_SERVER_PASSWORD is None: - storage = Redis( - port=self.context.config.REDIS_RESULT_STORAGE_SERVER_PORT, - host=self.context.config.REDIS_RESULT_STORAGE_SERVER_HOST, - db=self.context.config.REDIS_RESULT_STORAGE_SERVER_DB, - ) - else: - storage = Redis( - port=self.context.config.REDIS_RESULT_STORAGE_SERVER_PORT, - host=self.context.config.REDIS_RESULT_STORAGE_SERVER_HOST, - db=self.context.config.REDIS_RESULT_STORAGE_SERVER_DB, - password=self.context.config.REDIS_RESULT_STORAGE_SERVER_PASSWORD, - ) - - if self.shared_client: - Storage.storage = storage - return storage - def on_redis_error(self, fname, exc_type, exc_value): """Callback executed when there is a redis error. @@ -85,10 +48,8 @@ def on_redis_error(self, fname, exc_type, exc_value): :returns: Default value or raise the current exception """ - if self.shared_client: - Storage.storage = None - else: - self.storage = None + self.storage = None + self.set_shared_storage(None) if self.context.config.REDIS_RESULT_STORAGE_IGNORE_ERRORS is True: logger.error(f"Redis result storage failure: {exc_value}") diff --git a/tc_redis/storages/redis_storage.py b/tc_redis/storages/redis_storage.py index 9e7da0b..22586ce 100644 --- a/tc_redis/storages/redis_storage.py +++ b/tc_redis/storages/redis_storage.py @@ -2,16 +2,16 @@ import json from datetime import datetime, timedelta -from redis import Redis, RedisError -from tc_redis.utils import on_exception + +from redis import RedisError from thumbor.storages import BaseStorage from thumbor.utils import logger +from tc_redis.utils import on_exception +from tc_redis.base_storage import RedisBaseStorage -class Storage(BaseStorage): - - storage = None +class Storage(BaseStorage, RedisBaseStorage): def __init__(self, context, shared_client=True): """Initialize the RedisStorage @@ -21,42 +21,9 @@ def __init__(self, context, shared_client=True): """ BaseStorage.__init__(self, context) + RedisBaseStorage.__init__(self, context, "storage") self.shared_client = shared_client - self.storage = self.reconnect_redis() - - def get_storage(self): - """Get the storage instance. - - :return Redis: Redis instance - """ - - if self.storage: - return self.storage - self.storage = self.reconnect_redis() - - return self.storage - - def reconnect_redis(self): - if self.shared_client and Storage.storage: - return Storage.storage - - if self.context.config.REDIS_STORAGE_SERVER_PASSWORD is None: - storage = Redis( - port=self.context.config.REDIS_STORAGE_SERVER_PORT, - host=self.context.config.REDIS_STORAGE_SERVER_HOST, - db=self.context.config.REDIS_STORAGE_SERVER_DB, - ) - else: - storage = Redis( - port=self.context.config.REDIS_STORAGE_SERVER_PORT, - host=self.context.config.REDIS_STORAGE_SERVER_HOST, - db=self.context.config.REDIS_STORAGE_SERVER_DB, - password=self.context.config.REDIS_STORAGE_SERVER_PASSWORD, - ) - - if self.shared_client: - Storage.storage = storage - return storage + self.storage = self.get_storage() def on_redis_error(self, fname, exc_type, exc_value): """Callback executed when there is a redis error. @@ -67,10 +34,8 @@ def on_redis_error(self, fname, exc_type, exc_value): :returns: Default value or raise the current exception """ - if self.shared_client: - Storage.storage = None - else: - self.storage = None + self.storage = None + self.set_shared_storage(None) if self.context.config.REDIS_STORAGE_IGNORE_ERRORS is True: logger.error(f"[REDIS_STORAGE] {exc_value}") diff --git a/tests/fixtures/redis-sentinel/Dockerfile b/tests/fixtures/redis-sentinel/Dockerfile new file mode 100644 index 0000000..df4bf98 --- /dev/null +++ b/tests/fixtures/redis-sentinel/Dockerfile @@ -0,0 +1,21 @@ +FROM redis:6-alpine + +RUN mkdir -p /redis + +WORKDIR /redis + +COPY redis.conf . +COPY redis-secure.conf . +COPY sentinel.conf . +COPY sentinel-secure.conf . +COPY sentinel-entrypoint.sh /usr/local/bin/ + +RUN chown redis:redis /redis/* && \ + chmod +x /usr/local/bin/sentinel-entrypoint.sh + +EXPOSE 6379 +EXPOSE 6380 +EXPOSE 26379 +EXPOSE 26380 + +ENTRYPOINT ["sentinel-entrypoint.sh"] diff --git a/tests/fixtures/redis-sentinel/redis-secure.conf b/tests/fixtures/redis-sentinel/redis-secure.conf new file mode 100644 index 0000000..a628ace --- /dev/null +++ b/tests/fixtures/redis-sentinel/redis-secure.conf @@ -0,0 +1,3 @@ +daemonize yes +port 6380 +requirepass superpassword diff --git a/tests/fixtures/redis-sentinel/redis.conf b/tests/fixtures/redis-sentinel/redis.conf new file mode 100644 index 0000000..5959545 --- /dev/null +++ b/tests/fixtures/redis-sentinel/redis.conf @@ -0,0 +1,2 @@ +daemonize yes +port 6379 diff --git a/tests/fixtures/redis-sentinel/sentinel-entrypoint.sh b/tests/fixtures/redis-sentinel/sentinel-entrypoint.sh new file mode 100644 index 0000000..07be2bb --- /dev/null +++ b/tests/fixtures/redis-sentinel/sentinel-entrypoint.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +# Master +redis-server /redis/redis.conf +redis-server /redis/redis-secure.conf + +# Sentinel +redis-server /redis/sentinel.conf --sentinel +redis-server /redis/sentinel-secure.conf --sentinel + +sleep infinity +wait -n diff --git a/tests/fixtures/redis-sentinel/sentinel-secure.conf b/tests/fixtures/redis-sentinel/sentinel-secure.conf new file mode 100644 index 0000000..185d636 --- /dev/null +++ b/tests/fixtures/redis-sentinel/sentinel-secure.conf @@ -0,0 +1,13 @@ +daemonize yes +port 26380 + +dir /tmp + +requirepass superpassword +sentinel resolve-hostnames yes +sentinel announce-hostnames yes +sentinel monitor masterinstance localhost 6380 2 +sentinel down-after-milliseconds masterinstance 1000 +sentinel parallel-syncs masterinstance 1 +sentinel auth-pass masterinstance superpassword +sentinel failover-timeout masterinstance 1000 diff --git a/tests/fixtures/redis-sentinel/sentinel.conf b/tests/fixtures/redis-sentinel/sentinel.conf new file mode 100644 index 0000000..5ec6be1 --- /dev/null +++ b/tests/fixtures/redis-sentinel/sentinel.conf @@ -0,0 +1,10 @@ +daemonize yes +port 26379 + +dir /tmp + +sentinel resolve-hostnames yes +sentinel monitor masterinstance localhost 6379 2 +sentinel down-after-milliseconds masterinstance 1000 +sentinel parallel-syncs masterinstance 1 +sentinel failover-timeout masterinstance 1000 diff --git a/tests/test_redis_result_storage.py b/tests/test_redis_result_storage.py index f7479dc..05dee73 100644 --- a/tests/test_redis_result_storage.py +++ b/tests/test_redis_result_storage.py @@ -40,6 +40,11 @@ def setUp(self): ) self.storage = RedisStorage(self.ctx) + def test_should_be_instance_of_single_node(self): + expect(str(self.storage.get_storage())).to_equal( + "Redis>>" + ) + class CanStoreImage(RedisDBContext): def setUp(self): @@ -212,3 +217,59 @@ async def test_should_not_throw_an_exception(self): topic = await storage.get() expect(topic).to_equal(None) expect(topic).not_to_be_an_error() + + +class ConnectToRedisWithoutPassword(RedisDBContext): + def setUp(self): + super().setUp() + + self.cfg = Config( + REDIS_RESULT_STORAGE_SERVER_HOST="localhost", + REDIS_RESULT_STORAGE_SERVER_PORT=6379, + REDIS_RESULT_STORAGE_SERVER_DB=0, + ) + self.ctx = Context( + config=self.cfg, + server=get_server("ACME-SEC"), + ) + self.ctx.request = RequestParameters( + url=IMAGE_URL % 2, + ) + self.storage = RedisStorage(self.ctx) + + @pytest.mark.asyncio + async def test_should_be_in_catalog(self): + await self.storage.put(IMAGE_BYTES) + + topic = self.connection.get(f"result:{IMAGE_URL % 2}") + + expect(topic).not_to_be_null() + expect(topic).not_to_be_an_error() + + +class RedisModeInvalid(RedisDBContext): + def setUp(self): + super().setUp() + + self.cfg = Config( + REDIS_RESULT_STORAGE_SERVER_HOST="localhost", + REDIS_RESULT_STORAGE_SERVER_PORT=6379, + REDIS_RESULT_STORAGE_SERVER_DB=0, + REDIS_RESULT_STORAGE_MODE="test", + ) + self.ctx = Context( + config=self.cfg, + server=get_server("ACME-SEC"), + ) + self.ctx.request = RequestParameters( + url=IMAGE_URL % 2, + ) + + @pytest.mark.asyncio + async def test_should_raises_attribute_error(self): + with self.assertRaises(AttributeError) as error: + RedisStorage(self.ctx) + + expect(str(error.exception)).to_equal( + "Unknow value for REDIS_RESULT_STORAGE_MODE test. See README for more information." + ) diff --git a/tests/test_redis_sentinel_result_storage.py b/tests/test_redis_sentinel_result_storage.py new file mode 100644 index 0000000..812c219 --- /dev/null +++ b/tests/test_redis_sentinel_result_storage.py @@ -0,0 +1,287 @@ +# -*- coding: utf-8 -*- + +# thumbor imaging service +# https://github.com/globocom/thumbor/wiki + +# Licensed under the MIT license: +# http://www.opensource.org/licenses/mit-license +# Copyright (c) 2011 globo.com timehome@corp.globo.com + +from unittest import IsolatedAsyncioTestCase +from datetime import datetime, timedelta + +import redis +import pytest +from preggy import expect +from thumbor.context import Context, RequestParameters +from thumbor.config import Config + + +from tc_redis.result_storages.redis_result_storage import Storage as RedisStorage +from tests.fixtures.storage_fixtures import IMAGE_URL, IMAGE_BYTES, get_server + + +class RedisDBContext(IsolatedAsyncioTestCase): + def setUp(self): + self.sentinel = redis.Sentinel( + [("localhost", 26380)], + socket_timeout=1, + sentinel_kwargs={"password": "superpassword"}, + ) + self.connection = self.sentinel.master_for( + "masterinstance", socket_timeout=1, password="superpassword" + ) + self.cfg = Config( + REDIS_SENTINEL_RESULT_STORAGE_INSTANCES="localhost:26380", + REDIS_SENTINEL_RESULT_STORAGE_MASTER_INSTANCE="masterinstance", + REDIS_SENTINEL_RESULT_STORAGE_MASTER_PASSWORD="superpassword", + REDIS_SENTINEL_RESULT_STORAGE_PASSWORD="superpassword", + REDIS_RESULT_STORAGE_MODE="sentinel", + RESULT_STORAGE_EXPIRATION_SECONDS=60000, + ) + self.ctx = Context( + config=self.cfg, + server=get_server("ACME-SEC"), + ) + self.ctx.request = RequestParameters( + url=IMAGE_URL % 2, + ) + self.storage = RedisStorage(self.ctx) + + def test_should_be_instance_of_single_node(self): + expect(str(self.storage.get_storage())).to_equal( + "Redis" + ) + + +class CanStoreImage(RedisDBContext): + def setUp(self): + super().setUp() + + @pytest.mark.asyncio + async def test_should_be_in_catalog(self): + await self.storage.put(IMAGE_BYTES) + + topic = self.connection.get(f"result:{IMAGE_URL % 2}") + + expect(topic).not_to_be_null() + expect(topic).to_equal(IMAGE_BYTES) + + +class KnowsImageDoesNotExist(RedisDBContext): + @pytest.mark.asyncio + async def test_should_not_exist(self): + self.ctx.request.url = IMAGE_URL % 10000 + topic = await self.storage.get() + expect(topic).to_be_null() + expect(topic).not_to_be_an_error() + + +class GetMaxAgeFromRedisEnv(RedisDBContext): + def test_should_get_max_age_from_redis(self): + self.ctx.request.max_age = 10 + topic = self.storage.get_max_age() + expect(topic).not_to_be_null() + expect(topic).to_equal(self.cfg.RESULT_STORAGE_EXPIRATION_SECONDS) + + +class GetMaxAgeFromRequest(RedisDBContext): + def test_should_get_max_age_from_request(self): + max_age = 0 + self.ctx.request.max_age = max_age + topic = self.storage.get_max_age() + expect(topic).not_to_be_null() + expect(topic).to_equal(max_age) + + +class GetKeyFromRequest(RedisDBContext): + def test_should_get_key_from_request(self): + topic = self.storage.get_key_from_request() + expect(topic).not_to_be_null() + expect(topic).to_equal(f"result:{self.ctx.request.url}") + + +class GetWebpKeyFromRequest(RedisDBContext): + def setUp(self): + super().setUp() + + self.ctx.config.AUTO_WEBP = True + self.ctx.request.accepts_webp = True + + def test_should_get_webp_from_request(self): + topic = self.storage.get_key_from_request() + expect(topic).not_to_be_null() + expect(topic).to_equal(f"result:{self.ctx.request.url}/webp") + + +class CanGetAutoWebp(RedisDBContext): + def setUp(self): + super().setUp() + + self.ctx.config.AUTO_WEBP = True + self.ctx.request.accepts_webp = True + + def test_should_get_auto_webp(self): + topic = self.storage.get_key_from_request() + expect(topic).not_to_be_null() + expect(topic).to_equal(f"result:{self.ctx.request.url}/webp") + + +class CanNotGetAutoWebp(RedisDBContext): + def setUp(self): + super().setUp() + + self.ctx.config.AUTO_WEBP = True + self.ctx.request.accepts_webp = False + + def test_should_not_get_auto_webp(self): + topic = self.storage.get_key_from_request() + expect(topic).not_to_be_null() + expect(topic).to_equal(f"result:{self.ctx.request.url}") + + +class CanReadLastUpdatedFromStorage(RedisDBContext): + def test_should_get_last_updated_from_storage(self): + self.ctx.request.max_age = 0 + topic = self.storage.last_updated() + expect(topic).not_to_be_null() + expect(topic).to_equal(datetime.fromtimestamp(self.storage.start_time)) + + +class CanReadLastUpdatedFromImage(RedisDBContext): + def test_should_get_last_updated_from_image(self): + self.connection.set("test_last_updated", IMAGE_BYTES) + self.connection.expireat( + "test_last_updated", datetime.now() + timedelta(seconds=1000) + ) + self.ctx.request.max_age = 10 + + topic = self.storage.last_updated() + expect(topic).not_to_be_null() + expect(topic).to_be_lesser_than(datetime.now()) + + +class CanReadUnexpiryLastUpdated(RedisDBContext): + def test_should_get_unexpire_last_updated(self): + self.connection.set("test_last_updated", IMAGE_BYTES) + self.ctx.request.max_age = 10 + ttl = self.connection.ttl("test_last_updated") + topic = self.storage.last_updated() + expect(topic).not_to_be_null() + expect(ttl).to_equal(-1) + + +class CanRaiseErrors(RedisDBContext): + @pytest.mark.asyncio + async def test_should_throw_an_exception(self): + config = Config( + REDIS_SENTINEL_RESULT_STORAGE_INSTANCES="localhost:300", + REDIS_SENTINEL_RESULT_STORAGE_MASTER_INSTANCE="masterinstance", + REDIS_SENTINEL_RESULT_STORAGE_MASTER_PASSWORD="superpassword", + REDIS_SENTINEL_RESULT_STORAGE_PASSWORD="superpassword", + REDIS_RESULT_STORAGE_MODE="sentinel", + REDIS_RESULT_STORAGE_IGNORE_ERRORS=False, + ) + ctx = Context( + config=config, + server=get_server("ACME-SEC"), + ) + ctx.request = RequestParameters( + url=IMAGE_URL, + ) + storage = RedisStorage( + context=ctx, + shared_client=False, + ) + + try: + topic = await storage.get() + except Exception as redis_error: + expect(redis_error).not_to_be_null() + expect(redis_error).to_be_an_error_like(redis.RedisError) + + +class CanIgnoreErrors(RedisDBContext): + @pytest.mark.asyncio + async def test_should_not_throw_an_exception(self): + cfg = Config( + REDIS_SENTINEL_RESULT_STORAGE_INSTANCES="localhost:300", + REDIS_SENTINEL_RESULT_STORAGE_MASTER_INSTANCE="masterinstance", + REDIS_SENTINEL_RESULT_STORAGE_MASTER_PASSWORD="superpassword", + REDIS_SENTINEL_RESULT_STORAGE_PASSWORD="superpassword", + REDIS_RESULT_STORAGE_MODE="sentinel", + REDIS_RESULT_STORAGE_IGNORE_ERRORS=True, + ) + ctx = Context( + config=cfg, + server=get_server("ACME-SEC"), + ) + ctx.request = RequestParameters( + url=IMAGE_URL % 10, + ) + storage = RedisStorage( + context=ctx, + shared_client=False, + ) + + topic = await storage.get() + expect(topic).to_equal(None) + expect(topic).not_to_be_an_error() + + +class ConnectToRedisWithoutPassword(RedisDBContext): + def setUp(self): + super().setUp() + + self.sentinel = redis.Sentinel([("localhost", 26379)], socket_timeout=1) + self.connection = self.sentinel.master_for("masterinstance", socket_timeout=1) + + self.cfg = Config( + REDIS_SENTINEL_RESULT_STORAGE_INSTANCES="localhost:26379", + REDIS_SENTINEL_RESULT_STORAGE_MASTER_INSTANCE="masterinstance", + REDIS_SENTINEL_RESULT_STORAGE_MASTER_PASSWORD="superpassword", + REDIS_RESULT_STORAGE_MODE="sentinel", + ) + + self.ctx = Context( + config=self.cfg, + server=get_server("ACME-SEC"), + ) + self.ctx.request = RequestParameters( + url=IMAGE_URL % 2, + ) + self.storage = RedisStorage(self.ctx) + + @pytest.mark.asyncio + async def test_should_be_in_catalog(self): + await self.storage.put(IMAGE_BYTES) + + topic = self.connection.get(f"result:{IMAGE_URL % 2}") + + expect(topic).not_to_be_null() + expect(topic).not_to_be_an_error() + + +class RedisModeInvalid(RedisDBContext): + def setUp(self): + super().setUp() + + self.cfg = Config( + REDIS_SENTINEL_RESULT_STORAGE_SERVER_HOST="localhost", + REDIS_SENTINEL_RESULT_STORAGE_SERVER_PORT=6379, + REDIS_SENTINEL_RESULT_STORAGE_SERVER_DB=0, + REDIS_RESULT_STORAGE_MODE="test", + ) + self.ctx = Context( + config=self.cfg, + server=get_server("ACME-SEC"), + ) + + @pytest.mark.asyncio + async def test_should_raises_attribute_error(self): + with self.assertRaises(AttributeError) as error: + RedisStorage(self.ctx) + + expect(str(error.exception)).to_equal( + "Unknow value for REDIS_RESULT_STORAGE_MODE test. See README for more information." + ) diff --git a/tests/test_redis_sentinel_storage.py b/tests/test_redis_sentinel_storage.py new file mode 100644 index 0000000..e666987 --- /dev/null +++ b/tests/test_redis_sentinel_storage.py @@ -0,0 +1,320 @@ +# -*- coding: utf-8 -*- + +# thumbor imaging service +# https://github.com/globocom/thumbor/wiki + +# Licensed under the MIT license: +# http://www.opensource.org/licenses/mit-license +# Copyright (c) 2011 globo.com timehome@corp.globo.com + +from cmath import exp +from socket import getnameinfo +from unittest import IsolatedAsyncioTestCase + +import redis +import pytest +from preggy import expect +from thumbor.context import Context +from thumbor.config import Config + + +from tc_redis.storages.redis_storage import Storage as RedisStorage +from tests.fixtures.storage_fixtures import IMAGE_URL, IMAGE_BYTES, get_server + + +class RedisDBContext(IsolatedAsyncioTestCase): + def setUp(self): + self.sentinel = redis.Sentinel( + [("localhost", 26380)], + socket_timeout=1, + sentinel_kwargs={"password": "superpassword"}, + ) + self.connection = self.sentinel.master_for( + "masterinstance", socket_timeout=1, password="superpassword" + ) + self.cfg = Config( + REDIS_SENTINEL_STORAGE_INSTANCES="localhost:26380", + REDIS_SENTINEL_STORAGE_MASTER_INSTANCE="masterinstance", + REDIS_SENTINEL_STORAGE_MASTER_PASSWORD="superpassword", + REDIS_SENTINEL_STORAGE_PASSWORD="superpassword", + REDIS_STORAGE_MODE="sentinel", + ) + self.storage = RedisStorage( + Context(config=self.cfg, server=get_server("ACME-SEC")) + ) + + def test_should_be_instance_of_sentinel(self): + expect(str(self.storage.get_storage())).to_equal( + "Redis" + ) + + +class CanStoreImage(RedisDBContext): + @pytest.mark.asyncio + async def test_should_be_in_catalog(self): + await self.storage.put(IMAGE_URL % 1, IMAGE_BYTES) + + topic = self.connection.get(IMAGE_URL % 1) + + expect(topic).not_to_be_null() + expect(topic).not_to_be_an_error() + + +class KnowsImageExists(RedisDBContext): + @pytest.mark.asyncio + async def test_should_exist(self): + await self.storage.put(IMAGE_URL % 9999, IMAGE_BYTES) + topic = await self.storage.exists(IMAGE_URL % 9999) + expect(topic).not_to_be_null() + expect(topic).not_to_be_an_error() + expect(topic).to_be_true() + + +class KnowsImageDoesNotExist(RedisDBContext): + @pytest.mark.asyncio + async def test_should_not_exist(self): + topic = await self.storage.exists(IMAGE_URL % 10000) + expect(topic).not_to_be_null() + expect(topic).not_to_be_an_error() + expect(topic).to_be_false() + + +class CanRemoveImage(RedisDBContext): + @pytest.mark.asyncio + async def test_should_not_be_in_catalog(self): + await self.storage.put(IMAGE_URL % 10001, IMAGE_BYTES) + await self.storage.remove(IMAGE_URL % 10001) + topic = self.connection.get(IMAGE_URL % 10001) + expect(topic).not_to_be_an_error() + expect(topic).to_be_null() + + +class CanReRemoveImage(RedisDBContext): + @pytest.mark.asyncio + async def test_should_not_be_in_catalog(self): + await self.storage.remove(IMAGE_URL % 10001) + topic = self.connection.get(IMAGE_URL % 10001) + expect(topic).not_to_be_an_error() + expect(topic).to_be_null() + + +class CanGetImage(RedisDBContext): + @pytest.mark.asyncio + async def test_should_not_be_null(self): + await self.storage.put(IMAGE_URL % 2, IMAGE_BYTES) + topic = await self.storage.get(IMAGE_URL % 2) + expect(topic).not_to_be_null() + expect(topic).not_to_be_an_error() + + @pytest.mark.asyncio + async def test_should_have_proper_bytes(self): + await self.storage.put(IMAGE_URL % 2, IMAGE_BYTES) + topic = await self.storage.get(IMAGE_URL % 2) + expect(topic).to_equal(IMAGE_BYTES) + + +class CanRaiseErrors(RedisDBContext): + @pytest.mark.asyncio + async def test_should_throw_an_exception(self): + config = Config( + REDIS_SENTINEL_STORAGE_INSTANCES="localhost:379", + REDIS_SENTINEL_STORAGE_MASTER_INSTANCE="masterinstance", + REDIS_SENTINEL_STORAGE_MASTER_PASSWORD="superpassword", + REDIS_STORAGE_MODE="sentinel", + REDIS_STORAGE_IGNORE_ERRORS=False, + ) + storage = RedisStorage( + context=Context(config=config, server=get_server("ACME-SEC")), + shared_client=False, + ) + + try: + topic = await storage.exists(IMAGE_URL % 2) + except Exception as redis_error: + expect(redis_error).not_to_be_null() + expect(redis_error).to_be_an_error_like(redis.RedisError) + + +class IgnoreErrors(RedisDBContext): + def setUp(self): + super().setUp() + + self.cfg = Config( + REDIS_SENTINEL_STORAGE_INSTANCES="localhost:379", + REDIS_SENTINEL_STORAGE_MASTER_INSTANCE="masterinstance", + REDIS_SENTINEL_STORAGE_MASTER_PASSWORD="superpassword", + REDIS_STORAGE_MODE="sentinel", + REDIS_STORAGE_IGNORE_ERRORS=True, + ) + + self.storage = RedisStorage( + context=Context(config=self.cfg, server=get_server("ACME-SEC")), + shared_client=False, + ) + + @pytest.mark.asyncio + async def test_should_return_false(self): + result = await self.storage.exists(IMAGE_URL % 2) + expect(result).to_equal(False) + expect(result).not_to_be_an_error() + + @pytest.mark.asyncio + async def test_should_return_none(self): + result = await self.storage.get(IMAGE_URL % 2) + expect(result).to_equal(None) + expect(result).not_to_be_an_error() + + +class RaisesIfInvalidConfig(RedisDBContext): + @pytest.mark.asyncio + async def test_should_be_an_error(self): + config = Config( + REDIS_SENTINEL_STORAGE_INSTANCES="localhost:26380", + REDIS_SENTINEL_STORAGE_MASTER_INSTANCE="masterinstance", + REDIS_SENTINEL_STORAGE_MASTER_PASSWORD="superpassword", + REDIS_STORAGE_MODE="sentinel", + STORES_CRYPTO_KEY_FOR_EACH_IMAGE=True, + ) + storage = RedisStorage(Context(config=config, server=get_server(""))) + await storage.put(IMAGE_URL % 3, IMAGE_BYTES) + + try: + await storage.put_crypto(IMAGE_URL % 3) + except Exception as error: + expect(error).to_be_an_error_like(RuntimeError) + expect(error).to_have_an_error_message_of( + "STORES_CRYPTO_KEY_FOR_EACH_IMAGE can't be True if no " + "SECURITY_KEY specified" + ) + + +class GettingCryptoForANewImageReturnsNone(RedisDBContext): + @pytest.mark.asyncio + async def test_should_be_null(self): + config = Config( + REDIS_SENTINEL_STORAGE_INSTANCES="localhost:26380", + REDIS_SENTINEL_STORAGE_MASTER_INSTANCE="masterinstance", + REDIS_SENTINEL_STORAGE_MASTER_PASSWORD="superpassword", + REDIS_STORAGE_MODE="sentinel", + STORES_CRYPTO_KEY_FOR_EACH_IMAGE=True, + ) + storage = RedisStorage(Context(config=config, server=get_server("ACME-SEC"))) + + topic = await storage.get_crypto(IMAGE_URL % 9999) + expect(topic).to_be_null() + + +class DoesNotStoreIfConfigSaysNotTo(RedisDBContext): + @pytest.mark.asyncio + async def test_should_be_null(self): + await self.storage.put(IMAGE_URL % 5, IMAGE_BYTES) + await self.storage.put_crypto(IMAGE_URL % 5) + topic = await self.storage.get_crypto(IMAGE_URL % 5) + expect(topic).to_be_null() + + +class CanStoreCrypto(RedisDBContext): + def setUp(self): + super().setUp() + self.cfg = Config( + REDIS_SENTINEL_STORAGE_INSTANCES="localhost:26380", + REDIS_SENTINEL_STORAGE_MASTER_INSTANCE="masterinstance", + REDIS_SENTINEL_STORAGE_MASTER_PASSWORD="superpassword", + REDIS_STORAGE_MODE="sentinel", + STORES_CRYPTO_KEY_FOR_EACH_IMAGE=True, + ) + self.storage = RedisStorage( + Context(config=self.cfg, server=get_server("ACME-SEC")) + ) + + @pytest.mark.asyncio + async def test_should_not_be_null(self): + await self.storage.put(IMAGE_URL % 6, IMAGE_BYTES) + await self.storage.put_crypto(IMAGE_URL % 6) + topic = await self.storage.get_crypto(IMAGE_URL % 6) + expect(topic).not_to_be_null() + expect(topic).not_to_be_an_error() + + @pytest.mark.asyncio + async def test_should_have_proper_key(self): + await self.storage.put(IMAGE_URL % 6, IMAGE_BYTES) + await self.storage.put_crypto(IMAGE_URL % 6) + topic = await self.storage.get_crypto(IMAGE_URL % 6) + expect(topic).to_equal("ACME-SEC") + + +class CanStoreDetectorData(RedisDBContext): + @pytest.mark.asyncio + async def test_should_not_be_null(self): + await self.storage.put(IMAGE_URL % 7, IMAGE_BYTES) + await self.storage.put_detector_data(IMAGE_URL % 7, "some-data") + topic = await self.storage.get_detector_data(IMAGE_URL % 7) + expect(topic).not_to_be_null() + expect(topic).not_to_be_an_error() + + @pytest.mark.asyncio + async def test_should_equal_some_data(self): + await self.storage.put(IMAGE_URL % 7, IMAGE_BYTES) + await self.storage.put_detector_data(IMAGE_URL % 7, "some-data") + topic = await self.storage.get_detector_data(IMAGE_URL % 7) + expect(topic).to_equal("some-data") + + +class ReturnsNoneIfNoDetectorData(RedisDBContext): + @pytest.mark.asyncio + async def test_should_not_be_null(self): + topic = await self.storage.get_detector_data(IMAGE_URL % 10000) + expect(topic).to_be_null() + + +class ConnectToRedisWithoutPassword(RedisDBContext): + def setUp(self): + super().setUp() + + self.sentinel = redis.Sentinel([("localhost", 26379)], socket_timeout=1) + self.connection = self.sentinel.master_for("masterinstance", socket_timeout=1) + + self.cfg = Config( + REDIS_SENTINEL_STORAGE_INSTANCES="localhost:26379", + REDIS_SENTINEL_STORAGE_MASTER_INSTANCE="masterinstance", + REDIS_STORAGE_MODE="sentinel", + ) + + self.storage = RedisStorage( + context=Context(config=self.cfg, server=get_server("ACME-SEC")), + shared_client=False, + ) + + @pytest.mark.asyncio + async def test_should_be_in_catalog(self): + await self.storage.put(IMAGE_URL % 1, IMAGE_BYTES) + + topic = self.connection.get(IMAGE_URL % 1) + + expect(topic).not_to_be_null() + expect(topic).not_to_be_an_error() + + +class RedisModeInvalid(RedisDBContext): + def setUp(self): + super().setUp() + + self.cfg = Config( + REDIS_SENTINEL_STORAGE_SERVER_HOST="localhost", + REDIS_SENTINEL_STORAGE_SERVER_PORT=6379, + REDIS_SENTINEL_STORAGE_SERVER_DB=0, + REDIS_STORAGE_MODE="test", + ) + self.ctx = Context( + config=self.cfg, + server=get_server("ACME-SEC"), + ) + + @pytest.mark.asyncio + async def test_should_raises_attribute_error(self): + with self.assertRaises(AttributeError) as error: + RedisStorage(self.ctx) + + expect(str(error.exception)).to_equal( + "Unknow value for REDIS_STORAGE_MODE test. See README for more information." + ) diff --git a/tests/test_redis_storage.py b/tests/test_redis_storage.py index 3b3b4bf..238f8d8 100644 --- a/tests/test_redis_storage.py +++ b/tests/test_redis_storage.py @@ -35,6 +35,11 @@ def setUp(self): Context(config=self.cfg, server=get_server("ACME-SEC")) ) + def test_should_be_instance_of_single_node(self): + expect(str(self.storage.get_storage())).to_equal( + "Redis>>" + ) + class CanStoreImage(RedisDBContext): @pytest.mark.asyncio @@ -139,6 +144,11 @@ def setUp(self): shared_client=False, ) + def test_should_be_instance_of_single_node(self): + expect(str(self.storage.get_storage())).to_equal( + "Redis>>" + ) + @pytest.mark.asyncio async def test_should_return_false(self): result = await self.storage.exists(IMAGE_URL % 2) @@ -252,3 +262,53 @@ class ReturnsNoneIfNoDetectorData(RedisDBContext): async def test_should_not_be_null(self): topic = await self.storage.get_detector_data(IMAGE_URL % 10000) expect(topic).to_be_null() + + +class ConnectToRedisWithoutPassword(RedisDBContext): + def setUp(self): + super().setUp() + + self.cfg = Config( + REDIS_STORAGE_SERVER_HOST="localhost", + REDIS_STORAGE_SERVER_PORT=6379, + REDIS_STORAGE_SERVER_DB=0, + ) + + self.storage = RedisStorage( + context=Context(config=self.cfg, server=get_server("ACME-SEC")), + shared_client=False, + ) + + @pytest.mark.asyncio + async def test_should_be_in_catalog(self): + await self.storage.put(IMAGE_URL % 1, IMAGE_BYTES) + + topic = self.connection.get(IMAGE_URL % 1) + + expect(topic).not_to_be_null() + expect(topic).not_to_be_an_error() + + +class RedisModeInvalid(RedisDBContext): + def setUp(self): + super().setUp() + + self.cfg = Config( + REDIS_STORAGE_SERVER_HOST="localhost", + REDIS_STORAGE_SERVER_PORT=6379, + REDIS_STORAGE_SERVER_DB=0, + REDIS_STORAGE_MODE="test", + ) + self.ctx = Context( + config=self.cfg, + server=get_server("ACME-SEC"), + ) + + @pytest.mark.asyncio + async def test_should_raises_attribute_error(self): + with self.assertRaises(AttributeError) as error: + RedisStorage(self.ctx) + + expect(str(error.exception)).to_equal( + "Unknow value for REDIS_STORAGE_MODE test. See README for more information." + )