From 6f39e77e78ff9f351a0a3c097415984af83c63a6 Mon Sep 17 00:00:00 2001 From: Manilde van der Oord Date: Thu, 2 May 2024 17:01:59 +0200 Subject: [PATCH 1/3] refactor in apps --- .../management => auth_mail}/__init__.py | 0 src/{iiif => auth_mail}/authentication.py | 2 +- src/{iiif => auth_mail}/generate_token.py | 0 src/{iiif => auth_mail}/mailing.py | 0 .../templates/login_link.html | 0 src/auth_mail/urls.py | 11 ++ src/auth_mail/views.py | 45 ++++++++ src/iiif/image_handling.py | 2 +- src/iiif/image_server.py | 4 +- src/iiif/metadata.py | 2 +- src/iiif/parsing.py | 2 +- src/iiif/urls.py | 8 +- src/iiif/views.py | 101 +----------------- src/main/settings.py | 13 ++- src/main/urls.py | 2 + src/{iiif => main}/utils.py | 0 src/{iiif => main}/utils_azure.py | 0 .../commands => zip_consumer}/__init__.py | 0 src/zip_consumer/management/__init__.py | 0 .../management/commands/__init__.py | 0 .../management/commands/consume_zips.py | 2 +- .../queue_zip_consumer.py | 7 +- .../templates/download_zip.html | 0 src/zip_consumer/urls.py | 7 ++ src/zip_consumer/views.py | 73 +++++++++++++ src/{iiif => zip_consumer}/zip_tools.py | 2 +- tests/test_iiif.py | 8 +- tests/test_image_handling.py | 2 +- tests/test_utils.py | 6 +- tests/test_utils_azure.py | 2 +- tests/test_zip_endpoint.py | 16 +-- 31 files changed, 183 insertions(+), 134 deletions(-) rename src/{iiif/management => auth_mail}/__init__.py (100%) rename src/{iiif => auth_mail}/authentication.py (99%) rename src/{iiif => auth_mail}/generate_token.py (100%) rename src/{iiif => auth_mail}/mailing.py (100%) rename src/{iiif => auth_mail}/templates/login_link.html (100%) create mode 100644 src/auth_mail/urls.py create mode 100644 src/auth_mail/views.py rename src/{iiif => main}/utils.py (100%) rename src/{iiif => main}/utils_azure.py (100%) rename src/{iiif/management/commands => zip_consumer}/__init__.py (100%) create mode 100644 src/zip_consumer/management/__init__.py create mode 100644 src/zip_consumer/management/commands/__init__.py rename src/{iiif => zip_consumer}/management/commands/consume_zips.py (86%) rename src/{iiif => zip_consumer}/queue_zip_consumer.py (96%) rename src/{iiif => zip_consumer}/templates/download_zip.html (100%) create mode 100644 src/zip_consumer/urls.py create mode 100644 src/zip_consumer/views.py rename src/{iiif => zip_consumer}/zip_tools.py (96%) diff --git a/src/iiif/management/__init__.py b/src/auth_mail/__init__.py similarity index 100% rename from src/iiif/management/__init__.py rename to src/auth_mail/__init__.py diff --git a/src/iiif/authentication.py b/src/auth_mail/authentication.py similarity index 99% rename from src/iiif/authentication.py rename to src/auth_mail/authentication.py index b8aad3a..ef1a2f0 100644 --- a/src/iiif/authentication.py +++ b/src/auth_mail/authentication.py @@ -6,7 +6,7 @@ from django.http import HttpResponse from jwt.exceptions import DecodeError, ExpiredSignatureError, InvalidSignatureError -from iiif.utils import ImmediateHttpResponse +from main.utils import ImmediateHttpResponse log = logging.getLogger(__name__) diff --git a/src/iiif/generate_token.py b/src/auth_mail/generate_token.py similarity index 100% rename from src/iiif/generate_token.py rename to src/auth_mail/generate_token.py diff --git a/src/iiif/mailing.py b/src/auth_mail/mailing.py similarity index 100% rename from src/iiif/mailing.py rename to src/auth_mail/mailing.py diff --git a/src/iiif/templates/login_link.html b/src/auth_mail/templates/login_link.html similarity index 100% rename from src/iiif/templates/login_link.html rename to src/auth_mail/templates/login_link.html diff --git a/src/auth_mail/urls.py b/src/auth_mail/urls.py new file mode 100644 index 0000000..496c33c --- /dev/null +++ b/src/auth_mail/urls.py @@ -0,0 +1,11 @@ +from django.urls import path + +from auth_mail import views + +urlpatterns = [ + path( + "login-link-to-email/", + views.send_dataportaal_login_url_to_mail, + name="login_link_to_email_endpoint", + ), +] diff --git a/src/auth_mail/views.py b/src/auth_mail/views.py new file mode 100644 index 0000000..dcb24b3 --- /dev/null +++ b/src/auth_mail/views.py @@ -0,0 +1,45 @@ +import logging + +from django.conf import settings +from django.http import HttpResponse +from django.template.loader import render_to_string +from django.views.decorators.csrf import csrf_exempt +from django_ratelimit.decorators import ratelimit + +from auth_mail import authentication, mailing +from iiif import parsing +from main import utils + +log = logging.getLogger(__name__) + + +# TODO: limit to dataportaal urls +@csrf_exempt +@ratelimit( + key="ip", rate="3/d", block=False +) # TODO: Check django cache settings for rate limiter to work across parallel docker containers +def send_dataportaal_login_url_to_mail(request): + try: + # Some basic sanity checks + parsing.check_for_post(request) + payload = parsing.parse_payload(request) + email, origin_url = parsing.check_login_url_payload(payload) + parsing.check_email_validity(email) + + # Create the login url + token = authentication.create_mail_login_token(email, settings.JWT_SECRET_KEY) + login_url = origin_url + "?auth=" + token + + # Send the email + email_subject = "Toegang bouw- en omgevingsdossiers data.amsterdam.nl" + email_body = render_to_string("login_link.html", {"login_url": login_url}) + # TODO: move actually sending the email to a separate process + mailing.send_email(payload["email"], email_subject, email_body) + + except utils.ImmediateHttpResponse as e: + log.exception("ImmediateHttpResponse in login_url:") + return e.response + + # Respond with a 200 to signal success. + # The user will get an email asap with a login url + return HttpResponse() diff --git a/src/iiif/image_handling.py b/src/iiif/image_handling.py index 006605b..2a8d630 100644 --- a/src/iiif/image_handling.py +++ b/src/iiif/image_handling.py @@ -6,7 +6,7 @@ from django.http import HttpResponse from PIL import Image -from iiif import utils +from main import utils # Allow larger images too be processed. We can do this because we trust the source of the images Image.MAX_IMAGE_PIXELS = None diff --git a/src/iiif/image_server.py b/src/iiif/image_server.py index 0b5b738..7ec91a9 100644 --- a/src/iiif/image_server.py +++ b/src/iiif/image_server.py @@ -6,8 +6,8 @@ from django.http import HttpResponse from requests.exceptions import RequestException -from iiif import zip_tools -from iiif.utils import ImmediateHttpResponse +from main.utils import ImmediateHttpResponse +from zip_consumer import zip_tools log = logging.getLogger(__name__) diff --git a/src/iiif/metadata.py b/src/iiif/metadata.py index b6b84f6..876461c 100644 --- a/src/iiif/metadata.py +++ b/src/iiif/metadata.py @@ -5,7 +5,7 @@ from django.http import HttpResponse from requests.exceptions import RequestException -from iiif.utils import ImmediateHttpResponse +from main.utils import ImmediateHttpResponse log = logging.getLogger(__name__) diff --git a/src/iiif/parsing.py b/src/iiif/parsing.py index a2e8d38..809a1a7 100644 --- a/src/iiif/parsing.py +++ b/src/iiif/parsing.py @@ -6,7 +6,7 @@ from django.conf import settings from django.http import HttpResponse, HttpResponseNotAllowed -from iiif.utils import ImmediateHttpResponse +from main.utils import ImmediateHttpResponse log = logging.getLogger(__name__) diff --git a/src/iiif/urls.py b/src/iiif/urls.py index a8d742a..ebf70e7 100644 --- a/src/iiif/urls.py +++ b/src/iiif/urls.py @@ -1,13 +1,7 @@ from django.urls import path -from . import views +from iiif import views urlpatterns = [ - path( - "login-link-to-email/", - views.send_dataportaal_login_url_to_mail, - name="login_link_to_email_endpoint", - ), - path("zip/", views.request_multiple_files_in_zip, name="zip_endpoint"), path("", views.index, name="iiif_endpoint"), ] diff --git a/src/iiif/views.py b/src/iiif/views.py index b983c8b..0088a8b 100644 --- a/src/iiif/views.py +++ b/src/iiif/views.py @@ -1,18 +1,14 @@ -import json import logging -import uuid -from django.conf import settings from django.http import HttpResponse -from django.template.loader import render_to_string from django.views.decorators.cache import cache_control from django.views.decorators.csrf import csrf_exempt -from django_ratelimit.decorators import ratelimit -from iiif import authentication, image_server, mailing, parsing, utils, zip_tools +from auth_mail import authentication +from iiif import image_server, parsing from iiif.image_handling import crop_image, generate_info_json, scale_image from iiif.metadata import get_metadata -from iiif.utils_azure import store_blob_on_storage_account +from main import utils log = logging.getLogger(__name__) HOUR = 3600 @@ -32,6 +28,7 @@ def index(request, iiif_url): authentication.check_wabo_for_mail_login(is_mail_login, url_info) metadata, _ = get_metadata(url_info, iiif_url, {}) authentication.check_file_access_in_metadata(metadata, url_info, scope) + # TODO image from image_server cached? file_response, file_url = image_server.get_file( request.META, url_info, iiif_url, metadata ) @@ -66,93 +63,3 @@ def index(request, iiif_url): response_content, content_type, ) - - -# TODO: limit to dataportaal urls -@csrf_exempt -@ratelimit( - key="ip", rate="3/d", block=False -) # TODO: Check django cache settings for rate limiter to work across parallel docker containers -def send_dataportaal_login_url_to_mail(request): - try: - # Some basic sanity checks - parsing.check_for_post(request) - payload = parsing.parse_payload(request) - email, origin_url = parsing.check_login_url_payload(payload) - parsing.check_email_validity(email) - - # Create the login url - token = authentication.create_mail_login_token(email, settings.JWT_SECRET_KEY) - login_url = origin_url + "?auth=" + token - - # Send the email - email_subject = "Toegang bouw- en omgevingsdossiers data.amsterdam.nl" - email_body = render_to_string("login_link.html", {"login_url": login_url}) - # TODO: move actually sending the email to a separate process - mailing.send_email(payload["email"], email_subject, email_body) - - except utils.ImmediateHttpResponse as e: - log.exception("ImmediateHttpResponse in login_url:") - return e.response - - # Respond with a 200 to signal success. - # The user will get an email asap with a login url - return HttpResponse() - - -@csrf_exempt -def request_multiple_files_in_zip(request): - try: - parsing.check_for_post(request) - authentication.check_auth_availability(request) - read_jwt_token, is_mail_login = authentication.read_out_mail_jwt_token(request) - scope = authentication.get_max_scope(request, read_jwt_token) - email_address = parsing.get_email_address(request, read_jwt_token) - payload = parsing.parse_payload(request) - parsing.check_zip_payload(payload) - except utils.ImmediateHttpResponse as e: - log.error(e.response.content) - return e.response - - zip_info = { - "email_address": email_address, - "request_meta": request.META, - "urls": {}, - "scope": scope, - "is_mail_login": is_mail_login, - } - for full_url in payload["urls"]: - try: - iiif_url = parsing.strip_full_iiif_url(full_url) - url_info = parsing.get_url_info(iiif_url, True) - authentication.check_wabo_for_mail_login(is_mail_login, url_info) - - # We create a new dict with all the info so that we have it when we want to get and zip the files later - zip_info["urls"][iiif_url] = {"url_info": url_info} - - except utils.ImmediateHttpResponse as e: - log.error(e.response.content) - return e.response - - # The fact that we arrived here means that urls are formatted correctly and the info is extracted from it. - # It does NOT mean that the metadata exists or that the user is allowed to access all the files. This will - # be checked in the consumer. We now proceed with storing the info as a zip job so that a zip worker - # can pick it up. - - # Zip_info is too large for the request body (hard limit by Azure). So we use the claim check pattern. - # First, the jobs/zip_info is stored as a blob in the storage_account. Then we send a reference to the blob to the queue - blob_name = str(uuid.uuid4()) - zip_job = json.dumps( - { - key: zip_info[key] - for key in ["email_address", "scope", "is_mail_login", "urls"] - } - ) - store_blob_on_storage_account( - settings.STORAGE_ACCOUNT_CONTAINER_ZIP_QUEUE_JOBS_NAME, blob_name, zip_job - ) - zip_tools.store_zip_job(blob_name) - - # Respond with a 200 to signal success. - # The user will get an email once the files have been zipped by a worker. - return HttpResponse() diff --git a/src/main/settings.py b/src/main/settings.py index dc54cec..e9c319d 100644 --- a/src/main/settings.py +++ b/src/main/settings.py @@ -128,17 +128,24 @@ if TEMP_URL_EXPIRY_DAYS > 7: raise ValueError("TEMP_URL_EXPIRY_DAYS must be 7 days or less") - -INSTALLED_APPS = [ +# Application definition +DJANGO_APPS = [ "django.contrib.auth", "django.contrib.contenttypes", "django.contrib.sessions", "django.contrib.messages", "django.contrib.staticfiles", +] +THIRD_PARTY_APPS = [ + "corsheaders", +] +LOCAL_APPS = [ "iiif", "health", - "corsheaders", + "zip_consumer", + "auth_mail", ] +INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS MIDDLEWARE = [ "django.middleware.security.SecurityMiddleware", diff --git a/src/main/urls.py b/src/main/urls.py index 7dc4b14..367be88 100644 --- a/src/main/urls.py +++ b/src/main/urls.py @@ -18,6 +18,8 @@ urlpatterns = [ path("iiif/status/health", include("health.urls")), + path("iiif/", include("auth_mail.urls")), path("iiif/", include("iiif.urls")), + path("iiif/", include("zip_consumer.urls")), path("", include("health.urls")), ] diff --git a/src/iiif/utils.py b/src/main/utils.py similarity index 100% rename from src/iiif/utils.py rename to src/main/utils.py diff --git a/src/iiif/utils_azure.py b/src/main/utils_azure.py similarity index 100% rename from src/iiif/utils_azure.py rename to src/main/utils_azure.py diff --git a/src/iiif/management/commands/__init__.py b/src/zip_consumer/__init__.py similarity index 100% rename from src/iiif/management/commands/__init__.py rename to src/zip_consumer/__init__.py diff --git a/src/zip_consumer/management/__init__.py b/src/zip_consumer/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/zip_consumer/management/commands/__init__.py b/src/zip_consumer/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/iiif/management/commands/consume_zips.py b/src/zip_consumer/management/commands/consume_zips.py similarity index 86% rename from src/iiif/management/commands/consume_zips.py rename to src/zip_consumer/management/commands/consume_zips.py index b4fc5e5..2ef9c64 100644 --- a/src/iiif/management/commands/consume_zips.py +++ b/src/zip_consumer/management/commands/consume_zips.py @@ -2,7 +2,7 @@ from django.core.management.base import BaseCommand -from iiif.queue_zip_consumer import AzureZipQueueConsumer +from zip_consumer.queue_zip_consumer import AzureZipQueueConsumer logger = logging.getLogger(__name__) diff --git a/src/iiif/queue_zip_consumer.py b/src/zip_consumer/queue_zip_consumer.py similarity index 96% rename from src/iiif/queue_zip_consumer.py rename to src/zip_consumer/queue_zip_consumer.py index f3961d0..de34a75 100644 --- a/src/iiif/queue_zip_consumer.py +++ b/src/zip_consumer/queue_zip_consumer.py @@ -7,15 +7,18 @@ from django.conf import settings from django.template.loader import render_to_string -from iiif import authentication, image_server, mailing, utils, zip_tools +from auth_mail import authentication, mailing +from iiif import image_server from iiif.metadata import get_metadata -from iiif.utils_azure import ( +from main import utils +from main.utils_azure import ( create_storage_account_temp_url, get_blob_from_storage_account, get_queue_client, remove_blob_from_storage_account, store_file_on_storage_account, ) +from zip_consumer import zip_tools logger = logging.getLogger(__name__) diff --git a/src/iiif/templates/download_zip.html b/src/zip_consumer/templates/download_zip.html similarity index 100% rename from src/iiif/templates/download_zip.html rename to src/zip_consumer/templates/download_zip.html diff --git a/src/zip_consumer/urls.py b/src/zip_consumer/urls.py new file mode 100644 index 0000000..267968a --- /dev/null +++ b/src/zip_consumer/urls.py @@ -0,0 +1,7 @@ +from django.urls import path + +from zip_consumer import views + +urlpatterns = [ + path("zip/", views.request_multiple_files_in_zip, name="zip_endpoint"), +] diff --git a/src/zip_consumer/views.py b/src/zip_consumer/views.py new file mode 100644 index 0000000..3ad57c8 --- /dev/null +++ b/src/zip_consumer/views.py @@ -0,0 +1,73 @@ +import json +import logging +import uuid + +from django.conf import settings +from django.http import HttpResponse +from django.views.decorators.csrf import csrf_exempt + +from auth_mail import authentication +from iiif import parsing +from main import utils +from main.utils_azure import store_blob_on_storage_account +from zip_consumer import zip_tools + +log = logging.getLogger(__name__) + + +@csrf_exempt +def request_multiple_files_in_zip(request): + try: + parsing.check_for_post(request) + authentication.check_auth_availability(request) + read_jwt_token, is_mail_login = authentication.read_out_mail_jwt_token(request) + scope = authentication.get_max_scope(request, read_jwt_token) + email_address = parsing.get_email_address(request, read_jwt_token) + payload = parsing.parse_payload(request) + parsing.check_zip_payload(payload) + except utils.ImmediateHttpResponse as e: + log.error(e.response.content) + return e.response + + zip_info = { + "email_address": email_address, + "request_meta": request.META, + "urls": {}, + "scope": scope, + "is_mail_login": is_mail_login, + } + for full_url in payload["urls"]: + try: + iiif_url = parsing.strip_full_iiif_url(full_url) + url_info = parsing.get_url_info(iiif_url, True) + authentication.check_wabo_for_mail_login(is_mail_login, url_info) + + # We create a new dict with all the info so that we have it when we want to get and zip the files later + zip_info["urls"][iiif_url] = {"url_info": url_info} + + except utils.ImmediateHttpResponse as e: + log.error(e.response.content) + return e.response + + # The fact that we arrived here means that urls are formatted correctly and the info is extracted from it. + # It does NOT mean that the metadata exists or that the user is allowed to access all the files. This will + # be checked in the consumer. We now proceed with storing the info as a zip job so that a zip worker + # can pick it up. + + # Zip_info is too large for the request body (hard limit by Azure). So we use the claim check pattern. + # First, the jobs/zip_info is stored as a blob in the storage_account. Then we send a reference to the blob to the queue + blob_name = str(uuid.uuid4()) + zip_job = json.dumps( + { + key: zip_info[key] + for key in ["email_address", "scope", "is_mail_login", "urls"] + } + ) + store_blob_on_storage_account( + settings.STORAGE_ACCOUNT_CONTAINER_ZIP_QUEUE_JOBS_NAME, blob_name, zip_job + ) + zip_tools.store_zip_job(blob_name) + + # Respond with a 200 to signal success. + # The user will get an email once the files have been zipped by a worker. + return HttpResponse() diff --git a/src/iiif/zip_tools.py b/src/zip_consumer/zip_tools.py similarity index 96% rename from src/iiif/zip_tools.py rename to src/zip_consumer/zip_tools.py index 6752ef9..1224124 100644 --- a/src/iiif/zip_tools.py +++ b/src/zip_consumer/zip_tools.py @@ -5,7 +5,7 @@ from uuid import uuid4 from zipfile import ZipFile -from iiif.utils_azure import get_queue_client +from main.utils_azure import get_queue_client TMP_BOUWDOSSIER_ZIP_FOLDER = "/tmp/bouwdossier-zips/" ZIP_MESSAGE_VERSION_NAME = "zip_job_v1" diff --git a/tests/test_iiif.py b/tests/test_iiif.py index 7e6f753..6da739c 100644 --- a/tests/test_iiif.py +++ b/tests/test_iiif.py @@ -9,7 +9,7 @@ from django.test import override_settings from requests.exceptions import ConnectTimeout, RequestException -from iiif.authentication import ( +from auth_mail.authentication import ( RESPONSE_CONTENT_COPYRIGHT, RESPONSE_CONTENT_NO_DOCUMENT_IN_METADATA, RESPONSE_CONTENT_NO_TOKEN, @@ -17,7 +17,7 @@ RESPONSE_CONTENT_RESTRICTED, create_mail_login_token, ) -from iiif.generate_token import create_authz_token +from auth_mail.generate_token import create_authz_token from iiif.image_server import RESPONSE_CONTENT_ERROR_RESPONSE_FROM_IMAGE_SERVER from iiif.metadata import RESPONSE_CONTENT_ERROR_RESPONSE_FROM_METADATA_SERVER from tests.tools import MockResponse @@ -666,7 +666,7 @@ def setup_method(self): self.test_email_address, settings.SECRET_KEY ) - @patch("iiif.mailing.send_email") + @patch("auth_mail.mailing.send_email") def test_send_dataportaal_login_url_to_burger_email_address( self, mock_send_email, client ): @@ -734,7 +734,7 @@ def test_request_with_origin_url_not_in_whitelist_fails(self, client): assert response.status_code == 400 @override_settings(LOGIN_ORIGIN_URL_TLD_WHITELIST=["localhost"]) - @patch("iiif.mailing.send_email") + @patch("auth_mail.mailing.send_email") def test_request_with_localhost_and_port_in_origin_url_succeeds( self, mock_send_email, client ): diff --git a/tests/test_image_handling.py b/tests/test_image_handling.py index 2da1074..4162fa5 100644 --- a/tests/test_image_handling.py +++ b/tests/test_image_handling.py @@ -7,7 +7,7 @@ parse_scaling_string, scale_image, ) -from iiif.utils import ImmediateHttpResponse +from main.utils import ImmediateHttpResponse class TestImageFormatting: diff --git a/tests/test_utils.py b/tests/test_utils.py index 3c64091..e8ebad3 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -11,11 +11,10 @@ import pytz from django.conf import settings -from iiif.authentication import create_mail_login_token, img_is_public_copyright +from auth_mail.authentication import create_mail_login_token, img_is_public_copyright from iiif.image_server import create_file_url_and_headers, create_wabo_url from iiif.parsing import InvalidIIIFUrlError, get_email_address, get_info_from_iiif_url -from iiif.utils import ImmediateHttpResponse -from iiif.zip_tools import TMP_BOUWDOSSIER_ZIP_FOLDER, create_local_zip_file +from main.utils import ImmediateHttpResponse from tests.test_iiif import ( PRE_WABO_IMG_URL_NO_SCALING, PRE_WABO_IMG_URL_WITH_EXTRA_REFERENCE, @@ -25,6 +24,7 @@ WABO_IMG_URL, ) from tests.tools import filename_from_url, source_filename_from_url +from zip_consumer.zip_tools import TMP_BOUWDOSSIER_ZIP_FOLDER, create_local_zip_file log = logging.getLogger(__name__) timezone = pytz.timezone("UTC") diff --git a/tests/test_utils_azure.py b/tests/test_utils_azure.py index 182eccf..194c538 100644 --- a/tests/test_utils_azure.py +++ b/tests/test_utils_azure.py @@ -5,7 +5,7 @@ from azure.core.exceptions import ResourceExistsError from django.conf import settings -from iiif.utils_azure import get_blob_service_client, get_queue_client +from main.utils_azure import get_blob_service_client, get_queue_client def create_blob_container(container_name): diff --git a/tests/test_zip_endpoint.py b/tests/test_zip_endpoint.py index 6d7eb9c..e14b157 100644 --- a/tests/test_zip_endpoint.py +++ b/tests/test_zip_endpoint.py @@ -8,16 +8,14 @@ from azure.core.exceptions import ResourceNotFoundError from django.conf import settings -from iiif.authentication import ( +from auth_mail.authentication import ( RESPONSE_CONTENT_NO_WABO_WITH_MAIL_LOGIN, RESPONSE_CONTENT_RESTRICTED, RESPONSE_CONTENT_RESTRICTED_IN_ZIP, create_mail_login_token, ) -from iiif.generate_token import create_authz_token -from iiif.queue_zip_consumer import AzureZipQueueConsumer -from iiif.utils_azure import get_blob_from_storage_account, get_queue_client -from iiif.zip_tools import TMP_BOUWDOSSIER_ZIP_FOLDER +from auth_mail.generate_token import create_authz_token +from main.utils_azure import get_blob_from_storage_account, get_queue_client from tests.test_iiif import ( IMAGE_BINARY_DATA, PRE_WABO_IMG_URL_BASE, @@ -27,6 +25,8 @@ ) from tests.test_utils_azure import create_blob_container, create_queue from tests.tools import MockResponse +from zip_consumer.queue_zip_consumer import AzureZipQueueConsumer +from zip_consumer.zip_tools import TMP_BOUWDOSSIER_ZIP_FOLDER log = logging.getLogger(__name__) timezone = pytz.timezone("UTC") @@ -353,11 +353,11 @@ def test_get_restricted_image_with_restricted_scope_succeeds( assert response.status_code == 200 assert len(self.get_all_queue_messages()) == 1 - @patch("iiif.queue_zip_consumer.create_storage_account_temp_url") + @patch("zip_consumer.queue_zip_consumer.create_storage_account_temp_url") @patch("iiif.image_server.get_image_from_server") @patch("iiif.metadata.do_metadata_request") - @patch("iiif.mailing.send_email") - @patch("iiif.zip_tools.cleanup_local_files") + @patch("auth_mail.mailing.send_email") + @patch("zip_consumer.zip_tools.cleanup_local_files") @pytest.mark.parametrize( "scope, second_image_access, expected_line_end, expected_files", [ From 35697025d075dcb5552e943a32467e25de983f09 Mon Sep 17 00:00:00 2001 From: Manilde van der Oord Date: Thu, 2 May 2024 18:36:39 +0200 Subject: [PATCH 2/3] reorderd settings only used in tests --- tests/test_iiif.py | 52 ++++++++++---------------------------- tests/test_settings.py | 37 +++++++++++++++++++++++++++ tests/test_utils.py | 2 +- tests/test_zip_endpoint.py | 2 +- 4 files changed, 52 insertions(+), 41 deletions(-) create mode 100644 tests/test_settings.py diff --git a/tests/test_iiif.py b/tests/test_iiif.py index 6da739c..1509f6d 100644 --- a/tests/test_iiif.py +++ b/tests/test_iiif.py @@ -20,51 +20,25 @@ from auth_mail.generate_token import create_authz_token from iiif.image_server import RESPONSE_CONTENT_ERROR_RESPONSE_FROM_IMAGE_SERVER from iiif.metadata import RESPONSE_CONTENT_ERROR_RESPONSE_FROM_METADATA_SERVER +from tests.test_settings import ( + IMAGE_BINARY_DATA, + PRE_WABO_IMG_URL_NO_SCALING, + PRE_WABO_IMG_URL_WITH_EMPTY_SCALING, + PRE_WABO_IMG_URL_WITH_EXTRA_REFERENCE, + PRE_WABO_IMG_URL_WITH_REGION, + PRE_WABO_IMG_URL_WITH_REGION_NON_OVERLAPPING, + PRE_WABO_IMG_URL_WITH_SCALING, + PRE_WABO_INFO_JSON_URL, + WABO_IMG_URL, + IMAGE_BINARY_DATA_24x24x72x72, + IMAGE_BINARY_DATA_50x44, +) from tests.tools import MockResponse log = logging.getLogger(__name__) timezone = pytz.timezone("UTC") -def filename_from_url(url): - return url.split(":")[1].split("/")[0].replace("-", "/") - - -PRE_WABO_IMG_URL_BASE = "2/edepot:ST-00015-ST00000126_00001.jpg/" - -PRE_WABO_INFO_JSON_URL = PRE_WABO_IMG_URL_BASE + "info.json" - -PRE_WABO_IMG_URL_WITH_SCALING = PRE_WABO_IMG_URL_BASE + "full/50,50/0/default.jpg" -PRE_WABO_IMG_URL_WITH_EMPTY_SCALING = PRE_WABO_IMG_URL_BASE + "full//0/default.jpg" -PRE_WABO_FILE_NAME_WITH_SCALING = filename_from_url(PRE_WABO_IMG_URL_WITH_SCALING) - -PRE_WABO_IMG_URL_WITH_REGION = PRE_WABO_IMG_URL_BASE + "24,24,48,48/full/0/default.jpg" -PRE_WABO_IMG_URL_WITH_REGION_NON_OVERLAPPING = ( - PRE_WABO_IMG_URL_BASE + "10000,10000,48,48/full/0/default.jpg" -) - -PRE_WABO_IMG_URL_NO_SCALING = ( - "2/edepot:ST-00015-ST00000126_00001.jpg/full/full/0/default.jpg" -) -PRE_WABO_FILE_NAME_NO_SCALING = filename_from_url(PRE_WABO_IMG_URL_NO_SCALING) - -PRE_WABO_IMG_URL_WITH_EXTRA_REFERENCE = ( - "2/edepot:SQ1452-SQ-01452%20(2)-SQ10079651_00001.jpg/full/full/0/default.jpg" -) -PRE_WABO_IMG_FILE_NAME_WITH_EXTRA_REFERENCE = filename_from_url( - PRE_WABO_IMG_URL_WITH_EXTRA_REFERENCE -) - -WABO_IMG_URL = "2/wabo:SDZ-38657-4900487_628547/full/1000,900/0/default.jpg" - -with open("test-images/test-image-96x85.jpg", "rb") as file: - IMAGE_BINARY_DATA = file.read() -with open("test-images/test-image-50x44.jpg", "rb") as file: - IMAGE_BINARY_DATA_50x44 = file.read() -with open("test-images/test-image-cropped-24x24x72x72.jpg", "rb") as file: - IMAGE_BINARY_DATA_24x24x72x72 = file.read() - - class TestFileRetrievalWithAuthz: def setup_method(self): self.url = "/iiif/" diff --git a/tests/test_settings.py b/tests/test_settings.py new file mode 100644 index 0000000..6deaf2b --- /dev/null +++ b/tests/test_settings.py @@ -0,0 +1,37 @@ +def filename_from_url(url): + return url.split(":")[1].split("/")[0].replace("-", "/") + + +PRE_WABO_IMG_URL_BASE = "2/edepot:ST-00015-ST00000126_00001.jpg/" + +PRE_WABO_INFO_JSON_URL = PRE_WABO_IMG_URL_BASE + "info.json" + +PRE_WABO_IMG_URL_WITH_SCALING = PRE_WABO_IMG_URL_BASE + "full/50,50/0/default.jpg" +PRE_WABO_IMG_URL_WITH_EMPTY_SCALING = PRE_WABO_IMG_URL_BASE + "full//0/default.jpg" +PRE_WABO_FILE_NAME_WITH_SCALING = filename_from_url(PRE_WABO_IMG_URL_WITH_SCALING) + +PRE_WABO_IMG_URL_WITH_REGION = PRE_WABO_IMG_URL_BASE + "24,24,48,48/full/0/default.jpg" +PRE_WABO_IMG_URL_WITH_REGION_NON_OVERLAPPING = ( + PRE_WABO_IMG_URL_BASE + "10000,10000,48,48/full/0/default.jpg" +) + +PRE_WABO_IMG_URL_NO_SCALING = ( + "2/edepot:ST-00015-ST00000126_00001.jpg/full/full/0/default.jpg" +) +PRE_WABO_FILE_NAME_NO_SCALING = filename_from_url(PRE_WABO_IMG_URL_NO_SCALING) + +PRE_WABO_IMG_URL_WITH_EXTRA_REFERENCE = ( + "2/edepot:SQ1452-SQ-01452%20(2)-SQ10079651_00001.jpg/full/full/0/default.jpg" +) +PRE_WABO_IMG_FILE_NAME_WITH_EXTRA_REFERENCE = filename_from_url( + PRE_WABO_IMG_URL_WITH_EXTRA_REFERENCE +) + +WABO_IMG_URL = "2/wabo:SDZ-38657-4900487_628547/full/1000,900/0/default.jpg" + +with open("test-images/test-image-96x85.jpg", "rb") as file: + IMAGE_BINARY_DATA = file.read() +with open("test-images/test-image-50x44.jpg", "rb") as file: + IMAGE_BINARY_DATA_50x44 = file.read() +with open("test-images/test-image-cropped-24x24x72x72.jpg", "rb") as file: + IMAGE_BINARY_DATA_24x24x72x72 = file.read() diff --git a/tests/test_utils.py b/tests/test_utils.py index e8ebad3..001d9ee 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -15,7 +15,7 @@ from iiif.image_server import create_file_url_and_headers, create_wabo_url from iiif.parsing import InvalidIIIFUrlError, get_email_address, get_info_from_iiif_url from main.utils import ImmediateHttpResponse -from tests.test_iiif import ( +from tests.test_settings import ( PRE_WABO_IMG_URL_NO_SCALING, PRE_WABO_IMG_URL_WITH_EXTRA_REFERENCE, PRE_WABO_IMG_URL_WITH_REGION, diff --git a/tests/test_zip_endpoint.py b/tests/test_zip_endpoint.py index e14b157..079e718 100644 --- a/tests/test_zip_endpoint.py +++ b/tests/test_zip_endpoint.py @@ -16,7 +16,7 @@ ) from auth_mail.generate_token import create_authz_token from main.utils_azure import get_blob_from_storage_account, get_queue_client -from tests.test_iiif import ( +from tests.test_settings import ( IMAGE_BINARY_DATA, PRE_WABO_IMG_URL_BASE, PRE_WABO_IMG_URL_WITH_EXTRA_REFERENCE, From 91b4e3fbe8d1c9f5825374e7354c33c78c96b1eb Mon Sep 17 00:00:00 2001 From: Manilde van der Oord Date: Fri, 3 May 2024 12:24:06 +0200 Subject: [PATCH 3/3] setting correct order main.urls --- src/auth_mail/urls.py | 2 +- src/main/urls.py | 6 ++++-- src/zip_consumer/urls.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/auth_mail/urls.py b/src/auth_mail/urls.py index 496c33c..9ea3cc1 100644 --- a/src/auth_mail/urls.py +++ b/src/auth_mail/urls.py @@ -4,7 +4,7 @@ urlpatterns = [ path( - "login-link-to-email/", + "", views.send_dataportaal_login_url_to_mail, name="login_link_to_email_endpoint", ), diff --git a/src/main/urls.py b/src/main/urls.py index 367be88..858c58e 100644 --- a/src/main/urls.py +++ b/src/main/urls.py @@ -16,10 +16,12 @@ from django.urls import include, path +from main import settings + urlpatterns = [ path("iiif/status/health", include("health.urls")), - path("iiif/", include("auth_mail.urls")), + path("iiif/login-link-to-email/", include("auth_mail.urls")), + path("iiif/zip/", include("zip_consumer.urls")), path("iiif/", include("iiif.urls")), - path("iiif/", include("zip_consumer.urls")), path("", include("health.urls")), ] diff --git a/src/zip_consumer/urls.py b/src/zip_consumer/urls.py index 267968a..217604a 100644 --- a/src/zip_consumer/urls.py +++ b/src/zip_consumer/urls.py @@ -3,5 +3,5 @@ from zip_consumer import views urlpatterns = [ - path("zip/", views.request_multiple_files_in_zip, name="zip_endpoint"), + path("", views.request_multiple_files_in_zip, name="zip_endpoint"), ]