Feature/refactor in apps

src/iiif/authentication.py → src/auth_mail/authentication.py

@@ -6,7 +6,7 @@
 from django.http import HttpResponse
 from jwt.exceptions import DecodeError, ExpiredSignatureError, InvalidSignatureError
 
-from iiif.utils import ImmediateHttpResponse
+from main.utils import ImmediateHttpResponse
 
 log = logging.getLogger(__name__)
 

src/auth_mail/urls.py

@@ -0,0 +1,11 @@
+from django.urls import path
+
+from auth_mail import views
+
+urlpatterns = [
+    path(
+        "",
+        views.send_dataportaal_login_url_to_mail,
+        name="login_link_to_email_endpoint",
+    ),
+]

src/auth_mail/views.py

@@ -0,0 +1,45 @@
+import logging
+
+from django.conf import settings
+from django.http import HttpResponse
+from django.template.loader import render_to_string
+from django.views.decorators.csrf import csrf_exempt
+from django_ratelimit.decorators import ratelimit
+
+from auth_mail import authentication, mailing
+from iiif import parsing
+from main import utils
+
+log = logging.getLogger(__name__)
+
+
+# TODO: limit to dataportaal urls
+@csrf_exempt
+@ratelimit(
+    key="ip", rate="3/d", block=False
+)  # TODO: Check django cache settings for rate limiter to work across parallel docker containers
+def send_dataportaal_login_url_to_mail(request):
+    try:
+        # Some basic sanity checks
+        parsing.check_for_post(request)
+        payload = parsing.parse_payload(request)
+        email, origin_url = parsing.check_login_url_payload(payload)
+        parsing.check_email_validity(email)
+
+        # Create the login url
+        token = authentication.create_mail_login_token(email, settings.JWT_SECRET_KEY)
+        login_url = origin_url + "?auth=" + token
+
+        # Send the email
+        email_subject = "Toegang bouw- en omgevingsdossiers data.amsterdam.nl"
+        email_body = render_to_string("login_link.html", {"login_url": login_url})
+        # TODO: move actually sending the email to a separate process
+        mailing.send_email(payload["email"], email_subject, email_body)
+
+    except utils.ImmediateHttpResponse as e:
+        log.exception("ImmediateHttpResponse in login_url:")
+        return e.response
+
+    # Respond with a 200 to signal success.
+    # The user will get an email asap with a login url
+    return HttpResponse()

src/iiif/image_handling.py

@@ -6,7 +6,7 @@
 from django.http import HttpResponse
 from PIL import Image
 
-from iiif import utils
+from main import utils
 
 # Allow larger images too be processed. We can do this because we trust the source of the images
 Image.MAX_IMAGE_PIXELS = None

src/iiif/image_server.py

@@ -6,8 +6,8 @@
 from django.http import HttpResponse
 from requests.exceptions import RequestException
 
-from iiif import zip_tools
-from iiif.utils import ImmediateHttpResponse
+from main.utils import ImmediateHttpResponse
+from zip_consumer import zip_tools
 
 log = logging.getLogger(__name__)
 

src/iiif/metadata.py

@@ -5,7 +5,7 @@
 from django.http import HttpResponse
 from requests.exceptions import RequestException
 
-from iiif.utils import ImmediateHttpResponse
+from main.utils import ImmediateHttpResponse
 
 log = logging.getLogger(__name__)
 

src/iiif/parsing.py

@@ -6,7 +6,7 @@
 from django.conf import settings
 from django.http import HttpResponse, HttpResponseNotAllowed
 
-from iiif.utils import ImmediateHttpResponse
+from main.utils import ImmediateHttpResponse
 
 log = logging.getLogger(__name__)
 

src/iiif/urls.py

@@ -1,13 +1,7 @@
 from django.urls import path
 
-from . import views
+from iiif import views
 
 urlpatterns = [
-    path(
-        "login-link-to-email/",
-        views.send_dataportaal_login_url_to_mail,
-        name="login_link_to_email_endpoint",
-    ),
-    path("zip/", views.request_multiple_files_in_zip, name="zip_endpoint"),
     path("<path:iiif_url>", views.index, name="iiif_endpoint"),
 ]

src/iiif/views.py

@@ -1,18 +1,14 @@
-import json
 import logging
-import uuid
 
-from django.conf import settings
 from django.http import HttpResponse
-from django.template.loader import render_to_string
 from django.views.decorators.cache import cache_control
 from django.views.decorators.csrf import csrf_exempt
-from django_ratelimit.decorators import ratelimit
 
-from iiif import authentication, image_server, mailing, parsing, utils, zip_tools
+from auth_mail import authentication
+from iiif import image_server, parsing
 from iiif.image_handling import crop_image, generate_info_json, scale_image
 from iiif.metadata import get_metadata
-from iiif.utils_azure import store_blob_on_storage_account
+from main import utils
 
 log = logging.getLogger(__name__)
 HOUR = 3600
@@ -32,6 +28,7 @@ def index(request, iiif_url):
         authentication.check_wabo_for_mail_login(is_mail_login, url_info)
         metadata, _ = get_metadata(url_info, iiif_url, {})
         authentication.check_file_access_in_metadata(metadata, url_info, scope)
+        # TODO image from image_server cached?
         file_response, file_url = image_server.get_file(
             request.META, url_info, iiif_url, metadata
         )
@@ -66,93 +63,3 @@ def index(request, iiif_url):
         response_content,
         content_type,
     )
-
-
-# TODO: limit to dataportaal urls
-@csrf_exempt
-@ratelimit(
-    key="ip", rate="3/d", block=False
-)  # TODO: Check django cache settings for rate limiter to work across parallel docker containers
-def send_dataportaal_login_url_to_mail(request):
-    try:
-        # Some basic sanity checks
-        parsing.check_for_post(request)
-        payload = parsing.parse_payload(request)
-        email, origin_url = parsing.check_login_url_payload(payload)
-        parsing.check_email_validity(email)
-
-        # Create the login url
-        token = authentication.create_mail_login_token(email, settings.JWT_SECRET_KEY)
-        login_url = origin_url + "?auth=" + token
-
-        # Send the email
-        email_subject = "Toegang bouw- en omgevingsdossiers data.amsterdam.nl"
-        email_body = render_to_string("login_link.html", {"login_url": login_url})
-        # TODO: move actually sending the email to a separate process
-        mailing.send_email(payload["email"], email_subject, email_body)
-
-    except utils.ImmediateHttpResponse as e:
-        log.exception("ImmediateHttpResponse in login_url:")
-        return e.response
-
-    # Respond with a 200 to signal success.
-    # The user will get an email asap with a login url
-    return HttpResponse()
-
-
-@csrf_exempt
-def request_multiple_files_in_zip(request):
-    try:
-        parsing.check_for_post(request)
-        authentication.check_auth_availability(request)
-        read_jwt_token, is_mail_login = authentication.read_out_mail_jwt_token(request)
-        scope = authentication.get_max_scope(request, read_jwt_token)
-        email_address = parsing.get_email_address(request, read_jwt_token)
-        payload = parsing.parse_payload(request)
-        parsing.check_zip_payload(payload)
-    except utils.ImmediateHttpResponse as e:
-        log.error(e.response.content)
-        return e.response
-
-    zip_info = {
-        "email_address": email_address,
-        "request_meta": request.META,
-        "urls": {},
-        "scope": scope,
-        "is_mail_login": is_mail_login,
-    }
-    for full_url in payload["urls"]:
-        try:
-            iiif_url = parsing.strip_full_iiif_url(full_url)
-            url_info = parsing.get_url_info(iiif_url, True)
-            authentication.check_wabo_for_mail_login(is_mail_login, url_info)
-
-            # We create a new dict with all the info so that we have it when we want to get and zip the files later
-            zip_info["urls"][iiif_url] = {"url_info": url_info}
-
-        except utils.ImmediateHttpResponse as e:
-            log.error(e.response.content)
-            return e.response
-
-    # The fact that we arrived here means that urls are formatted correctly and the info is extracted from it.
-    # It does NOT mean that the metadata exists or that the user is allowed to access all the files. This will
-    # be checked in the consumer. We now proceed with storing the info as a zip job so that a zip worker
-    # can pick it up.
-
-    # Zip_info is too large for the request body (hard limit by Azure). So we use the claim check pattern.
-    # First, the jobs/zip_info is stored as a blob in the storage_account. Then we send a reference to the blob to the queue
-    blob_name = str(uuid.uuid4())
-    zip_job = json.dumps(
-        {
-            key: zip_info[key]
-            for key in ["email_address", "scope", "is_mail_login", "urls"]
-        }
-    )
-    store_blob_on_storage_account(
-        settings.STORAGE_ACCOUNT_CONTAINER_ZIP_QUEUE_JOBS_NAME, blob_name, zip_job
-    )
-    zip_tools.store_zip_job(blob_name)
-
-    # Respond with a 200 to signal success.
-    # The user will get an email once the files have been zipped by a worker.
-    return HttpResponse()

src/main/settings.py

@@ -128,17 +128,24 @@
 if TEMP_URL_EXPIRY_DAYS > 7:
     raise ValueError("TEMP_URL_EXPIRY_DAYS must be 7 days or less")
 
-
-INSTALLED_APPS = [
+# Application definition
+DJANGO_APPS = [
     "django.contrib.auth",
     "django.contrib.contenttypes",
     "django.contrib.sessions",
     "django.contrib.messages",
     "django.contrib.staticfiles",
+]
+THIRD_PARTY_APPS = [
+    "corsheaders",
+]
+LOCAL_APPS = [
     "iiif",
     "health",
-    "corsheaders",
+    "zip_consumer",
+    "auth_mail",
 ]
+INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS
 
 MIDDLEWARE = [
     "django.middleware.security.SecurityMiddleware",

src/main/urls.py

@@ -16,8 +16,12 @@
 
 from django.urls import include, path
 
+from main import settings
+
 urlpatterns = [
     path("iiif/status/health", include("health.urls")),
+    path("iiif/login-link-to-email/", include("auth_mail.urls")),
+    path("iiif/zip/", include("zip_consumer.urls")),
     path("iiif/", include("iiif.urls")),
     path("", include("health.urls")),
 ]

src/iiif/management/commands/consume_zips.py → src/zip_consumer/management/commands/consume_zips.py

@@ -2,7 +2,7 @@
 
 from django.core.management.base import BaseCommand
 
-from iiif.queue_zip_consumer import AzureZipQueueConsumer
+from zip_consumer.queue_zip_consumer import AzureZipQueueConsumer
 
 logger = logging.getLogger(__name__)
 

src/iiif/queue_zip_consumer.py → src/zip_consumer/queue_zip_consumer.py

@@ -7,15 +7,18 @@
 from django.conf import settings
 from django.template.loader import render_to_string
 
-from iiif import authentication, image_server, mailing, utils, zip_tools
+from auth_mail import authentication, mailing
+from iiif import image_server
 from iiif.metadata import get_metadata
-from iiif.utils_azure import (
+from main import utils
+from main.utils_azure import (
     create_storage_account_temp_url,
     get_blob_from_storage_account,
     get_queue_client,
     remove_blob_from_storage_account,
     store_file_on_storage_account,
 )
+from zip_consumer import zip_tools
 
 logger = logging.getLogger(__name__)
 

src/zip_consumer/urls.py

@@ -0,0 +1,7 @@
+from django.urls import path
+
+from zip_consumer import views
+
+urlpatterns = [
+    path("", views.request_multiple_files_in_zip, name="zip_endpoint"),
+]

src/zip_consumer/views.py

@@ -0,0 +1,73 @@
+import json
+import logging
+import uuid
+
+from django.conf import settings
+from django.http import HttpResponse
+from django.views.decorators.csrf import csrf_exempt
+
+from auth_mail import authentication
+from iiif import parsing
+from main import utils
+from main.utils_azure import store_blob_on_storage_account
+from zip_consumer import zip_tools
+
+log = logging.getLogger(__name__)
+
+
+@csrf_exempt
+def request_multiple_files_in_zip(request):
+    try:
+        parsing.check_for_post(request)
+        authentication.check_auth_availability(request)
+        read_jwt_token, is_mail_login = authentication.read_out_mail_jwt_token(request)
+        scope = authentication.get_max_scope(request, read_jwt_token)
+        email_address = parsing.get_email_address(request, read_jwt_token)
+        payload = parsing.parse_payload(request)
+        parsing.check_zip_payload(payload)
+    except utils.ImmediateHttpResponse as e:
+        log.error(e.response.content)
+        return e.response
+
+    zip_info = {
+        "email_address": email_address,
+        "request_meta": request.META,
+        "urls": {},
+        "scope": scope,
+        "is_mail_login": is_mail_login,
+    }
+    for full_url in payload["urls"]:
+        try:
+            iiif_url = parsing.strip_full_iiif_url(full_url)
+            url_info = parsing.get_url_info(iiif_url, True)
+            authentication.check_wabo_for_mail_login(is_mail_login, url_info)
+
+            # We create a new dict with all the info so that we have it when we want to get and zip the files later
+            zip_info["urls"][iiif_url] = {"url_info": url_info}
+
+        except utils.ImmediateHttpResponse as e:
+            log.error(e.response.content)
+            return e.response
+
+    # The fact that we arrived here means that urls are formatted correctly and the info is extracted from it.
+    # It does NOT mean that the metadata exists or that the user is allowed to access all the files. This will
+    # be checked in the consumer. We now proceed with storing the info as a zip job so that a zip worker
+    # can pick it up.
+
+    # Zip_info is too large for the request body (hard limit by Azure). So we use the claim check pattern.
+    # First, the jobs/zip_info is stored as a blob in the storage_account. Then we send a reference to the blob to the queue
+    blob_name = str(uuid.uuid4())
+    zip_job = json.dumps(
+        {
+            key: zip_info[key]
+            for key in ["email_address", "scope", "is_mail_login", "urls"]
+        }
+    )
+    store_blob_on_storage_account(
+        settings.STORAGE_ACCOUNT_CONTAINER_ZIP_QUEUE_JOBS_NAME, blob_name, zip_job
+    )
+    zip_tools.store_zip_job(blob_name)
+
+    # Respond with a 200 to signal success.
+    # The user will get an email once the files have been zipped by a worker.
+    return HttpResponse()