diff --git a/src/iiif/management/commands/consume_zips.py b/src/iiif/management/commands/consume_zips.py index bf6a61d..b4fc5e5 100644 --- a/src/iiif/management/commands/consume_zips.py +++ b/src/iiif/management/commands/consume_zips.py @@ -4,14 +4,17 @@ from iiif.queue_zip_consumer import AzureZipQueueConsumer -log = logging.getLogger(__name__) +logger = logging.getLogger(__name__) class Command(BaseCommand): help = "Start zip consumer to process zip requests" def handle(self, *args, **options): - log.info("Zip Consumer started") + logger.info("Zip Consumer started") - consumer = AzureZipQueueConsumer() - consumer.run() + try: + consumer = AzureZipQueueConsumer() + consumer.run() + except Exception as e: + logger.exception(e) diff --git a/src/iiif/queue_zip_consumer.py b/src/iiif/queue_zip_consumer.py index a1e2ca9..f3961d0 100644 --- a/src/iiif/queue_zip_consumer.py +++ b/src/iiif/queue_zip_consumer.py @@ -76,84 +76,74 @@ def run(self): def process_message(self, message): logger.info("Started process_message") - try: - job = json.loads(message.content) - if not job["version"] == self.MESSAGE_VERSION_NAME: - return - - # Get the job from the storage account - job_blob_name = job["data"] - blob_client, blob = get_blob_from_storage_account( - settings.STORAGE_ACCOUNT_CONTAINER_ZIP_QUEUE_JOBS_NAME, job_blob_name - ) - record = json.loads(blob) - # Prepare folder and report.txt file for downloads - ( - zipjob_uuid, - tmp_folder_path, - info_txt_contents, - ) = image_server.prepare_zip_downloads() - - # Get metadata and files from image servers - metadata_cache = {} - for iiif_url, image_info in record["urls"].items(): - fail_reason = None - metadata, metadata_cache = get_metadata( - image_info["url_info"], - iiif_url, - metadata_cache, - ) - try: - authentication.check_file_access_in_metadata( - metadata, image_info["url_info"], record["scope"] - ) - authentication.check_restricted_file( - metadata, image_info["url_info"] - ) - except utils.ImmediateHttpResponse as e: - fail_reason = e.response.content.decode("utf-8") - - info_txt_contents = image_server.download_file_for_zip( - iiif_url, - info_txt_contents, - image_info["url_info"], - fail_reason, - metadata, - None, # TODO: Remove parameter because not needed anymore (was: record["request_meta"]) - tmp_folder_path, - ) - # Store the info_file_along_with_the_image_files - zip_tools.save_file_to_folder( - tmp_folder_path, "report.txt", info_txt_contents + job = json.loads(message.content) + if not job["version"] == self.MESSAGE_VERSION_NAME: + return + + # Get the job from the storage account + job_blob_name = job["data"] + blob_client, blob = get_blob_from_storage_account( + settings.STORAGE_ACCOUNT_CONTAINER_ZIP_QUEUE_JOBS_NAME, job_blob_name + ) + record = json.loads(blob) + + # Prepare folder and report.txt file for downloads + ( + zipjob_uuid, + tmp_folder_path, + info_txt_contents, + ) = image_server.prepare_zip_downloads() + + # Get metadata and files from image servers + metadata_cache = {} + for iiif_url, image_info in record["urls"].items(): + fail_reason = None + metadata, metadata_cache = get_metadata( + image_info["url_info"], + iiif_url, + metadata_cache, ) + try: + authentication.check_file_access_in_metadata( + metadata, image_info["url_info"], record["scope"] + ) + authentication.check_restricted_file(metadata, image_info["url_info"]) + except utils.ImmediateHttpResponse as e: + fail_reason = e.response.content.decode("utf-8") - # Zip all files together - zip_file_path = zip_tools.create_local_zip_file( - zipjob_uuid, tmp_folder_path + info_txt_contents = image_server.download_file_for_zip( + iiif_url, + info_txt_contents, + image_info["url_info"], + fail_reason, + metadata, + None, # TODO: Remove parameter because not needed anymore (was: record["request_meta"]) + tmp_folder_path, ) - zip_file_name = os.path.basename(zip_file_path) + # Store the info_file_along_with_the_image_files + zip_tools.save_file_to_folder(tmp_folder_path, "report.txt", info_txt_contents) - blob_client, blob_service_client = store_file_on_storage_account( - settings.STORAGE_ACCOUNT_CONTAINER_NAME, zip_file_path, zip_file_name - ) + # Zip all files together + zip_file_path = zip_tools.create_local_zip_file(zipjob_uuid, tmp_folder_path) + zip_file_name = os.path.basename(zip_file_path) - temp_zip_download_url = create_storage_account_temp_url( - blob_client, blob_service_client - ) + blob_client, blob_service_client = store_file_on_storage_account( + settings.STORAGE_ACCOUNT_CONTAINER_NAME, zip_file_path, zip_file_name + ) - email_subject = "Downloadlink Bouw- en omgevingdossiers" - email_body = render_to_string( - "download_zip.html", {"temp_zip_download_url": temp_zip_download_url} - ) + temp_zip_download_url = create_storage_account_temp_url( + blob_client, blob_service_client + ) - mailing.send_email(record["email_address"], email_subject, email_body) + email_subject = "Downloadlink Bouw- en omgevingdossiers" + email_body = render_to_string( + "download_zip.html", {"temp_zip_download_url": temp_zip_download_url} + ) - remove_blob_from_storage_account( - settings.STORAGE_ACCOUNT_CONTAINER_ZIP_QUEUE_JOBS_NAME, job_blob_name - ) - zip_tools.cleanup_local_files(zip_file_path, tmp_folder_path) + mailing.send_email(record["email_address"], email_subject, email_body) - except Exception as e: - logger.exception(f"queue_zip_consumer error: {e}") - raise e + remove_blob_from_storage_account( + settings.STORAGE_ACCOUNT_CONTAINER_ZIP_QUEUE_JOBS_NAME, job_blob_name + ) + zip_tools.cleanup_local_files(zip_file_path, tmp_folder_path)