Merge pull request #154 from arXiv/fastly-purge

Fastly purge

ConversionContainer/source/config.py

@@ -2,7 +2,7 @@
 
 import os
 
-CLASSIC_DATABASE_URI = os.environ.get('CLASSIC_DATABASE_URI')
+CLASSIC_DATABASE_URI = os.environ['CLASSIC_DATABASE_URI']
 
 OUT_BUCKET_ARXIV_ID = os.environ['DOCUMENT_CONVERTED_BUCKET'] # Startup failure on miss
 IN_BUCKET_SUB_ID = os.environ['SUBMISSION_SOURCE_BUCKET'] # Startup failure on miss
@@ -22,4 +22,7 @@
 SQLALCHEMY_DATABASE_URI = CLASSIC_DATABASE_URI
 SQLALCHEMY_BINDS = { 'latexml': LATEXML_DB_URI }
 
+FASTLY_PURGE_KEY = os.environ.get('FASTLY_PURGE_KEY', 'no-key-dev')
+IS_DEV = os.environ.get('IS_DEV', True)
+
 LOCK_DIR = '/arxiv/locks'

ConversionContainer/source/publish/__init__.py

@@ -18,6 +18,7 @@
     move_sub_qa_to_doc_qa
 )
 from .watermark import make_published_watermark, insert_watermark
+from .fastly_purge import fastly_purge_abs
 
 logger = logging.getLogger()
 
@@ -88,9 +89,11 @@ def _publish (submission_id: int, paper_id: str, version: int):
 
         # Move log output from sub bucket to published bucket
         move_sub_qa_to_doc_qa (submission_id, paper_idv)
-        logger.info(f'Successfully wrote {submission_id}/{paper_idv} qa to doc bucket')         
+        logger.info(f'Successfully wrote {submission_id}/{paper_idv} qa to doc bucket')
 
-
+        # Purge abs page from fastly so we can see it
+        if not current_app.config['IS_DEV']:
+            fastly_purge_abs(paper_id, version, current_app.config['FASTLY_PURGE_KEY'])
 
     except Exception as e:
         try:

ConversionContainer/source/publish/fastly_purge.py

@@ -0,0 +1,31 @@
+import logging
+
+from flask import current_app
+
+def fastly_purge_abs (paper_id: str, version: int, fastly_key: str):
+    headers = {
+        "Fastly-Key": fastly_key,
+        "Accept": "application/json",
+    }
+    domains = ["arxiv.org", "web3.arxiv.org", "www.arxiv.org"]
+
+    for domain in domains:
+        url = f"https://{ domain }/abs/{ paper_id }"
+
+        response = requests.request("PURGE", url, headers=headers)
+
+        if response.status_code == 200:
+            success.append(url)
+        else:
+            status_code = 500
+            msg = 'Error purging'
+            failed.append(url)
+
+        url = f"https://{ domain }/abs/{ paper_id }v{ version }"
+
+        response = requests.request("PURGE", url, headers=headers)
+
+        if response.status_code == 200:
+            logging.info(f'successfully purged { url }')
+        else:
+            logging.warning(f'failed to purge { url }')