Merge pull request #672 from oduwsdl/f-string-indexer-conversion

Mv string formatting in indexer to use f-strings

ipwb/backends.py

@@ -94,6 +94,6 @@ def get_web_archive_index(path: str) -> str:
         return response
 
     raise ValueError((
-        'Unknown format of index file location: {}. Please provide ' +
-        'a valid local path, HTTP or FTP URL, or an IPFS QmHash.'
-    ).format(path))
+        f'Unknown format of index file location: {path}. Please provide '
+        f'a valid local path, HTTP or FTP URL, or an IPFS QmHash.'
+    ))

ipwb/indexer.py

@@ -82,7 +82,7 @@ def pushToIPFS(hstr, payload):
             payloadIPFSHash = pushBytesToIPFS(payload)
 
             if retryCount > 0:
-                m = 'Retrying succeeded after {0} attempts'.format(retryCount)
+                m = f'Retrying succeeded after {retryCount} attempts'
                 print(m)
             return [httpHeaderIPFSHash, payloadIPFSHash]
         except NewConnectionError as e:
@@ -91,9 +91,8 @@ def pushToIPFS(hstr, payload):
 
             sys.exit()
         except Exception as e:  # TODO: Do not use bare except
-            attemptCount = '{0}/{1}'.format(retryCount + 1, ipfsRetryCount)
-            logError('IPFS failed to add, ' +
-                     'retrying attempt {0}'.format(attemptCount))
+            attemptCount = f'{retryCount + 1}/{ipfsRetryCount}'
+            logError(f'IPFS failed to add, retrying attempt {attemptCount}')
             logError(sys.exc_info())
             traceback.print_tb(sys.exc_info()[-1])
 
@@ -272,7 +271,7 @@ def getCDXJLinesFromFile(warcPath, **encCompOpts):
                     (hstr, payload, nonce) = \
                         encrypt(hstr, payload, encryptionKey)
 
-            # print('Adding {0} to IPFS'.format(entry.get('url')))
+            # print(f'Adding {entry.get("url")} to IPFS')
             ipfsHashes = pushToIPFS(hstr, payload)
 
             if ipfsHashes is None:
@@ -291,8 +290,7 @@ def getCDXJLinesFromFile(warcPath, **encCompOpts):
                 record.rec_headers.get_header('WARC-Date'))
             mime = record.http_headers.get_header('content-type')
             obj = {
-                'locator': 'urn:ipfs/{0}/{1}'.format(
-                    httpHeaderIPFSHash, payloadIPFSHash),
+                'locator': f'urn:ipfs/{httpHeaderIPFSHash}/{payloadIPFSHash}',
                 'status_code': statusCode,
                 'mime_type': mime or '',
                 'original_uri': originaluri
@@ -306,19 +304,18 @@ def getCDXJLinesFromFile(warcPath, **encCompOpts):
 
             objJSON = json.dumps(obj)
 
-            cdxjLine = '{0} {1} {2}'.format(originaluri_surted,
-                                            timestamp, objJSON)
+            cdxjLine = f'{originaluri_surted} {timestamp} {objJSON}'
             cdxjLines.append(cdxjLine)  # + '\n'
         return cdxjLines
 
 
 def generateCDXJMetadata(cdxjLines=None):
     metadata = ['!context ["http://tools.ietf.org/html/rfc7089"]']
     metaVals = {
-        'generator': "InterPlanetary Wayback v.{0}".format(ipwbVersion),
-        'created_at': '{0}'.format(datetime.datetime.now().isoformat())
+        'generator': f'InterPlanetary Wayback {ipwbVersion}',
+        'created_at': datetime.datetime.now().isoformat()
     }
-    metaVals = '!meta {0}'.format(json.dumps(metaVals))
+    metaVals = f'!meta {json.dumps(metaVals)}'
     metadata.append(metaVals)
 
     return metadata
@@ -356,7 +353,7 @@ def verifyFileExists(warcPath):
 
 
 def showProgress(msg, i, n):
-    line = '{0}: {1}/{2}'.format(msg, i, n)
+    line = f'{msg}: {i}/{n}'
     print(line, file=sys.stderr, end='\r')
     # Clear status line, show complete msg
     if i == n - 1:

ipwb/util.py

@@ -193,7 +193,7 @@ def padDigits14(dtstr, validate=False):
         H = match.group(4) or '00'
         M = match.group(5) or '00'
         S = match.group(6) or '00'
-        dtstr = '{}{}{}{}{}{}'.format(Y, m, d, H, M, S)
+        dtstr = f'{Y}{m}{d}{H}{M}{S}'
     if validate:
         datetime.datetime.strptime(dtstr, '%Y%m%d%H%M%S')
     return dtstr
@@ -204,9 +204,9 @@ def fetch_remote_file(path):
         r = requests.get(path)
         return r.text
     except ConnectionError:
-        logError('File at {0} is unavailable.'.format(path))
+        logError(f'File at {path} is unavailable.')
     except Exception as E:
-        logError('An unknown error occurred trying to fetch {0}'.format(path))
+        logError(f'An unknown error occurred while trying to fetch {path}')
         logError(sys.exc_info()[0])
     return None
 
@@ -322,5 +322,5 @@ def checkForUpdate():
     if current != latest and current is not None:
         print('This version of ipwb is outdated.'
               ' Please run pip install --upgrade ipwb.')
-        print('* Latest version: {0}'.format(latest))
-        print('* Installed version: {0}'.format(current))
+        print(f'* Latest version: {latest}')
+        print(f'* Installed version: {current}')

release.sh

@@ -16,13 +16,12 @@ git add 'ipwb/'$FILE_NAME
 git commit -m "RELEASE: Bump version to "$VERSION_STRING
 
 # Create a tag in repo
-TAG_NAME='v'$VERSION_STRING
-git tag $TAG_NAME
+git tag $VERSION_STRING
 git push
-git push origin $TAG_NAME
+git push origin $VERSION_STRING
 
 # The `.github/workflows/dist.yml` Workflow is triggered automatically
 # when the repo is tagged by running this Shell script
-# or manually creating a relase in GH.
+# or manually creating a release in GitHub.
 # In either case, the Workflow will build the Python package
 # in GH's CI infrastructure and publish it to PyPI.

samples/indexes/froggie_badHeaderHash.cdxj

@@ -1,5 +1,5 @@
 !context ["http://tools.ietf.org/html/rfc7089"]
-!meta {"created_at": "2018-01-30T16:52:43.212606", "generator": "InterPlanetary Wayback v.0.2018.01.10.0435"}
+!meta {"created_at": "2018-01-30T16:52:43.212606", "generator": "InterPlanetary Wayback 0.2018.01.10.0435"}
 com,matkelly)/froggies/frog.png 20170301192639 {"locator": "urn:ipfs/QmUeko8zM7Xanwz6F9GtRH4rLAi4Poj3DMECGsci2BRQfs/QmPhMnX74cwqx2xgj9d3N3gTra8CzafXwSbUwU8xagMfqR", "original_uri": "http://matkelly.com/froggies/frog.png", "mime_type": "image/png", "status_code": "200"}
 com,matkelly)/robots.txt 20170301192639 {"locator": "urn:ipfs/Qmbk3Aju7u26Pzk356a43wY9eUCScAJiLPxhvwsMoVt7Pd/QmYNB85U2txRAAdLp6wvZSPvd8AQq8UcjZJ2azhv5h6NF7", "original_uri": "http://matkelly.com/robots.txt", "mime_type": "text/plain", "status_code": "200"}
 edu,odu,cs)/~mkelly/semester/2017_spring/remotefroggie.html 20170301192639 {"locator": "urn:ipfs/QmPdyY7Pm66iWtGpTc7PqK11hvsnYSKMVL57G69RiNjGcm/QmNZ6mKSSAXAmXEocQj5gT4y4kdcr5D2C173ubWJ6PSKEZ", "original_uri": "http://www.cs.odu.edu/~mkelly/semester/2017_spring/remoteFroggie.html", "mime_type": "text/html", "status_code": "200"}

samples/indexes/salam-home.cdxj

@@ -1,3 +1,3 @@
 !context ["http://tools.ietf.org/html/rfc7089"]
-!meta {"created_at": "2018-07-28T14:07:36.044446", "generator": "InterPlanetary Wayback v.0.2018.07.27.2357"}
+!meta {"created_at": "2018-07-28T14:07:36.044446", "generator": "InterPlanetary Wayback 0.2018.07.27.2357"}
 edu,odu,cs)/~salam/ 20160305192247 {"locator": "urn:ipfs/QmNkt4JbkTemhUDmua7JW5NaTQWCrVZbk2EvUvhhPm9NJP/QmQr2uoXCbmC5c1vLngeE9HU1CHfF7BVG2z98JR6DQNFoU", "original_uri": "http://www.cs.odu.edu/~salam/", "mime_type": "text/html", "status_code": "200"}

samples/indexes/sample-1.cdxj

@@ -1,5 +1,5 @@
 !context ["http://oduwsdl.github.io/contexts/cdxj"]
-!meta {"created_at": "2017-05-31T13:46:51.380204", "generator": "InterPlanetary Wayback v.0.2017.05.31.1322"}
+!meta {"created_at": "2017-05-31T13:46:51.380204", "generator": "InterPlanetary Wayback 0.2017.05.31.1322"}
 com,yahoo,search)/mrss/ 20130411205500 {"locator": "urn:ipfs/Qmd7gw1C84eaDqEcuGTQRj5CA2jKkiEE7ZVCa9pfkrruRY/QmVFeyXN7T3265XMwTrn86tbovTHzJJeHnus2HFCwmbaYh", "mime_type": "text/html", "status_code": "302"}
 org,bitchmagazine)/blogs/feed/ 20130411205457 {"locator": "urn:ipfs/QmbRZEuSFqT214MNZg2iSifq7DCdc9PNbcpb3rDHAHdhyv/Qmb4mXnzpjKwZUgRXw2hupjnTdx6LK71UbCNFCZeR9FvmP", "mime_type": "application/rss+xml", "status_code": "200"}
 org,bitchmagazine)/blogs/girls-of-color-in-dystopia 20130411210315 {"locator": "urn:ipfs/QmQZPeLTuUs4wk3nLzoSawpHEHqnfyZSV3C654pDPBeRMy/QmRDZij54ghXrKoigU9miXjp4crcuoPnKZtSWL81q3Q2As", "mime_type": "text/html", "status_code": "200"}

tests/test_memento.py

@@ -65,7 +65,7 @@ def test_acceptdatetime_status(warc, lookup, acceptdatetime, status):
 
     headers = {'Accept-Datetime': acceptdatetime}
 
-    resp = requests.get('http://localhost:5000/{}'.format(lookup),
+    resp = requests.get(f'http://localhost:5000/{lookup}',
                         allow_redirects=False, headers=headers)
     assert resp.status_code == status