From c600db59fdd18c7e5cb89df2a39d91956b85874f Mon Sep 17 00:00:00 2001
From: Fuhu Xia <fxia@reisystems.com>
Date: Thu, 25 Jul 2024 09:44:26 -0400
Subject: [PATCH 1/2] traverse IIS folders

---
 ckanext/spatial/harvesters/waf.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/ckanext/spatial/harvesters/waf.py b/ckanext/spatial/harvesters/waf.py
index c9299d5b..d7cc3377 100644
--- a/ckanext/spatial/harvesters/waf.py
+++ b/ckanext/spatial/harvesters/waf.py
@@ -261,8 +261,7 @@ def fetch_stage(self, harvest_object):
             parse.Word(parse.alphas)
         , adjacent=False, joinString=' ').setResultsName('date')
         ) \
-        + parse.Word(parse.nums).suppress() \
-        + parse.Literal('<A HREF=').suppress() \
+        + parse.SkipTo('<A HREF=', include=True).suppress() \
         + parse.quotedString.setParseAction(parse.removeQuotes).setResultsName('url')
 
 other = parse.SkipTo(parse.CaselessLiteral("<a href="), include=True).suppress() \

From 3d0a375fe98edc70a0d12efd2f4ac54f0e05b597 Mon Sep 17 00:00:00 2001
From: Fuhu Xia <fxia@reisystems.com>
Date: Tue, 13 Aug 2024 07:07:58 -0400
Subject: [PATCH 2/2] traverse IIS folders: dealing with iis folder url

---
 ckanext/spatial/harvesters/waf.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/ckanext/spatial/harvesters/waf.py b/ckanext/spatial/harvesters/waf.py
index d7cc3377..b49f59ed 100644
--- a/ckanext/spatial/harvesters/waf.py
+++ b/ckanext/spatial/harvesters/waf.py
@@ -1,5 +1,6 @@
 from __future__ import print_function
 
+import os
 from urllib.parse import urljoin
 import logging
 import hashlib
@@ -310,11 +311,13 @@ def _extract_waf(content, base_url, scraper, results = None, depth=0):
             continue
         if 'mailto:' in url:
             continue
-        if '..' not in url and url[0] != '/' and url[-1] == '/':
+        if '..' not in url and url[-1] == '/':
             new_depth = depth + 1
             if depth > 10:
                 log.info('Max WAF depth reached')
                 continue
+            # turn iis dir url '/some/full/path/' into apache/nginx style 'path/'
+            url = os.path.basename(url.rstrip('/')) + '/'
             new_url = urljoin(base_url, url)
             if not new_url.startswith(base_url):
                 continue