Recognise timestamp: used to generate unique string per request

omrilotan · May 29, 2024 · 4ccddfc · 4ccddfc
1 parent 1e172e6
commit 4ccddfc
Show file tree

Hide file tree

Showing 8 changed files with 133 additions and 35 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,9 @@
 # Changelog
 
+## [5.1.8](https://github.com/omrilotan/isbot/compare/v5.1.7...v5.1.8)
+
+- [Pattern] Recognise timestamp in user agent string - is used to generate unique strings for each request
+
 ## [5.1.7](https://github.com/omrilotan/isbot/compare/v5.1.6...v5.1.7)
 
 - [Pattern] Ignore NewsSapphire in-app browser (news app)

diff --git a/fixtures/browsers.yml b/fixtures/browsers.yml
@@ -560,8 +560,6 @@ Snapchat:
   - Mozilla/5.0 (iPhone; CPU iPhone OS 13_1_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Snapchat/10.69.5.72 (iPhone11,6; iOS 13.1.3; gzip)
   - Mozilla/5.0 (iPhone; CPU iPhone OS 13_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Mobile/15E148 Snapchat/10.77.5.59 (like Safari/604.1)
   - Mozilla/5.0 (iPhone; CPU iPhone OS 14_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Mobile/15E148 Snapchat/11.36.0.36 (like Safari/604.1)
-  - Mozilla/5.0 (Linux; Android 10; SM-G973U Build/QP1A.190711.020; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/79.0.3945.136 Mobile Safari/537.36Snapchat10.74.6.0 (SM-G973U; Android 10#G973USQS2CSL1#29; gzip)
-  - Mozilla/5.0 (Linux; Android 8.1.0; LG-Q710AL Build/O11019; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/73.0.3683.90 Mobile Safari/537.36Snapchat10.54.0.31 (LG-Q710AL; Android 8.1.0#1916317118700#27; gzip)
 Snowshoe:
   - Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.21 (KHTML, like Gecko) Snowshoe/1.0.0 Safari/537.21
 Sogou Explorer:
@@ -674,6 +672,8 @@ WeChat:
   - Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36 MicroMessenger/6.5.2.501 NetType/WIFI WindowsWechat QBCore/3.43.691.400 QQBrowser/9.0.2524.400
 Whale Browser:
   - Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Whale/0.7.33.5 Safari/537.36
+Wyzo:
+  - Mozilla/5.0 (Windows; U; Windows NT 6.1; zh-CN;; rv:1.9.0.9) Gecko/2009042410 Firefox/3.0.9 Wyzo/3.0.3
 Xumo Media Player:
   - Xumo/1.1 (Linux;Android 5.1.1) AmznExoPlayerLib/2.9.0
   - Xumo/2.6.19 (Linux;Android 7.1.1) ExoPlayerLib/2.9.0

diff --git a/fixtures/downloaded/downloaded b/fixtures/downloaded/downloaded
@@ -1 +1 @@
-Fri, 10 May 2024 12:22:50 GMT
+Wed, 29 May 2024 09:39:50 GMT
diff --git a/fixtures/downloaded/matomo-org.json b/fixtures/downloaded/matomo-org.json
@@ -926,6 +926,31 @@
   "LiveJournal.com ([email protected]; for https://www.livejournal.com/users/example/; 1 readers)",
   "bitdiscovery-suggestions",
   "Castopod/1.0",
+  "ivre-masscan/1.3 https://github.com/robertdavidgraham/",
   "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GeedoProductSearch; +http://www.geedo.com/product-search.html) Chrome/79.0.3945.88 Safari/537.36",
-  "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/121.0.6167.57 Safari/537.36 Elastic/Synthetics"
+  "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/121.0.6167.57 Safari/537.36 Elastic/Synthetics",
+  "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:77.0) Gecko/20100101 Firefox/77.0 Time/1710630666672",
+  "Mozilla/5.0 (Windows; U; Windows NT 6.1; zh-CN;; rv:1.9.0.9) Gecko/2009042410 Firefox/3.0.9 Wyzo/3.0.3 Time/1715201908281",
+  "WDG_Validator/1.6.2",
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; BuiltWith/1.0; +http://builtwith.com/biup) Chrome/74.0.3729.131 Safari/537.36",
+  "Google Chrome Mozilla/5.0 (Windows NT 10.0; Win64; x64; CONTACT; [email protected]) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36 (compatible; CrawlyProjectCrawler/0.1.3; [email protected] +https://crawlyproject.digitaldragon.dev/)",
+  "BDFetch",
+  "r00ts3c-owned-you",
+  "nvdorz",
+  "Root Slut",
+  "Abcd",
+  "NiggaBalls",
+  "Dark",
+  "BotPoke",
+  "url",
+  "facebookscraper/1.0( http://www.facebook.com/sharescraper_help.php)",
+  "PunkMap (https://github.com/openeasm/punkmap)",
+  "'Mozilla/5.0 (compatible; GenomeCrawlerd/1.0; +https://www.nokia.com/networks/ip-networks/deepfield/genome/)'",
+  "Gaisbot/3.0 ([email protected]; http://gais.cs.ccu.edu.tw/robot.php)",
+  "FAST-WebCrawler/3.8 (crawler at trd dot overture dot com; http://www.alltheweb.com/help/webmaster/crawler)",
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36 (scanner.ducks.party)",
+  "Mozilla/5.0 (compatible; Cloudflare-Smart-Transit/1.0; +https://www.cloudflare.com/",
+  "Mozilla/5.0 (Linux; Android 11; Moto G Power (2022)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Mobile Safari/537.36 CloudflareObservatory/1.0",
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36 CloudflareObservatory/1.0"
 ]
diff --git a/fixtures/downloaded/monperrus.json b/fixtures/downloaded/monperrus.json
@@ -1145,5 +1145,7 @@
   "Audisto Crawler (desktop; essential; +https://audisto.com/bot)",
   "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)",
   "claudebot",
-  "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; [email protected])"
+  "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; [email protected])",
+  "Mozilla/5.0 (compatible; Monsidobot/2.2; +http://monsido.com/bot.html; [email protected])",
+  "GroupMeBot/1.0"
 ]
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		Fri, 10 May 2024 12:22:50 GMT
		Wed, 29 May 2024 09:39:50 GMT