Add bots (#314)

* Add bots * Run export
JayBizzle · Jan 15, 2019 · 0f91f34 · 0f91f34
1 parent 28b16dd
commit 0f91f34
Show file tree

Hide file tree

Showing 4 changed files with 23 additions and 4 deletions.
diff --git a/raw/Crawlers.json b/raw/Crawlers.json
diff --git a/raw/Crawlers.txt b/raw/Crawlers.txt
@@ -47,6 +47,7 @@ AdAuth\/
 adbeat
 AddThis
 ADmantX
+AdminLabs
 adressendeutschland
 adscanner
 agentslug
@@ -306,6 +307,7 @@ FeedBooster
 FeedBucket
 FeedBunch\/
 FeedBurner
+feeder
 Feedly
 FeedshowOnline
 Feedspot
@@ -412,6 +414,7 @@ HAA(A)?RTLAND http client
 Haansoft
 hackney\/
 Hadi Agent
+HappyApps-WebCheck
 Hatena
 Havij
 HeadlessChrome
@@ -458,7 +461,6 @@ httrack
 huaweisymantec
 HubSpot 
 Humanlinks
-HyperZbozi\.cz Feeder
 i2kconnect\/
 Iblog
 ichiro
@@ -879,6 +881,7 @@ Seomoz
 SEOprofiler
 SEOsearch
 seoscanners
+seositecheckup
 SEOstats
 servernfo
 sexsearcher
@@ -986,6 +989,7 @@ Tenon\.io
 teoma
 terrainformatica
 Test Certificate Info
+testuri
 Tetrahedron
 The Drop Reaper
 The Expert HTML Source Viewer
@@ -1096,6 +1100,7 @@ WebDoc
 WebEnhancer
 WebFetch
 WebFuck
+WebGazer
 WebGo IS
 WebImageCollector
 WebImages
@@ -1114,6 +1119,7 @@ Website Quester
 websitepulse agent
 WebsiteQuester
 Websnapr
+WebSniffer
 Webster
 WebStripper
 WebSucker

diff --git a/src/Fixtures/Crawlers.php b/src/Fixtures/Crawlers.php
@@ -68,6 +68,7 @@ class Crawlers extends AbstractProvider
         'adbeat',
         'AddThis',
         'ADmantX',
+        'AdminLabs',
         'adressendeutschland',
         'adscanner',
         'agentslug',
@@ -327,6 +328,7 @@ class Crawlers extends AbstractProvider
         'FeedBucket',
         'FeedBunch\/',
         'FeedBurner',
+        'feeder',
         'Feedly',
         'FeedshowOnline',
         'Feedspot',
@@ -433,6 +435,7 @@ class Crawlers extends AbstractProvider
         'Haansoft',
         'hackney\/',
         'Hadi Agent',
+        'HappyApps-WebCheck',
         'Hatena',
         'Havij',
         'HeadlessChrome',
@@ -479,7 +482,6 @@ class Crawlers extends AbstractProvider
         'huaweisymantec',
         'HubSpot ',
         'Humanlinks',
-        'HyperZbozi\.cz Feeder',
         'i2kconnect\/',
         'Iblog',
         'ichiro',
@@ -900,6 +902,7 @@ class Crawlers extends AbstractProvider
         'SEOprofiler',
         'SEOsearch',
         'seoscanners',
+        'seositecheckup',
         'SEOstats',
         'servernfo',
         'sexsearcher',
@@ -1007,6 +1010,7 @@ class Crawlers extends AbstractProvider
         'teoma',
         'terrainformatica',
         'Test Certificate Info',
+        'testuri',
         'Tetrahedron',
         'The Drop Reaper',
         'The Expert HTML Source Viewer',
@@ -1117,6 +1121,7 @@ class Crawlers extends AbstractProvider
         'WebEnhancer',
         'WebFetch',
         'WebFuck',
+        'WebGazer',
         'WebGo IS',
         'WebImageCollector',
         'WebImages',
@@ -1135,6 +1140,7 @@ class Crawlers extends AbstractProvider
         'websitepulse agent',
         'WebsiteQuester',
         'Websnapr',
+        'WebSniffer',
         'Webster',
         'WebStripper',
         'WebSucker',

diff --git a/tests/crawlers.txt b/tests/crawlers.txt
@@ -3414,4 +3414,11 @@ Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.8
 Outlook-iOS/696.1188109.prod.iphone (2.102.0)
 Polymail/1.47 (ctrlla.Polymail; build:64; OS X 10.12.6) Alamofire/4.5.1
 nyawc/1.8.1 CPython/3.6.5 Linux/3.10.0-862.9.1.el7.x86_64
-commonscan.org cralwer v1.01
+commonscan.org cralwer v1.01
+AdminLabs
+WebGazer/1.0 (+https://www.webgazer.io)
+HappyApps-WebCheck/1.0
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6; +feeder.co) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36
+Mozilla/5.0 (compatible; TestURI; +http://testuri.org/)
+WebSniffer/1.0 (+http://websniffer.cc/)
+Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1 SeoSiteCheckup (https://seositecheckup.co