Wordfence Security – Firewall & Malware Scan - Version 5.1.5

Version Description

  • Fix: Bing crawler was being misidentified as human. Fixed.
  • Fix: Escaping HTML on whois records. Thanks Nikhil Srivastava, TechDefencelabs (http://techdefencelabs.com)
Download this release

Release Info

Developer mmaunder
Plugin Icon 128x128 Wordfence Security – Firewall & Malware Scan
Version 5.1.5
Comparing to
See all releases

Code changes from version 5.1.4 to 5.1.5

Files changed (5) hide show
  1. js/admin.js +1 -0
  2. lib/wfCrawl.php +5 -1192
  3. lib/wordfenceClass.php +1 -1
  4. readme.txt +5 -1
  5. wordfence.php +2 -2
js/admin.js CHANGED
@@ -1080,6 +1080,7 @@ window['wordfenceAdmin'] = {
1080
if(res.ok && res.result && res.result.rawdata && res.result.rawdata.length > 0){
1081
var rawhtml = "";
1082
for(var i = 0; i < res.result.rawdata.length; i++){
1083
res.result.rawdata[i] = res.result.rawdata[i].replace(/([^\s\t\r\n:;]+@[^\s\t\r\n:;\.]+\.[^\s\t\r\n:;]+)/, "<a href=\"mailto:$1\">$1<\/a>");
1084
res.result.rawdata[i] = res.result.rawdata[i].replace(/(https?:\/\/[^\/]+[^\s\r\n\t]+)/, "<a target=\"_blank\" href=\"$1\">$1<\/a>");
1085
var redStyle = "";
1080
if(res.ok && res.result && res.result.rawdata && res.result.rawdata.length > 0){
1081
var rawhtml = "";
1082
for(var i = 0; i < res.result.rawdata.length; i++){
1083
+ res.result.rawdata[i] = jQuery('<div />').text(res.result.rawdata[i]).html();
1084
res.result.rawdata[i] = res.result.rawdata[i].replace(/([^\s\t\r\n:;]+@[^\s\t\r\n:;\.]+\.[^\s\t\r\n:;]+)/, "<a href=\"mailto:$1\">$1<\/a>");
1085
res.result.rawdata[i] = res.result.rawdata[i].replace(/(https?:\/\/[^\/]+[^\s\r\n\t]+)/, "<a target=\"_blank\" href=\"$1\">$1<\/a>");
1086
var redStyle = "";
lib/wfCrawl.php CHANGED
@@ -2,10 +2,10 @@
2
require_once('wfUtils.php');
3
class wfCrawl {
4
public static function isCrawler($UA){
5
- foreach(self::$patterns as $pat){
6
- if(preg_match($pat . 'i', $UA)){
7
- return true;
8
- }
9
}
10
return false;
11
}
@@ -48,28 +48,6 @@ class wfCrawl {
48
return false;
49
}
50
}
51
- public static function isBingCrawler(){
52
- $UA = (isset($_SERVER['HTTP_USER_AGENT']) ? $_SERVER['HTTP_USER_AGENT'] : '');
53
- foreach(self::$bingPat as $pat){
54
- if(preg_match($pat . 'i', $UA)){
55
- return true;
56
- }
57
- }
58
- return false;
59
- }
60
- private static $bingPat = array(
61
- '@^Mozilla/5\\.0 \\(compatible; bingbot/2\\..*http\\://www\\.bing\\.com/bingbot\\.htm\\)$@',
62
- '@^msnbot\\-webmaster/1\\.0 \\(.*http\\://search\\.msn\\.com/msnbot\\.htm\\)$@',
63
- '@^msnbot\\-NewsBlogs/2\\..* \\(\\+http\\://search\\.msn\\.com/msnbot\\.htm\\)$@',
64
- '@^renlifangbot/1\\.0 \\(.http\\://search\\.msn\\.com/msnbot\\.htm\\)$@',
65
- '@^adidxbot/1\\.1 \\(.http\\://search\\.msn\\.com/msnbot\\.htm\\)$@',
66
- '@^msnbot\\-media/1\\.1.*$@',
67
- '@^msnbot\\-media/1\\.0.*$@',
68
- '@^msnbot\\-products$@',
69
- '@^msnbot/2\\.0b.*$@',
70
- '@^msnbot/1\\.0.*$@',
71
- '@^msnbot/1\\.1.*$@'
72
- );
73
public static function isGooglebot(){
74
$UA = (isset($_SERVER['HTTP_USER_AGENT']) ? $_SERVER['HTTP_USER_AGENT'] : '');
75
if(preg_match('/Googlebot\/\d\.\d/', $UA)){ // UA: Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html) or (rarely used): Googlebot/2.1 (+http://www.google.com/bot.html)
@@ -111,1171 +89,6 @@ class wfCrawl {
111
'@^AdsBot\\-Google.*$@',
112
'@^Google$@'
113
);
114
- private static $patterns = array(
115
- '@^Mozilla/5\\.0 \\(Windows; U; Windows NT 5\\.1; en\\-US\\) Speedy Spider \\(http\\://www\\.entireweb\\.com/about/search_tech/speedy_spider/\\)$@',
116
- '@^Mozilla/4\\.0 \\(compatible; MSIE .*; Windows NT; Girafabot; girafabot at girafa dot com; http\\://www\\.girafa\\.com\\)$@',
117
- '@^Mozilla/5\\.0 \\(compatible; MSIE 7\\.0; MSIE 6\\.0; ScanAlert; \\+http\\://www\\.scanalert\\.com/bot\\.jsp\\) Firefox/.*$@',
118
- '@^Mozilla/5\\.0 \\(compatible; Speedy Spider; http\\://www\\.entireweb\\.com/about/search_tech/speedy_spider/\\)$@',
119
- '@^Runnk online rss reader \\: http\\://www\\.runnk\\.com/ \\: RSS favorites \\: RSS ranking \\: RSS aggregator.*$@',
120
- '@^SaladSpoon/ShopSalad 1\\..* \\(Search Engine crawler for ShopSalad\\.com; .*; crawler\\@shopsalad\\.com\\)$@',
121
- '@^Mozilla/5\\.0 \\(compatible; Kyluka crawl; http\\://www\\.kyluka\\.com/crawl\\.html; crawl\\@kyluka\\.com\\)$@',
122
- '@^Mozilla/5\\.0 \\(X11; U; Linux.*; .*\\) AppleWebKit/.*\\(KHTML, like Gecko\\) Safari/.* Epiphany/2\\.30\\..*$@',
123
- '@^DomainCrawler/1\\.0 \\(info\\@domaincrawler\\.com; http\\://www\\.domaincrawler\\.com/domains/view/.*\\)$@',
124
- '@^Mozilla/5\\.0 \\(compatible; YoudaoBot/1\\..*; http\\://www\\.youdao\\.com/help/webmaster/spider/.*\\)$@',
125
- '@^Mozilla/5\\.0 \\(compatible; Yahoo\\! DE Slurp; http\\://help\\.yahoo\\.com/help/us/ysearch/slurp\\)$@',
126
- '@^Metaspinner/0\\.01 \\(Metaspinner; http\\://www\\.meta\\-spinner\\.de/; support\\@meta\\-spinner\\.de/\\)$@',
127
- '@^browscap updater; interval\\:weekly; server\\:rohan\\.doppy\\.nl; questions\\:support\\@doppy\\.nl;$@',
128
- '@^Mozilla/5\\.0 \\(compatible; Scrubby/.*; \\+http\\://www\\.scrubtheweb\\.com/abs/meta\\-check\\.html\\)$@',
129
- '@^Mozilla/5\\.0 \\(compatible; Yahoo\\! Slurp.*;.*http\\://help\\.yahoo\\.com/help/us/ysearch/slurp\\)$@',
130
- '@^Automated Browscap\\.ini Updater\\. To report issues contact us at\\+http\\://www\\.skycomp\\.ca$@',
131
- '@^Mozilla/4\\.0 \\(compatible; http\\://search\\.thunderstone\\.com/texis/websearch/about\\.html\\)$@',
132
- '@^Mozilla/5\\.0 \\(compatible; Baiduspider/2\\.0; \\+http\\://www\\.baidu\\.com/search/spider\\.html\\)$@',
133
- '@^Mozilla/5\\.0 \\(compatible; KaloogaBot; http\\://www\\.kalooga\\.com/info\\.html.page\\=crawler\\)$@',
134
- '@^compatible; Mozilla 4\\.0; MSIE 5\\.5; \\(SqwidgeBot v1\\.01 \\- http\\://www\\.sqwidge\\.com/bot/\\)$@',
135
- '@^Automated Browscap\\.ini Updater\\. Gary contact me at serge\\@skycomp\\.ca with problems\\.$@',
136
- '@^Mozilla/5\\.0 \\(compatible; Yahoo\\! Slurp China.*;.*http\\://misc\\.yahoo\\.com\\.cn/help\\.html\\)$@',
137
- '@^Mozilla/5\\.0 \\(compatible; Yahoo\\!\\-AdCrawler;.*http\\://help\\.yahoo\\.com/yahoo_adcrawler\\)$@',
138
- '@^OOZBOT/0\\.20 \\( http\\://www\\.setooz\\.com/oozbot\\.html ; agentname at setooz dot_com \\)$@',
139
- '@^Mozilla/5\\.0 \\(compatible; MSIE 6\\.0; Podtech Network; crawler_admin\\@podtech\\.net\\)$@',
140
- '@^Mozilla/5\\.0 \\(compatible; FriendFeedBot/0\\..*; \\+Http\\://friendfeed\\.com/about/bot\\)$@',
141
- '@^Mozilla/5\\.0 \\(compatible; SuchbaerBot/0\\..*; \\+http\\://bot\\.suchbaer\\.de/info\\.html\\)$@',
142
- '@^Mozilla/5\\.0 \\(Yahoo\\-MMCrawler/.*; mailto\\:vertical\\-crawl\\-support\\@yahoo\\-inc\\.com\\)$@',
143
- '@^Mozilla/5\\.0 \\(compatible; Chirp/1\\.0; \\+http\\://www\\.binarycanary\\.com/chirp\\.cfm\\)$@',
144
- '@^Mozilla/5\\.0 \\(Macintosh; .; PPC Mac OS X;.*\\) AppleWebKit/.* \\(.*\\) HistoryHound/.*$@',
145
- '@^Mozilla/5\\.0 \\(X11; U; Linux.*; .*; rv\\:1\\..*; aggregator\\:NewsMonster; .*\\) Gecko/.*$@',
146
- '@^Mozilla/5\\.0 \\(.*Mac OS X.*\\) AppleWebKit/.* \\(KHTML, like Gecko, Safari/.*\\) ADM/.*$@',
147
- '@^Mozilla/5\\.0 \\(compatible; del\\.icio\\.us\\-thumbnails/.*; .*\\) KHTML/.* \\(like Gecko\\)$@',
148
- '@^Cocoal\\.icio\\.us/1\\.0 \\(v43\\) \\(Mac OS X; http\\://www\\.scifihifi\\.com/cocoalicious\\)$@',
149
- '@^Simpy/.* \\(Simpy; http\\://www\\.simpy\\.com/.ref\\=bot; feedback at simpy dot com\\)$@',
150
- '@^Mozilla/5\\.0 \\(compatible; ActiveTouristBot.*; http\\://www\\.activetourist\\.com\\)$@',
151
- '@^Speedy Spider \\(http\\://www\\.entireweb\\.com/about/search_tech/speedy_spider/\\)$@',
152
- '@^Mozilla/5\\.0 \\(.*\\) Gecko/.* Firefox/2\\.0 OneRiot/1\\.0 \\(http\\://www\\.oneriot\\.com\\) $@',
153
- '@^Mozilla/4\\.0 \\(compatible; .*Vagabondo/.*; webcrawler at wise\\-guys dot nl; .*\\)$@',
154
- '@^Mozilla/5\\.0 \\(X11; U; Linux.*; .*; rv\\:1\\..*; aggregator\\:FeedParser; .*\\) Gecko/.*$@',
155
- '@^facebookexternalhit/.* \\(.http\\://www\\.facebook\\.com/externalhit_uatext\\.php\\).*$@',
156
- '@^Mozilla/5\\.0 \\(Windows; U; Windows .*; .*; rv\\:.*\\) Gecko/.* Firefox/.* webaroo/.*$@',
157
- '@^Mozilla/5\\.0 \\(compatible; LegalAnalysisAgent/1\\..*; http\\://www\\.legalx\\.net\\)$@',
158
- '@^Mozilla/5\\.0 \\(compatible; .*; http\\://www\\.80legs\\.com/spider\\.html;\\) Gecko/.*$@',
159
- '@^Tumblr/.* RSS syndication \\( http\\://www\\.tumblr\\.com/\\) \\(support\\@tumblr\\.com\\)$@',
160
- '@^Mozilla/5\\.0 \\(compatible; Plukkie/1\\..; http\\://www\\.botje\\.com/plukkie\\.htm\\)$@',
161
- '@^Mozilla/5\\.0 \\(compatible; Webscan v0\\..*; \\+http\\://otc\\.dyndns\\.org/webscan/\\)$@',
162
- '@^Mozilla/5\\.0 \\(compatible; Newz Crawler .*; http\\://www\\.newzcrawler\\.com/.\\)$@',
163
- '@^Mozilla/5\\.0 \\(Macintosh; U; .*Mac OS X; .*\\) AppleWebKit/.* \\(.*\\) Pandora/2\\..*$@',
164
- '@^Microsoft Data Access Internet Publishing Provider Protocol Discovery$@',
165
- '@^Mozilla/5\\.0 \\(compatible; JadynAveBot; .*http\\://www\\.jadynave\\.com/robot.*$@',
166
- '@^Mozilla/5\\.0 \\(compatible; Lipperhey Spider; http\\://www\\.lipperhey\\.com/\\)$@',
167
- '@^Mozilla/5\\.0 \\(compatible; NetcraftSurveyAgent/1\\.0; .*info\\@netcraft\\.com\\)$@',
168
- '@^Mozilla/5\\.0 \\(compatible; MojeekBot/.*; http\\://www\\.mojeek\\.com/bot\\.html\\)$@',
169
- '@^Mozilla/5\\.0 \\(compatible; spbot/.*; \\+http\\://www\\.seoprofiler\\.com/bot/ \\)$@',
170
- '@^Mozilla/5\\.0 \\(compatible; bingbot/2\\..*http\\://www\\.bing\\.com/bingbot\\.htm\\)$@',
171
- '@^Atrax Solutions atraxbot/0\\..*; http\\://www\\.atraxsolutions\\.com/atraxbot$@',
172
- '@^Mozilla/5\\.0 \\(.*; Rojo .*; http\\://www\\.rojo\\.com/corporate/help/agg; .*\\).*$@',
173
- '@^Mozilla/5\\.0 \\(X11; U; Linux.*; .*; rv\\:1\\..*; aggregator\\:Rojo; .*\\) Gecko/.*$@',
174
- '@^Mozilla/5\\.0 \\(compatible; DBLBot/1\\.0; .http\\://www\\.dontbuylists\\.com/\\)$@',
175
- '@^CC Metadata Scaper http\\://wiki\\.creativecommons\\.org/Metadata_Scraper$@',
176
- '@^Mozilla/4\\.7 \\(compatible; WhizBang; http\\://www\\.whizbang\\.com/crawler\\)$@',
177
- '@^Mozilla/4\\.0 \\(compatible; MSIE 6\\.0; Windows 98\\) Web Link Validator.*$@',
178
- '@^rssImagesBot/0\\.1 \\(.*http\\://herbert\\.groot\\.jebbink\\.nl/.app\\=rssImages\\)$@',
179
- '@^Mozilla/5\\.0 \\(compatible; DotBot/.*; http\\://www\\.dotnetdotcom\\.org/.*\\)$@',
180
- '@^Mozilla/5\\.0 \\(compatible; Twingly Recon; http\\://www\\.twingly\\.com/\\)$@',
181
- '@^Microsoft Data Access Internet Publishing Provider Cache Manager$@',
182
- '@^Knight/0\\.. \\(Zook Knight; http\\://knight\\.zook\\.in/; knight\\@zook\\.in\\)$@',
183
- '@^Mozilla/5\\.0 \\(compatible; MSIE 7\\.0 .http\\://www\\.europarchive\\.org\\)$@',
184
- '@^Mozilla/.* \\(compatible; linktiger/.*; .*http\\://www\\.linktiger\\.com.*\\)$@',
185
- '@^Mozilla/4\\.0 \\(compatible; Link Utility; http\\://net\\-promoter\\.com\\)$@',
186
- '@^Mozilla/5\\.0 \\(compatible; unwrapbot/2\\..*; \\+http\\://www\\.unwrap\\.jp.*\\)$@',
187
- '@^roboobot/1\\..* \\(roboo; http\\://wap\\.roboo\\.com; winter\\.pi\\@roboo\\.com\\)$@',
188
- '@^Mozilla/4\\.0 \\(compatible; MSIE .\\.0; GomezAgent .\\.0; Windows NT\\)$@',
189
- '@^Mozilla/5\\.0 \\(compatible; ClixSense; http\\://www\\.clixsense\\.com/\\)$@',
190
- '@^Mozilla/5\\.0 \\(compatible; Diffbot/0\\.1; \\+http\\://www\\.diffbot\\.com\\)$@',
191
- '@^mozilla/5\\.0 \\(compatible; genevabot \\+http\\://www\\.healthdash\\.com\\)$@',
192
- '@^Mozilla/5\\.0 \\(.*aggregator\\:TailRank; http\\://tailrank\\.com/robot\\).*$@',
193
- '@^Mozilla/5\\.0 \\(compatible; SEODat/0\\..* http\\://crawler\\.seodat\\.com\\)$@',
194
- '@^WebImages .* \\(.http\\://herbert\\.groot\\.jebbink\\.nl/.app\\=WebImages.\\)$@',
195
- '@^Mozilla/5\\.0 \\(compatible; ScoutJet; \\+http\\://www\\.scoutjet\\.com/\\)$@',
196
- '@^Mozilla/5\\.0 \\(compatible; aiHitBot.*/.*; \\+http\\://www\\.aihit\\.com/\\)$@',
197
- '@^e\\-SocietyRobot\\(http\\://www\\.yama\\.info\\.waseda\\.ac\\.jp/~yamana/es/\\)$@',
198
- '@^Mozilla/4\\.0 \\(compatible; MSIE 7\\.0; Windows NT 6\\.0; Webaroo/.*\\)$@',
199
- '@^Mozilla/4\\.0 \\(compatible; BOTW Feed Grabber; .*http\\://botw\\.org\\)$@',
200
- '@^\\(HTML Validator http\\://www\\.searchengineworld\\.com/validator/\\)$@',
201
- '@^FeedOnFeeds/0\\.1\\..* \\( http\\://minutillo\\.com/steve/feedonfeeds/\\)$@',
202
- '@^Mozilla/4\\.0 \\(compatible; MSIE 7\\.0; Win32\\) Link Commander 3\\.0$@',
203
- '@^Mozilla/5\\.0 \\(compatible; Crawly/1\\..*; \\+http\\://.*/crawler\\.html\\)$@',
204
- '@^TwengaBot\\-Discover \\(http\\://www\\.twenga\\.fr/bot\\-discover\\.html\\)$@',
205
- '@^Mozilla/5\\.0 \\(compatible; Seznam screenshot\\-generator 2\\.0;.*\\)$@',
206
- '@^OpenTaggerBot \\(http\\://www\\.opentagger\\.com/opentaggerbot\\.htm\\)$@',
207
- '@^Mozilla/.* \\(compatible; OffByOne; Windows.*\\) Webster Pro V3\\..*$@',
208
- '@^Mozilla/5\\.0 \\(compatible; Twitturls; \\+http\\://twitturls\\.com\\)$@',
209
- '@^InfociousBot \\(.http\\://corp\\.infocious\\.com/tech_crawler\\.php\\)$@',
210
- '@^Mozilla/5\\.0 \\(compatible; RSSMicro\\.com RSS/Atom Feed Robot\\)$@',
211
- '@^onCHECK Linkchecker von www\\.scientec\\.de fuer www\\.onsinn\\.de$@',
212
- '@^Mozilla/5\\.0 \\(compatible; Ezooms/1\\.0; ezooms\\.bot\\@gmail\\.com\\)$@',
213
- '@^Mozilla/4\\.0 \\(compatible; smartBot/1\\..*; checking links; .*\\)$@',
214
- '@^Healthbot/Health_and_Longevity_Project_\\(HealthHaven\\.com\\) $@',
215
- '@^Linguee Bot \\(http\\://www\\.linguee\\.com/bot; bot\\@linguee\\.com\\)$@',
216
- '@^livedoor FeedFetcher/0\\.0.* \\(http\\://reader\\.livedoor\\.com/;.*\\)$@',
217
- '@^msnbot\\-webmaster/1\\.0 \\(.*http\\://search\\.msn\\.com/msnbot\\.htm\\)$@',
218
- '@^msnbot\\-NewsBlogs/2\\..* \\(\\+http\\://search\\.msn\\.com/msnbot\\.htm\\)$@',
219
- '@^DuckDuckBot/.*; \\(.http\\://duckduckgo\\.com/duckduckbot\\.html\\)$@',
220
- '@^Mobitype .* \\(compatible; Mozilla/.*; MSIE .*\\..*; Windows .*\\)$@',
221
- '@^Mozilla/4\\.0 \\(compatible; BOTW Spider; .*http\\://botw\\.org\\)$@',
222
- '@^Microsoft Data Access Internet Publishing Provider DAV.*$@',
223
- '@^Mozilla/.*\\(.*redditbot/.*http\\://www\\.reddit\\.com/feedback.*\\)$@',
224
- '@^FAST MetaWeb Crawler \\(helpdesk at fastsearch dot com\\)$@',
225
- '@^voyager/2\\.0 \\(http\\://www\\.kosmix\\.com/html/crawler\\.html\\)$@',
226
- '@^eStyleSearch .* \\(compatible; MSIE 6\\.0; Windows NT 5\\.0\\)$@',
227
- '@^1st ZipCommander \\(Net\\) \\- http\\://www\\.zipcommander\\.com/$@',
228
- '@^Mozilla/5\\.0 \\(compatible; YandexBot/.*; MirrorDetector\\)$@',
229
- '@^Busiversebot/v1\\.0 \\(http\\://www\\.busiverse\\.com/bot\\.php\\)$@',
230
- '@^Mozilla/5\\.0 \\(Danger hiptop 3\\..*; U; rv\\:1\\.7\\..*\\) Gecko/.*$@',
231
- '@^Strategic Board Bot \\(.http\\://www\\.strategicboard\\.com\\)$@',
232
- '@^renlifangbot/1\\.0 \\(.http\\://search\\.msn\\.com/msnbot\\.htm\\)$@',
233
- '@^envolk.ITS.spider/.* \\(.http\\://www\\.envolk\\.com/envolk.*\\)$@',
234
- '@^Mozilla/5\\.0 http\\://www\\.inclue\\.com; graeme\\@inclue\\.com$@',
235
- '@^Mozilla/4\\.0 \\(compatible; MSIE 4\\.01; Vonna\\.com b o t\\)$@',
236
- '@^Mozilla/5\\.0 \\(.*http\\://gnomit\\.com/\\) Gecko/.* Gnomit/1\\.0$@',
237
- '@^Browscap Mirror System/1\\..* \\(browscap\\.giantrealm\\.com\\)$@',
238
- '@^Mozilla/5\\.0 \\(compatible; AdHitz; http\\://adhitz\\.com/\\)$@',
239
- '@^neTVision AG andreas\\.heidoetting\\@thomson\\-webcast\\.net$@',
240
- '@^FeedHub MetaDataFetcher/1\\.0 \\(http\\://www\\.feedhub\\.com\\)$@',
241
- '@^Pingdom\\.com_bot_version_.*_\\(http\\://www\\.pingdom\\.com/\\)$@',
242
- '@^MapoftheInternet\\.com.\\(.http\\://MapoftheInternet\\.com\\)$@',
243
- '@^T\\-Mobile Dash Mozilla/4\\.0 \\(.*\\) MSNBOT\\-MOBILE/1\\.1 \\(.*\\)$@',
244
- '@^Mozilla/4\\.0 \\(compatible; Advanced Email Extractor.*\\)$@',
245
- '@^Windows\\-RSS\\-Platform/2\\.0 \\(MSIE .\\.0; Windows NT .*\\..*\\)$@',
246
- '@^online link validator \\(http\\://www\\.dead\\-links\\.com/\\)$@',
247
- '@^Godzilla/.* \\(Basic.*; .*; Commodore C\\=64; .*; rv\\:1\\..*\\).*$@',
248
- '@^FeedHub FeedDiscovery/1\\.0 \\(http\\://www\\.feedhub\\.com\\)$@',
249
- '@^Mozilla/5\\.0 \\(Macintosh; Intel Mac OS X\\) Excel/12\\..*$@',
250
- '@^Mozilla/4\\.0 \\(compatible; MSIE 6\\.0; Bluecoat DRTR\\)$@',
251
- '@^Patwebbot \\(http\\://www\\.herz\\-power\\.de/technik\\.html\\)$@',
252
- '@^findfiles\\.net/.* \\(Robot;test_robot\\@gmx\\-topmail\\.de\\)$@',
253
- '@^Mozilla/5\\.0 \\(compatible; IPCheck Server Monitor.*\\)$@',
254
- '@^DataFountains/DMOZ Feature Vector Corpus Creator.*$@',
255
- '@^WCC Browscap Updater/0\\..* \\(PHP\\: file_get_contents\\)$@',
256
- '@^Inne\\: Mozilla/4\\.0 \\(compatible; Cerberian Drtrs.*\\)$@',
257
- '@^wwwster/.* \\(Beta, mailto\\:gue\\@cis\\.uni\\-muenchen\\.de\\)$@',
258
- '@^FeedHub FeedFetcher/1\\.0 \\(http\\://www\\.feedhub\\.com\\)$@',
259
- '@^WebmasterWorld StickyMail Server Header Checker.*$@',
260
- '@^SafariBookmarkChecker.*\\(.http\\://www\\.coriolis\\.ch/\\)$@',
261
- '@^Sosospider.\\(\\+http\\://help\\.soso\\.com/webspider\\.htm\\)$@',
262
- '@^adidxbot/1\\.1 \\(.http\\://search\\.msn\\.com/msnbot\\.htm\\)$@',
263
- '@^gosospider Mozilla/5\\.0 \\(compatible; GosoSpider.*\\)$@',
264
- '@^Mozilla/5\\.0 \\(compatible; YahooSeeker/M1A1\\-R2D2.*\\)$@',
265
- '@^Mozilla/5\\.0 \\(compatible; BMI/.* \\(Y\\!J\\-AGENT; 1\\.0\\)\\)$@',
266
- '@^UniversalFeedParser/4\\..* \\+http\\://feedparser\\.org/$@',
267
- '@^Mozilla/5\\.0 \\(compatible; FirstSearchBot/1\\.0; .*\\)$@',
268
- '@^Mozilla/4\\.0 \\(compatible; Y\\!J; for robot study.*\\)$@',
269
- '@^Windows\\-Live\\-Social\\-Object\\-Extractor\\-Engine/1\\.0$@',
270
- '@^Mozilla/5\\.0 \\(compatible; YandexImageResizer/.*\\).*$@',
271
- '@^Rome Client \\(http\\://tinyurl\\.com/64t5n\\) Ver\\: 0\\..*$@',
272
- '@^Made by ZmEu \\@ WhiteHat v0\\..* \\(www\\.WhiteHat\\.ro\\)$@',
273
- '@^Mozilla/4\\.0 \\(compatible; MSIE 4\\.01; Windows95\\)$@',
274
- '@^Mozilla/5\\.0 \\(SnapPreviewBot\\) Gecko/.* Firefox/.*$@',
275
- '@^exactseek\\-pagereaper\\-.* \\(crawler\\@exactseek\\.com\\)$@',
276
- '@^Liferea/1\\..* \\(Linux; .*; http\\://liferea\\.sf\\.net/\\)$@',
277
- '@^Mozilla/5\\.0 \\(compatible; YandexBlogs/.*; robot\\)$@',
278
- '@^Mozilla/5\\.0 \\(compatible; Yahoo\\! SearchMonkey.*\\)$@',
279
- '@^mp3Spider cn\\-search\\-devel at yahoo\\-inc dot com$@',
280
- '@^Mozilla/2\\.0 \\(compatible; NEWT ActiveX; Win32\\)$@',
281
- '@^GOFORITBOT \\(.http\\://www\\.goforit\\.com/about/.\\)$@',
282
- '@^psbot/.* \\(.http\\://www\\.picsearch\\.com/bot\\.html\\)$@',
283
- '@^inetbot/.* \\(.http\\://www\\.inetbot\\.com/bot\\.html\\)$@',
284
- '@^Mozilla/4\\.0 \\(compatible; Scumbot/.*; Linux/.*\\)$@',
285
- '@^Mozilla/5\\.0 \\(compatible; Exabot\\-Images/3\\.0.*\\)$@',
286
- '@^CipinetBot \\(http\\://www\\.cipinet\\.com/bot\\.html\\)$@',
287
- '@^Tasap\\-image\\-robot/0\\..* \\(http\\://www\\.tasap\\.com\\)$@',
288
- '@^Browser Capabilities Project \\- PHP Browscap.*$@',
289
- '@^Mozilla/5\\.0 \\(compatible; BecomeJPBot/2\\.3; .*\\)$@',
290
- '@^UofTDB_experiment.* \\(leehyun\\@cs\\.toronto\\.edu\\)$@',
291
- '@^ddetailsbot \\(http\\://www\\.displaydetails\\.com\\)$@',
292
- '@^Mozilla/4\\.0\\(.compatible; MSIE 6\\.0; Qihoo .*\\)$@',
293
- '@^CMS crawler \\(.http\\://buytaert\\.net/crawler/\\)$@',
294
- '@^Mozilla/5\\.0 \\(compatible;YodaoBot\\-Image/1\\..*\\)$@',
295
- '@^Mozilla/5\\.0 \\(compatible; BMF/.* \\(Y\\!J\\-AGENT\\)\\)$@',
296
- '@^Mozilla/5\\.0 \\(compatible; BMC/.* \\(Y\\!J\\-AGENT\\)\\)$@',
297
- '@^Mozilla/5\\.0 \\(compatible; Yahoo\\! Verifier/.*\\)$@',
298
- '@^Mozilla/5\\.0 \\(compatible; YandexFavicons/.*\\).*$@',
299
- '@^Susie \\(http\\://www\\.sync2it\\.com/bms/susie\\.php$@',
300
- '@^Mozilla/.\\.0 \\(compatible; Ask Jeeves/Teoma.*\\)$@',
301
- '@^Mozilla/5\\.0 \\(compatible; Butterfly/1\\.0; .*\\).*$@',
302
- '@^Mozilla/5\\.0 \\(compatible; archive\\.org_bot.*\\)$@',
303
- '@^Mozilla/4\\.0 \\(compatible; Cerberian Drtrs.*\\)$@',
304
- '@^Mozilla/5\\.0 \\(compatible; Viralheat Bot/.*\\) $@',
305
- '@^WAP_Browser/5\\.0 \\(compatible; YodaoBot/1\\..*\\)$@',
306
- '@^Tecomi Bot \\(http\\://www\\.tecomi\\.com/bot\\.htm\\)$@',
307
- '@^HTML2JPG Blackbox, http\\://www\\.html2jpg\\.com$@',
308
- '@^NextGenSearchBot.*\\(for information visit .*\\)$@',
309
- '@^Mozilla/5\\.0 \\(compatible; Quantcastbot/1\\..*\\)$@',
310
- '@^Mozilla/4\\.0 \\(compatible; ChangeDetection/.*$@',
311
- '@^Mozilla/5\\.0 \\(compatible; BuzzRankingBot/.*\\)$@',
312
- '@^Mozilla/5\\.0 \\(compatible; YandexMetrika/.*\\).*$@',
313
- '@^ZIBB Crawler \\(email address / WWW address\\)$@',
314
- '@^Mozilla/5\\.0 \\(compatible; YandexCatalog/.*\\).*$@',
315
- '@^Search Engine World Robots\\.txt Validator.*$@',
316
- '@^Mozilla/5\\.0 \\(compatible; EARTHCOM\\.info/.*\\)$@',
317
- '@^Mozilla/4\\.0 \\(compatible; SuperCleaner.*;.*\\)$@',
318
- '@^Html Link Validator \\(www\\.lithopssoft\\.com\\)$@',
319
- '@^Mozilla/5\\.0 \\(compatible; YandexImages/.*\\).*$@',
320
- '@^Mozilla/5\\.0 \\(compatible; YandexDirect/.*\\).*$@',
321
- '@^Mozilla/5\\.0 \\(compatible; YandexAddurl/.*\\).*$@',
322
- '@^envolk/.* \\(.http\\://www\\.envolk\\.com/envolk.*\\)$@',
323
- '@^Mozilla/5\\.0 \\(compatible; nextthing\\.org/.*\\)$@',
324
- '@^Mozilla/5\\.0 \\(compatible; CXL\\-FatAssANT.*\\)$@',
325
- '@^Mozilla/5\\.0 \\(compatible; Charlotte/.*; .*\\)$@',
326
- '@^Mozilla/4\\.0 \\(compatible; BorderManager.*\\)$@',
327
- '@^Mozilla/5\\.0 \\(compatible; Theophrastus/.*\\)$@',
328
- '@^Mozilla/5\\.0 \\(compatible; ParchBot/1\\.0;.*\\)$@',
329
- '@^BookmarkBase\\(2/;http\\://bookmarkbase\\.com\\)$@',
330
- '@^AideRSS/1\\.0 \\(aiderss\\.com\\); .* subscribers$@',
331
- '@^DotBot/.* \\(http\\://www\\.dotnetdotcom\\.org/.*\\)$@',
332
- '@^JRTwine Software Check Favorites Utility$@',
333
- '@^DocWeb Link Crawler \\(http\\://doc\\.php\\.net\\)$@',
334
- '@^Mozilla/5\\.0 \\(compatible; YandexMedia/.*\\).*$@',
335
- '@^FAST Data Search Document Retriever/4\\.0.*$@',
336
- '@^Mozilla/5\\.0 \\(compatible; YandexVideo/.*\\).*$@',
337
- '@^Mozilla/4\\.0 \\(compatible; Spider; Linux\\)$@',
338
- '@^Bloglines/.* \\(http\\://www\\.bloglines\\.com.*\\)$@',
339
- '@^Mozilla/5\\.0 \\(compatible; YandexNews/.*\\).*$@',
340
- '@^Mozilla/5\\.0 \\(compatible; ayna\\-crawler.*\\)$@',
341
- '@^Mozilla/4\\.0 \\(compatible; MyFamilyBot/.*\\)$@',
342
- '@^Microsoft Bing Mobile SocialStreams Bot$@',
343
- '@^http\\://hilfe\\.acont\\.de/bot\\.html ACONTBOT$@',
344
- '@^Mozilla/5\\.0 \\(compatible;.*newstin\\.com;.*\\)$@',
345
- '@^JS\\-Kit URL Resolver, http\\://js\\-kit\\.com/$@',
346
- '@^Mahalobot/1\\.0 \\(.http\\://www\\.mahalo\\.com/\\)$@',
347
- '@^WIRE/.* \\(Linux.*Bot,Robot,Spider,Crawler\\)$@',
348
- '@^Microsoft Office/.* \\(.*Picture Manager.*\\)$@',
349
- '@^Willow Internet Crawler by Twotrees V.*$@',
350
- '@^Visited by http\\://tools\\.geek\\-tools\\.org$@',
351
- '@^Mozilla/5\\.0 \\(compatible; YodaoBot/1\\..*\\)$@',
352
- '@^Mozilla/5\\.0 \\(compatible; YandexBot/.*\\).*$@',
353
- '@^OpenDNS Domain Crawler noc\\@opendns\\.com$@',
354
- '@^annotate_google; http\\://ponderer\\.org/.*$@',
355
- '@^Mozilla/5\\.0 \\(compatible; DNS\\-Digger/.*\\)$@',
356
- '@^Mozilla/5\\.0 \\(compatible; DKIMRepBot/.*\\)$@',
357
- '@^www\\.fi crawler, contact crawler\\@www\\.fi$@',
358
- '@^Mozilla/3\\.0 \\(compatible; Indy Library\\)$@',
359
- '@^Amico Alpha .* \\(.*\\) Gecko/.* AmicoAlpha/.*$@',
360
- '@^Kolinka Forum Search \\(www\\.kolinka\\.com\\)$@',
361
- '@^Mozilla/5\\.0 \\(compatible; MJ12bot/v1\\..*\\)$@',
362
- '@^Mozilla/5\\.0 \\(compatible; AboutUsBot/.*\\)$@',
363
- '@^Best Whois \\(http\\://www\\.bestwhois\\.net/\\)$@',
364
- '@^BlogPulseLive \\(support\\@blogpulse\\.com\\)$@',
365
- '@^ah\\-ha\\.com crawler \\(crawler\\@ah\\-ha\\.com\\)$@',
366
- '@^Mozilla/4\\.0 \\(compatible; ibisBrowser\\)$@',
367
- '@^Decode Framework 0\\..* browscap library$@',
368
- '@^AnswerBus \\(http\\://www\\.answerbus\\.com/\\)$@',
369
- '@^\\!Susie \\(http\\://www\\.sync2it\\.com/susie\\)$@',
370
- '@^Mozilla/5\\.0 \\(compatible\\) GM RSS Panel$@',
371
- '@^Mozilla/5\\.0 \\(compatible; Exabot/3\\.0.*\\)$@',
372
- '@^Microsoft Windows Network Diagnostics$@',
373
- '@^Mozilla/5\\.0 \\(compatible; NLCrawler/.*$@',
374
- '@^Mozilla/2\\.0 \\(compatible; Ask Jeeves\\)$@',
375
- '@^CatchBot/.*; \\+http\\://www\\.catchbot\\.com$@',
376
- '@^Microsoft Office Existence Discovery$@',
377
- '@^Feedster Crawler/.\\.0; Feedster, Inc\\.$@',
378
- '@^A1 Website Download/1\\..* \\(.*\\) miggibot$@',
379
- '@^Mozilla/4\\.0 \\(compatible; N\\-Stealth\\)$@',
380
- '@^Mozilla/.* \\(compatible; WebCapture.*\\)$@',
381
- '@^Mozilla/4\\.0 \\(compatible; DepSpid/.*\\)$@',
382
- '@^RixBot \\(http\\://babelserver\\.org/rix\\)$@',
383
- '@^Feedreader .* \\(Powered by Newsbrain\\)$@',
384
- '@^Microsoft Office Protocol Discovery$@',
385
- '@^Mozilla/5\\.0 \\(.*Google Keyword Tool.*\\)$@',
386
- '@^Mozilla/5\\.0 \\(compatible; Pogodak\\..*\\)$@',
387
- '@^Internet Content Rating Association$@',
388
- '@^check_http/.* \\(nagios\\-plugins 1\\.4\\..*\\)$@',
389
- '@^check_http/.* \\(nagios\\-plugins 1\\.3\\..*\\)$@',
390
- '@^check_http/.* \\(nagios\\-plugins 1\\.2\\..*\\)$@',
391
- '@^check_http/.* \\(nagios\\-plugins 1\\.1\\..*\\)$@',
392
- '@^PHP Browser Capabilities Project/.*$@',
393
- '@^Mono Browser Capabilities Updater.*$@',
394
- '@^.*Spinn3r.*http\\://spinn3r\\.com/robot.*$@',
395
- '@^Superpages URL Verification Engine$@',
396
- '@^kinjabot \\(http\\://www\\.kinja\\.com; .*\\)$@',
397
- '@^BlijbolReallySimpleAggregator/2\\.0.*$@',
398
- '@^Mozilla/4\\.0 \\(Compatible\\); URLBase.*$@',
399
- '@^Mozilla/4\\.0 \\(compatible; Getleft.*\\)$@',
400
- '@^Mozilla/5\\.0 \\(compatible; Webbot/.*\\)$@',
401
- '@^Mozilla/5\\.0 \\(.*Feedfetcher\\-Google.*\\)$@',
402
- '@^Feedfetcher\\-Google\\-iGoogleGadgets.*$@',
403
- '@^Mozilla/5\\.0 \\(compatible; Vermut.*\\)$@',
404
- '@^Pagebull http\\://www\\.pagebull\\.com/$@',
405
- '@^check_http/.* \\(nagios\\-plugins 1\\..*\\)$@',
406
- '@^SCEJ PSP BROWSER 0102pspNavigator$@',
407
- '@^Mozilla/4\\.0 \\(compatible; Arachmo\\)$@',
408
- '@^Mozilla/5\\.0 \\(compatible; NGBot/.*\\)$@',
409
- '@^Nozilla/P\\.N \\(Just for IDS woring\\)$@',
410
- '@^Fooky\\.com/ScorpionBot/ScoutOut;.*$@',
411
- '@^NewsGator FetchLinks extension/.*$@',
412
- '@^Mozilla/4\\.0 \\(compatible; Win32\\)$@',
413
- '@^AcadiaUniversityWebCensusClient$@',
414
- '@^MetaProducts Download Express/.*$@',
415
- '@^Mozilla/0\\.9.* no dos \\:\\) \\(Linux.*\\)$@',
416
- '@^Lorkyll .*\\..* \\-\\- lorkyll\\@444\\.net$@',
417
- '@^FAST\\-WebCrawler/.*/PartnerSite.*$@',
418
- '@^Mozilla/.* \\(Win32;.*Escape..*; .\\)$@',
419
- '@^Mozilla/5\\.0 CostaCider Search.*$@',
420
- '@^DataFountains/DMOZ Downloader.*$@',
421
- '@^Mozilla/5\\.0 \\(compatible; OsO;.*$@',
422
- '@^Mozilla/5\\.0 \\(RSS Reader Panel\\)$@',
423
- '@^Kapere \\(http\\://www\\.kapere\\.com\\)$@',
424
- '@^Mozilla/5\\.0 \\(.*Google Desktop.*\\)$@',
425
- '@^MFC Foundation Class Library.*$@',
426
- '@^Bookmark Renewal Check Agent.*$@',
427
- '@^Jigsaw/.* W3C_CSS_Validator.*/.*$@',
428
- '@^West Wind Internet Protocols.*$@',
429
- '@^YooW\\!/.* \\(.http\\://www\\.yoow\\.eu\\)$@',
430
- '@^WebIndexer/.* \\(Web Indexer; .*\\)$@',
431
- '@^SynapticSearch/AI Crawler 1\\..$@',
432
- '@^FAST\\-WebCrawler/.*.Multimedia.*$@',
433
- '@^PigBlock \\(Windows NT 5\\.1; U\\).*$@',
434
- '@^ConveraMultiMediaCrawler/0\\.1.*$@',
435
- '@^Mozilla/5\\.0 GurujiBot/1\\.0 \\(.*\\)$@',
436
- '@^Microsoft\\-WebDAV\\-MiniRedir/.*$@',
437
- '@^FAST\\-WebCrawler/.*/FirstPage.*$@',
438
- '@^Irish Blogs Aggregator/.*1\\.0.*$@',
439
- '@^ogeb browser , Version 1\\.1\\.0$@',
440
- '@^KakleBot \\- www\\.kakle\\.com/0\\.1$@',
441
- '@^.*Netcraft Web Server Survey.*$@',
442
- '@^HolmesBot \\(http\\://holmes\\.ge\\)$@',
443
- '@^Norbert the Spider\\(Burf\\.com\\)$@',
444
- '@^Epsilon SoftWorks\' MailMunky$@',
445
- '@^Miva \\(AlgoFeedback\\@miva\\.com\\)$@',
446
- '@^sitecheck\\.internetseer\\.com.*$@',
447
- '@^PDFBot \\(crawler\\@pdfind\\.com\\)$@',
448
- '@^FastSearch Web Crawler for.*$@',
449
- '@^Mozilla/.* \\(.*\\) \\- BrowseX \\(.*\\)$@',
450
- '@^POE\\-Component\\-Client\\-HTTP/.*$@',
451
- '@^Live \\(http\\://www\\.live\\.com/\\)$@',
452
- '@^.*Netcraft Webserver Survey.*$@',
453
- '@^http\\://www\\.yellowpages\\.com.*$@',
454
- '@^Microsoft Internet Explorer$@',
455
- '@^yp\\-crawl\\@attinteractive\\.com$@',
456
- '@^Newsgroupreporter LinkCheck$@',
457
- '@^Yahoo\\! Slurp/Site Explorer$@',
458
- '@^Der gro\\xdfe BilderSauger.*$@',
459
- '@^Mozilla/5\\.0 \\(Yahoo\\-Test/.*\\)$@',
460
- '@^searchbot admin\\@google\\.com$@',
461
- '@^.*E\\-Mail Address Extractor.*$@',
462
- '@^DomainsDB\\.net MetaCrawler.*$@',
463
- '@^AideRSS 2\\..* \\(postrank\\.com\\)$@',
464
- '@^LucidMedia ClickSense/4\\..$@',
465
- '@^Mozilla/5\\.0 gURLChecker/.*$@',
466
- '@^Mozilla/5\\.0 \\(YahooYSMcm.*\\)$@',
467
- '@^AideRSS/2\\.0 \\(aiderss\\.com\\)$@',
468
- '@^Windows\\-RSS\\-Platform/1\\.0.*$@',
469
- '@^Fast PartnerSite Crawler.*$@',
470
- '@^.*FAST Enterprise Crawler.*$@',
471
- '@^Mozilla/5\\.0 \\(.*\\) VoilaBot.*$@',
472
- '@^bot/.* \\(bot; .*bot\\@bot\\.bot\\)$@',
473
- '@^Google\\-Site\\-Verification.*$@',
474
- '@^Lincoln State Web Browser$@',
475
- '@^ICRA_Semantic_spider/0\\..$@',
476
- '@^Google OpenSocial agent.*$@',
477
- '@^Lycos\\-Spider_\\(modspider\\)$@',
478
- '@^ICRA_label_generator/1\\..$@',
479
- '@^ShowXML/1\\.0 libwww/5\\.4\\.0$@',
480
- '@^Nutch/0\\.. \\(OpenX Spider\\)$@',
481
- '@^FAST\\-WebCrawler/.*/Fresh.*$@',
482
- '@^Oracle Enterprise Search$@',
483
- '@^Tcl http client package.*$@',
484
- '@^appie.*\\(www\\.walhello\\.com\\)$@',
485
- '@^Kopiczek/.* \\(WyderOS.*; .*\\)$@',
486
- '@^IPiumBot laurion\\(dot\\)com$@',
487
- '@^BlogPulse \\(ISSpider\\-3\\..*\\)$@',
488
- '@^spider \\(tspyyp\\@tom\\.com\\)$@',
489
- '@^PubSub\\-RSS\\-Reader/.* \\(.*\\)$@',
490
- '@^gazz/.*\\(gazz\\@nttr\\.co\\.jp\\)$@',
491
- '@^blogsearchbot\\-pumpkin\\-2$@',
492
- '@^WebTrends Link Analyzer$@',
493
- '@^Domain Dossier utility.*$@',
494
- '@^Favorites Sweeper v\\.3\\..*$@',
495
- '@^ATA\\-Translation\\-Service$@',
496
- '@^CyberSpyder Link Test/.*$@',
497
- '@^XML Sitemaps Generator.*$@',
498
- '@^Bloglines Title Fetch/.*$@',
499
- '@^Shelob \\(shelob\\@gmx\\.net\\)$@',
500
- '@^SocialSpider\\-Finder/0\\..*$@',
501
- '@^sitemonitor\\@dnsvr\\.com/.*$@',
502
- '@^Myst Monitor Service v.*$@',
503
- '@^Open Web Analytics Bot.*$@',
504
- '@^Mozilla/5\\.0 \\(Twiceler.*\\)$@',
505
- '@^Extreme Picture Finder$@',
506
- '@^TargetYourNews\\.com bot$@',
507
- '@^Iltrovatore\\-.etaccio/.*$@',
508
- '@^JetBrains Omea Reader.*$@',
509
- '@^Microsoft URL Control.*$@',
510
- '@^Miscellaneous Browsers$@',
511
- '@^YahooSeeker/CafeKelsa.*$@',
512
- '@^AndroidDownloadManager$@',
513
- '@^Sogou develop spider/.*$@',
514
- '@^WinScripter iNet Tools$@',
515
- '@^websitepulse checker/.*$@',
516
- '@^HuaweiSymantecSpider/.*$@',
517
- '@^.*maxamine\\.com\\-\\-robot.*$@',
518
- '@^Mozilla/3\\.0 \\(INGRID/.*$@',
519
- '@^Netchart Adv Crawler.*$@',
520
- '@^.*BecomeBot\\@exava\\.com.*$@',
521
- '@^Browscap Mirror v1\\.30$@',
522
- '@^MSNBot\\-NewsBlogs/1\\.0.*$@',
523
- '@^Bookmark search tool.*$@',
524
- '@^.*Googlebot\\-Mobile/2\\..*$@',
525
- '@^Infoseek SideWinder/.*$@',
526
- '@^Mozilla/5\\.0 NewsFox/.*$@',
527
- '@^Cocoal\\.icio\\.us/.* \\(.*\\).*$@',
528
- '@^ezic\\.com http agent .*$@',
529
- '@^http\\://arachnode\\.net.*$@',
530
- '@^AdsBot\\-Google\\-Mobile.*$@',
531
- '@^Atomic_Email_Hunter/.*$@',
532
- '@^copyright sheriff \\(.*\\)$@',
533
- '@^iCopyright Conductor.*$@',
534
- '@^google \\(.*Enterprise.*\\)$@',
535
- '@^Mediapartners\\-Google.*$@',
536
- '@^NSO_Debugger_User/2\\.0$@',
537
- '@^Copyright/Plagiarism$@',
538
- '@^MSNBot\\-Academic/1\\.0.*$@',
539
- '@^GoogleFriendConnect.*$@',
540
- '@^Lycos\\-Spider_\\(T\\-Rex\\)$@',
541
- '@^googlebot\\-urlconsole$@',
542
- '@^8484 Boston Project.*$@',
543
- '@^Sogou Orion spider/.*$@',
544
- '@^.*Baiduspider\\-mobile.*$@',
545
- '@^TeragramWebcrawler/.*$@',
546
- '@^NetCarta_WebMapper/.*$@',
547
- '@^LinkLint\\-checkonly/.*$@',
548
- '@^.*Google Web Preview.*$@',
549
- '@^.*Web Link Validator.*$@',
550
- '@^Internet Exploiter/.*$@',
551
- '@^HenryTheMiragoRobot.*$@',
552
- '@^SOFTWING_TEAR_AGENT.*$@',
553
- '@^Funnel Web Profiler.*$@',
554
- '@^Radiation Retriever.*$@',
555
- '@^Microsoft Visio MSIE$@',
556
- '@^BackStreet Browser .*$@',
557
- '@^Wizz RSS News Reader$@',
558
- '@^Poodle_predictor_1\\.0$@',
559
- '@^Web Image Collector.*$@',
560
- '@^GigabotSiteSearch/.*$@',
561
- '@^semanticdiscovery/.*$@',
562
- '@^Sogou Push Spider/.*$@',
563
- '@^wadaino\\.jp\\-crawler.*$@',
564
- '@^Feedfetcher\\-Google.*$@',
565
- '@^Dumbot \\(version .*\\).*$@',
566
- '@^.*Baiduspider\\-image.*$@',
567
- '@^Seeker\\.lookseek\\.com$@',
568
- '@^Z\\-Add Link Checker.*$@',
569
- '@^Oracle Ultra Search$@',
570
- '@^Internet Explorer .*$@',
571
- //'@^WordPress\\-Do\\-P\\-/2\\..*$@',
572
- '@^Sogou\\-Test\\-Spider/.*$@',
573
- '@^PlantyNet_WebRobot.*$@',
574
- '@^ExactSeek Crawler/.*$@',
575
- '@^.*Baiduspider\\-video.*$@',
576
- '@^MetaGer\\-LinkChecker$@',
577
- '@^.*Fluffy the spider.*$@',
578
- '@^QuickFinder Crawler$@',
579
- '@^Website Downloader.*$@',
580
- '@^LightningDownload/.*$@',
581
- '@^Offline Downloader.*$@',
582
- '@^Sogou Pic Spider/.*$@',
583
- '@^Sogou head spider.*$@',
584
- '@^Site Valet Online.*$@',
585
- '@^.*LinksManager\\.com.*$@',
586
- '@^Twisted PageGetter$@',
587
- '@^Feed43 Proxy/.* \\(.*\\)$@',
588
- '@^AppleSyndication/.*$@',
589
- '@^BitTorrent Clients$@',
590
- '@^Link Valet Online.*$@',
591
- '@^Xenu.* Link Sleuth.*$@',
592
- '@^Flaming AttackBot.*$@',
593
- '@^Internet Explore .*$@',
594
- '@^Artera \\(Version .*\\)$@',
595
- '@^Forschungsportal/.*$@',
596
- '@^masidani_bot_v0\\.6.*$@',
597
- '@^magpie\\-crawler/1\\..*$@',
598
- '@^BrowscapUpdater1\\.0$@',
599
- '@^Searchmee\\! Spider.*$@',
600
- '@^W3C\\-mobileOK/DDC\\-.*$@',
601
- '@^DomainWatcher Bot.*$@',
602
- '@^PEAR HTTP_Request.*$@',
603
- '@^NetZip Downloader.*$@',
604
- '@^NexTools WebAgent.*$@',
605
- '@^Links4US\\-Crawler,.*$@',
606
- '@^LetsCrawl\\.com/1\\.0.*$@',
607
- '@^Website eXtractor.*$@',
608
- '@^FollowSite\\.com \\(.*\\)$@',
609
- '@^A \\.NET Web Crawler$@',
610
- '@^.*Exabot\\@exava\\.com.*$@',
611
- '@^.*Baiduspider\\-cpro.*$@',
612
- '@^.*Baiduspider\\-favo.*$@',
613
- '@^YahooExternalCache$@',
614
- '@^boitho\\.com\\-robot/.*$@',
615
- '@^.*Baiduspider\\-news.*$@',
616
- '@^NewsGatorOnline/.*$@',
617
- '@^Xerka WebBot v1\\..*$@',
618
- '@^msnbot\\-media/1\\.1.*$@',
619
- '@^Keyword Density/.*$@',
620
- '@^YahooVideoSearch.*$@',
621
- '@^Research Projects$@',
622
- '@^WebsiteExtractor.*$@',
623
- '@^msnbot\\-media/1\\.0.*$@',
624
- '@^Scooter/.*Y\\!CrawlX$@',
625
- '@^HatenaScreenshot.*$@',
626
- '@^AppEngine\\-Google.*$@',
627
- '@^Inet \\- Eureka App$@',
628
- '@^niXXieBot.Foster.*$@',
629
- '@^Hatena\\:\\:Crawler/.*$@',
630
- '@^sogou js robot\\(.*\\)$@',
631
- '@^sogou web spider.*$@',
632
- '@^Y\\!J SearchMonkey.*$@',
633
- '@^Talkro Web\\-Shot/.*$@',
634
- '@^Crawl_Application$@',
635
- '@^Tutorial Crawler.*$@',
636
- '@^REL Link Checker.*$@',
637
- '@^DomainsBotBot/1\\..*$@',
638
- '@^Star.*Downloader/.*$@',
639
- '@^FAST\\-WebCrawler/.*$@',
640
- '@^Xaldon WebSpider.*$@',
641
- '@^Offline Explorer.*$@',
642
- '@^BabalooSpider/1\\..*$@',
643
- '@^WWWeasel Robot v.*$@',
644
- '@^Exabot\\-Images/1\\.0$@',
645
- '@^Download Managers$@',
646
- '@^.*Baiduspider\\-ads.*$@',
647
- '@^Download Express.*$@',
648
- '@^WebAlta Crawler/.*$@',
649
- '@^E\\-Mail Harvesters$@',
650
- '@^Blue Coat Systems$@',
651
- '@^BaiduImageSpider.*$@',
652
- '@^Feeds Syndicators$@',
653
- '@^Hatena Bookmark/.*$@',
654
- '@^Social Networking$@',
655
- '@^LinkextractorPro.*$@',
656
- '@^Franklin Locator.*$@',
657
- '@^Missigua Locator.*$@',
658
- '@^InternetArchive/.*$@',
659
- '@^librabot/1\\.0 \\(.*\\)$@',
660
- '@^MSNBot\\-News/1\\.0.*$@',
661
- '@^Googlebot\\-Video.*$@',
662
- '@^Zend_Http_Client$@',
663
- '@^Version Checkers$@',
664
- '@^YahooFeedSeeker.*$@',
665
- '@^Yahoo\\-MMCrawler.*$@',
666
- '@^.*W3C\\-checklink/.*$@',
667
- '@^Web Downloader/.*$@',
668
- '@^UtilMind HTTPGet$@',
669
- '@^Desktop Sidebar.*$@',
670
- '@^Browscap updater$@',
671
- '@^Hatena Antenna/.*$@',
672
- '@^USyd\\-NLP\\-Spider.*$@',
673
- //'@^WordPress\\-B\\-/2\\..*$@',
674
- '@^STEROID Download$@',
675
- '@^NuSearch Spider.*$@',
676
- '@^Net\\:\\:Trackback/.*$@',
677
- '@^.*PhotoStickies/.*$@',
678
- '@^FeedDigest/.* \\(.*\\)$@',
679
- '@^BlueCoat ProxySG$@',
680
- '@^CerberianDrtrs/.*$@',
681
- '@^Download Wonder.*$@',
682
- '@^Download Master.*$@',
683
- '@^DownloadSession.*$@',
684
- '@^URL Spider Pro/.*$@',
685
- '@^ZeBot_www\\.ze\\.bz.*$@',
686
- '@^SquigglebotBot/.*$@',
687
- '@^ActiveBookmark .*$@',
688
- '@^Bookmark\\-Manager$@',
689
- '@^.*WinHttpRequest.*$@',
690
- '@^Internet Archive$@',
691
- '@^Toata dragostea.*$@',
692
- '@^TencentTraveler.*$@',
693
- '@^Googlebot\\-Image.*$@',
694
- '@^Google\\-Sitemaps.*$@',
695
- '@^Taiga web spider$@',
696
- '@^WebFilter Robot.*$@',
697
- '@^Microsoft BITS/.*$@',
698
- '@^RPT\\-HTTPClient/.*$@',
699
- '@^MSN Feed Manager$@',
700
- '@^Offline Browsers$@',
701
- '@^Go\\-Ahead\\-Got\\-It.*$@',
702
- '@^PluckItCrawler/.*$@',
703
- '@^fetch libfetch/.*$@',
704
- '@^General Crawlers$@',
705
- '@^YodaoBot/1\\..* \\(.*\\)$@',
706
- '@^.*altervista\\.org.*$@',
707
- '@^ConveraCrawler/.*$@',
708
- '@^Sogou Pic Agent$@',
709
- '@^Web Downloader.*$@',
710
- '@^Lsearch/sondeur$@',
711
- '@^FeedDemon/.* \\(.*\\)$@',
712
- '@^NewsAlloy/.* \\(.*\\)$@',
713
- '@^boitho\\.com\\-dc/.*$@',
714
- '@^MagpieRSS/.* \\(.*\\)$@',
715
- '@^Technoratibot/.*$@',
716
- '@^Googlebot/Test.*$@',
717
- '@^Website Quester$@',
718
- '@^SpeedDownload/.*$@',
719
- '@^UpTime Checker.*$@',
720
- '@^SmartDownload/.*$@',
721
- '@^MSR\\-ISRCCrawler$@',
722
- '@^OpenISearch/1\\..*$@',
723
- '@^ObjectsSearch/.*$@',
724
- '@^EmailCollector.*$@',
725
- '@^MFC_Tear_Sample$@',
726
- '@^msnbot\\-products$@',
727
- '@^WWW\\-Mechanize/.*$@',
728
- '@^CherryPicker.*/.*$@',
729
- '@^LinkedInBot/1\\..*$@',
730
- '@^Googlebot\\-News.*$@',
731
- '@^HyperEstraier/.*$@',
732
- '@^Webclipping\\.com$@',
733
- '@^Visicom Toolbar$@',
734
- '@^JUST\\-CRAWLER\\(.*\\)$@',
735
- '@^BitTorrentMac/.*$@',
736
- '@^Enigma Browser.*$@',
737
- '@^Tagyu Agent/1\\.0$@',
738
- '@^FeedValidator/.*$@',
739
- '@^Best of the Web$@',
740
- '@^HTML Validators$@',
741
- '@^libcurl\\-agent/.*$@',
742
- '@^libWeb/clsHTTP.*$@',
743
- '@^YRL_ODP_CRAWLER$@',
744
- '@^ScollSpider/2\\..*$@',
745
- '@^Bookmark Buddy.*$@',
746
- '@^.*Googlebot/2\\.1.*$@',
747
- '@^FLATARTS_FAVICO$@',
748
- '@^Pete\\-Spider/1\\..*$@',
749
- '@^Protopage/.* \\(.*\\)$@',
750
- '@^W3C_Validator/.*$@',
751
- '@^FreshDownload/.*$@',
752
- '@^UpdateBrowscap.*$@',
753
- '@^Internet Ninja.*$@',
754
- '@^URL Spider SQL.*$@',
755
- '@^Yahoo\\-MMAudVid.*$@',
756
- '@^CopyRightCheck.*$@',
757
- '@^Download Ninja.*$@',
758
- '@^Kontiki Client.*$@',
759
- '@^Download Demon.*$@',
760
- '@^ActiveRefresh.*$@',
761
- '@^.*avantbrowser.*$@',
762
- '@^Jayde Crawler.*$@',
763
- '@^Feedshow/.* \\(.*\\)$@',
764
- '@^ALeadSoftbot/.*$@',
765
- '@^VCI WebViewer.*$@',
766
- '@^Twitterbot/0\\..*$@',
767
- '@^W3C\\-mobileOK/.*$@',
768
- '@^.*NetNewsWire/.*$@',
769
- '@^.*NetVisualize.*$@',
770
- '@^Cogentbot/1\\...*$@',
771
- '@^Apple\\-PubSub/.*$@',
772
- '@^favorstarbot/.*$@',
773
- '@^Yahoo\\! Mindset$@',
774
- '@^FairAd Client.*$@',
775
- '@^Transmission/.*$@',
776
- '@^BigCliqueBOT/.*$@',
777
- '@^NetID\\.com Bot.*$@',
778
- '@^shaboyi spider$@',
779
- '@^WebDownloader.*$@',
780
- '@^StackRambler/.*$@',
781
- '@^HiddenMarket\\-.*$@',
782
- '@^naoFavicon4IE.*$@',
783
- '@^WebCrawler_1\\..*$@',
784
- '@^Kenjin Spider.*$@',
785
- '@^Kevin http\\://.*$@',
786
- '@^Y\\!J\\-BRP/YFSBJ.*$@',
787
- '@^Fast/AllTheWeb$@',
788
- '@^AdsBot\\-Google.*$@',
789
- '@^Search Engines$@',
790
- '@^hitcrawler_0\\..*$@',
791
- '@^RealDownload/.*$@',
792
- '@^Netprospector.*$@',
793
- '@^YadowsCrawler.*$@',
794
- '@^NetNewsWire.*/.*$@',
795
- '@^CydralSpider/.*$@',
796
- '@^Image Crawlers$@',
797
- '@^CrawlConvera.*$@',
798
- '@^ShopWiki/1\\.0.*$@',
799
- '@^.*Powermarks/.*$@',
800
- '@^GameSpyHTTP/.*$@',
801
- '@^AC\\-BaiduBot/.*$@',
802
- '@^www\\.zatka\\.com$@',
803
- '@^TheInformant.*$@',
804
- '@^Theme Spider.*$@',
805
- '@^.*Baiduspider.*$@',
806
- '@^Y\\!J\\-BRO/YFSJ.*$@',
807
- '@^GurujiBot/1\\..*$@',
808
- '@^Shim.Crawler.*$@',
809
- '@^GoGuidesBot/.*$@',
810
- '@^LinkSweeper/.*$@',
811
- '@^TurnitinBot/.*$@',
812
- '@^BlockNote\\.Net$@',
813
- '@^JCheckLinks/.*$@',
814
- '@^TutorGigBot/.*$@',
815
- '@^i1searchbot/.*$@',
816
- '@^exactseek\\.com$@',
817
- '@^LinkChecker/.*$@',
818
- '@^Link Checkers$@',
819
- '@^CazoodleBot/.*$@',
820
- '@^GetRightPro/.*$@',
821
- '@^P3P Validator$@',
822
- '@^ProWebWalker.*$@',
823
- '@^LocalcomBot/.*$@',
824
- '@^Y\\!J\\-VSC/ViSe.*$@',
825
- '@^Yahoo Mindset$@',
826
- '@^FeedGhost/1\\..*$@',
827
- '@^Twingly Recon$@',
828
- '@^TerrawizBot/.*$@',
829
- '@^URLy Warning.*$@',
830
- '@^.*TweakMASTER.*$@',
831
- '@^Y\\!J\\-BRQ/DLCK.*$@',
832
- '@^RedCell/.* \\(.*\\)$@',
833
- '@^Site Monitors$@',
834
- '@^.*MSIECrawler.*$@',
835
- '@^ExtractorPro.*$@',
836
- '@^EMAILsearcher$@',
837
- '@^GreatNews/1\\.0$@',
838
- '@^metatagsdir/.*$@',
839
- '@^Exabot\\-Test/.*$@',
840
- '@^IrssiUrlLog/.*$@',
841
- '@^Net Vampire/.*$@',
842
- '@^Covario\\-IDS/.*$@',
843
- '@^MonkeyCrawl/.*$@',
844
- '@^uTorrentMac/.*$@',
845
- '@^SharpReader/.*$@',
846
- '@^W3CLineMode/.*$@',
847
- '@^flatlandbot/.*$@',
848
- '@^WWW_Browser/.*$@',
849
- '@^MovableType/.*$@',
850
- '@^panscient\\.com$@',
851
- '@^FyberSpider.*$@',
852
- '@^Ace Explorer$@',
853
- '@^GroschoBot/.*$@',
854
- '@^NetRecorder.*$@',
855
- '@^GreenBrowser$@',
856
- '@^Jayde Online$@',
857
- '@^IconSurf/2\\..*$@',
858
- '@^FeedBurner/.*$@',
859
- '@^intraVnews/.*$@',
860
- '@^Netvibes \\(.*\\)$@',
861
- '@^Omnipelagos.*$@',
862
- '@^Exalead NG/.*$@',
863
- '@^WebEnhancer.*$@',
864
- '@^Atomic_Email$@',
865
- '@^LMQueueBot/.*$@',
866
- '@^ContactBot/.*$@',
867
- '@^EmailSiphon.*$@',
868
- '@^Mnogosearch.*$@',
869
- '@^BitTornado/.*$@',
870
- '@^BitTorrent/.*$@',
871
- '@^RLinkCheker.*$@',
872
- '@^CheckLinks/.*$@',
873
- '@^iVia Project$@',
874
- '@^Net_Vampire.*$@',
875
- '@^webcrawl\\.net$@',
876
- '@^.*AgentName/.*$@',
877
- '@^.*FDSE robot.*$@',
878
- '@^URL Control.*$@',
879
- '@^http generic$@',
880
- '@^URL_Access/.*$@',
881
- '@^HTMLParser/.*$@',
882
- '@^msnbot/2\\.0b.*$@',
883
- '@^NetMechanic.*$@',
884
- '@^IP.*Works\\!.*/.*$@',
885
- '@^iexplore\\.exe$@',
886
- '@^libtorrent/.*$@',
887
- '@^.*WebGrabber.*$@',
888
- '@^.*SqwidgeBot.*$@',
889
- '@^Seram Server$@',
890
- '@^DeepIndexer.*$@',
891
- '@^Y\\!J\\-BRG/GSC.*$@',
892
- '@^Hatena RSS/.*$@',
893
- '@^sogou spider$@',
894
- '@^WebSnatcher.*$@',
895
- '@^Webster Pro.*$@',
896
- '@^Feed\\:\\:Find/.*$@',
897
- '@^MSMOBOT/1\\.1.*$@',
898
- '@^WebGatherer.*$@',
899
- '@^WebStripper.*$@',
900
- '@^SiteSucker/.*$@',
901
- '@^NextopiaBOT.*$@',
902
- '@^gsa\\-crawler.*$@',
903
- '@^Nudelsalat/.*$@',
904
- '@^ia_archiver.*$@',
905
- '@^SiteSnagger.*$@',
906
- '@^SiteParser/.*$@',
907
- '@^W3C\\-WebCon/.*$@',
908
- '@^DISCo Pump .*$@',
909
- '@^ProxyTester.*$@',
910
- '@^YahooSeeker.*$@',
911
- '@^bitlybot/2\\..*$@',
912
- '@^Gulper Web .*$@',
913
- '@^CyberPatrol.*$@',
914
- '@^UbiCrawler/.*$@',
915
- '@^.*BecomeBot/.*$@',
916
- '@^User.*Agent\\:.*$@',
917
- '@^Yahoo\\-Blogs.*$@',
918
- '@^URI\\:\\:Fetch/.*$@',
919
- '@^Yahoo Pipes.*$@',
920
- '@^.*www4mail/.*$@',
921
- '@^NewsGator/.*$@',
922
- '@^Feeds Blogs$@',
923
- '@^msnbot/1\\.0.*$@',
924
- '@^DoCoMo/2\\.0.*$@',
925
- '@^KBeeBot/0\\..*$@',
926
- '@^WebWhacker.*$@',
927
- '@^SlySearch/.*$@',
928
- '@^DoCoMo/1\\.0.*$@',
929
- '@^.*EasyRider.*$@',
930
- '@^INGRID/.\\.0.*$@',
931
- '@^BitBeamer/.*$@',
932
- '@^Sqeobot/0\\..*$@',
933
- '@^Mail\\.Ru/1\\.0$@',
934
- '@^Translators$@',
935
- '@^Tarantula/.*$@',
936
- '@^Unicorn/1\\..*$@',
937
- '@^FANGCrawl/.*$@',
938
- '@^NG\\-Search/.*$@',
939
- '@^SITECHECKER$@',
940
- '@^NetReality.*$@',
941
- '@^Popdexter/.*$@',
942
- '@^msnbot/1\\.1.*$@',
943
- '@^Cynthia 1\\.0$@',
944
- '@^Search\\-10/.*$@',
945
- '@^WebPatrol/.*$@',
946
- '@^PicaLoader.*$@',
947
- '@^NameProtect$@',
948
- '@^.*WebMon .\\..*$@',
949
- '@^BasicHTTP/.*$@',
950
- '@^CFNetwork/.*$@',
951
- '@^CFSCHEDULE.*$@',
952
- '@^LinkAlarm/.*$@',
953
- '@^LinkWalker.*$@',
954
- '@^Ad Muncher.*$@',
955
- '@^FindLinks/.*$@',
956
- '@^ColdFusion.*$@',
957
- '@^Sunrise/0\\..*$@',
958
- '@^DataCha0s/.*$@',
959
- '@^IIITBOT/1\\..*$@',
960
- '@^YahooYSMcm.*$@',
961
- '@^Surf Knight$@',
962
- '@^SurveyBot/.*$@',
963
- '@^Robozilla/.*$@',
964
- '@^Zao\\-Crawler$@',
965
- '@^Amfibibot/.*$@',
966
- '@^Blaiz\\-Bee/.*$@',
967
- '@^SlimBrowser$@',
968
- '@^Eule.Robot.*$@',
969
- '@^.*Check&Get.*$@',
970
- '@^WebTrends/.*$@',
971
- '@^webcollage.*$@',
972
- '@^OutfoxBot/.*$@',
973
- '@^FOTOCHECKER$@',
974
- '@^Gregarius/.*$@',
975
- '@^SiteWinder.*$@',
976
- '@^ScoutAbout.*$@',
977
- '@^llssbot/1\\.0$@',
978
- '@^RepoMonkey.*$@',
979
- '@^Mozdex/0\\.7.*$@',
980
- '@^SuperHTTP/.*$@',
981
- '@^webbandit/.*$@',
982
- //'@^WordPress/.*$@',
983
- '@^Marvin v0\\.3$@',
984
- '@^Akregator/.*$@',
985
- '@^Web Magnet.*$@',
986
- '@^gnome\\-vfs/.*$@',
987
- '@^RssBandit/.*$@',
988
- '@^httpclient.*$@',
989
- '@^Lycos\\-Proxy$@',
990
- '@^HTTPFetch/.*$@',
991
- '@^HttpSession$@',
992
- '@^Mister PIX.*$@',
993
- '@^shelob v1\\..*$@',
994
- '@^SeznamBot/.*$@',
995
- '@^pixfinder/.*$@',
996
- '@^RssBar/1\\.2.*$@',
997
- '@^Camcrawler.*$@',
998
- '@^RedCarpet/.*$@',
999
- '@^SimplePie/.*$@',
1000
- '@^General RSS$@',
1001
- '@^LapozzBot/.*$@',
1002
- '@^.*CFNetwork.*$@',
1003
- '@^ICE_GetFile$@',
1004
- '@^ZipppBot/.*$@',
1005
- '@^BitComet/.*$@',
1006
- '@^KTorrent/.*$@',
1007
- '@^EuripBot/.*$@',
1008
- '@^Search Fst$@',
1009
- '@^Pajaczek/.*$@',
1010
- '@^USER_AGENT$@',
1011
- '@^VengaBot/.*$@',
1012
- '@^DiamondBot$@',
1013
- '@^antibot\\-V.*$@',
1014
- '@^JetBrains.*$@',
1015
- '@^Drupal \\(.*\\)$@',
1016
- '@^Exabot/2\\.0$@',
1017
- '@^My Browser$@',
1018
- '@^Exabot/3\\.0$@',
1019
- '@^DNSGroup/.*$@',
1020
- '@^BilgiBot/.*$@',
1021
- '@^TouchStone$@',
1022
- '@^iSEEKbot/.*$@',
1023
- '@^KRetrieve/$@',
1024
- '@^search\\.ch.*$@',
1025
- '@^Tkensaku/.*$@',
1026
- '@^SiteSpider$@',
1027
- '@^NavissoBot$@',
1028
- '@^Linknzbot.*$@',
1029
- '@^MaSagool/.*$@',
1030
- '@^miniRank/.*$@',
1031
- '@^YodaoBot/.*$@',
1032
- '@^HooWWWer/.*$@',
1033
- '@^SuperBot/.*$@',
1034
- '@^InetURL\\:/.*$@',
1035
- '@^PageNest/.*$@',
1036
- '@^httpunit/.*$@',
1037
- '@^Webcheck .*$@',
1038
- '@^NetPumper.*$@',
1039
- '@^P3P Client$@',
1040
- '@^NetSucker.*$@',
1041
- '@^sherlock/.*$@',
1042
- '@^EmailWolf.*$@',
1043
- '@^WebSauger.*$@',
1044
- '@^NewsFire/.*$@',
1045
- '@^MFHttpScan$@',
1046
- '@^W3CRobot/.*$@',
1047
- '@^Twitturly.*$@',
1048
- '@^LinkScan/.*$@',
1049
- '@^WorQmada/.*$@',
1050
- '@^.*LinkLint.*$@',
1051
- '@^WebCopier.*$@',
1052
- '@^URL2File/.*$@',
1053
- '@^webfetch/.*$@',
1054
- '@^RSS Menu/.*$@',
1055
- '@^WebReaper.*$@',
1056
- '@^lftp/3\\.2\\.1$@',
1057
- '@^Word Press$@',
1058
- '@^uTorrent/.*$@',
1059
- '@^rtorrent/.*$@',
1060
- '@^.*heritrix.*$@',
1061
- '@^AutoHotkey$@',
1062
- '@^GetRight/.*$@',
1063
- '@^GetSmart/.*$@',
1064
- '@^.*research.*$@',
1065
- '@^.*Pingdom.*$@',
1066
- '@^Harvest/.*$@',
1067
- '@^Net Probe$@',
1068
- '@^httperf/.*$@',
1069
- '@^SpankBot.*$@',
1070
- '@^.*ickHTTP.*$@',
1071
- '@^WebRipper$@',
1072
- '@^LeechGet.*$@',
1073
- '@^OCN\\-SOC/.*$@',
1074
- '@^Pageload.*$@',
1075
- '@^OSSProxy.*$@',
1076
- '@^MediaGet.*$@',
1077
- '@^woriobot.*$@',
1078
- '@^Steeler/.*$@',
1079
- '@^INetURL/.*$@',
1080
- '@^Entireweb$@',
1081
- '@^IlseBot/.*$@',
1082
- '@^MarsEdit.*$@',
1083
- '@^NewsGator$@',
1084
- '@^eventax/.*$@',
1085
- '@^cisco\\-IOS$@',
1086
- '@^polybot..*$@',
1087
- '@^Snapbot/.*$@',
1088
- '@^WinkBot/.*$@',
1089
- '@^WebAuto/.*$@',
1090
- '@^Checkbot.*$@',
1091
- '@^SiteBar/.*$@',
1092
- '@^URLBase/.*$@',
1093
- '@^Bookdog/.*$@',
1094
- '@^.*Linkman.*$@',
1095
- '@^sna\\-0\\.0\\..*$@',
1096
- '@^WebMiner.*$@',
1097
- '@^Weblide/.*$@',
1098
- '@^Hotzonu/.*$@',
1099
- '@^Microsoft$@',
1100
- '@^.*HTTrack.*$@',
1101
- '@^iGetter/.*$@',
1102
- '@^Deepindex$@',
1103
- '@^Faxobot/.*$@',
1104
- '@^Filangy/.*$@',
1105
- '@^MSProxy/.*$@',
1106
- '@^geniebot.*$@',
1107
- '@^Gaisbot/.*$@',
1108
- '@^Chilkat/.*$@',
1109
- '@^Scrubby/.*$@',
1110
- '@^WISEbot/.*$@',
1111
- '@^WebCorp/.*$@',
1112
- '@^updated/.*$@',
1113
- '@^TMCrawler$@',
1114
- '@^AutoMate5$@',
1115
- '@^DNS Tools$@',
1116
- '@^LOOQ/0\\.1.*$@',
1117
- '@^zibber\\-v.*$@',
1118
- '@^IWAgent/.*$@',
1119
- '@^Go\\!Zilla.*$@',
1120
- '@^Gozilla/.*$@',
1121
- '@^Prozilla.*$@',
1122
- '@^sEasyDL/.*$@',
1123
- '@^shareaza.*$@',
1124
- '@^PageDown.*$@',
1125
- '@^Seekbot/.*$@',
1126
- '@^Szukacz/.*$@',
1127
- '@^SygolBot.*$@',
1128
- '@^sproose/.*$@',
1129
- '@^Teleport.*$@',
1130
- '@^Vegas95/.*$@',
1131
- '@^ASPSeek/.*$@',
1132
- '@^Y\\!J\\-PSC.*$@',
1133
- '@^Netcraft$@',
1134
- '@^Y\\!J\\-NSC.*$@',
1135
- '@^Y\\!J\\-SRD.*$@',
1136
- '@^Y\\!OASIS.*$@',
1137
- '@^JPluck/.*$@',
1138
- '@^Gigabot.*$@',
1139
- '@^Y\\!J\\-DSC.*$@',
1140
- '@^Y\\!J\\-BSC.*$@',
1141
- '@^Scooter.*$@',
1142
- '@^Linkbot.*$@',
1143
- '@^WebTrans$@',
1144
- '@^Y\\!J\\-BRE.*$@',
1145
- '@^VadixBot$@',
1146
- '@^Y\\!J\\-BRI.*$@',
1147
- '@^Particls$@',
1148
- '@^YodaoBot$@',
1149
- '@^.*libwww.*$@',
1150
- '@^PEERbot.*$@',
1151
- '@^Spinne/.*$@',
1152
- '@^3D\\-FTP/.*$@',
1153
- '@^Pompos/.*$@',
1154
- '@^XSpider.*$@',
1155
- '@^CobWeb/.*$@',
1156
- '@^FlashGet$@',
1157
- '@^NetAnts.*$@',
1158
- '@^.*Larbin.*$@',
1159
- '@^WinTools$@',
1160
- '@^Facebook$@',
1161
- '@^IRLbot/.*$@',
1162
- '@^SBIder/.*$@',
1163
- '@^Lachesis$@',
1164
- '@^PycURL/.*$@',
1165
- '@^SBL\\-BOT.*$@',
1166
- '@^Wotbox/.*$@',
1167
- '@^Ocelli/.*$@',
1168
- '@^LeechFTP$@',
1169
- '@^PhpDig/.*$@',
1170
- '@^WhizBang$@',
1171
- '@^Azureus.*$@',
1172
- '@^Moozilla$@',
1173
- '@^HTTPGrab$@',
1174
- '@^n4p_bot.*$@',
1175
- '@^Holmes/.*$@',
1176
- '@^WebFetch$@',
1177
- '@^EasyDL/.*$@',
1178
- '@^WinHttp.*$@',
1179
- '@^NetSurf.*$@',
1180
- '@^Gaisbot.*$@',
1181
- '@^URLCHECK$@',
1182
- '@^FDM 1\\.x$@',
1183
- '@^Exalead$@',
1184
- '@^3wGet/.*$@',
1185
- '@^Deluge.*$@',
1186
- '@^.*Nutch.*$@',
1187
- '@^Gigabot$@',
1188
- '@^Rippers$@',
1189
- '@^AIBOT/.*$@',
1190
- '@^VMBot/.*$@',
1191
- '@^SynoBot$@',
1192
- '@^Myzilla$@',
1193
- '@^Beamer.*$@',
1194
- '@^Abacho.*$@',
1195
- '@^BDFetch$@',
1196
- '@^WeBoX/.*$@',
1197
- '@^Convera$@',
1198
- '@^MSRBOT.*$@',
1199
- '@^Webaroo$@',
1200
- '@^WebZIP.*$@',
1201
- '@^pavuk/.*$@',
1202
- '@^eCatch.*$@',
1203
- '@^Dumbot.*$@',
1204
- '@^moget/.*$@',
1205
- '@^SyncMgr$@',
1206
- '@^Reblog.*$@',
1207
- '@^.*naver.*$@',
1208
- '@^nicebot$@',
1209
- '@^Pockey.*$@',
1210
- '@^WebPix.*$@',
1211
- '@^Python.*$@',
1212
- '@^CTerm/.*$@',
1213
- '@^HLoader$@',
1214
- '@^IECheck$@',
1215
- '@^FeedHub$@',
1216
- '@^htdig/.*$@',
1217
- '@^.*Amiga.*$@',
1218
- '@^Foobot.*$@',
1219
- '@^cosmos.*$@',
1220
- '@^PMAFind$@',
1221
- '@^SMBot/.*$@',
1222
- '@^Qweery.*$@',
1223
- '@^btbot/.*$@',
1224
- '@^Snoopy.*$@',
1225
- '@^NPBot.*$@',
1226
- '@^WebGet$@',
1227
- '@^Custo.*$@',
1228
- '@^.*grub.*$@',
1229
- '@^hcat/.*$@',
1230
- '@^FavOrg$@',
1231
- '@^abot/.*$@',
1232
- '@^.*Zeus.*$@',
1233
- '@^Become$@',
1234
- '@^Nagios$@',
1235
- '@^DoCoMo$@',
1236
- '@^cz32ts$@',
1237
- '@^nabot.*$@',
1238
- '@^NV32ts$@',
1239
- '@^Lycos.*$@',
1240
- '@^Google$@',
1241
- '@^Hatena$@',
1242
- '@^BTSP/.*$@',
1243
- '@^Envolk$@',
1244
- '@^WWWC/.*$@',
1245
- '@^MQbot.*$@',
1246
- '@^.*java.*$@',
1247
- '@^DotBot$@',
1248
- '@^Yandex$@',
1249
- '@^Boitho$@',
1250
- '@^UCmore$@',
1251
- '@^Baidu$@',
1252
- '@^Zao/.*$@',
1253
- '@^Nutch$@',
1254
- '@^curl.*$@',
1255
- '@^Lycos$@',
1256
- '@^Sogou$@',
1257
- '@^Wget.*$@',
1258
- '@^FGet.*$@',
1259
- '@^DLC/.*$@',
1260
- '@^Yahoo$@',
1261
- '@^VSE/.*$@',
1262
- '@^sohu.*$@',
1263
- '@^slurp$@',
1264
- '@^b2w/.*$@',
1265
- '@^Ilse$@',
1266
- '@^FDM.*$@',
1267
- '@^lwp.*$@',
1268
- '@^ng/.*$@',
1269
- '@^CAST$@',
1270
- '@^NP/.*$@',
1271
- '@^Snap$@',
1272
- '@^DA .*$@',
1273
- '@^oBot$@',
1274
- '@^PHP.*$@',
1275
- '@^Ask$@',
1276
- '@^MSN$@',
1277
- '@^W3C$@',
1278
- '@^0$@'
1279
- );
1280
}
1281
?>
2
require_once('wfUtils.php');
3
class wfCrawl {
4
public static function isCrawler($UA){
5
+ $browscap = new wfBrowscap();
6
+ $b = $browscap->getBrowser($UA);
7
+ if($b && isset($b['Crawler']) && $b['Crawler']){
8
+ return true;
9
}
10
return false;
11
}
48
return false;
49
}
50
}
51
public static function isGooglebot(){
52
$UA = (isset($_SERVER['HTTP_USER_AGENT']) ? $_SERVER['HTTP_USER_AGENT'] : '');
53
if(preg_match('/Googlebot\/\d\.\d/', $UA)){ // UA: Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html) or (rarely used): Googlebot/2.1 (+http://www.google.com/bot.html)
89
'@^AdsBot\\-Google.*$@',
90
'@^Google$@'
91
);
92
+
93
}
94
?>
lib/wordfenceClass.php CHANGED
@@ -61,8 +61,8 @@ class wordfence {
61
//Remove all scheduled scans.
62
self::unscheduleAllScans();
63
64
if(wfConfig::get('deleteTablesOnDeact')){
65
- wfConfig::clearDiskCache();
66
$schema = new wfSchema();
67
$schema->dropAll();
68
foreach(array('wordfence_version', 'wordfenceActivated') as $opt){
61
//Remove all scheduled scans.
62
self::unscheduleAllScans();
63
64
+ wfConfig::clearDiskCache();
65
if(wfConfig::get('deleteTablesOnDeact')){
66
$schema = new wfSchema();
67
$schema->dropAll();
68
foreach(array('wordfence_version', 'wordfenceActivated') as $opt){
readme.txt CHANGED
@@ -3,7 +3,7 @@ Contributors: mmaunder
3
Tags: wordpress, security, performance, speed, caching, cache, caching plugin, wordpress cache, wordpress caching, wordpress security, security plugin, secure, anti-virus, malware, firewall, antivirus, virus, google safe browsing, phishing, scrapers, hacking, wordfence, securty, secrity, secure, two factor, cellphone sign-in, cellphone signin, cellphone, twofactor, security, secure, htaccess, login, log, users, login alerts, lock, chmod, maintenance, plugin, private, privacy, protection, permissions, 503, base64, injection, code, encode, script, attack, hack, hackers, block, blocked, prevent, prevention, RFI, XSS, CRLF, CSRF, SQL Injection, vulnerability, website security, WordPress security, security log, logging, HTTP log, error log, login security, personal security, infrastructure security, firewall security, front-end security, web server security, proxy security, reverse proxy security, secure website, secure login, two factor security, maximum login security, heartbleed, heart bleed, heartbleed vulnerability, openssl vulnerability, nginx, litespeed, php5-fpm, woocommerce support, woocommerce caching
4
Requires at least: 3.3.1
5
Tested up to: 3.9.1
6
- Stable tag: 5.1.4
7
8
Wordfence Security is a free enterprise class security and performance plugin that makes your site up to 50 times faster and more secure.
9
@@ -163,6 +163,10 @@ cause a security hole on your site.
163
164
== Changelog ==
165
166
= 5.1.4 =
167
* Feature: Auto updates for Wordfence! This is a much-requested feature by our power admin's. Enable the "Update Wordfence automatically when a new version is released" option on the Wordfence options page.
168
* Fix: Security fix. Thanks to Narendra Bhati from Suma Soft.
3
Tags: wordpress, security, performance, speed, caching, cache, caching plugin, wordpress cache, wordpress caching, wordpress security, security plugin, secure, anti-virus, malware, firewall, antivirus, virus, google safe browsing, phishing, scrapers, hacking, wordfence, securty, secrity, secure, two factor, cellphone sign-in, cellphone signin, cellphone, twofactor, security, secure, htaccess, login, log, users, login alerts, lock, chmod, maintenance, plugin, private, privacy, protection, permissions, 503, base64, injection, code, encode, script, attack, hack, hackers, block, blocked, prevent, prevention, RFI, XSS, CRLF, CSRF, SQL Injection, vulnerability, website security, WordPress security, security log, logging, HTTP log, error log, login security, personal security, infrastructure security, firewall security, front-end security, web server security, proxy security, reverse proxy security, secure website, secure login, two factor security, maximum login security, heartbleed, heart bleed, heartbleed vulnerability, openssl vulnerability, nginx, litespeed, php5-fpm, woocommerce support, woocommerce caching
4
Requires at least: 3.3.1
5
Tested up to: 3.9.1
6
+ Stable tag: 5.1.5
7
8
Wordfence Security is a free enterprise class security and performance plugin that makes your site up to 50 times faster and more secure.
9
163
164
== Changelog ==
165
166
+ = 5.1.5 =
167
+ * Fix: Bing crawler was being misidentified as human. Fixed.
168
+ * Fix: Escaping HTML on whois records. Thanks Nikhil Srivastava, TechDefencelabs (http://techdefencelabs.com)
169
+
170
= 5.1.4 =
171
* Feature: Auto updates for Wordfence! This is a much-requested feature by our power admin's. Enable the "Update Wordfence automatically when a new version is released" option on the Wordfence options page.
172
* Fix: Security fix. Thanks to Narendra Bhati from Suma Soft.
wordfence.php CHANGED
@@ -4,13 +4,13 @@ Plugin Name: Wordfence Security
4
Plugin URI: http://www.wordfence.com/
5
Description: Wordfence Security - Anti-virus, Firewall and High Speed Cache
6
Author: Wordfence
7
- Version: 5.1.4
8
Author URI: http://www.wordfence.com/
9
*/
10
if(defined('WP_INSTALLING') && WP_INSTALLING){
11
return;
12
}
13
- define('WORDFENCE_VERSION', '5.1.4');
14
if(get_option('wordfenceActivated') != 1){
15
add_action('activated_plugin','wordfence_save_activation_error'); function wordfence_save_activation_error(){ update_option('wf_plugin_act_error', ob_get_contents()); }
16
}
4
Plugin URI: http://www.wordfence.com/
5
Description: Wordfence Security - Anti-virus, Firewall and High Speed Cache
6
Author: Wordfence
7
+ Version: 5.1.5
8
Author URI: http://www.wordfence.com/
9
*/
10
if(defined('WP_INSTALLING') && WP_INSTALLING){
11
return;
12
}
13
+ define('WORDFENCE_VERSION', '5.1.5');
14
if(get_option('wordfenceActivated') != 1){
15
add_action('activated_plugin','wordfence_save_activation_error'); function wordfence_save_activation_error(){ update_option('wf_plugin_act_error', ob_get_contents()); }
16
}