Skip to content

Instantly share code, notes, and snippets.

@PrathameshSam

PrathameshSam/log_file_analyzer.sh

Last active March 19, 2021 12:22
Show Gist options
  • Save PrathameshSam/fa3ee9bc7259c04af65221ee4b330271 to your computer and use it in GitHub Desktop.
Save PrathameshSam/fa3ee9bc7259c04af65221ee4b330271 to your computer and use it in GitHub Desktop.
Download ZIP
Simple Unix script to fetch unique IP and corresponding user agents from a log file
Raw
log_file_analyzer.sh
#!/bin/bash
log_file=$1
if [ $# -eq 1 ]
then
output=`cat $1 | cut -d " " -f 1 -f 12- | uniq`
echo "$output"
else
echo "No file passed! Or too many files passed"
fi
Raw
unique_ip_unique_agent.sh
#!/bin/bash
log_file=$1
if [ $# -eq 1 ]
then
unique_ips=`cat $1 | cut -d " " -f 1 | uniq`
unique_user_agents=`cat $1 | cut -d " " -f 12- | uniq`
echo $"Unique IP's"
echo "$unique_ips"
echo "==========="
echo "Unique user agents"
echo "$unique_user_agents"
else
echo "No file passed! Or too many files passed"
fi
Raw
unique_ip_unique_agent_output.md 
➜  Desktop ./log_file_analyzer.sh access.log.2017-01-01
Unique IP's
14.139.187.130
68.180.228.229
207.46.13.146
68.180.228.229
123.125.67.226
85.115.32.180
144.32.170.29
85.115.32.180
144.32.170.29
195.154.191.64
95.28.180.251
166.88.123.66
220.181.108.117
202.188.40.138
211.144.0.163
101.226.33.200
202.188.40.138
211.144.0.163
119.147.146.60
82.81.92.138
77.88.47.100
202.188.40.138
103.21.125.80
70.53.215.244
68.180.228.229
207.46.13.95
162.217.31.85
94.252.88.177
47.189.17.206
94.252.88.177
71.10.36.99
66.249.66.105
172.56.42.13
185.83.162.4
123.125.71.95
180.76.15.25
180.76.15.12
207.46.13.146
66.220.146.31
220.181.108.182
61.251.236.11
98.220.75.1
24.146.203.160
113.108.192.2
180.151.172.46
95.27.26.171
46.229.168.67
157.55.39.75
46.229.168.65
46.229.168.68
46.229.168.65
46.229.168.69
46.229.168.66
46.229.168.71
46.229.168.74
46.229.168.68
46.229.168.74
46.229.168.67
46.229.168.72
46.229.168.68
46.229.168.70
46.229.168.69
46.229.168.70
46.229.168.66
46.229.168.65
46.229.168.68
46.229.168.66
46.229.168.70
46.229.168.66
46.229.168.73
46.229.168.74
46.229.168.73
46.229.168.68
46.229.168.65
81.163.131.56
46.229.168.68
46.229.168.71
46.229.168.67
46.229.168.66
46.229.168.71
46.229.168.72
46.229.168.71
46.229.168.65
46.229.168.72
46.229.168.68
46.229.168.69
46.229.168.72
46.229.168.69
46.229.168.72
46.229.168.71
46.229.168.74
46.229.168.69
46.229.168.70
46.229.168.68
46.229.168.71
46.229.168.66
46.229.168.69
46.229.168.67
46.229.168.70
46.229.168.67
46.229.168.68
46.229.168.74
46.229.168.72
46.229.168.74
46.229.168.65
46.229.168.68
46.229.168.73
46.229.168.68
46.229.168.73
46.229.168.67
46.229.168.72
46.229.168.74
46.229.168.73
46.229.168.74
46.229.168.67
46.229.168.69
46.229.168.73
46.229.168.70
46.229.168.67
46.229.168.70
46.229.168.67
68.180.228.229
46.229.168.67
46.229.168.65
46.229.168.74
46.229.168.65
46.229.168.66
46.229.168.72
46.229.168.71
46.229.168.72
46.229.168.67
46.229.168.65
46.229.168.67
46.229.168.73
46.229.168.69
46.229.168.74
46.229.168.68
46.229.168.73
36.110.120.151
46.229.168.69
46.229.168.70
46.229.168.67
46.229.168.74
46.229.168.65
46.229.168.72
46.229.168.73
46.229.168.65
46.229.168.68
46.229.168.65
46.229.168.71
46.229.168.67
46.229.168.72
46.229.168.68
46.229.168.74
46.229.168.66
46.229.168.71
123.125.67.224
123.125.71.76
46.229.168.71
46.229.168.74
46.229.168.67
46.229.168.73
46.229.168.68
46.229.168.70
46.229.168.66
46.229.168.74
46.229.168.68
46.229.168.74
58.16.42.148
46.229.168.65
46.229.168.68
46.229.168.70
46.229.168.69
46.229.168.73
46.229.168.72
46.229.168.67
46.229.168.66
46.229.168.68
46.229.168.65
46.229.168.71
46.229.168.72
46.229.168.67
46.229.168.66
46.229.168.68
46.229.168.66
46.229.168.65
46.229.168.71
46.229.168.73
46.229.168.74
46.229.168.71
46.229.168.74
46.229.168.71
46.229.168.67
66.249.93.64
66.249.93.95
46.229.168.69
46.229.168.73
46.229.168.68
46.229.168.72
46.229.168.68
46.229.168.69
46.229.168.65
46.229.168.72
46.229.168.65
46.229.168.71
46.229.168.67
46.229.168.73
46.229.168.70
46.229.168.73
46.229.168.69
46.229.168.71
46.229.168.65
46.229.168.70
46.229.168.69
46.229.168.73
46.229.168.65
46.229.168.71
46.229.168.74
46.229.168.66
46.229.168.68
46.229.168.69
46.229.168.70
46.229.168.71
46.229.168.66
46.229.168.74
46.229.168.72
46.229.168.70
46.229.168.74
46.229.168.66
46.229.168.70
46.229.168.72
46.229.168.67
46.229.168.65
46.229.168.67
46.229.168.74
46.229.168.69
46.229.168.74
46.229.168.71
46.229.168.67
46.229.168.68
46.229.168.71
46.229.168.65
46.229.168.73
65.55.210.255
46.229.168.70
46.229.168.65
46.229.168.73
46.229.168.68
46.229.168.73
46.229.168.68
46.229.168.71
46.229.168.73
46.229.168.69
46.229.168.68
46.229.168.74
46.229.168.71
46.229.168.65
69.30.210.242
46.229.168.73
46.229.168.71
46.229.168.72
46.229.168.74
46.229.168.68
46.229.168.74
46.229.168.69
46.229.168.70
46.229.168.71
46.229.168.74
46.229.168.69
46.229.168.72
46.229.168.67
46.229.168.70
46.229.168.72
46.229.168.66
46.229.168.72
46.229.168.67
46.229.168.72
46.229.168.70
46.229.168.65
46.229.168.66
46.229.168.73
46.229.168.66
46.229.168.69
46.229.168.72
46.229.168.65
46.229.168.74
46.229.168.68
46.229.168.73
46.229.168.67
46.229.168.71
46.229.168.69
46.229.168.65
46.229.168.68
46.229.168.69
46.229.168.73
46.229.168.65
46.229.168.70
46.229.168.72
46.229.168.69
46.229.168.68
46.229.168.69
46.229.168.65
46.229.168.74
46.229.168.67
46.229.168.73
46.229.168.68
46.229.168.65
54.172.242.152
46.229.168.66
46.229.168.67
46.229.168.69
46.229.168.72
46.229.168.66
46.229.168.67
46.229.168.72
46.229.168.65
46.229.168.67
46.229.168.69
46.229.168.66
46.229.168.71
46.229.168.70
46.229.168.71
46.229.168.65
46.229.168.67
46.229.168.71
46.229.168.73
46.229.168.71
46.229.168.67
46.229.168.74
46.229.168.73
46.229.168.69
46.229.168.71
46.229.168.70
46.229.168.69
46.229.168.72
46.229.168.69
46.229.168.68
46.229.168.67
46.229.168.74
46.229.168.67
46.229.168.66
46.229.168.65
46.229.168.69
46.229.168.65
46.229.168.74
46.229.168.72
46.229.168.68
46.229.168.70
46.229.168.73
46.229.168.65
46.229.168.73
46.229.168.72
46.229.168.65
46.229.168.66
46.229.168.71
46.229.168.69
46.229.168.74
46.229.168.67
46.229.168.74
46.229.168.70
46.229.168.74
46.229.168.66
46.229.168.69
46.229.168.65
46.229.168.71
46.229.168.67
46.229.168.72
46.229.168.68
46.229.168.70
221.4.169.197
46.229.168.65
46.229.168.67
46.229.168.73
221.4.169.197
46.229.168.66
46.229.168.65
54.172.242.152
46.229.168.70
46.229.168.71
46.229.168.70
46.229.168.68
46.229.168.71
46.229.168.65
46.229.168.73
46.229.168.68
46.229.168.73
66.249.66.197
66.249.66.103
66.249.66.209
66.249.66.105
46.229.168.65
46.229.168.68
46.229.168.66
46.229.168.65
46.229.168.68
46.229.168.71
46.229.168.65
46.229.168.73
46.229.168.67
46.229.168.73
46.229.168.67
46.229.168.73
46.229.168.69
46.229.168.65
46.229.168.71
46.229.168.67
46.229.168.66
46.229.168.70
46.229.168.67
46.229.168.73
46.229.168.74
46.229.168.65
46.229.168.71
46.229.168.74
46.229.168.66
46.229.168.73
46.229.168.65
46.229.168.70
46.229.168.66
46.229.168.72
46.229.168.68
46.229.168.74
46.229.168.69
46.229.168.74
46.229.168.70
46.229.168.72
46.229.168.71
46.229.168.68
46.229.168.72
46.229.168.69
46.229.168.67
46.229.168.74
46.229.168.72
46.229.168.65
46.229.168.70
46.229.168.71
46.229.168.73
46.229.168.72
46.229.168.65
46.229.168.67
46.229.168.70
46.229.168.65
46.229.168.69
46.229.168.68
46.229.168.74
46.229.168.65
46.229.168.71
46.229.168.68
46.229.168.66
46.229.168.70
46.229.168.74
46.229.168.71
46.229.168.74
46.229.168.65
46.229.168.66
46.229.168.72
46.229.168.71
46.229.168.73
46.229.168.67
46.229.168.66
46.229.168.72
66.102.6.129
66.102.6.159
104.131.86.190
113.108.192.2
185.182.49.31
70.106.209.116
37.144.52.198
66.249.66.101
176.195.113.223
66.249.83.84
66.249.83.83
157.55.39.100
131.253.27.116
207.46.13.157
123.125.67.224
180.76.15.33
104.162.192.217
96.250.56.15
24.142.90.210
207.46.13.157
68.180.228.229
78.68.160.67
157.55.39.232
69.250.190.181
180.76.15.148
60.180.157.119
123.57.239.25
121.40.39.151
173.82.77.179
101.200.199.203
101.200.204.113
123.57.237.215
101.200.205.6
101.200.191.237
182.92.7.12
101.200.204.198
123.57.239.25
121.40.39.151
178.252.127.220
75.118.9.134
157.55.39.232
157.55.39.100
66.249.83.82
66.249.83.84
157.55.39.100
77.75.76.161
88.198.16.153
180.76.15.140
68.180.228.229
88.185.102.251
52.212.146.96
123.125.71.85
185.100.84.82
31.185.104.21
180.121.191.241
68.54.170.68
211.210.103.197
103.21.125.85
157.49.22.184
211.42.54.241
5.106.255.253
42.236.10.100
164.124.106.147
199.16.157.181
62.81.238.27
157.55.39.232
54.234.67.133
109.66.159.240
118.33.113.8
109.66.159.240
211.110.178.7
211.244.83.219
118.33.113.8
104.200.107.62
109.66.159.240
121.52.146.107
117.200.32.210
111.91.137.215
87.201.213.114
118.33.113.8
27.5.119.139
83.42.194.191
109.86.71.185
87.255.155.12
109.64.164.19
140.112.24.181
140.112.24.217
140.112.24.213
140.112.24.181
140.112.24.217
140.112.24.247
140.112.24.223
188.25.158.195
1.202.222.146
188.25.158.195
103.195.73.100
36.110.120.151
46.229.164.102
211.249.40.21
211.249.40.24
121.52.146.107
94.139.69.2
155.94.170.213
157.55.39.75
211.249.40.11
117.201.11.139
103.195.73.100
78.68.160.67
223.102.16.121
106.38.241.146
===========
Unique user agents
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36" 
"Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 
"Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)" 
"Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 
"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.93 Safari/537.36" 
"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36" 
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36" 
"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36" 
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36" 
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36" 
"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)" 
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36" 
"Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)" 
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36" 
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36" 
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36" 
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36" 
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36" 
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95  Safari/537.36" 
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36" 
"Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)" 
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36" 
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.75 Safari/537.36 OPR/42.0.2393.85" 
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1" 
"Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 
"Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)" 
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/601.7.7 (KHTML, like Gecko)" 
"Safari/11601.7.7 CFNetwork/760.6.3 Darwin/15.6.0 (x86_64)" 
"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36" 
"Mozilla/5.0 (iPhone; CPU iPhone OS 10_2 like Mac OS X) AppleWebKit/602.3.12 (KHTML, like Gecko) Mobile/14C92" 
"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36" 
"Mozilla/5.0 (Linux; Android 6.0.1; SM-G900V Build/MMB29M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.91 Mobile Safari/537.36" 
"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" 
"Mozilla/5.0 (iPhone; CPU iPhone OS 10_2 like Mac OS X) AppleWebKit/602.3.12 (KHTML, like Gecko) Version/10.0 Mobile/14C92 Safari/602.1" 
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1" 
"Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)" 
"Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)" 
"facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)" 
"Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)" 
"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36" 
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36" 
"Mozilla/5.0 (Windows NT 6.3; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0" 
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)" 
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0" 
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36" 
"Mozilla/5.0 (compatible; SemrushBot/1.1~bl; +http://www.semrush.com/bot.html)" 
"Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)" 
"Mozilla/5.0 (compatible; SemrushBot/1.1~bl; +http://www.semrush.com/bot.html)" 
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.0.3705; .NET CLR 1.1.4322; Media Center PC 4.0)" 
"Mozilla/5.0 (compatible; SemrushBot/1.1~bl; +http://www.semrush.com/bot.html)" 
"Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 
"Mozilla/5.0 (compatible; SemrushBot/1.1~bl; +http://www.semrush.com/bot.html)" 
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)" 
"Mozilla/5.0 (compatible; SemrushBot/1.1~bl; +http://www.semrush.com/bot.html)" 
"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.93 Safari/537.36" 
"Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)" 
"Mozilla/5.0 (compatible; SemrushBot/1.1~bl; +http://www.semrush.com/bot.html)" 
"curl/7.19.7 (x86_64-redhat-linux-gnu) libcurl/7.19.7 NSS/3.19.1 Basic ECC zlib/1.2.3 libidn/1.18 libssh2/1.4.2" 
"Mozilla/5.0 (compatible; SemrushBot/1.1~bl; +http://www.semrush.com/bot.html)" 
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36 Google Favicon" 
"Mozilla/5.0 (compatible; SemrushBot/1.1~bl; +http://www.semrush.com/bot.html)" 
"Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 BingPreview/1.0b" 
"Mozilla/5.0 (compatible; SemrushBot/1.1~bl; +http://www.semrush.com/bot.html)" 
"Mozilla/5.0 (compatible; MJ12bot/v1.4.7; http://mj12bot.com/)" 
"Mozilla/5.0 (compatible; SemrushBot/1.1~bl; +http://www.semrush.com/bot.html)" 
"Python-urllib/2.7" 
"Mozilla/5.0 (compatible; SemrushBot/1.1~bl; +http://www.semrush.com/bot.html)" 
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)" 
"Mozilla/5.0 (compatible; SemrushBot/1.1~bl; +http://www.semrush.com/bot.html)" 
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)" 
"Mozilla/5.0 (compatible; SemrushBot/1.1~bl; +http://www.semrush.com/bot.html)" 
"Python-urllib/2.7" 
"Mozilla/5.0 (compatible; SemrushBot/1.1~bl; +http://www.semrush.com/bot.html)" 
"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" 
"Mozilla/5.0 (compatible; SemrushBot/1.1~bl; +http://www.semrush.com/bot.html)" 
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36 Google Favicon" 
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.99 Safari/537.36" 
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)" 
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5" 
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.95 Safari/537.36" 
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)" 
"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" 
"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)" 
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36 Google Favicon" 
"Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)" 
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534+ (KHTML, like Gecko) BingPreview/1.0b" 
"Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)" 
"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.93 Safari/537.36" 
"Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)" 
"Mozilla/5.0 (Windows NT 10.0; WOW64; rv:48.0) Gecko/20100101 Firefox/48.0" 
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0" 
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1" 
"Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)" 
"Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 
"Mozilla/5.0 (iPad; CPU OS 10_0_2 like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Mobile/14A456 Safari/602.1" 
"MobileSafari/602.1 CFNetwork/808.0.2 Darwin/16.0.0" 
"Mozilla/5.0 (iPad; CPU OS 10_0_2 like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Mobile/14A456 Safari/602.1" 
"MobileSafari/602.1 CFNetwork/808.0.2 Darwin/16.0.0" 
"Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)" 
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/600.8.9 (KHTML, like Gecko) Version/8.0.8 Safari/600.8.9" 
"Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B410 Safari/600.1.4" 
"com.apple.WebKit.WebContent/10600.8.9 CFNetwork/720.5.7 Darwin/14.5.0 (x86_64)" 
"Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)" 
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)" 
"Mozilla/5.0 (Ubuntu; X11; Linux x86_64; rv:8.0) Gecko/20100101 Firefox/8.0" 
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:45.9) Gecko/20100101 Goanna/3.0 Firefox/45.9 PaleMoon/27.0.3" 
"Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0" 
"Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)" 
"Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)" 
"Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)" 
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36 Google Favicon" 
"Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)" 
"Mozilla/5.0 (compatible; SeznamBot/3.2; +http://napoveda.seznam.cz/en/seznambot-intro/)" 
"Mozilla/5.0 (compatible; MJ12bot/v1.4.5; http://www.majestic12.co.uk/bot.php?+)" 
"Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)" 
"Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)" 
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1" 
"python-requests/2.12.1" 
"Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)" 
"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36" 
"Mozilla/5.0 (Windows NT 6.1; Trident/7.0; rv:11.0) like Gecko" 
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1" 
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36" 
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.75 Safari/537.36 OPR/42.0.2393.85" 
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0" 
"Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36" 
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36" 
"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36" 
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36 QIHU 360SE; 360Spider" 
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider" 
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36" 
"Twitterbot/1.0" 
"Mozilla/5.0 (Windows NT 10.0; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0" 
"Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)" 
"Go-http-client/1.1" 
"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:50.0) Gecko/20100101 Firefox/50.0" 
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.95 Safari/537.36" 
"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:50.0) Gecko/20100101 Firefox/50.0" 
"facebookexternalhit/1.1;kakaotalk-scrap/1.0;" 
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.95 Safari/537.36" 
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.95 Safari/537.36" 
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1" 
"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:50.0) Gecko/20100101 Firefox/50.0" 
"Mozilla/5.0 (Windows NT 10.0; rv:50.0) Gecko/20100101 Firefox/50.0" 
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1" 
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36" 
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0" 
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.95 Safari/537.36" 
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36" 
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36" 
"Mozilla/5.0 (Windows NT 5.1; rv:44.0) Gecko/20100101 Firefox/44.0" 
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36" 
"Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko" 
"Mozilla/5.0 (Windows NT 6.1; Win64; x64; Trident/7.0; rv:11.0) like Gecko" 
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.95 Safari/537.36" 
"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:44.0) Gecko/20100101 Firefox/44.0" 
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.100 Safari/537.36" 
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; 360space)" 
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.100 Safari/537.36" 
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1" 
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)" 
"Mozilla/5.0 (compatible; SemrushBot/1.1~bl; +http://www.semrush.com/bot.html)" 
"Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.1; Win64; x64; Windows NT 6.1; Trident/5.0)" 
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; .NET4.0C; .NET4.0E)" 
"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET4.0C; .NET4.0E)" 
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/601.7.7 (KHTML, like Gecko) Version/9.1.2 Safari/601.7.7" 
"Mozilla/5.0 (Windows NT 10.0; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0" 
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5" 
"Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)" 
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)" 
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1" 
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit/602.3.12 (KHTML, like Gecko)" 
"Safari/12602.3.12.0.1 CFNetwork/807.2.14 Darwin/16.3.0 (x86_64)" 
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)" 
"Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)"
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment