# 公司网站反映很慢，可能是一些页面的访问方法或者页面引起，通过程序统计nginx访问日志的页面和具体的action方法访问次数以及平均响应时间可以为程序开发的同事提供参考定位具体的代码
# 默认的nginx日志
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
现在加上响应时间，方便统计
log_format access '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for" "$http_host" "$upstream_response_time" "$request_time"' ;
发现统计出来的响应时间$upstream_response_time无法参与计算，于是去掉引号，变为如下，方便计算：
log_format main '$remote_addr - $remote_user [$time_local] "$request" ''$status $body_bytes_sent "$http_referer" ''"$http_user_agent" "$http_x_forwarded_for" "$http_host" $upstream_response_time $request_time';
不需要去掉双引号也可以，用python处理一下去掉两边的双引号即可，如下：
_time = _time.lstrip('"').rstrip('"')
具体的python代码
#encoding=utf-8from decimal import Decimal # 找到日志中的top n，日志格式如下 # 192.168.5.46 - - [01/Dec/2017:09:47:21 +0800] "GET /api_moblie_idol.php?action=get_user_guangying_comment&app_platform=android&channelId=S007&starid=6730&version=164&unique_id=A73308A15313C8E34B518CA515288A13&messageid=5a20ae347a11737a288b4738>ui_cid=03908d9fa39c8de124aa01688109b26c&page=7&type=latest HTTP/1.0" 200 70 "-" "-" "223.89.74.82" "data.android.idol001.com" 0.050 0.052 # 统计访问网站php页面和方法的次数，并且生成响应时间 def log_analysis(log_file, dpath, topn = 10):path=log_fileshandle = open(path, 'r')count = 1log_dict = {}log_dict2 = {}total_time = 0.0while True:
# 最好加上strip()碰到换行就无法识别了line = shandle.readline().strip()# 如果到了文件末尾就终止if line == '':breaknodes = line.split()# 192.168.9.187 - - [30/Nov/2017:18:28:35 +0800] "GET /api_moblie_idol.phpnique_id=5ACE6943 HTTP/1.0" 200 13950 "-" "idol/5900 (iPhone; iOS 10.3.2; Scale/3.00)" "171.221.169.54" "data.idol001.com" "0.144" "0.144" _url, _time = nodes[6], nodes[-2]# 使用?分割页面，获取url，通过action和&分割出action动作_tmp = _url.split('action')if len(_tmp) != 2:continue_url = _url.split('?')[0]_method = _tmp[1].split('&')[0]# print 'url:%s, method:%s, time:%s' % (_url,_method,_time)# 如果不是数字就跳出本次循环if _time == '-':continuetry:# {(url,method):count}当做字典的key# 统计url,method的次数if (_url, _method) not in log_dict:log_dict[(_url, _method)] = 1else:log_dict[(_url, _method)] = log_dict[(_url, _method)] + 1# 统计url,method的累计时间if (_url, _method) not in log_dict2:log_dict2[(_url, _method)] = Decimal(_time)else:log_dict2[(_url, _method)] = Decimal(_time) + Decimal(log_dict2[(_url, _method)])except Exception,e:continue# print log_dict# print log_dict2# 关闭文件句柄 shandle.close()# 对字典进行排序# ('/index', 'post'): 2rst_list = log_dict.items()# print rst_listfor j in range(topn):# 冒泡法根据rst_list中的count排序，找出访问量最大的10个IPfor i in range(0,len(rst_list) - 1):if rst_list[i][1] > rst_list[i+1][1]:temp = rst_list[i]rst_list[i] = rst_list[i+1]rst_list[i+1] = temp# 获取 topn 个数need_list = rst_list[-1:-topn - 1:-1]rt_list = []for _line, _num in need_list:_tmp_dict = {}_avg_time = Decimal(log_dict2[_line])/Decimal(_num)# print '_avg= %s' % _avg_time_tmp_dict['_num'] = _num_tmp_dict['_avg'] = _avg_timert_list.append((_line,_tmp_dict))# print rt_list# 打印出top 10访问日志，并写入网页中title = 'nginx访问日志'tbody = ''for k,v in rt_list:print v['_num'],k[1],k[0],v['_avg']tbody += '<tr>\n<td>%s</td><td>%s</td><td>%s</td><td>%s</td>\n<tr>\n' % (v['_num'],k[1],k[0],v['_avg'])html_tpl = '''<!DOCTYPE html><html><head><meta charset="utf-8"><title>{title}</title></head><body><table border="1" cellspacing="0" cellpadding="0" color='pink'><thead><tr cellspacing="0" cellpadding="0"><th>访问次数</th><th>method</th><th>url</th><th>平均响应时间</th></tr></thead>{tbody}</table></body></html>'''html_handle = open(dpath,'w')html_handle.write(html_tpl.format(title = title, tbody = tbody))html_handle.close()# 函数入口 if __name__ == '__main__':# nginx日志文件log_file = 'access_android_idol001_com.log'dpath = 'top1000.html'# topn 表示去top多少个# 不传，默认10个topn = 1000log_analysis(log_file,dpath,topn)
最终的效果：
