From 53122ec215760149a1421b926f03d3b464d13957 Mon Sep 17 00:00:00 2001 From: midoks Date: Thu, 21 Jul 2022 18:13:32 +0800 Subject: [PATCH] =?UTF-8?q?=E7=BD=91=E7=AB=99=E7=BB=9F=E8=AE=A1=20xx4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- plugins/mysql/init.d/mysql.tpl | 2 +- plugins/mysql/init.d/mysql8.0.tpl | 2 +- plugins/webstats/conf/config.json | 7 +- plugins/webstats/index.py | 16 +- plugins/webstats/js/setting.js | 122 +++++++++++--- plugins/webstats/lua/webstats_log.lua | 221 +++++++++++++++++++++----- 6 files changed, 298 insertions(+), 72 deletions(-) diff --git a/plugins/mysql/init.d/mysql.tpl b/plugins/mysql/init.d/mysql.tpl index ef861da5a..e4ee11b24 100644 --- a/plugins/mysql/init.d/mysql.tpl +++ b/plugins/mysql/init.d/mysql.tpl @@ -273,7 +273,7 @@ case "$mode" in then # Give extra arguments to mysqld with the my.cnf file. This script # may be overwritten at next upgrade. - $bindir/mysqld_safe --datadir="$datadir" --pid-file="$mysqld_pid_file_path" $other_args >/dev/null & + $bindir/mysqld_safe --defaults-file=$basedir/etc/my.cnf --datadir="$datadir" --pid-file="$mysqld_pid_file_path" $other_args >/dev/null & wait_for_pid created "$!" "$mysqld_pid_file_path"; return_value=$? # Make lock for RedHat / SuSE diff --git a/plugins/mysql/init.d/mysql8.0.tpl b/plugins/mysql/init.d/mysql8.0.tpl index c14f5ad3a..936ca0d7d 100755 --- a/plugins/mysql/init.d/mysql8.0.tpl +++ b/plugins/mysql/init.d/mysql8.0.tpl @@ -263,7 +263,7 @@ case "$mode" in then # Give extra arguments to mysqld with the my.cnf file. This script # may be overwritten at next upgrade. - $bindir/mysqld_safe --datadir="$datadir" --pid-file="$mysqld_pid_file_path" $other_args >/dev/null & + $bindir/mysqld_safe --defaults-file=$basedir/etc/my.cnf --datadir="$datadir" --pid-file="$mysqld_pid_file_path" $other_args >/dev/null & wait_for_pid created "$!" "$mysqld_pid_file_path"; return_value=$? # Make lock for RedHat / SuSE diff --git a/plugins/webstats/conf/config.json b/plugins/webstats/conf/config.json index 594e47f88..49b663802 100644 --- a/plugins/webstats/conf/config.json +++ b/plugins/webstats/conf/config.json @@ -34,8 +34,13 @@ 302, 303 ], + "exclude_ip":[ + "192.168.1.1" + ], + "exclude_url":[], + "ip_top_num": 50, + "uri_top_num": 50, "statistics_machine_access": true, - "exclude_url": [], "statistics_ip": true, "statistics_uri": true, "record_post_args": false, diff --git a/plugins/webstats/index.py b/plugins/webstats/index.py index 172770897..80d1f3db9 100755 --- a/plugins/webstats/index.py +++ b/plugins/webstats/index.py @@ -31,6 +31,11 @@ def getServerDir(): return mw.getServerDir() + '/' + getPluginName() +def getConf(): + conf = getServerDir() + "/lua/config.json" + return conf + + def getArgs(): args = sys.argv[2:] tmp = {} @@ -217,6 +222,13 @@ def reload(): return 'ok' +def getGloalConf(): + conf = getConf() + content = mw.readFile(conf) + content = json.loads(content) + return mw.returnJson(True, 'ok', content) + + def getLogsList(): args = getArgs() check = checkArgs(args, ['page', 'page_size', 'site']) @@ -263,8 +275,8 @@ if __name__ == "__main__": print(reload()) elif func == 'run_info': print(runInfo()) - elif func == 'conf': - print(getConf()) + elif func == 'get_global_conf': + print(getGloalConf()) elif func == 'get_logs_list': print(getLogsList()) else: diff --git a/plugins/webstats/js/setting.js b/plugins/webstats/js/setting.js index d86b167f4..b61accacb 100644 --- a/plugins/webstats/js/setting.js +++ b/plugins/webstats/js/setting.js @@ -1,36 +1,22 @@ function wsGlobalSetting(){ - var html = '
\ -
\ -
监控统计:
\ -
\ -
\ -
监控开关
\ - \ - \ - ?\ -
\ -
\ -
URI统计
\ - \ - \ - ?\ -
\ -
\ -
\ -
\ +//////////////////////////////////////////////// +wsPost('get_global_conf', '' ,{}, function(rdata){ + var rdata = $.parseJSON(rdata.data); + var rdata = rdata.data; + var html = '
\
\
统计:
\
\
\
IP统计
\ - \ + \ \ ?\
\
\
URI统计
\ - \ + \ \ ?\
\ @@ -42,7 +28,7 @@ function wsGlobalSetting(){
\
\
日志保存天数
\ - /天 \ + /天 \ \ ?\
\ @@ -61,7 +47,7 @@ function wsGlobalSetting(){ 记录请求原文\
\
\ - * 排除的请求不写入网站日志，不统计PV、UV、IP，只累计总请求、总流量数，如需多个请换行填写\ + * 准确识别CDN网络IP地址，请注意大小写，如需多个请换行填写\ \
\

\ @@ -71,8 +57,96 @@ function wsGlobalSetting(){

\
\
\ -
\
'; $(".soft-man-con").html(html); $('[data-toggle="tooltip"]').tooltip(); + + var common_tpl_tips = '* 准确识别CDN网络IP地址，请注意大小写，如需多个请换行填写'; + var common_tpl_area = ''; + + $('#webstats .tab-con textarea').text(rdata['global']['cdn_headers'].join('\n')); + $('#webstats .tab-nav span').click(function(e){ + $('#webstats .tab-nav span').removeClass('on'); + $(this).addClass('on'); + $('#webstats .tab-con').html(''); + + var typename = $(this).attr('data-type'); + if (typename == 'cdn_headers'){ + var content = $(common_tpl_tips).html('* 准确识别CDN网络IP地址，请注意大小写，如需多个请换行填写').prop('outerHTML'); + var area = $(common_tpl_area).html(rdata['global']['cdn_headers'].join('\n')).prop('outerHTML'); + + content += area; + $('#webstats .tab-con').html(content); + } else if (typename == 'exclude_extension'){ + + var content = $(common_tpl_tips).html('* 排除的请求不写入网站日志，不统计PV、UV、IP，只累计总请求、总流量数，如需多个请换行填写').prop('outerHTML'); + var area = $(common_tpl_area).html(rdata['global']['exclude_extension'].join('\n')).prop('outerHTML'); + content += area; + $('#webstats .tab-con').html(content); + } else if (typename == 'exclude_status'){ + var content = $(common_tpl_tips).html('* 排除的请求不写入网站日志，不统计PV、UV、IP，只累计总请求、总流量数，如需多个请换行填写').prop('outerHTML'); + var area = $(common_tpl_area).html(rdata['global']['exclude_status'].join('\n')).prop('outerHTML'); + content += area; + $('#webstats .tab-con').html(content); + } else if (typename == 'exclude_ip'){ + var txt = '
* 排除的IP不写入网站日志，不统计PV、UV、IP，只累计总请求、总流量数，如需多个请换行填写
\ +
* 支持 192.168.1.1-192.168.1.10格式排除区间IP
' + var content = $(common_tpl_tips).html(txt).prop('outerHTML'); + var area = $(common_tpl_area).html(rdata['global']['exclude_ip'].join('\n')).prop('outerHTML'); + content += area; + $('#webstats .tab-con').html(content); + } else if (typename == 'record_post_args'){ + var txt = '
记录请求原文说明：HTTP请求原文包括客户端请求详细参数，有助于分析或排查异常请求；
\ +
考虑到HTTP请求原文会占用额外存储空间，默认仅记录500错误请求原文。
' + var content = $(common_tpl_tips).html(txt).prop('outerHTML'); + + var check = '
\ + \ + \ +
'; + content+=check; + + $('#webstats .tab-con').html(content); + } else if ( typename == 'exclude_url'){ + var txt = '* 排除的请求不写入网站日志，不统计PV、UV、IP，只累计总请求、总流量数' + var content = $(common_tpl_tips).html(txt).prop('outerHTML'); + + var _text = ''; + var _tmp = rdata['global']['exclude_url']; + for(var i = 0; i<10; i++){ + if(typeof _tmp[i] == 'undefined'){ + _tmp[i] = {mode:'regular',url:''} + } + + _text += '\ + \ + \ + \ + \ + '; + } + + var list = '
\ + \ + \ + \ + \ + '+_text+'\ +
排除方式排除路径
\ +
'; + + content += list; + $('#webstats .tab-con').html(content); + } + + }); +}); +/////////////////////////////////////////////// } \ No newline at end of file diff --git a/plugins/webstats/lua/webstats_log.lua b/plugins/webstats/lua/webstats_log.lua index 7a3bece7f..ec790cd5e 100644 --- a/plugins/webstats/lua/webstats_log.lua +++ b/plugins/webstats/lua/webstats_log.lua @@ -69,6 +69,7 @@ log_by_lua_block { local config local today local day + local excluded --- default common var end --- local function init_var() @@ -184,7 +185,7 @@ log_by_lua_block { end if "table" == type(data) then - D("get_http_original:"..headers..data) + -- D("get_http_original:"..data) headers = table.concat(headers, data) end end @@ -255,8 +256,50 @@ log_by_lua_block { return unset_server_name end - -- exclude_func start - local function execute_status() + --------------------- exclude_func start -------------------------- + local function load_global_exclude_ip() + local load_key = "global_exclude_ip_load" + -- update global exclude ip + local global_exclude_ip = config["global"]["exclude_ip"] + if global_exclude_ip then + for i, _ip in pairs(global_exclude_ip) + do + -- global + if not cache:get("global_exclude_ip_".._ip) then + D("set global exclude ip: ".._ip) + cache:set("global_exclude_ip_".._ip, true) + end + end + end + -- set tag + cache:set(load_key, true) + end + + local function load_exclude_ip(input_server_name) + + local load_key = input_server_name .. "_exclude_ip_load" + local site_config = config[input_server_name] + + local site_exclude_ip = nil + if site_config then + site_exclude_ip = site_config["exclude_ip"] + end + + -- update server_name exclude ip + if site_exclude_ip then + for i, _ip in pairs(site_exclude_ip) + do + D("set exclude ip: ".._ip) + cache:set(input_server_name .. "_exclude_ip_".._ip, true) + end + end + + -- set tag + cache:set(load_key, true) + return true + end + + local function filter_status() if not config['exclude_status'] then return false end local the_status = tostring(ngx.status) for _,v in ipairs(config['exclude_status']) @@ -301,25 +344,71 @@ log_by_lua_block { end return false end - -- exclude_func end + + local function exclude_ip(input_server_name, ip) + -- 排除IP匹配，分网站单独的配置和全局配置两种方式 + local site_config = config[input_server_name] + local server_exclude_ips = nil + local check_server_exclude_ip = false + if site_config then + server_exclude_ips = site_config["exclude_ip"] + if not server_exclude_ips then + return false + end + for k, _ip in pairs(server_exclude_ips) + do + check_server_exclude_ip = true + break + end + end + D("server[" ..input_server_name.."]check exclude ip : "..tostring(check_server_exclude_ip)) + if check_server_exclude_ip then + if cache:get(input_server_name .. "_exclude_ip_"..ip) then + D("-Exclude server ip:"..ip) + return true + end + else + if cache:get("global_exclude_ip_"..ip) then + D("*Excluded global ip:"..ip) + return true + end + end + return false + end + --------------------- exclude_func end --------------------------- + --------------------- db start --------------------------- + local function update_stat(update_server_name, db, stat_table, key, columns) + -- 根据指定表名，更新统计数据 + if not columns then return end + local stmt = db:prepare(string.format("INSERT INTO %s(time) SELECT :time WHERE NOT EXISTS(SELECT time FROM %s WHERE time=:time);", stat_table, stat_table)) + stmt:bind_names{time=key} + local res, err = stmt:step() + stmt:finalize() + local update_sql = "UPDATE ".. stat_table .. " SET " .. columns + update_sql = update_sql .. " WHERE time=" .. key + status, errorString = db:exec(update_sql) + end + --------------------- db end --------------------------- + local function cache_logs() -- make new id local new_id = get_last_id(server_name) + + local excluded = false + local ip = get_client_ip_bylog() + excluded = filter_status() or exclude_extension() or exclude_url() or exclude_ip(server_name, ip) + D("new_id:"..new_id) - local err = nil - local ip_list = request_header["x-forwarded-for"] - local ip = get_client_ip_bylog() if ip and not ip_list then ip_list = ip end -- local request_time = ngx.var.request_time local request_time = get_request_time() - local client_port = ngx.var.remote_port local real_server_name = server_name local uri = ngx.var.uri @@ -351,7 +440,7 @@ log_by_lua_block { protocol=protocol, is_spider=0, request_time=request_time, - -- excluded=excluded, + excluded=excluded, request_headers=request_headers, ip_list=ip_list, client_port=client_port @@ -361,15 +450,30 @@ log_by_lua_block { local spider_stat_fields = "x" local client_stat_fields = "x" + if not excluded then + + if ngx.re.find("500,501,502,503,504,505,506,507,509,510,400,401,402,403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,421,422,423,424,425,426,449,451", tostring(status_code), "jo") then + local field = "status_"..status_code + request_stat_fields = request_stat_fields .. ","..field.."="..field.."+1" + end + -- debug("method:"..method) + local lower_method = string.lower(method) + if ngx.re.find("get,post,put,patch,delete", lower_method, "ijo") then + local field = "http_"..lower_method + request_stat_fields = request_stat_fields .. ","..field.."="..field.."+1" + end + end + local stat_fields = request_stat_fields..";"..client_stat_fields..";"..spider_stat_fields D("stat_fields:"..stat_fields) + cache_set(server_name, new_id, "stat_fields", stat_fields) cache_set(server_name, new_id, "log_kv", json.encode(kv)) end local function store_logs_line(db, stmt, input_server_name, lineno) local logvalue = cache_get(input_server_name, lineno, "log_kv") - -- D("store_logs_line:"..logvalue) if not logvalue then return false end + D("store_logs_line:"..logvalue) local logline = json.decode(logvalue) local time = logline["time"] @@ -389,36 +493,68 @@ log_by_lua_block { local server_name = logline["server_name"] local user_agent = logline["user_agent"] local request_headers = logline["request_headers"] + local excluded = logline["excluded"] - stmt:bind_names{ - time=time, - ip=ip, - domain=domain, - server_name=server_name, - method=method, - status_code=status_code, - uri=uri, - body_length=body_length, - referer=referer, - user_agent=user_agent, - protocol=protocol, - request_time=request_time, - is_spider=is_spider, - request_headers=request_headers, - ip_list=ip_list, - client_port=client_port, - } + local request_stat_fields = nil + local client_stat_fields = nil + local spider_stat_fields = nil + local stat_fields = cache_get(input_server_name, id, "stat_fields") + if stat_fields == nil then + D("Log stat fields is nil.") + D("Logdata:"..logvalue) + else + stat_fields = split_bylog(stat_fields, ";") + request_stat_fields = stat_fields[1] + client_stat_fields = stat_fields[2] + spider_stat_fields = stat_fields[3] - local res, err = stmt:step() - if tostring(res) == "5" then - D("Step res:"..tostring(res)) - D("Step err:"..tostring(err)) - D("Step数据库连接繁忙，稍候存储。") - return false + if "x" == client_stat_fields then + client_stat_fields = nil + end + + if "x" == spider_stat_fields then + spider_stat_fields = nil + end end - stmt:reset() - D("store_logs_line ok") + + if not excluded then + stmt:bind_names{ + time=time, + ip=ip, + domain=domain, + server_name=server_name, + method=method, + status_code=status_code, + uri=uri, + body_length=body_length, + referer=referer, + user_agent=user_agent, + protocol=protocol, + request_time=request_time, + is_spider=is_spider, + request_headers=request_headers, + ip_list=ip_list, + client_port=client_port, + } + + local res, err = stmt:step() + if tostring(res) == "5" then + D("Step res:"..tostring(res)) + D("Step err:"..tostring(err)) + D("Step数据库连接繁忙，稍候存储。") + return false + end + stmt:reset() + D("store_logs_line ok") + + update_stat(store_server_name, db, "client_stat", time_key, client_stat_fields) + update_stat(store_server_name, db, "spider_stat", time_key, spider_stat_fields) + D("stat ok") + end + + local time_key = logline["time_key"] + update_stat(input_server_name, db, "request_stat", time_key, request_stat_fields) return true end @@ -456,8 +592,6 @@ log_by_lua_block { -- end local stmt2 = nil - - if db ~= nil then stmt2 = db:prepare[[INSERT INTO web_logs( time, ip, domain, server_name, method, status_code, uri, body_length, @@ -468,7 +602,7 @@ log_by_lua_block { end if db == nil or stmt2 == nil then - D("网站监控报表数据库连接异常。") + D("web data db error") -- cache:set(storing_key, false) if db and db:isopen() then db:close() @@ -481,9 +615,8 @@ log_by_lua_block { for i=store_start, store_end, 1 do D("store_start:"..store_start..":store_end:".. store_end) if store_logs_line(db, stmt2, input_server_name, i) then - -- store_count = store_count + 1 cache_clear(input_server_name, i, "log_kv") - -- cache_clear(input_server_name, i, "STAT_FIELDS") + cache_clear(input_server_name, i, "stat_fields") end end end @@ -493,12 +626,11 @@ log_by_lua_block { if tostring(res) == "5" then D("Finalize res:"..tostring(res)) D("Finalize err:"..tostring(err)) - D("数据库连接繁忙，稍候存储.") + D("Finalize busy.") return true end local res, err = db:execute([[COMMIT]]) - if db and db:isopen() then db:close() end @@ -517,6 +649,9 @@ log_by_lua_block { D("c_name:"..c_name) D("server_name:"..server_name) + load_global_exclude_ip() + load_exclude_ip(server_name) + cache_logs() store_logs(server_name)