yan
/
mdserver-web
mirror of https://github.com/midoks/mdserver-web.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

up

This commit is contained in:
midoks 2022-10-17 23:29:57 +08:00
parent c501efdfac
commit c9a527e487
4 changed files with 71 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
plugins/webstats/index.py
View File

@ -1051,6 +1051,8 @@ def getUriStatList():
ftime = time.localtime(time.time())
day = ftime.tm_mday
# print(day)
field_day = "day" + str(day)
field_flow = "flow" + str(day)
# print(field_day, field_flow)

24
plugins/webstats/lua/webstats_common.lua
View File

@ -412,9 +412,11 @@ function _M.store_logs_line(self, db, stmt, input_sn, info)
end
stmt:reset()
self:update_stat( db, "client_stat", time_key, client_stat_fields)
self:update_stat( db, "spider_stat", time_key, spider_stat_fields)
-- self:D("stat ok")
local res ,err = self:update_stat( db, "client_stat", time_key, client_stat_fields)
-- self:D("step res:"..tostring(res) ..",step err:"..tostring(err))
local res ,err = self:update_stat( db, "spider_stat", time_key, spider_stat_fields)
-- self:D("step res:"..tostring(res) ..",step err:"..tostring(err))
-- self:D("stat ok"..)
-- only count non spider requests
local ok, err = self:statistics_uri(db, request_uri, ngx.md5(request_uri), body_length)
@ -478,7 +480,7 @@ function _M.statistics_uri(self, db, uri, uri_md5, body_length)
stat_sql = "INSERT INTO uri_stat(uri_md5,uri) SELECT \""..uri_md5.."\",\""..uri.."\" WHERE NOT EXISTS (SELECT uri_md5 FROM uri_stat WHERE uri_md5=\""..uri_md5.."\");"
local res, err = db:exec(stat_sql)
stat_sql = "UPDATE uri_stat SET "..day_column.."="..day_column.."+1,"..flow_column.."="..flow_column.."+"..body_length.." WHERE uri_md5=\""..uri_md5.."\""
stat_sql = "UPDATE uri_stat SET "..day_column.."="..day_column.."+1,"..flow_column.."="..flow_column.."+"..body_length.." WHERE uri_md5=\""..uri_md5.."\";"
local res, err = db:exec(stat_sql)
return true
end
@ -590,6 +592,20 @@ function _M.write_update_day(self, input_sn)
return self:write_file(_file, today, "w")
end
function _M.clean_stats(self, db, input_sn)
-- 清空 uri,ip 汇总的信息[昨日]
local update_day = self:load_update_day(input_sn)
if not update_day or update_day ~= today then
local update_sql = "UPDATE uri_stat SET "..day_column.."=0,"..flow_column.."=0"
db:exec(update_sql)
update_sql = "UPDATE ip_stat SET "..day_column.."=0,"..flow_column.."=0"
db:exec(update_sql)
self:write_update_day(input_sn)
end
end
function _M.lpop(self)
local cache = ngx.shared.mw_total
return cache:lpop(total_key)

55
plugins/webstats/lua/webstats_log.lua
View File

@ -14,11 +14,24 @@ log_by_lua_block {
local __C = require "webstats_common"
local C = __C:getInstance()
-- local redis = require "resty.redis"
-- local red = redis:new()
-- local ok, err = red:connect("127.0.0.1", 6379)
-- if not ok then
-- ngx.say("failed to connect: ", err)
-- return
-- end
-- red:auth("admin")
-- cache start ---
local cache = ngx.shared.mw_total
local function cache_set(server_name, id ,key, val)
local line_kv = "log_kv_"..server_name..'_'..id.."_"..key
-- cache:set(line_kv, val)
cache:set(line_kv, val)
end
@ -49,10 +62,12 @@ log_by_lua_block {
local auto_config = C:setInputSn(server_name)
local request_header = ngx.req.get_method()
local method = ngx.req.get_headers()
local excluded
local request_header = ngx.req.get_headers()
local method = ngx.req.get_method()
local excluded = false
local today = ngx.re.gsub(ngx.today(),'-','')
local day = os.date("%d")
local number_day = tonumber(day)
local day_column = "day"..number_day
@ -223,7 +238,7 @@ log_by_lua_block {
local kv = {
id=new_id,
time_key=time_key,
time=os.time(),
time=ngx.time(),
ip=ip,
domain=domain,
server_name=server_name,
@ -317,7 +332,8 @@ log_by_lua_block {
-- local key = C:getTotalKey()
-- ngx.shared.mw_total:rpush(key, push_data)
-- C:D("ddd")
-- C:D("stat_fields:"..stat_fields)
-- C:D("log_kv:"..json.encode(kv))
cache_set(server_name, new_id, "stat_fields", stat_fields)
cache_set(server_name, new_id, "log_kv", json.encode(kv))
@ -406,13 +422,14 @@ log_by_lua_block {
end
stmt:reset()
-- D("store_logs_line ok")
C:update_stat( db, "client_stat", time_key, client_stat_fields)
C:update_stat( db, "spider_stat", time_key, spider_stat_fields)
-- D("stat ok")
-- C:D("stat ok")
-- only count non spider requests
local ok, err = pcall(function() C:statistics_uri(db, request_uri, ngx.md5(request_uri), body_length) end)
local ok, err = pcall(function() C:statistics_ip(db, ip, body_length) end)
local ok, err = C:statistics_uri(db, request_uri, ngx.md5(request_uri), body_length)
local ok, err = C:statistics_ip(db, ip, body_length)
end
C:update_stat( db, "request_stat", time_key, request_stat_fields)
@ -454,6 +471,11 @@ log_by_lua_block {
return true
end
db:exec([[PRAGMA synchronous = 0]])
db:exec([[PRAGMA page_size = 4096]])
db:exec([[PRAGMA journal_mode = wal]])
db:exec([[PRAGMA journal_size_limit = 1073741824]])
local stmt2 = nil
if db ~= nil then
stmt2 = db:prepare[[INSERT INTO web_logs(
@ -472,23 +494,10 @@ log_by_lua_block {
end
return true
end
db:exec([[PRAGMA synchronous = 0]])
db:exec([[PRAGMA page_size = 4096]])
db:exec([[PRAGMA journal_mode = wal]])
db:exec([[PRAGMA journal_size_limit = 1073741824]])
status, errorString = db:exec([[BEGIN TRANSACTION]])
update_day = C:load_update_day(input_sn)
if not update_day or update_day ~= today then
local update_sql = "UPDATE uri_stat SET "..day_column.."=0,"..flow_column.."=0"
status, errorString = db:exec(update_sql)
update_sql = "UPDATE ip_stat SET "..day_column.."=0,"..flow_column.."=0"
status, errorString = db:exec(update_sql)
C:write_update_day(input_sn)
end
C:clean_stats(db, input_sn)
if store_end >= store_start then
for i=store_start, store_end, 1 do

18
plugins/webstats/t/index.py
View File

@ -74,13 +74,29 @@ def httpPost(url, data, timeout=10):
return str(ex)
def httpGet__UA(url, ua, timeout=10):
import urllib.request
headers = {'user-agent': ua}
try:
req = urllib.request.Request(url, headers=headers)
response = urllib.request.urlopen(req)
result = response.read().decode('utf-8')
return result
except Exception as e:
return str(e)
def test_OK():
'''
目录保存
'''
url = TEST_URL + "ok.txt"
print("ok test start")
url_val = httpGet(url, 10)
# Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36
# (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36
url_val = httpGet__UA(
url, "Mozilla / 5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit / 537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36")
print(url_val)
print("ok test end")