mirror of
https://github.com/midoks/mdserver-web.git
synced 2026-10-10 01:29:26 +08:00
up
This commit is contained in:
midoks
@@ -0,0 +1,56 @@
|
||||
import sys
|
||||
import os
|
||||
|
||||
|
||||
class LuaMaker:
|
||||
"""
|
||||
lua 处理器
|
||||
"""
|
||||
@staticmethod
|
||||
def makeLuaTable(table):
|
||||
"""
|
||||
table 转换为 lua table 字符串
|
||||
"""
|
||||
_tableMask = {}
|
||||
_keyMask = {}
|
||||
|
||||
def analysisTable(_table, _indent, _parent):
|
||||
if isinstance(_table, tuple):
|
||||
_table = list(_table)
|
||||
if isinstance(_table, list):
|
||||
_table = dict(zip(range(1, len(_table) + 1), _table))
|
||||
if isinstance(_table, dict):
|
||||
_tableMask[id(_table)] = _parent
|
||||
cell = []
|
||||
thisIndent = _indent + " "
|
||||
for k in _table:
|
||||
if sys.version_info[0] == 2:
|
||||
if type(k) not in [int, float, bool, list, dict, tuple]:
|
||||
k = k.encode()
|
||||
|
||||
if not (isinstance(k, str) or isinstance(k, int) or isinstance(k, float)):
|
||||
return
|
||||
key = isinstance(
|
||||
k, int) and "[" + str(k) + "]" or "[\"" + str(k) + "\"]"
|
||||
if _parent + key in _keyMask.keys():
|
||||
return
|
||||
_keyMask[_parent + key] = True
|
||||
var = None
|
||||
v = _table[k]
|
||||
if sys.version_info[0] == 2:
|
||||
if type(v) not in [int, float, bool, list, dict, tuple]:
|
||||
v = v.encode()
|
||||
if isinstance(v, str):
|
||||
var = "\"" + v + "\""
|
||||
elif isinstance(v, bool):
|
||||
var = v and "true" or "false"
|
||||
elif isinstance(v, int) or isinstance(v, float):
|
||||
var = str(v)
|
||||
else:
|
||||
var = analysisTable(v, thisIndent, _parent + key)
|
||||
cell.append(thisIndent + key + " = " + str(var))
|
||||
lineJoin = ",\n"
|
||||
return "{\n" + lineJoin.join(cell) + "\n" + _indent + "}"
|
||||
else:
|
||||
pass
|
||||
return analysisTable(table, "", "root")
|
||||
@@ -1,3 +1,9 @@
|
||||
PRAGMA synchronous = 0;
|
||||
PRAGMA page_size = 4096;
|
||||
PRAGMA journal_mode = wal;
|
||||
PRAGMA journal_size_limit = 1073741824;
|
||||
|
||||
|
||||
CREATE TABLE IF NOT EXISTS `web_logs` (
|
||||
`time` INTEGER,
|
||||
`ip` TEXT,
|
||||
|
||||
@@ -2,7 +2,8 @@
|
||||
<div class="bt-w-main">
|
||||
<div class="bt-w-menu">
|
||||
<p class="bgw" onclick="pluginService('webstats');">服务</p>
|
||||
<p onclick="pluginConfig('webstats');">概览</p>
|
||||
<p onclick="wsGlobalSetting();">概览</p>
|
||||
<p onclick="wsGlobalSetting();">网站日志</p>
|
||||
<p onclick="wsGlobalSetting();">全局设置</p>
|
||||
</div>
|
||||
<div class="bt-w-con pd15">
|
||||
|
||||
@@ -4,10 +4,12 @@ import sys
|
||||
import io
|
||||
import os
|
||||
import time
|
||||
import json
|
||||
|
||||
sys.path.append(os.getcwd() + "/class/core")
|
||||
import mw
|
||||
|
||||
|
||||
app_debug = False
|
||||
if mw.isAppleSystem():
|
||||
app_debug = True
|
||||
@@ -21,6 +23,10 @@ def getPluginDir():
|
||||
return mw.getPluginDir() + '/' + getPluginName()
|
||||
|
||||
|
||||
sys.path.append(getPluginDir() + "/class")
|
||||
from LuaMaker import LuaMaker
|
||||
|
||||
|
||||
def getServerDir():
|
||||
return mw.getServerDir() + '/' + getPluginName()
|
||||
|
||||
@@ -60,11 +66,17 @@ def status():
|
||||
return 'start'
|
||||
|
||||
|
||||
def pSqliteDb(dbname='web_logs'):
|
||||
file = getServerDir() + '/webstats.db'
|
||||
name = 'webstats'
|
||||
def pSqliteDb(dbname='web_logs', site_name='unset'):
|
||||
db_dir = getServerDir() + '/logs/' + site_name
|
||||
|
||||
if not os.path.exists(db_dir):
|
||||
mw.execShell('mkdir -p ' + db_dir)
|
||||
|
||||
name = 'logs'
|
||||
file = db_dir + '/' + name + '.db'
|
||||
|
||||
if not os.path.exists(file):
|
||||
conn = mw.M(dbname).dbPos(getServerDir(), name)
|
||||
conn = mw.M(dbname).dbPos(db_dir, name)
|
||||
sql = mw.readFile(getPluginDir() + '/conf/init.sql')
|
||||
sql_list = sql.split(';')
|
||||
for index in range(len(sql_list)):
|
||||
@@ -74,6 +86,30 @@ def pSqliteDb(dbname='web_logs'):
|
||||
return conn
|
||||
|
||||
|
||||
def makeSiteConfig():
|
||||
siteM = mw.M('sites')
|
||||
domainM = mw.M('domain')
|
||||
slist = siteM.field('id,name').where(
|
||||
'status=?', (1,)).order('id desc').select()
|
||||
|
||||
data = []
|
||||
for s in slist:
|
||||
tmp = {}
|
||||
tmp['name'] = s['name']
|
||||
|
||||
dlist = domainM.field('id,name').where(
|
||||
'pid=?', (s['id'],)).order('id desc').select()
|
||||
|
||||
_t = []
|
||||
for d in dlist:
|
||||
_t.append(d['name'])
|
||||
|
||||
tmp['domains'] = _t
|
||||
data.append(tmp)
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def initDreplace():
|
||||
|
||||
service_path = getServerDir()
|
||||
@@ -94,10 +130,29 @@ def initDreplace():
|
||||
mw.execShell('mkdir -p ' + lua_dir)
|
||||
lua_tpl = getPluginDir() + '/lua/web_stats_log.lua'
|
||||
content = mw.readFile(lua_tpl)
|
||||
# content = content.replace('{$SERVER_PATH}', service_path)
|
||||
content = content.replace('{$SERVER_APP}', service_path)
|
||||
content = content.replace('{$ROOT_PATH}', mw.getServerDir())
|
||||
mw.writeFile(lua_dst, content)
|
||||
|
||||
content = makeSiteConfig()
|
||||
for index in range(len(content)):
|
||||
pSqliteDb('web_log', content[index]['name'])
|
||||
|
||||
lua_site_json = lua_dir + "/sites.json"
|
||||
if not os.path.exists(lua_site_json):
|
||||
mw.writeFile(lua_site_json, json.dumps(content))
|
||||
|
||||
lua_site = lua_dir + "/sites.lua"
|
||||
if not os.path.exists(lua_site):
|
||||
config_sites = LuaMaker.makeLuaTable(content)
|
||||
sites_str = "return " + config_sites
|
||||
mw.writeFile(lua_site, sites_str)
|
||||
|
||||
log_path = getServerDir() + "/logs"
|
||||
if not os.path.exists(log_path):
|
||||
mw.execShell('mkdir -p ' + log_path)
|
||||
|
||||
debug_log = getServerDir() + "/debug.log"
|
||||
if not os.path.exists(debug_log):
|
||||
mw.execShell('mkdir -p ' + lua_dir)
|
||||
|
||||
@@ -74,23 +74,11 @@ Install_App()
|
||||
fi
|
||||
|
||||
# copy to code path
|
||||
if [ -f $serverPath/webstats/luarocks/lib/lua/5.1/lsqlite3.so ];then
|
||||
cp -rf $serverPath/webstats/luarocks/lib/lua/5.1/lsqlite3.so $serverPath/webstats/lua/lsqlite3.so
|
||||
DEFAULT_DIR=$serverPath/webstats/luarocks/lib/lua/5.1
|
||||
if [ -f ${DEFAULT_DIR}/lsqlite3.so ];then
|
||||
cp -rf ${DEFAULT_DIR}/lsqlite3.so $serverPath/webstats/lua/lsqlite3.so
|
||||
fi
|
||||
|
||||
# if [ ! -d $serverPath/source/webstats/luasql-2.6.0 ];then
|
||||
# wget -O $serverPath/source/webstats/luasql_2.6.0.tar.gz https://github.com/keplerproject/luasql/archive/refs/tags/2.6.0.tar.gz
|
||||
# cd $serverPath/source/webstats && tar xvf luasql_2.6.0.tar.gz
|
||||
# fi
|
||||
|
||||
# PATH=${serverPath}/openresty/luajit:${serverPath}/openresty/luajit/include/luajit-2.1:$PATH
|
||||
# export PATH
|
||||
# export LUA_INCDIR=${serverPath}/openresty/luajit/include/luajit-2.1
|
||||
# cd $serverPath/source/webstats/luasql-2.6.0 && make sqlite3
|
||||
|
||||
# $serverPath/webstats/luarocks/bin/luarocks install luasql-sqlite3 SQLITE_DIR=$serverPath/webstats/lua
|
||||
|
||||
|
||||
echo "${VERSION}" > $serverPath/webstats/version.pl
|
||||
echo '安装完成' > $install_tmp
|
||||
# cd $rootPath && python3 ${rootPath}/plugins/webstats/index.py start
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
log_by_lua_block {
|
||||
|
||||
local ver = '0.0.1'
|
||||
local max_log_id = 99999999999999
|
||||
local debug_mode = true
|
||||
|
||||
local unset_server_name = "unset"
|
||||
|
||||
|
||||
local cpath = "{$SERVER_APP}/lua/"
|
||||
if not package.cpath:find(cpath) then
|
||||
@@ -31,6 +34,7 @@ log_by_lua_block {
|
||||
end
|
||||
|
||||
|
||||
|
||||
-- cache start ---
|
||||
local cache = ngx.shared.mw_total
|
||||
local function cache_set(server_name, id ,key, val)
|
||||
@@ -52,30 +56,296 @@ log_by_lua_block {
|
||||
|
||||
-- domain config is import
|
||||
local server_name
|
||||
|
||||
local ip,today,day,body_length,method,config,cache_count
|
||||
local db = nil
|
||||
local json = require "cjson"
|
||||
local sqlite3 = require "lsqlite3"
|
||||
local request_header = ngx.req.get_headers()
|
||||
|
||||
|
||||
local today,day,method,config,cache_count
|
||||
--- default common var end ---
|
||||
|
||||
local function run_app()
|
||||
D("debug start")
|
||||
local function get_store_key()
|
||||
return os.date("%Y%m%d%H", os.time())
|
||||
end
|
||||
|
||||
local presult, err = pcall(
|
||||
function()
|
||||
json = require "cjson"
|
||||
sqlite3 = require "lsqlite3"
|
||||
local function get_length()
|
||||
local clen = ngx.var.body_bytes_sent
|
||||
if clen == nil then clen = 0 end
|
||||
return tonumber(clen)
|
||||
end
|
||||
|
||||
local function get_domain()
|
||||
local domain = request_header['host']
|
||||
if domain ~= nil then
|
||||
domain = string.gsub(domain, "_", ".")
|
||||
else
|
||||
domain = "unknown"
|
||||
end
|
||||
return domain
|
||||
end
|
||||
|
||||
local function get_server_name(c_name)
|
||||
local my_name = cache:get(c_name)
|
||||
if my_name then return my_name end
|
||||
|
||||
D("get_server_name start")
|
||||
|
||||
local determined_name = nil
|
||||
local sites = require "sites"
|
||||
D("get_server_name"..json.encode(sites))
|
||||
for _,v in ipairs(sites)
|
||||
do
|
||||
if c_name == v["name"] then
|
||||
cache:set(c_name, v['name'],3600)
|
||||
return v["name"]
|
||||
end
|
||||
)
|
||||
for _,d_name in ipairs(v['domains'])
|
||||
do
|
||||
if c_name == d_name then
|
||||
cache:set(c_name,v['name'],3600)
|
||||
return v['name']
|
||||
elseif string.find(d_name, "*") then
|
||||
new_domain = string.gsub(d_name, '*', '.*')
|
||||
debug("ngx server name:"..ngx.var.server_name.."/new_domain:"..new_domain)
|
||||
if string.find(c_name, new_domain) then
|
||||
-- cache:set(c_name, v['name'],3600)
|
||||
-- return v['name']
|
||||
-- debug("find deter name:"..v['name'])
|
||||
determined_name = v['name']
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
D("get_server_name end")
|
||||
if determined_name then
|
||||
cache:set(c_name, determined_name,3600)
|
||||
return determined_name
|
||||
end
|
||||
cache:set(c_name, unset_server_name, 3600)
|
||||
return unset_server_name
|
||||
end
|
||||
|
||||
|
||||
local function cache_logs()
|
||||
|
||||
-- make new id
|
||||
local new_id = nil
|
||||
local last_insert_id_key = server_name .. "_last_id"
|
||||
local err = nil
|
||||
|
||||
new_id, err = cache:incr(last_insert_id_key, 1, 0)
|
||||
cache:incr(cache_count_id_key, 1, 0)
|
||||
if new_id >= max_log_id then
|
||||
cache:set(last_insert_id_key, 1)
|
||||
new_id = cache:get(last_insert_id_key)
|
||||
end
|
||||
|
||||
D("new_id:"..new_id)
|
||||
|
||||
local ip_list = request_header["x-forwarded-for"]
|
||||
local ip = ngx.var.remote_addr
|
||||
if ip and not ip_list then
|
||||
ip_list = ip
|
||||
end
|
||||
|
||||
if not presult then
|
||||
D("depend on :"..tostring(err))
|
||||
local client_port = ngx.var.remote_port
|
||||
local request_time = ngx.var.request_time
|
||||
local real_server_name = server_name
|
||||
local uri = ngx.var.uri
|
||||
local status_code = ngx.status
|
||||
local protocol = ngx.var.server_protocol
|
||||
local request_uri = ngx.var.request_uri
|
||||
local time_key = get_store_key()
|
||||
local method = ngx.req.get_method()
|
||||
local body_length = get_length()
|
||||
local domain = get_domain()
|
||||
local referer = ngx.var.http_referer
|
||||
|
||||
|
||||
kv = {
|
||||
id=new_id,
|
||||
time_key=time_key,
|
||||
time=os.time(),
|
||||
ip=ip,
|
||||
domain=domain,
|
||||
server_name=server_name,
|
||||
real_server_name=real_server_name,
|
||||
method=method,
|
||||
status_code=status_code,
|
||||
uri=uri,
|
||||
request_uri=request_uri,
|
||||
body_length=body_length,
|
||||
referer=referer,
|
||||
-- user_agent=request_header['user-agent'],
|
||||
protocol=protocol,
|
||||
is_spider=0,
|
||||
request_time=request_time,
|
||||
-- excluded=excluded,
|
||||
-- request_headers="",
|
||||
ip_list=ip_list,
|
||||
client_port=client_port
|
||||
}
|
||||
|
||||
D(server_name..'__'..json.encode(kv))
|
||||
cache_set(server_name, new_id, "log_kv", json.encode(kv))
|
||||
end
|
||||
|
||||
local function store_logs_line(db, stmt, input_server_name, lineno)
|
||||
local logvalue = cache_get(input_server_name, lineno, "log_kv")
|
||||
D("store_logs_line:"..logvalue)
|
||||
if not logvalue then return false end
|
||||
local logline = json.decode(logvalue)
|
||||
|
||||
local time = logline["time"]
|
||||
local id = logline["id"]
|
||||
local protocol = logline["protocol"]
|
||||
local client_port = logline["client_port"]
|
||||
local status_code = logline["status_code"]
|
||||
local uri = logline["uri"]
|
||||
local method = logline["method"]
|
||||
local body_length = logline["body_length"]
|
||||
local referer = logline["referer"]
|
||||
local ip = logline["ip"]
|
||||
local ip_list = logline["ip_list"]
|
||||
local request_time = logline["request_time"]
|
||||
local is_spider = logline["is_spider"]
|
||||
local domain = logline["domain"]
|
||||
local server_name = logline["server_name"]
|
||||
|
||||
stmt:bind_names{
|
||||
time=time,
|
||||
ip=ip,
|
||||
domain=domain,
|
||||
server_name=server_name,
|
||||
method=method,
|
||||
status_code=status_code,
|
||||
uri=uri,
|
||||
body_length=body_length,
|
||||
referer=referer,
|
||||
user_agent="d",
|
||||
protocol=protocol,
|
||||
request_time=request_time,
|
||||
is_spider=is_spider,
|
||||
request_headers=".",
|
||||
ip_list=ip_list,
|
||||
client_port=client_port,
|
||||
}
|
||||
|
||||
local res, err = stmt:step()
|
||||
if tostring(res) == "5" then
|
||||
D("Step res:"..tostring(res))
|
||||
D("Step err:"..tostring(err))
|
||||
D("Step数据库连接繁忙，稍候存储。")
|
||||
return false
|
||||
end
|
||||
stmt:reset()
|
||||
|
||||
D("store_logs_line ok")
|
||||
end
|
||||
|
||||
local function store_logs(input_server_name)
|
||||
|
||||
local last_insert_id_key = input_server_name.."_last_id"
|
||||
local store_start_id_key = input_server_name.."_store_start"
|
||||
local last_id = cache:get(last_insert_id_key)
|
||||
local store_start = cache:get(store_start_id_key)
|
||||
if store_start == nil then
|
||||
store_start = 1
|
||||
end
|
||||
local store_end = last_id
|
||||
if store_end == nil then
|
||||
store_end = 1
|
||||
end
|
||||
|
||||
|
||||
local log_dir = "{$SERVER_APP}/logs"
|
||||
local db_path= log_dir .. '/' .. input_server_name .. "/logs.db"
|
||||
local db, err = sqlite3.open(db_path)
|
||||
D("sqlite3 path:"..db_path)
|
||||
|
||||
-- python3 ./plugins/webstats/index.py reload && echo "" > /Users/midoks/Desktop/mwdev/server/webstats/debug.log && wget http://t1.cn/
|
||||
|
||||
D("sqlite3 error:"..tostring(err))
|
||||
local stmt2 = nil
|
||||
|
||||
|
||||
if db ~= nil then
|
||||
stmt2 = db:prepare[[INSERT INTO web_logs(
|
||||
time, ip, domain, server_name, method, status_code, uri, body_length,
|
||||
referer, user_agent, protocol, request_time, is_spider, request_headers, ip_list, client_port)
|
||||
VALUES(:time, :ip, :domain, :server_name, :method, :status_code, :uri,
|
||||
:body_length, :referer, :user_agent, :protocol, :request_time, :is_spider,
|
||||
:request_headers, :ip_list, :client_port)]]
|
||||
end
|
||||
|
||||
if db == nil or stmt2 == nil then
|
||||
D("网站监控报表数据库连接异常。")
|
||||
-- cache:set(storing_key, false)
|
||||
if db and db:isopen() then
|
||||
db:close()
|
||||
end
|
||||
return true
|
||||
end
|
||||
|
||||
D("debug end")
|
||||
status, errorString = db:exec([[BEGIN TRANSACTION]])
|
||||
if store_end >= store_start then
|
||||
for i=store_start, store_end, 1 do
|
||||
D("store_start:"..store_start..":store_end:".. store_end)
|
||||
if store_logs_line(db, stmt2, input_server_name, i) then
|
||||
store_count = store_count + 1
|
||||
cache_clear(input_server_name, i, "log_kv")
|
||||
-- cache_clear(input_server_name, i, "STAT_FIELDS")
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
local res, err = stmt2:finalize()
|
||||
|
||||
if tostring(res) == "5" then
|
||||
D("Finalize res:"..tostring(res))
|
||||
D("Finalize err:"..tostring(err))
|
||||
D("数据库连接繁忙，稍候存储.")
|
||||
end
|
||||
|
||||
local res, err = db:execute([[COMMIT]])
|
||||
|
||||
if db and db:isopen() then
|
||||
db:close()
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
return run_app()
|
||||
local function run_app()
|
||||
D("------------ debug start ------------")
|
||||
|
||||
local c_name = ngx.var.server_name
|
||||
server_name = string.gsub(get_server_name(c_name),'_','.')
|
||||
|
||||
D("c_name:"..c_name)
|
||||
D("server_name:"..server_name)
|
||||
|
||||
cache_logs()
|
||||
store_logs(server_name)
|
||||
|
||||
D("------------ debug end ------------")
|
||||
end
|
||||
|
||||
|
||||
local function run_app_ok()
|
||||
if not debug_mode then return run_app() end
|
||||
|
||||
local presult, err = pcall(
|
||||
function()
|
||||
run_app()
|
||||
end
|
||||
)
|
||||
if not presult then
|
||||
D("err on :"..tostring(err))
|
||||
return true
|
||||
end
|
||||
end
|
||||
|
||||
return run_app_ok()
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user