diff --git a/plugins/op_waf/conf/init.sql b/plugins/op_waf/conf/init.sql new file mode 100644 index 000000000..b0f0be0c7 --- /dev/null +++ b/plugins/op_waf/conf/init.sql @@ -0,0 +1,15 @@ +PRAGMA synchronous = 0; +PRAGMA page_size = 4096; +PRAGMA journal_mode = wal; +PRAGMA journal_size_limit = 1073741824; + +CREATE TABLE IF NOT EXISTS `waf_history` ( + `time` INTEGER, + `ip` TEXT, + `domain` TEXT, + `server_name` TEXT, + `method` TEXT, + `status_code` INTEGER, + `uri` TEXT, + `reason` TEXT +); diff --git a/plugins/op_waf/index.py b/plugins/op_waf/index.py index c561c4500..822ca3c8c 100755 --- a/plugins/op_waf/index.py +++ b/plugins/op_waf/index.py @@ -72,6 +72,30 @@ def getConf(): return path +def pSqliteDb(dbname='waf_history'): + name = "waf" + db_dir = getServerDir() + '/logs/' + + if not os.path.exists(db_dir): + mw.execShell('mkdir -p ' + db_dir) + + file = db_dir + name + '.db' + if not os.path.exists(file): + conn = mw.M(dbname).dbPos(db_dir, name) + sql = mw.readFile(getPluginDir() + '/conf/init.sql') + sql_list = sql.split(';') + for index in range(len(sql_list)): + conn.execute(sql_list[index]) + else: + conn = mw.M(dbname).dbPos(db_dir, name) + + conn.execute("PRAGMA synchronous = 0") + conn.execute("PRAGMA page_size = 4096") + conn.execute("PRAGMA journal_mode = wal") + conn.execute("PRAGMA journal_size_limit = 1073741824") + return conn + + def initDomainInfo(): data = [] path_domains = getJsonPath('domains') @@ -317,6 +341,8 @@ def initDreplace(): initTotalInfo() autoMakeLuaConf() + pSqliteDb() + if not mw.isAppleSystem(): mw.execShell("chown -R www:www " + path) diff --git a/plugins/op_waf/t/test.sh b/plugins/op_waf/t/test.sh index 15dce272a..ce5c8a777 100755 --- a/plugins/op_waf/t/test.sh +++ b/plugins/op_waf/t/test.sh @@ -11,66 +11,3 @@ export PATH # ab -c 1000 -n 1000000 http://dev156.cachecha.com/ python3 index.py - - -# 安装 火焰图 -# sudo apt install elfutils -# sudo apt-get build-dep systemtap -# systemtap -# stap -e 'probe kernel.function("sys_open") {log("hello world") exit()}' -# stap -v -e 'probe vfs.read {printf("read performed\n"); exit()}' - - -# yum -y kernel-devel kernel-headers gcc elfutils -# stap -ve 'probe begin { log("hello systemtap!") exit() }' -# stap -e 'probe vfs.add_to_page_cache {printf("dev=%d, devname=%s, ino=%d, index=%d, nrpages=%d\n", dev, devname, ino, index, nrpages )}' -# git clone https://github.com/openresty/openresty-systemtap-toolkit -# http://openresty.org/en/build-systemtap.html - -# ./configure --prefix=/opt/stap --disable-docs --disable-publican --disable-refdocs CFLAGS="-g -O2" - -# apt-get install systemtap linux-image-`uname -r`-dbg linux-headers-`uname -r` -# /usr/share/doc/systemtap/README.Debian -# ./ngx-active-reqs -p 383774 - - -# wget -O kernel-debuginfo-$(uname -r).rpm http://debuginfo.centos.org/8/x86_64/kernel-debuginfo-$(uname -r).rpm - -# wget -O kernel-debuginfo-4.18.0-348.el8.x86_64.rpm http://debuginfo.centos.org/8/x86_64/Packages/kernel-debuginfo-4.18.0-348.el8.x86_64.rpm -# wget -O kernel-debuginfo-common-x86_64-4.18.0-348.el8.x86_64.rpm http://debuginfo.centos.org/8/x86_64/Packages/kernel-debuginfo-common-x86_64-4.18.0-348.el8.x86_64.rpm - -# rpm -ivh kernel-debuginfo-4.18.0-348.el8.x86_64.rpm -# rpm -ivh kernel-debuginfo-common-x86_64-4.18.0-348.el8.x86_64.rpm -# uname -r - -# yum install kernel-devel-4.18.0-358.el8.x86_64 -# yum install kernel-debuginfo-4.18.0-358.el8.x86_64 - -# yum search kernel-debuginfo -# kernel-devel - -# rpm -ivh kernel-debuginfo-4.18.0-358.el8.x86_64 -# yum install systemtap -y -# yum install perf -y - - -# perf record -F 99 -p 4452 -g -o test.data -- sleep 100 - - -# perf record -F 99 -p 153145 -g -o test.data -- sleep 100 - -# perf script -i test.data &> perf.unfold -# ./FlameGraph/stackcollapse-perf.pl perf.unfold &> perf.folded -# ./FlameGraph/flamegraph.pl perf.folded > perf.svg - -# -# git clone https://github.com/brendangregg/FlameGraph.git - -# git clone https://github.com/openresty/openresty-systemtap-toolkit -# ps -ef|grep openresty | grep -v grep | awk '{print $2}' -# ./ngx-active-reqs -p 153145 - - - - - diff --git a/plugins/op_waf/waf/lua/common.lua b/plugins/op_waf/waf/lua/common.lua index 28aa39c20..4ffa7aac0 100644 --- a/plugins/op_waf/waf/lua/common.lua +++ b/plugins/op_waf/waf/lua/common.lua @@ -25,21 +25,44 @@ function _M.new(self) server_name = '', global_tatal = nil, params = nil, + db = nil, } return setmetatable(self, mt) end - - function _M.getInstance(self) if rawget(self, "instance") == nil then rawset(self, "instance", self.new()) + self.initDB() end assert(self.instance ~= nil) return self.instance end +function _M.initDB(self) + if self.db then + return self.db + end + + local path = log_dir .. "/waf.db" + db, err = sqlite3.open(path) + + if err then + self:D("initDB err:"..tostring(err)) + return nil + end + + db:exec([[PRAGMA synchronous = 0]]) + db:exec([[PRAGMA cache_size = 8000]]) + db:exec([[PRAGMA page_size = 32768]]) + db:exec([[PRAGMA journal_mode = wal]]) + db:exec([[PRAGMA journal_size_limit = 1073741824]]) + + self.db = db + return db +end + function _M.setDebug(self, mode) debug_mode = mode end diff --git a/plugins/webstats/lua/webstats_log.lua b/plugins/webstats/lua/webstats_log.lua index 5588e7bbe..27a6b0f2d 100644 --- a/plugins/webstats/lua/webstats_log.lua +++ b/plugins/webstats/lua/webstats_log.lua @@ -262,7 +262,6 @@ log_by_lua_block { request_stat_fields = request_stat_fields .. ","..field.."="..field.."+1" end - -- D("method:"..method) local lower_method = string.lower(method) if ngx.re.find("get,post,put,patch,delete", lower_method, "ijo") then local field = "http_"..lower_method @@ -304,10 +303,8 @@ log_by_lua_block { local stat_fields = request_stat_fields..";"..client_stat_fields..";"..spider_stat_fields - - C:D("stat_fields:"..stat_fields) - -- cache_set(server_name, new_id, "stat_fields", stat_fields) - -- cache_set(server_name, new_id, "log_kv", json.encode(kv)) + cache_set(server_name, new_id, "stat_fields", stat_fields) + cache_set(server_name, new_id, "log_kv", json.encode(kv)) end local function cache_logs(input_sn) @@ -370,10 +367,7 @@ log_by_lua_block { } -- C:D(json.encode(kv)) - local request_stat_fields = { - req=1, - length=body_length, - } + local request_stat_fields = {req=1,length=body_length} local spider_stat_fields = {} local client_stat_fields = {} @@ -442,7 +436,7 @@ log_by_lua_block { } local push_data = json.encode(data) - -- C:D("push_data:"..push_data) + local key = C:getTotalKey() ngx.shared.mw_total:rpush(key, push_data) end