Skip to content

Commit

Permalink
支持按key查询
Browse files Browse the repository at this point in the history
  • Loading branch information
@jie123108
jie123108 committed Apr 1, 2017
1 parent 99693cf commit 6be627f
Show file tree
Hide file tree
Showing 9 changed files with 301 additions and 125 deletions.
25 changes: 17 additions & 8 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,23 +35,28 @@ Synopsis
-- stats by uri
stats.add_stats_config("stats_uri",
{selector={date="$date",key="$uri"}, update=update,index_keys={"date", "key"}})
{selector={date="$date",key="$uri"}, update=update,
indexes={{keys={'date', 'key'}, options={unique=true}},{keys={'key'}, options={}}} })
-- stats by arg
stats.add_stats_config("stats_arg",
{selector={date="$date",key="$arg_client_type"}, update=update,index_keys={"date", "key"}})
{selector={date="$date",key="$arg_client_type"}, update=update,
indexes={{keys={'date', 'key'}, options={unique=true}},{keys={'key'}, options={}}} })
-- stats by uri and args
stats.add_stats_config("stats_uri_arg",
{selector={date="$date",key="$uri?$arg_from"}, update=update,index_keys={"date", "key"}})
{selector={date="$date",key="$uri?$arg_from"}, update=update,
indexes={{keys={'date', 'key'}, options={unique=true}},{keys={'key'}, options={}}} })
-- stats by http request header
stats.add_stats_config("stats_header_in",
{selector={date="$date",key="city:$http_city"}, update=update,index_keys={"date", "key"}})
{selector={date="$date",key="city:$http_city"}, update=update,
indexes={{keys={'date', 'key'}, options={unique=true}},{keys={'key'}, options={}}} })
-- stats by http response header
stats.add_stats_config("stats_header_out",
{selector={date="$date",key="cache:$sent_http_cache"}, update=update,index_keys={"date", "key"}})
{selector={date="$date",key="cache:$sent_http_cache"}, update=update,
indexes={{keys={'date', 'key'}, options={unique=true}},{keys={'key'}, options={}}} })
local mongo_cfg = {host="192.168.1.201", port=27017, dbname="ngx_stats"}
local flush_interval = 2 -- second
Expand Down Expand Up @@ -182,7 +187,11 @@ stats_config:
selector={date='$date',key='$host'},
update={['$inc']= {count=1, ['hour_cnt.$hour']=1, ['status.$status']=1,
['req_time.all']="$request_time", ['req_time.$hour']="$request_time"}},
index_keys={'date', 'key'}, index_options={}}
indexes={
{keys={'date', 'key'}, options={unique=true}},
{keys={'key'}, options={}}
},
}
}
```
After this method is called, when you used stats.log(stats_name) method, you can use these predefined statistics.
Expand All @@ -199,7 +208,7 @@ The name will be used when calling the `stats.log(stats_name)` method.
* `selector` a mongodb query statement. like: `{date="$date",key="$host"}`
* `update` a mongodb update statement. like: `{["$inc"]= {count=1, ["hour_cnt.$hour"]=1, ["status.$status"]=1,
["req_time.all"]="$request_time", ["req_time.$hour"]="$request_time"}}`
* `index_keys` a table that contains all fields of the index.
* `indexes` a table that contains all fields of the index.

The `selector` and `update` configuration can use [variables](#variables). <br/>
Note that "$inc" is not a nginx variable, it's a mongodb's operator.
Expand Down
44 changes: 24 additions & 20 deletions lib/resty/stats/coll_util.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ comment: create collection index。
local mongo_dao = require("resty.stats.mongo_dao")
local util = require("resty.stats.util")
local t_ordered = require("resty.stats.orderedtable")
local json = require("resty.stats.json")

local _M = {}

Expand All @@ -15,35 +16,38 @@ local _M = {}
_M.exist_indexes = {}

-- create index if not exist!
function _M.create_coll_index(mongo_cfg, collection, index_keys, index_options)
function _M.create_coll_index(mongo_cfg, collection, indexes)
if _M.exist_indexes[collection] then
return true, "idx-exist"
end

local dao = mongo_dao:new(mongo_cfg, collection)
local ok, err = nil

local keys = t_ordered({})
for _, key in ipairs(index_keys) do
keys[key] = 1
end
local options = {unique=true}
if index_options then
for k, v in pairs(index_options) do
options[k] = v
for _, index_info in ipairs(indexes) do
ngx.log(ngx.ERR, "--- coll: ", tostring(collection), " index_info: ", json.dumps(index_info))
local index_keys = index_info.keys or index_info.index_keys
local index_options = index_info.options or index_info.index_options
local keys = t_ordered({})
for _, key in ipairs(index_keys) do
keys[key] = 1
end
end
ok, err, idx_name = dao:ensure_index(keys,options)
if ok then
ngx.log(ngx.INFO, "create index [",tostring(idx_name), "] for [", collection, "] success! ")
else
local err = tostring(err)
if(string.find(err, "already exists")) then
ok = true
local options = {}
if index_options then
for k, v in pairs(index_options) do
options[k] = v
end
end
ok, err, idx_name = dao:ensure_index(keys,options)
if ok then
ngx.log(ngx.INFO, "create index [",tostring(idx_name), "] for [", collection, "] success! ")
else
local err = tostring(err)
if(string.find(err, "already exists")) then
ok = true
end
ngx.log(ngx.ERR, "create index [",tostring(idx_name), "] for [", collection, "] failed! err:", err)
end
ngx.log(ngx.ERR, "create index [",tostring(idx_name), "] for [", collection, "] failed! err:", err)
end

-- dao:uninit()
if ok then
_M.exist_indexes[collection] = true
Expand Down
26 changes: 16 additions & 10 deletions lib/resty/stats/init.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,14 +74,18 @@ $hour,$minute,$second 分别为时分秒。
]]
-- selector 更新使用的查询子
-- update 更新语句。
-- index_keys 更新查询需要用到的索引的字段。
-- indexes 更新查询需要用到的索引的字段。
local def_stats_configs = {
stats_host={
selector={date='$date',key='$host'},
update={['$inc']= {count=1, ['hour_cnt.$hour']=1, ['status.$status']=1,
['req_time.all']="$request_time", ['req_time.$hour']="$request_time"}},
index_keys={'date', 'key'}, index_options={}}
indexes={
{keys={'date', 'key'}, options={unique=true}},
{keys={'key'}, options={}}
}
}
}

local stats_configs = {}

Expand Down Expand Up @@ -170,15 +174,18 @@ stats_name: stats name and table name.
stats_config:
-- selector: the mongodb update selector
-- update: the mongodb update statement
-- index_keys the index keys of the selector used.
-- indexes the indexes of the selector used.
eg.:
stats_name: stats_host
stats_config:
{
selector={date='$date',key='$host'},
update={['$inc']= {count=1, ['hour_cnt.$hour']=1, ['status.$status']=1,
['req_time.all']="$request_time", ['req_time.$hour']="$request_time"}},
index_keys={'date', 'key'}, index_options={}}
indexes={
{keys={'date', 'key'}, options={unique=true}},
{keys={'key'}, options={}}
}
}
]]
function _M.add_stats_config(stats_name, stats_config)
Expand Down Expand Up @@ -210,12 +217,11 @@ function _M.init(mongo_cfg, flush_interval, retry_interval)
check_config(stats_configs)
local function create_index_callback(premature, stats_configs, mongo_cfg)
for stats_name, stats_config in pairs(stats_configs) do
local index_keys = stats_config.index_keys
if index_keys then
local collection = stats_name
local ok, err = coll_util.create_coll_index(mongo_cfg, collection, index_keys,stats_config.index_options)
ngx.log(ngx.INFO, "create_coll_index(", collection, ") ok:",
tostring(ok), ", err:", tostring(err))
local indexes = stats_config.indexes
if indexes then
local collection = stats_name
local ok, err = coll_util.create_coll_index(mongo_cfg, collection, indexes)
ngx.log(ngx.INFO, "create_coll_index(", collection, ") ok:", tostring(ok), ", err:", tostring(err))
end
end
end
Expand Down
17 changes: 11 additions & 6 deletions t/nginx.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,23 +41,28 @@ http {
-- stats by uri
stats.add_stats_config("stats_uri",
{selector={date="$date",key="$uri"}, update=update,index_keys={"date", "key"}})
{selector={date="$date",key="$uri"}, update=update,
indexes={{keys={'date', 'key'}, options={unique=true}},{keys={'key'}, options={}}} })
-- stats by arg
stats.add_stats_config("stats_arg",
{selector={date="$date",key="$arg_client_type"}, update=update,index_keys={"date", "key"}})
{selector={date="$date",key="$arg_client_type"}, update=update,
indexes={{keys={'date', 'key'}, options={unique=true}},{keys={'key'}, options={}}} })
-- stats by uri and args
stats.add_stats_config("stats_uri_arg",
{selector={date="$date",key="$uri?$arg_from"}, update=update,index_keys={"date", "key"}})
{selector={date="$date",key="$uri?$arg_from"}, update=update,
indexes={{keys={'date', 'key'}, options={unique=true}},{keys={'key'}, options={}}} })
-- stats by http request header
stats.add_stats_config("stats_header_in",
{selector={date="$date",key="city:$http_city"}, update=update,index_keys={"date", "key"}})
{selector={date="$date",key="city:$http_city"}, update=update,
indexes={{keys={'date', 'key'}, options={unique=true}},{keys={'key'}, options={}}} })
-- stats by http response header
stats.add_stats_config("stats_header_out",
{selector={date="$date",key="cache:$sent_http_cache"}, update=update,index_keys={"date", "key"}})
{selector={date="$date",key="cache:$sent_http_cache"}, update=update,
indexes={{keys={'date', 'key'}, options={unique=true}},{keys={'key'}, options={}}} })
local mongo_cfg = {host="127.0.0.1", port=27017, dbname="ngx_stats"}
local flush_interval = 2 -- second
Expand Down
55 changes: 20 additions & 35 deletions view/filter.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,12 @@ function _M.requests_alt(stats)
return table.concat(alts)
end

function _M.ok(stats)
local function status_count(stats, begin, end_)
if stats.status then
local ok_count = 0
for status, count in pairs(stats.status) do
status = tonumber(status) or 0
if status >= 200 and status < 400 then
if status >= begin and status <= end_ then
ok_count = ok_count + tonumber(count)
end
end
Expand All @@ -42,15 +42,15 @@ function _M.ok(stats)
end
end

function _M.ok_alt(stats)
local function status_alt(stats, begin, end_)
if stats.status == nil then
return ""
end

local status_all = {}
for status, count in pairs(stats.status) do
local xstatus = tonumber(status) or 0
if xstatus >= 200 and xstatus < 400 then
if xstatus >= begin and xstatus <= end_ then
table.insert(status_all, status)
end
end
Expand All @@ -64,43 +64,28 @@ function _M.ok_alt(stats)
return table.concat(alts)
end

function _M.ok(stats)
return status_count(stats, 200, 399)
end

function _M.fail(stats)
if stats.status then
local fail_count = 0
for status, count in pairs(stats.status) do
status = tonumber(status) or 0
if status >= 400 then
fail_count = fail_count + tonumber(count)
end
end
return fail_count

else
return '0'
end
function _M.ok_alt(stats)
return status_alt(stats, 200, 399)
end

function _M.fail_alt(stats)
if stats.status == nil then
return ""
end
function _M.fail_4xx(stats)
return status_count(stats, 400, 499)
end

local status_all = {}
for status, count in pairs(stats.status) do
local xstatus = tonumber(status) or 0
if xstatus >= 400 then
table.insert(status_all, status)
end
end
function _M.fail_alt_4xx(stats)
return status_alt(stats, 400, 499)
end

table.sort(status_all)
function _M.fail_5xx(stats)
return status_count(stats, 500, 599)
end

local alts = {}
for i, status in ipairs(status_all) do
table.insert(alts, status .. ": " .. tostring(stats.status[status]))
end
return table.concat(alts)
function _M.fail_alt_5xx(stats)
return status_alt(stats, 500, 599)
end

function _M.avgtime(stats)
Expand Down
Loading

0 comments on commit 6be627f

Please sign in to comment.