https://my.oschina.net/MrYx3en/blog/525803

 

nginxerr.rulebase :

$ [root@10.10.10.254 test]# cat nginxerr.rulebase 
rule=A: %date:char-to:\x20% %time:time-24hr% [%level:char-to:\x5D%] %f1:char-to::%: %f2:char-to:\x20% %errmsg:char-to:,%, client: %client:ipv4%, server: %server:rest%"

#B和C不能同时存在一个rulebase文件中,否则会导致误解析。
rule=B: %date:char-to:\x20% %time:time-24hr% [%level:char-to:\x5D%] %f1:char-to::%: %f2:char-to:\x20% %errmsg:char-to:,%, client: %client:ipv4%, server: %server:char-to:,%, request: "%verb:word% %urlpath:char-to:\x3F%?%urlparam:char-to:\x20% HTTP/%httpversion:char-to:\x22%", upstream: %upstream:char-to:,%, host: %host:rest%

rule=C: %date:char-to:\x20% %time:time-24hr% [%level:char-to:\x5D%] %f1:char-to::%: %f2:char-to:\x20% %errmsg:char-to:,%, client: %client:ipv4%, server: %server:char-to:,%, request: "%verb:word% %urlpath:char-to:\x20% HTTP/%httpversion:char-to:\x22%", upstream: %upstream:char-to:\x2C%, host: %host:rest%

rule=D: %date:char-to:\x20% %time:time-24hr% [%level:char-to:\x5D%] %f1:char-to::%: %f2:char-to:\x20% %errmsg:char-to:,%, client: %client:ipv4%, server: %server:char-to:,%, request: "%verb:word% %urlpath:char-to:\x3F%?%urlparam:char-to:\x20% HTTP/%httpversion:char-to:\x22%", host: %host:rest%

rule=F: %errmsg:rest%

\\

liblognormalizer命令解析:

$ lognormalizer -r nginxerr.rulebase -e json -T < test0.log > normalized.log

\\

解析后的格式(Json):

$ [root@10.10.10.254 test]# head -n 3 normalized.log "host""\"api.weibo.cn\"""httpversion""1.1""urlparam""gsid=_2A257MJvbDeTxGeRG4loS9yrEzj-IHXVWZ6gTrDV6PUJbrdANLWT1kWpTgI6SCDQQPwBgknJMn2cCApowHA..&wm=3333_2001&i=9130105&b=1&from=1055093010&c=iphone&v_p=24&skin=default&v_f=1&s=3c44f071&lang=zh_CN&ua=iPhone8,1__weibo__5.5.0__iphone__os9.0&sflag=1""urlpath""\/2\/client\/addlog_batch""verb""POST""server""api.v5.weibo.cn""client""118.189.6.88""errmsg""client intended to send too large body: 9742073 bytes""f2""*48466998""f1""21852#0""level""error""time""19:45:10""date""2015\/11\/01""event.tags": [ "D" ] }
{ "host""\"api.weibo.cn\"""httpversion""1.1""urlparam""gsid=_2A257MZL9DeTxGeRP41MZ8i3MzzuIHXVWZqE1rDV6PENPuNIMGlKVlGgHwPxa1DkwlDmCgLaR9Sntltky&wm=3333_2001&i=1db5f43&b=0&from=1055093010&c=iphone&v_p=24&skin=default&v_f=1&s=2e61a91e&lang=en_US&ua=iPhone7,2__weibo__5.5.0__iphone__os9.1&sflag=1""urlpath""\/2\/client\/addlog_batch""verb""POST""server""api.v5.weibo.cn""client""124.213.122.214""errmsg""client intended to send too large body: 35119444 bytes""f2""*48594758""f1""41204#0""level""error""time""19:50:03""date""2015\/11\/01""event.tags": [ "D" ] }
{ "host""\"api.weibo.cn\"""httpversion""1.1""urlparam""gsid=_2A257MKPHDeTxGeRI7VsY9ijIyDmIHXVWZ7APrDV6PUJbrdAKLW_FkWp8DjDOdVlsqU29TOw80hZnkvFfwA..&wm=3333_2001&i=5bb2307&b=1&from=1054093010&c=iphone&v_p=22&skin=default&v_f=1&s=0af1f487&lang=en_US&ua=iPhone7,2__weibo__5.4.0__iphone__os9.1&sflag=1""urlpath""\/2\/client\/addlog_batch""verb""POST""server""api.v5.weibo.cn""client""125.34.2.67""errmsg""client intended to send too large body: 19501344 bytes""f2""*48323317""f1""839#0""level""error""time""19:52:43""date""2015\/11\/01""event.tags": [ "D" ] }

rsyslog配置文件:

$ [root@10.10.10.254 test]# cat nginx-error.conf 
template(name="nginxerrorTemplate" type="list")
{
    constant(value="{\"@timestamp\":\"")    property(name="timereported" dateFormat="rfc3339")
    constant(value="\","\"fromhost\":\"")   property(name="hostname")
    constant(value="\","\"programname\":\"")    property(name="programname")
    constant(value="\",")    property(name="msg" position.from="2")
}
ruleset(name="forwardRulesetlocal7nginxerror")
{
    action(
        type="mmnormalize"
        ruleBase="nginx-error-1.rulebase"
    )
    action(
        type="omfile"
        file=""
        template="nginxerrorTemplate"
    )
}
#request中不包含urlparam
ruleset(name="forwardRulesetlocal7nginxerror-nourlparam")
{
    action(
        type="mmnormalize"
        ruleBase="nginx-error-1.rulebase"
    )
    action(
        type="omfile"
        file=""
        template="nginxerrorTemplate"
    )
}
if ($syslogfacility-text == "local6" and $programname == "nginx-error"then
{
    if ($msg contains ["?"]) then
    {
        call forwardRulesetlocal7nginxerror
    } else {
        call forwardRulesetlocal7nginxerror-nourlparam
    }
}

发表评论

邮箱地址不会被公开。 必填项已用*标注