本文链接:https://blog.csdn.net/wanglei_storage/article/details/82663706
input {

file {
path => “/data/logs/nginx_logs/access.log”
type => “dev-chuiyi-site-landing-1”
#        codec => json
start_position => “end”
}
}

filter {

grok {
patterns_dir => “/usr/local/logstash-6.3.2/vendor/bundle/jruby/2.3.0/gems/logstash-patterns-core-4.1.2/patterns”     # 定义logtash中自定义模式目录所在的位置
match => {
“message” => “%{NGINX_LOGS}”
}
#match => [
#    “message”,”%{NGINX_LOGS}”,
#    “message”,”%{USERHOST:userhost} %{USERNAME:username}”
#]

add_field => {                      # 新增name_%{remote_addr}_[0|1]字段
“name_%{type}_0” => “hello world 0”
“name_%{type}_1” => “hello world 1”
“name_%{type}_2” => “hello world 2”
“coerce_value” => “null”
}

remove_field => [“message”]         # 删除message字段
remove_field => [“name_%{type}_0”]
}

if [request] == “HEAD / HTTP/1.0” {     # 判断request字段是否匹配,如果匹配则drop进行丢弃该条信息,不传输至es
drop {}
}

geoip {
source => “remote_addr”
database => “/usr/local/src/GeoLite2-City_20180807/GeoLite2-City.mmdb”
fields => [“country_name”,”region_code”, “city_name”, “ip”]
}

mutate {
copy => {                           # 将原有的字段进行拷贝一份并命名
“remote_addr” => “client_addr”
}
convert => {                        # 将原有的字段的值进行类型转换,可转换的类型为: integer/float/string/boolean
“status” => “integer”
“request_time” => “float”
}
split => {                          # 将原有字段的string进行split以指定的符号(分隔符)进行分隔,分隔后变为一组数组
“remote_addr” => “.”
}
join => {                           # 将原有字段的array进行join以指定的的符号(分隔符)进行合并,合并后变为一串字符串
“remote_addr” => “-”
}
update => {                         # 用于替换原有字段的值,如果原有字段不存在,则不执行任何操作,原有字段不支持使用%{}变量,但值可以调用%{}变量
“upstream_response_time” => “%{remote_addr}”
}
replace => {                        # 用于替换原有字段的值,如果原有字段不存在,则新增一个字段,新增的字段不支持使用%{}变量,但值可以调用%{}变量
“new_replace” => “%{remote_addr}”
}
gsub => [                           # 用于替换原有字段的值,第一个元素为字段名,第二个为正则或匹配的字符串,第三个为要替换的值(只支持字符串替换操作)
“time_local”, “\d{2}/[A-Za-z]{3}/[\d:]+”, “this is time”,
“request”, “HTTP”, “http”
]
rename => {                         # 将字段http_referer重命名为http_source
“http_referer” => “http_source”
}
}

}

output {
redis {
host => “10.168.216.1”
port => “6379”
key => “redis_auth”
data_type => “list”
password => “2RMYFdlZNSSsqKYi”
}
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
关于grok nginx正则
cat /usr/local/logstash-6.3.2/vendor/bundle/jruby/2.3.0/gems/logstash-patterns-core-4.1.2/patterns

REMOTE_ADDR              ^([\d\.]+)
TIME_LOCAL               [^\s]+\s\+\d{4}
REQUEST                  [A-Z]+\s.*\sHTTP/\d\.\d
STATUS                   \d{3}
BODY_BYTES_SENT          \d+
HTTP_REFERER             [^\|]*
HTTP_USER_AGENT          [^\|]*
HTTP_X_FORWARDED_FOR     [\d.]+|-
UPSTREAM_ADDR            [^\|]*
UPSTREAM_STATUS          [\d]{3}|-
REQUEST_TIME             [0-9]*\.[0-9]+
UPSTREAM_RESPONSE_TIME   ([0-9]*\.[0-9]+|-)$

NGINX_LOGS %{REMOTE_ADDR:remote_addr}\|%{TIME_LOCAL:time_local}\|%{REQUEST:request}\|%{STATUS:status}\|%{BODY_BYTES_SENT:body_bytes_sent}\|%{HTTP_REFERER:http_referer}\|%{HTTP_USER_AGENT:http_user_agent}\|%{HTTP_X_FORWARDED_FOR:http_x_forwarded_for}\|%{UPSTREAM_ADDR:upstream_addr}\|%{UPSTREAM_STATUS:upstream_status}\|%{REQUEST_TIME:request_time}\|%{UPSTREAM_RESPONSE_TIME:upstream_response_time}

USERHOST [\d\.]+
USERNAME [a-zA-Z]+
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
关于nginx日志格式

log_format  main    ‘$remote_addr|$time_local|$request|’
‘$status|$body_bytes_sent|$http_referer|’
‘$http_user_agent|$http_x_forwarded_for|’
‘$upstream_addr|$upstream_status|$request_time|$upstream_response_time’;
1
2
3
4

————————————————
版权声明:本文为CSDN博主「亦非我所愿丶」的原创文章,遵循 CC 4.0 BY-SA 版权协议,转载请附上原文出处链接及本声明。
原文链接:https://blog.csdn.net/wanglei_storage/article/details/82663706

发表评论

邮箱地址不会被公开。 必填项已用*标注