## Logstash grok 示例
## 正则名称对应的内容参考：https://github.com/logstash-plugins/logstash-patterns-core/blob/master/patterns/grok-patterns
# Example 1
#   2018-03-02 00:02:18 pythonfile.py[line:1] INFO aaa bbb ccc
#   %{TIMESTAMP_ISO8601:log_time} %{USERNAME:code_file}\[line:%{NUMBER:code_line}\] %{WORD:log_level}%{SPACE}%{GREEDYDATA:content} 
# Example 2 - stackoverflow
#   14:46:16.603 [http-nio-8080-exec-4] INFO  METERING - msg=93e6dd5e-c009-46b3-b9eb-f753ee3b889a CREATE_JOB job=a820018e-7ad7-481a-97b0-bd705c3280ad data=71b1652e-16c8-4b33-9a57-f5fcb3d5de92
#   %{TIME:timestamp} \[%{USERNAME:http}\] %{WORD:loglevel}%{SPACE}%{WORD:logtype} - msg=%{UUID:msg} %{WORD:action} job=%{UUID:job} data=%{UUID:data}
# Example 3
#   E123 18:28:11.070340   25673 gofile.go:184] aaa bbb ccc
#   %{USERNAME:code}%{SPACE}%{TIME:log_time}%{SPACE}%{NUMBER:pid}%{SPACE}%{USERNAME:code_file}:%{NUMBER:code_line}\]%{SPACE}%{GREEDYDATA:content}
 
## 
input {
	# 对应filebeat的ouput
    beats {
        port => 10086
        ssl => false
    }
}
filter {
    grok {
		# 对一样日志进行匹配，获取里面的信息
		# %{TIME:log_time} 表示：在这个位置，使用TIME对应的正则表达式进行匹配，把结果填到log_time这个字段
        match => {"message" => "%{USERNAME:log_type}%{SPACE}%{TIME:log_time}%{SPACE}%{NUMBER:pid}%{SPACE}%{USERNAME:code_file}:%{NUMBER:code_line}\]%{SPACE}%{GREEDYDATA:content}"}
    }
    mutate {
        rename => {
            "source" => "log_file"
            "host" => "log_host"
        }
        add_field => {
            # 由于日志中只有时间没有日期，先添加一个包含日期的字段
            "datetime" => "%{+yyyy-MM-dd} %{log_time}"
            "log_level" => "%{[fields][level]}"
        }
    }
    date {
		# 日志中的时间转换为ES中的时间
        match => [
            "datetime",
            "YYYY-MM-dd HH:mm:ss.SSSSSS"
        ]
        target => "log_time"
    }
    mutate {
        remove_field => [
            "message",
            "tags",
            "offset",
            "prospector",
            "@version",
            "beat",
            "tags",
            "fields",
            "datetime"
        ]
    }
}
output {
	# 实际上只需要输出到es即可，此处要设置index分表的名称规则
    stdout { codec => rubydebug }
    elasticsearch {
        index => "template_name-%{+YYYY.MM.dd}"
        document_type => "template_name"
        hosts => ["1.2.3.4:123","1.2.3.5:123"]
    }
}