1. nginx access log 测试文件
# cat /usr/share/logstash/nginx_access_geoip_test.log
33.55.69.101 - - [24/May/2018:01:50:26 +0800] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36"
222.174.69.101 - - [24/May/2018:01:52:11 +0800] "GET /abc HTTP/1.1" 404 3652 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36"
2. logstash 配置
input {
file {
# 指定文件路径
path => "/usr/share/logstash/nginx_access_geoip_test.log"
type => "nginx_access"
start_position => "beginning"
}
}
filter {
grok {
# 匹配日志内容，分割字段
match => {
"message" => '%{IPORHOST:remote_ip} - %{DATA:user_name} \[%{HTTPDATE:time}\] "%{WORD:method} %{DATA:uri} HTTP/%{NUMBER:http_version}" %{NUMBER:response_code} %{NUMBER:bytes} "%{DATA:referrer}" "%{DATA:agent}"'
}
}
mutate {
# bytes 字段由 string 类型转换为 number 类型，以便可以做 sum 之类的统计分析
convert => { "bytes" => "integer" }
}
date {
# 用 time 字段展示日志原来的北京时间
match => [ "time", "dd/MMM/YYYY:HH:mm:ss Z" ]
locale => en
}
geoip {
# 对 来源 IP 做 geoip 查询归类
source => "remote_ip"
target => "geoip"
}
useragent {
source => "agent"
target => "user_agent"
}
}
output {
stdout { codec => "rubydebug" }
}
3. 测试结果
# /usr/share/logstash/bin/logstash -f /usr/share/logstash/geoip.conf
WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults
Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs errors to the console
[INFO ] 2018-06-01 23:17:06.036 [main] scaffold - Initializing module {:module_name=>"fb_apache", :directory=>"/usr/share/logstash/modules/fb_apache/configuration"}
[INFO ] 2018-06-01 23:17:06.044 [main] scaffold - Initializing module {:module_name=>"netflow", :directory=>"/usr/share/logstash/modules/netflow/configuration"}
[WARN ] 2018-06-01 23:17:06.529 [LogStash::Runner] multilocal - Ignoring the 'pipelines.yml' file because modules or command line options are specified
[INFO ] 2018-06-01 23:17:06.814 [LogStash::Runner] runner - Starting Logstash {"logstash.version"=>"6.2.3"}
[INFO ] 2018-06-01 23:17:07.111 [Api Webserver] agent - Successfully started Logstash API endpoint {:port=>9601}
[INFO ] 2018-06-01 23:17:09.025 [Ruby-0-Thread-1: /usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/stud-0.0.23/lib/stud/task.rb:22] pipeline - Starting pipeline {:pipeline_id=>"main", "pipeline.workers"=>2, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>50}
[INFO ] 2018-06-01 23:17:09.134 [[main]-pipeline-manager] geoip - Using geoip database {:path=>"/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/logstash-filter-geoip-5.0.3-java/vendor/GeoLite2-City.mmdb"}
[INFO ] 2018-06-01 23:17:10.125 [Ruby-0-Thread-1: /usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/stud-0.0.23/lib/stud/task.rb:22] pipeline - Pipeline started succesfully {:pipeline_id=>"main", :thread=>"#<Thread:0x2610db05@/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:246 sleep>"}
[INFO ] 2018-06-01 23:17:10.180 [Ruby-0-Thread-1: /usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/stud-0.0.23/lib/stud/task.rb:22] agent - Pipelines running {:count=>1, :pipelines=>["main"]}
{
"uri" => "/",
"bytes" => 0,
"@version" => "1",
"method" => "GET",
"host" => "devops-01",
"geoip" => {
"ip" => "33.55.69.101",
"country_code3" => "US",
"latitude" => 37.751,
"country_code2" => "US",
"longitude" => -97.822,
"continent_code" => "NA",
"location" => {
"lon" => -97.822,
"lat" => 37.751
},
"country_name" => "United States"
},
"http_version" => "1.1",
"time" => "24/May/2018:01:50:26 +0800",
"message" => "33.55.69.101 - - [24/May/2018:01:50:26 +0800] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36\"",
"@timestamp" => 2018-05-23T17:50:26.000Z,
"remote_ip" => "33.55.69.101",
"response_code" => "304",
"user_name" => "-",
"agent" => "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36",
"type" => "nginx_access",
"path" => "/usr/share/logstash/nginx_access_geoip_test.log",
"referrer" => "-",
"user_agent" => {
"device" => "Other",
"os_name" => "Windows 10",
"major" => "64",
"patch" => "3282",
"build" => "",
"name" => "Chrome",
"minor" => "0",
"os" => "Windows 10"
}
}
{
"uri" => "/abc",
"bytes" => 3652,
"@version" => "1",
"method" => "GET",
"host" => "devops-01",
"geoip" => {
"ip" => "222.174.69.101",
"city_name" => "Jinan",
"location" => {
"lon" => 116.9972,
"lat" => 36.6683
},
"country_name" => "China",
"country_code3" => "CN",
"region_name" => "Shandong",
"latitude" => 36.6683,
"country_code2" => "CN",
"longitude" => 116.9972,
"timezone" => "Asia/Shanghai",
"continent_code" => "AS",
"region_code" => "37"
},
"http_version" => "1.1",
"time" => "24/May/2018:01:52:11 +0800",
"message" => "222.174.69.101 - - [24/May/2018:01:52:11 +0800] \"GET /abc HTTP/1.1\" 404 3652 \"-\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36\"",
"@timestamp" => 2018-05-23T17:52:11.000Z,
"remote_ip" => "222.174.69.101",
"response_code" => "404",
"user_name" => "-",
"agent" => "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36",
"type" => "nginx_access",
"path" => "/usr/share/logstash/nginx_access_geoip_test.log",
"referrer" => "-",
"user_agent" => {
"device" => "Other",
"os_name" => "Windows 10",
"major" => "64",
"patch" => "3282",
"build" => "",
"name" => "Chrome",
"minor" => "0",
"os" => "Windows 10"
}
}
网友评论