Logstash geoip 随机故障

Question

我正在使用 logstash 来处理来自 dnsmasq 的日志。 在 DNS 响应中，我使用“geoip”过滤器来使用位置信息丰富请求。不幸的是，一些请求包含了地理位置信息，而另一些则没有。

地理位置代码：

      geoip {
        source => "serverip"
      }

不带地理位置的示例 IP 列表

104.156.81.217
104.156.85.217
104.16.92.65
104.16.93.65
104.16.94.65
104.16.95.65
104.16.96.65
104.20.5.131
104.20.6.131
104.20.77.18
104.20.78.18
104.244.43.135
104.244.43.167
104.244.43.231
104.244.43.39
104.244.43.7
104.28.30.27
104.28.31.27
104.40.196.5
104.41.231.130
104.45.95.112
104.47.151.128
104.71.97.80
104.84.200.206
104.90.129.122
104.90.176.199
104.90.176.77
104.94.60.210
104.98.119.204
104.98.150.212
162.255.119.124
185.118.208.20
185.19.196.101
185.54.150.54
185.63.147.12
191.232.139.13
191.233.80.151
191.239.8.125
192.229.233.25
23.101.51.170
23.196.235.245
23.196.247.114
23.196.249.86
23.196.255.139
23.197.0.60
23.199.209.223
23.235.33.217
23.235.37.217
23.97.173.24

成功进入：

{
         "message" => "May 27 18:17:16 dnsmasq[385]: reply www.google.com is 216.58.213.228",
        "@version" => "1",
      "@timestamp" => "2016-05-27T18:17:17.147Z",
            "path" => "/var/log/dnsmasq.log",
            "host" => "dns",
            "type" => "dnsmasq",
    "reqtimestamp" => "May 27 18:17:16",
         "program" => "dnsmasq",
             "pid" => "385",
          "action" => "reply",
          "domain" => "www.google.com",
        "function" => "is",
        "serverip" => "216.58.213.228",
           "geoip" => {
                      "ip" => "216.58.213.228",
           "country_code2" => "US",
           "country_code3" => "USA",
            "country_name" => "United States",
          "continent_code" => "NA",
             "region_name" => "CA",
               "city_name" => "Mountain View",
             "postal_code" => "94043",
                "latitude" => 37.41919999999999,
               "longitude" => -122.0574,
                "dma_code" => 807,
               "area_code" => 650,
                "timezone" => "America/Los_Angeles",
        "real_region_name" => "California",
                "location" => [
            [0] -122.0574,
            [1] 37.41919999999999
        ]
    }
}

输入失败：

{
         "message" => "May 27 18:15:50 dnsmasq[385]: reply e5884.d.akamaiedge.net is 23.197.8.251",
        "@version" => "1",
      "@timestamp" => "2016-05-27T18:15:51.697Z",
            "path" => "/var/log/dnsmasq.log",
            "host" => "dns",
            "type" => "dnsmasq",
    "reqtimestamp" => "May 27 18:15:50",
         "program" => "dnsmasq",
             "pid" => "385",
          "action" => "reply",
          "domain" => "e5884.d.akamaiedge.net",
        "function" => "is",
        "serverip" => "23.197.8.251"
}

完成 Logstash 配置：

input {
  file {
    path => "/var/log/dnsmasq.log"
    start_position => "beginning"
    type => "dnsmasq"
  }
}

# Mar 15 20:13:05 dnsmasq[346]: query[A] imap.gmail.com from 192.168.0.140
# Mar  2 20:38:45 dnsmasq-dhcp[11856]: DHCPACK(eth0) 192.168.0.152 60:67:20:72:df:00 E0199149
# Mar 15 21:55:34 dnsmasq-dhcp[346]: 3806132383 DHCPACK(eth0) 192.168.0.80 04:0c:ce:d1:af:18 Air-de-irobot
# Mar 16 08:54:31 dnsmasq-dhcp[346]: 4280587370 DHCPACK(eth0) 192.168.0.158 48:9d:24:ae:0e:00 BB-JP
# Mar 16 08:18:49 dnsmasq[346]: /etc/pihole/gravity.list ssl.google-analytics.com is 192.168.0.2

filter {
  if [type] == "dnsmasq" {
    grok {
      match =>  [ "message", "%{SYSLOGTIMESTAMP:reqtimestamp} %{USER:program}\[%{NONNEGINT:pid}\]\: ?(%{NONNEGINT:num} )?%{NOTSPACE:action} %{IP:clientip} %{MAC:clientmac} ?(%{HOSTNAME:clientname})?"]
      match =>  [ "message", "%{SYSLOGTIMESTAMP:reqtimestamp} %{USER:program}\[%{NONNEGINT:pid}\]\: ?(%{NONNEGINT:num} )?%{USER:action}?(\[%{USER:subaction}\])? %{NOTSPACE:domain} %{NOTSPACE:function} %{IP:clientip}"]

      match =>  [ "message", "%{SYSLOGTIMESTAMP:reqtimestamp} %{USER:program}\[%{NONNEGINT:pid}\]\: %{NOTSPACE:action} %{DATA:data}"]
    }

    if [action] =~ "DHCPACK" {
      if ![clientname] {
        mutate {
          add_field => { "clientname" => "No name" }
        }
      }
      aggregate {
        task_id => "%{clientip}"
        code => "map['clientmac'] = event['clientmac']; map['clientname'] = event['clientname'];"
        map_action => "create_or_update"
        # timeout = 0 sets the timeout to the default value 1800 seconds.
        timeout => 172800
      }
    } else if [action] == "query" {
      aggregate {
        task_id => "%{clientip}"
        code => "event['clientmac'] = map['clientmac']; event['clientname'] = map['clientname']"
        map_action => "update"
      }
      if ![clientname] {
        mutate {
          add_field => { "clientname" => "%{clientip}" }
        }
      }
      if ![clientmac] {
        mutate {
          add_field => { "clientmac" => "%{clientip}" }
        }
      }
    } else if [action] == "reply" {
        mutate {
          rename => { "clientip" => "serverip" }
        }
      geoip {
        source => "serverip"
      }
    } else
    {
      drop{}
    }
  }
}
output {
# elasticsearch { hosts => ["localhost:9200"] }
 stdout { codec => rubydebug }
}

Answer 1

从调试日志中可以看出，最新的 Logstash 版本 (2.3.2) 捆绑了 logstash geoip filter 2.0.7，而后者又包含了 2013 年的旧 Maxmind 的 GeoIP 数据库 (GeoLiteCity-2013-01-18.dat)

Using geoip database {:path=>"/usr/local/Cellar/logstash/2.3.2/vendor/bundle/jruby/1.9/gems/logstash-filter-geoip-2.0.7/vendor/GeoLiteCity-2013-01-18.dat", :level=>:info, :file=>"logstash/filters/geoip.rb", :line=>"97", :method=>"register"}

Maxmind 有一个新的 GeoIP2 服务，您可以尝试here，它会正确地对上面列表中的所有 IP 进行地理编码。

就 Logstash geoip 而言，从 GeoIP 切换到 GeoIP2 has been made in March 并将在 next Logstash 5.0 version 中提供。