【发布时间】：2020-08-25 12:04:27
【问题描述】：
我有这样的 logstash 配置：
input {
beats {
port => '5044'
}
}
filter {
grok {
patterns_dir => ['./patterns']
match => { 'message' => '%{TIME:timestamp}(","Event\s)(?<Event>([^"]+))(","Form\s)?(?<Form>([^"]+))?(","ParentType\s)?(?<parent_type>([^"]+))?(","ParentName\s)?(?<parent_name>([^"]+))?(","Type\s)?(?<type>([^"]+))?(","Name\s)?(?<Name_of_form>([^"]+))?(","Detail\s)?(?<Detail>([^"]+))?(","t=)?(?<t>([\d]+))?(",")?(?<Status>(end|beg))?' }
add_tag => [ '%{Status}' ]
}
dissect {
mapping => {
'[log][file][path]' => 'C:\Logs\%{login}\%{server}\%{user}\%{filename}.txt'
}
}
date {
match => [ 'timestamp', 'dd.MM.yyyy HH:mm:ss' ]
}
mutate {
add_field => { 'uniqueEvent' => '%{Event}_%{filename}' }
}
elapsed {
unique_id_field => 'uniqueEvent'
start_tag => 'beg'
end_tag => 'end'
new_event_on_match => false
}
if 'elapsed' in [tags] {
aggregate {
task_id => '%{uniqueEvent}'
code => 'map["duration"] = [(event.get("elapsed_time")*1000).to_i]'
map_action => 'create'
}
}
mutate {
remove_field => ['timestamp', 'ecs', 'log', 'tags', 'message', '@version', 'filename', 'input', 'host', 'agent', 't', 'parent_type', 'parent_name', 'type']
rename => {'elapsed_time' => 'Event_duration'}
}
}
output {
elasticsearch {
hosts => ['localhost:9200']
index => 'logs-%{+dd.MM.YYYY}'
}
}
在elasticsearch中，我需要的字段总是以不同的顺序显示，这有点令人气愤。我只有几个文本字段、两个日期字段和一个数字字段。 Elasticsearch 本身完美地定义了类型，但我希望所有数据在每个索引中都按严格的顺序排列。这个可以吗？
映射应该是这样一个严格的顺序：
"mappings": {
"properties": {
"@timestamp": {
"type": "date"
},
"Event": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"Form": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"Name_of_form": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"Detail": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"Event_duration": {
"type": "float"
},
"elapsed_timestamp_start": {
"type": "date"
},
"user": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"login": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"server": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"Status": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"uniqueEvent": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
我尝试根据this 主题的第一个也是唯一一个指南来实现这一点，但没有任何效果。根据其他人的文章，我在logstash目录下创建了template.json，但是启动时这个模板不起作用，错误类型是：这个模板不在elasticsearch中。
如何正确创建此文件？如何将其正确写入输入块？这一切都会奏效吗？
感谢您的帮助！
报错还是这样：
[2020-08-25T19:33:13,365][INFO ][logstash.outputs.elasticsearch][main] Using mapping template from {:path=>"C:\\Users\\moral\\elk_stack\\logstash-7.8.1\\template\\template.json"}
[2020-08-25T19:33:13,406][INFO ][logstash.outputs.elasticsearch][main] Attempting to install template {:manage_template=>{"template"=>"logs-*", "version"=>50001, "settings"=>{"index.refresh_interval"=>"5s"}, "mappings"=>{"_default_"=>{"properties"=>{"@timestamp"=>{"type"=>"date"}, "Event"=>{"type"=>"text", "fields"=>{"keyword"=>{"type"=>"keyword", "ignore_above"=>256}}}, "Form"=>{"type"=>"text", "fields"=>{"keyword"=>{"type"=>"keyword", "ignore_above"=>256}}}, "Name_of_form"=>{"type"=>"text", "fields"=>{"keyword"=>{"type"=>"keyword", "ignore_above"=>256}}}, "Detail"=>{"type"=>"text", "fields"=>{"keyword"=>{"type"=>"keyword", "ignore_above"=>256}}}, "Event_duration"=>{"type"=>"float"}, "elapsed_timestamp_start"=>{"type"=>"date"}, "user"=>{"type"=>"text", "fields"=>{"keyword"=>{"type"=>"keyword", "ignore_above"=>256}}}, "login"=>{"type"=>"text", "fields"=>{"keyword"=>{"type"=>"keyword", "ignore_above"=>256}}}, "server"=>{"type"=>"text", "fields"=>{"keyword"=>{"type"=>"keyword", "ignore_above"=>256}}}, "Status"=>{"type"=>"text", "fields"=>{"keyword"=>{"type"=>"keyword", "ignore_above"=>256}}}, "uniqueEvent"=>{"type"=>"text", "fields"=>{"keyword"=>{"type"=>"keyword", "ignore_above"=>256}}}}}}}}
[2020-08-25T19:33:13,421][INFO ][logstash.outputs.elasticsearch][main] Installing elasticsearch template to _template/log-template
[2020-08-25T19:33:13,468][ERROR][logstash.outputs.elasticsearch][main] Failed to install template. {:message=>"Got response code '400' contacting Elasticsearch at URL 'http://localhost:9200/_template/log-template'", :class=>"LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError", :backtrace=>["C:/Users/moral/elk_stack/logstash-7.8.1/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.5.1-java/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb:80:in `perform_request'", "C:/Users/moral/elk_stack/logstash-7.8.1/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.5.1-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:332:in `perform_request_to_url'", "C:/Users/moral/elk_stack/logstash-7.8.1/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.5.1-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:319:in `block in perform_request'", "C:/Users/moral/elk_stack/logstash-7.8.1/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.5.1-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:414:in `with_connection'", "C:/Users/moral/elk_stack/logstash-7.8.1/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.5.1-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:318:in `perform_request'", "C:/Users/moral/elk_stack/logstash-7.8.1/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.5.1-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:326:in `block in Pool'", "C:/Users/moral/elk_stack/logstash-7.8.1/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.5.1-java/lib/logstash/outputs/elasticsearch/http_client.rb:352:in `template_put'", "C:/Users/moral/elk_stack/logstash-7.8.1/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.5.1-java/lib/logstash/outputs/elasticsearch/http_client.rb:86:in `template_install'", "C:/Users/moral/elk_stack/logstash-7.8.1/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.5.1-java/lib/logstash/outputs/elasticsearch/template_manager.rb:28:in `install'", "C:/Users/moral/elk_stack/logstash-7.8.1/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.5.1-java/lib/logstash/outputs/elasticsearch/template_manager.rb:16:in `install_template'", "C:/Users/moral/elk_stack/logstash-7.8.1/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.5.1-java/lib/logstash/outputs/elasticsearch/common.rb:205:in `install_template'", "C:/Users/moral/elk_stack/logstash-7.8.1/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.5.1-java/lib/logstash/outputs/elasticsearch/common.rb:49:in `block in setup_after_successful_connection'"]}
template.json 文件：
{
"template" : "logs-*",
"version" : 50001,
"settings" : {
"index.refresh_interval" : "5s"
},
"mappings" : {
"_default_" : {
"properties": {
"@timestamp": {
"type": "date"
},
"Event": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"Form": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"Name_of_form": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"Detail": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"Event_duration": {
"type": "float"
},
"elapsed_timestamp_start": {
"type": "date"
},
"user": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"login": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"server": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"Status": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"uniqueEvent": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
}
【问题讨论】：
标签： elasticsearch logstash elastic-stack logstash-grok logstash-configuration