我正在尝试在logstash v7.17中使用数据流和索引模板
实现此目的的正确的 elasticsearch 输出配置是什么？
选项 1 在模板中使用 data_stream -> 失败
output {
elasticsearch {
hosts => ["https://elasticsearch-master:9200"]
index => "microservice-%{+YYYY.MM.dd}"
template => "/usr/share/logstash/templates/microservices.json"
# template_overwrite => false
template_name => "microservices"
}
}
/usr/share/logstash/templates/microservices.json {
"index_patterns": "microservice-*",
"template": {
"settings" : {
"index" : {
"number_of_shards" : "1",
"number_of_replicas" : "1"
}
},
"mappings" : {
"properties" : {
"@timestamp" : {
"type" : "date"
},
"@version" : {
"type" : "keyword"
},
"host" : {
"type" : "keyword"
},
"level" : {
"type" : "keyword"
},
"service" : {
"type" : "keyword"
},
"type" : {
"type" : "keyword"
}
}
}
},
"data_stream" : {
"hidden" : false
}
}
Logstash 日志（调试模式）：
18:37:34.459 [[.monitoring-logstash]-pipeline-manager] INFO logstash.outputs.elasticsearchmonitoring - Config is not compliant with data streams. `data_stream => auto` resolved to `false`
18:37:34.470 [Ruby-0-Thread-14: :1] INFO logstash.outputs.elasticsearchmonitoring - Config is not compliant with data streams. `data_stream => auto` resolved to `false`
18:37:34.564 [[.monitoring-logstash]-pipeline-manager] WARN logstash.javapipeline - 'pipeline.ordered' is enabled and is likely less efficient, consider disabling if preserving event order is not necessary
18:37:34.666 [[main]-pipeline-manager] WARN logstash.outputs.elasticsearch - Restored connection to ES instance {:url=>"https://elastic:xxxxxx@elasticsearch-master:9200/"}
18:37:34.678 [[main]-pipeline-manager] INFO logstash.outputs.elasticsearch - Elasticsearch version determined (7.16.3) {:es_version=>7}
18:37:34.737 [[main]-pipeline-manager] WARN logstash.outputs.elasticsearch - Detected a 6.x and above cluster: the `type` event field won't be used to determine the document _type {:es_version=>7}
18:37:34.850 [[main]-pipeline-manager] DEBUG logstash.outputs.elasticsearch - Not eligible for data streams because ecs_compatibility is not enabled. Elasticsearch data streams require that events adhere to the Elastic Common Schema. While `ecs_compatibility` can be set for this individual Elasticsearch output plugin, doing so will not fix schema conflicts caused by upstream plugins in your pipeline. To avoid mapping conflicts, you will need to use ECS-compatible field names and datatypes throughout your pipeline. Many plugins support an `ecs_compatibility` mode, and the `pipeline.ecs_compatibility` setting can be used to opt-in for all plugins in a pipeline.
18:37:34.850 [[main]-pipeline-manager] INFO logstash.outputs.elasticsearch - Config is not compliant with data streams. `data_stream => auto` resolved to `false`
18:37:35.059 [Ruby-0-Thread-17: :1] INFO logstash.outputs.elasticsearch - Using mapping template from {:path=>"/usr/share/logstash/templates/microservices.json"}
18:37:35.067 [Ruby-0-Thread-17: :1] DEBUG logstash.outputs.elasticsearch - Attempting to install template {:template=>{"index_patterns"=>"microservice-*", "template"=>{"settings"=>{"index"=>{"number_of_shards"=>"1", "number_of_replicas"=>"1", "refresh_interval"=>"5s"}}, "mappings"=>{"properties"=>{"@timestamp"=>{"type"=>"date"}, "@version"=>{"type"=>"keyword"}, "host"=>{"type"=>"text", "fields"=>{"keyword"=>{"type"=>"keyword", "ignore_above"=>256}}}, "level"=>{"type"=>"keyword"}, "service"=>{"type"=>"keyword"}, "type"=>{"type"=>"keyword"}}}}, "data_stream"=>{"hidden"=>false}}}
18:37:35.135 [[main]-pipeline-manager] INFO logstash.javapipeline - Starting pipeline {:pipeline_id=>"main", "pipeline.workers"=>1, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>50, "pipeline.max_inflight"=>125, "pipeline.sources"=>["/usr/share/logstash/pipeline/logstash.conf", "/usr/share/logstash/pipeline/uptime.conf"], :thread=>"#<Thread:0x38ca1ed0@/usr/share/logstash/logstash-core/lib/logstash/pipeline_action/create.rb:54 run>"}
18:37:35.136 [[.monitoring-logstash]-pipeline-manager] INFO logstash.javapipeline - Starting pipeline {:pipeline_id=>".monitoring-logstash", "pipeline.workers"=>1, "pipeline.batch.size"=>2, "pipeline.batch.delay"=>50, "pipeline.max_inflight"=>2, "pipeline.sources"=>["monitoring pipeline"], :thread=>"#<Thread:0x3d81d305 run>"}
18:37:35.346 [Ruby-0-Thread-17: :1] INFO logstash.outputs.elasticsearch - Installing Elasticsearch template {:name=>"microservices"}
18:37:35.677 [Ruby-0-Thread-17: :1] ERROR logstash.outputs.elasticsearch - Failed to install template {:message=>"Got response code '400' contacting Elasticsearch at URL 'https://elasticsearch-master:9200/_template/microservices'", :exception=>LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError, :backtrace=>["/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-11.4.1-java/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb:84:in `perform_request'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-11.4.1-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:324:in `perform_request_to_url'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-11.4.1-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:311:in `block in perform_request'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-11.4.1-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:398:in `with_connection'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-11.4.1-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:310:in `perform_request'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-11.4.1-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:318:in `block in Pool'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-11.4.1-java/lib/logstash/outputs/elasticsearch/http_client.rb:408:in `template_put'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-11.4.1-java/lib/logstash/outputs/elasticsearch/http_client.rb:85:in `template_install'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-11.4.1-java/lib/logstash/outputs/elasticsearch/template_manager.rb:29:in `install'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-11.4.1-java/lib/logstash/outputs/elasticsearch/template_manager.rb:17:in `install_template'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-11.4.1-java/lib/logstash/outputs/elasticsearch.rb:494:in `install_template'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-11.4.1-java/lib/logstash/outputs/elasticsearch.rb:318:in `finish_register'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-11.4.1-java/lib/logstash/outputs/elasticsearch.rb:283:in `block in register'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-11.4.1-java/lib/logstash/plugin_mixins/elasticsearch/common.rb:149:in `block in after_successful_connection'"]}
选项 2 在输出中使用 data_streams -> 失败
output {
elasticsearch {
hosts => ["https://elasticsearch-master:9200"]
data_stream => true
#data_stream_type => "logs"
#data_stream_dataset => "microservices"
#data_stream_namespace => ""
index => "microservice-%{+YYYY.MM.dd}"
template => "/usr/share/logstash/templates/microservices.json"
# template_overwrite => false
# ecs_compatibility => "v1"
template_name => "microservices"
}
}
日志是：
18:41:47.356 [[main]-pipeline-manager] ERROR logstash.outputs.elasticsearch - Invalid data stream configuration, following parameters are not supported: {"template"=>"/usr/share/logstash/templates/microservices.json", "template_name"=>"microservices", "index"=>"microservice-%{+YYYY.MM.dd}"}
18:41:47.357 [Ruby-0-Thread-16: :1] ERROR logstash.outputs.elasticsearch - Invalid data stream configuration, following parameters are not supported: {"template"=>"/usr/share/logstash/templates/microservices.json", "template_name"=>"microservices", "index"=>"microservice-%{+YYYY.MM.dd}"}
18:41:47.400 [[main]-pipeline-manager] ERROR logstash.javapipeline - Pipeline error {:pipeline_id=>"main", :exception=>#<LogStash::ConfigurationError: Invalid data stream configuration: ["template", "template_name", "index"]>, :backtrace=>["/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-11.4.1-java/lib/logstash/outputs/elasticsearch/data_stream_support.rb:68:in `check_data_stream_config!'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-11.4.1-java/lib/logstash/outputs/elasticsearch/data_stream_support.rb:33:in `data_stream_config?'", "/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-11.4.1-java/lib/logstash/outputs/elasticsearch.rb:296:in `register'", "org/logstash/config/ir/compiler/OutputStrategyExt.java:131:in `register'", "org/logstash/config/ir/compiler/AbstractOutputDelegatorExt.java:68:in `register'", "/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:232:in `block in register_plugins'", "org/jruby/RubyArray.java:1821:in `each'", "/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:231:in `register_plugins'", "/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:589:in `maybe_setup_out_plugins'", "/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:244:in `start_workers'", "/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:189:in `run'", "/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:141:in `block in start'"], "pipeline.sources"=>["/usr/share/logstash/pipeline/logstash.conf", "/usr/share/logstash/pipeline/uptime.conf"], :thread=>"#<Thread:0x98bdee3 run>"}
选项 3) 手动 -> 有效
首先，通过 API 调用手动创建索引模板：
PUT _index_template/microservices/usr/share/logstash/templates/microservices.json然后：
output {
elasticsearch {
hosts => ["https://elasticsearch-master:9200"]
index => "microservice-test"
action => "create"
}
}
但我不想执行此手动步骤。我想使用logstash输出选项来管理data_stream +索引名称+索引模板。
在
data_streamelasticsearchindextemplatetemplate_nametype、dataset特定命名方案和一个
namespace在您的情况下，类型似乎是
microservicelogsgeneric，默认命名空间是
default。所以如果你的
elasticsearch
输出看起来像这样......
output {
elasticsearch {
hosts => ["https://elasticsearch-master:9200"]
data_stream => true
data_stream_type => "microservice"
}
}
...您的数据将存储在名为 microservice-generic-default
的数据流中，并且该数据流将与您的索引模板匹配
microservice-*，因此您可以开始了。您只需要确保使用以下命令提前创建索引模板，因为Logstash仅支持
旧索引模板不支持数据流
PUT _index_template/microservices
{
"index_patterns": "microservice-*",
"template": {
"settings" : {
"index" : {
"number_of_shards" : "1",
"number_of_replicas" : "1"
}
},
"mappings" : {
"properties" : {
"@timestamp" : {
"type" : "date"
},
"@version" : {
"type" : "keyword"
},
"host" : {
"type" : "keyword"
},
"level" : {
"type" : "keyword"
},
"service" : {
"type" : "keyword"
},
"type" : {
"type" : "keyword"
}
}
}
},
"data_stream" : {
"hidden" : false
}
}
创建此索引模板后，您可以运行 Logstash 管道，它将起作用。