使用 TCP 从一个 logstash 发送到另一个

Question

我正在尝试将事件从一个 logstash 服务器转发到另一个。最后一个 logstash 服务器应该将接收到的日志写入文件

在我添加的源logstash中：

output {
    tcp {
        host => "host"
        port => port
    }
}

目标logstash有：

input {
  tcp {
    port => port
  }
}

output {
  file {
    path => "/var/log/project/log"
  }
}

这是启动日志：

{:timestamp=>"2016-01-15T23:12:30.884000+0100", :message=>"Reading config file", :file=>"logstash/agent.rb", :level=>:debug, :line=>"309", :method=>"local_config"}
{:timestamp=>"2016-01-15T23:12:31.011000+0100", :message=>"Compiled pipeline code:\n        @inputs = []\n        @filters = []\n        @outputs = []\n        @periodic_flushers = []\n        @shutdown_flushers = []\n\n          @input_tcp_1 = plugin(\"input\", \"tcp\", LogStash::Util.hash_merge_many({ \"port\" => port }))\n\n          @inputs << @input_tcp_1\n\n          @output_file_2 = plugin(\"output\", \"file\", LogStash::Util.hash_merge_many({ \"path\" => (\"/var/log/project/log\") }))\n\n          @outputs << @output_file_2\n\n  def filter_func(event)\n    events = [event]\n    @logger.debug? && @logger.debug(\"filter received\", :event => event.to_hash)\n    events\n  end\n  def output_func(event)\n    @logger.debug? && @logger.debug(\"output received\", :event => event.to_hash)\n    @output_file_2.handle(event)\n    \n  end", :level=>:debug, :file=>"logstash/pipeline.rb", :line=>"29", :method=>"initialize"}
{:timestamp=>"2016-01-15T23:12:31.027000+0100", :message=>"Plugin not defined in namespace, checking for plugin file", :type=>"input", :name=>"tcp", :path=>"logstash/inputs/tcp", :level=>:debug, :file=>"logstash/plugin.rb", :line=>"133", :method=>"lookup"}
{:timestamp=>"2016-01-15T23:12:31.070000+0100", :message=>"Plugin not defined in namespace, checking for plugin file", :type=>"codec", :name=>"line", :path=>"logstash/codecs/line", :level=>:debug, :file=>"logstash/plugin.rb", :line=>"133", :method=>"lookup"}
{:timestamp=>"2016-01-15T23:12:31.101000+0100", :message=>"config LogStash::Codecs::Line/@charset = \"UTF-8\"", :level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"111", :method=>"config_init"}
{:timestamp=>"2016-01-15T23:12:31.120000+0100", :message=>"config LogStash::Inputs::Tcp/@port = 35364", :level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"111", :method=>"config_init"}
{:timestamp=>"2016-01-15T23:12:31.127000+0100", :message=>"config LogStash::Inputs::Tcp/@debug = false", :level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"111", :method=>"config_init"}
{:timestamp=>"2016-01-15T23:12:31.135000+0100", :message=>"config LogStash::Inputs::Tcp/@codec = <LogStash::Codecs::Line charset=>\"UTF-8\">", :level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"111", :method=>"config_init"}
{:timestamp=>"2016-01-15T23:12:31.138000+0100", :message=>"config LogStash::Inputs::Tcp/@add_field = {}", :level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"111", :method=>"config_init"}
{:timestamp=>"2016-01-15T23:12:31.146000+0100", :message=>"config LogStash::Inputs::Tcp/@host = \"0.0.0.0\"", :level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"111", :method=>"config_init"}
{:timestamp=>"2016-01-15T23:12:31.154000+0100", :message=>"config LogStash::Inputs::Tcp/@data_timeout = -1", :level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"111", :method=>"config_init"}
{:timestamp=>"2016-01-15T23:12:31.162000+0100", :message=>"config LogStash::Inputs::Tcp/@mode = \"server\"", :level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"111", :method=>"config_init"}
{:timestamp=>"2016-01-15T23:12:31.169000+0100", :message=>"config LogStash::Inputs::Tcp/@ssl_enable = false", :level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"111", :method=>"config_init"}
{:timestamp=>"2016-01-15T23:12:31.172000+0100", :message=>"config LogStash::Inputs::Tcp/@ssl_verify = false", :level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"111", :method=>"config_init"}
{:timestamp=>"2016-01-15T23:12:31.175000+0100", :message=>"config LogStash::Inputs::Tcp/@ssl_key_passphrase = <password>", :level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"111", :method=>"config_init"}
{:timestamp=>"2016-01-15T23:12:31.179000+0100", :message=>"Plugin not defined in namespace, checking for plugin file", :type=>"output", :name=>"file", :path=>"logstash/outputs/file", :level=>:debug, :file=>"logstash/plugin.rb", :line=>"133", :method=>"lookup"}
{:timestamp=>"2016-01-15T23:12:31.219000+0100", :message=>"Plugin not defined in namespace, checking for plugin file", :type=>"codec", :name=>"plain", :path=>"logstash/codecs/plain", :level=>:debug, :file=>"logstash/plugin.rb", :line=>"133", :method=>"lookup"}
{:timestamp=>"2016-01-15T23:12:31.246000+0100", :message=>"config LogStash::Codecs::Plain/@charset = \"UTF-8\"", :level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"111", :method=>"config_init"}
{:timestamp=>"2016-01-15T23:12:31.263000+0100", :message=>"config LogStash::Outputs::File/@path = \"/var/log/project/log\"", :level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"111", :method=>"config_init"}
{:timestamp=>"2016-01-15T23:12:31.269000+0100", :message=>"config LogStash::Outputs::File/@type = \"\"", :level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"111", :method=>"config_init"}
{:timestamp=>"2016-01-15T23:12:31.281000+0100", :message=>"config LogStash::Outputs::File/@tags = []", :level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"111", :method=>"config_init"}
{:timestamp=>"2016-01-15T23:12:31.284000+0100", :message=>"config LogStash::Outputs::File/@exclude_tags = []", :level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"111", :method=>"config_init"}
{:timestamp=>"2016-01-15T23:12:31.297000+0100", :message=>"config LogStash::Outputs::File/@codec = <LogStash::Codecs::Plain charset=>\"UTF-8\">", :level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"111", :method=>"config_init"}
{:timestamp=>"2016-01-15T23:12:31.309000+0100", :message=>"config LogStash::Outputs::File/@workers = 1", :level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"111", :method=>"config_init"}
{:timestamp=>"2016-01-15T23:12:31.312000+0100", :message=>"config LogStash::Outputs::File/@flush_interval = 2", :level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"111", :method=>"config_init"}
{:timestamp=>"2016-01-15T23:12:31.331000+0100", :message=>"config LogStash::Outputs::File/@gzip = false", :level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"111", :method=>"config_init"}
{:timestamp=>"2016-01-15T23:12:31.334000+0100", :message=>"config LogStash::Outputs::File/@filename_failure = \"_filepath_failures\"", :level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"111", :method=>"config_init"}
{:timestamp=>"2016-01-15T23:12:31.364000+0100", :message=>"Starting tcp input listener", :address=>"0.0.0.0:35364", :level=>:info, :file=>"logstash/inputs/tcp.rb", :line=>"92", :method=>"register"}
{:timestamp=>"2016-01-15T23:12:31.425000+0100", :message=>"Pipeline started", :level=>:info, :file=>"logstash/pipeline.rb", :line=>"87", :method=>"run"}
{:timestamp=>"2016-01-15T23:13:53.014000+0100", :message=>"Accepted connection", :client=>"xx.xx.xx.xx:51272", :server=>"0.0.0.0:port", :level=>:debug, :file=>"logstash/inputs/tcp.rb", :line=>"146", :method=>"client_thread"}
{:timestamp=>"2016-01-15T23:13:53.025000+0100", :message=>"config LogStash::Codecs::Line/@charset = \"UTF-8\"", :level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"111", :method=>"config_init"}

我可以使用netstat 看到客户端和服务器之间的连接，还可以看到包含tcpdump 事件的数据包。 /var/log/project 归 logstash 用户所有。

但是，没有文件被创建，没有文件被写入，没有关于那个的日志

Answer 1

我不确定问题是出在 TCP 还是其他方面，但从 TCP 切换到伐木工人的输入和输出效果很好

Answer 2

我知道这个问题很老了，但是我正在回答或猜测的任何方式

问题可能是日志存储可能没有打开文件 /var/log/project/log 的权限。

检查日志存储日志文件以获取日志存储应用程序。

我之前遇到过这个问题，就是这个问题。