ELK + R as log storage 2. Installation and configuration

 input { rabbitmq { auto_delete => false durable => true exchange => "logstash-exchange" exclusive => false host => '1.1.1.1' key => "logstash-routing-key" password => "guest" prefetch_count => 4 queue => "logstash-filter" type => "logstash-filter-input" user => "guest" threads => 3 arguments => {"x-ha-policy" => "all"} } } # Filter all data filter { if [type] == "apache-access" { grok { match => [ "message", "%{COMBINEDAPACHELOG}" ] add_field => [ "severity", "normal" ] } } else if [type] == "eventlog" { mutate { update => { "host" => "None" } add_field => { "severity" => "None" } } } if [type] =~ "apache" { mutate { add_field => { "Message" => "%{message}" } } } if [type] == "eventlog" { mutate { gsub => [ 'message', '[\[\]\\]', ""] } mutate { gsub => [ 'Message', ':', "-"] } mutate { gsub => [ 'Message', '"', "'"] } json { source => "message" } mutate { update => { "host" => "%{Hostname}" } update => { "severity" => "%{Severity}" } } } if [_jsonparsefailure] { mutate { update => { "host" => "Unknown" } update => { "severity" => "Unknown" } } } } # Output to elasticsearch balancer output { elasticsearch { cluster => "CLUSTER_NAME" protocol => "http" host => ['2.2.2.2'] index => "logstash-%{+YYYY.MM.dd}" } } 

 input { # Logstash to Logstash lumberjack { codec => "json" port => 3333 ssl_certificate => "/etc/logstash/ls.crt" ssl_key => "/etc/logstash/ls.key" } # Nxlog to Logstash tcp { type => "eventlog" port => 3515 codec => "json" } } output { rabbitmq { durable => true exchange => "logstash-exchange" key => "logstash-routing-key" exchange_type => "direct" host => '1.1.1.1' password => "guest" persistent => true user => "guest" } #file { path => "/var/log/raw/%{host}/%{+YYYY-MM-dd}.log" } } 

 global log /dev/log local0 log /dev/log local1 notice chroot /var/lib/haproxy user haproxy group haproxy daemon defaults log global mode http option tcplog option dontlognull contimeout 5000 clitimeout 50000 srvtimeout 50000 listen to_listeners_event_log :3515 mode tcp balance roundrobin option tcpka server one 1.1.1.1:3515 check inter 5000 server two 2.2.2.2:3515 check inter 5000 … server last nnnn:3515 check inter 5000 listen to_listeners_event_log :3333 mode tcp balance roundrobin option tcpka server one 1.1.1.1:3333 check inter 5000 server two 2.2.2.2:3333 check inter 5000 … server last nnnn:3333 check inter 5000 listen stats :1936 stats show-legends stats refresh 5s stats hide-version stats uri / 

 #define ROOT C:\Program Files\nxlog define ROOT C:\Program Files (x86)\nxlog Moduledir %ROOT%\modules CacheDir %ROOT%\data Pidfile %ROOT%\data\nxlog.pid SpoolDir %ROOT%\data LogFile %ROOT%\data\nxlog.log <Extension json> Module xm_json </Extension> # Windows Event Log <Input eventlog> # Uncomment im_msvistalog for Windows Vista/2008 and later Module im_msvistalog Query <QueryList> \ <Query Id='1'> \ <Select Path="Application">*[System[(Level=1 or Level=2 or Level=3 or Level=4)]]</Select> \ <Select Path='Security'>*</Select> \ <Select Path="System">*[System[(Level=1 or Level=2 or Level=3 or Level=4)]]</Select> \ </Query> \ </QueryList> Exec $Message = replace ($Message, '"', "'"); # Uncomment im_mseventlog for Windows XP/2000/2003 # Module im_mseventlog Exec to_json(); </Input> <Output out> Module om_tcp # Ip of HAProxy listeners server Host 192.168.0.14 # Port in Listeners Port 3515 </Output> <Route 1> Path eventlog => out </Route>