ELK stack na Ubuntu 17.04

# Java
sudo add-apt-repository -y ppa:webupd8team/java
sudo apt-get update
sudo apt-get -y install oracle-java8-installer

sudo update-java-alternatives --set java-8-oracle
java -XshowSettings 2>&1 | grep -e 'java.home' | awk '{print "JAVA_HOME="$3}' | sed "s/\/jre//g" >> /etc/environment
odhlásit a přihlásit (ze sudo), aby se projevila změna v environment

# Elastic applications
wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
sudo apt-get install apt-transport-https
echo "deb https://artifacts.elastic.co/packages/5.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-5.x.list
sudo apt-get update && sudo apt-get -y install elasticsearch kibana logstash

# Elasticsearch
vi /etc/elasticsearch/elasticsearch.yml
network.host: 127.0.0.1
http.max_header_size: 64kb
http.max_initial_line_length: 32kb
http.compression_level: 5

sudo systemctl restart elasticsearch
sudo systemctl daemon-reload
sudo systemctl enable elasticsearch

# Kibana
vi /etc/kibana/kibana.yml
server.host: "10.1.8.50"
logging.quiet: true

sudo systemctl daemon-reload
sudo systemctl enable kibana
sudo systemctl start kibana

# Logstash - SSL certifikát pro nahrávání logů z klientů
sudo mkdir -p /etc/pki/tls/certs
sudo mkdir /etc/pki/tls/private
cd /etc/pki/tls
sudo openssl req -subj '/CN=elk.cesal.cz/' -x509 -days 3650 -batch -nodes -newkey rsa:2048 -keyout private/logstash-forwarder.key -out certs/logstash-forwarder.crt

# Logstash
# ### Inputs
vi /etc/logstash/conf.d/02-beats-input.conf

input {
  beats {
    port => 5044
    ssl => true
    ssl_certificate => "/etc/pki/tls/certs/logstash-forwarder.crt"
    ssl_key => "/etc/pki/tls/private/logstash-forwarder.key"
  }
}

# ### Filters
vi /etc/logstash/conf.d/10-syslog-filter.conf

filter {
  if [type] == "syslog" {
    grok {
      match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
      add_field => [ "received_at", "%{@timestamp}" ]
      add_field => [ "received_from", "%{host}" ]
    }
    syslog_pri { }
    date {
      match => [ "syslog_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
    }
  } else if [type] == "apache_access" {
    grok {
      match => { "message" => "%{HTTPD_COMBINEDLOG}" }
    }
    date {
      match => [ "timestamp" , "dd/MMM/yyyy:HH:mm:ss Z" ]
    }
  }
}


# ### Outputs
vi /etc/logstash/conf.d/30-elasticsearch-output.conf

output {
  elasticsearch {
    hosts => ["localhost:9200"]
    sniffing => true
    manage_template => false
    index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
    document_type => "%{[@metadata][type]}"
  }
}

# Logstash config test
/usr/share/logstash/bin/logstash -t -f /etc/logstash/conf.d/

# Logstash - update plugins
/usr/share/logstash/bin/logstash-plugin update
/usr/share/logstash/bin/logstash-plugin install logstash-filter-cidr

# Logstash - service
sudo systemctl daemon-reload
sudo systemctl enable logstash
sudo systemctl restart logstash

# Filebeat do Elasticsearch
curl -O https://gist.githubusercontent.com/thisismitch/3429023e8438cc25b86c/raw/d8c479e2a1adcea8b1fe86570e42abab0f10f364/filebeat-index-template.json
curl -XPUT 'http://localhost:9200/_template/filebeat?pretty' -H 'Content-Type: application/json' -d@filebeat-index-template.json

# Firewall
vi /etc/iptables.rules

# Logstash
-A INPUT -p tcp -m tcp --dport 5044 -j ACCEPT
-A INPUT -p udp -m udp --dport 5044 -j ACCEPT
# Kibana
-A INPUT -p tcp -m tcp --dport 5601 -j ACCEPT

iptables-restore < /etc/iptables.rules

Na každém z klientů:


# Z ELK serveru zkopírovat certifikát na klienta:
scp /etc/pki/tls/certs/logstash-forwarder.crt peter@10.1.8.99:/tmp

# Na klientovi - přesunout certifikát
sudo mkdir -p /etc/pki/tls/certs
sudo mv /tmp/logstash-forwarder.crt /etc/pki/tls/certs/

# Nainstalovat Filebeat
wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
sudo apt-get install apt-transport-https
echo "deb https://artifacts.elastic.co/packages/5.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-5.x.list
sudo apt-get update && sudo apt-get -y install filebeat

# Konfigurace Filebeat
https://www.elastic.co/guide/en/beats/filebeat/current/configuration-filebeat-options.html

ll /var/log/*.log /var/log/*.err /var/log/syslog /var/log/apache2/*.log /var/log/apache2/*.err /var/log/mysql/error.log
vi /etc/filebeat/filebeat.yml
  
- input_type: log
  paths:
    - /var/log/syslog
    - /var/log/syslog.1
    - /var/log/auth.log
    - /var/log/auth.log.1
    - /var/log/mail.log
    - /var/log/mail.log.1
    - /var/log/mail.err
    - /var/log/mail.err.1
    - /var/vmail/dovecot-deliver.log
  document_type: syslog

- input_type: log
  paths:
    - /var/log/apache2/access.log
    - /var/log/apache2/other_vhosts_access.log
  document_type: apache_access


Delete or comment out the entire Elasticsearch output section
Chceme to do Logstash, ne do Elasticu:

output.logstash:
  # The Logstash hosts
  hosts: ["elk.cesal.cz:5044"]
  bulk_max_size: 1024

  # Optional SSL. By default is off.
  # List of root certificates for HTTPS server verifications
  ssl.certificate_authorities: ["/etc/pki/tls/certs/logstash-forwarder.crt"]


sudo systemctl daemon-reload
sudo systemctl enable filebeat
sudo systemctl restart filebeat

# Logy filebeat na klinetovi:
tail -f /var/log/syslog /var/log/filebeat/filebeat


# Filebeat nainstalovat také na ELK serveru a importovat schéma do Kibany
/usr/share/filebeat/scripts/import_dashboards

tohle není třeba:
wget https://artifacts.elastic.co/downloads/beats/beats-dashboards/beats-dashboards-5.4.0.zip
sudo apt-get -y install unzip
unzip beats-dashboards-*.zip
rm beats-dashboards-*.zip
cd beats-dashboards-*


# ### Znovuposlání logů Filebeatem
systemctl stop filebeat && rm /var/lib/filebeat/registry
systemctl restart filebeat

# ### Vypsání a smazání indexů v Elasticu
curl -XGET 'http://localhost:9200/_cat/indices?v'
curl -XDELETE 'http://localhost:9200/filebeat*'

# Smazání jen některých dat (podle fieldu)
POST filebeat-*/_delete_by_query?conflicts=proceed
{
  "query": { 
    "match": {
      "received_from": "aaa"
    }
  }
}

Custom grok patterns:

/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-4.1.0/patterns

JAVA_CLASS_NAME ([a-zA-Z$_][\.\a-zA-Z$_0-9]*?)
JAVA_CLASS_LINE (\d+)
LOG4J2_SQBRACKET .+?
CZECH_DATE_TIME %{MONTHDAY}\.%{MONTHNUM}\.%{YEAR} %{HOUR}:%{MINUTE}:%{SECOND}

do filteru:

^\[%{LOG4J2_SQBRACKET:log_level}%{SPACE}\] %{CZECH_DATE_TIME} \[%{LOG4J2_SQBRACKET:java_thread}\] \(%{JAVA_CLASS_NAME:java_class_name}(?:\.java)?\:%{JAVA_CLASS_LINE:java_class_line}\) *\- *%{DATA:log_message}$

systemctl restart logstash && tail -f /var/log/syslog /var/log/logstash/logstash-plain.log

David Česal

ELK stack na Ubuntu 17.04