Merge pull request #19 from shazChaudhry/6.4.0

6.4.0

Author: shazChaudhry

README.md

@@ -49,7 +49,7 @@ SSH in to the master node of the Docker Swarm cluster allocated to running Elast
   * `cd docker-elastic`
 
 * Deploy Elastic stack by running the following commands:
-  * `export ELASTIC_VERSION=6.3.2`
+  * `export ELASTIC_VERSION=6.4.0`
   * `docker network create --driver overlay elastic`
   * `docker stack deploy --compose-file docker-compose.yml elastic` _(This will deploy a reverse proxy, logstash, Kibana and 2x Elasticsearch instances in Master / data nodes configuration. Please note that Elasticsearch is configured to start as a global service which means data nodes will be scalled out automatically as soon as new nodes are added to the docker swarm cluster. Here is an explaination on various Elasticsearch cluster nodes: https://discuss.elastic.co/t/node-types-in-an-elasticsearch-cluster/25488)_
 * Check status of the stack services by running the following commands:
@@ -62,7 +62,7 @@ SSH in to the master node of the Docker Swarm cluster allocated to running Elast
 SSH in to the master node of the Docker Swarm cluster allocated to running containerized custom applicatins and beats. Clone this repo and change directory as per the instructions in the previous section
 
 Execute the following commands to deploy filebeat and metricbeat:
-  * `export ELASTIC_VERSION=6.3.2`
+  * `export ELASTIC_VERSION=6.4.0`
   * `docker network create --driver overlay elastic`
   * Edit "filebeat-docker-compose.yml" file. Change environment variables for Kibana and Elasticseaerch hosts
   * `docker stack deploy --compose-file filebeat-docker-compose.yml filebeat`  _(Filebeat starts as a global service on all docker swarm nodes. It is only configured to picks up container logs for all services at '`/var/lib/docker/containers/*/*.log`' (container stdout and stderr logs) and forward thtem to Elasticsearch. These logs will then be available under filebeat index in Kibana. You will need to add additional configurations for other log locations. You may wish to read [Docker Reference Architecture: Docker Logging Design and Best Practices](https://success.docker.com/article/docker-reference-architecture-docker-logging-design-and-best-practices))_

elk/beats/filebeat/config/filebeat.yml

@@ -18,24 +18,56 @@ filebeat.modules:
   log:
     enabled: false
 
-#=========================== Filebeat prospectors ==============================
+#========================== Filebeat autodiscover ==============================
+# Autodiscover allows you to detect changes in the system and spawn new modules or inputs as they happen.
+filebeat.autodiscover:
+# Autodiscover docker containers and parse logs
+  providers:
+    - type: docker
+      templates:
+          config:
+            - type: docker
+              containers.ids:
+                - "${data.docker.container.id}"
+
+#=========================== Filebeat inputs ==============================
 filebeat.inputs:
-- type: log
+#------------------------------ Docker input --------------------------------
+- type: docker
   enabled: true
+  containers.ids:
+    - "*"
   paths:
-    - '/var/lib/docker/containers/*/*.log'
-  json.message_key: log
-  json.add_error_key: true
-  json.keys_under_root: true
+    - /var/lib/docker/containers/${data.docker.container.id}/*.log
+  # json.message_key: log
+  # json.add_error_key: true
+  # json.keys_under_root: true
+  exclude_lines: ["^\\s+[\\-`('.|_]"]  # drop asciiart lines
+  multiline.pattern: "^\t|^[[:space:]]+(at|...)|^Caused by:"
+  multiline.match: after
   processors:
     - add_docker_metadata: ~
     - add_cloud_metadata: ~
     - add_locale: ~
-  multiline.pattern: "^\t|^[[:space:]]+(at|...)|^Caused by:"
-  multiline.match: after
+
+#------------------------------ Log input --------------------------------
 # - type: log
 #   enabled: true
 #   paths:
+#     - '/var/lib/docker/containers/*/*.log'
+#   json.message_key: log
+#   json.add_error_key: true
+#   json.keys_under_root: true
+#   processors:
+#     - add_docker_metadata: ~
+#     - add_cloud_metadata: ~
+#     - add_locale: ~
+#   multiline.pattern: "^\t|^[[:space:]]+(at|...)|^Caused by:"
+#   multiline.match: after
+#
+# - type: log
+#   enabled: false
+#   paths:
 #     # path to jenkins build logs
 #     - /var/lib/docker/volumes/jenkins_home/_data/jobs/*/builds/*/log
 #   json.message_key: log