Merge "Adding the ELK Code"
authorAimee Ukasick <aimeeu.opensource@gmail.com>
Fri, 23 Feb 2018 14:57:49 +0000 (14:57 +0000)
committerGerrit Code Review <gerrit@acumos.org>
Fri, 23 Feb 2018 14:57:49 +0000 (14:57 +0000)
14 files changed:
elk-stack/docker-compose.yml [new file with mode: 0644]
elk-stack/elasticsearch/Dockerfile [new file with mode: 0644]
elk-stack/elasticsearch/config/elasticsearch.yml [new file with mode: 0644]
elk-stack/elasticsearch/pom.xml [new file with mode: 0644]
elk-stack/filebeat/Dockerfile [new file with mode: 0644]
elk-stack/filebeat/config/filebeat.yml [new file with mode: 0644]
elk-stack/filebeat/pom.xml [new file with mode: 0644]
elk-stack/kibana/Dockerfile [new file with mode: 0644]
elk-stack/kibana/config/kibana.yml [new file with mode: 0644]
elk-stack/kibana/pom.xml [new file with mode: 0644]
elk-stack/logstash/Dockerfile [new file with mode: 0644]
elk-stack/logstash/config/logstash.yml [new file with mode: 0644]
elk-stack/logstash/pipeline/logstash.conf [new file with mode: 0644]
elk-stack/logstash/pom.xml [new file with mode: 0644]

diff --git a/elk-stack/docker-compose.yml b/elk-stack/docker-compose.yml
new file mode 100644 (file)
index 0000000..0501258
--- /dev/null
@@ -0,0 +1,55 @@
+version: '2'
+
+services:
+
+  elasticsearch:
+    build: elasticsearch/
+    volumes:
+      - ./elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
+    ports:
+      - "9200:9200"
+      - "9300:9300"
+    environment:
+      ES_JAVA_OPTS: "-Xmx256m -Xms256m"
+    networks:
+      - elk
+
+  logstash:
+    build: logstash/
+    volumes:
+      - ./logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml
+      - ./logstash/pipeline:/usr/share/logstash/pipeline
+    ports:
+      - "5000:5000"
+    environment:
+      LS_JAVA_OPTS: "-Xmx256m -Xms256m"
+    networks:
+      - elk
+    depends_on:
+      - elasticsearch
+
+  kibana:
+    build: kibana/
+    volumes:
+      - ./kibana/config/:/usr/share/kibana/config
+    ports:
+      - "5601:5601"
+    networks:
+      - elk
+    depends_on:
+      - elasticsearch
+
+  filebeat:
+    build: filebeat/
+    volumes:
+     - acumos-logs:/filebeat-logs    
+    networks:
+      - elk
+    depends_on:
+      - logstash         
+  
+
+networks:
+
+  elk:
+    driver: bridge
\ No newline at end of file
diff --git a/elk-stack/elasticsearch/Dockerfile b/elk-stack/elasticsearch/Dockerfile
new file mode 100644 (file)
index 0000000..b534c5c
--- /dev/null
@@ -0,0 +1,4 @@
+# https://github.com/elastic/elasticsearch-docker
+FROM docker.elastic.co/elasticsearch/elasticsearch:5.5.1
+# Add your elasticsearch plugins setup here
+# Example: RUN elasticsearch-plugin install analysis-icu
diff --git a/elk-stack/elasticsearch/config/elasticsearch.yml b/elk-stack/elasticsearch/config/elasticsearch.yml
new file mode 100644 (file)
index 0000000..b3f96bb
--- /dev/null
@@ -0,0 +1,26 @@
+---
+## Default Elasticsearch configuration from elasticsearch-docker.
+## from https://github.com/elastic/elasticsearch-docker/blob/master/build/elasticsearch/elasticsearch.yml
+#
+cluster.name: "docker-cluster"
+network.host: 0.0.0.0
+
+# minimum_master_nodes need to be explicitly set when bound on a public IP
+# set to 1 to allow single node clusters
+# Details: https://github.com/elastic/elasticsearch/pull/17288
+discovery.zen.minimum_master_nodes: 1
+
+## Use single node discovery in order to disable production mode and avoid bootstrap checks
+## see https://www.elastic.co/guide/en/elasticsearch/reference/current/bootstrap-checks.html
+#
+discovery.type: single-node
+
+## Disable X-Pack
+## see https://www.elastic.co/guide/en/x-pack/current/xpack-settings.html
+##     https://www.elastic.co/guide/en/x-pack/current/installing-xpack.html#xpack-enabling
+#
+xpack.security.enabled: false
+xpack.monitoring.enabled: false
+xpack.ml.enabled: false
+xpack.graph.enabled: false
+xpack.watcher.enabled: false
diff --git a/elk-stack/elasticsearch/pom.xml b/elk-stack/elasticsearch/pom.xml
new file mode 100644 (file)
index 0000000..f901aa3
--- /dev/null
@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+        <!-- This file publishes Docker name and tag for the LF Jenkins build script -->
+        <groupId>IGNORED</groupId>
+        <!-- docker image name -->
+        <artifactId>acumos-elasticsearch</artifactId>
+        <!-- docker tag -->
+        <version>1.0-SNAPSHOT</version>
+</project>
+
+
diff --git a/elk-stack/filebeat/Dockerfile b/elk-stack/filebeat/Dockerfile
new file mode 100644 (file)
index 0000000..be46001
--- /dev/null
@@ -0,0 +1,6 @@
+# https://hub.docker.com/r/prima/filebeat/
+FROM docker.elastic.co/beats/filebeat:6.0.1
+COPY /config/filebeat.yml /usr/share/filebeat/filebeat.yml
+USER root
+RUN chown filebeat /usr/share/filebeat/filebeat.yml && chmod go-w /usr/share/filebeat/filebeat.yml
+USER filebeat
diff --git a/elk-stack/filebeat/config/filebeat.yml b/elk-stack/filebeat/config/filebeat.yml
new file mode 100644 (file)
index 0000000..82643bb
--- /dev/null
@@ -0,0 +1,128 @@
+###################### Filebeat Configuration Example #########################
+
+# This file is an example configuration file highlighting only the most common
+# options. The filebeat.full.yml file from the same directory contains all the
+# supported options with more comments. You can use it as a reference.
+#
+# You can find the full configuration reference here:
+# https://www.elastic.co/guide/en/beats/filebeat/index.html
+
+#=========================== Filebeat prospectors =============================
+
+filebeat.prospectors:
+
+# Each - is a prospector. Most options can be set at the prospector level, so
+# you can use different prospectors for various configurations.
+# Below are the prospector specific configurations.
+
+- input_type: log
+  #enabled: true
+  # Paths that should be crawled and fetched. Glob based paths.
+  paths:
+    - /filebeat-logs/portal-be/*.log
+    - /filebeat-logs/ccds/*.log
+    - /filebeat-logs/EELF/*.log
+    - /filebeat-logs/portal-fe/*.log
+    - /filebeat-logs/on-boarding/*.log
+    - /filebeat-logs/dsce/*.log
+    - /filebeat-logs/federated-gateway/*.log
+
+   #registry_file:
+   # - /home/cognitamaster/filebeat/registry
+
+  #encoding: utf-16le-bom
+  document_type: log
+  #fields_under_root: true
+  # Exclude lines. A list of regular expressions to match. It drops the lines that are
+  # matching any regular expression from the list.
+  #exclude_lines: ["^DBG"]
+
+  # Include lines. A list of regular expressions to match. It exports the lines that are
+  # matching any regular expression from the list.
+  #include_lines: ["^ERR", "^WARN"]
+
+  # Exclude files. A list of regular expressions to match. Filebeat drops the files that
+  # are matching any regular expression from the list. By default, no files are dropped.
+  #exclude_files: [".gz$"]
+
+  # Optional additional fields. These field can be freely picked
+  # to add additional information to the crawled log files for filtering
+  #fields:
+  #  level: debug
+  #  review: 1
+
+  ### Multiline options
+
+  # Mutiline can be used for log messages spanning multiple lines. This is common
+  # for Java Stack Traces or C-Line Continuation
+
+  # The regexp Pattern that has to be matched. The example pattern matches all lines starting with [
+  #multiline.pattern: '^Caused by:'
+  #multiline.pattern: '^[[:space:]]'
+  #"^\t|^[[:space:]]+(at|...)|^Caused by:"
+  multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2},[0-9]{3}'
+  multiline.negate: true
+  multiline.match: after
+  # Defines if the pattern set under pattern should be negated or not. Default is false.
+  #multiline.negate: false
+
+  # Match can be set to "after" or "before". It is used to define if lines should be append to a pattern
+  # that was (not) matched before or after or as long as a pattern is not matched based on negate.
+  # Note: After is the equivalent to previous and before is the equivalent to to next in Logstash
+  #multiline.match: after
+
+
+#================================ General =====================================
+
+# The name of the shipper that publishes the network data. It can be used to group
+# all the transactions sent by a single shipper in the web interface.
+#name:
+
+# The tags of the shipper are included in their own field with each
+# transaction published.
+#tags: ["service-X", "web-tier"]
+
+# Optional fields that you can specify to add additional information to the
+# output.
+#fields:
+#  env: staging
+
+#================================ Outputs =====================================
+
+# Configure what outputs to use when sending the data collected by the beat.
+# Multiple outputs may be used.
+
+#-------------------------- Elasticsearch output ------------------------------
+#output.elasticsearch:
+  # Array of hosts to connect to.
+  #hosts: ["localhost:9200"]
+
+  # Optional protocol and basic auth credentials.
+  #protocol: "https"
+  #username: "elastic"
+  #password: "changeme"
+
+#----------------------------- Logstash output --------------------------------
+output.logstash:
+  # The Logstash hosts
+  hosts: ["${LOGSTASH_HOST}:${LOGSTASH_PORT}"]
+  # Optional SSL. By default is off.
+  # List of root certificates for HTTPS server verifications
+  #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
+
+  # Certificate for SSL client authentication
+  #ssl.certificate: "/etc/pki/client/cert.pem"
+
+  # Client Certificate Key
+  #ssl.key: "/etc/pki/client/cert.key"
+
+#================================ Logging =====================================
+
+# Sets log level. The default log level is info.
+# Available log levels are: critical, error, warning, info, debug
+#logging.level: debug
+
+# At debug level, you can selectively enable logging only for some components.
+# To enable all selectors use ["*"]. Examples of other selectors are "beat",
+# "publish", "service".
+#logging.selectors: ["*"]
diff --git a/elk-stack/filebeat/pom.xml b/elk-stack/filebeat/pom.xml
new file mode 100644 (file)
index 0000000..082e247
--- /dev/null
@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+        <!-- This file publishes Docker name and tag for the LF Jenkins build script -->
+        <groupId>IGNORED</groupId>
+        <!-- docker image name -->
+        <artifactId>acumos-filebeat</artifactId>
+        <!-- docker tag -->
+        <version>1.0-SNAPSHOT</version>
+</project>
+
+
diff --git a/elk-stack/kibana/Dockerfile b/elk-stack/kibana/Dockerfile
new file mode 100644 (file)
index 0000000..58797d4
--- /dev/null
@@ -0,0 +1,4 @@
+# https://github.com/elastic/kibana-docker
+FROM docker.elastic.co/kibana/kibana:5.5.1
+# Add your kibana plugins setup here
+# Example: RUN kibana-plugin install <name|url>
\ No newline at end of file
diff --git a/elk-stack/kibana/config/kibana.yml b/elk-stack/kibana/config/kibana.yml
new file mode 100644 (file)
index 0000000..50bba63
--- /dev/null
@@ -0,0 +1,17 @@
+---
+## Default Kibana configuration from kibana-docker.
+## from https://github.com/elastic/kibana-docker/blob/master/build/kibana/config/kibana.yml
+#
+server.name: kibana
+server.host: "0"
+elasticsearch.url: http://elasticsearch:9200
+
+## Disable X-Pack
+## see https://www.elastic.co/guide/en/x-pack/current/xpack-settings.html
+##     https://www.elastic.co/guide/en/x-pack/current/installing-xpack.html#xpack-enabling
+#
+xpack.security.enabled: false
+xpack.monitoring.enabled: false
+xpack.ml.enabled: false
+xpack.graph.enabled: false
+xpack.reporting.enabled: false
\ No newline at end of file
diff --git a/elk-stack/kibana/pom.xml b/elk-stack/kibana/pom.xml
new file mode 100644 (file)
index 0000000..e48690b
--- /dev/null
@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+        <!-- This file publishes Docker name and tag for the LF Jenkins build script -->
+        <groupId>IGNORED</groupId>
+        <!-- docker image name -->
+        <artifactId>acumos-kibana</artifactId>
+        <!-- docker tag -->
+        <version>1.0-SNAPSHOT</version>
+</project>
+
+
diff --git a/elk-stack/logstash/Dockerfile b/elk-stack/logstash/Dockerfile
new file mode 100644 (file)
index 0000000..3c59a96
--- /dev/null
@@ -0,0 +1,26 @@
+# https://github.com/elastic/logstash-docker
+FROM docker.elastic.co/logstash/logstash:5.5.1
+#FROM docker.elastic.co/logstash/logstash:6.0.1
+
+# Add your logstash plugins setup here
+# Example: RUN logstash-plugin install logstash-filter-json
+# Install plugin for development
+##RUN /opt/logstash/bin/logstash-plugin install --development
+##RUN /opt/logstash/bin/logstash-plugin list
+
+
+# Prepare directory & file for test
+##USER root
+##RUN mkdir -p /test
+
+#ADD spec /test/spec
+#ADD pipeline /test/pipeline
+##ADD spec/*.rb /test/spec
+##ADD pipeline/logstash.conf /test/pipeline/logstash.conf
+##ADD run.sh /usr/local/bin/run.sh
+
+##RUN chown logstash /usr/local/bin/run.sh && chmod +x /usr/local/bin/run.sh
+
+##USER logstash
+##CMD ["/usr/local/bin/run.sh"]
+#USER logstash
diff --git a/elk-stack/logstash/config/logstash.yml b/elk-stack/logstash/config/logstash.yml
new file mode 100644 (file)
index 0000000..f9d6053
--- /dev/null
@@ -0,0 +1,11 @@
+---
+## Default Logstash configuration from logstash-docker.
+## from https://github.com/elastic/logstash-docker/blob/master/build/logstash/config/logstash.yml
+#
+http.host: "0.0.0.0"
+path.config: /usr/share/logstash/pipeline
+
+## Disable X-Pack
+## see https://www.elastic.co/guide/en/x-pack/current/xpack-settings.html
+##     https://www.elastic.co/guide/en/x-pack/current/installing-xpack.html#xpack-enabling
+#
diff --git a/elk-stack/logstash/pipeline/logstash.conf b/elk-stack/logstash/pipeline/logstash.conf
new file mode 100644 (file)
index 0000000..6cb5c37
--- /dev/null
@@ -0,0 +1,39 @@
+input {
+  beats {
+        port => 5000
+        codec => plain {
+                 charset => "ISO-8859-1"
+                 }        
+  }
+}
+filter {
+     
+  if ([source] =~ /access.log$/){
+       grok {
+          match => { "message" => "%{TIMESTAMP_ISO8601:timestamp}\|%{WORD:moduleName}\|%{WORD:userAgent}\|%{NOTSPACE:urlDefault}\|%{LOGLEVEL:loglevel}\|\|%{GREEDYDATA:unstructured_data}\|%{SPACE}%{SPACE}(?<actualData>(.|\r|\n)*)" }
+       }
+  }
+  if ([source] =~ /application.log$/){
+       grok {
+          match => { "message" => "%{TIMESTAMP_ISO8601:timestamp}\|\|\|%{NOTSPACE:app}\|\|\|\|%{LOGLEVEL:loglevel}\|\|\|\|\|\|\|%{SPACE}%{GREEDYDATA:actualData}" }
+       }
+  }
+  if ([source] =~ /debug.log$/){
+       grok {
+          match => { "message" => "%{TIMESTAMP_ISO8601:timestamp}\|\|\|%{NOTSPACE:app}\|\|\|\|%{LOGLEVEL:loglevel}\|\|\|\|\|%{SPACE}%{GREEDYDATA:packageName}\|\|%{SPACE}(?<actualData>(.|\r|\n)*)" }
+       }
+  }
+  if ([source] =~ /error.log$/){
+       grok {
+          match => { "message" => "%{TIMESTAMP_ISO8601:timestamp}\|\|\|%{NOTSPACE:app}\|\|\|\|%{LOGLEVEL:loglevel}\|\|\|\|\|%{SPACE}%{GREEDYDATA:packageName}\|\|%{SPACE}(?<actualData>(.|\r|\n)*)" }
+       }
+  }
+
+}
+
+
+output {
+        elasticsearch { hosts => ["elasticsearch:9200"]
+       }
+
+}
diff --git a/elk-stack/logstash/pom.xml b/elk-stack/logstash/pom.xml
new file mode 100644 (file)
index 0000000..3ca8f15
--- /dev/null
@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+        <!-- This file publishes Docker name and tag for the LF Jenkins build script -->
+        <groupId>IGNORED</groupId>
+        <!-- docker image name -->
+        <artifactId>acumos-logstash</artifactId>
+        <!-- docker tag -->
+        <version>1.0-SNAPSHOT</version>
+</project>