Updated ELK stack to implement log standards 46/2846/1
authorGUPTA <pg0057291@techmahindra.com>
Fri, 14 Sep 2018 17:10:08 +0000 (13:10 -0400)
committerGUPTA <pg0057291@techmahindra.com>
Fri, 14 Sep 2018 17:10:08 +0000 (13:10 -0400)
Issue-ID: ACUMOS-1091

Change-Id: I015cce56b13181133f82567d0442bb7384932a0f
Signed-off-by: GUPTA <pg0057291@techmahindra.com>
docs/release-notes.rst
elk-stack/elasticsearch/pom.xml
elk-stack/kibana/pom.xml
elk-stack/logstash/pipeline/logstash.conf
elk-stack/logstash/pom.xml
filebeat/config/filebeat.yml
filebeat/pom.xml
metricbeat/pom.xml

index 19ea662..1fd43f6 100644 (file)
 Platform Operations, Administration, and Management (OA&M) Release Notes
 ========================================================================
 
+Version 1.18.1, 14 September 2018
+---------------------------------
+
+-  Updated ELK, filebeat and metricbeat as per standard log specification (ACUMOS-1091).
+
 Version 1.18.0, 5 September 2018
 --------------------------------
 
index 3ce1d4e..7e42029 100644 (file)
@@ -25,7 +25,7 @@
         <!-- docker image name -->
         <artifactId>acumos-elasticsearch</artifactId>
         <!-- docker tag -->
-        <version>1.18.0-SNAPSHOT</version>
+        <version>1.18.1-SNAPSHOT</version>
 </project>
 
 
index ed58d30..ec42df7 100644 (file)
@@ -25,7 +25,7 @@
         <!-- docker image name -->
         <artifactId>acumos-kibana</artifactId>
         <!-- docker tag -->
-        <version>1.18.0-SNAPSHOT</version>
+        <version>1.18.1-SNAPSHOT</version>
 </project>
 
 
index 11d129f..0225c08 100644 (file)
@@ -145,33 +145,36 @@ jdbc {
 
   beats {
         port => 5000
-        codec => plain {
-                 charset => "ISO-8859-1"
-                 }        
+        codec => multiline {
+                 pattern => "^%{TIMESTAMP_ISO8601}"
+                 negate => true
+                 what => "previous"
+                 }                      
   }
 }
-filter {
-     
-  if ([source] =~ /access.log$/){
-       grok {
-          match => { "message" => "%{TIMESTAMP_ISO8601:timestamp}\|%{WORD:moduleName}\|%{WORD:userAgent}\|%{NOTSPACE:urlDefault}\|%{LOGLEVEL:loglevel}\|\|%{GREEDYDATA:unstructured_data}\|%{SPACE}%{SPACE}(?<actualData>(.|\r|\n)*)" }
-       }
-  }
-  if ([source] =~ /application.log$/){
-       grok {
-          match => { "message" => "%{TIMESTAMP_ISO8601:timestamp}\|\|\|%{NOTSPACE:app}\|\|\|\|%{LOGLEVEL:loglevel}\|\|\|\|\|\|\|%{SPACE}%{GREEDYDATA:actualData}" }
-       }
-  }
-  if ([source] =~ /debug.log$/){
+filter {  
+
+    if ([source] =~ /audit.log$/){
        grok {
-          match => { "message" => "%{TIMESTAMP_ISO8601:timestamp}\|\|\|%{NOTSPACE:app}\|\|\|\|%{LOGLEVEL:loglevel}\|\|\|\|\|%{SPACE}%{GREEDYDATA:packageName}\|\|%{SPACE}(?<actualData>(.|\r|\n)*)" }
+          match => { "message" => "%{DATA:LogTimestamp}\|%{DATA:EntryTimestamp}\|%{DATA:InvokeTimestamp}\|%{DATA:RequestID}\|%{DATA:InvocationID}\|%{DATA:InstanceUUID}\|%{DATA:ServiceInstanceID}\|%{DATA:Thread}\|%{DATA:ServiceName}\|%{DATA:PartnerName}\|%{DATA:StatusCode}\|%{DATA:ResponseCode}\|%{DATA:ResponseDescription}\|%{DATA:level}\|%{DATA:Severity}\|%{DATA:ServerIPAddress}\|%{DATA:ElapsedTime}\|%{DATA:ServerFQDN}\|%{DATA:ClientIPAddress}\|%{DATA:VirtualServerName}\|%{DATA:ContextName}\|%{DATA:TargetEntity}\|%{DATA:TargetServiceName}\|%{DATA:TargetElement}\|%{DATA:User}\|%{DATA:p_logger}\|%{DATA:p_mdc}\|%{DATA:p_message}\|%{DATA:p_marker}" }
        }
-  }
-  if ([source] =~ /error.log$/){
+    }
+    if ([source] =~ /debug.log$/){
        grok {
-          match => { "message" => "%{TIMESTAMP_ISO8601:timestamp}\|\|\|%{NOTSPACE:app}\|\|\|\|%{LOGLEVEL:loglevel}\|\|\|\|\|%{SPACE}%{GREEDYDATA:packageName}\|\|%{SPACE}(?<actualData>(.|\r|\n)*)" }
+          match => { "message" => "%{DATA:LogTimestamp}\|%{DATA:EntryTimestamp}\|%{DATA:InvokeTimestamp}\|%{DATA:RequestID}\|%{DATA:InvocationID}\|%{DATA:InstanceUUID}\|%{DATA:ServiceInstanceID}\|%{DATA:Thread}\|%{DATA:ServiceName}\|%{DATA:PartnerName}\|%{DATA:StatusCode}\|%{DATA:ResponseCode}\|%{DATA:ResponseDescription}\|%{DATA:level}\|%{DATA:Severity}\|%{DATA:ServerIPAddress}\|%{DATA:ElapsedTime}\|%{DATA:ServerFQDN}\|%{DATA:ClientIPAddress}\|%{DATA:VirtualServerName}\|%{DATA:ContextName}\|%{DATA:TargetEntity}\|%{DATA:TargetServiceName}\|%{DATA:TargetElement}\|%{DATA:User}\|%{DATA:p_logger}\|%{DATA:p_mdc}\|%{DATA:p_message}\|%{DATA:p_marker}" }
        }
-  }
+    }
+
+    if ([source] =~ /error.log$/){
+         grok {
+            match => { "message" => "%{DATA:LogTimestamp}\|%{DATA:EntryTimestamp}\|%{DATA:InvokeTimestamp}\|%{DATA:RequestID}\|%{DATA:InvocationID}\|%{DATA:InstanceUUID}\|%{DATA:ServiceInstanceID}\|%{DATA:Thread}\|%{DATA:ServiceName}\|%{DATA:PartnerName}\|%{DATA:StatusCode}\|%{DATA:ResponseCode}\|%{DATA:ResponseDescription}\|%{DATA:level}\|%{DATA:Severity}\|%{DATA:ServerIPAddress}\|%{DATA:ElapsedTime}\|%{DATA:ServerFQDN}\|%{DATA:ClientIPAddress}\|%{DATA:VirtualServerName}\|%{DATA:ContextName}\|%{DATA:TargetEntity}\|%{DATA:TargetServiceName}\|%{DATA:TargetElement}\|%{DATA:User}\|%{DATA:p_logger}\|%{DATA:p_mdc}\|%{DATA:p_message}\|(?<p_marker>(.|\r|\n)*)" }
+         }
+    }
+       if ([source] =~ /metrics.log$/){
+         grok {
+            match => { "message" => "%{DATA:LogTimestamp}\|%{DATA:EntryTimestamp}\|%{DATA:InvokeTimestamp}\|%{DATA:RequestID}\|%{DATA:InvocationID}\|%{DATA:InstanceUUID}\|%{DATA:ServiceInstanceID}\|%{DATA:Thread}\|%{DATA:ServiceName}\|%{DATA:PartnerName}\|%{DATA:StatusCode}\|%{DATA:ResponseCode}\|%{DATA:ResponseDescription}\|%{DATA:level}\|%{DATA:Severity}\|%{DATA:ServerIPAddress}\|%{DATA:ElapsedTime}\|%{DATA:ServerFQDN}\|%{DATA:ClientIPAddress}\|%{DATA:VirtualServerName}\|%{DATA:ContextName}\|%{DATA:TargetEntity}\|%{DATA:TargetServiceName}\|%{DATA:TargetElement}\|%{DATA:User}\|%{DATA:p_logger}\|%{DATA:p_mdc}\|%{DATA:p_message}\|(?<p_marker>(.|\r|\n)*)" }
+         }
+    }
 
 }
 
index 802af1d..f196767 100644 (file)
@@ -25,5 +25,5 @@
         <!-- docker image name -->
         <artifactId>acumos-logstash</artifactId>
         <!-- docker tag -->
-        <version>1.18.0-SNAPSHOT</version>
+        <version>1.18.1-SNAPSHOT</version>
 </project>
index 883d860..f29c0a1 100644 (file)
@@ -37,14 +37,7 @@ filebeat.prospectors:
   #enabled: true
   # Paths that should be crawled and fetched. Glob based paths.
   paths:
-    - /filebeat-logs/portal-be/*.log
-    - /filebeat-logs/ccds/*.log
-    - /filebeat-logs/EELF/*.log
-    - /filebeat-logs/portal-fe/*.log
-    - /filebeat-logs/on-boarding/*.log
-    - /filebeat-logs/dsce/*.log
-    - /filebeat-logs/federated-gateway/*.log
-
+    - /filebeat-logs/*/*.log
 
   #encoding: utf-16le-bom
   document_type: log
index 5de476e..78a6f08 100644 (file)
@@ -25,7 +25,7 @@
         <!-- docker image name -->
         <artifactId>acumos-filebeat</artifactId>
         <!-- docker tag -->
-        <version>1.18.0-SNAPSHOT</version>
+        <version>1.18.1-SNAPSHOT</version>
 </project>
 
 
index e4d4cd1..895f81f 100644 (file)
@@ -25,7 +25,7 @@
         <!-- docker image name -->
         <artifactId>acumos-metricbeat</artifactId>
         <!-- docker tag -->
-        <version>1.18.0-SNAPSHOT</version>
+        <version>1.18.1-SNAPSHOT</version>
 </project>