Added filter for unstructured logs 61/2861/2
authorAMIT M <am00474504@techmahindra.com>
Mon, 17 Sep 2018 14:22:50 +0000 (19:52 +0530)
committerAMIT M <am00474504@techmahindra.com>
Mon, 17 Sep 2018 15:29:07 +0000 (20:59 +0530)
Change-Id: I97c738c418e3326fb8e6ef397edb77d05f5ffa2f
Issue-ID: ACUMOS-1091
Signed-off-by: AMIT M <am00474504@techmahindra.com>
elk-stack/logstash/pipeline/logstash.conf
filebeat/config/filebeat.yml

index 0225c08..fd07066 100644 (file)
@@ -158,22 +158,22 @@ filter {
        grok {
           match => { "message" => "%{DATA:LogTimestamp}\|%{DATA:EntryTimestamp}\|%{DATA:InvokeTimestamp}\|%{DATA:RequestID}\|%{DATA:InvocationID}\|%{DATA:InstanceUUID}\|%{DATA:ServiceInstanceID}\|%{DATA:Thread}\|%{DATA:ServiceName}\|%{DATA:PartnerName}\|%{DATA:StatusCode}\|%{DATA:ResponseCode}\|%{DATA:ResponseDescription}\|%{DATA:level}\|%{DATA:Severity}\|%{DATA:ServerIPAddress}\|%{DATA:ElapsedTime}\|%{DATA:ServerFQDN}\|%{DATA:ClientIPAddress}\|%{DATA:VirtualServerName}\|%{DATA:ContextName}\|%{DATA:TargetEntity}\|%{DATA:TargetServiceName}\|%{DATA:TargetElement}\|%{DATA:User}\|%{DATA:p_logger}\|%{DATA:p_mdc}\|%{DATA:p_message}\|%{DATA:p_marker}" }
        }
-    }
-    if ([source] =~ /debug.log$/){
+    } else if ([source] =~ /debug.log$/){
        grok {
           match => { "message" => "%{DATA:LogTimestamp}\|%{DATA:EntryTimestamp}\|%{DATA:InvokeTimestamp}\|%{DATA:RequestID}\|%{DATA:InvocationID}\|%{DATA:InstanceUUID}\|%{DATA:ServiceInstanceID}\|%{DATA:Thread}\|%{DATA:ServiceName}\|%{DATA:PartnerName}\|%{DATA:StatusCode}\|%{DATA:ResponseCode}\|%{DATA:ResponseDescription}\|%{DATA:level}\|%{DATA:Severity}\|%{DATA:ServerIPAddress}\|%{DATA:ElapsedTime}\|%{DATA:ServerFQDN}\|%{DATA:ClientIPAddress}\|%{DATA:VirtualServerName}\|%{DATA:ContextName}\|%{DATA:TargetEntity}\|%{DATA:TargetServiceName}\|%{DATA:TargetElement}\|%{DATA:User}\|%{DATA:p_logger}\|%{DATA:p_mdc}\|%{DATA:p_message}\|%{DATA:p_marker}" }
        }
-    }
-
-    if ([source] =~ /error.log$/){
+    } else if ([source] =~ /error.log$/){
          grok {
             match => { "message" => "%{DATA:LogTimestamp}\|%{DATA:EntryTimestamp}\|%{DATA:InvokeTimestamp}\|%{DATA:RequestID}\|%{DATA:InvocationID}\|%{DATA:InstanceUUID}\|%{DATA:ServiceInstanceID}\|%{DATA:Thread}\|%{DATA:ServiceName}\|%{DATA:PartnerName}\|%{DATA:StatusCode}\|%{DATA:ResponseCode}\|%{DATA:ResponseDescription}\|%{DATA:level}\|%{DATA:Severity}\|%{DATA:ServerIPAddress}\|%{DATA:ElapsedTime}\|%{DATA:ServerFQDN}\|%{DATA:ClientIPAddress}\|%{DATA:VirtualServerName}\|%{DATA:ContextName}\|%{DATA:TargetEntity}\|%{DATA:TargetServiceName}\|%{DATA:TargetElement}\|%{DATA:User}\|%{DATA:p_logger}\|%{DATA:p_mdc}\|%{DATA:p_message}\|(?<p_marker>(.|\r|\n)*)" }
          }
-    }
-       if ([source] =~ /metrics.log$/){
+    } else if ([source] =~ /metrics.log$/){
          grok {
             match => { "message" => "%{DATA:LogTimestamp}\|%{DATA:EntryTimestamp}\|%{DATA:InvokeTimestamp}\|%{DATA:RequestID}\|%{DATA:InvocationID}\|%{DATA:InstanceUUID}\|%{DATA:ServiceInstanceID}\|%{DATA:Thread}\|%{DATA:ServiceName}\|%{DATA:PartnerName}\|%{DATA:StatusCode}\|%{DATA:ResponseCode}\|%{DATA:ResponseDescription}\|%{DATA:level}\|%{DATA:Severity}\|%{DATA:ServerIPAddress}\|%{DATA:ElapsedTime}\|%{DATA:ServerFQDN}\|%{DATA:ClientIPAddress}\|%{DATA:VirtualServerName}\|%{DATA:ContextName}\|%{DATA:TargetEntity}\|%{DATA:TargetServiceName}\|%{DATA:TargetElement}\|%{DATA:User}\|%{DATA:p_logger}\|%{DATA:p_mdc}\|%{DATA:p_message}\|(?<p_marker>(.|\r|\n)*)" }
          }
+    } else {
+         grok {
+            match => { "message" => "(?<unstructuredlog>(.|\r|\n)*)" }
+         }
     }
 
 }
index f29c0a1..e4573c2 100644 (file)
@@ -115,6 +115,7 @@ filebeat.prospectors:
 output.logstash:
   # The Logstash hosts
   hosts: ["${LOGSTASH_HOST}:${LOGSTASH_PORT}"]
+  bulk_max_size: 1024
   # Optional SSL. By default is off.
   # List of root certificates for HTTPS server verifications
   #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]