Update logstash queries for database version 2.0.4 69/3669/1
authorGUPTA <pg0057291@techmahindra.com>
Fri, 25 Jan 2019 01:58:57 +0000 (20:58 -0500)
committerGUPTA <pg0057291@techmahindra.com>
Fri, 25 Jan 2019 02:05:26 +0000 (21:05 -0500)
Issue-ID: ACUMOS-2403

Change-Id: I6848475f269869fdfb7a445b20c834c339487605
Signed-off-by: GUPTA <pg0057291@techmahindra.com>
docs/release-notes.rst
elk-stack/elasticsearch/pom.xml
elk-stack/kibana/pom.xml
elk-stack/logstash/config/logstash.yml
elk-stack/logstash/pipeline/logstash.conf
elk-stack/logstash/pom.xml
filebeat/pom.xml
metricbeat/pom.xml

index 6f8cced..cb2c6f5 100644 (file)
 Platform Operations, Administration, and Management (OA&M) Release Notes
 ========================================================================
 
+Version 2.0.4, 24 January 2019
+-------------------------------
+* Update logstash queries for database version 2.0.4 (`ACUMOS-2403 <https://jira.acumos.org/browse/ACUMOS-2403>`_)
+
 Version 2.0.0, 18 December 2018
 -------------------------------
 * kibana dashboard verbose logstash logs are crashing the log server (`ACUMOS-2151 <https://jira.acumos.org/browse/ACUMOS-2151>`_)
index 641184c..411d9e8 100644 (file)
@@ -25,7 +25,7 @@
         <!-- docker image name -->
         <artifactId>acumos-elasticsearch</artifactId>
         <!-- docker tag -->
-        <version>2.0.0-SNAPSHOT</version>
+        <version>2.0.4-SNAPSHOT</version>
 </project>
 
 
index 9cac87a..1a82d6a 100644 (file)
@@ -25,7 +25,7 @@
         <!-- docker image name -->
         <artifactId>acumos-kibana</artifactId>
         <!-- docker tag -->
-        <version>2.0.0-SNAPSHOT</version>
+        <version>2.0.4-SNAPSHOT</version>
 </project>
 
 
index fa41837..71c7793 100644 (file)
@@ -21,4 +21,5 @@
 #
 http.host: "0.0.0.0"
 path.config: /usr/share/logstash/pipeline
+log.level: error
 
index ba34e7c..3d8654d 100644 (file)
@@ -27,7 +27,7 @@ jdbc {
        jdbc_validate_connection => true
           schedule => "0 * * * *"
        statement => "SELECT * FROM C_SOLUTION_DOWNLOAD"
-       type => "solution_downloaded"
+       sql_log_level => "error"       
        tags => ["database"]
       }
 jdbc {
@@ -38,8 +38,8 @@ jdbc {
        jdbc_password => "${ACUMOS_ELK_JDBC_PASSWORD}"
        jdbc_validate_connection => true
           schedule => "* * * * *"
-       statement => "SELECT USER_ID,FIRST_NAME,MIDDLE_NAME,LAST_NAME,ORG_NAME,EMAIL,LOGIN_NAME,ACTIVE_YN,LAST_LOGIN_DATE,CREATED_DATE,MODIFIED_DATE,LOGIN_FAIL_COUNT FROM C_USER"
-       type => "no_of_user"
+       statement => "SELECT USER_ID,FIRST_NAME,MIDDLE_NAME,LAST_NAME,ORG_NAME,EMAIL,LOGIN_NAME,ACTIVE_YN,LAST_LOGIN_DATE,CREATED_DATE,MODIFIED_DATE,LOGIN_FAIL_COUNT,LOGIN_PASS_EXPIRE_DATE,LOGIN_FAIL_DATE FROM C_USER"
+       sql_log_level => "error"
        tags => ["database"]
       }
 jdbc {
@@ -51,7 +51,7 @@ jdbc {
        jdbc_validate_connection => true
           schedule => "0 * * * *"
        statement => "SELECT * FROM C_SOLUTION_DEPLOYMENT"
-       type => "solution_deployed"
+          sql_log_level => "error"
        tags => ["database"]
       }
 jdbc {
@@ -62,8 +62,8 @@ jdbc {
        jdbc_password => "${ACUMOS_ELK_JDBC_PASSWORD}"
        jdbc_validate_connection => true
           schedule => "* * * * *"
-       statement => "SELECT SOLUTION_ID,NAME,DESCRIPTION,USER_ID,ACTIVE_YN,MODEL_TYPE_CD,TOOLKIT_TYPE_CD,CREATED_DATE,MODIFIED_DATE FROM C_SOLUTION"
-       type => "all_solution"
+       statement => "SELECT SOLUTION_ID,NAME,VIEW_COUNT,DOWNLOAD_COUNT,RATING_AVG_TENTHS,RATING_COUNT,FEATURED_YN,USER_ID,ACTIVE_YN,MODEL_TYPE_CD,TOOLKIT_TYPE_CD,CREATED_DATE,MODIFIED_DATE,LAST_DOWNLOAD FROM C_SOLUTION"
+          sql_log_level => "error"
        tags => ["database"]
       }
 jdbc {
@@ -75,98 +75,30 @@ jdbc {
        jdbc_validate_connection => true
           schedule => "* * * * *"
        statement => "SELECT * FROM C_SOLUTION_REV"
-       type => "revision_solution"
-       tags => ["database"]
-      }
-jdbc {
-          jdbc_driver_library => "${ACUMOS_ELK_JDBC_DRIVER_LIBRARY}"
-       jdbc_driver_class => "${ACUMOS_ELK_JDBC_DRIVER_CLASS}"
-          jdbc_connection_string => "${ACUMOS_ELK_JDBC_CONNECTION_STRING}"
-       jdbc_user => "${ACUMOS_ELK_JDBC_USERNAME}"
-       jdbc_password => "${ACUMOS_ELK_JDBC_PASSWORD}"
-       jdbc_validate_connection => true
-          schedule => "0 * * * *"
-       statement => "SELECT r.REVISION_ID,r.SOLUTION_ID,r.CREATED_DATE,r.MODIFIED_DATE,r.ACCESS_TYPE_CD,r.VALIDATION_STATUS_CD,s.SOLUTION_ID,s.ACTIVE_YN,s.MODEL_TYPE_CD,s.TOOLKIT_TYPE_CD,s.CREATED_DATE,s.MODIFIED_DATE FROM C_SOLUTION s inner join C_SOLUTION_REV r ON s.SOLUTION_ID = r.SOLUTION_ID AND r.ACCESS_TYPE_CD='OR' and s.ACTIVE_YN ='Y'"
-       type => "company_model"
-       tags => ["database"]       
-      }
-jdbc {
-          jdbc_driver_library => "${ACUMOS_ELK_JDBC_DRIVER_LIBRARY}"
-       jdbc_driver_class => "${ACUMOS_ELK_JDBC_DRIVER_CLASS}"
-          jdbc_connection_string => "${ACUMOS_ELK_JDBC_CONNECTION_STRING}"
-       jdbc_user => "${ACUMOS_ELK_JDBC_USERNAME}"
-       jdbc_password => "${ACUMOS_ELK_JDBC_PASSWORD}"
-       jdbc_validate_connection => true
-          schedule => "0 * * * *"
-       statement => "SELECT r.REVISION_ID,r.SOLUTION_ID,r.CREATED_DATE,r.MODIFIED_DATE,r.ACCESS_TYPE_CD,r.VALIDATION_STATUS_CD,s.SOLUTION_ID,s.ACTIVE_YN,s.MODEL_TYPE_CD,s.TOOLKIT_TYPE_CD,s.CREATED_DATE,s.MODIFIED_DATE FROM C_SOLUTION s inner join C_SOLUTION_REV r ON s.SOLUTION_ID = r.SOLUTION_ID AND r.ACCESS_TYPE_CD='PB' and s.ACTIVE_YN ='Y'"
-       type => "public_model"
-       tags => ["database"]
-      }
-jdbc {
-          jdbc_driver_library => "${ACUMOS_ELK_JDBC_DRIVER_LIBRARY}"
-       jdbc_driver_class => "${ACUMOS_ELK_JDBC_DRIVER_CLASS}"
-          jdbc_connection_string => "${ACUMOS_ELK_JDBC_CONNECTION_STRING}"
-       jdbc_user => "${ACUMOS_ELK_JDBC_USERNAME}"
-       jdbc_password => "${ACUMOS_ELK_JDBC_PASSWORD}"
-       jdbc_validate_connection => true
-          schedule => "0 * * * *"
-       statement => "SELECT r.REVISION_ID,r.SOLUTION_ID,r.CREATED_DATE,r.MODIFIED_DATE,r.ACCESS_TYPE_CD,r.VALIDATION_STATUS_CD,s.SOLUTION_ID,s.ACTIVE_YN,s.MODEL_TYPE_CD,s.TOOLKIT_TYPE_CD,s.CREATED_DATE,s.MODIFIED_DATE FROM C_SOLUTION s inner join C_SOLUTION_REV r ON s.SOLUTION_ID = r.SOLUTION_ID AND r.ACCESS_TYPE_CD='PR' and s.ACTIVE_YN ='Y'"
-       type => "private_model"
-       tags => ["database"]
-      }
-jdbc {
-          jdbc_driver_library => "${ACUMOS_ELK_JDBC_DRIVER_LIBRARY}"
-       jdbc_driver_class => "${ACUMOS_ELK_JDBC_DRIVER_CLASS}"
-          jdbc_connection_string => "${ACUMOS_ELK_JDBC_CONNECTION_STRING}"
-       jdbc_user => "${ACUMOS_ELK_JDBC_USERNAME}"
-       jdbc_password => "${ACUMOS_ELK_JDBC_PASSWORD}"
-       jdbc_validate_connection => true
-          schedule => "0 * * * *"
-       statement => "select s.SOLUTION_ID, s.NAME Model_Name, u.First_Name, u.Last_Name, r.ACCESS_TYPE_CD, s.Created_Date, s.Modified_date, s.ACTIVE_YN from C_SOLUTION s, C_USER u, C_SOLUTION_REV r where s.USER_ID = u.USER_ID and s.SOLUTION_ID = r.SOLUTION_ID order by u.Last_Name"
-       type => "solutions"
-       tags => ["database"]
-      }
-jdbc {
-          jdbc_driver_library => "${ACUMOS_ELK_JDBC_DRIVER_LIBRARY}"
-       jdbc_driver_class => "${ACUMOS_ELK_JDBC_DRIVER_CLASS}"
-          jdbc_connection_string => "${ACUMOS_ELK_JDBC_CONNECTION_STRING}"
-       jdbc_user => "${ACUMOS_ELK_JDBC_USERNAME}"
-       jdbc_password => "${ACUMOS_ELK_JDBC_PASSWORD}"
-       jdbc_validate_connection => true
-          schedule => "0 * * * *"
-       statement => "select solution.name,CONCAT(user.first_name,' ',user.last_name) from c_solution solution inner join c_user user on user.USER_ID = solution.USER_ID"
-       type => "name_model"
+          sql_log_level => "error"
        tags => ["database"]
       }
 
 
+
   beats {
         port => 5000
   }
 }
 filter {  
 
-    if ([source] =~ /audit.log$/){
-       grok {
-          match => { "message" => "%{DATA:LogTimestamp}\|%{DATA:EntryTimestamp}\|%{DATA:InvokeTimestamp}\|%{DATA:RequestID}\|%{DATA:InvocationID}\|%{DATA:InstanceUUID}\|%{DATA:ServiceInstanceID}\|%{DATA:Thread}\|%{DATA:ServiceName}\|%{DATA:PartnerName}\|%{DATA:StatusCode}\|%{DATA:ResponseCode}\|%{DATA:ResponseDescription}\|%{DATA:level}\|%{DATA:Severity}\|%{DATA:ServerIPAddress}\|%{DATA:ElapsedTime}\|%{DATA:ServerFQDN}\|%{DATA:ClientIPAddress}\|%{DATA:VirtualServerName}\|%{DATA:ContextName}\|%{DATA:TargetEntity}\|%{DATA:TargetServiceName}\|%{DATA:TargetElement}\|%{DATA:User}\|%{DATA:p_logger}\|%{DATA:p_mdc}\|%{DATA:p_message}\|%{DATA:p_marker}" }
-       }
-    } else if ([source] =~ /debug.log$/){
-       grok {
-          match => { "message" => "%{DATA:LogTimestamp}\|%{DATA:EntryTimestamp}\|%{DATA:InvokeTimestamp}\|%{DATA:RequestID}\|%{DATA:InvocationID}\|%{DATA:InstanceUUID}\|%{DATA:ServiceInstanceID}\|%{DATA:Thread}\|%{DATA:ServiceName}\|%{DATA:PartnerName}\|%{DATA:StatusCode}\|%{DATA:ResponseCode}\|%{DATA:ResponseDescription}\|%{DATA:level}\|%{DATA:Severity}\|%{DATA:ServerIPAddress}\|%{DATA:ElapsedTime}\|%{DATA:ServerFQDN}\|%{DATA:ClientIPAddress}\|%{DATA:VirtualServerName}\|%{DATA:ContextName}\|%{DATA:TargetEntity}\|%{DATA:TargetServiceName}\|%{DATA:TargetElement}\|%{DATA:User}\|%{DATA:p_logger}\|%{DATA:p_mdc}\|%{DATA:p_message}\|%{DATA:p_marker}" }
-       }
-    } else if ([source] =~ /error.log$/){
-         grok {
-            match => { "message" => "%{DATA:LogTimestamp}\|%{DATA:EntryTimestamp}\|%{DATA:InvokeTimestamp}\|%{DATA:RequestID}\|%{DATA:InvocationID}\|%{DATA:InstanceUUID}\|%{DATA:ServiceInstanceID}\|%{DATA:Thread}\|%{DATA:ServiceName}\|%{DATA:PartnerName}\|%{DATA:StatusCode}\|%{DATA:ResponseCode}\|%{DATA:ResponseDescription}\|%{DATA:level}\|%{DATA:Severity}\|%{DATA:ServerIPAddress}\|%{DATA:ElapsedTime}\|%{DATA:ServerFQDN}\|%{DATA:ClientIPAddress}\|%{DATA:VirtualServerName}\|%{DATA:ContextName}\|%{DATA:TargetEntity}\|%{DATA:TargetServiceName}\|%{DATA:TargetElement}\|%{DATA:User}\|%{DATA:p_logger}\|%{DATA:p_mdc}\|%{DATA:p_message}\|(?<p_marker>(.|\r|\n)*)" }
-         }
-    } else if ([source] =~ /metrics.log$/){
-         grok {
-            match => { "message" => "%{DATA:LogTimestamp}\|%{DATA:EntryTimestamp}\|%{DATA:InvokeTimestamp}\|%{DATA:RequestID}\|%{DATA:InvocationID}\|%{DATA:InstanceUUID}\|%{DATA:ServiceInstanceID}\|%{DATA:Thread}\|%{DATA:ServiceName}\|%{DATA:PartnerName}\|%{DATA:StatusCode}\|%{DATA:ResponseCode}\|%{DATA:ResponseDescription}\|%{DATA:level}\|%{DATA:Severity}\|%{DATA:ServerIPAddress}\|%{DATA:ElapsedTime}\|%{DATA:ServerFQDN}\|%{DATA:ClientIPAddress}\|%{DATA:VirtualServerName}\|%{DATA:ContextName}\|%{DATA:TargetEntity}\|%{DATA:TargetServiceName}\|%{DATA:TargetElement}\|%{DATA:User}\|%{DATA:p_logger}\|%{DATA:p_mdc}\|%{DATA:p_message}\|(?<p_marker>(.|\r|\n)*)" }
-         }
-    } else {
+   kv {
+    source => "Mdc"
+    field_split => ","
+    trim_key => "\s"
+    trim_value => "\s"
+  }
+
          grok {
-            match => { "message" => "(?<unstructuredlog>(.|\r|\n)*)" }
+            match => { "message" => "%{TIMESTAMP_ISO8601:LogTimestamp}\t%{GREEDYDATA:Thread}\t%{LOGLEVEL:Level}\t%{JAVACLASS:Logger}\t%{GREEDYDATA:Marker}\t(?:[^\t]+\t)*(?<Mdc>.*)\t%{GREEDYDATA:Message}\t" }
          }
-    }
+
+
 
 }
 
@@ -176,8 +108,7 @@ output {
        if "database" in [tags]{
        elasticsearch {
        hosts => ["elasticsearch:9200"]
-       index => "testdb"
-       document_type => "%{type}"
+       index => "testdb"       
        }
       }
     else {
index 3621886..a22e350 100644 (file)
@@ -25,5 +25,5 @@
         <!-- docker image name -->
         <artifactId>acumos-logstash</artifactId>
         <!-- docker tag -->
-        <version>2.0.0-SNAPSHOT</version>
+        <version>2.0.4-SNAPSHOT</version>
 </project>
index 6c50999..b42ad2a 100644 (file)
@@ -25,7 +25,7 @@
         <!-- docker image name -->
         <artifactId>acumos-filebeat</artifactId>
         <!-- docker tag -->
-        <version>2.0.0-SNAPSHOT</version>
+        <version>2.0.4-SNAPSHOT</version>
 </project>
 
 
index fa61c5a..19195c0 100644 (file)
@@ -25,7 +25,7 @@
         <!-- docker image name -->
         <artifactId>acumos-metricbeat</artifactId>
         <!-- docker tag -->
-        <version>2.0.0-SNAPSHOT</version>
+        <version>2.0.4-SNAPSHOT</version>
 </project>