ba34e7c0cfd5bf2c11011b864e3941145b568c8a
[platform-oam.git] / elk-stack / logstash / pipeline / logstash.conf
1 # ===============LICENSE_START=======================================================
2 # Acumos Apache-2.0
3 # ===================================================================================
4 # Copyright (C) 2017-2018 AT&T Intellectual Property & Tech Mahindra. All rights reserved.
5 # ===================================================================================
6 # This Acumos software file is distributed by AT&T and Tech Mahindra
7 # under the Apache License, Version 2.0 (the "License");
8 # you may not use this file except in compliance with the License.
9 # You may obtain a copy of the License at
10 #
11 # http://www.apache.org/licenses/LICENSE-2.0
12 #
13 # This file is distributed on an "AS IS" BASIS,
14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 # See the License for the specific language governing permissions and
16 # limitations under the License.
17 # ===============LICENSE_END=========================================================
18 input {
19
20
21 jdbc {
22            jdbc_driver_library => "${ACUMOS_ELK_JDBC_DRIVER_LIBRARY}"
23        jdbc_driver_class => "${ACUMOS_ELK_JDBC_DRIVER_CLASS}"
24            jdbc_connection_string => "${ACUMOS_ELK_JDBC_CONNECTION_STRING}"
25        jdbc_user => "${ACUMOS_ELK_JDBC_USERNAME}"
26        jdbc_password => "${ACUMOS_ELK_JDBC_PASSWORD}"
27        jdbc_validate_connection => true
28            schedule => "0 * * * *"
29        statement => "SELECT * FROM C_SOLUTION_DOWNLOAD"
30        type => "solution_downloaded"
31        tags => ["database"]
32       }
33 jdbc {
34             jdbc_driver_library => "${ACUMOS_ELK_JDBC_DRIVER_LIBRARY}"
35        jdbc_driver_class => "${ACUMOS_ELK_JDBC_DRIVER_CLASS}"
36            jdbc_connection_string => "${ACUMOS_ELK_JDBC_CONNECTION_STRING}"
37        jdbc_user => "${ACUMOS_ELK_JDBC_USERNAME}"
38        jdbc_password => "${ACUMOS_ELK_JDBC_PASSWORD}"
39        jdbc_validate_connection => true
40            schedule => "* * * * *"
41        statement => "SELECT USER_ID,FIRST_NAME,MIDDLE_NAME,LAST_NAME,ORG_NAME,EMAIL,LOGIN_NAME,ACTIVE_YN,LAST_LOGIN_DATE,CREATED_DATE,MODIFIED_DATE,LOGIN_FAIL_COUNT FROM C_USER"
42        type => "no_of_user"
43        tags => ["database"]
44       }
45 jdbc {
46            jdbc_driver_library => "${ACUMOS_ELK_JDBC_DRIVER_LIBRARY}"
47        jdbc_driver_class => "${ACUMOS_ELK_JDBC_DRIVER_CLASS}"
48            jdbc_connection_string => "${ACUMOS_ELK_JDBC_CONNECTION_STRING}"
49        jdbc_user => "${ACUMOS_ELK_JDBC_USERNAME}"
50        jdbc_password => "${ACUMOS_ELK_JDBC_PASSWORD}"
51        jdbc_validate_connection => true
52            schedule => "0 * * * *"
53        statement => "SELECT * FROM C_SOLUTION_DEPLOYMENT"
54        type => "solution_deployed"
55        tags => ["database"]
56       }
57 jdbc {
58            jdbc_driver_library => "${ACUMOS_ELK_JDBC_DRIVER_LIBRARY}"
59        jdbc_driver_class => "${ACUMOS_ELK_JDBC_DRIVER_CLASS}"
60            jdbc_connection_string => "${ACUMOS_ELK_JDBC_CONNECTION_STRING}"
61        jdbc_user => "${ACUMOS_ELK_JDBC_USERNAME}"
62        jdbc_password => "${ACUMOS_ELK_JDBC_PASSWORD}"
63        jdbc_validate_connection => true
64            schedule => "* * * * *"
65        statement => "SELECT SOLUTION_ID,NAME,DESCRIPTION,USER_ID,ACTIVE_YN,MODEL_TYPE_CD,TOOLKIT_TYPE_CD,CREATED_DATE,MODIFIED_DATE FROM C_SOLUTION"
66        type => "all_solution"
67        tags => ["database"]
68       }
69 jdbc {
70            jdbc_driver_library => "${ACUMOS_ELK_JDBC_DRIVER_LIBRARY}"
71        jdbc_driver_class => "${ACUMOS_ELK_JDBC_DRIVER_CLASS}"
72            jdbc_connection_string => "${ACUMOS_ELK_JDBC_CONNECTION_STRING}"
73        jdbc_user => "${ACUMOS_ELK_JDBC_USERNAME}"
74        jdbc_password => "${ACUMOS_ELK_JDBC_PASSWORD}"
75        jdbc_validate_connection => true
76            schedule => "* * * * *"
77        statement => "SELECT * FROM C_SOLUTION_REV"
78        type => "revision_solution"
79        tags => ["database"]
80       }
81 jdbc {
82            jdbc_driver_library => "${ACUMOS_ELK_JDBC_DRIVER_LIBRARY}"
83        jdbc_driver_class => "${ACUMOS_ELK_JDBC_DRIVER_CLASS}"
84            jdbc_connection_string => "${ACUMOS_ELK_JDBC_CONNECTION_STRING}"
85        jdbc_user => "${ACUMOS_ELK_JDBC_USERNAME}"
86        jdbc_password => "${ACUMOS_ELK_JDBC_PASSWORD}"
87        jdbc_validate_connection => true
88            schedule => "0 * * * *"
89        statement => "SELECT r.REVISION_ID,r.SOLUTION_ID,r.CREATED_DATE,r.MODIFIED_DATE,r.ACCESS_TYPE_CD,r.VALIDATION_STATUS_CD,s.SOLUTION_ID,s.ACTIVE_YN,s.MODEL_TYPE_CD,s.TOOLKIT_TYPE_CD,s.CREATED_DATE,s.MODIFIED_DATE FROM C_SOLUTION s inner join C_SOLUTION_REV r ON s.SOLUTION_ID = r.SOLUTION_ID AND r.ACCESS_TYPE_CD='OR' and s.ACTIVE_YN ='Y'"
90        type => "company_model"
91        tags => ["database"]        
92       }
93 jdbc {
94            jdbc_driver_library => "${ACUMOS_ELK_JDBC_DRIVER_LIBRARY}"
95        jdbc_driver_class => "${ACUMOS_ELK_JDBC_DRIVER_CLASS}"
96            jdbc_connection_string => "${ACUMOS_ELK_JDBC_CONNECTION_STRING}"
97        jdbc_user => "${ACUMOS_ELK_JDBC_USERNAME}"
98        jdbc_password => "${ACUMOS_ELK_JDBC_PASSWORD}"
99        jdbc_validate_connection => true
100            schedule => "0 * * * *"
101        statement => "SELECT r.REVISION_ID,r.SOLUTION_ID,r.CREATED_DATE,r.MODIFIED_DATE,r.ACCESS_TYPE_CD,r.VALIDATION_STATUS_CD,s.SOLUTION_ID,s.ACTIVE_YN,s.MODEL_TYPE_CD,s.TOOLKIT_TYPE_CD,s.CREATED_DATE,s.MODIFIED_DATE FROM C_SOLUTION s inner join C_SOLUTION_REV r ON s.SOLUTION_ID = r.SOLUTION_ID AND r.ACCESS_TYPE_CD='PB' and s.ACTIVE_YN ='Y'"
102        type => "public_model"
103        tags => ["database"]
104       }
105 jdbc {
106            jdbc_driver_library => "${ACUMOS_ELK_JDBC_DRIVER_LIBRARY}"
107        jdbc_driver_class => "${ACUMOS_ELK_JDBC_DRIVER_CLASS}"
108            jdbc_connection_string => "${ACUMOS_ELK_JDBC_CONNECTION_STRING}"
109        jdbc_user => "${ACUMOS_ELK_JDBC_USERNAME}"
110        jdbc_password => "${ACUMOS_ELK_JDBC_PASSWORD}"
111        jdbc_validate_connection => true
112            schedule => "0 * * * *"
113        statement => "SELECT r.REVISION_ID,r.SOLUTION_ID,r.CREATED_DATE,r.MODIFIED_DATE,r.ACCESS_TYPE_CD,r.VALIDATION_STATUS_CD,s.SOLUTION_ID,s.ACTIVE_YN,s.MODEL_TYPE_CD,s.TOOLKIT_TYPE_CD,s.CREATED_DATE,s.MODIFIED_DATE FROM C_SOLUTION s inner join C_SOLUTION_REV r ON s.SOLUTION_ID = r.SOLUTION_ID AND r.ACCESS_TYPE_CD='PR' and s.ACTIVE_YN ='Y'"
114        type => "private_model"
115        tags => ["database"]
116       }
117 jdbc {
118            jdbc_driver_library => "${ACUMOS_ELK_JDBC_DRIVER_LIBRARY}"
119        jdbc_driver_class => "${ACUMOS_ELK_JDBC_DRIVER_CLASS}"
120            jdbc_connection_string => "${ACUMOS_ELK_JDBC_CONNECTION_STRING}"
121        jdbc_user => "${ACUMOS_ELK_JDBC_USERNAME}"
122        jdbc_password => "${ACUMOS_ELK_JDBC_PASSWORD}"
123        jdbc_validate_connection => true
124            schedule => "0 * * * *"
125        statement => "select s.SOLUTION_ID, s.NAME Model_Name, u.First_Name, u.Last_Name, r.ACCESS_TYPE_CD, s.Created_Date, s.Modified_date, s.ACTIVE_YN from C_SOLUTION s, C_USER u, C_SOLUTION_REV r where s.USER_ID = u.USER_ID and s.SOLUTION_ID = r.SOLUTION_ID order by u.Last_Name"
126        type => "solutions"
127        tags => ["database"]
128       }
129 jdbc {
130            jdbc_driver_library => "${ACUMOS_ELK_JDBC_DRIVER_LIBRARY}"
131        jdbc_driver_class => "${ACUMOS_ELK_JDBC_DRIVER_CLASS}"
132            jdbc_connection_string => "${ACUMOS_ELK_JDBC_CONNECTION_STRING}"
133        jdbc_user => "${ACUMOS_ELK_JDBC_USERNAME}"
134        jdbc_password => "${ACUMOS_ELK_JDBC_PASSWORD}"
135        jdbc_validate_connection => true
136            schedule => "0 * * * *"
137        statement => "select solution.name,CONCAT(user.first_name,' ',user.last_name) from c_solution solution inner join c_user user on user.USER_ID = solution.USER_ID"
138        type => "name_model"
139        tags => ["database"]
140       }
141
142
143   beats {
144         port => 5000
145   }
146 }
147 filter {  
148
149     if ([source] =~ /audit.log$/){
150        grok {
151           match => { "message" => "%{DATA:LogTimestamp}\|%{DATA:EntryTimestamp}\|%{DATA:InvokeTimestamp}\|%{DATA:RequestID}\|%{DATA:InvocationID}\|%{DATA:InstanceUUID}\|%{DATA:ServiceInstanceID}\|%{DATA:Thread}\|%{DATA:ServiceName}\|%{DATA:PartnerName}\|%{DATA:StatusCode}\|%{DATA:ResponseCode}\|%{DATA:ResponseDescription}\|%{DATA:level}\|%{DATA:Severity}\|%{DATA:ServerIPAddress}\|%{DATA:ElapsedTime}\|%{DATA:ServerFQDN}\|%{DATA:ClientIPAddress}\|%{DATA:VirtualServerName}\|%{DATA:ContextName}\|%{DATA:TargetEntity}\|%{DATA:TargetServiceName}\|%{DATA:TargetElement}\|%{DATA:User}\|%{DATA:p_logger}\|%{DATA:p_mdc}\|%{DATA:p_message}\|%{DATA:p_marker}" }
152        }
153     } else if ([source] =~ /debug.log$/){
154        grok {
155           match => { "message" => "%{DATA:LogTimestamp}\|%{DATA:EntryTimestamp}\|%{DATA:InvokeTimestamp}\|%{DATA:RequestID}\|%{DATA:InvocationID}\|%{DATA:InstanceUUID}\|%{DATA:ServiceInstanceID}\|%{DATA:Thread}\|%{DATA:ServiceName}\|%{DATA:PartnerName}\|%{DATA:StatusCode}\|%{DATA:ResponseCode}\|%{DATA:ResponseDescription}\|%{DATA:level}\|%{DATA:Severity}\|%{DATA:ServerIPAddress}\|%{DATA:ElapsedTime}\|%{DATA:ServerFQDN}\|%{DATA:ClientIPAddress}\|%{DATA:VirtualServerName}\|%{DATA:ContextName}\|%{DATA:TargetEntity}\|%{DATA:TargetServiceName}\|%{DATA:TargetElement}\|%{DATA:User}\|%{DATA:p_logger}\|%{DATA:p_mdc}\|%{DATA:p_message}\|%{DATA:p_marker}" }
156        }
157     } else if ([source] =~ /error.log$/){
158          grok {
159             match => { "message" => "%{DATA:LogTimestamp}\|%{DATA:EntryTimestamp}\|%{DATA:InvokeTimestamp}\|%{DATA:RequestID}\|%{DATA:InvocationID}\|%{DATA:InstanceUUID}\|%{DATA:ServiceInstanceID}\|%{DATA:Thread}\|%{DATA:ServiceName}\|%{DATA:PartnerName}\|%{DATA:StatusCode}\|%{DATA:ResponseCode}\|%{DATA:ResponseDescription}\|%{DATA:level}\|%{DATA:Severity}\|%{DATA:ServerIPAddress}\|%{DATA:ElapsedTime}\|%{DATA:ServerFQDN}\|%{DATA:ClientIPAddress}\|%{DATA:VirtualServerName}\|%{DATA:ContextName}\|%{DATA:TargetEntity}\|%{DATA:TargetServiceName}\|%{DATA:TargetElement}\|%{DATA:User}\|%{DATA:p_logger}\|%{DATA:p_mdc}\|%{DATA:p_message}\|(?<p_marker>(.|\r|\n)*)" }
160          }
161     } else if ([source] =~ /metrics.log$/){
162          grok {
163             match => { "message" => "%{DATA:LogTimestamp}\|%{DATA:EntryTimestamp}\|%{DATA:InvokeTimestamp}\|%{DATA:RequestID}\|%{DATA:InvocationID}\|%{DATA:InstanceUUID}\|%{DATA:ServiceInstanceID}\|%{DATA:Thread}\|%{DATA:ServiceName}\|%{DATA:PartnerName}\|%{DATA:StatusCode}\|%{DATA:ResponseCode}\|%{DATA:ResponseDescription}\|%{DATA:level}\|%{DATA:Severity}\|%{DATA:ServerIPAddress}\|%{DATA:ElapsedTime}\|%{DATA:ServerFQDN}\|%{DATA:ClientIPAddress}\|%{DATA:VirtualServerName}\|%{DATA:ContextName}\|%{DATA:TargetEntity}\|%{DATA:TargetServiceName}\|%{DATA:TargetElement}\|%{DATA:User}\|%{DATA:p_logger}\|%{DATA:p_mdc}\|%{DATA:p_message}\|(?<p_marker>(.|\r|\n)*)" }
164          }
165     } else {
166          grok {
167             match => { "message" => "(?<unstructuredlog>(.|\r|\n)*)" }
168          }
169     }
170
171 }
172
173
174 output {
175        stdout { codec => json_lines }
176        if "database" in [tags]{
177        elasticsearch {
178        hosts => ["elasticsearch:9200"]
179        index => "testdb"
180        document_type => "%{type}"
181        }
182       }
183     else {
184      elasticsearch {
185        hosts => ["elasticsearch:9200"]
186        index => "logstash"
187            }
188       }
189 }