3d15181b9d03329931c018a61e887f973ba9e009
[platform-oam.git] / elk-stack / logstash / pipeline / logstash.conf
1 # ===============LICENSE_START=======================================================
2 # Acumos Apache-2.0
3 # ===================================================================================
4 # Copyright (C) 2017-2018 AT&T Intellectual Property & Tech Mahindra. All rights reserved.
5 # ===================================================================================
6 # This Acumos software file is distributed by AT&T and Tech Mahindra
7 # under the Apache License, Version 2.0 (the "License");
8 # you may not use this file except in compliance with the License.
9 # You may obtain a copy of the License at
10 #
11 # http://www.apache.org/licenses/LICENSE-2.0
12 #
13 # This file is distributed on an "AS IS" BASIS,
14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 # See the License for the specific language governing permissions and
16 # limitations under the License.
17 # ===============LICENSE_END=========================================================
18 input {
19
20
21 jdbc {
22            jdbc_driver_library => "${ACUMOS_ELK_JDBC_DRIVER_LIBRARY:/usr/share/logstash/vendor/bundle/mariadb-java-client-2.1.0.jar}"
23        jdbc_driver_class => "${ACUMOS_ELK_JDBC_DRIVER_CLASS:org.mariadb.jdbc.Driver}"
24            jdbc_connection_string => "${ACUMOS_ELK_JDBC_CONNECTION_STRING:jdbc:mariadb://cognita-ist2-vm01-db.eastus.cloudapp.azure.com:3306/acumosist_1_14_0}"
25        jdbc_user => "${ACUMOS_ELK_JDBC_USERNAME:acumos_ro}"
26        jdbc_password => "${ACUMOS_ELK_JDBC_PASSWORD:istro2017}"
27        jdbc_validate_connection => true
28        statement => "SELECT * FROM C_SOLUTION_DOWNLOAD"
29        type => "solution_downloaded"
30        tags => ["database"]
31       }
32 jdbc {
33             jdbc_driver_library => "${ACUMOS_ELK_JDBC_DRIVER_LIBRARY:/usr/share/logstash/vendor/bundle/mariadb-java-client-2.1.0.jar}"
34        jdbc_driver_class => "${ACUMOS_ELK_JDBC_DRIVER_CLASS:org.mariadb.jdbc.Driver}"
35            jdbc_connection_string => "${ACUMOS_ELK_JDBC_CONNECTION_STRING:jdbc:mariadb://cognita-ist2-vm01-db.eastus.cloudapp.azure.com:3306/acumosist_1_14_0}"
36        jdbc_user => "${ACUMOS_ELK_JDBC_USERNAME:acumos_ro}"
37        jdbc_password => "${ACUMOS_ELK_JDBC_PASSWORD:istro2017}"
38        jdbc_validate_connection => true
39        statement => "SELECT * FROM C_USER"
40        type => "no_of_user"
41        tags => ["database"]
42       }
43 jdbc {
44            jdbc_driver_library => "${ACUMOS_ELK_JDBC_DRIVER_LIBRARY:/usr/share/logstash/vendor/bundle/mariadb-java-client-2.1.0.jar}"
45        jdbc_driver_class => "${ACUMOS_ELK_JDBC_DRIVER_CLASS:org.mariadb.jdbc.Driver}"
46            jdbc_connection_string => "${ACUMOS_ELK_JDBC_CONNECTION_STRING:jdbc:mariadb://cognita-ist2-vm01-db.eastus.cloudapp.azure.com:3306/acumosist_1_14_0}"
47        jdbc_user => "${ACUMOS_ELK_JDBC_USERNAME:acumos_ro}"
48        jdbc_password => "${ACUMOS_ELK_JDBC_PASSWORD:istro2017}"
49        jdbc_validate_connection => true
50        statement => "SELECT * FROM C_SOLUTION_DEPLOYMENT"
51        type => "solution_deployed"
52        tags => ["database"]
53       }
54 jdbc {
55            jdbc_driver_library => "${ACUMOS_ELK_JDBC_DRIVER_LIBRARY:/usr/share/logstash/vendor/bundle/mariadb-java-client-2.1.0.jar}"
56        jdbc_driver_class => "${ACUMOS_ELK_JDBC_DRIVER_CLASS:org.mariadb.jdbc.Driver}"
57            jdbc_connection_string => "${ACUMOS_ELK_JDBC_CONNECTION_STRING:jdbc:mariadb://cognita-ist2-vm01-db.eastus.cloudapp.azure.com:3306/acumosist_1_14_0}"
58        jdbc_user => "${ACUMOS_ELK_JDBC_USERNAME:acumos_ro}"
59        jdbc_password => "${ACUMOS_ELK_JDBC_PASSWORD:istro2017}"
60        jdbc_validate_connection => true
61        statement => "SELECT * FROM C_SOLUTION"
62        type => "all_solution"
63        tags => ["database"]
64       }
65 jdbc {
66            jdbc_driver_library => "${ACUMOS_ELK_JDBC_DRIVER_LIBRARY:/usr/share/logstash/vendor/bundle/mariadb-java-client-2.1.0.jar}"
67        jdbc_driver_class => "${ACUMOS_ELK_JDBC_DRIVER_CLASS:org.mariadb.jdbc.Driver}"
68            jdbc_connection_string => "${ACUMOS_ELK_JDBC_CONNECTION_STRING:jdbc:mariadb://cognita-ist2-vm01-db.eastus.cloudapp.azure.com:3306/acumosist_1_14_0}"
69        jdbc_user => "${ACUMOS_ELK_JDBC_USERNAME:acumos_ro}"
70        jdbc_password => "${ACUMOS_ELK_JDBC_PASSWORD:istro2017}"
71        jdbc_validate_connection => true
72        statement => "SELECT * FROM C_SOLUTION WHERE ACCESS_TYPE_CD='OR'"
73        type => "company_model"
74        tags => ["database"]
75       }
76 jdbc {
77            jdbc_driver_library => "${ACUMOS_ELK_JDBC_DRIVER_LIBRARY:/usr/share/logstash/vendor/bundle/mariadb-java-client-2.1.0.jar}"
78        jdbc_driver_class => "${ACUMOS_ELK_JDBC_DRIVER_CLASS:org.mariadb.jdbc.Driver}"
79            jdbc_connection_string => "${ACUMOS_ELK_JDBC_CONNECTION_STRING:jdbc:mariadb://cognita-ist2-vm01-db.eastus.cloudapp.azure.com:3306/acumosist_1_14_0}"
80        jdbc_user => "${ACUMOS_ELK_JDBC_USERNAME:acumos_ro}"
81        jdbc_password => "${ACUMOS_ELK_JDBC_PASSWORD:istro2017}"
82        jdbc_validate_connection => true
83        statement => "SELECT * FROM C_SOLUTION WHERE ACCESS_TYPE_CD='PB'"
84        type => "public_model"
85        tags => ["database"]
86       }
87 jdbc {
88            jdbc_driver_library => "${ACUMOS_ELK_JDBC_DRIVER_LIBRARY:/usr/share/logstash/vendor/bundle/mariadb-java-client-2.1.0.jar}"
89        jdbc_driver_class => "${ACUMOS_ELK_JDBC_DRIVER_CLASS:org.mariadb.jdbc.Driver}"
90            jdbc_connection_string => "${ACUMOS_ELK_JDBC_CONNECTION_STRING:jdbc:mariadb://cognita-ist2-vm01-db.eastus.cloudapp.azure.com:3306/acumosist_1_14_0}"
91        jdbc_user => "${ACUMOS_ELK_JDBC_USERNAME:acumos_ro}"
92        jdbc_password => "${ACUMOS_ELK_JDBC_PASSWORD:istro2017}"
93        jdbc_validate_connection => true
94        statement => "select solution.name,CONCAT(user.first_name,' ',user.last_name) from c_solution solution inner join c_user user on user.USER_ID = solution.OWNER_ID"
95        type => "name_model"
96        tags => ["database"]
97       }
98
99
100
101
102
103   beats {
104         port => 5000
105         codec => plain {
106                  charset => "ISO-8859-1"
107                  }        
108   }
109 }
110 filter {
111      
112   if ([source] =~ /access.log$/){
113        grok {
114           match => { "message" => "%{TIMESTAMP_ISO8601:timestamp}\|%{WORD:moduleName}\|%{WORD:userAgent}\|%{NOTSPACE:urlDefault}\|%{LOGLEVEL:loglevel}\|\|%{GREEDYDATA:unstructured_data}\|%{SPACE}%{SPACE}(?<actualData>(.|\r|\n)*)" }
115        }
116   }
117   if ([source] =~ /application.log$/){
118        grok {
119           match => { "message" => "%{TIMESTAMP_ISO8601:timestamp}\|\|\|%{NOTSPACE:app}\|\|\|\|%{LOGLEVEL:loglevel}\|\|\|\|\|\|\|%{SPACE}%{GREEDYDATA:actualData}" }
120        }
121   }
122   if ([source] =~ /debug.log$/){
123        grok {
124           match => { "message" => "%{TIMESTAMP_ISO8601:timestamp}\|\|\|%{NOTSPACE:app}\|\|\|\|%{LOGLEVEL:loglevel}\|\|\|\|\|%{SPACE}%{GREEDYDATA:packageName}\|\|%{SPACE}(?<actualData>(.|\r|\n)*)" }
125        }
126   }
127   if ([source] =~ /error.log$/){
128        grok {
129           match => { "message" => "%{TIMESTAMP_ISO8601:timestamp}\|\|\|%{NOTSPACE:app}\|\|\|\|%{LOGLEVEL:loglevel}\|\|\|\|\|%{SPACE}%{GREEDYDATA:packageName}\|\|%{SPACE}(?<actualData>(.|\r|\n)*)" }
130        }
131   }
132
133 }
134
135
136 output {
137        stdout { codec => json_lines }
138        if "database" in [tags]{
139        elasticsearch {
140        hosts => ["elasticsearch:9200"]
141        index => "testdb"
142        document_type => "%{type}"
143        }
144       }
145     else {
146      elasticsearch {
147        hosts => ["elasticsearch:9200"]
148        index => "logstash"
149            }
150       }
151 }