Logstash 学习笔记

OrangBus大约 2 分钟

gork:https://gitbook.curiouser.top/origin/logstash-filter-grok.html

收集nginx日志

 log_format tcp_format '$time_local|$remote_addr|$protocol|$status|$bytes_sent|$bytes_received|$session_time|$upstream_addr|$upstream_bytes_sent|$upstream_bytes_received|$upstream_connect_time';

nginx日志示例

192.168.2.50 - - [17/Jun/2023:18:02:50 +0800] "GET / HTTP/1.1" 403 548 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
192.168.2.50 - - [17/Jun/2023:18:02:52 +0800] "GET /favicon.ico HTTP/1.1" 404 548 "http://home.com:888/" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
192.168.2.50 - - [19/Jun/2023:09:36:45 +0800] "GET /phpmyadmin_7238930f423aadc9 HTTP/1.1" 301 162 "http://home.com:10086/" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
192.168.2.50 - - [19/Jun/2023:09:36:46 +0800] "GET /phpmyadmin_7238930f423aadc9/ HTTP/1.1" 200 3735 "http://home.com:10086/" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
192.168.2.50 - - [19/Jun/2023:09:36:46 +0800] "GET /phpmyadmin_7238930f423aadc9/js/vendor/codemirror/addon/hint/show-hint.css?v=5.0.4 HTTP/1.1" 200 623 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
192.168.2.50 - - [19/Jun/2023:09:36:46 +0800] "GET /phpmyadmin_7238930f423aadc9/themes/pmahomme/jquery/jquery-ui.css HTTP/1.1" 200 9737 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
192.168.2.50 - - [19/Jun/2023:09:36:46 +0800] "GET /phpmyadmin_7238930f423aadc9/js/vendor/codemirror/addon/lint/lint.css?v=5.0.4 HTTP/1.1" 200 1331 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
192.168.2.50 - - [19/Jun/2023:09:36:46 +0800] "GET /phpmyadmin_7238930f423aadc9/js/vendor/codemirror/lib/codemirror.css?v=5.0.4 HTTP/1.1" 200 2803 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
192.168.2.50 - - [19/Jun/2023:09:36:46 +0800] "GET /phpmyadmin_7238930f423aadc9/themes/pmahomme/css/theme.css?v=5.0.4&nocache=4971196866ltr&server=1 HTTP/1.1" 200 24345 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
192.168.2.50 - - [19/Jun/2023:09:36:46 +0800] "GET /phpmyadmin_7238930f423aadc9/js/vendor/jquery/jquery.min.js?v=5.0.4 HTTP/1.1" 200 34502 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
192.168.2.50 - - [19/Jun/2023:09:36:46 +0800] "GET /phpmyadmin_7238930f423aadc9/js/vendor/sprintf.js?v=5.0.4 HTTP/1.1" 200 2763 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
192.168.2.50 - - [19/Jun/2023:09:36:46 +0800] "GET /phpmyadmin_7238930f423aadc9/js/vendor/jquery/jquery-migrate.js?v=5.0.4 HTTP/1.1" 200 6855 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
192.168.2.50 - - [19/Jun/2023:09:36:46 +0800] "GET /phpmyadmin_7238930f423aadc9/js/ajax.js?v=5.0.4 HTTP/1.1" 200 9863 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
192.168.2.50 - - [19/Jun/2023:09:36:46 +0800] "GET /phpmyadmin_7238930f423aadc9/js/keyhandler.js?v=5.0.4 HTTP/1.1" 200 1158 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
192.168.2.50 - - [19/Jun/2023:09:36:46 +0800] "GET /phpmyadmin_7238930f423aadc9/js/vendor/bootstrap/bootstrap.bundle.min.js?v=5.0.4 HTTP/1.1" 200 25957 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
192.168.2.50 - - [19/Jun/2023:09:36:46 +0800] "GET /phpmyadmin_7238930f423aadc9/js/vendor/js.cookie.js?v=5.0.4 HTTP/1.1" 200 1633 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
192.168.2.50 - - [19/Jun/2023:09:36:46 +0800] "GET /phpmyadmin_7238930f423aadc9/js/vendor/jquery/jquery.event.drag-2.2.js?v=5.0.4 HTTP/1.1" 200 4726 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
192.168.2.50 - - [19/Jun/2023:09:36:46 +0800] "GET /phpmyadmin_7238930f423aadc9/js/vendor/jquery/jquery.validate.js?v=5.0.4 HTTP/1.1" 200 15775 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
192.168.2.50 - - [19/Jun/2023:09:36:46 +0800] "GET /phpmyadmin_7238930f423aadc9/js/vendor/jquery/jquery-ui.min.js?v=5.0.4 HTTP/1.1" 200 79936 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"

logstash 规则

input {
  file {
      path => "/home/orangbus/Docker/elastic/weblogs/nginx/acess/gaobuba.com.log"
      type => "gaobuba_access_log" # 日志
      start_position => "beginning"
    }
    file {
          path => "/home/orangbus/Docker/elastic/weblogs/nginx/acess/gaozhidazhuan.com.log"
          type => "gaozhidazhuan_access_log" # 日志
          start_position => "beginning"
    }
}

filter {
    grok {
        match => {
        # $time_local|$remote_addr|$protocol|$status|$bytes_sent|$bytes_received|$session_time|$upstream_addr|$upstream_bytes_sent|$upstream_bytes_received|$upstream_connect_time
        # 192.168.2.50 - - [14/Jun/2023:14:43:28 +0800] "POST /api/login?phone=18388112576&password=admin666 HTTP/1.1" 500 6628 "-" "PostmanRuntime/7.31.1"
            "message" => '%{IPORHOST:client_ip} - - \[%{HTTPDATE:timestamp}\] \"%{WORD:http_method} %{DATA:request_path} HTTP/%{NUMBER:http_version}\" %{NUMBER:http_status} %{NUMBER:response_size} \"%{DATA:referrer}\" \"%{DATA:user_agent}\"'
        }
    }

}

output {
  stdout  {
    codec => rubydebug
  }
#  elasticsearch {
#    hosts => ["127.0.0.1:9200"]
#    index => "logstash_nginx_access_log"
#    user => "elastic"
#    password => "admin666"
#  }
}

规则设置

nginx

mysql

input {
  file {
      path => "/home/orangbus/Docker/elastic/weblogs/mysql/xjt-2020-slow.log"
      type => "xjt2020_mysql_slow_log" # 日志
      start_position => "beginning"
    }
}

filter {
    grok {
        match => {
            "message" =>[
                "^# Time: %{TIMESTAMP_ISO8601:log_timestamp}$",
                "^# Query_time: %{NUMBER:query_time}  Lock_time: %{NUMBER:lock_time} Rows_sent: %{NUMBER:rows_sent}  Rows_examined: %{NUMBER:rows_examined}$",
                "^SET timestamp＝%{NUMBER:timestamp};$",
                "^SELECT %{GREEDYDATA:query};$"
            ]
        }
    }
}

output {
#  stdout  {
#    codec => rubydebug
#  }
  elasticsearch {
    hosts => ["127.0.0.1:9200"]
    index => "xjt2020_mysql_slow"
    user => "elastic"
    password => "admin666"
  }
}