Logstash - apache web log 예제 및 실행

1. 사용법 stdin, stdout

bin/logstash -e 'input { stdin { } } output { stdout { } }'

 

hello

 

-> 

{

  "message" => "hello",

  "@version" => "1",

  "host" => "jh",

  "@timestamp" => 2022-11-07T15:25.14.320Z

}

 

---

/bin/logstash -f ../config/test.conf

 

input {
  stdin { }
}

output {
  stdout { }
}

 

동일하게 실행됨

 

---

weblog-sample.log.zip

6.42MB

https://www.elastic.co/guide/en/logstash/7.10/plugins-filters-grok.html

Grok filter plugin | Logstash Reference [7.10] | Elastic

https://github.com/logstash-plugins/logstash-patterns-core/blob/main/patterns/legacy/httpd

GitHub - logstash-plugins/logstash-patterns-core

HTTPDUSER %{EMAILADDRESS}|%{USER}
HTTPDERROR_DATE %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}

# Log formats
HTTPD_COMMONLOG %{IPORHOST:clientip} %{HTTPDUSER:ident} %{HTTPDUSER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" (?:-|%{NUMBER:response}) (?:-|%{NUMBER:bytes})
HTTPD_COMBINEDLOG %{HTTPD_COMMONLOG} %{QS:referrer} %{QS:agent}

# Error logs
HTTPD20_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{LOGLEVEL:loglevel}\] (?:\[client %{IPORHOST:clientip}\] ){0,1}%{GREEDYDATA:message}
HTTPD24_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[(?:%{WORD:module})?:%{LOGLEVEL:loglevel}\] \[pid %{POSINT:pid}(:tid %{NUMBER:tid})?\]( \(%{POSINT:proxy_errorcode}\)%{DATA:proxy_message}:)?( \[client %{IPORHOST:clientip}:%{POSINT:clientport}\])?( %{DATA:errorcode}:)? %{GREEDYDATA:message}
HTTPD_ERRORLOG %{HTTPD20_ERRORLOG}|%{HTTPD24_ERRORLOG}

# Deprecated
COMMONAPACHELOG %{HTTPD_COMMONLOG}
COMBINEDAPACHELOG %{HTTPD_COMBINEDLOG}

grok 패턴 적용한 conf 파일 

input {
#     stdin { }
    file {
        path => "C:/Users/Admin/Desktop/data/test-sample.log"
        # start_position => "beginning"
    }
}
 
filter {
#   grok {
#     match => { "message" => "%{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] \"%{NOTSPACE:transaction_id}\" \"%{WORD:method} %{DATA:request} HTTP/%{N
# UMBER:httpversion}\" %{NUMBER:response} (?:-|%{NUMBER:bytes})"}
#   }
  grok {
    match => { "message" => "%{HTTPD_COMMONLOG}"}
  }
  date {
    match => [ "timestamp" , "dd/MMM/yyyy:HH:mm:ss Z" ]
  }
}
 
output {
#   elasticsearch {
#         hosts => ["localhost:9200"]
#         index => "apache--%{+YYYY.MM.dd}"
#   }
  stdout {
        codec => rubydebug
  }
}

 

test-sample.log 파일에 데이터를 추가하면 stdout 에 나오게 된다.

1건의 데이터를 확인해보고 추후 데이터 넣어서 

대시보드 까지 만들었으나, 후에 데이터 확인하여 다른 Visualize 생성 해봐야겠다!

 

---

input {
	# echo '메시지' | nc localhost 9200
    # 명령으로 메시지 전달 가능
    #tcp {
    #	port => 9200
    #}
    #beats {
    #	port => 5044	
    #}
    file {
        path => "C:/Users/Admin/Desktop/data/test-sample.log"
        start_position => "beginning"
    }
}

filter {
	mutate {
    	remove_field => ["agent"]
    }
    
    grok {
    	match => { "message" => "%{COMBINEDAPACHELOG}" }
    }
    
    geoip {
    	source => "clientip"
    }
    
    useragent {
    	source => "agent"
        target => "useragent"
    }
    
    mutate {
    	convert => {
        	"bytes" => "integer"
        }
    }
    
    date {
    	match => ["timestamp", "dd/MMM/yyyy:HH:mm:ss Z"]
    }
    
    mutate {
    	remove_field => [ "timestamp", "host", "@version", "agent" ]
    }
}

output {
	stdout {
    	codec => "dots"
    }
    
    elasticsearch {
    	index => "apachelog-%{+yyyy.mm.dd}"
        hosts => ["...:9200"]
        user => "..."
        password => "..."
    }
}

참고