Elastic: Logstash
설명
- 실시간 파이프라이닝 기능을 갖춘 오픈 소스 데이터 수집 엔진
- 다양한 소스에서 데이터 수집 및 필터 후 다양한 저장소로 출력
- 입력
- https://www.elastic.co/guide/en/logstash/current/input-plugins.html
- 형식이나 복잡성과 관계 없이 데이터 수집 가능
- beats, elasticsearch, exec, file, github, heartbeat, http, imap, jdbc, kafka, log4j, syslog, …
- 필터
- https://www.elastic.co/guide/en/logstash/current/filter-plugins.html
- grok
- 비정형 데이터 분석 후 정형 데이터를 생성
- geoip
- IP 주소에 대한 지리적 정보를 추가
- …
- 출력
- https://www.elastic.co/guide/en/logstash/current/output-plugins.html
- 데이터를 특정 대상으로 전달
- 파이프라인의 마지막 단계
- csv, elasticsearch, email, exec, kafka, stdout, syslog, …
- Persistent Queue
- https://www.elastic.co/guide/en/logstash/current/persistent-queues.html
- 정상/비정상 종료 시 메세지 손실에 대해 최소 1회 전달을 보장
- DLQ
- https://www.elastic.co/guide/en/logstash/current/dead-letter-queues.html
- 처리할 수 없는 이벤트 발생 시 데이터 유실을 방지하기 위해 삭제 대신 DLQ에 저장 가능
- DLQ에 저장된 이벤트 내용으로는 원본 이벤트, 미처리 이유, 큐에 저장된 시간 등이 포함
- 현재 Elasticsearch 출력만 지원
yaml
- filebeat와 metricbeat의 데이터를 받아서 각각 다른 alias로 elasticsearch에 저장
- filebeat는 에러 로그 여부를 판단하여 http 전송
- metricbeat는 정적 매핑을 위해 템플릿 설정
apiVersion: v1
kind: ConfigMap
metadata:
name: logstash-configmap
namespace: default
data:
logstash.yml: |
log.level: debug
http.host: "127.0.0.1"
#path.config: /usr/share/logstash/pipeline
pipeline.workers: 2
pipelines.yml: |
- pipeline.id: main
path.config: "/usr/share/logstash/pipeline/main.yml"
- pipeline.id: filebeat
path.config: "/usr/share/logstash/pipeline/filebeat.yml"
- pipeline.id: filebeat_http
path.config: "/usr/share/logstash/pipeline/filebeat-http.yml"
- pipeline.id: metricbeat
path.config: "/usr/share/logstash/pipeline/metricbeat.yml"
main.yml: |
input {
beats {
port => 5044
}
}
output {
if [agent][type] == "filebeat" {
pipeline {
send_to => filebeat
}
} else if [agent][type] == "metricbeat" {
pipeline {
send_to => metricbeat
}
}
}
filebeat.yml: |
input {
pipeline {
address => filebeat
}
}
output {
if "level=error" in [message] or "[ERROR]" in [message] or [message] =~ /^E\d* *([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9])/ {
pipeline {
send_to => filebeat_http
}
}
elasticsearch {
hosts => ["http://elasticsearch:9200"]
ilm_rollover_alias => "filebeat"
ilm_pattern => "{now/d}-000001"
ilm_policy => "filebeat-ilm-policy"
}
}
filebeat-http.yml: |
input {
pipeline {
address => filebeat_http
}
}
output {
http {
url => "http://127.0.0.1:10000/test"
content_type => "application/json"
http_method => "post"
format => "json"
#mapping => ["field1", "value1", "text", "%{message}"]
#headers => ["Authorization", "xxx"]
message => "%{message}"
pool_max => 1000
pool_max_per_route => 500
}
}
metricbeat.yml: |
input {
pipeline {
address => metricbeat
}
}
output {
elasticsearch {
hosts => ["http://elasticsearch:9200"]
ilm_rollover_alias => "metricbeat-prometheus"
ilm_pattern => "{now/d}-000001"
ilm_policy => "metricbeat-prometheus-ilm-policy"
template => "/usr/share/logstash/template/template.json"
template_name => "metricbeat-prometheus-template"
template_overwrite => true
}
}
---
apiVersion: v1
kind: ConfigMap
metadata:
name: metricbeat-prometheus-template
namespace: default
data:
template.json: |
{
"index_patterns": [
"metricbeat-prometheus-*"
],
"settings" : {
"index" : {
"number_of_shards" : "1",
"number_of_replicas" : "1",
"lifecycle.name": "metricbeat-prometheus-ilm-policy",
"lifecycle.rollover_alias": "metricbeat-prometheus"
}
},
"mappings" : {
"properties" : {
"@timestamp" : {
"type" : "date"
},
"agent" : {
"properties" : {
"ephemeral_id" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"hostname" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"id" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"name" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"type" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"version" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"ecs" : {
"properties" : {
"version" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"event" : {
"properties" : {
"dataset" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"duration" : {
"type" : "long"
},
"module" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"host" : {
"properties" : {
"name" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"metricset" : {
"properties" : {
"name" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"period" : {
"type" : "long"
}
}
},
"prometheus" : {
"properties" : {
"labels" : {
"properties" : {
"area" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"branch" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"breaker" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"build_date" : {
"type" : "date"
},
"build_hash" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"cache" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"cluster" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"cluster_uuid" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"code" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"color" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"device" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"es_client_node" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"es_data_node" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"es_ingest_node" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"es_master_node" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"gc" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"goversion" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"handler" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"host" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"index" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"instance" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"job" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"lucene_version" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"method" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"mount" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"name" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"node" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"path" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"pool" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"quantile" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"revision" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"role" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"shard" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"type" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"url" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"version" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
}
}
},
"service" : {
"properties" : {
"address" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"type" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
}
}
}
}
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: logstash
namespace: default
spec:
replicas: 1
selector:
matchLabels:
app: logstash
template:
metadata:
labels:
app: logstash
spec:
containers:
- name: logstash
image: logstash:7.14.0
ports:
- containerPort: 5044
volumeMounts:
- name: config-volume
mountPath: /usr/share/logstash/config/logstash.yml
subPath: logstash.yml
- name: logstash-pipeline-volume
mountPath: /usr/share/logstash/config/pipelines.yml
subPath: pipelines.yml
- name: main-pipeline-volume
mountPath: /usr/share/logstash/pipeline/main.yml
subPath: main.yml
- name: filebeat-pipeline-volume
mountPath: /usr/share/logstash/pipeline/filebeat.yml
subPath: filebeat.yml
- name: filebeat-http-pipeline-volume
mountPath: /usr/share/logstash/pipeline/filebeat-http.yml
subPath: filebeat-http.yml
- name: metricbeat-pipeline-volume
mountPath: /usr/share/logstash/pipeline/metricbeat.yml
subPath: metricbeat.yml
- name: template-volume
mountPath: /usr/share/logstash/template/template.json
subPath: template.json
volumes:
- name: config-volume
configMap:
name: logstash-configmap
items:
- key: logstash.yml
path: logstash.yml
- name: logstash-pipeline-volume
configMap:
name: logstash-configmap
items:
- key: pipelines.yml
path: pipelines.yml
- name: main-pipeline-volume
configMap:
name: logstash-configmap
items:
- key: main.yml
path: main.yml
- name: filebeat-pipeline-volume
configMap:
name: logstash-configmap
items:
- key: filebeat.yml
path: filebeat.yml
- name: filebeat-http-pipeline-volume
configMap:
name: logstash-configmap
items:
- key: filebeat-http.yml
path: filebeat-http.yml
- name: metricbeat-pipeline-volume
configMap:
name: logstash-configmap
items:
- key: metricbeat.yml
path: metricbeat.yml
- name: template-volume
configMap:
name: metricbeat-prometheus-template
---
apiVersion: v1
kind: Service
metadata:
name: logstash
namespace: default
spec:
selector:
app: logstash
ports:
- protocol: TCP
port: 5044
targetPort: 5044
type: ClusterIP
