[Elastic][observability] Logstash

8 minute read
설명

실시간 파이프라이닝 기능을 갖춘 오픈 소스 데이터 수집 엔진
다양한 소스에서 데이터 수집 및 필터 후 다양한 저장소로 출력
입력
- https://www.elastic.co/guide/en/logstash/current/input-plugins.html
- 형식이나 복잡성과 관계 없이 데이터 수집 가능
- beats, elasticsearch, exec, file, github, heartbeat, http, imap, jdbc, kafka, log4j, syslog, …
필터
- https://www.elastic.co/guide/en/logstash/current/filter-plugins.html
- grok
  - 비정형 데이터 분석 후 정형 데이터를 생성
- geoip
  - IP 주소에 대한 지리적 정보를 추가
- …
출력
- https://www.elastic.co/guide/en/logstash/current/output-plugins.html
- 데이터를 특정 대상으로 전달
- 파이프라인의 마지막 단계
  - csv, elasticsearch, email, exec, kafka, stdout, syslog, …
Persistent Queue
- https://www.elastic.co/guide/en/logstash/current/persistent-queues.html
- 정상/비정상 종료 시 메세지 손실에 대해 최소 1회 전달을 보장
DLQ
- https://www.elastic.co/guide/en/logstash/current/dead-letter-queues.html
- 처리할 수 없는 이벤트 발생 시 데이터 유실을 방지하기 위해 삭제 대신 DLQ에 저장 가능
- DLQ에 저장된 이벤트 내용으로는 원본 이벤트, 미처리 이유, 큐에 저장된 시간 등이 포함
- 현재 Elasticsearch 출력만 지원
yaml

filebeat와 metricbeat의 데이터를 받아서 각각 다른 alias로 elasticsearch에 저장
filebeat는 에러 로그 여부를 판단하여 http 전송
metricbeat는 정적 매핑을 위해 템플릿 설정
 apiVersion: v1
 kind: ConfigMap
 metadata:
   name: logstash-configmap
   namespace: default
 data:
   logstash.yml: |
     log.level: debug
     http.host: "127.0.0.1"
     #path.config: /usr/share/logstash/pipeline
     pipeline.workers: 2
   pipelines.yml: |
     - pipeline.id: main
       path.config: "/usr/share/logstash/pipeline/main.yml"
    
     - pipeline.id: filebeat
       path.config: "/usr/share/logstash/pipeline/filebeat.yml"
    
     - pipeline.id: filebeat_http
       path.config: "/usr/share/logstash/pipeline/filebeat-http.yml"
    
     - pipeline.id: metricbeat
       path.config: "/usr/share/logstash/pipeline/metricbeat.yml"
   main.yml: |
     input {
       beats {
         port => 5044
       }
     }
     output {
       if [agent][type] == "filebeat" {
         pipeline {
           send_to => filebeat
         }
       } else if [agent][type] == "metricbeat" {
         pipeline {
           send_to => metricbeat
         }
       }
     }
   filebeat.yml: |
     input {
       pipeline { 
         address => filebeat
       }
     }
     output {
 	  if "level=error" in [message] or "[ERROR]" in [message] or  [message] =~ /^E\d* *([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9])/ {
         pipeline {
           send_to => filebeat_http
         }
       }
    
       elasticsearch {
         hosts => ["http://elasticsearch:9200"]
         ilm_rollover_alias => "filebeat"
         ilm_pattern => "{now/d}-000001"
         ilm_policy => "filebeat-ilm-policy"
       }
     }
   filebeat-http.yml: |
     input {
       pipeline { 
         address => filebeat_http
       }
     }
     output {
       http {
         url => "http://127.0.0.1:10000/test"
         content_type => "application/json"
         http_method => "post"
         format => "json"
         #mapping => ["field1", "value1", "text", "%{message}"]
         #headers => ["Authorization", "xxx"]
         message => "%{message}"
         pool_max  => 1000
         pool_max_per_route => 500
       }
     }
   metricbeat.yml: |
     input {
       pipeline {
         address => metricbeat
       }
     }
     output {
       elasticsearch {
         hosts => ["http://elasticsearch:9200"]
         ilm_rollover_alias => "metricbeat-prometheus"
         ilm_pattern => "{now/d}-000001"
         ilm_policy => "metricbeat-prometheus-ilm-policy"
         template => "/usr/share/logstash/template/template.json"
         template_name => "metricbeat-prometheus-template"
         template_overwrite => true
       }
     }
    
 ---
 apiVersion: v1
 kind: ConfigMap
 metadata:
   name: metricbeat-prometheus-template
   namespace: default
 data:
   template.json: |
     {
     "index_patterns": [
       "metricbeat-prometheus-*"
     ],
     "settings" : {
       "index" : {
         "number_of_shards" : "1",
         "number_of_replicas" : "1",
         "lifecycle.name": "metricbeat-prometheus-ilm-policy",
         "lifecycle.rollover_alias": "metricbeat-prometheus"
       }
     },
     "mappings" : {
       "properties" : {
         "@timestamp" : {
           "type" : "date"
         },
         "agent" : {
           "properties" : {
             "ephemeral_id" : {
               "type" : "text",
               "fields" : {
                 "keyword" : {
                   "type" : "keyword",
                   "ignore_above" : 256
                 }
               }
             },
             "hostname" : {
               "type" : "text",
               "fields" : {
                 "keyword" : {
                   "type" : "keyword",
                   "ignore_above" : 256
                 }
               }
             },
             "id" : {
               "type" : "text",
               "fields" : {
                 "keyword" : {
                   "type" : "keyword",
                   "ignore_above" : 256
                 }
               }
             },
             "name" : {
               "type" : "text",
               "fields" : {
                 "keyword" : {
                   "type" : "keyword",
                   "ignore_above" : 256
                 }
               }
             },
             "type" : {
               "type" : "text",
               "fields" : {
                 "keyword" : {
                   "type" : "keyword",
                   "ignore_above" : 256
                 }
               }
             },
             "version" : {
               "type" : "text",
               "fields" : {
                 "keyword" : {
                   "type" : "keyword",
                   "ignore_above" : 256
                 }
               }
             }
           }
         },
         "ecs" : {
           "properties" : {
             "version" : {
               "type" : "text",
               "fields" : {
                 "keyword" : {
                   "type" : "keyword",
                   "ignore_above" : 256
                 }
               }
             }
           }
         },
         "event" : {
           "properties" : {
             "dataset" : {
               "type" : "text",
               "fields" : {
                 "keyword" : {
                   "type" : "keyword",
                   "ignore_above" : 256
                 }
               }
             },
             "duration" : {
               "type" : "long"
             },
             "module" : {
               "type" : "text",
               "fields" : {
                 "keyword" : {
                   "type" : "keyword",
                   "ignore_above" : 256
                 }
               }
             }
           }
         },
         "host" : {
           "properties" : {
             "name" : {
               "type" : "text",
               "fields" : {
                 "keyword" : {
                   "type" : "keyword",
                   "ignore_above" : 256
                 }
               }
             }
           }
         },
         "metricset" : {
           "properties" : {
             "name" : {
               "type" : "text",
               "fields" : {
                 "keyword" : {
                   "type" : "keyword",
                   "ignore_above" : 256
                 }
               }
             },
             "period" : {
               "type" : "long"
             }
           }
         },
         "prometheus" : {
           "properties" : {
             "labels" : {
               "properties" : {
                 "area" : {
                   "type" : "text",
                   "fields" : {
                     "keyword" : {
                       "type" : "keyword",
                       "ignore_above" : 256
                     }
                   }
                 },
                 "branch" : {
                   "type" : "text",
                   "fields" : {
                     "keyword" : {
                       "type" : "keyword",
                       "ignore_above" : 256
                     }
                   }
                 },
                 "breaker" : {
                   "type" : "text",
                   "fields" : {
                     "keyword" : {
                       "type" : "keyword",
                       "ignore_above" : 256
                     }
                   }
                 },
                 "build_date" : {
                   "type" : "date"
                 },
                 "build_hash" : {
                   "type" : "text",
                   "fields" : {
                     "keyword" : {
                       "type" : "keyword",
                       "ignore_above" : 256
                     }
                   }
                 },
                 "cache" : {
                   "type" : "text",
                   "fields" : {
                     "keyword" : {
                       "type" : "keyword",
                       "ignore_above" : 256
                     }
                   }
                 },
                 "cluster" : {
                   "type" : "text",
                   "fields" : {
                     "keyword" : {
                       "type" : "keyword",
                       "ignore_above" : 256
                     }
                   }
                 },
                 "cluster_uuid" : {
                   "type" : "text",
                   "fields" : {
                     "keyword" : {
                       "type" : "keyword",
                       "ignore_above" : 256
                     }
                   }
                 },
                 "code" : {
                   "type" : "text",
                   "fields" : {
                     "keyword" : {
                       "type" : "keyword",
                       "ignore_above" : 256
                     }
                   }
                 },
                 "color" : {
                   "type" : "text",
                   "fields" : {
                     "keyword" : {
                       "type" : "keyword",
                       "ignore_above" : 256
                     }
                   }
                 },
                 "device" : {
                   "type" : "text",
                   "fields" : {
                     "keyword" : {
                       "type" : "keyword",
                       "ignore_above" : 256
                     }
                   }
                 },
                 "es_client_node" : {
                   "type" : "text",
                   "fields" : {
                     "keyword" : {
                       "type" : "keyword",
                       "ignore_above" : 256
                     }
                   }
                 },
                 "es_data_node" : {
                   "type" : "text",
                   "fields" : {
                     "keyword" : {
                       "type" : "keyword",
                       "ignore_above" : 256
                     }
                   }
                 },
                 "es_ingest_node" : {
                   "type" : "text",
                   "fields" : {
                     "keyword" : {
                       "type" : "keyword",
                       "ignore_above" : 256
                     }
                   }
                 },
                 "es_master_node" : {
                   "type" : "text",
                   "fields" : {
                     "keyword" : {
                       "type" : "keyword",
                       "ignore_above" : 256
                     }
                   }
                 },
                 "gc" : {
                   "type" : "text",
                   "fields" : {
                     "keyword" : {
                       "type" : "keyword",
                       "ignore_above" : 256
                     }
                   }
                 },
                 "goversion" : {
                   "type" : "text",
                   "fields" : {
                     "keyword" : {
                       "type" : "keyword",
                       "ignore_above" : 256
                     }
                   }
                 },
                 "handler" : {
                   "type" : "text",
                   "fields" : {
                     "keyword" : {
                       "type" : "keyword",
                       "ignore_above" : 256
                     }
                   }
                 },
                 "host" : {
                   "type" : "text",
                   "fields" : {
                     "keyword" : {
                       "type" : "keyword",
                       "ignore_above" : 256
                     }
                   }
                 },
                 "index" : {
                   "type" : "text",
                   "fields" : {
                     "keyword" : {
                       "type" : "keyword",
                       "ignore_above" : 256
                     }
                   }
                 },
                 "instance" : {
                   "type" : "text",
                   "fields" : {
                     "keyword" : {
                       "type" : "keyword",
                       "ignore_above" : 256
                     }
                   }
                 },
                 "job" : {
                   "type" : "text",
                   "fields" : {
                     "keyword" : {
                       "type" : "keyword",
                       "ignore_above" : 256
                     }
                   }
                 },
                 "lucene_version" : {
                   "type" : "text",
                   "fields" : {
                     "keyword" : {
                       "type" : "keyword",
                       "ignore_above" : 256
                     }
                   }
                 },
                 "method" : {
                   "type" : "text",
                   "fields" : {
                     "keyword" : {
                       "type" : "keyword",
                       "ignore_above" : 256
                     }
                   }
                 },
                 "mount" : {
                   "type" : "text",
                   "fields" : {
                     "keyword" : {
                       "type" : "keyword",
                       "ignore_above" : 256
                     }
                   }
                 },
                 "name" : {
                   "type" : "text",
                   "fields" : {
                     "keyword" : {
                       "type" : "keyword",
                       "ignore_above" : 256
                     }
                   }
                 },
                 "node" : {
                   "type" : "text",
                   "fields" : {
                     "keyword" : {
                       "type" : "keyword",
                       "ignore_above" : 256
                     }
                   }
                 },
                 "path" : {
                   "type" : "text",
                   "fields" : {
                     "keyword" : {
                       "type" : "keyword",
                       "ignore_above" : 256
                     }
                   }
                 },
                 "pool" : {
                   "type" : "text",
                   "fields" : {
                     "keyword" : {
                       "type" : "keyword",
                       "ignore_above" : 256
                     }
                   }
                 },
                 "quantile" : {
                   "type" : "text",
                   "fields" : {
                     "keyword" : {
                       "type" : "keyword",
                       "ignore_above" : 256
                     }
                   }
                 },
                 "revision" : {
                   "type" : "text",
                   "fields" : {
                     "keyword" : {
                       "type" : "keyword",
                       "ignore_above" : 256
                     }
                   }
                 },
                 "role" : {
                   "type" : "text",
                   "fields" : {
                     "keyword" : {
                       "type" : "keyword",
                       "ignore_above" : 256
                     }
                   }
                 },
                 "shard" : {
                   "type" : "text",
                   "fields" : {
                     "keyword" : {
                       "type" : "keyword",
                       "ignore_above" : 256
                     }
                   }
                 },
                 "type" : {
                   "type" : "text",
                   "fields" : {
                     "keyword" : {
                       "type" : "keyword",
                       "ignore_above" : 256
                     }
                   }
                 },
                 "url" : {
                   "type" : "text",
                   "fields" : {
                     "keyword" : {
                       "type" : "keyword",
                       "ignore_above" : 256
                     }
                   }
                 },
                 "version" : {
                   "type" : "text",
                   "fields" : {
                     "keyword" : {
                       "type" : "keyword",
                       "ignore_above" : 256
                     }
                   }
                 }
               }
             }
           }
         },
         "service" : {
           "properties" : {
             "address" : {
               "type" : "text",
               "fields" : {
                 "keyword" : {
                   "type" : "keyword",
                   "ignore_above" : 256
                 }
               }
             },
             "type" : {
               "type" : "text",
               "fields" : {
                 "keyword" : {
                   "type" : "keyword",
                   "ignore_above" : 256
                 }
               }
             }
           }
         }
       }
     }
     }
    
 ---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: logstash
   namespace: default
 spec:
   replicas: 1
   selector:
     matchLabels:
       app: logstash
   template:
     metadata:
       labels:
         app: logstash
     spec:
       containers:
         - name: logstash
           image: logstash:7.14.0
           ports:
             - containerPort: 5044
           volumeMounts:
             - name: config-volume
               mountPath: /usr/share/logstash/config/logstash.yml
               subPath: logstash.yml
             - name: logstash-pipeline-volume
               mountPath: /usr/share/logstash/config/pipelines.yml
               subPath: pipelines.yml
             - name: main-pipeline-volume
               mountPath: /usr/share/logstash/pipeline/main.yml
               subPath: main.yml
             - name: filebeat-pipeline-volume
               mountPath: /usr/share/logstash/pipeline/filebeat.yml
               subPath: filebeat.yml
             - name: filebeat-http-pipeline-volume
               mountPath: /usr/share/logstash/pipeline/filebeat-http.yml
               subPath: filebeat-http.yml
             - name: metricbeat-pipeline-volume
               mountPath: /usr/share/logstash/pipeline/metricbeat.yml
               subPath: metricbeat.yml
             - name: template-volume
               mountPath: /usr/share/logstash/template/template.json
               subPath: template.json
       volumes:
         - name: config-volume
           configMap:
             name: logstash-configmap
             items:
               - key: logstash.yml
                 path: logstash.yml
         - name: logstash-pipeline-volume
           configMap:
             name: logstash-configmap
             items:
               - key: pipelines.yml
                 path: pipelines.yml
         - name: main-pipeline-volume
           configMap:
             name: logstash-configmap
             items:
               - key: main.yml
                 path: main.yml
         - name: filebeat-pipeline-volume
           configMap:
             name: logstash-configmap
             items:
               - key: filebeat.yml
                 path: filebeat.yml
         - name: filebeat-http-pipeline-volume
           configMap:
             name: logstash-configmap
             items:
               - key: filebeat-http.yml
                 path: filebeat-http.yml
         - name: metricbeat-pipeline-volume
           configMap:
             name: logstash-configmap
             items:
               - key: metricbeat.yml
                 path: metricbeat.yml
         - name: template-volume
           configMap:
             name: metricbeat-prometheus-template
    
 ---
 apiVersion: v1
 kind: Service
 metadata:
   name: logstash
   namespace: default
 spec:
   selector:
     app: logstash
   ports:
     - protocol: TCP
       port: 5044
       targetPort: 5044
   type: ClusterIP
Share on

Twitter Facebook LinkedIn
chp

[Elastic][observability] Logstash

설명

yaml

Share on

You may also enjoy

[database] PG Back Web

[database] DuckDB

[Kubernetes] External Secrets Operator

[messaging] Centrifugo
