Multi-nodes mode

Multi-nodes mode는 아래 파일들의 내부 항목에 node 클러스터를 설정하기 위한 내용이 포함되어 있다.
향후에 추가될 logstash와의 연동, docker log의 syslog 통합, elasticsearch text analysis를 위한 inverted index mapping 등의 고급 설정들은 Single mode (Collecting and Analysis for syslog via ELK 8.17.x | Basic single mode)로 가이드 할 것이다.

Single mode에 Multi-nodes mode로의 설정을 추가/병합하는 방법으로 간단히 할 수 있을 것 이다.

./.env

# Password for the 'elastic' user (at least 6 characters)
ELASTIC_PASSWORD=changeme
# Password for the 'kibana_system' user (at least 6 characters)
KIBANA_PASSWORD=changeme
# Version of Elastic products
STACK_VERSION=8.17.3
# Set the cluster name
CLUSTER_NAME=docker-cluster
# Set to 'basic' or 'trial' to automatically start the 30-day trial
LICENSE=basic
#LICENSE=trial
# Port to expose Elasticsearch HTTP API to the host
ES_PORT=9200
#ES_PORT=127.0.0.1:9200
# Port to expose Kibana to the host
KIBANA_PORT=5601
#KIBANA_PORT=80
LOGSTASH_PORT=5045
LOGSTASH_SYSLOG_PORT=5140
LOGSTASH_MONITORING_PORT=9601
# Increase or decrease based on the available host memory (in bytes)
# MEM_LIMIT=1073741824 # 1GiB
MEM_LIMIT=2147483648 # 2GiB
# MEM_LIMIT=6442450944 # 6GiB
# Project namespace (defaults to the current folder name if not set)
COMPOSE_PROJECT_NAME=elk-basic

./elasticsearch/Dockerfile 동일

./kibana/Dockerfile 동일

./logstash/Dockerfile 동일

./docker-compose.yml

services:
  setup:
    image: elk/elasticsearch:${STACK_VERSION}
    build:
      context: elasticsearch/
      args:
        ELK_VERSION: ${STACK_VERSION}
    volumes:
      - certs:/usr/share/elasticsearch/config/certs
    user: "0"
    command: >
      bash -c '
        if [ x${ELASTIC_PASSWORD} == x ]; then
          echo "Set the ELASTIC_PASSWORD environment variable in the .env file";
          exit 1;
        elif [ x${KIBANA_PASSWORD} == x ]; then
          echo "Set the KIBANA_PASSWORD environment variable in the .env file";
          exit 1;
        fi;
        if [ ! -f config/certs/ca.zip ]; then
          echo "Creating CA";
          bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip;
          unzip config/certs/ca.zip -d config/certs;
        fi;
        if [ ! -f config/certs/certs.zip ]; then
          echo "Creating certs";
          echo -ne \
          "instances:\n"\
          "  - name: es01\n"\
          "    dns:\n"\
          "      - es01\n"\
          "      - localhost\n"\
          "    ip:\n"\
          "      - 127.0.0.1\n"\
          "  - name: es02\n"\
          "    dns:\n"\
          "      - es02\n"\
          "      - localhost\n"\
          "    ip:\n"\
          "      - 127.0.0.1\n"\
          "  - name: es03\n"\
          "    dns:\n"\
          "      - es03\n"\
          "      - localhost\n"\
          "    ip:\n"\
          "      - 127.0.0.1\n"\
          > config/certs/instances.yml;
          bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key;
          unzip config/certs/certs.zip -d config/certs;
        fi;
        echo "Setting file permissions"
        chown -R root:root config/certs;
        find . -type d -exec chmod 755 \{\} \;;
        find . -type f -exec chmod 644 \{\} \;;
        echo "Waiting for Elasticsearch availability";
        until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done;
        echo "Setting kibana_system password";
        until curl -s -X POST --cacert config/certs/ca/ca.crt -u "elastic:${ELASTIC_PASSWORD}" -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done;
        echo "All done!";
      '
    healthcheck:
      test:
        - "CMD-SHELL"
        - "[ -f config/certs/es01/es01.crt ]"
      interval: 1s
      timeout: 5s
      retries: 120
  es01:
    depends_on:
      setup:
        condition: service_healthy
    image: elk/elasticsearch:${STACK_VERSION}
    build:
      context: elasticsearch/
      args:
        ELK_VERSION: ${STACK_VERSION}
    volumes:
      - certs:/usr/share/elasticsearch/config/certs
      - esdata01:/usr/share/elasticsearch/data
    ports:
      - ${ES_PORT}:9200
    environment:
      node.name: es01
      cluster.name: ${CLUSTER_NAME}
      cluster.initial_master_nodes: es01,es02,es03
      discovery.seed_hosts: es02,es03
      ELASTIC_PASSWORD: ${ELASTIC_PASSWORD}
      bootstrap.memory_lock: "true"
      xpack.security.enabled: "true"
      xpack.security.http.ssl.enabled: "true"
      xpack.security.http.ssl.key: certs/es01/es01.key
      xpack.security.http.ssl.certificate: certs/es01/es01.crt
      xpack.security.http.ssl.certificate_authorities: certs/ca/ca.crt
      xpack.security.transport.ssl.enabled: "true"
      xpack.security.transport.ssl.key: certs/es01/es01.key
      xpack.security.transport.ssl.certificate: certs/es01/es01.crt
      xpack.security.transport.ssl.certificate_authorities: certs/ca/ca.crt
      xpack.security.transport.ssl.verification_mode: certificate
      xpack.license.self_generated.type: ${LICENSE}
      xpack.ml.use_auto_machine_memory_percent: "true"
    mem_limit: ${MEM_LIMIT}
    ulimits:
      memlock:
        soft: -1
        hard: -1
    healthcheck:
      test:
        - CMD-SHELL
        - curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'
      interval: 10s
      timeout: 10s
      retries: 120
  es02:
    depends_on:
      - es01
    image: elk/elasticsearch:${STACK_VERSION}
    build:
      context: elasticsearch/
      args:
        ELK_VERSION: ${STACK_VERSION}
    volumes:
      - certs:/usr/share/elasticsearch/config/certs
      - esdata02:/usr/share/elasticsearch/data
    environment:
      node.name: es02
      cluster.name: ${CLUSTER_NAME}
      cluster.initial_master_nodes: es01,es02,es03
      discovery.seed_hosts: es01,es03
      ELASTIC_PASSWORD: ${ELASTIC_PASSWORD}
      bootstrap.memory_lock: "true"
      xpack.security.enabled: "true"
      xpack.security.http.ssl.enabled: "true"
      xpack.security.http.ssl.key: certs/es02/es02.key
      xpack.security.http.ssl.certificate: certs/es02/es02.crt
      xpack.security.http.ssl.certificate_authorities: certs/ca/ca.crt
      xpack.security.transport.ssl.enabled: "true"
      xpack.security.transport.ssl.key: certs/es02/es02.key
      xpack.security.transport.ssl.certificate: certs/es02/es02.crt
      xpack.security.transport.ssl.certificate_authorities: certs/ca/ca.crt
      xpack.security.transport.ssl.verification_mode: certificate
      xpack.license.self_generated.type: ${LICENSE}
      xpack.ml.use_auto_machine_memory_percent: "true"
    mem_limit: ${MEM_LIMIT}
    ulimits:
      memlock:
        soft: -1
        hard: -1
    healthcheck:
      test:
        - CMD-SHELL
        - curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'
      interval: 10s
      timeout: 10s
      retries: 120
  es03:
    depends_on:
      - es02
    image: elk/elasticsearch:${STACK_VERSION}
    build:
      context: elasticsearch/
      args:
        ELK_VERSION: ${STACK_VERSION}
    volumes:
      - certs:/usr/share/elasticsearch/config/certs
      - esdata03:/usr/share/elasticsearch/data
    environment:
      node.name: es03
      cluster.name: ${CLUSTER_NAME}
      cluster.initial_master_nodes: es01,es02,es03
      discovery.seed_hosts: es01,es02
      ELASTIC_PASSWORD: ${ELASTIC_PASSWORD}
      bootstrap.memory_lock: "true"
      xpack.security.enabled: "true"
      xpack.security.http.ssl.enabled: "true"
      xpack.security.http.ssl.key: certs/es03/es03.key
      xpack.security.http.ssl.certificate: certs/es03/es03.crt
      xpack.security.http.ssl.certificate_authorities: certs/ca/ca.crt
      xpack.security.transport.ssl.enabled: "true"
      xpack.security.transport.ssl.key: certs/es03/es03.key
      xpack.security.transport.ssl.certificate: certs/es03/es03.crt
      xpack.security.transport.ssl.certificate_authorities: certs/ca/ca.crt
      xpack.security.transport.ssl.verification_mode: certificate
      xpack.license.self_generated.type: ${LICENSE}
      xpack.ml.use_auto_machine_memory_percent: "true"
    mem_limit: ${MEM_LIMIT}
    ulimits:
      memlock:
        soft: -1
        hard: -1
    healthcheck:
      test:
        - CMD-SHELL
        - curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'
      interval: 10s
      timeout: 10s
      retries: 120
  kibana:
    depends_on:
      es01:
        condition: service_healthy
      es02:
        condition: service_healthy
      es03:
        condition: service_healthy
    image: elk/kibana:${STACK_VERSION}
    build:
      context: kibana/
      args:
        ELK_VERSION: ${STACK_VERSION}
    volumes:
      - certs:/usr/share/kibana/config/certs
      - kibanadata:/usr/share/kibana/data
    ports:
      - ${KIBANA_PORT}:5601
    environment:
      SERVERNAME: kibana
      ELASTICSEARCH_HOSTS: https://es01:9200
      ELASTICSEARCH_USERNAME: kibana_system
      ELASTICSEARCH_PASSWORD: ${KIBANA_PASSWORD}
      ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES: config/certs/ca/ca.crt
    mem_limit: ${MEM_LIMIT}
    healthcheck:
      test:
        - CMD-SHELL
        - curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'
      interval: 10s
      timeout: 10s
      retries: 120
  logstash:
    depends_on:
      es:
        condition: service_healthy
    image: elk/logstash:${STACK_VERSION}
    build:
      context: logstash/
      args:
        ELK_VERSION: ${STACK_VERSION}
    volumes:
      - certs:/usr/share/logstash/config/certs
      - logstash_pipeline:/usr/share/logstash/pipeline
    ports:
      - ${LOGSTASH_PORT}:5044                   # Beats, Logstash Forwarder, etc.
      - ${LOGSTASH_MONITORING_PORT}:9600        # Logstash Monitoring API
      - ${LOGSTASH_SYSLOG_PORT}:5140
    environment:
      xpack.monitoring.elasticsearch.hosts: "https://es:9200"
      xpack.monitoring.elasticsearch.username: elastic
      xpack.monitoring.elasticsearch.password: changeme
      xpack.monitoring.elasticsearch.ssl.certificate_authority: config/certs/ca/ca.crt
      ELASTICSEARCH_HOSTS: "https://es:9200"
      ELASTICSEARCH_USERNAME: "elastic"
      ELASTICSEARCH_PASSWORD: ${ELASTIC_PASSWORD}
      ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES: config/certs/ca/ca.crt
    mem_limit: ${MEM_LIMIT}
    healthcheck:
      test:
        [
          "CMD-SHELL",
          "curl -s -I http://localhost:9600 | grep -q 'HTTP/1.1 200 OK'",
        ]
      interval: 10s
      timeout: 10s
      retries: 120
volumes:
  certs:
    driver: local
  esdata01:
    driver: local
  esdata02:
    driver: local
  esdata03:
    driver: local
  kibanadata:
    driver: local
  logstash_pipeline:
    driver: local