Docker容器搭建ELK日志分析系统

文章目录

Docker容器搭建ELK日志分析系统

资源列表

操作系统	配置	主机名	IP	所需软件
CentOS 7.9	2C4G	elk	192.168.93.165	Docker 26.1.2

基础环境

关闭防火墙

systemctl stop firewalld
systemctl disable firewalld

关闭内核安全机制

setenforce 0
sed -i "s/^SELINUX=.*/SELINUX=disabled/g" /etc/selinux/config

修改主机名

hostnamectl set-hostname elk

一、创建容器网络

[root@elk ~]# docker network create elk
44c014de1c02291f28ebcdc734e4bfb820e2a1e2aa61b2758517af60661ee4d7
[root@elk ~]# docker network list | grep elk
44c014de1c02   elk       bridge    local

二、创建容器挂载目录

[root@elk ~]# mkdir -p /mnt/{elasticsearch,logstash,kibana}

三、构建systemctl镜像

目的：为了更改管理容器中的ELK服务

# 创建工作目录
[root@elk ~]# mkdir systemctl
[root@elk ~]# cd systemctl/
[root@elk systemctl]# cat Dockerfile 
# 指定基础镜像
FROM centos:7
ENV container docker
RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == \
systemd-tmpfiles-setup.service ] || rm -f $i; done); \
rm -f /lib/systemd/system/multi-user.target.wants/*; \
rm -f /etc/systemd/system/*.wants/*; \
rm -f /lib/systemd/system/local-fs.target.wants/*; \
rm -f /lib/systemd/system/sockets.target.wants/*udev*; \
rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \
rm -f /lib/systemd/system/basic.target.wants/*; \
rm -f /lib/systemd/system/anaconda.target.wants/*;
# 在容器创建一个挂载点，这个挂载点将会跟宿主机或者别的容器交互
VOLUME ["/sys/fs/cgroup"]

# 构建镜像
[root@elk systemctl]# docker build -t systemctl:elk .

三、构建Elasticsearch镜像

存储日志数据

3.1、构建Elasticsearch

# 创建ELK工作目录
[root@elk ~]# mkdir elk
[root@elk ~]# cd elk/
# 创建工作目录
[root@elk elk]# mkdir elasticsearch
[root@elk elk]# cd elasticsearch/
[root@elk elasticsearch]# cat Dockerfile 
# 指定基础镜像
FROM systemctl:elk
# 安装java环境
COPY jdk-8u202-linux-x64.rpm /root
RUN rpm -ivh /root/jdk-8u202-linux-x64.rpm
RUN echo "export JAVA_HOME=/usr/java/jdk1.8.0_202-amd64/" >> /etc/profile && echo "export CLASSPATH=$JAVA_HOME/lib/tools.jar:$JAVA_HOME/lib/dt.jar" >> /etc/profile && echo "export PATH=$JAVA_HOME/bin:$PATH" >> /etc/profile
RUN source /etc/profile
# 安装elasticsearch
COPY elasticsearch-5.5.0.rpm /root
RUN rpm -ivh /root/elasticsearch-5.5.0.rpm
COPY elasticsearch.yml /etc/elasticsearch/
RUN mkdir -p /data/elk_data && chown -R elasticsearch:elasticsearch /data/elk_data/ && 

# ES配置文件
[root@elk elasticsearch]# grep -v "^#" elasticsearch.yml
cluster.name: es
node.name: ES1
path.data: /data/elk_data
path.logs: /var/log/elasticsearch/
bootstrap.memory_lock: false
network.host: 0.0.0.0
http.port: 9200
discovery.zen.ping.unicast.hosts: ["ES1"]

3.2、构建镜像

[root@elk elasticsearch]# ls
Dockerfile               elasticsearch.yml
elasticsearch-5.5.0.rpm  jdk-8u202-linux-x64.rpm
[root@elk elasticsearch]# docker build -t elasticsearch:latest .

3.3、启动容器

[root@elk elasticsearch]# docker run -d --name elasticsearch -p 9200:9200 -p 9300:9300 --network elk -v /mnt/elasticsearch:/mnt/elasticsearch --privileged -v /sys/fs/cgroup:/sys/fs/cgroup:ro elasticsearch:latest /sbin/init

3.4、进入容器

[root@elk elasticsearch]# docker exec -it elasticsearch bash
[root@41f5c697c0c3 ~]# chown -R elasticsearch:elasticsearch elasticsearch.yml
[root@41f5c697c0c3 /]# systemctl start elasticsearch
[root@41f5c697c0c3 /]# systemctl enable elasticsearch
[root@41f5c697c0c3 /]# yum -y install net-tools
[root@41f5c697c0c3 ~]# netstat -anpt | grep 9200
tcp6       0      0 :::9200                 :::*                    LISTEN      168/java            
[root@41f5c697c0c3 ~]# netstat -anpt | grep 9300
tcp6       0      0 :::9300

3.5、查看节点信息

# 查看节点信息
[root@elk ~]# curl http://192.168.93.165:9200
{
  "name" : "ES1",
  "cluster_name" : "es",
  "cluster_uuid" : "qtJ4glpZQSuJFqU7zSOR1w",
  "version" : {
    "number" : "5.5.0",
    "build_hash" : "260387d",
    "build_date" : "2017-06-30T23:16:05.735Z",
    "build_snapshot" : false,
    "lucene_version" : "6.6.0"
  },
  "tagline" : "You Know, for Search"
}

# 查看群集健康状况，status直为green，表示节点健康运行
[root@elk ~]# curl -XGET 'http://192.168.93.165:9200/_cluster/health?pretty'
{
  "cluster_name" : "es",
  "status" : "green",
  "timed_out" : false,
  "number_of_nodes" : 1,
  "number_of_data_nodes" : 1,
  "active_primary_shards" : 0,
  "active_shards" : 0,
  "relocating_shards" : 0,
  "initializing_shards" : 0,
  "unassigned_shards" : 0,
  "delayed_unassigned_shards" : 0,
  "number_of_pending_tasks" : 0,
  "number_of_in_flight_fetch" : 0,
  "task_max_waiting_in_queue_millis" : 0,
  "active_shards_percent_as_number" : 100.0
}

四、构建Logstash镜像

收集日志、处理日志、输出日志（把处理好的日志输出给Elasticsearch）

4.1、构建Logstash镜像

[root@elk ~]# cd elk/
[root@elk elk]# mkdir logstash
[root@elk elk]# cd logstash/
[root@elk logstash]# cat Dockerfile 
FROM systemctl:elk
# 安装java环境
COPY jdk-8u202-linux-x64.rpm /root
RUN rpm -ivh /root/jdk-8u202-linux-x64.rpm && echo "export JAVA_HOME=/usr/java/jdk1.8.0_202-amd64/" >> /etc/profile && echo "export CLASSPATH=$JAVA_HOME/lib/tools.jar:$JAVA_HOME/lib/dt.jar" >> /etc/profile && echo "export PATH=$JAVA_HOME/bin:$PATH" >> /etc/profile && source /etc/profile
# 安装logstash
COPY logstash-5.5.1.rpm /root/
RUN rpm -ivh /root/logstash-5.5.1.rpm && ln -s /usr/share/logstash/bin/logstash /usr/local/bin/

4.2、构建镜像

[root@elk logstash]# ls
Dockerfile  jdk-8u202-linux-x64.rpm  logstash-5.5.1.rpm
[root@elk logstash]# docker build -t logstash .

4.3、启动容器

# 将/var/log/httpd目录挂载到容器中的/var/log/httpd中，等下会安装httpd服务，logstash会收集/var/log/httpd中的日志
[root@elk logstash]# docker run -d --network elk --privileged -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /mnt/logstash:/mnt/logstash -v /var/log/httpd:/var/log/httpd --name logstash logstash:latest /sbin/init
[root@ec3e7bdf85c2 ~]# systemctl start logstash
[root@ec3e7bdf85c2 ~]# systemctl enable logstash

4.4、进入容器收集日志

4.4.1、安装Apache

在宿主机上安装Apache服务

[root@elk ~]# yum -y install httpd
[root@elk ~]# systemctl start httpd && systemctl enable httpd

# 多访问几次，让httpd访问日志有内容
[root@elk ~]# curl 127.0.0.1
[root@elk ~]# curl 127.0.0.1
[root@elk ~]# curl 127.0.0.1
[root@elk ~]# curl 127.0.0.1

# 查看日志内容
[root@elk ~]# cat /var/log/httpd/access_log 
127.0.0.1 - - [04/Jun/2024:05:17:37 -0400] "GET / HTTP/1.1" 403 4897 "-" "curl/7.29.0"
127.0.0.1 - - [04/Jun/2024:05:17:37 -0400] "GET / HTTP/1.1" 403 4897 "-" "curl/7.29.0"
127.0.0.1 - - [04/Jun/2024:05:17:37 -0400] "GET / HTTP/1.1" 403 4897 "-" "curl/7.29.0"
127.0.0.1 - - [04/Jun/2024:05:17:38 -0400] "GET / HTTP/1.1" 403 4897 "-" "curl/7.29.0"
127.0.0.1 - - [04/Jun/2024:05:17:38 -0400] "GET / HTTP/1.1" 403 4897 "-" "curl/7.29.0"
127.0.0.1 - - [04/Jun/2024:05:17:38 -0400] "GET / HTTP/1.1" 403 4897 "-" "curl/7.29.0"
127.0.0.1 - - [04/Jun/2024:05:17:39 -0400] "GET / HTTP/1.1" 403 4897 "-" "curl/7.29.0"

4.4.2、收集Apache日志

进入容器收集日志

[root@elk logstash]# docker exec -it logstash bash

# 创建收集日志文件，将收集到的日志，输出给192.168.93.165:9200端口，也就是ES服务
[root@ec3e7bdf85c2 /]# cd /etc/logstash/conf.d/
[root@ec3e7bdf85c2 conf.d]# cat apache_log.conf 
input {
	file {
	    # 收集Apache访问日志
		path => "/var/log/httpd/access_log"
		# 类型指定为access
		type => "access"
		# 从开始处收集
		start_position => "beginning"
	}
	file {
	    # 收集Apache错误日志
		path => "/var/log/httpd/error_log"
		# 类型指定为error
		type => "error"
		# 从开始处收集
		start_position => "beginning"
	}
}
output{
    # 如果类型为access，即Apache访问日志
	if [type] == "access" {
	    # 输出到elasticsearch
		elasticsearch {
		    # elasticsearch监听地址及端口
			hosts => ["192.168.93.165:9200"]
			# 指定索引格式
			index => "apache_access-%{+YYYY.MM.dd}"
			}
		}
    # 如果类型为error，即Apache错误日志
	if [type] == "error" {
	     # 输出到elasticsearch
		elasticsearch {
		    # elasticsearch监听地址及端口
			hosts => ["192.168.93.165:9200"]
			# 指定索引格式
			index => "apache_error-%{+YYYY.MM.dd}"
			}
		}
}

# 开始收集日志，回显是前台运行的，输出之后ctrl+c终止即可
[root@ec3e7bdf85c2 ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/apache_log.conf

查看是否在Elasticsearch中创建索引

# 索引创建成功，里面是apache的访问日志和错误日志内容
[root@elk ~]# curl -XGET "localhost:9200/_cat/indices?v"
health status index                    uuid                   pri rep docs.count docs.deleted store.size pri.store.size
yellow open   apache_error-2024.06.04  aMWtWv-OStmiKOvnUz2btQ   5   1         12            0     45.3kb         45.3kb
yellow open   apache_access-2024.06.04 9UaofAv1T5GPWyANz5D55w   5   1          7            0       28kb           28kb

五、构建Kibana镜像

作用：以Web的方式展示日志

5.1、构建Kibana镜像

[root@elk ~]# cd elk/
[root@elk elk]# mkdir kibana
[root@elk elk]# cd kibana/
[root@elk kibana]# cat Dockerfile 
FROM systemctl:elk
# 安装java环境
COPY jdk-8u202-linux-x64.rpm /root
RUN rpm -ivh /root/jdk-8u202-linux-x64.rpm
RUN echo "export JAVA_HOME=/usr/java/jdk1.8.0_202-amd64/" >> /etc/profile && echo "export CLASSPATH=$JAVA_HOME/lib/tools.jar:$JAVA_HOME/lib/dt.jar" >> /etc/profile && echo "export PATH=$JAVA_HOME/bin:$PATH" >> /etc/profile
RUN source /etc/profile
# 安装kibana
COPY kibana-5.5.1-x86_64.rpm /root
RUN rpm -ivh /root/kibana-5.5.1-x86_64.rpm
COPY kibana.yml /etc/kibana/
EXPOSE 5601

[root@elk kibana]# grep -v "#" kibana.yml | grep -v "^?" | grep -v "^$"
server.port: 5601
server.host: "0.0.0.0"
elasticsearch.url: "http://192.168.93.165:9200"
kibana.index: ".kibana"

5.2、构建镜像

[root@elk kibana]# ls                                                        
Dockerfile  jdk-8u202-linux-x64.rpm  kibana-5.5.1-x86_64.rpm  kibana.yml
[root@elk kibana]# docker build -t kibana .

5.3、启动容器

[root@elk kibana]# docker run -d --name kibana -p 5601:5601 --privileged -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /mnt/kibana:/mnt/kibana kibana:latest /sbin/init

5.4、进入容器

[root@elk kibana]# docker exec -it kibana bash
[root@2db31a060306 /]# systemctl start kibana.service 
[root@2db31a060306 /]# yum -y install net-tools
[root@2db31a060306 /]# netstat -anpt | grep 5601
tcp        0      0 0.0.0.0:5601            0.0.0.0:*               LISTEN      54/node