上篇文章我们说过了EFK 日志系统收集K8s日志之容器标准输出日志，今天我们来谈谈 k8s中EFK收集服务日志

1、收集容器中日志文件

大致思路：

在Pod中增加一个容器运行日志采集器，使用emtyDir共享日志目录让日志采集器读取到业务容器的日志文件

PS: 收集容器中日志文件所需要的Pod Yaml 文件在 https://github.com/fxkjnj/kubernetes/elk-for-kubernetes/es-single-node/app-tomcat-filebeat-log 目录下

编写dockerfile，创建一个标准的tomcat8 镜像

PS: 确保本机有docker 的环境, 如果没有部署docker 可以参考我的另一篇文章 https://www.fxkjnj.com/?p=2732

当然如果不想自己制作镜像，也可以使用我制作好的tomcat8 镜像 docker pull feixiangkeji974907/tomcat-test:v8

创建软件目录，下载tomcat8, jdk1.8

[root@master-1 es-single-node]# mkdir app-tomcat-filebeat-log

[root@master-1 es-single-node]# cd app-tomcat-filebeat-log
[root@master-1 app-tomcat-filebeat-log]# http://jpg.fxkjnj.com/ruanjian/apache-tomcat-8.5.39.tar.gz
[root@master-1 app-tomcat-filebeat-log]# http://jpg.fxkjnj.com/ruanjian/jdk1.8.0_66.tar.gz

编写dockerfile

cat >  Dockerfile  << EOF
FROM centos
MAINTAINER fxkjnj.com fxkj
EXPOSE 8080
WORKDIR /opt

#ADD jdk1.8
    COPY jdk1.8.0_66.tar.gz /opt
    RUN tar zxf /opt/jdk1.8.0_66.tar.gz -C /usr/local/ && rm -rf /opt/jdk1.8.0_66.tar.gz
    RUN ln -s /usr/local/jdk1.8.0_66 /usr/local/jdk
#环境变量/etc/profile
    ENV JAVA_HOME /usr/local/jdk
    ENV CLASSPATH=.:$JAVA_HOME/jre/lib/rt.jar:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
    ENV PATH $PATH:$JAVA_HOME/bin

#ADD tomcat8
    COPY apache-tomcat-8.5.39.tar.gz /opt
    RUN tar zxf apache-tomcat-8.5.39.tar.gz -C /usr/local  && rm -rf apache-tomcat-8.5.39.tar.gz
    RUN mv /usr/local/apache-tomcat-8.5.39 /usr/local/tomcat

#CMD
ENTRYPOINT /usr/local/tomcat/bin/startup.sh && tail -f /usr/local/tomcat/logs/catalina.out
EOF

构建镜像

[root@master-1 app-tomcat-filebeat-log]# docker build -t feixiangkeji974907/tomcat-test:v8 /root/kubernetes/elk-for-kubernetes/es-single-node/app-tomcat-filebeat-log/
Sending build context to Docker daemon  191.2MB
Step 1/14 : FROM centos
latest: Pulling from library/centos
7a0437f04f83: Pull complete 
Digest: sha256:5528e8b1b1719d34604c87e11dcd1c0a20bedf46e83b5632cdeac91b8c04efc1
Status: Downloaded newer image for centos:latest
 ---> 300e315adb2f
Step 2/14 : MAINTAINER fxkjnj.com fxkj
 ---> Running in c6960bcfe61f
Removing intermediate container c6960bcfe61f
 ---> 4d90c5f058e4
Step 3/14 : EXPOSE 8080
 ---> Running in 4b74564852a6
Removing intermediate container 4b74564852a6
 ---> 1d513bed4b8a
Step 4/14 : WORKDIR /opt
 ---> Running in ba66ad1e1f2b
Removing intermediate container ba66ad1e1f2b
 ---> af3d2848cd2a
Step 5/14 : COPY jdk1.8.0_66.tar.gz /opt
 ---> 5407bdfd840e
Step 6/14 : RUN tar zxf /opt/jdk1.8.0_66.tar.gz -C /usr/local/ && rm -rf /opt/jdk1.8.0_66.tar.gz
 ---> Running in 969ef89b2a29
Removing intermediate container 969ef89b2a29
 ---> 84717736fc66
Step 7/14 : RUN ln -s /usr/local/jdk1.8.0_66 /usr/local/jdk
 ---> Running in 3e2a24de56fd
Removing intermediate container 3e2a24de56fd
 ---> 807c98672e7f
Step 8/14 : ENV JAVA_HOME /usr/local/jdk
 ---> Running in c1f21968d26c
Removing intermediate container c1f21968d26c
 ---> a24e93067d43
Step 9/14 : ENV CLASSPATH=.:$JAVA_HOME/jre/lib/rt.jar:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
 ---> Running in 1bc184124271
Removing intermediate container 1bc184124271
 ---> 50e6aa9d66f9
Step 10/14 : ENV PATH $PATH:$JAVA_HOME/bin
 ---> Running in 104d6ee96bfb
Removing intermediate container 104d6ee96bfb
 ---> 7ff4d81f456c
Step 11/14 : COPY apache-tomcat-8.5.39.tar.gz /opt
 ---> 4815155b0c9f
Step 12/14 : RUN tar zxf apache-tomcat-8.5.39.tar.gz -C /usr/local  && rm -rf /opt/apache-tomcat-8.5.39.zip
 ---> Running in b5d13adfbf93
Removing intermediate container b5d13adfbf93
 ---> 49413a5efaed
Step 13/14 : RUN mv /usr/local/apache-tomcat-8.5.39 /usr/local/tomcat
 ---> Running in a2ea891bb8b2
Removing intermediate container a2ea891bb8b2
 ---> 6c71db7365e9
Step 14/14 : ENTRYPOINT /usr/local/tomcat/bin/startup.sh && tail -f /usr/local/tomcat/logs/catalina.out
 ---> Running in f01fa6926b74
Removing intermediate container f01fa6926b74
 ---> 0686065360e3
Successfully built 0686065360e3
Successfully tagged feixiangkeji974907/tomcat-test:v8

测试下镜像，启动容器

 [root@master-1 app-tomcat-filebeat-log]# docker run --name tomcat -itd -p 80:8080 feixiangkeji974907/tomcat-test:v8

访问tomcat: http://192.168.31.61 可以看到首页效果

如果需要替换war包操作。可以将上面制作的tomcat8 镜像为基础镜像，在写一个dockerfile。我这里提供一下

cat >  Dockerfile  << EOF
FROM feixiangkeji974907/tomcat-test:v8
MAINTAINER fxkjnj.com fxkj
COPY app.war /opt
RUN unzip /opt/app.war -d /usr/local/tomcat/webapps/ && rm -rf /opt/app.war
EOF


[root@master-1 app-tomcat-filebeat-log]# docker build -t tomcat-app:v1 .

创建 app-tomcat-log-logfile.yaml 文件，并加入 Filebeat 来收集tomcat容器日志

cat >  app-tomcat-log-logfile.yaml   << EOF
apiVersion: apps/v1
kind: Deployment
metadata:
  name: tomcat-logfile
spec:
  replicas: 3
  selector:
    matchLabels:
      project: tomcat-app
      app: tomcat-logfile
  template:
    metadata:
      labels:
        project: tomcat-app
        app: tomcat-logfile
    spec:
      containers:
      # 应用容器
      - name: tomcat
        image: feixiangkeji974907/tomcat-test:v8
        # 将数据卷挂载到日志目录
        volumeMounts:
        - name: tomcat-logs 
          mountPath: /usr/local/tomcat/logs
      # 日志采集器容器
      - name: filebeat
        image: elastic/filebeat:7.9.2 
        args: [
          "-c", "/etc/filebeat.yml",
          "-e",
        ]
        resources:
          requests:
            cpu: 100m
            memory: 100Mi
          limits:
            memory: 500Mi
        securityContext:
          runAsUser: 0
        volumeMounts:
        # 挂载filebeat配置文件
        - name: filebeat-config
          mountPath: /etc/filebeat.yml
          subPath: filebeat.yml
        # 将数据卷挂载到日志目录
        - name: tomcat-logs 
          mountPath: /usr/local/tomcat/logs
      # 数据卷共享日志目录
      volumes:
      - name: tomcat-logs
        emptyDir: {}
      - name: filebeat-config
        configMap:
          name: filebeat-tomcat-config
---
apiVersion: v1
kind: Service
metadata:
  name: app-log-logfile
spec:
  ports:
  - port: 80
    protocol: TCP
    targetPort: 8080
  selector:
    project: tomcat-app
    app: tomcat-logfile
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: filebeat-tomcat-config
  
data:
  # 配置文件保存在ConfigMap
  filebeat.yml: |-
    filebeat.inputs:
      - type: log
        paths:
          - /usr/local/tomcat/logs/localhost_access_log.*
        # tags: ["access-log"]
        # fields_under_root，如果值为ture，那么fields 字段存储在输出文档的顶级位置，如果与filebeat中字段冲突，自定义字段会覆盖其他字段
        fields_under_root: true
        fields:
          project: tomcat-app
          app: tomcat-logfile
        #自定义ES的索引需要把ilm设置为false
        #定义模板的相关信息
    setup.ilm.enabled: false
    setup.template.name: "tomcat-access"
    setup.template.pattern: "tomcat-access-*"

    output.elasticsearch:
      hosts: ['elasticsearch.ops:9200']
      index: "tomcat-access-%{+yyyy.MM.dd}"
EOF



[root@master-1 app-tomcat-filebeat-log]# kubectl apply -f app-tomcat-log-logfile.yaml 
deployment.apps/tomcat-logfile created
service/app-log-logfile created
configmap/filebeat-tomcat-config created

查看tomcat pod，service 状态

[root@master-1 es-single-node]# kubectl get pods
NAME                              READY   STATUS    RESTARTS   AGE
tomcat-logfile-694d588b78-7k97g   2/2     Running   0          5m36s
tomcat-logfile-694d588b78-phnxt   2/2     Running   0          5m36s
tomcat-logfile-694d588b78-vmp25   2/2     Running   0          5m36s

[root@master-1 es-single-node]# kubectl get svc
NAME              TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)   AGE
app-log-logfile   ClusterIP   10.0.0.194   <none>        80/TCP    5m40s

登陆kibana 管理索引，添加索引模式

索引管理：

（一般只要有数据入到ES中就会有索引出现，如果没有出现可以试着访问下业务使其产生日志输出到ES中）

点击左边的 Stack Management 中的索引管理可以看到一个名词为tomcat-access-2021.03.08的索引，状态为open

添加索引模式:

点击左边的 Stack Management 中的索引模式，创建索引模式

输入索引模式名称：tomcat-access-*

表示可以匹配到上面的索引 tomcat-access-2021.03.08

选择@timestamp 时间字段

访问tomcat 的Pod 使其产生日志

[root@node-1 ~]# curl -I 10.0.0.194
HTTP/1.1 200 
Content-Type: text/html;charset=UTF-8
Transfer-Encoding: chunked
Date: Mon, 08 Mar 2021 10:11:17 GMT