EFK架构图

EFK

安装

0.helm

官方地址：https://github.com/elastic/helm-charts

helm init --upgrade
#报错 storage.googleapis.com，使用阿里云的镜像
helm init --upgrade -i registry.cn-hangzhou.aliyuncs.com/google_containers/tiller:v2.5.1 --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts
helm list
helm delete --purge abc
kubectl get pods --namespace=logs -l app=elasticsearch-master -w

1.安装elasticsearch

$ kubectl create ns logs
$ helm repo add elastic https://helm.elastic.co
$ helm fetch elastic/elasticsearch
#修改values.yaml文件中pv为storageClass动态分配
 volumeClaimTemplate:
  accessModes: [ "ReadWriteOnce" ]
  storageClassName: "alicloud-disk-available"
  resources:
    requests:
      storage: 30Gi
$ helm install -n elasticsearch --namespace=logs ./elasticsearch
$ helm status elasticsearch

2.安装kibana

$ helm fetch elastic/kibana
#修改values.yaml文件中service为nodePort类型
 service:
  type: NodePort
  port: 5601
  nodePort: 30056
$ helm install -n kibana  --namespace=logs ./kibana

3.安装filebeat

$ helm fetch elastic/filebeat
#默认读取的是节点/var/lib下的所有文件
# Root directory where Filebeat will write data to in order to persist registry data across pod restarts (file position and other metadata).
hostPathRoot: /var/lib
$ helm install -n filebeat  --namespace=logs ./filebeat

4.在pod种创建filebeat

############################## ybu 
# api
apiVersion: apps/v1beta2
kind: Deployment
metadata:
  name: dc-new-api-ybu
  labels:
    app: dc-new-api-ybu
spec:
  replicas: 1
  selector:
    matchLabels:
      app: dc-new-api-ybu
  template:
    metadata:
      labels:
        app: dc-new-api-ybu
    spec:
      containers:
        - env:  # 这个导致 阿里云上的filebeat起不来
            - name: KUBERNETES_SERVICE_HOST
              value: '0'
          image: 'registry.cn-beijing.aliyuncs.com/filebeat:v2'
          name: filebeat
          volumeMounts:
            - mountPath: /app/logs
              name: log
            - mountPath: /etc/filebeat/
              name: filebeat-config
        - image: 'registry.cn-beijing.aliyuncs.com/dc-new-ybu:latest'
          imagePullPolicy: Always
          name: dc-new-api-ybu
          ports:
            - containerPort: 8001
          volumeMounts:
            - mountPath: /app/logs
              name: log
      volumes:
        - name: log
          emptyDir: {}
        - name: filebeat-config
          configMap:
            name: filebeat-config
      imagePullSecrets:
        - name: docker-image

---
# api service
apiVersion: v1
kind: Service
metadata:
  name: dc-new-api-ybu
spec:
  ports:
    - port: 8001
      protocol: TCP
      targetPort: 8001
  selector:
    app: dc-new-api-ybu
  type: ClusterIP

---
# api route
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: qstyuntu
  namespace: logs
spec:
  rules:
    - host: qst.ali.com
      http:
        paths:
          - backend:
              serviceName: dc-new-api-ybu
              servicePort: 8001
            path: /
---
#filebeat-config
apiVersion: v1
kind: ConfigMap
metadata:
  name: filebeat-config
data:
  filebeat.yml: |
    filebeat.inputs:
    - type: log
      enabled: true
      paths:
        - /app/logs/*.log
      multiline.pattern: ^\[
      multiline.negate: true
      multiline.match: after
    setup.template.settings:
      index.number_of_shards: 1
    setup.kibana:
      host: "kibana-kibana:5601"
    output.elasticsearch:
      hosts: ["elasticsearch-master:9200"]
    processors:
      - add_host_metadata: ~
      - add_cloud_metadata: ~
      - add_docker_metadata: ~
      - add_kubernetes_metadata: ~

上边的filebeat:v2 镜像我放到github上了https://github.com/Qtong121/docker-images

参照
https://www.cnblogs.com/Dev0ps/p/11127417.html
https://www.kubernetes.org.cn/3435.html
https://www.jianshu.com/p/c60cbed8364a
https://www.qikqiak.com/post/install-efk-stack-on-k8s/
https://jimmysong.io/kubernetes-handbook/practice/app-log-collection.html