使用 elk stack监控kubernetes

启动elasticsearch+kibana

为了快速启动这里直接使用docker

docker run -d -v /etc/localtime:/etc/localtime -p 9200:9200 -p 9300:9300 --name=elasticsearch -e "discovery.type=single-node" docker.elastic.co/elasticsearch/elasticsearch:7.4.1
docker run -d -v /etc/localtime:/etc/localtime --link elasticsearch:elasticsearch -p 5601:5601 docker.elastic.co/kibana/kibana:7.4.1

集群安装

创建集群

cat >> kubeadm.config << EOF
apiVersion: kubeadm.k8s.io/v1beta2
bootstrapTokens:
- groups:
 - system:bootstrappers:kubeadm:default-node-token
 token: abcdef.0123456789abcdef
 ttl: 24h0m0s
 usages:
 - signing
 - authentication
kind: InitConfiguration
localAPIEndpoint:
 advertiseAddress: 10.10.8.42
 bindPort: 6443
nodeRegistration:
 criSocket: /var/run/dockershim.sock
 name: 10.10.8.42
 taints:
 - effect: NoSchedule
 key: node-role.kubernetes.io/master
---
apiServer:
 timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta2
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns:
 type: CoreDNS
etcd:
 local:
 dataDir: /var/lib/etcd
imageRepository: registry.aliyuncs.com/google_containers
kind: ClusterConfiguration
kubernetesVersion: v1.16.0
networking:
 dnsDomain: cluster.local
 serviceSubnet: 200.0.0.1/16
 podSubnet: 10.201.0.0/16
controlPlaneEndpoint: "10.10.8.200"
scheduler: {}
EOF
kubeadm init --config=kubeadm.config

设置网络

我们选用了canal插件

kubectl apply -f https://docs.projectcalico.org/v3.8/manifests/canal.yaml

安装kube-state-metrics

kube-state-metrics 用于通过apiserver获取k8s集群及创建对象的状态

git clone https://github.com/kubernetes/kube-state-metrics.git
cd kube-state-metrics/examples/standard/
kubectl apply -f .

安装metricbeat

git clone https://github.com/elastic/beats.git
cd beats/deploy/kubernetes/metricbeat/
# 修改镜像版本
sed -i "s/%VERSION%/7.4.1/g" *
# 修改对应es的host后执行
kubectl apply -f .

安装filebeat

cd beats/deploy/kubernetes/metricbeat/
# 修改对应es的host
TODO
kubectl apply -f .

安装heartbeat

cat >> heartbeat.yaml << EOF
---
apiVersion: v1
kind: ConfigMap
metadata:
 name: heartbeat-deployment-config
 namespace: kube-system
 labels:
 k8s-app: heartbeat
data:
 heartbeat.yml: |-
 heartbeat.autodiscover:
 providers:
 - type: kubernetes
 templates:
 - config:
 - type: icmp
 hosts: ["${data.host}"]
 schedule: '*/5 * * * * * *'
 cloud.auth: ${ELASTIC_CLOUD_AUTH}
 cloud.id: ${ELASTIC_CLOUD_ID}
 output.elasticsearch:
 hosts: ${ELASTICSEARCH_HOSTS}
 username: ${ELASTICSEARCH_USERNAME}
 password: ${ELASTICSEARCH_PASSWORD}
 setup.kibana:
 host: ${KIBANA_HOST}
---
apiVersion: apps/v1beta1
kind: Deployment
metadata:
 name: heartbeat
 namespace: kube-system
 labels:
 k8s-app: heartbeat
spec:
 template:
 metadata:
 labels:
 k8s-app: heartbeat
 spec:
 serviceAccountName: heartbeat
 hostNetwork: true
 dnsPolicy: ClusterFirstWithHostNet
 containers:
 - name: heartbeat
 image: docker.elastic.co/beats/heartbeat:7.2.0
 args: [
 "-c", "/etc/heartbeat.yml",
 "-e",
 ]
 env:
 - name: ELASTIC_CLOUD_ID
 - name: ELASTIC_CLOUD_AUTH
 - name: ELASTICSEARCH_HOSTS
 value: "10.10.8.42"
 - name: KIBANA_HOST
 value: "10.10.8.42"
 - name: ELASTICSEARCH_USERNAME
 value: "admin"
 - name: ELASTICSEARCH_PASSWORD
 value: "admin"
 - name: NODE_NAME
 valueFrom:
 fieldRef:
 fieldPath: spec.nodeName
 securityContext:
 runAsUser: 0
 resources:
 limits:
 memory: 200Mi
 requests:
 cpu: 100m
 memory: 100Mi
 volumeMounts:
 - name: config
 mountPath: /etc/heartbeat.yml
 readOnly: true
 subPath: heartbeat.yml
 volumes:
 - name: config
 configMap:
 defaultMode: 0600
 name: heartbeat-deployment-config
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
 name: heartbeat
subjects:
- kind: ServiceAccount
 name: heartbeat
 namespace: kube-system
roleRef:
 kind: ClusterRole
 name: heartbeat
 apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
 name: heartbeat
 labels:
 k8s-app: heartbeat
rules:
- apiGroups: [""]
 resources:
 - nodes
 - namespaces
 - events
 - pods
 verbs: ["get", "list", "watch"]
- apiGroups: ["extensions"]
 resources:
 - replicasets
 verbs: ["get", "list", "watch"]
- apiGroups: ["apps"]
 resources:
 - statefulsets
 - deployments
 verbs: ["get", "list", "watch"]
- apiGroups:
 - ""
 resources:
 - nodes/stats
 verbs:
 - get
---
apiVersion: v1
kind: ServiceAccount
metadata:
 name: heartbeat
 namespace: kube-system
 labels:
 k8s-app: heartbeat
---
EOF
kubectl apply -f heartbeat.yaml

效果展示

pod列表

pod日志

pod监控

pod网络

​

閱讀更多 有點技術 的文章

關鍵字: Docker Beats DNS