k8s部署高可用etcd集群（SSL）

创建etcd命名空间

kubectl create ns etcd

生成etcd证书secret

下载cfssl二进制包

mkdir -p /home/etcd/ssl && cd /home/etcd/ssl
wget https://github.com/cloudflare/cfssl/releases/download/v1.6.4/cfssl_1.6.4_linux_amd64
wget https://github.com/cloudflare/cfssl/releases/download/v1.6.4/cfssljson_1.6.4_linux_amd64
wget https://github.com/cloudflare/cfssl/releases/download/v1.6.4/cfssl-certinfo_1.6.4_linux_amd64
mv cfssl_1.6.4_linux_amd64  /usr/bin/cfssl
mv cfssljson_1.6.4_linux_amd64 /usr/bin/cfssljson
mv cfssl-certinfo_1.6.4_linux_amd64 /usr/bin/cfssl-certinfo
chmod +x /usr/bin/cfssl*

生成ssl证书

# 证书域名可根据命名空间自行修改
cat > ca-csr.json <<EOF
{
  "CN": "Kubernetes",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "Beijing",
      "L": "Beijing",
      "O": "Kubernetes",
      "OU": "CA"
    }
  ]
}
EOF

cfssl gencert -initca ca-csr.json | cfssljson -bare ca

cat > ca-config.json <<EOF
{
  "signing": {
    "default": {
      "expiry": "876000h"
    },
    "profiles": {
      "kubernetes": {
        "usages": [
          "signing",
          "key encipherment",
          "server auth",
          "client auth"
        ],
        "expiry": "876000h"
      }
    }
  }
}
EOF

cat > etcd-csr.json <<EOF
{
  "CN": "etcd",
  "hosts": [
    "*.etcd-headless",
    "*.etcd-headless.etcd",
    "*.etcd-headless.etcd.svc",
    "*.etcd-headless.etcd.svc.cluster",
    "*.etcd-headless.etcd.svc.cluster.local",
    "*.etcd",
    "*.etcd.svc",
    "*.etcd.svc.cluster",
    "*.etcd.svc.cluster.local"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "Beijing",
      "L": "Beijing",
      "O": "etcd",
      "OU": "etcd"
    }
  ]
}
EOF

cfssl gencert \
  -ca=ca.pem \
  -ca-key=ca-key.pem \
  -config=ca-config.json \
  -profile=kubernetes \
  etcd-csr.json | cfssljson -bare etcd

创建ssl证书secret密钥

kubectl -n etcd create secret generic etcd-certs --from-file=cert.pem=etcd.pem --from-file=key.pem=etcd-key.pem --from-file=ca.crt=ca.pem

配置etcd

添加repo源

helm repo add bitnami https://charts.bitnami.com/bitnami
helm repo update

配置values.yaml

cd /home/etcd/
helm pull bitnami/etcd
tar zxvf etcd-9.5.0.tgz
vim etcd/values.yaml   # 修改如下即可
---
  client:
    secureTransport: true # 启动 https，karmada 只通过 https 连接 etcd
    enableAuthentication: true
    useAutoTLS: false
    existingSecret: etcd-certs # 我们创建的 etcd 相关证书保存在这个 secret 中
    caFilename: "ca.crt"
  peer:
    secureTransport: true
    useAutoTLS: false
    existingSecret: etcd-certs
    enableAuthentication: true
    caFilename: "ca.crt"
initialClusterState: "new"  # 不配置该项，会导致单个节点重启后报错etcdserver: member not found
replicaCount: 3
resources:  # 配置第一个resources即可，initcontainer和cronjob容器可不配置资源限制
  requests:
    cpu: 100m
    memory: 512Mi
  limits:
    cpu: 500m
    memory: 1Gi
persistence:
  enabled: true # 启用数据持久化
  storageClass: "csi-cbs" # 修改为自己的sc
  size: 10Gi
---

# 如报错无法识别seccompProfile，则删除以下内容
  seccompProfile:
    type: RuntimeDefault

部署etcd集群

cd /home/etcd/
helm upgrade --install etcd ./etcd -f ./etcd/values.yaml --namespace etcd

kubectl get pod -n etcd
NAME     READY   STATUS    RESTARTS   AGE
etcd-0   1/1     Running   0          29m
etcd-1   1/1     Running   0          29m
etcd-2   1/1     Running   0          24m

验证etcd集群状态

初步验证

kubectl exec -it etcd-0 -n etcd sh

$ etcdctl  --user=root:$ETCD_ROOT_PASSWORD --endpoints=https://etcd-0.etcd-headless.etcd.svc.cluster.local:2379,https://etcd-1.etcd-headless.etcd.svc.cluster.local:2379,https://etcd-2.etcd-headless.etcd.svc.cluster.local:2379 --cacert=/opt/bitnami/etcd/certs/client/ca.crt --cert=/opt/bitnami/etcd/certs/client/cert.pem --key=/opt/bitnami/etcd/certs/client/key.pem endpoint health -w table
+----------------------------------------------------------+--------+------------+-------+
|                         ENDPOINT                         | HEALTH |    TOOK    | ERROR |
+----------------------------------------------------------+--------+------------+-------+
| https://etcd-2.etcd-headless.etcd.svc.cluster.local:2379 |   true |  1.40932ms |       |
| https://etcd-1.etcd-headless.etcd.svc.cluster.local:2379 |   true | 1.627304ms |       |
| https://etcd-0.etcd-headless.etcd.svc.cluster.local:2379 |   true | 1.379927ms |       |
+----------------------------------------------------------+--------+------------+-------+

$ etcdctl  --user=root:$ETCD_ROOT_PASSWORD --endpoints=https://etcd-0.etcd-headless.etcd.svc.cluster.local:2379,https://etcd-1.etcd-headless.etcd.svc.cluster.local:2379,https://etcd-2.etcd-headless.etcd.svc.cluster.local:2379 --cacert=/opt/bitnami/etcd/certs/client/ca.crt --cert=/opt/bitnami/etcd/certs/client/cert.pem --key=/opt/bitnami/etcd/certs/client/key.pem member list -w table
+------------------+---------+--------+----------------------------------------------------------+---------------------------------------------------------------------------------------------------+------------+
|        ID        | STATUS  |  NAME  |                        PEER ADDRS                        |                                           CLIENT ADDRS                                            | IS LEARNER |
+------------------+---------+--------+----------------------------------------------------------+---------------------------------------------------------------------------------------------------+------------+
| 353f075dede680ca | started | etcd-1 | https://etcd-1.etcd-headless.etcd.svc.cluster.local:2380 | https://etcd-1.etcd-headless.etcd.svc.cluster.local:2379,https://etcd.etcd.svc.cluster.local:2379 |      false |
| 383cd4868f3344c8 | started | etcd-2 | https://etcd-2.etcd-headless.etcd.svc.cluster.local:2380 | https://etcd-2.etcd-headless.etcd.svc.cluster.local:2379,https://etcd.etcd.svc.cluster.local:2379 |      false |
| 5d62df9077746deb | started | etcd-0 | https://etcd-0.etcd-headless.etcd.svc.cluster.local:2380 | https://etcd-0.etcd-headless.etcd.svc.cluster.local:2379,https://etcd.etcd.svc.cluster.local:2379 |      false |
+------------------+---------+--------+----------------------------------------------------------+---------------------------------------------------------------------------------------------------+------------+

$ etcdctl  --user=root:$ETCD_ROOT_PASSWORD --endpoints=https://etcd-0.etcd-headless.etcd.svc.cluster.local:2379,https://etcd-1.etcd-headless.etcd.svc.cluster.local:2379,https://etcd-2.etcd-headless.etcd.svc.cluster.local:2379 --cacert=/opt/bitnami/etcd/certs/client/ca.crt --cert=/opt/bitnami/etcd/certs/client/cert.pem --key=/opt/bitnami/etcd/certs/client/key.pem endpoint status -w table
+----------------------------------------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
|                         ENDPOINT                         |        ID        | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
+----------------------------------------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| https://etcd-0.etcd-headless.etcd.svc.cluster.local:2379 | 5d62df9077746deb |   3.5.9 |   20 kB |     false |      false |         7 |        814 |                814 |        |
| https://etcd-1.etcd-headless.etcd.svc.cluster.local:2379 | 353f075dede680ca |   3.5.9 |   20 kB |     false |      false |         7 |        815 |                815 |        |
| https://etcd-2.etcd-headless.etcd.svc.cluster.local:2379 | 383cd4868f3344c8 |   3.5.9 |   20 kB |      true |      false |         7 |        816 |                816 |        |
+----------------------------------------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
# 当前leader是etcd-2

故障自愈测试

# 删除当前leader节点pod
kubectl delete pod etcd-2 -n etcd

# 查看pod状态
kubectl get pod -n etcd
NAME     READY   STATUS    RESTARTS   AGE
etcd-0   1/1     Running   0          34m
etcd-1   1/1     Running   0          34m
etcd-2   1/1     Running   0          2m46s

#查看当前集群信息
$ etcdctl  --user=root:$ETCD_ROOT_PASSWORD --endpoints=https://etcd-0.etcd-headless.etcd.svc.cluster.local:2379,https://etcd-1.etcd-headless.etcd.svc.cluster.local:2379,https://etcd-2.etcd-headless.etcd.svc.cluster.local:2379 --cacert=/opt/bitnami/etcd/certs/client/ca.crt --cert=/opt/bitnami/etcd/certs/client/cert.pem --key=/opt/bitnami/etcd/certs/client/key.pem endpoint health -w table
+----------------------------------------------------------+--------+-------------+-------+
|                         ENDPOINT                         | HEALTH |    TOOK     | ERROR |
+----------------------------------------------------------+--------+-------------+-------+
| https://etcd-1.etcd-headless.etcd.svc.cluster.local:2379 |   true | 71.681742ms |       |
| https://etcd-0.etcd-headless.etcd.svc.cluster.local:2379 |   true |  9.803507ms |       |
| https://etcd-2.etcd-headless.etcd.svc.cluster.local:2379 |   true |  1.084359ms |       |
+----------------------------------------------------------+--------+-------------+-------+
$ etcdctl  --user=root:$ETCD_ROOT_PASSWORD --endpoints=https://etcd-0.etcd-headless.etcd.svc.cluster.local:2379,https://etcd-1.etcd-headless.etcd.svc.cluster.local:2379,https://etcd-2.etcd-headless.etcd.svc.cluster.local:2379 --cacert=/opt/bitnami/etcd/certs/client/ca.crt --cert=/opt/bitnami/etcd/certs/client/cert.pem --key=/opt/bitnami/etcd/certs/client/key.pem member list -w table
+------------------+---------+--------+----------------------------------------------------------+---------------------------------------------------------------------------------------------------+------------+
|        ID        | STATUS  |  NAME  |                        PEER ADDRS                        |                                           CLIENT ADDRS                                            | IS LEARNER |
+------------------+---------+--------+----------------------------------------------------------+---------------------------------------------------------------------------------------------------+------------+
| 353f075dede680ca | started | etcd-1 | https://etcd-1.etcd-headless.etcd.svc.cluster.local:2380 | https://etcd-1.etcd-headless.etcd.svc.cluster.local:2379,https://etcd.etcd.svc.cluster.local:2379 |      false |
| 383cd4868f3344c8 | started | etcd-2 | https://etcd-2.etcd-headless.etcd.svc.cluster.local:2380 | https://etcd-2.etcd-headless.etcd.svc.cluster.local:2379,https://etcd.etcd.svc.cluster.local:2379 |      false |
| 5d62df9077746deb | started | etcd-0 | https://etcd-0.etcd-headless.etcd.svc.cluster.local:2380 | https://etcd-0.etcd-headless.etcd.svc.cluster.local:2379,https://etcd.etcd.svc.cluster.local:2379 |      false |
+------------------+---------+--------+----------------------------------------------------------+---------------------------------------------------------------------------------------------------+------------+
$ etcdctl  --user=root:$ETCD_ROOT_PASSWORD --endpoints=https://etcd-0.etcd-headless.etcd.svc.cluster.local:2379,https://etcd-1.etcd-headless.etcd.svc.cluster.local:2379,https://etcd-2.etcd-headless.etcd.svc.cluster.local:2379 --cacert=/opt/bitnami/etcd/certs/client/ca.crt --cert=/opt/bitnami/etcd/certs/client/cert.pem --key=/opt/bitnami/etcd/certs/client/key.pem endpoint status -w table
+----------------------------------------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
|                         ENDPOINT                         |        ID        | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
+----------------------------------------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| https://etcd-0.etcd-headless.etcd.svc.cluster.local:2379 | 5d62df9077746deb |   3.5.9 |   20 kB |      true |      false |         8 |       1480 |               1480 |        |
| https://etcd-1.etcd-headless.etcd.svc.cluster.local:2379 | 353f075dede680ca |   3.5.9 |   20 kB |     false |      false |         8 |       1483 |               1483 |        |
| https://etcd-2.etcd-headless.etcd.svc.cluster.local:2379 | 383cd4868f3344c8 |   3.5.9 |   20 kB |     false |      false |         8 |       1484 |               1484 |        |
+----------------------------------------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
# leader变为了etcd-0