etcd安装

1、单节点部署

Install | etcd

1.1、安装

# centos7环境安装etcd
#
# 下载etcd
wget https://github.com/etcd-io/etcd/releases/download/v3.5.11/etcd-v3.5.11-linux-amd64.tar.gz
 
# 解压
tar -xf etcd-v3.5.11-linux-amd64.tar.gz
 
# 进入 etcd-v3.5.11-linux-amd64目录
cd etcd-v3.5.11-linux-amd64
 
# 将etcd、etcdctl、etcdutl移动到/usr/local/bin目录
mv etcd /usr/local/bin/etcd
mv etcdctl /usr/local/bin/etcdctl
mv etcdutl /usr/local/bin/etcdutl
 
# 查看etcd版本
etcd --version
etcdctl version
etcdutl version

1.2、运行

# 运行etcd
# 会在当前目录创建data-dir="default.etcd"，wal-dir="default.etcd/member/wal"
etcd

# 指定参数运行
etcd --listen-client-urls=http://$PRIVATE_IP:2379 \
   --advertise-client-urls=http://$PRIVATE_IP:2379


# 查看集群状态
etcdctl --endpoints=$ENDPOINTS endpoint status
etcdctl --endpoints=$ENDPOINTS endpoint health

2、集群部署

2.1、使用http通信

操作步骤如下：

在每个节点运行etcd

在172.17.0.2节点查看集群

etcd部分参数说明：

节点IP：172.17.0.2、172.17.0.3、172.17.0.4（所有节点安装etcd）

cluster-state：new

token：etcd-token

cluster=etcd-node-1=http://172.17.0.2:2380,etcd-node-2=http://172.17.0.3:2380,etcd-node-3=http://172.17.0.2:2380

172.17.0.2节点：

etcd --data-dir=data.etcd --name etcd-node-1 \
	--initial-advertise-peer-urls http://172.17.0.2:2380 \
	--listen-peer-urls http://172.17.0.2:2380 \
	--advertise-client-urls http://172.17.0.2:2379 \
	--listen-client-urls http://172.17.0.2:2379 \
	--initial-cluster etcd-node-1=http://172.17.0.2:2380,etcd-node-2=http://172.17.0.3:2380,etcd-node-3=http://172.17.0.4:2380 \
	--initial-cluster-state new \
	--initial-cluster-token etcd-token \
    --peer-key-file peer.key \
    --peer-cert-file peer.cert

172.17.0.3节点：

etcd --data-dir=data.etcd --name etcd-node-2 \
	--initial-advertise-peer-urls http://172.17.0.3:2380 \
	--listen-peer-urls http://172.17.0.3:2380 \
	--advertise-client-urls http://172.17.0.3:2379 \
	--listen-client-urls http://172.17.0.3:2379 \
	--initial-cluster etcd-node-1=http://172.17.0.2:2380,etcd-node-2=http://172.17.0.3:2380,etcd-node-3=http://172.17.0.4:2380 \
	--initial-cluster-state new \
	--initial-cluster-token etcd-token

172.17.0.4节点：

etcd --data-dir=data.etcd --name etcd-node-3 \
	--initial-advertise-peer-urls http://172.17.0.4:2380 \
	--listen-peer-urls http://172.17.0.4:2380 \
	--advertise-client-urls http://172.17.0.4:2379 \
	--listen-client-urls http://172.17.0.4:2379 \
	--initial-cluster etcd-node-1=http://172.17.0.2:2380,etcd-node-2=http://172.17.0.3:2380,etcd-node-3=http://172.17.0.4:2380 \
	--initial-cluster-state new \
	--initial-cluster-token etcd-token

172.17.0.2节点，查看集群：

# 1、查看集群
etcdctl --endpoints=172.17.0.2:2379,172.17.0.3:2379,172.17.0.4:2379 member list

# 显示结果
17396d3aa5468ea, started, etcd-node-1, http://172.17.0.2:2380, http://172.17.0.2:2379, false
3a9e24ab09adc359, started, etcd-node-2, http://172.17.0.3:2380, http://172.17.0.3:2379, false
c79b72d1b9e2d891, started, etcd-node-3, http://172.17.0.4:2380, http://172.17.0.4:2379, false

# ===============================================================================
# 2、移除节点
etcdctl --endpoints=172.17.0.2:2379,172.17.0.3:2379,172.17.0.4:2379 member remove c79b72d1b9e2d891

# 显示结果
Member c79b72d1b9e2d891 removed from cluster 8cc616c226560de7

# 3、查看集群
etcdctl --endpoints=172.17.0.2:2379,172.17.0.3:2379,172.17.0.4:2379 member list

# 显示结果
17396d3aa5468ea, started, etcd-node-1, http://172.17.0.2:2380, http://172.17.0.2:2379, false
3a9e24ab09adc359, started, etcd-node-2, http://172.17.0.3:2380, http://172.17.0.3:2379, false


# ===============================================================================
# 4、添加节点
etcdctl --endpoints=172.17.0.2:2379,172.17.0.3:2379 member add etcd-node-3 --peer-urls=http://172.17.0.4:2380

# 显示结果
Member 99d9b4191b7df3f2 added to cluster 8cc616c226560de7

ETCD_NAME="etcd-node-3"
ETCD_INITIAL_CLUSTER="etcd-node-1=http://172.17.0.2:2380,etcd-node-2=http://172.17.0.3:2380,etcd-node-3=http://172.17.0.4:2380"
ETCD_INITIAL_ADVERTISE_PEER_URLS="http://172.17.0.4:2380"
ETCD_INITIAL_CLUSTER_STATE="existing"

# 5、查看集群
etcdctl --endpoints=172.17.0.2:2379,172.17.0.3:2379,172.17.0.4:2379 member list

# 显示结果
17396d3aa5468ea, started, etcd-node-1, http://172.17.0.2:2380, http://172.17.0.2:2379, false
3a9e24ab09adc359, started, etcd-node-2, http://172.17.0.3:2380, http://172.17.0.3:2379, false
91fd2ad3e8b7b64f, unstarted, , http://172.17.0.4:2380, , false


# 6、172.17.0.4节点执行
rm -rf ./etcd-node-3.etcd

# 7、172.17.0.4节点执行
etcd --data-dir=data.etcd --name etcd-node-3 \
	--initial-advertise-peer-urls http://172.17.0.4:2380 \
	--listen-peer-urls http://172.17.0.4:2380 \
	--advertise-client-urls http://172.17.0.4:2379 \
	--listen-client-urls http://172.17.0.4:2379 \
	--initial-cluster etcd-node-1=http://172.17.0.2:2380,etcd-node-2=http://172.17.0.3:2380,etcd-node-3=http://172.17.0.4:2380 \
	--initial-cluster-state existing \
	--initial-cluster-token etcd-token

# 8、查看集群
etcdctl --endpoints=172.17.0.2:2379,172.17.0.3:2379,172.17.0.4:2379 member list

# 显示结果
17396d3aa5468ea, started, etcd-node-1, http://172.17.0.2:2380, http://172.17.0.2:2379, false
2e6bc6b95d92f714, started, etcd-node-3, http://172.17.0.4:2380, http://172.17.0.4:2379, false
3a9e24ab09adc359, started, etcd-node-2, http://172.17.0.3:2380, http://172.17.0.3:2379, false

2.2、使用https通信

操作步骤如下：

在172.17.0.2节点安装cfssl，生成证书

将证书复制到其他节点（172.17.0.3、172.17.0.4）

在每个节点运行etcd

在172.17.0.2节点查看集群

安装cfssl

cfssl简单使用-CSDN博客

# 1、下载cfssl、cfssljson、cfssl-certinfo
# cfssl：用于签发证书
# cfssljson：将cfssl签发生成的证书(json格式)变成文件承载式文件
# cfssl-certinfo：验证查看证书信息
wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 -O /usr/local/bin/cfssl
wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 -O /usr/local/bin/cfssljson
wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64 -O /usr/local/bin/cfssl-certinfo
 
# 2、给cfssl、cfssljson、cfssl-certinfo添加可执行权限
chmod +x /usr/local/bin/cfssl*

生成证书

配置文件：ca-config.json、ca-csr.json、etcd-csr.json

文件内容来源：https://github.com/etcd-io/etcd/tree/main/hack/tls-setup

ca-config.json文件：

{
  "signing": {
    "default": {
        "usages": [
          "signing",
          "key encipherment",
          "server auth",
          "client auth"
        ],
        "expiry": "876000h"
    }
  }
}

ca-csr.json文件：

{
  "CN": "Autogenerated CA",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "O": "Honest Achmed's Used Certificates",
      "OU": "Hastily-Generated Values Divison",
      "L": "San Francisco",
      "ST": "California",
      "C": "US"
    }
  ]
}

etcd-csr.json文件：

{
  "CN": "etcd",
  "hosts": [
    "localhost",
    "127.0.0.1",
    "172.17.0.2",
    "172.17.0.3",
    "172.17.0.4"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "O": "autogenerated",
      "OU": "etcd cluster",
      "L": "the internet"
    }
  ]
}

执行命令：

# 创建目录
mkdir -p etcd/ssl

# 切换目录
cd etcd/ssl

# 创建文件，复制上述ca-config.json、ca-csr.json、etcd-csr.json内容到对应文件
touch {ca-config.json,ca-csr.json,etcd-csr.json}

# 生成CA根证书及其私钥
cfssl gencert -initca ca-csr.json | cfssljson -bare ca

# 根据CA根证书及其私钥签名生成目标证书和私钥
cfssl gencert -ca ca.pem -ca-key ca-key.pem -config ca-config.json etcd-csr.json | cfssljson -bare etcd

# 复制etcd目录到其他节点
scp -r /root/etcd 172.17.0.3:/root/etcd
scp -r /root/etcd 172.17.0.4:/root/etcd

172.17.0.2节点：

etcd --data-dir=data.etcd --name etcd-node-1 \
    --initial-advertise-peer-urls https://172.17.0.2:2380 \
    --listen-peer-urls https://172.17.0.2:2380 \
    --advertise-client-urls https://172.17.0.2:2379 \
    --listen-client-urls https://172.17.0.2:2379 \
    --initial-cluster 'etcd-node-1=https://172.17.0.2:2380,etcd-node-2=https://172.17.0.3:2380,etcd-node-3=https://172.17.0.4:2380' \
    --initial-cluster-state new \
    --initial-cluster-token etcd-token \
    --cert-file=/root/etcd/ssl/etcd.pem \
    --key-file=/root/etcd/ssl/etcd-key.pem \
    --peer-cert-file=/root/etcd/ssl/etcd.pem \
    --peer-key-file=/root/etcd/ssl/etcd-key.pem \
    --peer-client-cert-auth \
    --peer-trusted-ca-file=/root/etcd/ssl/ca.pem

172.17.0.3节点：

etcd --data-dir=data.etcd --name etcd-node-2 \
    --initial-advertise-peer-urls https://172.17.0.3:2380 \
    --listen-peer-urls https://172.17.0.3:2380 \
    --advertise-client-urls https://172.17.0.3:2379 \
    --listen-client-urls https://172.17.0.3:2379 \
    --initial-cluster 'etcd-node-1=https://172.17.0.2:2380,etcd-node-2=https://172.17.0.3:2380,etcd-node-3=https://172.17.0.4:2380' \
    --initial-cluster-state new \
    --initial-cluster-token etcd-token \
    --cert-file=/root/etcd/ssl/etcd.pem \
    --key-file=/root/etcd/ssl/etcd-key.pem \
    --peer-cert-file=/root/etcd/ssl/etcd.pem \
    --peer-key-file=/root/etcd/ssl/etcd-key.pem \
    --peer-client-cert-auth \
    --peer-trusted-ca-file=/root/etcd/ssl/ca.pem

172.17.0.4节点：

etcd --data-dir=data.etcd --name etcd-node-3 \
    --initial-advertise-peer-urls https://172.17.0.4:2380 \
    --listen-peer-urls https://172.17.0.4:2380 \
    --advertise-client-urls https://172.17.0.4:2379 \
    --listen-client-urls https://172.17.0.4:2379 \
    --initial-cluster 'etcd-node-1=https://172.17.0.2:2380,etcd-node-2=https://172.17.0.3:2380,etcd-node-3=https://172.17.0.4:2380' \
    --initial-cluster-state new \
    --initial-cluster-token etcd-token \
    --cert-file=/root/etcd/ssl/etcd.pem \
    --key-file=/root/etcd/ssl/etcd-key.pem \
    --peer-cert-file=/root/etcd/ssl/etcd.pem \
    --peer-key-file=/root/etcd/ssl/etcd-key.pem \
    --peer-client-cert-auth \
    --peer-trusted-ca-file=/root/etcd/ssl/ca.pem

172.17.0.2节点，查看集群：

# 查看集群
etcdctl --endpoints=172.17.0.2:2379,172.17.0.3:2379,172.17.0.4:2379 \
  --cert=/root/etcd/ssl/etcd.pem \
  --key=/root/etcd/ssl/etcd-key.pem \
  --cacert=/root/etcd/ssl/ca.pem \
  member list

# 显示结果
c6b958fbe52963, started, etcd-node-1, https://172.17.0.2:2380, https://172.17.0.2:2379, false
5f334165954101b, started, etcd-node-3, https://172.17.0.4:2380, https://172.17.0.4:2379, false
964941aff35ec5da, started, etcd-node-2, https://172.17.0.3:2380, https://172.17.0.3:2379, false


# 不带证书访问集群
etcdctl --endpoints=172.17.0.2:2379,172.17.0.3:2379,172.17.0.4:2379   member list

# 显示结果
{"level":"warn","ts":"2024-01-20T21:58:12.352461Z","logger":"etcd-client","caller":"v3@v3.5.11/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000374380/172.17.0.2:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"error reading server preface: EOF\""}
Error: context deadline exceeded

详见：

操作 Kubernetes 中的 etcd 集群 | Kubernetes

Clustering Guide | etcd

How to Set Up a Demo etcd Cluster | etcd

How to Add and Remove Members | etcd

3、docker部署

Run etcd clusters inside containers | etcd

3.1、使用http通信

# 当前主机IP：10.0.8.13
# 创建目录
mkdir /root/etcd-data


# 当前主机IP：10.0.8.13
# 创建容器
docker run \
  -p 2379:2379 \
  -p 2380:2380 \
  --volume=/root/etcd-data:/root/etcd-data \
  --name etcd quay.io/coreos/etcd:latest \
  /usr/local/bin/etcd \
  --data-dir=/root/etcd-data --name node1 \
  --initial-advertise-peer-urls http://10.0.8.13:2380 \
  --listen-peer-urls http://0.0.0.0:2380 \
  --advertise-client-urls http://10.0.8.13:2379 \
  --listen-client-urls http://0.0.0.0:2379 \
  --initial-cluster node1=http://10.0.8.13:2380


# 当前主机IP：10.0.8.13
# 查看集群状态
etcdctl --endpoints=10.0.8.13:2379 member list

# 显示结果
942908011bbade83, started, node1, http://10.0.8.13:2380, http://10.0.8.13:2379, false

3.2、使用https通信

# 当前主机IP：10.0.8.13
# 创建目录
mkdir -p /root/etcd/{data,ssl}

# 生成证书
# 参考：“2.2、使用https通信” 的 “生成证书”
# 修改“etcd-csr.json文件” 的 “hosts” 字段，hosts值应为 “主机IP”

# 当前主机IP：10.0.8.13
# 创建容器
docker run \
  -p 2379:2379 \
  -p 2380:2380 \
  --volume=/root/etcd:/root/etcd \
  --name etcd quay.io/coreos/etcd:latest \
  /usr/local/bin/etcd \
  --data-dir=/root/etcd/data --name node1 \
  --initial-advertise-peer-urls https://10.0.8.13:2380 \
  --listen-peer-urls https://0.0.0.0:2380 \
  --advertise-client-urls https://10.0.8.13:2379 \
  --listen-client-urls https://0.0.0.0:2379 \
  --cert-file=/root/etcd/ssl/etcd.pem \
  --key-file=/root/etcd/ssl/etcd-key.pem \
  --peer-cert-file=/root/etcd/ssl/etcd.pem \
  --peer-key-file=/root/etcd/ssl/etcd-key.pem \
  --peer-client-cert-auth \
  --peer-trusted-ca-file=/root/etcd/ssl/ca.pem


# 当前主机IP：10.0.8.13
# 查看集群
etcdctl --endpoints=10.0.8.13:2379 \
  --cert=/root/etcd/ssl/etcd.pem \
  --key=/root/etcd/ssl/etcd-key.pem \
  --cacert=/root/etcd/ssl/ca.pem \
  member list

# 显示结果
b0b9626eea1088ab, started, node1, https://10.0.8.13:2380, https://10.0.8.13:2379, false


# 当前主机IP：10.0.8.13
# 不带证书访问集群
etcdctl --endpoints=10.0.8.13:2379 member list

# 显示结果
{"level":"warn","ts":"2024-01-21T15:42:38.997455+0800","logger":"etcd-client","caller":"v3@v3.5.11/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00007c000/10.0.8.13:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"error reading server preface: read tcp 10.0.8.13:56464->10.0.8.13:2379: read: connection reset by peer\""}
Error: context deadline exceeded