etcd高可用安装

一.用openssl生成证书安装 二.用cfssl生成证书安装

用OPENSSL生成证书安装

mkdir etcd_ssl
cd etcd_ssl

openssl genrsa -out ca.key 2048

openssl req -x509 -new -nodes -key ca.key -subj "/CN=test-k8s" -days 3650 -out ca.crt

cat > etcd-ca.conf <<EOF
[ req ]
default_bits = 2048
prompt = no
default_md = sha256
req_extensions = req_ext
distinguished_name = dn

[ dn ]
C = CN
ST = HuBei
L = WuHan
O = etcd
OU = jiaparts
CN = test-k8s

[ req_ext ]
subjectAltName = @alt_names

[ alt_names ]
DNS.1 = localhost
DNS.2 = test-01
DNS.3 = test-02
DNS.4 = test-03
IP.1 = 127.0.0.1
IP.2 = 172.23.210.22
IP.3 = 172.23.210.23
IP.4 = 172.23.210.24

[ v3_ext ]
authorityKeyIdentifier=keyid,issuer:always
basicConstraints=CA:FALSE
keyUsage=keyEncipherment,dataEncipherment
extendedKeyUsage=serverAuth,clientAuth
subjectAltName=@alt_names
EOF

openssl genrsa -out etcd.key 2048
openssl req -new -key etcd.key -out etcd.csr -config etcd-ca.conf 
openssl x509 -req -in etcd.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out etcd.crt -days 3650 -extensions v3_ext -extfile etcd-ca.conf 
openssl verify -CAfile ca.crt etcd.crt 

etcd安装

tar -zxf etcd-v3.3.13-linux-amd64.tar.gz
cp etcd-v3.3.13/etcd /usr/bin/
cp etcd-v3.3.13/etcdctl /usr/bin/

etcd配置

mkdir -p /etc/etcd/ssl
cat > /etc/etcd/etcd.conf <<EOF
# [Member Flags]
# ETCD_ELECTION_TIMEOUT=1000
# ETCD_HEARTBEAT_INTERVAL=100
# 指定etcd的数据目录
ETCD_NAME=test-01	# test-02 test-03
ETCD_DATA_DIR=/var/lib/etcd/

# [Cluster Flags]
# ETCD_AUTO_COMPACTION_RETENTIO:N=0
ETCD_INITIAL_CLUSTER_STATE=new
ETCD_ADVERTISE_CLIENT_URLS=https://172.3.210.22:2379	# 172.23.210.23 172.23.210.22
ETCD_INITIAL_ADVERTISE_PEER_URLS=https://172.3.210.22:2380	# 172.23.210.23 172.23.210.22
ETCD_LISTEN_CLIENT_URLS=https://172.3.210.22:2379,https://127.0.0.1:2379	# 172.23.210.23 172.23.210.22
ETCD_INITIAL_CLUSTER_TOKEN=etcd-cluster
ETCD_LISTEN_PEER_URLS=https://172.3.210.22:2380	# 172.23.210.23 172.23.210.22
ETCD_INITIAL_CLUSTER="test-01=https://172.3.210.22:2380,test-02=https://172.23.210.23:2380,test-03=https://172.23.210.24:2380"

# [Proxy Flags]
ETCD_PROXY=off

# [Security flags]
# ETCD_CLIENT_CERT_AUTH=
# ETCD_PEER_CLIENT_CERT_AUTH=
# 指定etcd的公钥证书和私钥
ETCD_TRUSTED_CA_FILE=/etc/etcd/ssl/ca.crt
ETCD_CERT_FILE=/etc/etcd/ssl/etcd.crt
ETCD_KEY_FILE=/etc/etcd/ssl/etcd.key
# 指定etcd的Peers通信的公钥证书和私钥
ETCD_PEER_TRUSTED_CA_FILE=/etc/etcd/ssl/ca.crt
ETCD_PEER_CERT_FILE=/etc/etcd/ssl/etcd.crt
ETCD_PEER_KEY_FILE=/etc/etcd/ssl/etcd.key

# [Profiling flags]
# ETCD_METRICS={{ etcd_metrics }}
EOF

etc服务控制

cat > /usr/lib/systemd/system/etcd.service <<EOF
[Unit]
Description=etcd server
After=network.target
After=network-online.target
Wants=network-online.target

[Service]
Type=notify
WorkingDirectory=/var/lib/etcd/
EnvironmentFile=-/etc/etcd/etcd.conf
ExecStart=/usr/local/bin/etcd
NotifyAccess=all
Restart=always
RestartSec=5s
LimitNOFILE=40000

[Install]
WantedBy=multi-user.target
EOF

etcd服务控制

systemctl start etcd
systemctl enable etcd

etcd健康检查

etcdctl cluster-health

---

用cfssl生成集群tls证书安装

下载安装cfssl 和cfssjson

curl~ -s -L -o /usr/bin/cfssl https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
curl -s -L -o /usr/bin/cfssljson https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
chmo~d +x /usr/bin/{cfssl,cfssljson}

创建个目录用来放要生成的证书

mkdir ~/cfssl
cd ~/cfssl

生成CA配置文件

cfssl print-defaults config > ca-config.json
cfssl print-defaults csr > ca-csr.json

将内容修成如下,expiry的值为10年,表示生成的证书有效期

cat ca-config.json

{
	"signing": {
		"default": {
			"expiry": "87600h"
		},
		"profiles": {
			"server": { # 服务器证书由服务器使用，并由客户端验证服务器身份。例如docker服务器或kube-apiserver
				"expiry": "87600h",
				"usages": [
					"signing",
					"key encipherment",
					"server auth"
				]
			},
			"client": { # 客户端证书用于按服务器对客户端进行身份验证。例如etcdctl，etcd proxy或者docker客户端。
				"expiry": "87600h",
				"usages": [
					"signing",
					"key encipherment",
					"client auth"
				]
			},
			"peer": { # 对等证书由etcd集群成员使用，因为它们以两种方式相互通信。本文档就是用这个证书
				"expiry": "87600h",
				"usages": [
					"signing",
					"key encipherment",
					"server auth",
					"client auth"
				]
			}
		}
	}
}

修改ca-csr.json内容为如下

cat ca-csr.json

{
	"CN": "etcd server",
	"hosts": [
		"localhost",
		"127.0.0.1",
		"172.23.210.22",
		"172.23.210.23",
		"172.23.210.24",
		"test-01",
		"test-02",
		"test-03"
	],
	"key": {
		"algo": "rsa",
		"size": 2048
	},
	"names": [
		{
			"C": "CN",
			"L": "HB",
			"ST": "Wu Han"
		}
	]
}

生成CA证书

cfssl gencert -initca ca-csr.json | cfssljson -bare ca -

会得到以下文件：

ca-key.pem
ca.csr
ca.pem

生成对等证书

etcd tls就是用这个证书

cfssl print-defaults csr > peer.json

内容如下：

cat peer.json
{
	"CN": "etcd",
	"hosts": [
		"localhost",
		"127.0.0.1",
		"172.23.210.22",
		"172.23.210.23",
		"172.23.210.24"
	],
	"key": {
		"algo": "rsa",
		"size": 2048
	},
	"names": [{
		"C": "CN",
		"L": "HB",
		"ST": "Wu Han"
	}]
}

执行生成命令

cfssl gencert -hostname="172.23.210.22,172.23.210.23,172.23.210.24" -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=peer peer.json | cfssljson -bare etcd

# 如果不加-hostname参数,生成证书后会报下面的错误,cfssl 1.2版本暂未修复这个BUG
[WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for websites. For more information see the Baseline Requirements for the Issuance and Management of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org); specifically, section 10.2.3 ("Information Requirements").

生成如下文件

etcd.pem
etcd-key.pem
etct.csr

将ca.pem etcd.pem etcd-key.pem复制到/etc/etcd/ssl/目录下

etcd安装

tar -zxf etcd-v3.3.13-linux-amd64.tar.gz
cp etcd-v3.3.13/etcd /usr/bin/
cp etcd-v3.3.13/etcdctl /usr/bin/

etcd配置文件

cat /etc/etcd/etcd.conf
# [Member Flags]
# ETCD_ELECTION_TIMEOUT=1000
# ETCD_HEARTBEAT_INTERVAL=100
# 指定etcd的数据目录
ETCD_NAME=test-01
ETCD_DATA_DIR=/var/lib/etcd/

# [Cluster Flags]
# ETCD_AUTO_COMPACTION_RETENTIO:N=0
ETCD_INITIAL_CLUSTER_STATE=new		# new是新集群，existing表示加入已有集群
ETCD_ADVERTISE_CLIENT_URLS=https://172.23.210.22:2379   # 服务器2、3修改IP为172.23.210.23、24
ETCD_INITIAL_ADVERTISE_PEER_URLS=https://172.23.210.22:2380 # 服务器2、3修改IP为172.23.210.23、24
ETCD_LISTEN_CLIENT_URLS=https://172.23.210.22:2379,https://127.0.0.1:2379   # 服务器2、3修改IP为172.23.210.23、24
ETCD_INITIAL_CLUSTER_TOKEN=etcd-cluster		# 集群token
ETCD_LISTEN_PEER_URLS=https://172.23.210.22:2380    # 服务器2、3修改IP为172.23.210.23、24
ETCD_INITIAL_CLUSTER="test-01=https://172.23.210.22:2380,test-02=https://172.23.210.23:2380,test-03=https://172.23.210.24:2380"

# [Proxy Flags]
ETCD_PROXY=off
[Security flags]
ETCD_CERT_FILE="/etc/etcd/ssl/etcd.pem"     
ETCD_KEY_FILE="/etc/etcd/ssl/etcd-key.pem" 
ETCD_CLIENT_CERT_AUTH="true" 
ETCD_TRUSTED_CA_FILE="/etc/etcd/ssl/ca.pem" 
ETCD_AUTO_TLS="true" 
ETCD_PEER_CERT_FILE="/etc/etcd/ssl/etcd.pem" 
ETCD_PEER_KEY_FILE="/etc/etcd/ssl/etcd-key.pem" 
ETCD_PEER_CLIENT_CERT_AUTH="true" 
ETCD_PEER_TRUSTED_CA_FILE="/etc/etcd/ssl/ca.pem" 
ETCD_PEER_AUTO_TLS="true"

etc服务控制

cat > /usr/lib/systemd/system/etcd.service <<EOF
[Unit]
Description=etcd server
After=network.target
After=network-online.target
Wants=network-online.target

[Service]
Type=notify
WorkingDirectory=/var/lib/etcd/
EnvironmentFile=-/etc/etcd/etcd.conf
ExecStart=/usr/local/bin/etcd
NotifyAccess=all
Restart=always
RestartSec=5s
LimitNOFILE=40000

[Install]
WantedBy=multi-user.target
EOF

etcd服务控制

systemctl start etcd
systemctl enable etcd

etcd健康检查

etcdctl --ca-file=/etc/etcd/ssl/ca.pem --cert-file=/etc/etcd/ssl/etcd.pem --key-file=/etc/etcd/ssl/etcd-key.pem --endpoints="https://172.23.210.22:2379,https://172.23.210.23:2379,https://172.23.210.24:2379" cluster-health

member 22d860099d5a23a4 is healthy: got healthy result from https://172.23.210.24:2379
member b8c31f277f3aec2f is healthy: got healthy result from https://172.23.210.22:2379
member c3d95832e4f4a6e7 is healthy: got healthy result from https://172.23.210.23:2379


etcdctl --ca-file=/etc/etcd/ssl/ca.pem --cert-file=/etc/etcd/ssl/etcd.pem --key-file=/etc/etcd/ssl/etcd-key.pem --endpoints="https://172.23.210.22:2379,https://172.23.210.23:2379,https://172.23.210.24:2379" member list

22d860099d5a23a4: name=test-03 peerURLs=https://172.23.210.24:2380 clientURLs=https://172.23.210.24:2379 isLeader=false
b8c31f277f3aec2f: name=test-01 peerURLs=https://172.23.210.22:2380 clientURLs=https://172.23.210.22:2379 isLeader=true
c3d95832e4f4a6e7: name=test-02 peerURLs=https://172.23.210.23:2380 clientURLs=https://172.23.210.23:2379 isLeader=false

---

注意事项: 注意 /var/lib/etcd/ 目录下不要有失败环境的遗留文件 注意节点时间同步,建议通过ntpd来同步时间