rke2-install

Posted on In

1 系统准备

1.1 DNS 配置

或者修改/etc/hosts 文件

1
2
3
100.64.20.101 k3s-001
100.64.20.102 k3s-002
100.64.20.103 k3s-003

1.2. disable SELINUX

1.3. linux module

1
2
3
4
5
6
# cat /etc/sysctl.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
# modprobe br_netfilter
# sysctl -p

1.4. 修改Firewall 规则

master 需要open TCP 9345 port

1.5. networkmanager 的修改

1
2
3
# cat /etc/NetworkManager/conf.d/rke2-canal.conf
[keyfile]
unmanaged-devices=interface-name:cali*;interface-name:flannel*

2. 安装rke2

1
2
3
4
Master node:
# curl -sfL http://rancher-mirror.rancher.cn/rke2/install.sh | INSTALL_RKE2_MIRROR=cn sh -
worker node:
# curl -sfL http://rancher-mirror.rancher.cn/rke2/install.sh | INSTALL_RKE2_MIRROR=cn INSTALL_RKE2_TYPE="agent"  sh -

指定一个安装的版本

1
curl -sfL http://rancher-mirror.rancher.cn/rke2/install.sh | INSTALL_RKE2_VERIOIN= vX.Y.Z- INSTALL_RKE2_MIRROR=cn sh -

2.1 启动 第一个 rke2 master 节点

1
2
# systemctl enable rke2-server
# systemctl start rke2-server

在第一次启动后，在/var/lib/rancher/rke2 目录下会创建 rke2 的环境

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
root@k3s-001:/var/lib/rancher/rke2# ls -lh
total 12K
drwxr-xr-x 7 root root 4.0K Nov 19 08:08 agent
lrwxrwxrwx 1 root root   58 Nov 19 08:03 bin -> /var/lib/rancher/rke2/data/v1.21.6-rke2r1-fd8a733b61b5/bin
drwxr-xr-x 3 root root 4.0K Nov 19 08:03 data
drwx------ 7 root root 4.0K Nov 19 08:07 server
root@k3s-001:/var/lib/rancher/rke2# ls -lh ./bin/
total 269M
-rwxr-xr-x 1 root root  33M Nov 19 08:03 containerd
-rwxr-xr-x 1 root root 6.2M Nov 19 08:03 containerd-shim
-rwxr-xr-x 1 root root  11M Nov 19 08:03 containerd-shim-runc-v1
-rwxr-xr-x 1 root root  11M Nov 19 08:03 containerd-shim-runc-v2
-rwxr-xr-x 1 root root  23M Nov 19 08:03 crictl
-rwxr-xr-x 1 root root  19M Nov 19 08:03 ctr
-rwxr-xr-x 1 root root  47M Nov 19 08:03 kubectl
-rwxr-xr-x 1 root root 112M Nov 19 08:03 kubelet
-rwxr-xr-x 1 root root  11M Nov 19 08:03 runc
-rwxr-xr-x 1 root root 307K Nov 19 08:03 socat
root@k3s-001:/var/lib/rancher/rke2#

在 /etc/rancher/rke2 目录下会生成 rke2.yaml 的文件。这是一个 kubectl 的配置文件，用于集群管理。

1
2
3
4
root@k3s-001:~# ls -l /etc/rancher/rke2/
total 4
-rw------- 1 root root 2961 Nov 19 08:03 rke2.yaml
root@k3s-001:~#

通过kubectl 管理k3s 集群

1
2
export KUBECONFIG=/etc/rancher/rke2/rke2.yaml
export PATH=$PATH:/var/lib/rancher/rke2/bin

查看k3s cluster(rke2)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22

root@k3s-001:~# kubectl get pod -A
NAMESPACE     NAME                                                   READY   STATUS      RESTARTS   AGE
kube-system   cloud-controller-manager-k3s-001                       1/1     Running     0          19h
kube-system   etcd-k3s-001                                           1/1     Running     0          19h
kube-system   helm-install-rke2-canal-nwqr2                          0/1     Completed   0          19h
kube-system   helm-install-rke2-coredns-86s8w                        0/1     Completed   0          19h
kube-system   helm-install-rke2-ingress-nginx-t2bvr                  0/1     Completed   0          19h
kube-system   helm-install-rke2-metrics-server-2q2f7                 0/1     Completed   0          19h
kube-system   kube-apiserver-k3s-001                                 1/1     Running     0          19h
kube-system   kube-controller-manager-k3s-001                        1/1     Running     0          19h
kube-system   kube-proxy-k3s-001                                     1/1     Running     0          19h
kube-system   kube-scheduler-k3s-001                                 1/1     Running     0          19h
kube-system   rke2-canal-46v6k                                       2/2     Running     0          19h
kube-system   rke2-coredns-rke2-coredns-5c84cbd6bd-tdlcn             1/1     Running     0          19h
kube-system   rke2-coredns-rke2-coredns-autoscaler-78bdc5d6c-gj4w6   1/1     Running     0          19h
kube-system   rke2-ingress-nginx-controller-gxszn                    1/1     Running     0          19h
kube-system   rke2-metrics-server-5df7d77b5b-p2j7t                   1/1     Running     0          19h
root@k3s-001:~# kubectl get node -A
NAME      STATUS   ROLES                       AGE   VERSION
k3s-001   Ready    control-plane,etcd,master   19h   v1.21.6+rke2r1
root@k3s-001:~#

修改第一个节点的 rke2 配置文件：

1
2
3
4
5
6
7
8
9
10
# cat /etc/rancher/rke2/config.yaml

# token path /var/lib/rancher/rke2/server/node-token
token: K10b74ab83885c946d61159a734036a4f964f54acd80a5a74042cf9adff34a983b0::server:82ed82eb9668d57283c1efd84c5c1a05
node-name: "k3s-001"
# node-taint:
node-label:
  - "node=Master"
  - "k3s-001=Master"

重启rke2 server

1
2
# systemctl daemon-reload
# systemctl restart rke2-server

2.2 添加第二个master 节点

在第二个节点上添加 配置文件(先不要启动第二节点的rke2-server 服务，如果之前启动过，那么会有一些配置文件不一致导致无法加入集群)

第二个节点的配置文件：

1
2
3
4
5
6
7
8
9
10
11
12
13
# mkdir -p /etc/rancher/rke2/
# cat /etc/rancher/rke2/config.yaml

# token path /var/lib/rancher/rke2/server/node-token
server: "https://100.64.20.101:9345"
# token path /var/lib/rancher/rke2/server/node-token (from first master node)
token: K10b74ab83885c946d61159a734036a4f964f54acd80a5a74042cf9adff34a983b0::server:82ed82eb9668d57283c1efd84c5c1a05

node-name: "k3s-002"
# node-taint:
node-label:
  - "node=Master"
  - "k3s-002=Master"

2.3 添加第三个节点作为work 节点

1
2
3
4
5
6
7
8
9
10
11
12
# mkdir -p /etc/rancher/rke2/
# cat /etc/rancher/rke2/config.yaml

# token path /var/lib/rancher/rke2/server/node-token
server: "https://100.64.20.101:9345"
# token path /var/lib/rancher/rke2/server/node-token (from first master node)
token: K10b74ab83885c946d61159a734036a4f964f54acd80a5a74042cf9adff34a983b0::server:82ed82eb9668d57283c1efd84c5c1a05
node-name: "k3s-003"
# node-taint:
node-label:
  - "node=Worker"
  - "k3s-003=Worker"
1
2
systemctl daemon-reload
systemctl restart rke2-agent

检查 containerd

1
#/var/lib/rancher/rke2/bin/crictl --runtime-endpoint=unix:///run/k3s/containerd/containerd.sock ps

配置master 上的管理cli 环境变量

1
2
3
4
5
6
# cd $HOME
mkdir .kube
ln -s /etc/rancher/rke2/rke2.yaml ~/.kube/config
chmod 600 ~/.kube/config
ln -s /var/lib/rancher/rke2/agent/etc/crictl.yaml /etc/crictl.yaml
export PATH=$PATH:/var/lib/rancher/rke2/bin

container 的image rergistory -TBD

1
2
3
4
5
6
7
8
cat /etc/rancher/rke2/registries.yaml

mirrors:
  aliyuns.com:
    endpoint:
      - ""
configs:

配置etcd的 snapshots

1
2
3
4
5
6
7
8
9
10
token: K10b74ab83885c946d61159a734036a4f964f54acd80a5a74042cf9adff34a983b0::server:82ed82eb9668d57283c1efd84c5c1a05
node-name: "k3s-001"
# node-taint:
node-label:
  - "node=Master"
  - "k3s-001=Master"

etcd-snapshot-retention: 2
# /var/lib/rancher/rke2/server/db/snapshots/
etcd-snapshot-schedule-cron: '*/5 * * * *'