问题

k8s node节点加入到集群时卡住 “[preflight] Running pre-flight checks”

# master节点重新生成加入命令
kubeadm token create --ttl 0 --print-join-command

参考

注意

k8s1.24+使用containerd而不再使用docker，因此使用k8s1.23版本

环境

k8s: 1.23.0
ubuntu: 20.04
docker: 23
docker-compose: 1.25.4

主机	IP	角色
xcrj01	192.168.66.10	k8s-master01
xcrj02	192.168.66.20	k8s-node01
xcrj03	192.168.66.21	k8s-node02

hyper-v ubuntu20.04

参考

hyper-v koolshare

参考

docker

参考
修改daemon.json

{"registry-mirrors": ["https://docker.mirrors.ustc.edu.cn","http://hub-mirror.c.163.com","http://registry.docker-cn.com"],"exec-opts": ["native.cgroupdriver=systemd"]
}

创建systemd所需目录

mkdir -p /etc/systemd/system/docker.service.d

docker-compose

参考

k8s

master

# 切换root用户
sudo su -# 更新源
apt update
# 升级包
apt upgrade -y# 设置hostname，可选
# hostnamectl set-hostname k8s-master01# 备份hosts文件
cp /etc/hosts /etc/hosts-bk
# 添加hosts，见下
vim /etc/hosts# 开启IPVS负载均衡，见下# 设置k8s所需内核参数，见下
vim /etc/sysctl.d/kubernetes.conf
# 应用内核参数
sysctl --system# docker
# 见大标题docker# 永久关闭swap。pod运行在swap分区中会大大影响效率
swapoff -a && sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab
# 禁用SELINUX：开启会限制服务进程访问资源
setenforce 0 && sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config
# 安装必要包
apt install -y curl gnupg2 software-properties-common apt-transport-https ca-certificates
# 开启路由功能
sysctl -w net.ipv4.ip_forward=1
# 重启虚拟机系统# kubernetes
# add aliyun/kubernetes apt key
curl https://mirrors.aliyun.com/kubernetes/apt/doc/apt-key.gpg | sudo apt-key add -
# add aliyun/Kubernetes apt repository
apt-add-repository "deb https://mirrors.aliyun.com/kubernetes/apt/ kubernetes-xenial main"
# 检查更新
apt update
# 安装kubelet kubeadm kubectl
apt install -y kubelet=1.23.0-00 kubeadm=1.23.0-00 kubectl=1.23.0-00
# 锁定版本，不随apt upgrade更新
apt-mark hold kubelet kubeadm kubectl
# 查看kubeadm版本
kubeadm version
# 生成初始化文件
kubeadm config print init-defaults > kubeadm-config.yaml
# 修改初始化文件，见下
# 初始化kubernetes
kubeadm init --config=kubeadm-config.yaml --v=6 --upload-certs | tee kubeadm-init.log# 下面的命令来自 保存打印的内容，推荐使用普通用户
su xcrj01
## 普通用户执行这个，To start using your cluster, you need to run the following as a 
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
## root用户执行这个，Alternatively, if you are the root user, you can run
export KUBECONFIG=/etc/kubernetes/admin.conf# 配置CNI网络
# You should now deploy a pod network to the cluster. 安装calico网络组件
# kubectl apply -f "https://docs.projectcalico.org/manifests/calico.yaml"
# 暂时使用Flannel（基于L2）构建扁平化的网络，以后升级到Calico（完全基于L3）
sudo kubectl apply -f "https://github.com/flannel-io/flannel/blob/master/Documentation/kube-flannel.yml"

hosts

   #master01vi /etc/hosts192.168.66.10 k8s-master01192.168.66.20 k8s-node01192.168.66.21 k8s-node02#node01vi /etc/hosts192.168.66.10 k8s-master01192.168.66.20 k8s-node01#node02vi /etc/hosts192.168.66.10 k8s-master01192.168.66.21 k8s-node02

开启IPVS负载均衡

modprobe br_netfilter

或

cat > /etc/sysconfig/modules/ipvs.modules <<EOF
#!/bin/bash
modprobe br_netfilter
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack
EOF
#操作上面创建的文件
chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep -e ip_vs -e nf_conntrack_ipv4

kubernetes.conf

net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1

kubeadm-config.yaml
增加podSubnet；修改kubernetesVersion；修改advertiseAddress（masterIP）；修改imageRepository为阿里云

   apiVersion: kubeadm.k8s.io/v1beta2bootstrapTokens:- groups:- system:bootstrappers:kubeadm:default-node-tokentoken: abcdef.0123456789abcdefttl: 24h0m0susages:- signing- authenticationkind: InitConfigurationlocalAPIEndpoint:advertiseAddress: 192.168.66.10bindPort: 6443nodeRegistration:criSocket: /var/run/dockershim.sockname: k8s-master01taints:- effect: NoSchedulekey: node-role.kubernetes.io/master---apiServer:timeoutForControlPlane: 4m0sapiVersion: kubeadm.k8s.io/v1beta2certificatesDir: /etc/kubernetes/pkiclusterName: kubernetescontrollerManager: {}dns:type: CoreDNSetcd:local:dataDir: /var/lib/etcdimageRepository: registry.aliyuncs.com/google_containerskind: ClusterConfigurationkubernetesVersion: v1.23.0networking:dnsDomain: cluster.localpodSubnet: 10.244.0.0/16serviceSubnet: 10.96.0.0/12scheduler: {}

node

# 切换root用户
sudo su -# 更新源
apt update
# 升级包
apt upgrade -y# 设置hostname，可选
# hostnamectl set-hostname k8s-master01# 备份hosts文件
cp /etc/hosts /etc/hosts-bk
# 添加hosts，见下
vim /etc/hosts# 开启IPVS负载均衡，见下# 设置k8s所需内核参数，见下
vim /etc/sysctl.d/kubernetes.conf
# 应用内核参数
sysctl --system# docker
# 见大标题docker# 永久关闭swap。pod运行在swap分区中会大大影响效率
swapoff -a && sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab
# 禁用SELINUX：开启会限制服务进程访问资源
setenforce 0 && sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config
# 安装必要包
apt install -y curl gnupg2 software-properties-common apt-transport-https ca-certificates
# 开启路由功能
sysctl -w net.ipv4.ip_forward=1
# 重启虚拟机系统# kubernetes
# add aliyun/kubernetes apt key
curl https://mirrors.aliyun.com/kubernetes/apt/doc/apt-key.gpg | sudo apt-key add -
# add aliyun/Kubernetes apt repository
apt-add-repository "deb https://mirrors.aliyun.com/kubernetes/apt/ kubernetes-xenial main"
# 检查更新
apt update
# 安装kubelet kubeadm kubectl
apt install -y kubelet=1.23.0-00 kubeadm=1.23.0-00 kubectl=1.23.0-00
# 锁定版本，不随apt upgrade更新
apt-mark hold kubelet kubeadm kubectl
# 查看kubeadm版本
kubeadm version# 下面的命令来自 保存打印的内容，推荐使用普通用户
su xcrj02
# 来自kubeadm-init.log文件，192.168.66.10 masterIP
sudo kubeadm join 192.168.66.10:6443 --token xxcbh2.xdulqkbvvrup3b90 \--discovery-token-ca-cert-hash sha256:b84434080d676bf402aba832343faf07f119b2d261b95440ff47d67fd4d78eee

hosts

   #master01vi /etc/hosts192.168.66.10 k8s-master01192.168.66.20 k8s-node01192.168.66.21 k8s-node02#node01vi /etc/hosts192.168.66.10 k8s-master01192.168.66.20 k8s-node01#node02vi /etc/hosts192.168.66.10 k8s-master01192.168.66.21 k8s-node02

开启IPVS负载均衡

modprobe br_netfilter

或

cat > /etc/sysconfig/modules/ipvs.modules <<EOF
#!/bin/bash
modprobe br_netfilter
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack
EOF
#操作上面创建的文件
chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep -e ip_vs -e nf_conntrack_ipv4

kubernetes.conf

net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1