k8s高可用HAproxy容器化快速部署
1.基础环境配置
1.设置系统主机名以及 Host 文件的相互解析
[root@master3 /root] master3 #cat /etc/hosts 127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 192.168.83.101 master1 192.168.83.102 master2 192.168.83.103 master3 192.168.83.104 node1 192.168.83.105 node22.安装软件依赖
[root@master3 /root] master3 #yum install -y conntrack ntpdate ntp ipvsadm ipset jq iptables curl sysstat libseccomp wget vim git3.关闭安全
[root@master1 /root] master1 #systemctl stop firewalld && systemctl disable firewalld #yum -y install iptables-services && systemctl start iptables && systemctl enable iptables && iptables -F && service iptables save [root@master3 /root] master3 #swapoff -a && sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab [root@master3 /root] master3 #setenforce 0 && sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config4.调整内核参数，k8s相关
cat > kubernetes.conf << EOF net.bridge.bridge-nf-call-iptables=1 net.bridge.bridge-nf-call-ip6tables=1 net.ipv4.ip_forward=1 net.ipv4.tcp_tw_recycle=0 vm.swappiness=0 vm.overcommit_memory=1 vm.panic_on_oom=0 fs.inotify.max_user_instances=8192 fs.inotify.max_user_watches=1048576 fs.file-max=52706963 fs.nr_open=52706963 net.ipv6.conf.all.disable_ipv6=1 net.netfilter.nf_conntrack_max=2310720 EOF [root@master1 /root] master1 #cp kubernetes.conf /etc/sysctl.d/kubernetes.conf [root@master1 /root] master1 #sysctl -p /etc/sysctl.d/kubernetes.conf5.系统服务调整
[root@master1 /root] master1 #timedatectl set-timezone Asia/Shanghai #timedatectl set-local-rtc 0 将当前的 UTC 时间写入硬件时钟 #systemctl restart rsyslog #systemctl restart crond #systemctl stop postfix && systemctl disable postfix #mkdir /var/log/journal # 持久化保存日志的目录 #mkdir /etc/systemd/journald.conf.d #cat > /etc/systemd/journald.conf.d/99-prophet.conf << EOF [Journal] # 持久化保存到磁盘 Storage=persistent # 压缩历史日志 Compress=yes SyncIntervalSec=5m RateLimitInterval=30s RateLimitBurst=1000 # 最大占用空间 10G SystemMaxUse=10G # 单日志文件最大 200M SystemMaxFileSize=200M # 日志保存时间 2 周 MaxRetentionSec=2week # 不将日志转发到 syslog ForwardToSyslog=no EOF #systemctl restart systemd-journald6.可选升级内核
rpm -Uvh http://www.elrepo.org/elrepo-release-7.0-3.el7.elrepo.noarch.rpm # 安装完成后检查 /boot/grub2/grub.cfg 中对应内核 menuentry 中是否包含 initrd16 配置，如果没有，再安装 一次！ yum --enablerepo=elrepo-kernel install -y kernel-lt # 设置开机从新内核启动 grub2-set-default "CentOS Linux (4.4.182-1.el7.elrepo.x86_64) 7 (Core)" # 重启后安装内核源文件 yum --enablerepo=elrepo-kernel install kernel-lt-devel-$(uname -r) kernel-lt-headers-$(uname -r)7.可选关闭 NUMA
cp /etc/default/grub{,.bak} vim /etc/default/grub # 在 GRUB_CMDLINE_LINUX 一行添加 `numa=off` 参数，如下所示： diff /etc/default/grub.bak /etc/default/grub 6c6 < GRUB_CMDLINE_LINUX="crashkernel=auto rd.lvm.lv=centos/root rhgb quiet" --- > GRUB_CMDLINE_LINUX="crashkernel=auto rd.lvm.lv=centos/root rhgb quiet numa=off" cp /boot/grub2/grub.cfg{,.bak} grub2-mkconfig -o /boot/grub2/grub.cfg
2.安装k8s服务软件
1.ipvs配置
#modprobe br_netfilter #cat > /etc/sysconfig/modules/ipvs.modules << EOF #!/bin/bash modprobe -- ip_vs modprobe -- ip_vs_rr modprobe -- ip_vs_wrr modprobe -- ip_vs_sh modprobe -- nf_conntrack_ipv4 EOF #chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep -e ip_vs -e nf_conntrack_ipv42.安装docker
**确认docker适配，1.15建议docker18.09 ** #yum install -y yum-utils device-mapper-persistent-data lvm2 #yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo #yum list docker-ce --showduplicates | sort -r #查看docker版本 #yum update -y && yum install -y docker-ce #mkdir /etc/docker #cat > /etc/docker/daemon.json <<EOF { "exec-opts": ["native.cgroupdriver=systemd"], "log-driver": "json-file", "log-opts": { "max-size": "100m" } } EOF #mkdir -p /etc/systemd/system/docker.service.d #systemctl daemon-reload && systemctl restart docker && systemctl enable docker3.在主节点启动Haproxy 与 Keepalived 容器
#mkdir /usr/local/kuberbetes #cd !$ [root@master1 /usr/local/kuberbetes] master1 #ls -alh -rw-r--r-- 1 root root 71M Aug 5 2019 haproxy.tar -rw-r--r-- 1 root root 16M Aug 5 2019 keepalived.tar -rw-r--r-- 1 root root 231M Aug 5 2019 kubeadm-basic.images.tar.gz -rwxr-xr-x 1 root root 201 Jun 21 2019 load-images.sh -rw-r--r-- 1 root root 1.5K Aug 5 2019 start.keep.tar.gz #tar xf kubeadm-basic.images.tar.gz #ls -alh kubeadm-basic.images -rw------- 1 root root 199M Aug 5 2019 apiserver.tar -rw------- 1 root root 39M Aug 5 2019 coredns.tar -rw------- 1 root root 247M Aug 5 2019 etcd.tar -rw------- 1 root root 153M Aug 5 2019 kubec-con-man.tar -rw------- 1 root root 737K Aug 5 2019 pause.tar -rw------- 1 root root 81M Aug 5 2019 proxy.tar -rw------- 1 root root 79M Aug 5 2019 scheduler.tar # 修改脚本导入镜像 #docker images REPOSITORY TAG IMAGE ID CREATED SIZE k8s.gcr.io/kube-controller-manager v1.15.1 d75082f1d121 2 years ago 159MB k8s.gcr.io/kube-proxy v1.15.1 89a062da739d 2 years ago 82.4MB k8s.gcr.io/kube-scheduler v1.15.1 b0b3c4c404da 2 years ago 81.1MB k8s.gcr.io/kube-apiserver v1.15.1 68c3eb07bfc3 2 years ago 207MB k8s.gcr.io/coredns 1.3.1 eb516548c180 2 years ago 40.3MB k8s.gcr.io/etcd 3.3.10 2c4adeb21b4f 2 years ago 258MB wise2c/keepalived-k8s latest 0ba6a7862982 3 years ago 14MB wise2c/haproxy-k8s latest fde31577093d 3 years ago 71.1MB k8s.gcr.io/pause 3.1 da86e6ba6ca1 3 years ago 742kB #tar xf start.keep.tar.gz #mv /usr/local/kuberbetes/data / #[root@master1 /data/lb] master1 #tail /data/lb/etc/haproxy.cfg backend be_k8s_6443 mode tcp timeout queue 1h timeout server 1h timeout connect 1h log global balance roundrobin server rancher01 192.168.83.101:6443 server rancher02 192.168.83.102:6443 server rancher02 192.168.83.103:6443 # 第一次启动haproxy.cfg文件先只留下一个server [root@master1 /data/lb] master1 #cat /data/lb/start-haproxy.sh #!/bin/bash MasterIP1=192.168.83.101 MasterIP2=192.168.83.102 MasterIP3=192.168.83.103 MasterPort=6443 docker run -d --restart=always --name HAProxy-K8S -p 6444:6444 \ -e MasterIP1=$MasterIP1 \ -e MasterIP2=$MasterIP2 \ -e MasterIP3=$MasterIP3 \ -e MasterPort=$MasterPort \ -v /data/lb/etc/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg \ wise2c/haproxy-k8s #cat /data/lb/start-keepalived.sh #!/bin/bash VIRTUAL_IP=192.168.83.100 INTERFACE=ens33 NETMASK_BIT=24 CHECK_PORT=6444 RID=10 VRID=160 MCAST_GROUP=224.0.0.18 docker run -itd --restart=always --name=Keepalived-K8S \ --net=host --cap-add=NET_ADMIN \ -e VIRTUAL_IP=$VIRTUAL_IP \ -e INTERFACE=$INTERFACE \ -e CHECK_PORT=$CHECK_PORT \ -e RID=$RID \ -e VRID=$VRID \ -e NETMASK_BIT=$NETMASK_BIT \ -e MCAST_GROUP=$MCAST_GROUP \ wise2c/keepalived-k8s #修改VIP和网卡名称 [root@master1 /data/lb] master1 #ip a 2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000 link/ether 00:0c:29:0e:5f:da brd ff:ff:ff:ff:ff:ff inet 192.168.83.101/24 brd 192.168.83.255 scope global ens33 valid_lft forever preferred_lft forever inet 192.168.83.100/24 scope global secondary ens33 #确认已添加VIP #netstat -tunlp | grep 64444.安装Kubeadm（主从配置）
#cat < /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF
#yum -y install kubeadm-1.15.1 kubectl-1.15.1 kubelet-1.15.1
#systemctl enable kubelet.service && systemctl start kubelet.service5.初始化主节点
#kubeadm config print init-defaults > kubeadm-config.yaml修改kubeadm-config.yaml
advertiseAddress: 192.168.83.101 #修改master的地址 clusterName: kubernetes controlPlaneEndpoint: "192.168.83.100:6444" #添加VIP controllerManager: {} kubernetesVersion: v1.15.1#kubeadm init --config=kubeadm-config.yaml --experimental-upload-certs | tee kubeadm-init.log6.初始化集群遇到问题
#kubeadm reset #iptables -F && iptables -t nat -F && iptables -t mangle -F && iptables -X 其他参考 #ifconfig cni0 down #ip link delete cni0 #ifconfig flannel.1 down #ip link delete flannel.1 #rm -rf /var/lib/cni/7.加入主节点及其余工作节点
Your Kubernetes control-plane has initialized successfully! To start using your cluster, you need to run the following as a regular user: mkdir -p $HOME/.kube sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config sudo chown $(id -u):$(id -g) $HOME/.kube/config You should now deploy a pod network to the cluster. Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at: https://kubernetes.io/docs/concepts/cluster-administration/addons/ You can now join any number of the control-plane node running the following command on each as root: kubeadm join 192.168.122.80:6444 --token abcdef.0123456789abcdef \ --discovery-token-ca-cert-hash sha256:1ad02d18fc7256b9f5d5b58ae9e023113b51d7fdb0b14401cafe98a22ae89303 \ --control-plane --certificate-key 17c904507248e347d6f8b232008c7f5c6a3c972cb955ff33b37753b6349efc85 Please note that the certificate-key gives access to cluster sensitive data, keep it secret! As a safeguard, uploaded-certs will be deleted in two hours; If necessary, you can use "kubeadm init phase upload-certs --upload-certs" to reload certs afterward. Then you can join any number of worker nodes by running the following on each as root: kubeadm join 192.168.122.80:6444 --token abcdef.0123456789abcdef \ --discovery-token-ca-cert-hash sha256:1ad02d18fc7256b9f5d5b58ae9e023113b51d7fdb0b14401cafe98a22ae89303 # 注意 同样start容器haproxy和keepalived,配置信息仍用第一个。 按照初始化提示命令安装8.部署网络
kubectl apply -f kube-flannel.yaml kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/2140ac876ef134e0ed5af15c65e414cf26827915/Documentation/kube-flannel.yml
3.调整master高可用
1.haproxy重启
添加剩余server，然后start容器添加rancher
[root@k8s-master1 /data/lb/etc] k8s-master1 #cat haproxy.cfg global log 127.0.0.1 local0 log 127.0.0.1 local1 notice maxconn 4096 #chroot /usr/share/haproxy #user haproxy #group haproxy daemon defaults log global mode http option httplog option dontlognull retries 3 option redispatch timeout connect 5000 timeout client 50000 timeout server 50000 frontend stats-front bind *:8081 mode http default_backend stats-back frontend fe_k8s_6444 bind *:6444 mode tcp timeout client 1h log global option tcplog default_backend be_k8s_6443 acl is_websocket hdr(Upgrade) -i WebSocket acl is_websocket hdr_beg(Host) -i ws backend stats-back mode http balance roundrobin stats uri /haproxy/stats stats auth pxcstats:secret backend be_k8s_6443 mode tcp timeout queue 1h timeout server 1h timeout connect 1h log global balance roundrobin server rancher01 192.168.122.81:6443 server rancher02 192.168.122.82:6443 server rancher03 192.168.122.83:6443重拉HA容器
#docker ps | grep HAProxy-K8S 06cd27740576 wise2c/haproxy-k8s "/docker-entrypoint.…" 11 minutes ago Up 11 minutes 0.0.0.0:6444->6444/tcp HAProxy-K8S #docker rm -f HAProxy-K8S && bash /data/lb/start-haproxy.sh2.调整kube
APIServer中的VIP调整为本节点ip
#vim /root/.kube/config apiVersion: v1 clusters: - cluster: certificate-authority-data: server: https://192.168.122.81:6444 #每个master调整为本节点ip,不通过vip name: kubernetes contexts: - context: cluster: kubernetes user: kubernetes-admin name: kubernetes-admin@kubernetes current-context: kubernetes-admin@kubernetes kind: Config preferences: {} users: - name: kubernetes-admin
4.验证
Etcd 集群状态查看
kubectl -n kube-system exec etcd-k8s-master01 -- etcdctl \ --endpoints=https://192.168.92.10:2379 \ --ca-file=/etc/kubernetes/pki/etcd/ca.crt \ --cert-file=/etc/kubernetes/pki/etcd/server.crt \ --key-file=/etc/kubernetes/pki/etcd/server.key cluster-health kubectl get endpoints kube-controller-manager --namespace=kube-system -o yaml kubectl get endpoints kube-scheduler --namespace=kube-system -o yaml.