0%

Kubernetes高可用集群搭建

介绍

Kubernetes是一个开源的,用于管理云平台中多个主机上的容器化的应用,Kubernetes的目标是让部署容器化的应用简单并且高效(powerful),Kubernetes提供了应用部署,规划,更新,维护的一种机制。

准备工作

  • 6台虚拟机,分别为3个master和3个node

部署基础环境

以下操作需要在所有虚拟机上都执行

环境调整

  1. 检查MAC地址和UUID是否重复

    1
    2
    ifconfig -a
    cat /sys/class/dmi/id/product_uuid
  2. 设置系统语言

    1
    echo 'LANG="en_US.UTF-8"' >> /etc/profile;source /etc/profile
  3. 同步系统时间

    注意:一定要先同步好时间,如果等k8s搭建好了在更新,可能会出现k8s证书过期的问题。

    1
    2
    timedatectl set-timezone Asia/Shanghai
    yum install -y ntp && timedatectl set-ntp yes
  4. 关闭防火墙

    1
    systemctl disable firewalld && systemctl stop firewalld
  5. 关闭swap

    1
    2
    swapoff -a
    sed -ri '/^[^#]*swap/s@^@#@' /etc/fstab
  6. 禁用selinux

    1
    2
    setenforce 0
    sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config
  7. 优化内核

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    cat > /etc/sysctl.d/k8s.conf << EOF
    net.bridge.bridge-nf-call-ip6tables = 1
    net.bridge.bridge-nf-call-iptables = 1
    EOF

    cat > /etc/sysctl.d/aliyun.conf << EOF
    net.ipv6.conf.all.disable_ipv6 = 1
    net.ipv6.conf.default.disable_ipv6 = 1
    net.ipv6.conf.lo.disable_ipv6 = 1

    vm.swappiness = 0
    net.ipv4.neigh.default.gc_stale_time=120


    # see details in https://help.aliyun.com/knowledge_detail/39428.html
    net.ipv4.conf.all.rp_filter=0
    net.ipv4.conf.default.rp_filter=0
    net.ipv4.conf.default.arp_announce = 2
    net.ipv4.conf.lo.arp_announce=2
    net.ipv4.conf.all.arp_announce=2


    # see details in https://help.aliyun.com/knowledge_detail/41334.html
    net.ipv4.tcp_max_tw_buckets = 5000
    net.ipv4.tcp_syncookies = 1
    net.ipv4.tcp_max_syn_backlog = 1024
    net.ipv4.tcp_synack_retries = 2
    kernel.sysrq=1
    EOF

    sysctl --system

    确保在此步骤之前已加载了 br_netfilter 模块。这可以通过运行 lsmod | grep br_netfilter 来完成。要显示加载它,请调用 modprobe br_netfilter

  8. 安装ipvs

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    yum install -y ipset ipvsadm

    cat > /etc/sysconfig/modules/ipvs.modules <<EOF
    #!/bin/bash
    modprobe -- ip_vs
    modprobe -- ip_vs_rr
    modprobe -- ip_vs_wrr
    modprobe -- ip_vs_sh
    modprobe -- nf_conntrack_ipv4
    EOF

    chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep -e ip_vs -e nf_conntrack_ipv4

安装Docker

  1. 安装依赖包

    1
    yum install -y yum-utils device-mapper-persistent-data lvm2
  2. 添加docker源

    1
    2
    3
    yum-config-manager \
    --add-repo \
    https://download.docker.com/linux/centos/docker-ce.repo
  3. 安装docker

    1
    2
    3
    4
    yum install -y \
    containerd.io-1.2.10 \
    docker-ce-19.03.4 \
    docker-ce-cli-19.03.4
  4. 配置Docker

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    mkdir /etc/docker
    cat > /etc/docker/daemon.json <<EOF
    {
    "exec-opts": ["native.cgroupdriver=systemd"],
    "log-driver": "json-file",
    "log-opts": {
    "max-size": "100m"
    },
    "storage-driver": "overlay2",
    "storage-opts": [
    "overlay2.override_kernel_check=true"
    ],
    "registry-mirrors": ["https://5onzn5tq.mirror.aliyuncs.com"]
    }
    EOF

    mkdir -p /etc/systemd/system/docker.service.d

    systemctl enable docker
    systemctl start docker
  1. 设置Docker代理(可选)

    如果使用的是国内节点这步是必做的,不然k8s的镜像是拉不到的。

    1
    2
    3
    4
    5
    cat <<EOF > /etc/systemd/system/docker.service.d/http-proxy.conf
    [Service]
    Environment="HTTP_PROXY=http://xxx.xxx.com:1080" "HTTPS_PROXY=http://xxx.xxx.com:1080"
    EOF
    systemctl daemon-reload && systemctl restart docker

    重启后运行命令systemctl show --property=Environment docker,当出现下面提示就表示成功了

    1
    Environment=HTTP_PROXY=http://xxx.xxx.com:1080 HTTPS_PROXY=http://xxx.xxx.com:1080

安装Kubernetes组件

  1. 添加源

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    cat <<EOF > /etc/yum.repos.d/kubernetes.repo
    [kubernetes]
    name=Kubernetes
    baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
    enabled=1
    gpgcheck=0
    repo_gpgcheck=0
    gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
    http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
    EOF
  2. 安装组件

    1
    2
    yum install -y kubelet kubeadm kubectl --disableexcludes=kubernetes
    systemctl enable --now kubelet

集群配置

高可用配置

在三个master节点上都需要执行以下操作

  1. 安装HAProxy

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    mkdir /etc/haproxy
    cat > /etc/haproxy/haproxy.cfg <<EOF
    global
    log 127.0.0.1 local0 err
    maxconn 50000
    uid 99
    gid 99
    #daemon
    nbproc 1
    pidfile haproxy.pid

    defaults
    mode http
    log 127.0.0.1 local0 err
    maxconn 50000
    retries 3
    timeout connect 5s
    # 以下两个超时时间过短会出现kubectl执行exec、logs命令时自动断开
    timeout client 600s
    timeout server 600s
    timeout check 2s

    frontend k8s_api_servers
    bind 0.0.0.0:8443
    mode tcp
    default_backend k8s_api_servers

    backend k8s_api_servers
    mode tcp
    balance roundrobin
    server k8s-master-node1 192.168.2.100:6443 weight 1 maxconn 1000 check inter 2000 rise 2 fall 3
    server k8s-master-node2 192.168.2.101:6443 weight 1 maxconn 1000 check inter 2000 rise 2 fall 3
    server k8s-master-node3 192.168.2.102:6443 weight 1 maxconn 1000 check inter 2000 rise 2 fall 3
    EOF

    docker run -d --name haproxy --restart always -v /etc/haproxy/:/usr/local/etc/haproxy:ro -p 8443:8443 haproxy:2.1.2
  2. 安装keepalived

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    mkdir /etc/keepalived
    cat > /etc/keepalived/keepalived.conf << EOF
    ! Configuration File for keepalived

    global_defs {
    router_id k8s-master-node1 #使用主机名即可,注意每个节点不相同
    }

    vrrp_instance VI_1 {
    state MASTER #当前节点在此虚拟路由器上的初始状态;只能有一个是MASTER,余下的都应该为BACKUP,启动后根据priority竞选;
    interface ens192 #VIP漂移到的网卡
    virtual_router_id 51 #当前虚拟路由器的惟一标识,范围是0-255, 所有节点保持一致
    priority 100 #当前主机在此虚拟路由器中的优先级;范围1-254;MASTER 至少高于BACKUP50
    advert_int 1
    authentication {
    auth_type PASS
    auth_pass csdc456csdc
    }
    virtual_ipaddress {
    192.168.2.99/24 # vip
    }
    }
    EOF

    docker run -d --name keepalived --restart always --cap-add=NET_ADMIN --net=host \
    -v /etc/keepalived/keepalived.conf:/container/service/keepalived/assets/keepalived.conf \
    osixia/keepalived:2.0.19 --copy-service

Master端配置

  1. 初始化master

    首先先导出配置文件

    1
    kubeadm config print init-defaults > kubeadm.yaml

    修改或新增以下带有注释的内容

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    39
    40
    41
    42
    43
    44
    45
    apiVersion: kubeadm.k8s.io/v1beta2
    bootstrapTokens:
    - groups:
    - system:bootstrappers:kubeadm:default-node-token
    token: abcdef.0123456789abcdef
    ttl: 24h0m0s
    usages:
    - signing
    - authentication
    kind: InitConfiguration
    localAPIEndpoint:
    advertiseAddress: 192.168.2.100 # 修改为本机ip
    bindPort: 6443
    nodeRegistration:
    criSocket: /var/run/dockershim.sock
    name: k8s-master-node1
    taints:
    - effect: NoSchedule
    key: node-role.kubernetes.io/master
    ---
    apiServer:
    timeoutForControlPlane: 4m0s
    apiVersion: kubeadm.k8s.io/v1beta2
    certificatesDir: /etc/kubernetes/pki
    clusterName: kubernetes
    controlPlaneEndpoint: "192.168.2.99:8443" # VIP地址 + HAProxy监听的端口
    controllerManager: {}
    dns:
    type: CoreDNS
    etcd:
    local:
    dataDir: /var/lib/etcd
    imageRepository: k8s.gcr.io
    kind: ClusterConfiguration
    kubernetesVersion: v1.17.0
    networking:
    dnsDomain: cluster.local
    podSubnet: 10.244.0.0/16 # 指定pod网络子网,使用fannel网络必须使用这个CIDR
    serviceSubnet: 10.96.0.0/12
    scheduler: {}

    ---
    apiVersion: kubeproxy.config.k8s.io/v1alpha1
    kind: KubeProxyConfiguration
    mode: ipvs # kube-proxy模式改为ipvs

    然后就可以开始初始化了

    1
    kubeadm init --config kubeadm.yaml --upload-certs | tee kubeadm-init.log

    当出现以下提示就表示master已经初始化成功了

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    .....
    [kubelet-finalize] Updating "/etc/kubernetes/kubelet.conf" to point to a rotatable kubelet client certificate and key
    [addons] Applied essential addon: CoreDNS
    [endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
    [addons] Applied essential addon: kube-proxy

    Your Kubernetes control-plane has initialized successfully!

    To start using your cluster, you need to run the following as a regular user:

    mkdir -p $HOME/.kube
    sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
    sudo chown $(id -u):$(id -g) $HOME/.kube/config

    You should now deploy a pod network to the cluster.
    Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
    https://kubernetes.io/docs/concepts/cluster-administration/addons/

    You can now join any number of the control-plane node running the following command on each as root:

    kubeadm join 192.168.2.99:8443 --token abcdef.0123456789abcdef \
    --discovery-token-ca-cert-hash sha256:352f92cbfcf163bb2adc6d580d1xxxxx33cf8d99d78020392aa123cefb597027 \
    --control-plane --certificate-key 8f53e757596677f86b7006aa21fd3869dxxxxx58dfe100a58808cb9dd5342232

    Please note that the certificate-key gives access to cluster sensitive data, keep it secret!
    As a safeguard, uploaded-certs will be deleted in two hours; If necessary, you can use
    "kubeadm init phase upload-certs --upload-certs" to reload certs afterward.

    Then you can join any number of worker nodes by running the following on each as root:

    kubeadm join 192.168.2.99:8443 --token abcdef.0123456789abcdef \
    --discovery-token-ca-cert-hash sha256:352f92cbfcf163bb2adc6xxxxx108f5c33cf8d99d78020392aa123cefb597027

    按照提示运行

    1
    2
    3
    mkdir -p $HOME/.kube
    sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
    sudo chown $(id -u):$(id -g) $HOME/.kube/config

    提示: 日志中有两条kubeadm join命令,其中第一条是用于添加master节点的,第二条是用于添加工作节点的。

  2. 应用flannel网络

    1
    kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/2140ac876ef134e0ed5af15c65e414cf26827915/Documentation/kube-flannel.yml
  1. 添加剩余的master节点

    使用第一条命令添加节点,如果没问题就会出现以下内容

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    ...
    This node has joined the cluster and a new control plane instance was created:

    * Certificate signing request was sent to apiserver and approval was received.
    * The Kubelet was informed of the new secure connection details.
    * Control plane (master) label and taint were applied to the new node.
    * The Kubernetes control plane instances scaled up.
    * A new etcd member was added to the local/stacked etcd cluster.

    To start administering your cluster from this node, you need to run the following as a regular user:

    mkdir -p $HOME/.kube
    sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
    sudo chown $(id -u):$(id -g) $HOME/.kube/config

    uid 99
    Run 'kubectl get nodes' to see this node join the cluster.

    然后也按照提示执行以下命令即可

    1
    2
    3
    mkdir -p $HOME/.kube
    sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
    sudo chown $(id -u):$(id -g) $HOME/.kube/config

Node端配置

运行第二条kubeadm join命令,当出现以下提示时就表示加入工作节点成功了

1
2
3
4
5
6
7
8
9
10
11
12
13
14
[preflight] Running pre-flight checks
[preflight] Reading configuration from the cluster...
[preflight] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -oyaml'
[kubelet-start] Downloading configuration for the kubelet from the "kubelet-config-1.17" ConfigMap in the kube-system namespace
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Starting the kubelet
[kubelet-start] Waiting for the kubelet to perform the TLS Bootstrap...

This node has joined the cluster:
* Certificate signing request was sent to apiserver and a response was received.
* The Kubelet was informed of the new secure connection details.

Run 'kubectl get nodes' on the control-plane to see this node join the cluster.

检查集群状态

至此集群已经搭建成功了,可使用kubectl get nodes查看每个节点的状态,当全部Ready了就可以使用了。

1
2
3
4
5
6
7
NAME               STATUS   ROLES    AGE   VERSION
k8s-master-node1 Ready master 54m v1.17.2
k8s-master-node2 Ready master 51m v1.17.2
k8s-master-node3 Ready master 51m v1.17.2
k8s-worker-node1 Ready <none> 51m v1.17.2
k8s-worker-node2 Ready <none> 51m v1.17.2
k8s-worker-node3 Ready <none> 51m v1.17.2

收尾工作

  1. 取消Docker代理 (可选)

    1
    2
    rm -f /etc/systemd/system/docker.service.d/http-proxy.conf
    systemctl daemon-reload && systemctl restart docker