cat > etcd-csr.json <<EOF
{
  "CN": "etcd",
  "hosts": [
    "127.0.0.1",
    "192.168.10.81",
    "192.168.10.82",
    "192.168.10.83"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "GuangZhou",
      "L": "GuangZhou",
      "O": "k8s",
      "OU": "System"
    }
  ]
}
EOF

cfssl gencert -ca=ca.pem \
    -ca-key=ca-key.pem \
    -config=ca-config.json \
    -profile=kubernetes etcd-csr.json | cfssljson -bare etcd

ETCDHOSTS=(192.168.10.81 192.168.10.82 192.168.10.83)
for i in ${ETCDHOSTS[@]}; do scp -r /etc/etcd root@$i:/etc/; done

Oct 10 03:36:05 node1 etcd: rejected connection from "192.168.10.83:52514" (error "remote error: tls: bad certificate", ServerName "")
Oct 10 03:36:05 node1 etcd: rejected connection from "192.168.10.83:52518" (error "remote error: tls: bad certificate", ServerName "")
Oct 10 03:36:05 node1 etcd: rejected connection from "192.168.10.82:57950" (error "remote error: tls: bad certificate", ServerName "")
Oct 10 03:36:05 node1 etcd: rejected connection from "192.168.10.82:57952" (error "remote error: tls: bad certificate", ServerName "")

cat > /etc/haproxy/haproxy.cfg <<EOF
global
    log         127.0.0.1 local2
    chroot      /var/lib/haproxy
    pidfile     /var/run/haproxy.pid
    maxconn     4000
    user        haproxy
    group       haproxy
    daemon
    stats socket /var/lib/haproxy/stats

defaults
    mode                    http
    log                     global
    option                  httplog
    option                  dontlognull
    option http-server-close
    option forwardfor       except 127.0.0.0/8
    option                  redispatch
    retries                 3
    timeout http-request    10s
    timeout queue           1m
    timeout connect         10s
    timeout client          1m
    timeout server          1m
    timeout http-keep-alive 10s
    timeout check           10s
    maxconn                 3000

frontend kubernetes-apiserver
    mode                 tcp
    bind                 *:8443
    option               tcplog
    default_backend      kubernetes-apiserver

backend kubernetes-apiserver
    mode        tcp
    balance     roundrobin
    server      k8s01 192.168.10.81:6443 check
    server      k8s02 192.168.10.82:6443 check
    server      k8s03 192.168.10.83:6443 check

listen stats
    bind                 *:1080
    stats auth           admin:sunday
    stats refresh        5s
    stats realm          HAProxy\ Statistics
    stats uri            /haproxy-stats
EOF

systemctl enable haproxy 
systemctl start haproxy

ETCDHOSTS=(192.168.10.81 192.168.10.82 192.168.10.83)
for i in ${ETCDHOSTS[@]}; do scp /etc/haproxy/haproxy.cfg root@$i:/etc/haproxy/haproxy.cfg; done

cat > /etc/keepalived/keepalived.conf <<EOF
! Configuration File for keepalived
global_defs {
   router_id keepalived
}

vrrp_script check_haproxy {
    script "killall -0 haproxy"
    interval 2
    weight -3
    fall 2
    rise 2
}

vrrp_instance VI_1 {
    state MASTER
    interface eth0
    virtual_router_id 51
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass Sunday
    }
    virtual_ipaddress {
        192.168.10.80
    }
    track_script {
        check_haproxy
    }
}
EOF


ETCDHOSTS=(192.168.10.81 192.168.10.82 192.168.10.83)
for i in ${ETCDHOSTS[@]}; do 
    scp /etc/keepalived/keepalived.conf root@$i:/etc/keepalived/keepalived.conf
    ssh root@${ETCDHOSTS[1]} "sed -i -e 's#MASTER#BACKUP#g' -e 's#priority 100#priority 99#g' /etc/keepalived/keepalived.conf"
    ssh root@${ETCDHOSTS[2]} "sed -i -e 's#MASTER#BACKUP#g' -e 's#priority 100#priority 98#g' /etc/keepalived/keepalived.conf"
done

kubeadm init --config kubeadm-config.yaml --upload-certs
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

cat << EOF > busybox.yaml
apiVersion: v1
kind: Pod
metadata:
  name: busybox
  namespace: default
spec:
  containers:
  - name: busybox
    image: busybox:1.28
    command:
      - sleep
      - "3600"
    imagePullPolicy: IfNotPresent
  restartPolicy: Always
EOF

kubectl apply -f busybox.yaml 
kubectl get pods busybox
kubectl exec -ti busybox -- nslookup kubernetes.default

Server:    10.96.0.10
Address 1: 10.96.0.10 kube-dns.kube-system.svc.cluster.local

Name:      kubernetes.default
Address 1: 10.96.0.1 kubernetes.default.svc.cluster.local

kubectl logs -l  k8s-app=kibana-logging -nkube-system