一、前言

? ? 二進(jìn)制部署1.23.15版本k8s集群，etcd集群部署與k8s集群節(jié)點(diǎn)復(fù)用，手動(dòng)頒發(fā)集群證書

? ? 主機(jī)信息如下

二、部署

? ?集群所有節(jié)點(diǎn)主機(jī)基礎(chǔ)優(yōu)化

? ?1.關(guān)閉防火墻

? ??systemctl stop firewalld &&?systemctl disable firewalld

? ?2.關(guān)閉selinux服務(wù)

setenforce 0? #臨時(shí)關(guān)閉

sed -i 's/enforcing/disabled/' /etc/selinux/config #永久關(guān)閉

3.關(guān)閉系統(tǒng)交換分區(qū)

swapoff -a #臨時(shí)關(guān)閉

sed -ri 's/.*swap.*/#&/' /etc/fstab? #永久關(guān)閉

4.設(shè)置主機(jī)名稱(三臺(tái)主機(jī)名稱不一樣按照表格上的主機(jī)名填)

hostnamectl set-hostname k8s-master01

5.編輯hosts文件

cat > /etc/hosts << EOF

10.1.60.125 k8s-master01

10.1.60.126 k8s-node01

10.1.60.127 k8s-node02

10.1.60.128 k8s-node03

EOF

6.配置時(shí)間同步

yum -y install chrony

systemctl strat chronyd &&?systemctl enbale?chronyd

7.配置ipvs

cat > /etc/sysconfig/modules/ipvs.modules <<EOF

#!/bin/bash

modprobe -- ip_vs

modprobe -- ip_vs_rr

modprobe -- ip_vs_wrr

modprobe -- ip_vs_sh

modprobe -- nf_conntrack_ipv4

EOF

#授權(quán)并生效ipvs配置

chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules

yum install -y ipset ipvsadm

cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf

overlay

br_netfilter

EOF

?sudo modprobe overlay &&?sudo modprobe br_netfilter

8.將橋接的IPv4流量傳遞到iptables

cat > /etc/sysctl.d/k8s.conf << EOF

net.bridge.bridge-nf-call-ip6tables = 1

net.bridge.bridge-nf-call-iptables = 1

net.ipv4.ip_forward = 1

EOF

sysctl --system? #配置生效

9.配置所有主機(jī)互相免密登錄

ssh-keygen

ssh-copy-id -i ~/.ssh/id_rsa.pub 10.1.60.126 ? #將公鑰拷貝到126主機(jī)上，實(shí)現(xiàn)免密登錄

ssh-copy-id -i ~/.ssh/id_rsa.pub 10.1.60.127

ssh-copy-id -i ~/.ssh/id_rsa.pub 10.1.60.128

部署k8s集群服務(wù)?

安裝證書生成工具cfssl

wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64
chmod +x cfssl_linux-amd64 cfssljson_linux-amd64 cfssl-certinfo_linux-amd64
mv cfssl_linux-amd64 /usr/local/bin/cfssl
mv cfssljson_linux-amd64 /usr/local/bin/cfssljson
mv cfssl-certinfo_linux-amd64 /usr/local/bin/cfssl-certinfo

創(chuàng)建證書存儲(chǔ)目錄

mkdir -p ~/TLS/{etcd,k8s}
cd TLS/etcd

創(chuàng)建證書文件

vi ca-config.json

{
  "signing": {
    "default": {
      "expiry": "87600h"
    },
    "profiles": {
      "www": {
         "expiry": "87600h",
         "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ]
      }
    }
  }
}

vi?ca-csr.json

{
    "CN": "etcd CA",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "ShenZhen",
            "ST": "ShenZhen"
        }
    ]
}

生成根證書

cfssl gencert -initca ca-csr.json | cfssljson -bare ca -

查看目錄下是否生成ca-key.pem和ca.pem這兩個(gè)根證書

ls *.pem

創(chuàng)建etcd證書申請(qǐng)文件

vi?server-csr.json

{
    "CN": "etcd",
    "hosts": [
    "10.1.60.125",    #填寫etcd集群所有節(jié)點(diǎn)的ip地址
    "10.1.60.126",
    "10.1.60.127"
   
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C:": "CN",
            "L": "ShenZhen",
            "ST": "ShenZhen"
        }
    ]
}

?生成etcd證書

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes server-csr.json | cfssljson -bare server

查看目錄下是否生成server-key.pem server.pem這兩個(gè)證書文件

ls *.pem

部署etcd集群

?查看k8s集群版本對(duì)應(yīng)版本的etcd版本，我這里使用的是3.5.1版本的etcd

安裝包下載參考：https://github.com/etcd-io/etcd/releases/download/v3.5.1/etcd-v3.5.1-linux-amd64.tar.gz

創(chuàng)建etcd工作目錄并解壓安裝包?

cd ~
mkdir -p /opt/etcd/{bin,cfg,ssl}?
wget https://github.com/etcd-io/etcd/releases/download/v3.5.1/etcd-v3.5.1-linux-amd64.tar.gz
tar -zxvf etcd-v3.5.1-linux-amd64.tar.gz
mv etcd-v3.5.1-linux-amd64/{etcd,etcdctl} /opt/etcd/bin/

創(chuàng)建etcd配置文件

vi /opt/etcd/cfg/etcd.conf

#[Member]
ETCD_NAME="etcd-1"   #etcd集群中的節(jié)點(diǎn)名稱
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"   #etcd數(shù)據(jù)目錄
ETCD_LISTEN_PEER_URLS="https://10.1.60.125:2380"      #etcd節(jié)點(diǎn)通訊地址端口
ETCD_LISTEN_CLIENT_URLS="https://10.1.60.125:2379"    #etcd客戶端訪問地址端口
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://10.1.60.125:2380"  #集群通訊地址端口
ETCD_ADVERTISE_CLIENT_URLS="https://10.1.60.125:2379"       #集群客戶端通訊地址端口
ETCD_INITIAL_CLUSTER="etcd-1=https://10.1.60.125:2380,etcd-2=https://10.1.60.126:2380,etcd-3=https://10.1.60.127:2380"        #etcd集群所有節(jié)點(diǎn)地址和端口
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"   #集群之間通訊的token
ETCD_INITIAL_CLUSTER_STATE="new"          #加入etcd集群的狀態(tài)，new是新集群，existing是加入已有集群

使用systemd管理etcd服務(wù)

vi?/usr/lib/systemd/system/etcd.service

[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
[Service]
Type=notify
EnvironmentFile=/opt/etcd/cfg/etcd.conf    #指定配置文件
ExecStart=/opt/etcd/bin/etcd \
    --cert-file=/opt/etcd/ssl/server.pem \     #指定證書
    --key-file=/opt/etcd/ssl/server-key.pem \
    --peer-cert-file=/opt/etcd/ssl/server.pem \
    --peer-key-file=/opt/etcd/ssl/server-key.pem \
    --trusted-ca-file=/opt/etcd/ssl/ca.pem \
    --peer-trusted-ca-file=/opt/etcd/ssl/ca.pem \
    --logger=zap
Restart=on-failure
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target

拷貝生成的etcd證書到etcd工作目錄下

cp ~/TLS/etcd/ca*pem /opt/etcd/ssl/
cp ~/TLS/etcd/server*pem /opt/etcd/ssl/

將節(jié)點(diǎn)1上生成的關(guān)于etcd的所有文件拷貝到其余兩個(gè)etcd節(jié)點(diǎn)上

#拷貝到etcd節(jié)點(diǎn)二上
scp -r /opt/etcd/ root@10.1.60.126:/opt/
scp /usr/lib/systemd/system/etcd.service root@10.1.60.126:/usr/lib/systemd/system/
#拷貝到etcd節(jié)點(diǎn)三上
scp -r /opt/etcd/ root@10.1.60.127:/opt/
scp /usr/lib/systemd/system/etcd.service root@10.1.60.127:/usr/lib/systemd/system/

拷貝完成后編輯兩個(gè)節(jié)點(diǎn)的etcd配置文件

vi?/opt/etcd/cfg/etcd.conf

#[Member]
ETCD_NAME="etcd-2"        #更改此項(xiàng)為不同的節(jié)點(diǎn)名稱
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://10.1.60.126:2380"     #更改此項(xiàng)為該節(jié)點(diǎn)對(duì)應(yīng)的ip地址
ETCD_LISTEN_CLIENT_URLS="https://10.1.60.126:2379"    #更改此項(xiàng)為該節(jié)點(diǎn)對(duì)應(yīng)的ip地址
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://10.1.60.126:2380"  #更改此項(xiàng)為該節(jié)點(diǎn)對(duì)應(yīng)的ip地址
ETCD_ADVERTISE_CLIENT_URLS="https://10.1.60.126:2379"    #更改此項(xiàng)為該節(jié)點(diǎn)對(duì)應(yīng)的ip地址
ETCD_INITIAL_CLUSTER="etcd-1=https://10.1.60.125:2380,etcd-2=https://10.1.60.126:2380,etcd-3=https://10.1.60.127:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster" 
ETCD_INITIAL_CLUSTER_STATE="new"

etcd集群所有節(jié)點(diǎn)啟動(dòng)etcd服務(wù)

systemctl daemon-reload

systemctl?start etcd &&?systemctl?enable etcd

etcd集群所有節(jié)點(diǎn)服務(wù)啟動(dòng)后查看狀態(tài)是否正常

ETCDCTL_API=3 /opt/etcd/bin/etcdctl --cacert=/opt/etcd/ssl/ca.pem --cert=/opt/etcd/ssl/server.pem --key=/opt/etcd/ssl/server-key.pem --endpoints="https://10.1.60.125:2379,https://10.1.60.126:2379,https://10.1.60.127:2379" endpoint health

輸出以下信息表面etcd集群狀態(tài)正常

k8s集群所有節(jié)點(diǎn) 安裝docker服務(wù)

docker版本也需要對(duì)應(yīng)k8s版本，這里選用19.03.9版本

下載參考：https://download.docker.com/linux/static/stable/x86_64/docker-19.03.9.tgz

下載安裝包并解壓?

cd ~
wget https://download.docker.com/linux/static/stable/x86_64/docker-19.03.9.tgz
tar -zxvf docker-19.03.9.tgz
mv docker/* /usr/bin

?使用systemd管理docker服務(wù)

vi?/usr/lib/systemd/system/docker.service

[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network-online.target firewalld.service
Wants=network-online.target

[Service]
Type=notify
ExecStart=/usr/bin/dockerd
ExecReload=/bin/kill -s HUP $MAINPID
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
TimeoutStartSec=0
Delegate=yes
KillMode=process
Restart=on-failure
StartLimitBurst=3
StartLimitInterval=60s

[Install]
WantedBy=multi-user.target

配置docker鏡像加速

mkdir /etc/docker

vi /etc/docker/daemon.json

{
  "registry-mirrors": ["https://62d7sxch.mirror.aliyuncs.com"]   #配置阿里云鏡像加速地址
}

啟動(dòng)docker服務(wù)

systemctl daemon-reload

systemctl start docker &&?systemctl enable docker

創(chuàng)建k8s集群自簽證書文件

cd ~/TLS/k8s

vi ca-config.json

{
  "signing": {
    "default": {
      "expiry": "87600h"
    },
    "profiles": {
      "kubernetes": {
         "expiry": "87600h",
         "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ]
      }
    }
  }
}

vi?ca-csr.json

{
    "CN": "kubernetes",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "ShenZhen",
            "ST": "ShenZhen",
            "O": "k8s",
            "OU": "System"
        }
    ]
}

生成根證書

cfssl gencert -initca ca-csr.json | cfssljson -bare ca -

查看是否生成ca-key.pem和ca.pem這兩個(gè)根證書

ls *.pem

創(chuàng)建kube-apiserver證書申請(qǐng)文件

vi?server-csr.json

{
    "CN": "kubernetes",
    "hosts": [
      "20.0.0.1",        #定義k8s集群地址，若是一個(gè)局域網(wǎng)中有多個(gè)k8s集群，k8s集群地址的網(wǎng)段不能相同，否則會(huì)沖突
      "127.0.0.1",
      "10.1.60.125",    #填寫k8s集群節(jié)點(diǎn)地址
      "10.1.60.126",
      "10.1.60.127",
      "10.1.60.128",
      "kubernetes",
      "kubernetes.default",
      "kubernetes.default.svc",
      "kubernetes.default.svc.cluster",
      "kubernetes.default.svc.cluster.local"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "ShenZhen",
            "ST": "ShenZhen",
            "O": "k8s",
            "OU": "System"
        }
    ]
}

?生成kube-apiserver證書

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes server-csr.json | cfssljson -bare server

查看生成的證書

ls server*pem

創(chuàng)建kube-proxy證書申請(qǐng)文件

vi?kube-proxy-csr.json

{
  "CN": "system:kube-proxy",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "L": "ShenZhen",
      "ST": "ShenZhen",
      "O": "k8s",
      "OU": "System"
    }
  ]
}

?生成kube-proxy證書

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy

查看生成的證書

ls kube-proxy*pem

部署master節(jié)點(diǎn)所需服務(wù)

?在gitlab中下載1.23.10版本的k8s server安裝包

參考：https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.23.md#downloads-for-v12310

?

下載并解壓k8s server安裝包

cd ~
wget https://dl.k8s.io/v1.23.10/kubernetes-server-linux-amd64.tar.gz
tar -zxvf kubernetes-server-linux-amd64.tar.gz

創(chuàng)建k8s工作目錄

mkdir -p /opt/kubernetes/{bin,cfg,ssl,logs}
cd kubernetes/server/bin
cp kube-apiserver kube-scheduler kube-controller-manager /opt/kubernetes/bin
cp kubectl /usr/bin/

部署kube-apiserver服務(wù)

創(chuàng)建kube-apiserver配置文件

vi?/opt/kubernetes/cfg/kube-apiserver.conf

KUBE_APISERVER_OPTS="--allow-privileged=true \   #啟用授權(quán)
--feature-gates=RemoveSelfLink=false \   #K8S1.20后移除了對(duì)SelfLink的支持，而NFS需要SelfLink的支持
--etcd-servers=https://10.1.60.125:2379,https://10.1.60.126:2379,https://10.1.60.127:2379 \     #配置etcd集群所有節(jié)點(diǎn)地址和客戶端端口
--bind-address=10.1.60.125 \    #apiserver服務(wù)監(jiān)聽地址
--secure-port=6443 \          #apiserver服務(wù)監(jiān)聽端口
--advertise-address=10.1.60.125 \     #k8s集群通訊地址
--service-cluster-ip-range=20.0.0.0/24 \   #k8s集群網(wǎng)絡(luò)地址段，不能和局域網(wǎng)中的其它k8s集群地址段相同
--enable-admission-plugins=NodeRestriction \     #準(zhǔn)入控制插件
--authorization-mode=RBAC,Node \       #授權(quán)模式
--enable-bootstrap-token-auth=true \   #啟用 bootstrap-tokenren認(rèn)證，為了讓node節(jié)點(diǎn)kubelet自由的頒發(fā)證書
--token-auth-file=/opt/kubernetes/cfg/token.csv \    #指定token文件
--service-node-port-range=30000-32767 \            #定義nodeport使用端口范圍
--kubelet-client-certificate=/opt/kubernetes/ssl/server.pem \   #指定apiserver訪問kubelet客戶端的證書
--kubelet-client-key=/opt/kubernetes/ssl/server-key.pem \     #指定apiserver訪問kubelet客戶端的證書
--tls-cert-file=/opt/kubernetes/ssl/server.pem  \        #指定apiserver https證書
--tls-private-key-file=/opt/kubernetes/ssl/server-key.pem \       #指定apiserver https證書
--client-ca-file=/opt/kubernetes/ssl/ca.pem \
--service-account-key-file=/opt/kubernetes/ssl/ca-key.pem \
--service-account-issuer=api \
--service-account-signing-key-file=/opt/kubernetes/ssl/ca-key.pem \
--etcd-cafile=/opt/etcd/ssl/ca.pem \         #指定連接etcd集群證書
--etcd-certfile=/opt/etcd/ssl/server.pem \
--etcd-keyfile=/opt/etcd/ssl/server-key.pem \
--requestheader-client-ca-file=/opt/kubernetes/ssl/ca.pem \
--proxy-client-cert-file=/opt/kubernetes/ssl/server.pem \
--proxy-client-key-file=/opt/kubernetes/ssl/server-key.pem \
--requestheader-allowed-names=kubernetes \
--requestheader-extra-headers-prefix=X-Remote-Extra- \
--requestheader-group-headers=X-Remote-Group \
--requestheader-username-headers=X-Remote-User \
--enable-aggregator-routing=true \
--audit-log-maxage=30 \
--audit-log-maxbackup=3 \
--audit-log-maxsize=100 \
--audit-log-path=/opt/kubernetes/logs/k8s-audit.log"

?拷貝根證書和apiserver證書到k8s工作目錄中

cp ~/TLS/k8s/ca*pem /opt/kubernetes/ssl/
cp ~/TLS/k8s/server*pem /opt/kubernetes/ssl/

生成token文件所需的密鑰

head -c 16 /dev/urandom | od -An -t x | tr -d ' '

使用生成的密鑰編輯token文件

vi?/opt/kubernetes/cfg/token.csv

c5cdf6375ed70ce66c52de20eb350bb6,kubelet-bootstrap,10001,"system:node-bootstrapper"

使用systemd管理kube-apiserver服務(wù)

vi?/usr/lib/systemd/system/kube-apiserver.service

[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes

[Service]
EnvironmentFile=/opt/kubernetes/cfg/kube-apiserver.conf
ExecStart=/opt/kubernetes/bin/kube-apiserver $KUBE_APISERVER_OPTS
Restart=on-failure

[Install]
WantedBy=multi-user.target

?啟動(dòng)kube-apiserver服務(wù)

systemctl daemon-reload

systemctl start kube-apiserver &&?systemctl enable kube-apiserver

部署kube-controller-manager服務(wù)

編輯kube-controller-manager服務(wù)配置文件

vi?/opt/kubernetes/cfg/kube-controller-manager.conf

KUBE_CONTROLLER_MANAGER_OPTS="--leader-elect=true \
--kubeconfig=/opt/kubernetes/cfg/kube-controller-manager.kubeconfig \
--bind-address=127.0.0.1 \
--allocate-node-cidrs=true \
--cluster-cidr=20.244.0.0/16 \    #指定k8s pod服務(wù)使用的網(wǎng)段
--service-cluster-ip-range=20.0.0.0/24 \    #指定k8s ervice服務(wù)使用的網(wǎng)段
--cluster-signing-cert-file=/opt/kubernetes/ssl/ca.pem \    #指定kubelet頒發(fā)證書所需的根證書
--cluster-signing-key-file=/opt/kubernetes/ssl/ca-key.pem  \
--root-ca-file=/opt/kubernetes/ssl/ca.pem \
--service-account-private-key-file=/opt/kubernetes/ssl/ca-key.pem \
--cluster-signing-duration=87600h0m0s"   #指定kubelete頒發(fā)證書的有效期

創(chuàng)建controller-manager服務(wù)證書配置文件

cd?~/TLS/k8s

vi?kube-controller-manager-csr.json

{
  "CN": "system:kube-controller-manager",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "L": "ShenZhen", 
      "ST": "ShenZhen",
      "O": "system:masters",
      "OU": "System"
    }
  ]
}

生成證書文件

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-controller-manager-csr.json | cfssljson -bare kube-controller-manager

查看生成的證書文件

ls kube-controller-manager*.pem

創(chuàng)建生成kubeconfig文件腳本

vi?kube-controller-manager.sh

KUBE_CONFIG="/opt/kubernetes/cfg/kube-controller-manager.kubeconfig"
KUBE_APISERVER="https://10.1.60.125:6443"   #填寫kube-apiserver服務(wù)地址和端口

kubectl config set-cluster kubernetes \
  --certificate-authority=/opt/kubernetes/ssl/ca.pem \
  --embed-certs=true \
  --server=${KUBE_APISERVER} \
  --kubeconfig=${KUBE_CONFIG}
kubectl config set-credentials kube-controller-manager \
  --client-certificate=./kube-controller-manager.pem \
  --client-key=./kube-controller-manager-key.pem \
  --embed-certs=true \
  --kubeconfig=${KUBE_CONFIG}
kubectl config set-context default \
  --cluster=kubernetes \
  --user=kube-controller-manager \
  --kubeconfig=${KUBE_CONFIG}
kubectl config use-context default --kubeconfig=${KUBE_CONFIG}

賦權(quán)執(zhí)行腳本生成kubeconfig文件

chmod +x?kube-controller-manager.sh

sh?kube-controller-manager.sh

使用systemd管理kube-controller-manager服務(wù)

vi?/usr/lib/systemd/system/kube-controller-manager.service

[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/kubernetes/kubernetes

[Service]
EnvironmentFile=/opt/kubernetes/cfg/kube-controller-manager.conf
ExecStart=/opt/kubernetes/bin/kube-controller-manager $KUBE_CONTROLLER_MANAGER_OPTS
Restart=on-failure

[Install]
WantedBy=multi-user.target

啟動(dòng)kube-controller-manager服務(wù)

systemctl daemon-reload
systemctl start kube-controller-manager && systemctl enable kube-controller-manager

部署kube-scheduler服務(wù)

編輯kube-scheduler服務(wù)配置文件

vi?/opt/kubernetes/cfg/kube-scheduler.conf

KUBE_SCHEDULER_OPTS="--leader-elect \
--kubeconfig=/opt/kubernetes/cfg/kube-scheduler.kubeconfig \
--bind-address=127.0.0.1"

創(chuàng)建kube-scheduler服務(wù)證書配置文件

vi?kube-scheduler-csr.json

{
  "CN": "system:kube-scheduler",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "L": "ShenZhen",
      "ST": "ShenZhen",
      "O": "system:masters",
      "OU": "System"
    }
  ]
}

生成證書文件

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-scheduler-csr.json | cfssljson -bare kube-scheduler

查看生成的證書文件

ls kube-scheduler*.pem

?編輯生成kubeconfig配置文件腳本

vi?kube-scheduler.sh

KUBE_CONFIG="/opt/kubernetes/cfg/kube-scheduler.kubeconfig"
KUBE_APISERVER="https://10.1.60.125:6443"

kubectl config set-cluster kubernetes \
  --certificate-authority=/opt/kubernetes/ssl/ca.pem \
  --embed-certs=true \
  --server=${KUBE_APISERVER} \
  --kubeconfig=${KUBE_CONFIG}
kubectl config set-credentials kube-scheduler \
  --client-certificate=./kube-scheduler.pem \
  --client-key=./kube-scheduler-key.pem \
  --embed-certs=true \
  --kubeconfig=${KUBE_CONFIG}
kubectl config set-context default \
  --cluster=kubernetes \
  --user=kube-scheduler \
  --kubeconfig=${KUBE_CONFIG}
kubectl config use-context default --kubeconfig=${KUBE_CONFIG}

賦權(quán)執(zhí)行腳本生成kubeconfig文件

chmod +x kube-scheduler.sh
sh kube-scheduler.sh

使用systemd管理kube-scheduler服務(wù)

vi?/usr/lib/systemd/system/kube-scheduler.service

[Unit]
Description=Kubernetes Scheduler
Documentation=https://github.com/kubernetes/kubernetes

[Service]
EnvironmentFile=/opt/kubernetes/cfg/kube-scheduler.conf
ExecStart=/opt/kubernetes/bin/kube-scheduler $KUBE_SCHEDULER_OPTS
Restart=on-failurea

[Install]
WantedBy=multi-user.target

啟動(dòng)kube-scheduler服務(wù)

systemctl daemon-reload
systemctl start kube-scheduler && systemctl enable kube-scheduler

創(chuàng)建kubectl證書配置文件

vi?admin-csr.json

{
  "CN": "admin",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "L": "ShenZhen",
      "ST": "ShenZhen",
      "O": "system:masters",
      "OU": "System"
    }
  ]
}

生成證書文件

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes admin-csr.json | cfssljson -bare admin

查看生成的證書文件

ls admin*.pem

?編輯生成kubeconfig的配置文件腳本

vi kubectl.sh

KUBE_CONFIG="/root/.kube/config"
KUBE_APISERVER="https://10.1.60.125:6443"

kubectl config set-cluster kubernetes \
  --certificate-authority=/opt/kubernetes/ssl/ca.pem \
  --embed-certs=true \
  --server=${KUBE_APISERVER} \
  --kubeconfig=${KUBE_CONFIG}
kubectl config set-credentials cluster-admin \
  --client-certificate=./admin.pem \
  --client-key=./admin-key.pem \
  --embed-certs=true \
  --kubeconfig=${KUBE_CONFIG}
kubectl config set-context default \
  --cluster=kubernetes \
  --user=cluster-admin \
  --kubeconfig=${KUBE_CONFIG}
kubectl config use-context default --kubeconfig=${KUBE_CONFIG}

創(chuàng)建kubectl的kubeconfig存放目錄

mkdir /root/.kube

賦權(quán)執(zhí)行腳本生成kubeconfig文件

chmod +x kubectl.sh
sh kubectl.sh

使用kubectl命令查看master節(jié)點(diǎn)上的組件是否正常運(yùn)行

kubectl get cs

?授權(quán)kubelet-bootstrap用戶允許請(qǐng)求證書

kubectl create clusterrolebinding kubelet-bootstrap \
--clusterrole=system:node-bootstrapper \
--user=kubelet-bootstrap
#若想刪除則使用以下命令
#kubectl delete clusterrolebinding kubelet-bootstrap

部署work節(jié)點(diǎn)所需服務(wù)

master節(jié)點(diǎn)也作為work節(jié)點(diǎn)

在master節(jié)點(diǎn)拷貝work節(jié)點(diǎn)所需的組件服務(wù)到k8s工作目錄

cd kubernetes/server/bin
cp kubelet kube-proxy /opt/kubernetes/bin    #將work節(jié)點(diǎn)所需服務(wù)組件拷貝到k8s工作目錄

部署kubelet服務(wù)

編輯kubelet配置文件

vi?/opt/kubernetes/cfg/kubelet.conf

KUBELET_OPTS="--hostname-override=k8s-master01-test \
--network-plugin=cni \
--kubeconfig=/opt/kubernetes/cfg/kubelet.kubeconfig \
--bootstrap-kubeconfig=/opt/kubernetes/cfg/bootstrap.kubeconfig \
--config=/opt/kubernetes/cfg/kubelet-config.yml \
--cert-dir=/opt/kubernetes/ssl \
--pod-infra-container-image=registry.cn-hangzhou.aliyuncs.com/google-containers/pause-amd64:3.0"   #管理Pod網(wǎng)絡(luò)容器的鏡像

創(chuàng)建kubelet定義參數(shù)文件

vi?/opt/kubernetes/cfg/kubelet-config.yml

kind: KubeletConfiguration
apiVersion: kubelet.config.k8s.io/v1beta1
address: 0.0.0.0
port: 10250
readOnlyPort: 10255
cgroupDriver: cgroupfs
clusterDNS:
- 20.0.0.2        #定義dns服務(wù)的ip地址，后面部署coredns服務(wù)會(huì)使用到
clusterDomain: cluster.local 
failSwapOn: false
authentication:
  anonymous:
    enabled: false
  webhook:
    cacheTTL: 2m0s
    enabled: true
  x509:
    clientCAFile: /opt/kubernetes/ssl/ca.pem 
authorization:
  mode: Webhook
  webhook:
    cacheAuthorizedTTL: 5m0s
    cacheUnauthorizedTTL: 30s
evictionHard:
  imagefs.available: 15%
  memory.available: 100Mi
  nodefs.available: 10%
  nodefs.inodesFree: 5%
maxOpenFiles: 1000000
maxPods: 110

編輯生成kubeconfig的配置文件腳本

vi kubelet.sh

KUBE_CONFIG="/opt/kubernetes/cfg/bootstrap.kubeconfig"
KUBE_APISERVER="https://10.1.60.125:6443"
TOKEN="c5cdf6375ed70ce66c52de20eb350bb6"   #該token需要和上面生成的token.csv中的token一致

kubectl config set-cluster kubernetes \
  --certificate-authority=/opt/kubernetes/ssl/ca.pem \
  --embed-certs=true \
  --server=${KUBE_APISERVER} \
  --kubeconfig=${KUBE_CONFIG}
kubectl config set-credentials "kubelet-bootstrap" \
  --token=${TOKEN} \
  --kubeconfig=${KUBE_CONFIG}
kubectl config set-context default \
  --cluster=kubernetes \
  --user="kubelet-bootstrap" \
  --kubeconfig=${KUBE_CONFIG}
kubectl config use-context default --kubeconfig=${KUBE_CONFIG}

賦權(quán)并執(zhí)行腳本生成kubeconfig文件

chmod +x kubelet.sh
sh kubelet.sh

使用systemd管理kublete服務(wù)

[Unit]
Description=Kubernetes Kubelet
After=docker.service

[Service]
EnvironmentFile=/opt/kubernetes/cfg/kubelet.conf
ExecStart=/opt/kubernetes/bin/kubelet $KUBELET_OPTS
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target

啟動(dòng)kubelet服務(wù)

systemctl daemon-reload
systemctl start kubelet && systemctl enable kubelet

審批kubelet證書申請(qǐng)并加入集群

kubectl get csr   #將該命令輸出的申請(qǐng)名稱填入下面命令中審批
kubectl certificate approve node-csr-ldQlmnDLt-PFbwy2w5NIkwNB8eS4uHc3-hGvSu97elo
kubectl get node  #查看集群節(jié)點(diǎn)，就會(huì)發(fā)現(xiàn)有一個(gè)master節(jié)點(diǎn)

部署kube-proxy服務(wù)

編輯kube-proxy配置文件

vi?/opt/kubernetes/cfg/kube-proxy.conf

KUBE_PROXY_OPTS="--config=/opt/kubernetes/cfg/kube-proxy-config.yml"

創(chuàng)建kube-proxy定義參數(shù)文件

vi?/opt/kubernetes/cfg/kube-proxy-config.yml

kind: KubeProxyConfiguration
apiVersion: kubeproxy.config.k8s.io/v1alpha1
bindAddress: 0.0.0.0
metricsBindAddress: 0.0.0.0:10249
clientConnection:
  kubeconfig: /opt/kubernetes/cfg/kube-proxy.kubeconfig
hostnameOverride: k8s-master01-test
clusterCIDR: 20.244.0.0/16

創(chuàng)建kube-proxy生成證書的配置文件

vi?kube-proxy-csr.json

{
  "CN": "system:kube-proxy",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "L": "ShenZhen",
      "ST": "ShenZhen",
      "O": "k8s",
      "OU": "System"
    }
  ]
}

生成證書

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy

查看生成的證書文件

ls kube-proxy*.pem

?

編輯生成kubeconfig的配置文件腳本

vi kube-proxy.sh

KUBE_CONFIG="/opt/kubernetes/cfg/kube-proxy.kubeconfig"
KUBE_APISERVER="https://10.1.60.125:6443"

kubectl config set-cluster kubernetes \
  --certificate-authority=/opt/kubernetes/ssl/ca.pem \
  --embed-certs=true \
  --server=${KUBE_APISERVER} \
  --kubeconfig=${KUBE_CONFIG}
kubectl config set-credentials kube-proxy \
  --client-certificate=./kube-proxy.pem \
  --client-key=./kube-proxy-key.pem \
  --embed-certs=true \
  --kubeconfig=${KUBE_CONFIG}
kubectl config set-context default \
  --cluster=kubernetes \
  --user=kube-proxy \
  --kubeconfig=${KUBE_CONFIG}
kubectl config use-context default --kubeconfig=${KUBE_CONFIG}

賦權(quán)并執(zhí)行腳本生成kubeconfig文件

chmod +x kubelet.sh
sh kubelet.sh

使用systemd管理kube-proxy服務(wù)

vi?/usr/lib/systemd/system/kube-proxy.service

[Unit]
Description=Kubernetes Proxy
After=network.target

[Service]
EnvironmentFile=/opt/kubernetes/cfg/kube-proxy.conf
ExecStart=/opt/kubernetes/bin/kube-proxy $KUBE_PROXY_OPTS
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target

啟動(dòng)kube-proxy服務(wù)

systemctl daemon-reload
systemctl start kube-proxy  && systemctl enable kube-proxy

部署calico網(wǎng)絡(luò)組件

參考：k8s集群使用calico網(wǎng)絡(luò)組件_k8s集群與calico版本對(duì)應(yīng)關(guān)系-CSDN博客

授權(quán) apiserver服務(wù)訪問kubelet

vi /opt/kubernetes/cfg/apiserver-to-kubelet-rbac.yaml

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  annotations:
    rbac.authorization.kubernetes.io/autoupdate: "true"
  labels:
    kubernetes.io/bootstrapping: rbac-defaults
  name: system:kube-apiserver-to-kubelet
rules:
  - apiGroups:
      - ""
    resources:
      - nodes/proxy
      - nodes/stats
      - nodes/log
      - nodes/spec
      - nodes/metrics
      - pods/log
    verbs:
      - "*"
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: system:kube-apiserver
  namespace: ""
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:kube-apiserver-to-kubelet
subjects:
  - apiGroup: rbac.authorization.k8s.io
    kind: User
    name: kubernetes

work節(jié)點(diǎn)加入集群

在其余work節(jié)點(diǎn)創(chuàng)建k8s工作目錄

mkdir -p /usr/local/k8s/kubernetes/{bin,cfg,ssl,logs}

在master節(jié)點(diǎn)上拷貝證書以及組件服務(wù)到所有work節(jié)點(diǎn)上

scp -r /opt/kubernetes root@10.1.60.126:/opt/
scp -r /usr/lib/systemd/system/{kubelet,kube-proxy}.service root@10.1.60.126:/usr/lib/systemd/system

scp -r /opt/kubernetes root@10.1.60.127:/opt/
scp -r /usr/lib/systemd/system/{kubelet,kube-proxy}.service root@10.1.60.127:/usr/lib/systemd/system

scp -r /opt/kubernetes root@10.1.60.128:/opt/
scp -r /usr/lib/systemd/system/{kubelet,kube-proxy}.service root@10.1.60.128:/usr/lib/systemd/system

?在拷貝文件后的所有work節(jié)點(diǎn)刪除kubelet的證書文件和kubeconfig文件

rm -f /opt/kubernetes/cfg/kubelet.kubeconfig 
rm -f /opt/kubernetes/ssl/kubelet*

#這些文件是審批后自動(dòng)生成的，所以每個(gè)節(jié)點(diǎn)的文件都是不一樣的

更改kubelet的配置文件

vi?/opt/kubernetes/cfg/kubelet.conf

--hostname-override=k8s-node01   #所有work節(jié)點(diǎn)都更改為自身對(duì)應(yīng)的主機(jī)名稱

更改kube-proxy的定義參數(shù)文件?

vi?/opt/kubernetes/cfg/kube-proxy-config.yml

hostnameOverride: k8s-node01   #所有work節(jié)點(diǎn)都更改為自身對(duì)應(yīng)的主機(jī)名稱

所有work節(jié)點(diǎn)啟動(dòng)kubelet和kube-proxy服務(wù)

systemctl daemon-reload
systemctl start kubelet kube-proxy
systemctl enable kubelet kube-proxy

?在master節(jié)點(diǎn)審批work節(jié)點(diǎn)請(qǐng)求，使得work節(jié)點(diǎn)加入集群

kubectl get csr  #查看審批請(qǐng)求，加審批請(qǐng)求的名稱填入以下審批命令中
kubectl certificate approve node-csr-RWx2DiMmRgLi8IsrpzwHthGve2W_YpIk3cccs8ZLNIk  #審批請(qǐng)求
kubectl get node  查看集群會(huì)發(fā)現(xiàn)，節(jié)點(diǎn)已加入集群

部署coredns組件

參考：k8s部署coredns_k8s pod怎么綁定coredns-CSDN博客?

至此k8s集群部署完成

集群網(wǎng)絡(luò)功能驗(yàn)證

?部署busybox

vi busybox.yaml

apiVersion: v1
kind: Pod
metadata:
  name: busybox
  namespace: default
spec:
  containers:
  - name: busybox
    image: busybox:1.28
    command:
      - sleep
      - "3600"
    imagePullPolicy: IfNotPresent
  restartPolicy: Always

創(chuàng)建pod

kubectl apply -f busybox.yaml

進(jìn)入pod驗(yàn)證

kubectl exec -it bosybox /bin/sh
nslookup kubernetes

nslookup kube-dns.kube-system

集群網(wǎng)絡(luò)解析正常?文章來源地址http://www.zghlxwxcb.cn/news/detail-839309.html

到了這里，關(guān)于k8s1.23.15集群二進(jìn)制部署的文章就介紹完了。如果您還想了解更多內(nèi)容，請(qǐng)?jiān)谟疑辖撬阉鱐OY模板網(wǎng)以前的文章或繼續(xù)瀏覽下面的相關(guān)文章，希望大家以后多多支持TOY模板網(wǎng)！