环境准备

  • Oracle Cloud
  • Oracle Linux 8
  • Ampere ARM 处理器
  • 2 cores, 12g memory, bandwidth 2gbps

开启 Google BBR TCP 加速

执行命令 1

wget https://raw.githubusercontent.com/bannedbook/fanqiang/master/v2ss/server-cfg/sysctl.conf -O -> /etc/sysctl.conf

执行命令 2

sysctl -p

成功后输出

[root@k8s-master ~]# sysctl -p
fs.file-max = 65535
net.ipv6.conf.all.accept_ra = 2
net.core.default_qdisc = fq
net.ipv4.tcp_congestion_control = bbr

安装 trojan

一键安装脚本

#安装/更新
source <(curl -sL https://git.io/trojan-install)

#卸载
source <(curl -sL https://git.io/trojan-install) --remove
  • 命令行修改 trojan 端口为 38300
  • 修改/etc/systemd/system/trojan-web.service 配置文件 在/usr/local/bin/trojan web 后面添加 -p 8081 然后运行: bash systemctl daemon-reload systemctl restart trojan-web
  • 修改 trojan-web 端口
    vim /usr/local/etc/trojan/config.json
    将 remote_port 改为 8081
    "remote_port": 8081,

关闭防火墙

停止

systemctl stop firewalld

禁止开机启动

systemctl disable firewalld

Docker & Containerd 部署

设置仓库

使用官方源地址

sudo yum-config-manager \
    --add-repo \
    https://download.docker.com/linux/centos/docker-ce.repo

docker & containerd 安装

sudo yum install docker-ce docker-ce-cli containerd.io

docker 配置

将 driver 由 cgroup 改为 systemd

sudo tee /etc/docker/daemon.json <<-'EOF'
{
"exec-opts": ["native.cgroupdriver=systemd"]
}
EOF

sudo systemctl daemon-reload && systemctl restart docker

containerd 配置

  • 生成默认配置文件
containerd config default > /etc/containerd/config.toml
  • 配置 cgroup 驱动程序 systemd
vim /etc/containerd/config.toml
114 [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
125   SystemdCgroup = true
  • 应用所有更改后, 重新启动 containerd
systemctl restart containerd && systemctl enable containerd

netstat -anput | grep containerd

Kubernetes 部署

允许 iptables 检查桥接流量

sudo modprobe br_netfilter

sudo lsmod | grep br_netfilter

cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
br_netfilter
EOF

cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF
sudo sysctl --system

检查所需端口

yum install -y nc

nc 127.0.0.1 6443

禁用 swap 分区

# 暂时关闭SWAP分区
swapoff -a


# 永久禁用SWAP分区
vim /etc/fstab
# 注释掉下面行
/.swapfile      none    swap    sw,comment=cloudconfig  0       0

添加官方 repo

cat <<EOF | sudo tee /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-\$basearch
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
exclude=kubelet kubeadm kubectl
EOF

设置 SELinux

# 将 SELinux 设置为 permissive 模式（相当于将其禁用）
sudo setenforce 0
sudo sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config

安装 kubelet, kubectl, kubeadm

sudo yum install -y kubelet kubeadm kubectl --disableexcludes=kubernetes

sudo systemctl enable --now kubelet

初始化集群

kubeadm init --pod-network-cidr=10.244.0.0/16 --v=5

配置默认的 endpoints

crictl config runtime-endpoint unix:///run/containerd/containerd.sock
crictl config image-endpoint unix:///run/containerd/containerd.sock

更新用户配置文件

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

flannel 网络部署

wget https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml

kubectl apply -f kube-flannel.yml

遇到“open /run/flannel/subnet.env: no such file or directory“解决办法

新建/run/flannel/subnet.env文件, 写入内容

FLANNEL_NETWORK=10.244.0.0/16
FLANNEL_SUBNET=10.244.0.1/24
FLANNEL_MTU=1450
FLANNEL_IPMASQ=true

Dashboard 部署

设置 Master 节点可调度

kubectl taint nodes k8s-master node-role.kubernetes.io/control-plane:NoSchedule-
kubectl taint nodes --all node-role.kubernetes.io/master-

dashboard 安装

kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.6.0/aio/deploy/recommended.yaml

代理访问

nohup kubectl proxy &

使用 http://localhost:8001/api/v1/namespaces/kubernetes-dashboard/services/https:kubernetes-dashboard:/proxy/ 访问

创建 admin-user

kubectl apply -f dashboard-adminuser.yaml

dashboard-adminuser.yaml 详情:

apiVersion: v1
kind: ServiceAccount
metadata:
  name: admin-user
  namespace: kubernetes-dashboard

创建集群角色绑定

kubectl apply -f rbac.yaml

rbac.yaml 详情:

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: admin-user
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: admin-user
  namespace: kubernetes-dashboard

创建登录 token

kubectl -n kubernetes-dashboard create token admin-user

安装 nginx 及申请 https 证书

  • 安装 nginx
    yum install -y nginx
  • 修改配置文件
    # For more information on configuration, see:
#   * Official English Documentation: http://nginx.org/en/docs/
#   * Official Russian Documentation: http://nginx.org/ru/docs/

user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log;
pid /run/nginx.pid;

# Load dynamic modules. See /usr/share/doc/nginx/README.dynamic.
include /usr/share/nginx/modules/*.conf;

events {
    worker_connections 1024;
}

http {
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                    '$status $body_bytes_sent "$http_referer" '
                    '"$http_user_agent" "$http_x_forwarded_for"';

    access_log  /var/log/nginx/access.log  main;

    sendfile            on;
    tcp_nopush          on;
    tcp_nodelay         on;
    keepalive_timeout   65;
    types_hash_max_size 2048;

    include             /etc/nginx/mime.types;
    default_type        application/octet-stream;

    # Load modular configuration files from the /etc/nginx/conf.d directory.
    # See http://nginx.org/en/docs/ngx_core_module.html#include
    # for more information.
    include /etc/nginx/conf.d/*.conf;

    server {
        listen       80 default_server;
        listen       [::]:80 default_server;
        server_name  code.ihasy.com;

        return 301 https://code.ihasy.com$request_uri;

        # Load configuration files for the default server block.
        include /etc/nginx/default.d/*.conf;

        location / {
        }

        error_page 404 /404.html;
            location = /40x.html {
        }

        error_page 500 502 503 504 /50x.html;
            location = /50x.html {
        }
    }

    # Settings for a TLS enabled server.
    server {
        listen       443 ssl http2 default_server;
        listen       [::]:443 ssl http2 default_server;
        server_name  code.ihasy.com;
        root         /usr/share/nginx/html;

        ssl_certificate "/root/.ca/ihasy.com/ihasy.com.pem";
        ssl_certificate_key "/root/.ca/ihasy.com/ihasy.com.key";
        ssl_session_cache shared:SSL:1m;
        ssl_session_timeout  10m;
        ssl_ciphers PROFILE=SYSTEM;
        ssl_prefer_server_ciphers on;

        # Load configuration files for the default server block.
        include /etc/nginx/default.d/*.conf;

        location / {
            proxy_pass http://127.0.0.1:8080/;
            proxy_set_header Upgrade $http_upgrade;
            proxy_set_header Connection upgrade;
            proxy_set_header Accept-Encoding gzip;
            proxy_set_header Origin https://$host;
            proxy_set_header Host $host;
            proxy_set_header X-Forwarded-For $remote_addr;
        }

        error_page 404 /404.html;
            location = /40x.html {
        }

        error_page 500 502 503 504 /50x.html;
            location = /50x.html {
        }
    }

}

更改 dashboard 过期时间

进入 dashboard 找到 namespace 为 kubernetes-dashboard 的命名空间, 找到 containers 节点, 在 args 参数下添加

- '--token-ttl=43200'

最终结果为

      containers:
        - name: kubernetes-dashboard
          image: kubernetesui/dashboard:v2.6.0
          args:
            - '--auto-generate-certificates'
            - '--namespace=kubernetes-dashboard'
            - '--token-ttl=43200'

设置 Master 节点防火墙

  • 开启防火墙
    firewall-cmd --permanent --add-port=6443/tcp
  • 重新加载
    firewall-cmd --reload
  • 确认是否开放
    firewall-cmd --list-all

稍等片刻, 便大功告成! 😄

NAME     STATUS   ROLES           AGE     VERSION   INTERNAL-IP     EXTERNAL-IP   OS-IMAGE                KERNEL-VERSION                CONTAINER-RUNTIME
master   Ready    control-plane   4m27s   v1.24.2   172.18.54.227   <none>        CentOS Linux 7 (Core)   3.10.0-1160.66.1.el7.x86_64   containerd://1.6.6
Table of Contents