k3s containerd cilium hubble 集群快速上手

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

# 脚本安装
$ curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash

# Debian/Ubuntu
$ curl https://baltocdn.com/helm/signing.asc | gpg --dearmor | sudo tee /usr/share/keyrings/helm.gpg > /dev/null
$ sudo apt-get install apt-transport-https --yes
$ echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/helm.gpg] https://baltocdn.com/helm/stable/debian/ all main" | sudo tee /etc/apt/sources.list.d/helm-stable-debian.list
$ sudo apt update
$ sudo apt install -y helm

# windows
$ scoop install helm
# macOS
$ brew install helm

# 如果在本机适用，需要设置环境变量（非必需）
# 因为默认读取 $HOME/.kube/config 也可以在安装时设置到 helm 默认路径
export KUBECONFIG=/etc/rancher/k3s/k3s.yaml

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

# 当前源
$ helm repo list
# 海外源
$ helm repo add stable https://charts.helm.sh/stable

# 导入阿里源 国内代理
$ helm repo remove stable
$ helm repo add stable https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts
$ helm repo update stable

# 导入 云原生应用市场 https://hub.grapps.cn/
$ helm repo add appstore https://charts.grapps.cn
$ helm repo update appstore

# 更新全部
$ helm repo update

# 移除某个 repo
$ helm repo remove [some]

1
2
3
4

# search hub
$ helm search hub postgresql
# search repo
$ helm search repo nginx

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

$ CLI_ARCH=amd64
$ if [ "$(uname -m)" = "aarch64" ]; then CLI_ARCH=arm64; fi

$ CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/main/stable.txt)
# 确认 cilium 安装版本
$ echo $CILIUM_CLI_VERSION

$ curl -L --fail --remote-name-all https://github.com/cilium/cilium-cli/releases/download/${CILIUM_CLI_VERSION}/cilium-linux-${CLI_ARCH}.tar.gz
$ curl -L --fail --remote-name-all https://github.com/cilium/cilium-cli/releases/download/${CILIUM_CLI_VERSION}/cilium-linux-${CLI_ARCH}.tar.gz.sha256sum

## use proxy
$ CILIUM_CLI_VERSION=$(curl -s https://mirror.ghproxy.com/https://raw.githubusercontent.com/cilium/cilium-cli/main/stable.txt)
# 确认 cilium 安装版本
$ echo $CILIUM_CLI_VERSION
$ curl -L --fail --remote-name-all "https://mirror.ghproxy.com/https://github.com/cilium/cilium-cli/releases/download/${CILIUM_CLI_VERSION}/cilium-linux-${CLI_ARCH}.tar.gz"
$ curl -L --fail --remote-name-all "https://mirror.ghproxy.com/https://github.com/cilium/cilium-cli/releases/download/${CILIUM_CLI_VERSION}/cilium-linux-${CLI_ARCH}.tar.gz.sha256sum"

# check download
$ sha256sum --check cilium-linux-${CLI_ARCH}.tar.gz.sha256sum

# install cilium
$ sudo tar xzvfC cilium-linux-${CLI_ARCH}.tar.gz /usr/local/bin
# remove package
$ rm "cilium-linux-${CLI_ARCH}.tar.gz" "cilium-linux-${CLI_ARCH}.tar.gz.sha256sum"

# check install
$ cilium version

1
2

# 生成 K3S_TOKEN 后面会用到
$ openssl rand -hex 32

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85

# 设置主机上的其他非特权用户将能读取它，这个根据实际情况来（非必需）
$ export K3S_KUBECONFIG_MODE=644

# 设置 数据库连接环境变量（非必需）--cluster-init 为内嵌 etcd 模式
$ export K3S_DATASTORE_ENDPOINT=""

# 设置对外服务 ip （非必需）
$ export K3S_INSTALL_ADVERTISE_ADDRESS="192.168.50.56"

## 这些环境变量就是必须设置的

# 这里设置共享密钥用于将 server 或 agent 加入集群的令牌
# 令牌使用 ${ENV_GEN_K3S_TOKEN}
$ export K3S_TOKEN=${ENV_GEN_K3S_TOKEN}
# 设置 首个 server 地址，用于其他节点加入
$ export K3S_INSTALL_FIRST_ADVERTISE_ADDRESS="192.168.50.56"
# 设置私有 harbor 的域名或者 IP
$ export K3S_TLS_SUBJECT_ALTERNATIVE_NAME_HOSTS="192.168.50.50"
# 设置安装版本 https://github.com/k3s-io/k3s/releases 版本
$ export INSTALL_K3S_VERSION=v1.28.1+k3s1

## 设置安装命令
# 这个方式为 使用嵌入式 etcd 初始化新集群，设置 server 对外节点地址，设置配置文件位置
# 使用自定义 CNI 禁用运行 kube-proxy
$ export INSTALL_K3S_EXEC="server --tls-san=${K3S_TLS_SUBJECT_ALTERNATIVE_NAME_HOSTS} --cluster-init --flannel-backend=none --disable-network-policy --kube-apiserver-arg service-node-port-range=20000-39999 --write-kubeconfig $HOME/.kube/config --write-kubeconfig-mode 0644"

## 开始安装 k3s
# check env
$ echo $K3S_TOKEN $INSTALL_K3S_EXEC
# then
$ curl -sfL https://get.k3s.io | sh -
# 或者使用国内镜像安装
$ curl -sfL https://rancher-mirror.oss-cn-beijing.aliyuncs.com/k3s/k3s-install.sh | \
  INSTALL_K3S_MIRROR=cn sh -s -

# 添加其他参数（非必需）
$ curl -sfL https://rancher-mirror.oss-cn-beijing.aliyuncs.com/k3s/k3s-install.sh | \
  INSTALL_K3S_MIRROR=cn sh -s - \
  --tls-san <harbor_domain> \
  --advertise-address=${K3S_INSTALL_ADVERTISE_ADDRESS} \
  --system-default-registry registry.cn-hangzhou.aliyuncs.com

# 或者下载到本地 install-k3s.sh
$ curl -fL -o install-k3s.sh https://get.k3s.io
$ cat install-k3s.sh | INSTALL_K3S_MIRROR=cn sh -s - \
  --system-default-registry registry.cn-hangzhou.aliyuncs.com

## 检查服务运行
$ systemctl status k3s.service

# 配置 KUBECONFIG
$ export KUBECONFIG=/etc/rancher/k3s/k3s.yaml

## 使用 cilium cli 安装 cilium
# get version of cilium https://github.com/cilium/cilium/releases/latest
$ curl https://github.com/cilium/cilium/raw/main/stable.txt
# get version by proxy
$ curl https://mirror.ghproxy.com/https://raw.githubusercontent.com/cilium/cilium/main/stable.txt
# now latest is v1.14.4
# match k3s default podCIDR 10.42.0.0/16
$ cilium install --version 1.14.4 \
  --set=ipam.operator.clusterPoolIPv4PodCIDRList="10.42.0.0/16"
🔮 Auto-detected Kubernetes kind: K3s
ℹ️  Using Cilium version 1.14.4
🔮 Auto-detected cluster name: default

# wait cilium install
$ cilium status --wait

## 安装有错误，卸载安装
$ cilium uninstall
# k3s 下必须手动删除 cilium_host、cilium_net 和 cilium_vxlan 接口
# 如果不这样做，你可能会在 K3s 停止时丢失与主机的网络连接
$ ip link show | grep cilium
$ ip link delete cilium_host
$ ip link delete cilium_net
$ ip link delete cilium_vxlan
# 并且 需要删除 cilium 的 iptables 规则
$ iptables-save | grep -iv cilium | iptables-restore
$ ip6tables-save | grep -iv cilium | ip6tables-restore
# 确认删除
$ sudo iptables-save | grep cilium
$ sudo ip6tables-save | grep cilium
# 确认上面的执行完成后
$ /usr/local/bin/k3s-uninstall.sh

1
2
3
4

$ cat /etc/systemd/system/k3s.service
内容为
–advertise-address=
为 公网或者内网 ip

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20

# 查看 kubectl 配置，这里已经被修改路径 该文件同 /etc/rancher/k3s/k3s.yaml
$ cat $HOME/.kube/config

$ kubectl get nodes
NAME        STATUS   ROLES                       AGE   VERSION
server1     Ready    control-plane,etcd,master   28m   vX.Y.Z

# 或者
$ kubectl get pods -n kube-system -o wide
# 确认基础节点运行正常
$ kubectl get pods -A

# check cilium status
$ cilium status
Deployment             cilium-operator    Desired: 1, Ready: 1/1, Available: 1/1
DaemonSet              cilium             Desired: 1, Ready: 1/1, Available: 1/1
Containers:            cilium             Running: 1
                       cilium-operator    Running: 1
Cluster Pods:          5/5 managed by Cilium
Helm chart version:    1.14.4

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75

# 设置主机上的其他非特权用户将能读取它，这个根据实际情况来（非必需）
$ export K3S_KUBECONFIG_MODE=644

# 设置 数据库连接环境变量（非必需）--cluster-init 为内嵌 etcd 模式
$ export K3S_DATASTORE_ENDPOINT=""

## 这些环境变量就是必须设置的

# 这里设置共享密钥用于将 server 或 agent 加入集群的令牌
# 令牌使用首个节点安装时生成的 ${ENV_GEN_K3S_TOKEN}
$ export K3S_TOKEN=${ENV_GEN_K3S_TOKEN}
# 设置 首个 server 地址，用于其他节点加入
$ export K3S_INSTALL_FIRST_ADVERTISE_ADDRESS="192.168.50.56"
# 设置私有 harbor 的域名或者 IP
$ export K3S_TLS_SUBJECT_ALTERNATIVE_NAME_HOSTS="192.168.50.50"
# 设置安装版本 https://github.com/k3s-io/k3s/releases 版本
$ export INSTALL_K3S_VERSION=v1.28.1+k3s1

## 设置安装命令
# 这个方式为 使用嵌入式 etcd 初始化新集群，设置 server 对外节点地址
# 设置 master 对外节点地址，设置配置文件位置
# 使用自定义 CNI 禁用运行 kube-proxy
$ export INSTALL_K3S_EXEC="server --server https://${K3S_INSTALL_FIRST_ADVERTISE_ADDRESS}:6443 --tls-san=${K3S_TLS_SUBJECT_ALTERNATIVE_NAME_HOSTS} --flannel-backend=none --disable-network-policy --kube-apiserver-arg service-node-port-range=20000-39999 --write-kubeconfig $HOME/.kube/config --write-kubeconfig-mode 0644"

## 开始安装
# check env
$ echo $K3S_TOKEN $K3S_INSTALL_FIRST_ADVERTISE_ADDRESS $INSTALL_K3S_EXEC
# then
$ curl -sfL https://get.k3s.io | sh -
# 或者使用国内镜像安装
$ curl -sfL https://rancher-mirror.oss-cn-beijing.aliyuncs.com/k3s/k3s-install.sh | \
  INSTALL_K3S_MIRROR=cn sh -s -

## 检查服务运行
$ systemctl status k3s.service

# 配置 KUBECONFIG
$ export KUBECONFIG=/etc/rancher/k3s/k3s.yaml

## 使用 cilium cli 安装 cilium
# get version of cilium https://github.com/cilium/cilium/releases/latest
$ curl https://github.com/cilium/cilium/raw/main/stable.txt
# get version by proxy
$ curl https://mirror.ghproxy.com/https://raw.githubusercontent.com/cilium/cilium/main/stable.txt
# now latest is v1.14.4
$ cilium install --version 1.14.4
🔮 Auto-detected Kubernetes kind: K3s
ℹ️  Using Cilium version 1.14.4
🔮 Auto-detected cluster name: default

# wait cilium install
$ cilium status --wait
# 安装成功后效果
Deployment             cilium-operator    Desired: 1, Ready: 1/1, Available: 1/1
DaemonSet              cilium             Desired: 2, Ready: 2/2, Available: 2/2
Containers:            cilium             Running: 2
                       cilium-operator    Running: 1
Cluster Pods:          6/6 managed by Cilium

## 安装有错误，卸载安装
$ cilium uninstall
# k3s下必须手动删除 cilium_host、cilium_net 和 cilium_vxlan 接口
# 如果不这样做，你可能会在 K3s 停止时丢失与主机的网络连接
$ ip link show | grep cilium
$ ip link delete cilium_host
$ ip link delete cilium_net
$ ip link delete cilium_vxlan
# 并且 需要删除 cilium 的 iptables 规则
$ iptables-save | grep -iv cilium | iptables-restore
$ ip6tables-save | grep -iv cilium | ip6tables-restore
# 确认删除
$ sudo iptables-save | grep cilium
$ sudo ip6tables-save | grep cilium
# 确认上面的执行完成后
$ /usr/local/bin/k3s-uninstall.sh

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41

## 这些环境变量就是必须设置的

# 这里设置共享密钥用于将 server 或 agent 加入集群的令牌
# 令牌使用首个节点安装时生成的 ${ENV_GEN_K3S_TOKEN}
$ export K3S_TOKEN=${ENV_GEN_K3S_TOKEN}
# 设置安装版本 https://github.com/k3s-io/k3s/releases 版本
$ export INSTALL_K3S_VERSION=v1.28.1+k3s1
# 设置集群节点地址
$ export K3S_INSTALL_FIRST_ADVERTISE_ADDRESS="192.168.50.56"
$ export K3S_URL=https://${K3S_INSTALL_FIRST_ADVERTISE_ADDRESS}:6443
# 安装配置 agent
$ export INSTALL_K3S_EXEC="agent --server https://${K3S_INSTALL_FIRST_ADVERTISE_ADDRESS}:6443"

## 开始安装
# check env
$ echo $K3S_TOKEN $K3S_URL $INSTALL_K3S_EXEC
# then
$ curl -sfL https://get.k3s.io | sh -
# 或者使用国内镜像安装
$ curl -sfL https://rancher-mirror.oss-cn-beijing.aliyuncs.com/k3s/k3s-install.sh | \
  INSTALL_K3S_MIRROR=cn sh -s -

# 或者使用这个命令来，作为已有 server 的工作节点
$ k3s agent --server ${K3S_URL} --token ${K3S_TOKEN}

# 安装有错误，卸载 agent 安装
# k3s下必须手动删除 cilium_host、cilium_net 和 cilium_vxlan 接口
# 如果不这样做，你可能会在 K3s 停止时丢失与主机的网络连接
$ ip link show | grep cilium
$ ip link delete cilium_host
$ ip link delete cilium_net
$ ip link delete cilium_vxlan
# 并且 需要删除 cilium 的 iptables 规则
$ iptables-save | grep -iv cilium | iptables-restore
$ ip6tables-save | grep -iv cilium | ip6tables-restore
# 确认删除
$ sudo iptables-save | grep cilium
$ sudo ip6tables-save | grep cilium
# 确认上面的执行完成后
$ /usr/local/bin/k3s-agent-uninstall.sh

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

## 验证
# 借助 cilium service list 命令，我们可以验证 Cilium 的 eBPF kube-proxy 替代程序是否创建了新的 NodePort 服务
$ kubectl -n kube-system exec ds/cilium -- cilium service list
# 使用主机名空间中的 iptables 验证是否存在针对该服务的 iptables 规则
$ sudo iptables-save | grep KUBE-SVC
# 因为是使用 cilium cli 安装，iptables 规则不会为空，对于快速上手来说，足够使用

## test
$ cilium connectivity test
# test with args external to https://www.baidu.com
$ cilium connectivity test --request-timeout 30s --connect-timeout 10s --external-target www.baidu.com --external-ip 183.2.172.42

1

$ kubectl -n kube-system exec ds/cilium -- cilium status

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

# get version of cilium https://github.com/cilium/cilium/releases/latest
$ curl https://github.com/cilium/cilium/raw/main/stable.txt
# get version by proxy
$ curl https://mirror.ghproxy.com/https://raw.githubusercontent.com/cilium/cilium/main/stable.txt
# now latest is v1.14.4

$ helm repo add cilium https://helm.cilium.io/
$ helm upgrade cilium cilium/cilium --version 1.14.4 \
   --namespace kube-system \
   --reuse-values \
   --set hubble.relay.enabled=true \
   --set hubble.ui.enabled=true

# or enable by cilium
$ cilium hubble enable

# check
$ cilium status
    /¯¯\
 /¯¯\__/¯¯\    Cilium:             OK
 \__/¯¯\__/    Operator:           OK
 /¯¯\__/¯¯\    Envoy DaemonSet:    disabled (using embedded mode)
 \__/¯¯\__/    Hubble Relay:       OK
    \__/       ClusterMesh:        disable

# Hubble Relay:       OK

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35

$ HUBBLE_ARCH=amd64
$ if [ "$(uname -m)" = "aarch64" ]; then HUBBLE_ARCH=arm64; fi

# get version
$ HUBBLE_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/hubble/main/stable.txt)
$ echo "https://github.com/cilium/hubble/releases/download/$HUBBLE_VERSION/hubble-linux-${HUBBLE_ARCH}.tar.gz"
$ curl -L --fail --remote-name-all https://github.com/cilium/hubble/releases/download/$HUBBLE_VERSION/hubble-linux-${HUBBLE_ARCH}.tar.gz
$ curl -L --fail --remote-name-all https://github.com/cilium/hubble/releases/download/$HUBBLE_VERSION/hubble-linux-${HUBBLE_ARCH}.tar.gz.sha256sum
## use proxy
$ HUBBLE_VERSION=$(curl -s https://mirror.ghproxy.com/https://raw.githubusercontent.com/cilium/hubble/main/stable.txt)
$ echo "https://github.com/cilium/hubble/releases/download/$HUBBLE_VERSION/hubble-linux-${HUBBLE_ARCH}.tar.gz"
$ curl -L --fail --remote-name-all "https://mirror.ghproxy.com/https://github.com/cilium/hubble/releases/download/$HUBBLE_VERSION/hubble-linux-${HUBBLE_ARCH}.tar.gz"
$ curl -L --fail --remote-name-all "https://mirror.ghproxy.com/https://github.com/cilium/hubble/releases/download/$HUBBLE_VERSION/hubble-linux-${HUBBLE_ARCH}.tar.gz.sha256sum"

# check download
$ sha256sum --check hubble-linux-${HUBBLE_ARCH}.tar.gz.sha256sum
# pass check then install
$ sudo tar xzvfC hubble-linux-${HUBBLE_ARCH}.tar.gz /usr/local/bin
$ rm "hubble-linux-${HUBBLE_ARCH}.tar.gz" "hubble-linux-${HUBBLE_ARCH}.tar.gz.sha256sum"

# 默认情况下 hubble 服务绑定在 localhost:4245
# 如果需要，可以修改 hubble 服务配置
$ hubble config set server 0.0.0.0:4245

# 为了访问Hubble API，从本地计算机创建一个转发到Hubble服务的端口。这将允许您将Hubble客户端连接到本地端口4245并访问库伯内特斯集群中的Hubble中继服务。
# This will allow you to connect the Hubble client to the local port 4245 and access the Hubble Relay service in your Kubernetes cluster.
# more doc see https://kubernetes.io/docs/tasks/access-application-cluster/port-forward-access-application-cluster/
$ cilium hubble port-forward&

# check install
$ hubble status
Healthcheck (via 0.0.0.0:4245): Ok
Current/Max Flows: 16,380/16,380 (100.00%)
Flows/s: 50.64
Connected Nodes: 4/4

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

# 您还可以查询流API并查找流
$ hubble observe

# 修改 hubble 服务配置
$ hubble config set server 0.0.0.0:4245

# 检查 hubble 配置
$ hubble config view

# 如果修改配置需要
$ cilium hubble disable
$ cilium hubble enable

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

# 启用 hubble UI
$ cilium hubble enable --ui
$ cilium hubble ui --open-browser=false
# 默认情况下 hubble ui 会映射到端口 12000 上

# 检查 正在运行 Hubble UI 的 pod
$ kubectl get pods -n kube-system -l k8s-app=hubble-ui
# 查看详细标签
$ kubectl get pods -n kube-system -l k8s-app=hubble-ui --show-labels
# 查看网络映射
$ kubectl get pods -n kube-system -o wide -l 'k8s-app=hubble-ui'
# 查看 svc 状态
$ kubectl get svc -n kube-system -o wide -l 'k8s-app=hubble-ui'

1
2

# 例如暴露 Hubble UI 前端 到 192.168.50.56 端口 39997
$ kubectl port-forward -n kube-system deployment/hubble-ui 39997:8081 --address=192.168.50.56

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

kind: Service
apiVersion: v1
metadata:
  namespace: kube-system
  name: hubble-ui-np
  labels:
    k8s-app: hubble-ui-np
spec:
  selector:
    k8s-app: hubble-ui
  ports:
    - name: http
      port: 8081
      targetPort: 8081
      nodePort: 39998
  type: NodePort

1
2

# 查看映射
$ kubectl get svc -n kube-system -o wide -l 'k8s-app=hubble-ui-np'

1

$ cilium connectivity test --request-timeout 30s --connect-timeout 10s

1
2

# 查看当前所有的 Cilium Endpoints
$ kubectl get ciliumendpoints.cilium.io -A

1
2

# 查看当前所有的 Cilium Identity
$ kubectl get ciliumidentities.cilium.io -A

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

$ cat << EOF | kubectl apply -f -
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-nginx
spec:
  selector:
    matchLabels:
      run: my-nginx
  replicas: 2
  template:
    metadata:
      labels:
        run: my-nginx
    spec:
      containers:
      - name: my-nginx
        image: nginx
        ports:
        - containerPort: 80
EOF
# 观察创建
$ kubectl get pods --watch

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22

$ kubectl expose deployment my-nginx --type=NodePort --port=80
service/my-nginx exposed

# 查看 NodePort 服务端口等信息
$ kubectl get svc my-nginx
NAME       TYPE       CLUSTER-IP     EXTERNAL-IP   PORT(S)        AGE
my-nginx   NodePort   10.43.209.89   <none>        80:30995/TCP   18s

# 借助 cilium service list 命令
# 可以验证 Cilium 的 eBPF kube-proxy 替代程序是否创建了新的 NodePort 服务
# 在本例中，创建了端口号为 30995 的服务
$ kubectl -n kube-system exec ds/cilium -- cilium service list
...
21   10.43.209.89:80      ClusterIP      1 => 10.42.0.202:80 (active)
                                         2 => 10.42.1.188:80 (active)

# 使用主机名空间中的 iptables 验证是否存在针对该服务的 iptables 规则
$ sudo iptables-save | grep KUBE-SVC
...
-A KUBE-SERVICES -d 10.43.209.89/32 -p tcp -m comment --comment "default/my-nginx cluster IP" -m tcp --dport 80 -j KUBE-SVC-L65ENXXZWWSAPRCR
...
-A KUBE-SVC-L65ENXXZWWSAPRCR ! -s 10.42.0.0/16 -d 10.43.209.89/32 -p tcp -m comment --comment "default/my-nginx cluster IP" -m tcp --dport 80 -j KUBE-MARK-MASQ

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

# 获取 svc my-nginx 端口号
$ node_port=$(kubectl get svc my-nginx -o=jsonpath='{@.spec.ports[0].nodePort}')
# 测试通断
# localhost+NodePort
$ curl 127.0.0.1:$node_port
# eth0+NodePort
$ curl 192.168.50.56:$node_port
# ClusterIP
$ curl 10.43.209.89:80
# 本机 PodIP
$ curl 10.42.0.202:80
# 其他 Node PodIP
$ curl 10.42.1.188:80

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22

$ kubectl delete svc my-nginx
$ cat << EOF | kubectl delete -f -
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-nginx
spec:
  selector:
    matchLabels:
      run: my-nginx
  replicas: 2
  template:
    metadata:
      labels:
        run: my-nginx
    spec:
      containers:
      - name: my-nginx
        image: nginx
        ports:
        - containerPort: 80
EOF

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46

## 移除 hubble
# remove cilium hubble
$ cilium hubble disable

## 卸载 cilium
# remove cilium install by cli
$ cilium uninstall

# 如果已经移除 cilium 则不需要在其他节点执行 上面的移除

## 删除 cilium 节点网络配置
# 删除前确认状态
$ cilium status
    /¯¯\
 /¯¯\__/¯¯\    Cilium:             1 errors
 \__/¯¯\__/    Operator:           disabled
 /¯¯\__/¯¯\    Envoy DaemonSet:    disabled (using embedded mode)
 \__/¯¯\__/    Hubble Relay:       disabled
    \__/       ClusterMesh:        disabled
...

# k3s下必须手动删除 cilium_host、cilium_net 和 cilium_vxlan 接口
# 如果不这样做，你可能会在 K3s 停止时丢失与主机的网络连接
$ sudo ip link delete cilium_host
$ sudo ip link delete cilium_net
$ sudo ip link delete cilium_vxlan
# 确认删除 与主机的网络连接
$ ip link show | grep cilium
# 并且 需要删除 cilium 的 iptables 规则
$ sudo iptables-save | grep -iv cilium | sudo iptables-restore
$ sudo ip6tables-save | grep -iv cilium | sudo ip6tables-restore
# 确认删除
$ sudo iptables-save | grep cilium
$ sudo ip6tables-save | grep cilium

## 可选，删除部署的 KUBE 防火墙规则
$ sudo iptables-save | grep -iv 'KUBE-' | sudo iptables-restore
$ sudo ip6tables-save | grep -iv 'KUBE-' | sudo iptables-restore
$ sudo iptables-save | grep 'KUBE-'
$ sudo ip6tables-save | grep 'KUBE-'

## 卸载 k3s 节点
# 确认上面的执行完成后 agent 卸载
$ /usr/local/bin/k3s-agent-uninstall.sh
# 确认上面的执行完成后 server 卸载
$ /usr/local/bin/k3s-uninstall.sh