问题一¶
问题： [利用host-gw模式提升集群网络性能](http://49.7.203.222:3000/#/kubernetes-advanced/cni?id=利用host-gw模式提升集群网络性能)
有一个步骤要编辑修改flannel的网络后端： $ kubectl edit cm kube-flannel-cfg -n kube-system
当前并没有这个pod
原因：安装flannel插件的时候，课件提供的下载地址 下载下来的版本已经不是课程教学的版本了。
wget https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
-------------------------------------
v0.19.2 版本 课程教学 v0.16.1版本
# 问题kube-system 没有 kube-flannel-cfg 现在在线edit 哪个？
[root@k8s-master ~]# kubectl -n kube-flannel get cm
NAME DATA AGE
kube-flannel-cfg 2 10d
kube-root-ca.crt 1 10d
[root@k8s-master ~]# kubectl -n kube-system get po
NAME READY STATUS RESTARTS AGE
coredns-59d64cd4d4-kpl7s 1/1 Running 7 9d
coredns-59d64cd4d4-lb7tv 1/1 Running 7 9d
etcd-k8s-master 1/1 Running 4 9d
kube-apiserver-k8s-master 1/1 Running 4 9d
kube-controller-manager-k8s-master 1/1 Running 4 43h
kube-proxy-7fsvw 1/1 Running 7 9d
kube-proxy-rsz76 1/1 Running 6 9d
kube-proxy-tzqfv 1/1 Running 7 9d
kube-scheduler-k8s-master 1/1 Running 5 43h
metrics-server-5dc985c965-xmhtb 1/1 Running 1 23h
[root@k8s-master ~]# kubectl get ns
NAME STATUS AGE
default Active 9d
demo Active 43h
kube-flannel Active 9d # 发现有一个单独的kuke-flannel空间
kube-node-lease Active 9d
kube-public Active 9d
kube-system Active 9d
kubernetes-dashboard Active 41h
luffy Active 9d
[root@k8s-master ~]# kubectl get po -n kube-flannel
NAME READY STATUS RESTARTS AGE
kube-flannel-ds-dsjmv 1/1 Running 9 9d
kube-flannel-ds-m5ktm 1/1 Running 8 9d
kube-flannel-ds-t6mrz 1/1 Running 6 9d
问题2: 手动编辑了 kube-flannel.yml 文件后操作¶
[root@k8s-master ~]# vi kube-flannel.yml #手动更新yml文件后操作
kube-flannel v0.19.2版本操作记录
[root@k8s-master ~]# vi kube-flannel.yml
82 net-conf.json: |
83 {
84 "Network": "10.244.0.0/16",
85 "Backend": {
86 "Type": "host-gw" #修改
87 }
88 }
[root@k8s-master ~]# kubectl apply -f kube-flannel.yml
# 重建Flannel的Pod
[root@k8s-master ~]# kubectl -n kube-flannel get po
NAME READY STATUS RESTARTS AGE
kube-flannel-ds-dsjmv 1/1 Running 9 9d
kube-flannel-ds-m5ktm 1/1 Running 8 9d
kube-flannel-ds-t6mrz 1/1 Running 6 9d
[root@k8s-master ~]# kubectl -n kube-flannel delete po kube-flannel-ds-dsjmv kube-flannel-ds-m5ktm kube-flannel-ds-t6mrz
[root@k8s-master ~]# kubectl -n kube-flannel logs -f kube-flannel-ds-bp7tm
I1020 02:47:11.776468 1 main.go:207] CLI flags config: {etcdEndpoints:http://127.0.0.1:4001,http://127.0.0.1:2379 etcdPrefix:/coreos.com/network etcdKeyfile: etcdCertfile: etcdCAFile: etcdUsername: etcdPassword: version:false kubeSubnetMgr:true kubeApiUrl: kubeAnnotationPrefix:flannel.alpha.coreos.com kubeConfigFile: iface:[eth0] ifaceRegex:[] ipMasq:true ifaceCanReach: subnetFile:/run/flannel/subnet.env publicIP: publicIPv6: subnetLeaseRenewMargin:60 healthzIP:0.0.0.0 healthzPort:0 iptablesResyncSeconds:5 iptablesForwardRules:true netConfPath:/etc/kube-flannel/net-conf.json setNodeNetworkUnavailable:true}
W1020 02:47:11.776579 1 client_config.go:614] Neither --kubeconfig nor --master was specified. Using the inClusterConfig. This might not work.
I1020 02:47:11.883481 1 kube.go:120] Waiting 10m0s for node controller to sync
I1020 02:47:11.883521 1 kube.go:401] Starting kube subnet manager
I1020 02:47:12.976013 1 kube.go:127] Node controller sync successful
I1020 02:47:12.976081 1 main.go:227] Created subnet manager: Kubernetes Subnet Manager - k8s-slave1
I1020 02:47:12.976093 1 main.go:230] Installing signal handlers
I1020 02:47:12.976455 1 main.go:467] Found network config - Backend type: host-gw
I1020 02:47:12.978362 1 match.go:259] Using interface with name eth0 and address 10.211.55.26
I1020 02:47:12.978393 1 match.go:281] Defaulting external address to interface address (10.211.55.26)
I1020 02:47:13.074899 1 kube.go:350] Setting NodeNetworkUnavailable
I1020 02:47:13.082687 1 main.go:345] Setting up masking rules
I1020 02:47:13.283635 1 main.go:366] Changing default FORWARD chain policy to ACCEPT
I1020 02:47:13.283814 1 main.go:379] Wrote subnet file to /run/flannel/subnet.env
I1020 02:47:13.283825 1 main.go:383] Running backend.
I1020 02:47:13.284439 1 main.go:404] Waiting for all goroutines to exit
I1020 02:47:13.284495 1 route_network.go:55] Watching for new subnet leases
W1020 02:47:13.285026 1 route_network.go:87] Ignoring non-host-gw subnet: type=vxlan # 这里提示有问题吗
I1020 02:47:13.285040 1 route_network.go:92] Subnet added: 10.244.0.0/24 via 10.211.55.25
W1020 02:47:13.285272 1 route_network.go:151] Replacing existing route to {Ifindex: 4 Dst: 10.244.0.0/24 Src: <nil> Gw: 10.244.0.0 Flags: [onlink] Table: 254 Realm: 0} with {Ifindex: 2 Dst: 10.244.0.0/24 Src: <nil> Gw: 10.211.55.25 Flags: [] Table: 0 Realm: 0}
I1020 02:47:13.576997 1 iptables.go:177] bootstrap done
I1020 02:47:13.578330 1 iptables.go:177] bootstrap done
I1020 02:47:19.624118 1 route_network.go:92] Subnet added: 10.244.2.0/24 via 10.211.55.27
W1020 02:47:19.624715 1 route_network.go:151] Replacing existing route to {Ifindex: 4 Dst: 10.244.2.0/24 Src: <nil> Gw: 10.244.2.0 Flags: [onlink] Table: 254 Realm: 0} with {Ifindex: 2 Dst: 10.244.2.0/24 Src: <nil> Gw: 10.211.55.27 Flags: [] Table: 0 Realm: 0}
[root@k8s-master ~]# route -n
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
0.0.0.0 10.211.55.1 0.0.0.0 UG 100 0 0 eth0
10.211.55.0 0.0.0.0 255.255.255.0 U 100 0 0 eth0
10.244.0.0 0.0.0.0 255.255.255.0 U 0 0 0 cni0
10.244.1.0 10.211.55.26 255.255.255.0 UG 0 0 0 eth0 #节点1ip
10.244.2.0 10.211.55.27 255.255.255.0 UG 0 0 0 eth0 #节点2ip
172.17.0.0 0.0.0.0 255.255.0.0 U 0 0 0 docker0
# 测验 slave1的容器ping slave2的容器
[root@k8s-master ~]# kubectl -n luffy get po -owide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
myblog-76d54c49b6-26br5 1/1 Running 2 17h 10.244.1.37 k8s-slave1 <none> <none>
myblog-76d54c49b6-bfkvf 1/1 Running 0 13h 10.244.1.42 k8s-slave1 <none> <none>
mysql-864b4c85b5-9cdds 1/1 Running 0 17h 10.244.1.40 k8s-slave1 <none> <none>
[root@k8s-master ~]# kubectl -n luffy get hpa
NAME REFERENCE TARGETS MINPODS MAXPODS REPLICAS AGE
hpa-myblog Deployment/myblog 64%/80%, 2%/20% 1 3 2 13h
[root@k8s-master ~]# kubectl -n luffy delete hpa hpa-myblog # 先删除hpa对myblog的限制
horizontalpodautoscaler.autoscaling "hpa-myblog" deleted
[root@k8s-master ~]# kubectl -n luffy get hpa
No resources found in luffy namespace.
[root@k8s-master ~]# kubectl -n luffy scale deployment myblog --replicas=5
deployment.apps/myblog scaled
[root@k8s-master ~]# kubectl -n luffy get po
NAME READY STATUS RESTARTS AGE
myblog-76d54c49b6-26br5 1/1 Running 2 17h
myblog-76d54c49b6-bfkvf 1/1 Running 0 13h
myblog-76d54c49b6-nn4bb 0/1 Running 0 10s
myblog-76d54c49b6-sj4q6 0/1 Running 0 10s
myblog-76d54c49b6-w9wqd 0/1 Running 0 10s
mysql-864b4c85b5-9cdds 1/1 Running 0 17h
[root@k8s-master ~]# kubectl -n luffy get po -owide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
myblog-76d54c49b6-26br5 1/1 Running 2 17h 10.244.1.37 k8s-slave1 <none> <none>
myblog-76d54c49b6-bfkvf 1/1 Running 0 13h 10.244.1.42 k8s-slave1 <none> <none>
myblog-76d54c49b6-nn4bb 1/1 Running 0 17s 10.244.1.44 k8s-slave1 <none> <none>
myblog-76d54c49b6-sj4q6 1/1 Running 0 17s 10.244.0.21 k8s-master <none> <none>
myblog-76d54c49b6-w9wqd 1/1 Running 0 17s 10.244.1.43 k8s-slave1 <none> <none>
mysql-864b4c85b5-9cdds 1/1 Running 0 17h 10.244.1.40 k8s-slave1 <none> <none>
[root@k8s-master ~]# kubectl -n luffy get no
NAME STATUS ROLES AGE VERSION
k8s-master Ready control-plane,master 9d v1.21.5
k8s-slave1 Ready <none> 9d v1.21.5
k8s-slave2 Ready <none> 9d v1.21.5
[root@k8s-master ~]# kubectl -n luffy scale deployment myblog --replicas=10
deployment.apps/myblog scaled
[root@k8s-master ~]# kubectl -n luffy get po -owide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
myblog-76d54c49b6-26br5 1/1 Running 2 17h 10.244.1.37 k8s-slave1 <none> <none>
myblog-76d54c49b6-2v6x9 0/1 Running 0 5s 10.244.0.23 k8s-master <none> <none>
myblog-76d54c49b6-8cwc7 0/1 Running 0 5s 10.244.1.45 k8s-slave1 <none> <none>
myblog-76d54c49b6-bfkvf 1/1 Running 0 13h 10.244.1.42 k8s-slave1 <none> <none>
myblog-76d54c49b6-h6jx7 0/1 Running 0 5s 10.244.0.22 k8s-master <none> <none>
myblog-76d54c49b6-hrswf 0/1 Running 0 5s 10.244.1.46 k8s-slave1 <none> <none>
myblog-76d54c49b6-k5vbk 0/1 Running 0 5s 10.244.0.24 k8s-master <none> <none>
myblog-76d54c49b6-nn4bb 1/1 Running 0 52s 10.244.1.44 k8s-slave1 <none> <none>
myblog-76d54c49b6-sj4q6 1/1 Running 0 52s 10.244.0.21 k8s-master <none> <none>
myblog-76d54c49b6-w9wqd 1/1 Running 0 52s 10.244.1.43 k8s-slave1 <none> <none>
mysql-864b4c85b5-9cdds 1/1 Running 0 17h 10.244.1.40 k8s-slave1 <none> <none>
[root@k8s-master ~]# kubectl cordon k8s-slave1 #以上操作死活没跑到slave2机器上，这里把slave1设置不可调度
node/k8s-slave1 cordoned
[root@k8s-master ~]# kubectl -n luffy scale deployment myblog --replicas=16
deployment.apps/myblog scaled
[root@k8s-master ~]# kubectl -n luffy get po -owide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
myblog-76d54c49b6-26br5 1/1 Running 2 17h 10.244.1.37 k8s-slave1 <none> <none>
myblog-76d54c49b6-2v6x9 1/1 Running 0 3m51s 10.244.0.23 k8s-master <none> <none>
myblog-76d54c49b6-8cwc7 1/1 Running 0 3m51s 10.244.1.45 k8s-slave1 <none> <none>
myblog-76d54c49b6-bfkvf 1/1 Running 0 13h 10.244.1.42 k8s-slave1 <none> <none>
myblog-76d54c49b6-h6jx7 1/1 Running 0 3m51s 10.244.0.22 k8s-master <none> <none>
myblog-76d54c49b6-hrswf 1/1 Running 0 3m51s 10.244.1.46 k8s-slave1 <none> <none>
myblog-76d54c49b6-hsfhj 0/1 Running 0 6s 10.244.0.27 k8s-master <none> <none>
myblog-76d54c49b6-k5vbk 1/1 Running 0 3m51s 10.244.0.24 k8s-master <none> <none>
myblog-76d54c49b6-nfrh5 0/1 Running 0 6s 10.244.0.25 k8s-master <none> <none>
myblog-76d54c49b6-nn4bb 1/1 Running 0 4m38s 10.244.1.44 k8s-slave1 <none> <none>
myblog-76d54c49b6-q4bb8 0/1 Running 0 6s 10.244.0.29 k8s-master <none> <none>
myblog-76d54c49b6-rsdhp 0/1 Running 0 6s 10.244.0.28 k8s-master <none> <none>
myblog-76d54c49b6-sj4q6 1/1 Running 0 4m38s 10.244.0.21 k8s-master <none> <none>
myblog-76d54c49b6-w9wqd 1/1 Running 0 4m38s 10.244.1.43 k8s-slave1 <none> <none>
myblog-76d54c49b6-xdxpm 0/1 Running 0 6s 10.244.0.26 k8s-master <none> <none>
myblog-76d54c49b6-xvm84 0/1 Running 0 6s 10.244.0.30 k8s-master <none> <none>
mysql-864b4c85b5-9cdds 1/1 Running 0 17h 10.244.1.40 k8s-slave1 <none> <none>
[root@k8s-master ~]# kubectl -n luffy scale deployment myblog --replicas=1
deployment.apps/myblog scaled
[root@k8s-master ~]# kubectl -n luffy get po -owide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
myblog-76d54c49b6-26br5 1/1 Running 2 17h 10.244.1.37 k8s-slave1 <none> <none>
myblog-76d54c49b6-2v6x9 1/1 Running 0 5m37s 10.244.0.23 k8s-master <none> <none>
myblog-76d54c49b6-8cwc7 1/1 Running 0 5m37s 10.244.1.45 k8s-slave1 <none> <none>
myblog-76d54c49b6-bfkvf 1/1 Running 0 13h 10.244.1.42 k8s-slave1 <none> <none>
myblog-76d54c49b6-h6jx7 1/1 Running 0 5m37s 10.244.0.22 k8s-master <none> <none>
myblog-76d54c49b6-hrswf 1/1 Running 0 5m37s 10.244.1.46 k8s-slave1 <none> <none>
myblog-76d54c49b6-hsfhj 1/1 Running 0 112s 10.244.0.27 k8s-master <none> <none>
myblog-76d54c49b6-k5vbk 1/1 Running 0 5m37s 10.244.0.24 k8s-master <none> <none>
myblog-76d54c49b6-nfrh5 1/1 Running 0 112s 10.244.0.25 k8s-master <none> <none>
myblog-76d54c49b6-nn4bb 1/1 Running 0 6m24s 10.244.1.44 k8s-slave1 <none> <none>
myblog-76d54c49b6-q4bb8 1/1 Running 0 112s 10.244.0.29 k8s-master <none> <none>
myblog-76d54c49b6-rsdhp 1/1 Running 0 112s 10.244.0.28 k8s-master <none> <none>
myblog-76d54c49b6-sj4q6 1/1 Running 0 6m24s 10.244.0.21 k8s-master <none> <none>
myblog-76d54c49b6-w9wqd 1/1 Running 0 6m24s 10.244.1.43 k8s-slave1 <none> <none>
myblog-76d54c49b6-xdxpm 1/1 Running 0 112s 10.244.0.26 k8s-master <none> <none>
myblog-76d54c49b6-xvm84 1/1 Running 0 112s 10.244.0.30 k8s-master <none> <none>
mysql-864b4c85b5-9cdds 1/1 Running 0 17h 10.244.1.40 k8s-slave1 <none> <none>
[root@k8s-master ~]# kubectl -n luffy get po -owide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
myblog-76d54c49b6-26br5 1/1 Running 2 17h 10.244.1.37 k8s-slave1 <none> <none>
myblog-76d54c49b6-2v6x9 1/1 Running 0 5m39s 10.244.0.23 k8s-master <none> <none>
myblog-76d54c49b6-8cwc7 1/1 Running 0 5m39s 10.244.1.45 k8s-slave1 <none> <none>
myblog-76d54c49b6-bfkvf 1/1 Running 0 13h 10.244.1.42 k8s-slave1 <none> <none>
myblog-76d54c49b6-h6jx7 1/1 Running 0 5m39s 10.244.0.22 k8s-master <none> <none>
myblog-76d54c49b6-hrswf 1/1 Running 0 5m39s 10.244.1.46 k8s-slave1 <none> <none>
myblog-76d54c49b6-hsfhj 1/1 Running 0 114s 10.244.0.27 k8s-master <none> <none>
myblog-76d54c49b6-k5vbk 1/1 Running 0 5m39s 10.244.0.24 k8s-master <none> <none>
myblog-76d54c49b6-nfrh5 1/1 Running 0 114s 10.244.0.25 k8s-master <none> <none>
myblog-76d54c49b6-nn4bb 1/1 Running 0 6m26s 10.244.1.44 k8s-slave1 <none> <none>
myblog-76d54c49b6-q4bb8 1/1 Running 0 114s 10.244.0.29 k8s-master <none> <none>
myblog-76d54c49b6-rsdhp 1/1 Running 0 114s 10.244.0.28 k8s-master <none> <none>
myblog-76d54c49b6-sj4q6 1/1 Running 0 6m26s 10.244.0.21 k8s-master <none> <none>
myblog-76d54c49b6-w9wqd 1/1 Running 0 6m26s 10.244.1.43 k8s-slave1 <none> <none>
myblog-76d54c49b6-xdxpm 1/1 Running 0 114s 10.244.0.26 k8s-master <none> <none>
myblog-76d54c49b6-xvm84 1/1 Running 0 114s 10.244.0.30 k8s-master <none> <none>
mysql-864b4c85b5-9cdds 1/1 Running 0 17h 10.244.1.40 k8s-slave1 <none> <none>
[root@k8s-master ~]# kubectl -n luffy get po -owide #依然没有到slave2机器上
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
myblog-76d54c49b6-26br5 1/1 Running 2 17h 10.244.1.37 k8s-slave1 <none> <none>
myblog-76d54c49b6-2v6x9 1/1 Running 0 5m58s 10.244.0.23 k8s-master <none> <none>
myblog-76d54c49b6-8cwc7 1/1 Running 0 5m58s 10.244.1.45 k8s-slave1 <none> <none>
myblog-76d54c49b6-bfkvf 1/1 Running 0 13h 10.244.1.42 k8s-slave1 <none> <none>
myblog-76d54c49b6-h6jx7 1/1 Running 0 5m58s 10.244.0.22 k8s-master <none> <none>
myblog-76d54c49b6-hrswf 1/1 Running 0 5m58s 10.244.1.46 k8s-slave1 <none> <none>
myblog-76d54c49b6-hsfhj 1/1 Running 0 2m13s 10.244.0.27 k8s-master <none> <none>
myblog-76d54c49b6-k5vbk 1/1 Running 0 5m58s 10.244.0.24 k8s-master <none> <none>
myblog-76d54c49b6-nfrh5 1/1 Running 0 2m13s 10.244.0.25 k8s-master <none> <none>
myblog-76d54c49b6-nn4bb 1/1 Running 0 6m45s 10.244.1.44 k8s-slave1 <none> <none>
myblog-76d54c49b6-q4bb8 1/1 Running 0 2m13s 10.244.0.29 k8s-master <none> <none>
myblog-76d54c49b6-rsdhp 1/1 Running 0 2m13s 10.244.0.28 k8s-master <none> <none>
myblog-76d54c49b6-sj4q6 1/1 Running 0 6m45s 10.244.0.21 k8s-master <none> <none>
myblog-76d54c49b6-w9wqd 1/1 Running 0 6m45s 10.244.1.43 k8s-slave1 <none> <none>
myblog-76d54c49b6-xdxpm 1/1 Running 0 2m13s 10.244.0.26 k8s-master <none> <none>
myblog-76d54c49b6-xvm84 1/1 Running 0 2m13s 10.244.0.30 k8s-master <none> <none>
mysql-864b4c85b5-9cdds 1/1 Running 0 17h 10.244.1.40 k8s-slave1 <none> <none>
[root@k8s-master ~]# kubectl -n luffy scale deployment myblog --replicas=2 # 调整回来2个副本
deployment.apps/myblog scaled
[root@k8s-master ~]# kubectl -n luffy get po -owide # 发现slave1机器容器没有消失
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
myblog-76d54c49b6-26br5 1/1 Running 2 17h 10.244.1.37 k8s-slave1 <none> <none>
myblog-76d54c49b6-2v6x9 1/1 Running 0 6m12s 10.244.0.23 k8s-master <none> <none>
myblog-76d54c49b6-8cwc7 1/1 Running 0 6m12s 10.244.1.45 k8s-slave1 <none> <none>
myblog-76d54c49b6-bfkvf 1/1 Running 0 13h 10.244.1.42 k8s-slave1 <none> <none>
myblog-76d54c49b6-h6jx7 1/1 Running 0 6m12s 10.244.0.22 k8s-master <none> <none>
myblog-76d54c49b6-hrswf 1/1 Running 0 6m12s 10.244.1.46 k8s-slave1 <none> <none>
myblog-76d54c49b6-hsfhj 1/1 Running 0 2m27s 10.244.0.27 k8s-master <none> <none>
myblog-76d54c49b6-k5vbk 1/1 Running 0 6m12s 10.244.0.24 k8s-master <none> <none>
myblog-76d54c49b6-nfrh5 1/1 Running 0 2m27s 10.244.0.25 k8s-master <none> <none>
myblog-76d54c49b6-nn4bb 1/1 Running 0 6m59s 10.244.1.44 k8s-slave1 <none> <none>
myblog-76d54c49b6-q4bb8 1/1 Running 0 2m27s 10.244.0.29 k8s-master <none> <none>
myblog-76d54c49b6-rsdhp 1/1 Running 0 2m27s 10.244.0.28 k8s-master <none> <none>
myblog-76d54c49b6-sj4q6 1/1 Running 0 6m59s 10.244.0.21 k8s-master <none> <none>
myblog-76d54c49b6-w9wqd 1/1 Running 0 6m59s 10.244.1.43 k8s-slave1 <none> <none>
myblog-76d54c49b6-xdxpm 1/1 Running 0 2m27s 10.244.0.26 k8s-master <none> <none>
myblog-76d54c49b6-xvm84 1/1 Running 0 2m27s 10.244.0.30 k8s-master <none> <none>
mysql-864b4c85b5-9cdds 1/1 Running 0 17h 10.244.1.40 k8s-slave1 <none> <none>
[root@k8s-master ~]# kubectl drain k8s-slave1
node/k8s-slave1 already cordoned
error: unable to drain node "k8s-slave1", aborting command...
There are pending nodes to be drained:
k8s-slave1
cannot delete Pods not managed by ReplicationController, ReplicaSet, Job, DaemonSet or StatefulSet (use --force to override): demo/default-mem-demo
cannot delete DaemonSet-managed Pods (use --ignore-daemonsets to ignore): kube-flannel/kube-flannel-ds-bp7tm, kube-system/kube-proxy-7fsvw
[root@k8s-master ~]# kubectl drain k8s-slave1
node/k8s-slave1 already cordoned
error: unable to drain node "k8s-slave1", aborting command...
There are pending nodes to be drained:
k8s-slave1
cannot delete Pods not managed by ReplicationController, ReplicaSet, Job, DaemonSet or StatefulSet (use --force to override): demo/default-mem-demo
cannot delete DaemonSet-managed Pods (use --ignore-daemonsets to ignore): kube-flannel/kube-flannel-ds-bp7tm, kube-system/kube-proxy-7fsvw
[root@k8s-master ~]# kubectl get no
\NAME STATUS ROLES AGE VERSION
k8s-master Ready control-plane,master 9d v1.21.5
k8s-slave1 Ready,SchedulingDisabled <none> 9d v1.21.5
k8s-slave2 Ready <none> 9d v1.21.5
# 注意：若node节点上存在daemonsets控制器创建的pod,则需要使用--ignore-daemonsets忽略错误错误警告
[root@k8s-master ~]# kubectl drain k8s-slave1 --ignore-daemonsets
node/k8s-slave1 already cordoned
error: unable to drain node "k8s-slave1", aborting command...
There are pending nodes to be drained:
k8s-slave1
error: cannot delete Pods not managed by ReplicationController, ReplicaSet, Job, DaemonSet or StatefulSet (use --force to override): demo/default-mem-demo
# 恢复可调度
[root@k8s-master ~]# kubectl uncordon k8s-slave1
node/k8s-slave1 uncordoned
[root@k8s-master ~]# kubectl get no
NAME STATUS ROLES AGE VERSION
k8s-master Ready control-plane,master 10d v1.21.5
k8s-slave1 Ready <none> 10d v1.21.5
k8s-slave2 Ready <none> 10d v1.21.5
问题： 按照课件操作的代码， pv pvc 都配置好的。 启动的容器只是在创建状态，并没有启动。¶
这个部分老师没有做演示直接带过了
[root@k8s-master pvc]# kubectl get pv
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE
nfs-pv 1Gi RWX Retain Bound default/pvc-nfs 53m
[root@k8s-master pvc]# kubectl get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
pvc-nfs Bound nfs-pv 1Gi RWX 9s
[root@k8s-master pvc]# cat deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: nfs-pvc
spec:
replicas: 1
selector: #指定Pod的选择器
matchLabels:
app: nginx
template:
metadata:
labels:
app: nginx
spec:
containers:
- name: nginx
image: nginx:alpine
imagePullPolicy: IfNotPresent
ports:
- containerPort: 80
name: web
volumeMounts: #挂载容器中的目录到pvc nfs中的目录
- name: www
mountPath: /usr/share/nginx/html
volumes:
- name: www
persistentVolumeClaim: #指定pvc
claimName: pvc-nfs
[root@k8s-master pvc]# kubectl apply -f deployment.yaml
deployment.apps/nfs-pvc created
[root@k8s-master pvc]# kubectl get deploy
NAME READY UP-TO-DATE AVAILABLE AGE
nfs-pvc 0/1 1 0 10s
[root@k8s-master pvc]# kubectl get po
NAME READY STATUS RESTARTS AGE
nfs-pvc-7bf65c788-qbhns 0/1 ContainerCreating 0 14s
[root@k8s-master pvc]# kubectl logs nfs-pvc-7bf65c788-qbhns
Error from server (BadRequest): container "nginx" in pod "nfs-pvc-7bf65c788-qbhns" is waiting to start: ContainerCreating
[root@k8s-master pvc]# kubectl describe pod nfs-pvc-7bf65c788-klkx9 #查看容器详情
Name: nfs-pvc-7bf65c788-klkx9
Namespace: default
Priority: 0
Node: k8s-slave1/10.211.55.26
Start Time: Thu, 20 Oct 2022 23:56:10 +0800
Labels: app=nginx
pod-template-hash=7bf65c788
Annotations: <none>
Status: Pending
IP:
IPs: <none>
Controlled By: ReplicaSet/nfs-pvc-7bf65c788
Containers:
nginx:
Container ID:
Image: nginx:alpine
Image ID:
Port: 80/TCP
Host Port: 0/TCP
State: Waiting
Reason: ContainerCreating
Ready: False
Restart Count: 0
Environment: <none>
Mounts:
/usr/share/nginx/html from www (rw)
/var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-7gv77 (ro)
Conditions:
Type Status
Initialized True
Ready False
ContainersReady False
PodScheduled True
Volumes:
www:
Type: PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
ClaimName: pvc-nfs
ReadOnly: false
kube-api-access-7gv77:
Type: Projected (a volume that contains injected data from multiple sources)
TokenExpirationSeconds: 3607
ConfigMapName: kube-root-ca.crt
ConfigMapOptional: <nil>
DownwardAPI: true
QoS Class: BestEffort
Node-Selectors: <none>
Tolerations: node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Warning FailedMount 15m (x23 over 74m) kubelet Unable to attach or mount volumes: unmounted volumes=[www], unattached volumes=[www kube-api-access-7gv77]: timed out waiting for the condition
Warning FailedMount 46s (x45 over 76m) kubelet MountVolume.SetUp failed for volume "nfs-pv" : mount failed: exit status 32
Mounting command: mount
Mounting arguments: -t nfs 10.211.55.26:/data/k8s/nginx /var/lib/kubelet/pods/dabee207-be5c-4e06-927e-fe86bd6dc88b/volumes/kubernetes.io~nfs/nfs-pv
Output: mount.nfs: mounting 10.211.55.26:/data/k8s/nginx failed, reason given by server: No such file or directory
# nfs服务器上创建目录
[root@k8s-slave1 k8s]# mkdir /data/k8s/nginx
[root@k8s-master pvc]# kubectl delete deploy nfs-pvc
deployment.apps "nfs-pvc" deleted
[root@k8s-master pvc]# kubectl apply -f deployment.yaml
deployment.apps/nfs-pvc created
[root@k8s-master pvc]# kubectl get po
NAME READY STATUS RESTARTS AGE
nfs-pvc-7bf65c788-h2sqk 1/1 Running 0 5s
[root@k8s-master pvc]#
问题，helm 安装wordpress 浏览器访问不了¶
[root@k8s-master linux-amd64]# helm repo add stable https://charts.bitnami.com/bitnami
Error: looks like "https://charts.bitnami.com/bitnami" is not a valid chart repository or cannot be reached: Get https://charts.bitnami.com/bitnami/index.yaml: dial tcp: lookup charts.bitnami.com on 10.211.55.1:53: read udp 10.211.55.25:35013->10.211.55.1:53: i/o timeout
[root@k8s-master linux-amd64]# helm repo add stable https://charts.bitnami.com/bitnami
"stable" has been added to your repositories
[root@k8s-master linux-amd64]# helm repo update
Hang tight while we grab the latest from your chart repositories...
...Successfully got an update from the "stable" chart repository
Update Complete. ⎈ Happy Helming!⎈
[root@k8s-master linux-amd64]# helm search repo wordpress
NAME CHART VERSION APP VERSION DESCRIPTION
stable/wordpress 15.2.6 6.0.3 WordPress is the world's most popular blogging ...
stable/wordpress-intel 2.1.12 6.0.3 WordPress for Intel is the most popular bloggin...
[root@k8s-master linux-amd64]# kubectl create namespace wordpress
namespace/wordpress created
[root@k8s-master linux-amd64]# helm -n wordpress install wordpress stable/wordpress --set mariadb.primary.persistence.enabled=false --set service.type=ClusterIP --set ingress.enabled=true --set persistence.enabled=false --set ingress.hostname=wordpress.luffy.com
NAME: wordpress
LAST DEPLOYED: Fri Oct 21 13:41:45 2022
NAMESPACE: wordpress
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
CHART NAME: wordpress
CHART VERSION: 15.2.6
APP VERSION: 6.0.3
** Please be patient while the chart is being deployed **
Your WordPress site can be accessed through the following DNS name from within your cluster:
wordpress.wordpress.svc.cluster.local (port 80)
To access your WordPress site from outside the cluster follow the steps below:
1. Get the WordPress URL and associate WordPress hostname to your cluster external IP:
export CLUSTER_IP=$(minikube ip) # On Minikube. Use: `kubectl cluster-info` on others K8s clusters
echo "WordPress URL: http://wordpress.luffy.com/"
echo "$CLUSTER_IP wordpress.luffy.com" | sudo tee -a /etc/hosts
2. Open a browser and access WordPress using the obtained URL.
3. Login with the following credentials below to see your blog:
echo Username: user
echo Password: $(kubectl get secret --namespace wordpress wordpress -o jsonpath="{.data.wordpress-password}" | base64 -d)
[root@k8s-master linux-amd64]# kubectl get secret --namespace wordpress wordpress -o jsonpath="{.data.wordpress-password}" | base64 -d
ONs40PGjLf[root@k8s-master linux-amd64]# kubectl cluster-info
Kubernetes control plane is running at https://10.211.55.25:6443
CoreDNS is running at https://10.211.55.25:6443/api/v1/namespaces/kube-system/services/kube-dns:dns/proxy
To further debug and diagnose cluster problems, use 'kubectl cluster-info dump'.
[root@k8s-master linux-amd64]# helm -n wordpress ls
NAME NAMESPACE REVISION UPDATED STATUS CHART APP VERSION
wordpress wordpress 1 2022-10-21 13:41:45.706167389 +0800 CST deployed wordpress-15.2.6 6.0.3
[root@k8s-master linux-amd64]# helm -n wordpress get all
Error: "helm get all" requires 1 argument
Usage: helm get all RELEASE_NAME [flags]
[root@k8s-master linux-amd64]# kubectl -n wordpress get all
NAME READY STATUS RESTARTS AGE
pod/wordpress-745967ff4-cmnj9 1/1 Running 0 6m27s
pod/wordpress-mariadb-0 1/1 Running 0 6m27s
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/wordpress ClusterIP 10.101.126.165 <none> 80/TCP,443/TCP 6m27s
service/wordpress-mariadb ClusterIP 10.102.7.97 <none> 3306/TCP 6m27s
NAME READY UP-TO-DATE AVAILABLE AGE
deployment.apps/wordpress 1/1 1 1 6m27s
NAME DESIRED CURRENT READY AGE
replicaset.apps/wordpress-745967ff4 1 1 1 6m27s
NAME READY AGE
statefulset.apps/wordpress-mariadb 1/1 6m27s
[root@k8s-master helm3]# helm -n wordpress ls
NAME NAMESPACE REVISION UPDATED STATUS CHART APP VERSION
wordpress wordpress 1 2022-10-21 13:41:45.706167389 +0800 CST deployed wordpress-15.2.6 6.0.3
[root@k8s-master helm3]# kubectl -n wordpress get all
NAME READY STATUS RESTARTS AGE
pod/wordpress-745967ff4-cmnj9 1/1 Running 0 15m
pod/wordpress-mariadb-0 1/1 Running 0 15m
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/wordpress ClusterIP 10.101.126.165 <none> 80/TCP,443/TCP 15m
service/wordpress-mariadb ClusterIP 10.102.7.97 <none> 3306/TCP 15m
NAME READY UP-TO-DATE AVAILABLE AGE
deployment.apps/wordpress 1/1 1 1 15m
NAME DESIRED CURRENT READY AGE
replicaset.apps/wordpress-745967ff4 1 1 1 15m
NAME READY AGE
statefulset.apps/wordpress-mariadb 1/1 15m
[root@k8s-master helm3]# kubectl -n wordpress get po
NAME READY STATUS RESTARTS AGE
wordpress-745967ff4-cmnj9 1/1 Running 0 16m
wordpress-mariadb-0 1/1 Running 0 16m
[root@k8s-master helm3]# kubectl -n wordpress get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
wordpress ClusterIP 10.101.126.165 <none> 80/TCP,443/TCP 17m
wordpress-mariadb ClusterIP 10.102.7.97 <none> 3306/TCP 17m
[root@k8s-master helm3]# kubectl -n wordpress ing
Error: flags cannot be placed before plugin name: -n
[root@k8s-master helm3]# kubectl -n wordpress get ing
NAME CLASS HOSTS ADDRESS PORTS AGE
wordpress <none> wordpress.luffy.com 80 17m
[root@k8s-master helm3]# kubectl -n wordpress get cm
NAME DATA AGE
kube-root-ca.crt 1 18m
wordpress-mariadb 1 18m
[root@k8s-master helm3]# kubectl -n wordpress get secrets
NAME TYPE DATA AGE
default-token-ck95p kubernetes.io/service-account-token 3 19m
sh.helm.release.v1.wordpress.v1 helm.sh/release.v1 1 18m
wordpress Opaque 1 18m
wordpress-mariadb Opaque 2 18m
wordpress-mariadb-token-tbllp kubernetes.io/service-account-token 3 18m
[root@k8s-master helm3]# kubectl -n wordpress get ing
NAME CLASS HOSTS ADDRESS PORTS AGE
wordpress <none> wordpress.luffy.com 80 23m
[root@k8s-master helm3]# kubectl -n wordpress describe ing
Name: wordpress
Namespace: wordpress
Address:
Default backend: default-http-backend:80 (<error: endpoints "default-http-backend" not found>)
Rules:
Host Path Backends
---- ---- --------
wordpress.luffy.com
/ wordpress:http (10.244.0.32:8080)
Annotations: meta.helm.sh/release-name: wordpress
meta.helm.sh/release-namespace: wordpress
Events: <none>
[root@k8s-master helm3]# curl -HHost:wordpress.luffy.com 10.211.55.25:80
curl: (7) Failed connect to 10.211.55.25:80; 拒绝连接
原因： 没装ingress-controller
问题 helm plugin push 安装不成功¶
[root@k8s-master helm3]# tar zxf helm-push_0.8.1_linux_amd64.tar.gz -C helm-push
[root@k8s-master helm3]# cd helm-push/
[root@k8s-master helm-push]# helm plugin install ./
Error: plugin already exists
[root@k8s-master helm-push]# helm plugin ls
NAME VERSION DESCRIPTION
cm-push 0.10.3 Push chart package to ChartMuseum
[root@k8s-master helm-push]# cd ../
[root@k8s-master helm3]# ls
harbor harbor.ca.crt helm-push helm-v3.2.4-linux-amd64.tar.gz nginx
harbor-1.10.1.tgz harbor-pvc.yaml helm-push_0.8.1_linux_amd64.tar.gz linux-amd64
[root@k8s-master helm-push]# helm push
Error: unknown command "push" for "helm"
Did you mean this?
pull
Run 'helm --help' for usage.
# 注意这里要使用
helm cm-push
而不是
helm push
# 0.10.3版本 使用cm-push
# harbor 是当前目录下 chart文件包解压的目录。 ca-file也是当前目录下的文件
[root@k8s-master helm3]# helm cm-push harbor luffy --ca-file=harbor.ca.crt -u admin -p Harbor12345
Pushing harbor-1.10.1.tgz to luffy...
Done.
问题 kubectl -n monitor get cm alertmanager -o yaml 输出的内容带有 \n 没有正确的显示¶
[root@k8s-master alertmanager]# cat config.yml
apiVersion: v1
data:
config.yml: |
global:
resolve_timeout: 5m
smtp_smarthost: 'smtp.163.com:25'
smtp_from: '343264992@163.com'
smtp_auth_username: '343264992@163.com'
smtp_auth_password: 'TTDIQQDRGRNEWJGI'
smtp_require_tls: false
route:
group_by: ['alertname']
group_wait: 30s
group_interval: 10m
repeat_interval: 1d
receiver: default
routes:
- {}
receivers:
- name: 'default'
email_configs:
- to: '343264992@qq.com'
send_resolved: true
kind: ConfigMap
metadata:
name: alertmanager
namespace: monitor
[root@k8s-master alertmanager]# kubectl apply -f config.yml
configmap/alertmanager unchanged
[root@k8s-master alertmanager]# kubectl -n monitor get cm alertmanager -o yaml
apiVersion: v1
data:
config.yml: "global: \n resolve_timeout: 5m\n smtp_smarthost: 'smtp.163.com:25'\n
\ smtp_from: '343264992@163.com'\n smtp_auth_username: '343264992@163.com'\n
\ smtp_auth_password: 'TTDIQQDRGRNEWJGI'\n smtp_require_tls: false\nroute:\n
\ group_by: ['alertname']\n group_wait: 30s\n group_interval: 10m\n repeat_interval:
1d\n receiver: default\n routes:\n - {}\nreceivers:\n- name: 'default'\n email_configs:\n
\ - to: '343264992@qq.com'\n send_resolved: true\n"
kind: ConfigMap
metadata:
annotations:
kubectl.kubernetes.io/last-applied-configuration: |
{"apiVersion":"v1","data":{"config.yml":"global: \n resolve_timeout: 5m\n smtp_smarthost: 'smtp.163.com:25'\n smtp_from: '343264992@163.com'\n smtp_auth_username: '343264992@163.com'\n smtp_auth_password: 'TTDIQQDRGRNEWJGI'\n smtp_require_tls: false\nroute:\n group_by: ['alertname']\n group_wait: 30s\n group_interval: 10m\n repeat_interval: 1d\n receiver: default\n routes:\n - {}\nreceivers:\n- name: 'default'\n email_configs:\n - to: '343264992@qq.com'\n send_resolved: true\n"},"kind":"ConfigMap","metadata":{"annotations":{},"name":"alertmanager","namespace":"monitor"}}
creationTimestamp: "2022-10-27T08:16:43Z"
name: alertmanager
namespace: monitor
resourceVersion: "1026788"
selfLink: /api/v1/namespaces/monitor/configmaps/alertmanager
uid: 855361f5-3f7c-4f2e-a36d-0f96ad1e86b1
# 解决 yaml 文件使用 cat -A ***.yml 查看每一行后面不要有空格。 上部分\n解决
# 想要完整好使用 kubectl create --from-file=
问题：获取apiserver 错误 curl: (7) Failed connect to 10.96.0.1:6443; 拒绝连接¶
[root@k8s-master ~]# kubectl -n monitor describe secrets prometheus-token-g25wf
Name: prometheus-token-g25wf
Namespace: monitor
Labels: <none>
Annotations: kubernetes.io/service-account.name: prometheus
kubernetes.io/service-account.uid: 3bfb1b24-e537-424e-90d7-42136172e0db
Type: kubernetes.io/service-account-token
Data
====
ca.crt: 1066 bytes
namespace: 7 bytes
token: eyJhbGciOiJSUzI1NiIsImtpZCI6Ik9VMi1HX3FFMlBUT193OUo3ZWI4eDh3aE9pc0dTYXMyQWRMNnRHNHJtMWsifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJtb25pdG9yIiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZWNyZXQubmFtZSI6InByb21ldGhldXMtdG9rZW4tZzI1d2YiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC5uYW1lIjoicHJvbWV0aGV1cyIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50LnVpZCI6IjNiZmIxYjI0LWU1MzctNDI0ZS05MGQ3LTQyMTM2MTcyZTBkYiIsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDptb25pdG9yOnByb21ldGhldXMifQ.Yq5onkrUCRkUbF5NuMsjVmXo4DOC3tDHsTCcucaEITeAI3T67g-WWnnEPntCZgKA8-_0pAQUB0oyVWCOTzpOeHsVsLk0sU2Sdik6hvTARu__Ljwra1NRfEf8S_PJDFKsb6iYWCnMj4ximepWX2zpn5rPJqR2VZ-cDgxNrKMOUODgh1_r_zr7Zi9RXyGp7g4jVnxnfe4RXCCylq2WVeX0C5w5kpw2vfIq9HOWhip94sm00iffv7YOn4ACGXfQ0SLZRJyUMv7wXmksliFe85KTHMerqMBdaOf6dE3ylYoqMnQuKSQ5gCnLOWMcX2XjSZ7li1lPNvQntPePCGPG7mdImA
[root@k8s-master ~]# kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 15d
nginx ClusterIP None <none> 80/TCP 32h
[root@k8s-master ~]# curl -k -H "Authorization: Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6Ik9VMi1HX3FFMlBUT193OUo3ZWI4eDh3aE9pc0dTYXMyQWRMNnRHNHJtMWsifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJtb25pdG9yIiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZWNyZXQubmFtZSI6InByb21ldGhldXMtdG9rZW4tZzI1d2YiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC5uYW1lIjoicHJvbWV0aGV1cyIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50LnVpZCI6IjNiZmIxYjI0LWU1MzctNDI0ZS05MGQ3LTQyMTM2MTcyZTBkYiIsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDptb25pdG9yOnByb21ldGhldXMifQ.Yq5onkrUCRkUbF5NuMsjVmXo4DOC3tDHsTCcucaEITeAI3T67g-WWnnEPntCZgKA8-_0pAQUB0oyVWCOTzpOeHsVsLk0sU2Sdik6hvTARu__Ljwra1NRfEf8S_PJDFKsb6iYWCnMj4ximepWX2zpn5rPJqR2VZ-cDgxNrKMOUODgh1_r_zr7Zi9RXyGp7g4jVnxnfe4RXCCylq2WVeX0C5w5kpw2vfIq9HOWhip94sm00iffv7YOn4ACGXfQ0SLZRJyUMv7wXmksliFe85KTHMerqMBdaOf6dE3ylYoqMnQuKSQ5gCnLOWMcX2XjSZ7li1lPNvQntPePCGPG7mdImA" https://10.96.0.1:6443/metrics
curl: (7) Failed connect to 10.96.0.1:6443; 拒绝连接 # 改成443就可以了
[root@k8s-master prometheus]# kubectl describe svc kubernetes
Name: kubernetes
Namespace: default
Labels: component=apiserver
provider=kubernetes
Annotations: <none>
Selector: <none>
Type: ClusterIP
IP Family Policy: SingleStack
IP Families: IPv4
IP: 10.96.0.1
IPs: 10.96.0.1
Port: https 443/TCP
TargetPort: 6443/TCP
Endpoints: 10.211.55.25:6443
Session Affinity: None
Events: <none>
curl -k -H "Authorization: Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6Ik9VMi1HX3FFMlBUT193OUo3ZWI4eDh3aE9pc0dTYXMyQWRMNnRHNHJtMWsifQ.eyJhdWQiOlsiaHR0cHM6Ly9rdWJlcm5ldGVzLmRlZmF1bHQuc3ZjLmNsdXN0ZXIubG9jYWwiXSwiZXhwIjoxNjk4MjAyOTMyLCJpYXQiOjE2NjY2NjY5MzIsImlzcyI6Imh0dHBzOi8va3ViZXJuZXRlcy5kZWZhdWx0LnN2Yy5jbHVzdGVyLmxvY2FsIiwia3ViZXJuZXRlcy5pbyI6eyJuYW1lc3BhY2UiOiJtb25pdG9yIiwicG9kIjp7Im5hbWUiOiJwcm9tZXRoZXVzLTdkNjc5OWM4NGMtdGc3NzQiLCJ1aWQiOiI2ODg5ZTY1OC1kNjRiLTQwZjctYjVkNi1hNjc2YWMxNmVmNTgifSwic2VydmljZWFjY291bnQiOnsibmFtZSI6InByb21ldGhldXMiLCJ1aWQiOiIzYmZiMWIyNC1lNTM3LTQyNGUtOTBkNy00MjEzNjE3MmUwZGIifSwid2FybmFmdGVyIjoxNjY2NjcwNTM5fSwibmJmIjoxNjY2NjY2OTMyLCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6bW9uaXRvcjpwcm9tZXRoZXVzIn0.Jh_l5odIF72yamDKO9nDQoTxX6QRSZkpLchNxIa7dA53FZgog-jSCToitSN2qJcB5-24q-JQ9F8ZezwhPU5IAbUVz9i2NalcFpooCIrjRaYaqpS8IOk3I9SvPIcwyQQn6iJryvFzRhaj9t5h0eC1UxHpvpX8ziQgrlEmCh_FBX1IMZ85hed3Is8sBdaZLVrImUcuIdL5aEW9fbnOQKOP3TgNQ-Y__K-inf9sfAypU7bciti-L6axbWAgXtX40CfB5WV1ApyBLopMVDl06UypN8JD3yRQPfsf-CZ49K7a_5f8H8MDGgW8Z4Vcbw0u9gJ7xYFLMpOnEaVPadndTL4T4A" https://10.211.55.25:6443/metrics #此处可以获取到数据
#kubesphere 安装问题¶
[root@k8s-master kubesphere]# kubectl get sc
NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE
nfs (default) luffy.com/nfs Delete Immediate false 18d
[root@k8s-master kubesphere]# ll
总用量 20
-rw-r--r--. 1 root root 10072 11月 8 10:37 cluster-configuration.yaml
-rw-r--r--. 1 root root 4553 11月 8 10:37 kubesphere-installer.yaml
[root@k8s-master kubesphere]# kubectl apply -f kubesphere-installer.yaml
customresourcedefinition.apiextensions.k8s.io/clusterconfigurations.installer.kubesphere.io created
namespace/kubesphere-system created
serviceaccount/ks-installer created
clusterrole.rbac.authorization.k8s.io/ks-installer created
clusterrolebinding.rbac.authorization.k8s.io/ks-installer created
deployment.apps/ks-installer created
[root@k8s-master kubesphere]# kubectl apply -f cluster-configuration.yaml
clusterconfiguration.installer.kubesphere.io/ks-installer created
[root@k8s-master kubesphere]# kubectl -n kubesphere-system get po
NAME READY STATUS RESTARTS AGE
ks-installer-895b8994d-t6sm9 0/1 Error 0 56s
[root@k8s-master kubesphere]# kubectl -n kubesphere-system describe po ks-installer-895b8994d-t6sm9
Name: ks-installer-895b8994d-t6sm9
Namespace: kubesphere-system
Priority: 0
Node: k8s-slave1/10.211.55.26
Start Time: Tue, 08 Nov 2022 10:38:19 +0800
Labels: app=ks-installer
pod-template-hash=895b8994d
Annotations: <none>
Status: Running
IP: 10.244.1.39
IPs:
IP: 10.244.1.39
Controlled By: ReplicaSet/ks-installer-895b8994d
Containers:
installer:
Container ID: docker://e1b47b2dc9eb01f62faf01b225e9c8dbb8cd061d8eeb52703f300693f8f1d225
Image: kubesphere/ks-installer:v3.3.1
Image ID: docker-pullable://kubesphere/ks-installer@sha256:9f69a411f38337e21ee3f10b23bfd968ba3d034076822809a533aa90a3ebf56c
Port: <none>
Host Port: <none>
State: Terminated
Reason: Error
Exit Code: 1
Started: Tue, 08 Nov 2022 10:38:54 +0800
Finished: Tue, 08 Nov 2022 10:38:58 +0800
Ready: False
Restart Count: 0
Limits:
cpu: 1
memory: 1Gi
Requests:
cpu: 20m
memory: 100Mi
Environment: <none>
Mounts:
/etc/localtime from host-time (ro)
/var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-zxgdw (ro)
Conditions:
Type Status
Initialized True
Ready False
ContainersReady False
PodScheduled True
Volumes:
host-time:
Type: HostPath (bare host directory volume)
Path: /etc/localtime
HostPathType:
kube-api-access-zxgdw:
Type: Projected (a volume that contains injected data from multiple sources)
TokenExpirationSeconds: 3607
ConfigMapName: kube-root-ca.crt
ConfigMapOptional: <nil>
DownwardAPI: true
QoS Class: Burstable
Node-Selectors: <none>
Tolerations: node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Normal Scheduled 74s default-scheduler Successfully assigned kubesphere-system/ks-installer-895b8994d-t6sm9 to k8s-slave1
Normal Pulled 39s kubelet Successfully pulled image "kubesphere/ks-installer:v3.3.1" in 34.232813059s
Normal Created 39s kubelet Created container installer
Normal Started 39s kubelet Started container installer
Normal Pulling 34s (x2 over 73s) kubelet Pulling image "kubesphere/ks-installer:v3.3.1"
[root@k8s-master kubesphere]# kubectl -n kubesphere-system get po
NAME READY STATUS RESTARTS AGE
ks-installer-895b8994d-t6sm9 0/1 CrashLoopBackOff 2 2m39s
[root@k8s-master kubesphere]# kubectl -n kubesphere-system logs -f ks-installer-895b8994d-t6sm9
2022-11-08T10:40:24+08:00 INFO : shell-operator latest
2022-11-08T10:40:24+08:00 INFO : Use temporary dir: /tmp/shell-operator
2022-11-08T10:40:24+08:00 INFO : Initialize hooks manager ...
2022-11-08T10:40:24+08:00 INFO : Search and load hooks ...
2022-11-08T10:40:24+08:00 INFO : Load hook config from '/hooks/kubesphere/installRunner.py'
2022-11-08T10:40:24+08:00 INFO : HTTP SERVER Listening on 0.0.0.0:9115
2022-11-08T10:40:24+08:00 INFO : Load hook config from '/hooks/kubesphere/schedule.sh'
2022-11-08T10:40:24+08:00 INFO : Initializing schedule manager ...
2022-11-08T10:40:24+08:00 INFO : KUBE Init Kubernetes client
2022-11-08T10:40:24+08:00 INFO : KUBE-INIT Kubernetes client is configured successfully
2022-11-08T10:40:24+08:00 INFO : MAIN: run main loop
2022-11-08T10:40:24+08:00 INFO : MAIN: add onStartup tasks
2022-11-08T10:40:24+08:00 INFO : Running schedule manager ...
2022-11-08T10:40:24+08:00 INFO : MSTOR Create new metric shell_operator_live_ticks
2022-11-08T10:40:24+08:00 INFO : QUEUE add all HookRun@OnStartup
2022-11-08T10:40:24+08:00 INFO : MSTOR Create new metric shell_operator_tasks_queue_length
2022-11-08T10:40:24+08:00 ERROR : error getting GVR for kind 'ClusterConfiguration': unable to retrieve the complete list of server APIs: custom.metrics.k8s.io/v1beta1: the server is currently unable to handle the request, custom.metrics.k8s.io/v1beta2: the server is currently unable to handle the request, external.metrics.k8s.io/v1beta1: the server is currently unable to handle the request
2022-11-08T10:40:24+08:00 ERROR : Enable kube events for hooks error: unable to retrieve the complete list of server APIs: custom.metrics.k8s.io/v1beta1: the server is currently unable to handle the request, custom.metrics.k8s.io/v1beta2: the server is currently unable to handle the request, external.metrics.k8s.io/v1beta1: the server is currently unable to handle the request
2022-11-08T10:40:27+08:00 INFO : TASK_RUN Exit: program halts.
解决过程：
[root@k8s-master kubesphere]# kubectl get apiservice
NAME SERVICE AVAILABLE AGE
v1. Local True 28d
v1.admissionregistration.k8s.io Local True 28d
v1.apiextensions.k8s.io Local True 28d
v1.apps Local True 28d
v1.authentication.k8s.io Local True 28d
v1.authorization.k8s.io Local True 28d
v1.autoscaling Local True 28d
v1.batch Local True 28d
v1.certificates.k8s.io Local True 28d
v1.coordination.k8s.io Local True 28d
v1.discovery.k8s.io Local True 28d
v1.events.k8s.io Local True 28d
v1.networking.k8s.io Local True 28d
v1.node.k8s.io Local True 28d
v1.policy Local True 28d
v1.rbac.authorization.k8s.io Local True 28d
v1.scheduling.k8s.io Local True 28d
v1.storage.k8s.io Local True 28d
v1alpha1.extensions.istio.io Local True 17m
v1alpha1.install.istio.io Local True 17m
v1alpha1.installer.kubesphere.io Local True 8m1s
v1alpha1.telemetry.istio.io Local True 17m
v1alpha3.networking.istio.io Local True 24h
v1beta1.admissionregistration.k8s.io Local True 28d
v1beta1.apiextensions.k8s.io Local True 28d
v1beta1.authentication.k8s.io Local True 28d
v1beta1.authorization.k8s.io Local True 28d
v1beta1.batch Local True 28d
v1beta1.certificates.k8s.io Local True 28d
v1beta1.coordination.k8s.io Local True 28d
v1beta1.custom.metrics.k8s.io monitor/custom-metrics-apiserver False (ServiceNotFound) 11d
v1beta1.discovery.k8s.io Local True 28d
v1beta1.events.k8s.io Local True 28d
v1beta1.extensions Local True 28d
v1beta1.external.metrics.k8s.io monitor/custom-metrics-apiserver False (ServiceNotFound) 11d
v1beta1.flowcontrol.apiserver.k8s.io Local True 28d
v1beta1.metrics.k8s.io kube-system/metrics-server True 20d
v1beta1.networking.istio.io Local True 17m
v1beta1.networking.k8s.io Local True 28d
v1beta1.node.k8s.io Local True 28d
v1beta1.policy Local True 28d
v1beta1.rbac.authorization.k8s.io Local True 28d
v1beta1.scheduling.k8s.io Local True 28d
v1beta1.security.istio.io Local True 17m
v1beta1.storage.k8s.io Local True 28d
v1beta2.custom.metrics.k8s.io monitor/custom-metrics-apiserver False (ServiceNotFound) 11d
v2beta1.autoscaling Local True 28d
v2beta2.autoscaling Local True 28d
[root@k8s-master kubesphere]# kubectl delete apiservice v1beta1.custom.metrics.k8s.io
apiservice.apiregistration.k8s.io "v1beta1.custom.metrics.k8s.io" deleted
[root@k8s-master kubesphere]# kubectl delete apiservice v1beta1.external.metrics.k8s.io
apiservice.apiregistration.k8s.io "v1beta1.external.metrics.k8s.io" deleted
[root@k8s-master kubesphere]# kubectl delete apiservice v1beta2.custom.metrics.k8s.io
apiservice.apiregistration.k8s.io "v1beta2.custom.metrics.k8s.io" deleted
[root@k8s-master kubesphere]# kubectl -n kubesphere-system delete po ks-installer-895b8994d-t6sm9
pod "ks-installer-895b8994d-t6sm9" deleted
[root@k8s-master kubesphere]# kubectl -n kubesphere-system get po
NAME READY STATUS RESTARTS AGE
ks-installer-895b8994d-nln2p 1/1 Running 0 106s