k8s中部署基于nfs的StorageClass

部署nfs服务

1.1 创建基础镜像(选做)

如果以docker的形式部署nfs server, 参考此步骤, 若否, 该步骤可忽略。

mkdir /data/nfs -p
chmod 755 /data/nfs

# NFS默认端口: 111、2049、20048
docker run -d \
--privileged \
--name nfs_server \
-p 111:111/tcp \
-p 111:111/udp \
-p 2049:2049/tcp \
-p 2049:2049/udp \
-p 30001-30005:30001-30005/tcp \
-p 30001-30005:30001-30005/udp \
-v /data/nfs:/data/nfs \
centos:centos7.9.2009 /usr/sbin/init 

# docker exec -it nfs_server bash

# =================== 生成镜像并推送到私有仓库 =================== #
docker commit -p -a "langpf" -m "centos7.9.2009, nfs_server" 25c6d01e41b0 rpp-harbor.com/my-project/nfs_server:centos7.9.2009-u

$. docker save -o nfs_server-centos7.9.2009-u.tar rpp-harbor.com/my-project/nfs_server:centos7.9.2009-u
$. docker load --input nfs_server-centos7.9.2009-u.tar

$. docker push rpp-harbor.com/my-project/nfs_server:centos7.9.2009-u

1.2 部署NFS Server

# step 1(docker部署时执行)
docker exec -it nfs_server bash
chmod 755 /data/nfs

# step 2
yum -y install nfs-utils rpcbind
yum install net-tools -y

# step 3
$. vi /etc/exports
# 写入
/data/nfs 192.168.1.0/24(rw,sync,all_squash,anonuid=0,anongid=0)
/data/nfs 172.17.0.0/24(rw,sync,root_squash)
# -----
/data/nfs 172.17.0.0/24(rw,sync,root_squash)
/data/nfs 192.168.31.0/24(insecure,fsid=0,rw,sync,no_root_squash,acl)
# 编译生效
$. exportfs -arv

# ===================== 修改nfs端口号 ===================== #
# 1. 修改 /etc/sysconfig/nfs 配置
$. vi /etc/sysconfig/nfs
# 在文末加入
RQUOTAD_PORT=30001
LOCKD_TCPPORT=30002
LOCKD_UDPPORT=30002
MOUNTD_PORT=30003
STATD_PORT=30004
STATD_OUTGOING_PORT=30005

# 2. 修改 /etc/modprobe.d/lockd.conf 配置
$. vi /etc/modprobe.d/lockd.conf
# 在/etc/modprobe.d/lockd.conf中添加以下设置：
options lockd nlm_tcpport=30002
options lockd nlm_udpport=30002

# 3. 重新加载NFS配置和服务：
systemctl restart rpcbind
systemctl restart nfs
#systemctl restart nfs-config
#systemctl restart nfs-idmap
#systemctl restart nfs-lock

# strep systemctl start rpcbind
systemctl start nfs

# step
systemctl status rpcbind
systemctl status nfs
systemctl enable rpcbind
systemctl enable nfs

# 4.  
exportfs -arv

# 校验配置等
showmount -e
showmount -e 192.168.31.133
# -----
showmount -e 192.168.1.161
rpcinfo -p 
rpcinfo -p 192.168.1.161

部署nfs客户端

需要在k8s所有节点部署nfs客户端，在k8s所有节点部署执行如下命令, 部署nfs客户端：

# ========================= 客户端 ========================= #
# 普通客户端需要先安装nfs, 执行如下命令
# k8s集群要挂载nfs, 需要在所有节点执行如下命令
yum -y install nfs-utils 

# =================== 安装后nfs在客户端执行以下命令验证 ================ #
# ----------
showmount -e 172.17.0.2
showmount -e 192.168.31.133
showmount -e 192.168.1.161

# -----------
mount -t nfs 172.17.0.2:/data/nfs /data -o proto=tcp -o nolock
mount -t nfs 192.168.31.133:/data/nfs /data2 -o proto=tcp -o nolock
umount /data

# -----------
mount -t nfs 192.168.1.161:/data/nfs /data2 -o proto=tcp -o nolock
umount /data2

# ===================== 其它命令 ====================== #
# 在服务端执行，查看nfs状态信息
nfsstat -s

# 在客户端执行，查看nfs挂载信息
nfsstat -m

# 查看服务端的远程共享信息，此命令会调用服务端的20048端口
showmount -e 192.168.202.151

# 查看rpcbind注册的所有端口号
rpcinfo -p
rpcinfo -p 192.168.1.161

# 在服务端执行，重新挂载/etc/exports中配置
exportfs -r

创建sc,pv,pvc,pod资源

k8s命令行执行如下命令, 创建资源:

kubectl apply -f nfs-sc-pv-pvc-pvc-pod.yaml

nfs-sc-pv-pvc-pvc-pod.yaml文件：

# kubectl config set-context $(kubectl config current-context) --namespace=rpp-nfs-ns
# kubectl config get-contexts
# kubectl get ns

# kubectl get pod -n rpp-nfs-ns
# kubectl describe pod rpp-nfs-pod
# kubectl exec -it rpp-nfs-pod -- sh
# kubectl logs rpp-nfs-pod  
# kubectl logs -f rpp-nfs-pod  

# kubectl get pv
# kubectl describe pv rpp-nfs-pv

# kubectl get pvc
# kubectl describe pvc rpp-nfs-pvc

# kubectl get sc
# kubectl describe sc rpp-nfs-sc    
# ns
apiVersion: v1
kind: Namespace
metadata:
  name: rpp-nfs-ns
---      
# 卷插件为NFS的存储类,
apiVersion: storage.k8s.io/v1  
kind: StorageClass  
metadata:  
  name: rpp-nfs-sc
  # namespace: rpp-nfs-ns # 无ns属性
provisioner: kubernetes.io/nfs  
#parameters:  
#  path: /data/nfs
#  server: 192.168.31.133
---
# 创建NFS类型的PV, 并绑定SC
apiVersion: v1  
kind: PersistentVolume  
metadata:  
  name: rpp-nfs-pv
  # namespace: rpp-nfs-ns # 无ns属性
spec:  
  capacity:  
    storage: 5Gi  
  accessModes:  
    - ReadWriteMany  
  persistentVolumeReclaimPolicy: Retain  
  storageClassName: rpp-nfs-sc
  volumeMode: Filesystem  
  nfs:  
    server: 192.168.1.161
    path: /data/nfs  
    readOnly: false
---
# 创建PVC, 指定StorageClass为rpp-nfs-sc, 并请求1GB的存储空间
apiVersion: v1
kind: PersistentVolumeClaim
metadata:  
  name: rpp-nfs-pvc
  # namespace: rpp-nfs-ns # 无ns属性
spec:
  storageClassName: rpp-nfs-sc
  accessModes:
  - ReadWriteMany
  resources:
    requests:
      storage: 1Gi
---
# POD挂载PVC
apiVersion: v1
kind: Pod
metadata:
  name: rpp-nfs-pod
  namespace: rpp-nfs-ns
spec:
  containers:
  - name: springboot-docker2  # 容器名
    image: rpp-harbor.com/my-project/spring-boot-docker2:0.0.1-SNAPSHOT
    volumeMounts:
    - name: rpp-volume
      mountPath: /data2
    ports:                 # 端口
    - containerPort: 8600  # 容器暴露的端口
      name: business-port
    - containerPort: 8800
      name: actuator-port
  volumes:  
  - name: rpp-volume  
    persistentVolumeClaim:  
      claimName: rpp-nfs-pvc