Hike News
Hike News

创建NFS的StorageClass

容器 本文总阅读量次

前言

在Kubernetes的几种网络存储中，NFS是成本较低、使用简单的一种方案。

但是NFS存储不建议用在生产环境，因为我们测试环境的MySQL数据库部署在NFS上都经常出问题，比如nfs4_reclaim_open_state: Lock reclaim failed和kernel:NMI watchdog: BUG: soft lockup - CPU#0 stuck for 26s。

NFS服务端

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
yum install -y net-tools lsof nfs-utils rpcbind
mkdir /data/nfs -p
vi /etc/exports
#挂载NFS服务器上的/data/nfs/目录到自己的文件系统中，rw表示可读写，no_root_squash 是让root保持权限
/data/nfs/ *(insecure,rw,no_root_squash)

关闭防火墙
systemctl stop firewalld
先为rpcbind和nfs做开机启动：(必须先启动rpcbind服务)
systemctl enable rpcbind.service
systemctl enable nfs-server.service
然后分别启动rpcbind和nfs服务：
systemctl start rpcbind.service
systemctl start nfs-server.service
 
exportfs -r
#可以查看到已经ok
exportfs
/home/nfs 192.168.248.0/24

NFS客户端

1
2
3
4
5
6
7
8
9
10
11
12
#安装nfs工具
yum  install -y nfs-utils
#建立挂载目录
mkdir /data
#挂载nfs
mount -t nfs 192.168.80.145:/data/nfs /data
卸载挂载
umount /data
 
查看是目录挂载状态
df -h
showmount  -e 192.168.80.145

创建NFS-StorageClass

nfs-rbac.yaml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
kind: ServiceAccount
apiVersion: v1
metadata:
  name: nfs-client-provisioner
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: nfs-client-provisioner-runner
rules:
  - apiGroups: [""]
    resources: ["persistentvolumes"]
    verbs: ["get", "list", "watch", "create", "delete"]
  - apiGroups: [""]
    resources: ["persistentvolumeclaims"]
    verbs: ["get", "list", "watch", "update"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["storageclasses"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["events"]
    verbs: ["create", "update", "patch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: run-nfs-client-provisioner
subjects:
  - kind: ServiceAccount
    name: nfs-client-provisioner
    namespace: default
roleRef:
  kind: ClusterRole
  name: nfs-client-provisioner-runner
  apiGroup: rbac.authorization.k8s.io
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: leader-locking-nfs-client-provisioner
rules:
  - apiGroups: [""]
    resources: ["endpoints"]
    verbs: ["get", "list", "watch", "create", "update", "patch"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: leader-locking-nfs-client-provisioner
subjects:
  - kind: ServiceAccount
    name: nfs-client-provisioner
    namespace: default
roleRef:
  kind: Role
  name: leader-locking-nfs-client-provisioner
  apiGroup: rbac.authorization.k8s.io

nfs-deployment.yaml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
kind: Deployment
apiVersion: extensions/v1beta1
metadata:
  name: nfs-client-provisioner
spec:
  replicas: 1
  strategy:
    type: Recreate
  template:
    metadata:
      labels:
        app: nfs-client-provisioner
    spec:
      tolerations:
        - key: node-env
          value: pre
          effect: NoSchedule
          operator: Equal
      priorityClassName: cluster
      serviceAccountName: nfs-client-provisioner
      containers:
        - name: nfs-client-provisioner
          image: quay.io/external_storage/nfs-client-provisioner:latest
          volumeMounts:
            - name: nfs-client-root
              mountPath: /persistentvolumes
          env:
            - name: PROVISIONER_NAME
              value: nfs-provisioner
            - name: NFS_SERVER
              value: ##NFS_SERVER_IP##
            - name: NFS_PATH
              value: ##NFS_PATH##
      volumes:
        - name: nfs-client-root
          nfs:
            server: ##NFS_SERVER_IP##
            path: ##NFS_PATH##

nfs-storage.yaml

1
2
3
4
5
6
7
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: nfs-storage
provisioner: nfs-provisioner # or choose another name, must match deployment's env PROVISIONER_NAME'
parameters:
  archiveOnDelete: "false"

test-nfs-storage.yaml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: test-claim
  annotations:
    volume.beta.kubernetes.io/storage-class: "nfs-storage"
spec:
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 1Mi

---

kind: Pod
apiVersion: v1
metadata:
  name: test-pod
spec:
  containers:
    - name: test-pod
      image: busybox
      command:
        - "/bin/sh"
      args:
        - "-c"
        - "touch /mnt/SUCCESS && exit 0 || exit 1"
      volumeMounts:
        - name: nfs-pvc
          mountPath: "/mnt"
  restartPolicy: "Never"
  volumes:
    - name: nfs-pvc
      persistentVolumeClaim:
        claimName: test-claim

创建顺序

  1. nfs-rbac.yaml
  2. nfs-deployment.yaml
  3. nfs-storage.yaml

参数说明

nfs-deployment.yaml

  • ##NFS_SERVER_IP##是NFS服务端的IP，根据实际IP进行替换。
  • ##NFS_PATH##是NFS服务端的目录，根据实际目录进行替换。

部署

1
2
3
4
5
NFS_SERVER_IP=192.168.80.145  #换成自己的实际IP
NFS_PATH=/data/nfs  # 换成自己的实际目录
kubectl apply -f nfs-rbac.yaml
sed "s|##NFS_SERVER_IP##|${NFS_SERVER_IP}|g;s|##NFS_PATH##|${NFS_PATH}|g" nfs-deployment.yaml | kubectl apply -f -
kubectl apply -f nfs-storage.yaml

验证方法

1
kubectl apply -f test-nfs-storage.yaml

可选：设置默认存储

设置这个StorageClass为Kubernetes的默认存储

1
2
3
4
 kubectl patch storageclass nfs-storage -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'
 [root@master1 nfs-storage]# kubectl get sc
NAME                    PROVISIONER                    AGE
nfs-storage (default)   nfs-provisioner                12m

参考

https://www.cnblogs.com/lixiuran/p/7117000.html

https://github.com/kubernetes-incubator/external-storage/tree/master/nfs-client

容器