diff --git a/.gitignore b/.gitignore index 69b6f20..a46496c 100644 --- a/.gitignore +++ b/.gitignore @@ -24,7 +24,7 @@ _deps # Output of the go coverage tool, specifically when used with LiteIDE *.out - +.history # Dependency directories (remove the comment below to include it) # vendor/ diff --git a/README.md b/README.md index a9bd7fd..0cfb6d2 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,47 @@ -# +# NFS + + +利用 k8s 部署 NFS 存储服务 + +## 没有 `NFS` 服务 + +使用 `server-and-client` 目录中的文件部署nfs 服务 + +需要注意根据实际数据持久化的节点,修改 `statefulset.yaml` 下面位置 + +```yaml + nodeSelector: + kubernetes.io/hostname: test-n3 + volumes: + - name: export-volume + hostPath: + path: /srv +``` + +`read-pod.yaml` 和 `write-pod.yaml` 是用来测试服务是否搭建成功的。 + +## 已经存在 `NFS` 服务 + +已经存在外部 `NFS` 如果 `k8s` 需要 持久化使用已存在的 `NFS`,使用 `client-only` 中的文件部署,`nfs provisioner` + +需要注意根据实际 NFS 的服务和 export 的目录,修改 `deployment.yaml` 下面位置 + +```yaml + volumeMounts: + - name: nfs-client-root + mountPath: /persistentvolumes + env: + - name: PROVISIONER_NAME + value: k8s-sigs.io/nfs-provisioner + - name: NFS_SERVER + value: 10.8.30.152 # nfs 服务 + - name: NFS_PATH + value: /data # export 的目录 + volumes: + - name: nfs-client-root + nfs: + server: 10.8.30.152 # nfs 服务 + path: /data # export 的目录 +``` + diff --git a/client-only/class.yaml b/client-only/class.yaml new file mode 100644 index 0000000..cb0a351 --- /dev/null +++ b/client-only/class.yaml @@ -0,0 +1,7 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: k8s-sigs-nfs-client +provisioner: k8s-sigs.io/nfs-provisioner # or choose another name, must match deployment's env PROVISIONER_NAME' +parameters: + archiveOnDelete: "false" diff --git a/client-only/deployment.yaml b/client-only/deployment.yaml new file mode 100644 index 0000000..e1b5b69 --- /dev/null +++ b/client-only/deployment.yaml @@ -0,0 +1,39 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nfs-client-provisioner + labels: + app: nfs-client-provisioner + # replace with namespace where provisioner is deployed + namespace: default +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: nfs-client-provisioner + template: + metadata: + labels: + app: nfs-client-provisioner + spec: + serviceAccountName: nfs-client-provisioner + containers: + - name: nfs-client-provisioner + image: registry.cn-hangzhou.aliyuncs.com/gcr_k8s_containers/nfs-subdir-external-provisioner:v4.0.2 + volumeMounts: + - name: nfs-client-root + mountPath: /persistentvolumes + env: + - name: PROVISIONER_NAME + value: k8s-sigs.io/nfs-provisioner + - name: NFS_SERVER + value: 10.8.30.152 + - name: NFS_PATH + value: /data + volumes: + - name: nfs-client-root + nfs: + server: 10.8.30.152 + path: /data diff --git a/client-only/kustomization.yaml b/client-only/kustomization.yaml new file mode 100644 index 0000000..445c6c9 --- /dev/null +++ b/client-only/kustomization.yaml @@ -0,0 +1,4 @@ +resources: + - class.yaml + - rbac.yaml + - deployment.yaml diff --git a/client-only/rbac.yaml b/client-only/rbac.yaml new file mode 100644 index 0000000..28dbb68 --- /dev/null +++ b/client-only/rbac.yaml @@ -0,0 +1,68 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: nfs-client-provisioner + # replace with namespace where provisioner is deployed + namespace: default +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: nfs-client-provisioner-runner +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "update", "patch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: run-nfs-client-provisioner +subjects: + - kind: ServiceAccount + name: nfs-client-provisioner + # replace with namespace where provisioner is deployed + namespace: default +roleRef: + kind: ClusterRole + name: nfs-client-provisioner-runner + apiGroup: rbac.authorization.k8s.io +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: leader-locking-nfs-client-provisioner + # replace with namespace where provisioner is deployed + namespace: default +rules: + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "list", "watch", "create", "update", "patch"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: leader-locking-nfs-client-provisioner + # replace with namespace where provisioner is deployed + namespace: default +subjects: + - kind: ServiceAccount + name: nfs-client-provisioner + # replace with namespace where provisioner is deployed + namespace: default +roleRef: + kind: Role + name: leader-locking-nfs-client-provisioner + apiGroup: rbac.authorization.k8s.io diff --git a/client-only/test-claim.yaml b/client-only/test-claim.yaml new file mode 100644 index 0000000..e19d45f --- /dev/null +++ b/client-only/test-claim.yaml @@ -0,0 +1,11 @@ +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: test-claim +spec: + storageClassName: k8s-sigs-nfs-client + accessModes: + - ReadWriteMany + resources: + requests: + storage: 1Mi diff --git a/client-only/test-pod.yaml b/client-only/test-pod.yaml new file mode 100644 index 0000000..fd15060 --- /dev/null +++ b/client-only/test-pod.yaml @@ -0,0 +1,21 @@ +kind: Pod +apiVersion: v1 +metadata: + name: test-pod +spec: + containers: + - name: test-pod + image: busybox:stable + command: + - "/bin/sh" + args: + - "-c" + - "touch /mnt/SUCCESS && exit 0 || exit 1" + volumeMounts: + - name: nfs-pvc + mountPath: "/mnt" + restartPolicy: "Never" + volumes: + - name: nfs-pvc + persistentVolumeClaim: + claimName: test-claim diff --git a/server-and-client/claim.yaml b/server-and-client/claim.yaml new file mode 100644 index 0000000..5ce777c --- /dev/null +++ b/server-and-client/claim.yaml @@ -0,0 +1,11 @@ +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: nfs +spec: + storageClassName: gcr-nfs + accessModes: + - ReadWriteMany + resources: + requests: + storage: 1Mi diff --git a/server-and-client/class.yaml b/server-and-client/class.yaml new file mode 100644 index 0000000..ef39635 --- /dev/null +++ b/server-and-client/class.yaml @@ -0,0 +1,7 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: gcr-nfs +provisioner: gcr.io/nfs +mountOptions: + - vers=4.1 diff --git a/server-and-client/psp.yaml b/server-and-client/psp.yaml new file mode 100644 index 0000000..03fd080 --- /dev/null +++ b/server-and-client/psp.yaml @@ -0,0 +1,23 @@ +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: nfs-provisioner +spec: + fsGroup: + rule: RunAsAny + allowedCapabilities: + - DAC_READ_SEARCH + - SYS_RESOURCE + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - configMap + - downwardAPI + - emptyDir + - persistentVolumeClaim + - secret + - hostPath diff --git a/server-and-client/rbac.yaml b/server-and-client/rbac.yaml new file mode 100644 index 0000000..0d96ad1 --- /dev/null +++ b/server-and-client/rbac.yaml @@ -0,0 +1,61 @@ +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: nfs-provisioner-runner +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "update", "patch"] + - apiGroups: [""] + resources: ["services", "endpoints"] + verbs: ["get"] + - apiGroups: ["extensions"] + resources: ["podsecuritypolicies"] + resourceNames: ["nfs-provisioner"] + verbs: ["use"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: run-nfs-provisioner +subjects: + - kind: ServiceAccount + name: nfs-provisioner + # replace with namespace where provisioner is deployed + namespace: default +roleRef: + kind: ClusterRole + name: nfs-provisioner-runner + apiGroup: rbac.authorization.k8s.io +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: leader-locking-nfs-provisioner +rules: + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "list", "watch", "create", "update", "patch"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: leader-locking-nfs-provisioner +subjects: + - kind: ServiceAccount + name: nfs-provisioner + # replace with namespace where provisioner is deployed + namespace: default +roleRef: + kind: Role + name: leader-locking-nfs-provisioner + apiGroup: rbac.authorization.k8s.io diff --git a/server-and-client/read-pod.yaml b/server-and-client/read-pod.yaml new file mode 100644 index 0000000..17ff766 --- /dev/null +++ b/server-and-client/read-pod.yaml @@ -0,0 +1,22 @@ +kind: Pod +apiVersion: v1 +metadata: + name: read-pod +spec: + containers: + - name: read-pod + image: busybox:1.24 + command: + - "/bin/sh" + args: + - "-c" + - "test -f /mnt/SUCCESS && exit 0 || exit 1" + volumeMounts: + - name: nfs-pvc + mountPath: "/mnt" + restartPolicy: "Never" + volumes: + - name: nfs-pvc + persistentVolumeClaim: + claimName: nfs + diff --git a/server-and-client/statefulset.yaml b/server-and-client/statefulset.yaml new file mode 100644 index 0000000..e232659 --- /dev/null +++ b/server-and-client/statefulset.yaml @@ -0,0 +1,125 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: nfs-provisioner +--- +kind: Service +apiVersion: v1 +metadata: + name: nfs-provisioner + labels: + app: nfs-provisioner +spec: + ports: + - name: nfs + port: 2049 + - name: nfs-udp + port: 2049 + protocol: UDP + - name: nlockmgr + port: 32803 + - name: nlockmgr-udp + port: 32803 + protocol: UDP + - name: mountd + port: 20048 + - name: mountd-udp + port: 20048 + protocol: UDP + - name: rquotad + port: 875 + - name: rquotad-udp + port: 875 + protocol: UDP + - name: rpcbind + port: 111 + - name: rpcbind-udp + port: 111 + protocol: UDP + - name: statd + port: 662 + - name: statd-udp + port: 662 + protocol: UDP + selector: + app: nfs-provisioner +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: nfs-provisioner +spec: + selector: + matchLabels: + app: nfs-provisioner + serviceName: "nfs-provisioner" + replicas: 1 + template: + metadata: + labels: + app: nfs-provisioner + spec: + serviceAccount: nfs-provisioner + terminationGracePeriodSeconds: 10 + containers: + - name: nfs-provisioner + image: registry.cn-hangzhou.aliyuncs.com/gcr_k8s_containers/nfs-provisioner:v3.0.0 + ports: + - name: nfs + containerPort: 2049 + - name: nfs-udp + containerPort: 2049 + protocol: UDP + - name: nlockmgr + containerPort: 32803 + - name: nlockmgr-udp + containerPort: 32803 + protocol: UDP + - name: mountd + containerPort: 20048 + - name: mountd-udp + containerPort: 20048 + protocol: UDP + - name: rquotad + containerPort: 875 + - name: rquotad-udp + containerPort: 875 + protocol: UDP + - name: rpcbind + containerPort: 111 + - name: rpcbind-udp + containerPort: 111 + protocol: UDP + - name: statd + containerPort: 662 + - name: statd-udp + containerPort: 662 + protocol: UDP + securityContext: + capabilities: + add: + - DAC_READ_SEARCH + - SYS_RESOURCE + args: + - "-provisioner=gcr.io/nfs" + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: SERVICE_NAME + value: nfs-provisioner + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: export-volume + mountPath: /export + nodeSelector: + kubernetes.io/hostname: test-n3 + volumes: + - name: export-volume + hostPath: + path: /srv diff --git a/server-and-client/write-pod.yaml b/server-and-client/write-pod.yaml new file mode 100644 index 0000000..cc532f2 --- /dev/null +++ b/server-and-client/write-pod.yaml @@ -0,0 +1,22 @@ +kind: Pod +apiVersion: v1 +metadata: + name: write-pod +spec: + containers: + - name: write-pod + image: busybox:1.24 + command: + - "/bin/sh" + args: + - "-c" + - "touch /mnt/SUCCESS && exit 0 || exit 1" + volumeMounts: + - name: nfs-pvc + mountPath: "/mnt" + restartPolicy: "Never" + volumes: + - name: nfs-pvc + persistentVolumeClaim: + claimName: nfs +