BlaineEXE/nfs-deploy-prototype.yaml

## nfs-deploy-prototype.yaml
# Apply this file once the single-replica nfs-test.yaml is deployed
# You must install sssd-client into the nfs-ganesha container once this file is applied
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: rook-ceph-nfs
    app.kubernetes.io/component: cephnfses.ceph.rook.io
    app.kubernetes.io/created-by: rook-ceph-operator
    app.kubernetes.io/instance: my-nfs-a
    app.kubernetes.io/managed-by: rook-ceph-operator
    app.kubernetes.io/name: ceph-nfs
    app.kubernetes.io/part-of: my-nfs
    ceph-version: 17.2.1-0
    ceph_daemon_id: my-nfs-a
    ceph_daemon_type: nfs
    ceph_nfs: my-nfs
    instance: a
    nfs: my-nfs-a
    rook-version: v1.9.0-alpha.0.412.g86b14d446
    rook.io/operator-namespace: rook-ceph
    rook_cluster: rook-ceph
  name: rook-ceph-nfs-my-nfs-a
  namespace: rook-ceph
spec:
  progressDeadlineSeconds: 600
  replicas: 1
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      app: rook-ceph-nfs
      ceph_daemon_id: my-nfs-a
      ceph_nfs: my-nfs
      instance: a
      nfs: my-nfs-a
      rook_cluster: rook-ceph
  strategy:
    rollingUpdate:
      maxSurge: 25%
      maxUnavailable: 25%
    type: RollingUpdate
  template:
    metadata:
      annotations:
        config-hash: 297d986b464262ce71982e9a186f9692
      labels:
        app: rook-ceph-nfs
        app.kubernetes.io/component: cephnfses.ceph.rook.io
        app.kubernetes.io/created-by: rook-ceph-operator
        app.kubernetes.io/instance: my-nfs-a
        app.kubernetes.io/managed-by: rook-ceph-operator
        app.kubernetes.io/name: ceph-nfs
        app.kubernetes.io/part-of: my-nfs
        ceph_daemon_id: my-nfs-a
        ceph_daemon_type: nfs
        ceph_nfs: my-nfs
        instance: a
        nfs: my-nfs-a
        rook.io/operator-namespace: rook-ceph
        rook_cluster: rook-ceph
      name: rook-ceph-nfs-my-nfs-a
    spec:
      affinity: {}
      containers:
        - args:
            - -F
            - -L
            - STDERR
            - -p
            - /var/run/ganesha/ganesha.pid
            - -N
            - NIV_DEBUG
          command:
            - ganesha.nfsd
          env:
            - name: CONTAINER_IMAGE
              value: quay.io/ceph/ceph:v17
            - name: POD_NAME
              valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: metadata.name
            - name: POD_NAMESPACE
              valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: metadata.namespace
            - name: NODE_NAME
              valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: spec.nodeName
            - name: POD_MEMORY_LIMIT
              valueFrom:
                resourceFieldRef:
                  divisor: "0"
                  resource: limits.memory
            - name: POD_MEMORY_REQUEST
              valueFrom:
                resourceFieldRef:
                  divisor: "0"
                  resource: requests.memory
            - name: POD_CPU_LIMIT
              valueFrom:
                resourceFieldRef:
                  divisor: "1"
                  resource: limits.cpu
            - name: POD_CPU_REQUEST
              valueFrom:
                resourceFieldRef:
                  divisor: "0"
                  resource: requests.cpu
            - name: ROOK_CEPH_MON_HOST
              valueFrom:
                secretKeyRef:
                  key: mon_host
                  name: rook-ceph-config
            - name: ROOK_CEPH_MON_INITIAL_MEMBERS
              valueFrom:
                secretKeyRef:
                  key: mon_initial_members
                  name: rook-ceph-config
          image: quay.io/ceph/ceph:v17
          imagePullPolicy: IfNotPresent
          name: nfs-ganesha
          # BIG NOTE: ganesha container needs package 'sssd-client' package installed
          resources: {}
          securityContext:
            privileged: false
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
          volumeMounts:
            - mountPath: /etc/ceph
              name: etc-ceph
            - mountPath: /etc/ceph/keyring-store/
              name: rook-ceph-nfs-my-nfs-a-keyring
              readOnly: true
            - mountPath: /etc/ganesha
              name: ganesha-config
            - mountPath: /run/dbus
              name: run-dbus
            - mountPath: /var/lib/sss/pipes
              name: sssd-sockets
            - mountPath: /etc/nsswitch.conf
              name: nsswitch-conf
              subPath: nsswitch.conf
        - args:
            - --nofork
            - --system
            - --nopidfile
            - --nosyslog
          command:
            - dbus-daemon
          env:
            - name: CONTAINER_IMAGE
              value: quay.io/ceph/ceph:v17
            - name: POD_NAME
              valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: metadata.name
            - name: POD_NAMESPACE
              valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: metadata.namespace
            - name: NODE_NAME
              valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: spec.nodeName
            - name: POD_MEMORY_LIMIT
              valueFrom:
                resourceFieldRef:
                  divisor: "0"
                  resource: limits.memory
            - name: POD_MEMORY_REQUEST
              valueFrom:
                resourceFieldRef:
                  divisor: "0"
                  resource: requests.memory
            - name: POD_CPU_LIMIT
              valueFrom:
                resourceFieldRef:
                  divisor: "1"
                  resource: limits.cpu
            - name: POD_CPU_REQUEST
              valueFrom:
                resourceFieldRef:
                  divisor: "0"
                  resource: requests.cpu
          image: quay.io/ceph/ceph:v17
          imagePullPolicy: IfNotPresent
          name: dbus-daemon
          resources: {}
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
          volumeMounts:
            - mountPath: /run/dbus
              name: run-dbus
        # vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv SSSD vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv #
        - name: sssd-daemon
          image: quay.io/brgardne/sssd:latest
          imagePullPolicy: Always
          resources: {}
          volumeMounts:
            # sssd uses dbus only for internal communications, so no shared dbus connection needed
            # mounting dbus causes failures for unknown reasons, and is only necessary to
            # commmunicate w/ sssd's dbus api (not used by nfs-ganesha)
            # - mountPath: /run/dbus
            #   name: run-dbus
            - mountPath: /etc/sssd/conf.d
              name: sssd-config
            - mountPath: /var/lib/sss/pipes
              name: sssd-sockets
            # - mountPath: /etc/nsswitch.conf
            #   name: nsswitch-conf
            #   subPath: nsswitch.conf
        # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ SSSD ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #
      dnsPolicy: ClusterFirst
      initContainers:
        - command:
            - /bin/bash
            - -c
            - |
              set -xEeuo pipefail

              cat << EOF > /etc/ceph/ceph.conf
              [global]
              mon_host = $(ROOK_CEPH_MON_HOST)

              [client.nfs-ganesha.my-nfs.a]
              keyring = /etc/ceph/keyring-store/keyring
              EOF

              chmod 444 /etc/ceph/ceph.conf

              cat /etc/ceph/ceph.conf
          env:
            - name: ROOK_CEPH_MON_HOST
              valueFrom:
                secretKeyRef:
                  key: mon_host
                  name: rook-ceph-config
            - name: ROOK_CEPH_MON_INITIAL_MEMBERS
              valueFrom:
                secretKeyRef:
                  key: mon_initial_members
                  name: rook-ceph-config
          image: quay.io/ceph/ceph:v17
          imagePullPolicy: IfNotPresent
          name: generate-minimal-ceph-conf
          resources: {}
          securityContext:
            privileged: false
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
          volumeMounts:
            - mountPath: /etc/ceph
              name: etc-ceph
            - mountPath: /etc/ceph/keyring-store/
              name: rook-ceph-nfs-my-nfs-a-keyring
              readOnly: true
        # copy the sssd sockets (pipes) starting content to the dir where it will be shared between containers
        - name: copy-sssd-sockets
          image: quay.io/brgardne/sssd:latest
          command:
            - bash
            - -c
            - |
              set -ex
              cp -a -v /var/lib/sss/pipes/* /tmp/var/lib/sss/pipes/.
              ls -alF -R /tmp/var/lib/sss/pipes
          volumeMounts:
            - mountPath: /tmp/var/lib/sss/pipes
              name: sssd-sockets
      restartPolicy: Always
      schedulerName: default-scheduler
      securityContext: {}
      terminationGracePeriodSeconds: 30
      tolerations:
        - effect: NoExecute
          key: node.kubernetes.io/unreachable
          operator: Exists
          tolerationSeconds: 5
      volumes:
        - emptyDir: {}
          name: etc-ceph
        - name: rook-ceph-nfs-my-nfs-a-keyring
          secret:
            defaultMode: 420
            secretName: rook-ceph-nfs-my-nfs-a-keyring
        - configMap:
            defaultMode: 420
            items:
              - key: config
                path: ganesha.conf
            name: rook-ceph-nfs-my-nfs-a
          name: ganesha-config
        - emptyDir: {}
          name: run-dbus
        - name: sssd-config
          configMap:
            name: sssd-config
            defaultMode: 0600 # required
        - name: sssd-sockets
          emptyDir: {}
        - name: nsswitch-conf
          configMap:
            name: nsswitch-conf

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: sssd-config
  namespace: rook-ceph
data:
  50-ldap.conf: |
    [domain/default]
    id_provider = ldap
    #access_provider = simple
    access_provider = ldap
    ldap_access_filter = memberOf=cn=rook,ou=groups,dc=rook,dc=net
    #autofs_provider = ldap
    auth_provider = ldap
    #chpass_provider = ldap
    ldap_uri = ldap://ldap-test-server.default.svc.cluster.local/
    ldap_search_base = dc=rook,dc=net
    ldap_default_bind_dn = cn=admin,dc=rook,dc=net
    ldap_default_authtok_type = password
    ldap_default_authtok = admin
    # ldap_id_use_start_tls = true
    # cache_credentials = true
    # ldap_tls_cacertdir = /etc/openldap/certs
    ldap_tls_reqcert = allow
    ldap_user_search_base = ou=users,dc=rook,dc=net
    ldap_group_search_base = ou=groups,dc=rook,dc=net
    debug_level = 9
    # enumerate = true

    [sssd]
    services = nss
    domains = default
    config_file_version = 2
    enable_files_domain = false
    debug_level = 9

    [nss]
    local_negative_timeout = 0
    debug_level = 9
    filter_users = root
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: nsswitch-conf
  namespace: rook-ceph
data:
  nsswitch.conf: |
    passwd: sss
    group: sss
    netgroup: sss
	# Apply this file once the single-replica nfs-test.yaml is deployed
	# You must install sssd-client into the nfs-ganesha container once this file is applied
	apiVersion: apps/v1
	kind: Deployment
	metadata:
	labels:
	app: rook-ceph-nfs
	app.kubernetes.io/component: cephnfses.ceph.rook.io
	app.kubernetes.io/created-by: rook-ceph-operator
	app.kubernetes.io/instance: my-nfs-a
	app.kubernetes.io/managed-by: rook-ceph-operator
	app.kubernetes.io/name: ceph-nfs
	app.kubernetes.io/part-of: my-nfs
	ceph-version: 17.2.1-0
	ceph_daemon_id: my-nfs-a
	ceph_daemon_type: nfs
	ceph_nfs: my-nfs
	instance: a
	nfs: my-nfs-a
	rook-version: v1.9.0-alpha.0.412.g86b14d446
	rook.io/operator-namespace: rook-ceph
	rook_cluster: rook-ceph
	name: rook-ceph-nfs-my-nfs-a
	namespace: rook-ceph
	spec:
	progressDeadlineSeconds: 600
	replicas: 1
	revisionHistoryLimit: 10
	selector:
	matchLabels:
	app: rook-ceph-nfs
	ceph_daemon_id: my-nfs-a
	ceph_nfs: my-nfs
	instance: a
	nfs: my-nfs-a
	rook_cluster: rook-ceph
	strategy:
	rollingUpdate:
	maxSurge: 25%
	maxUnavailable: 25%
	type: RollingUpdate
	template:
	metadata:
	annotations:
	config-hash: 297d986b464262ce71982e9a186f9692
	labels:
	app: rook-ceph-nfs
	app.kubernetes.io/component: cephnfses.ceph.rook.io
	app.kubernetes.io/created-by: rook-ceph-operator
	app.kubernetes.io/instance: my-nfs-a
	app.kubernetes.io/managed-by: rook-ceph-operator
	app.kubernetes.io/name: ceph-nfs
	app.kubernetes.io/part-of: my-nfs
	ceph_daemon_id: my-nfs-a
	ceph_daemon_type: nfs
	ceph_nfs: my-nfs
	instance: a
	nfs: my-nfs-a
	rook.io/operator-namespace: rook-ceph
	rook_cluster: rook-ceph
	name: rook-ceph-nfs-my-nfs-a
	spec:
	affinity: {}
	containers:
	- args:
	- -F
	- -L
	- STDERR
	- -p
	- /var/run/ganesha/ganesha.pid
	- -N
	- NIV_DEBUG
	command:
	- ganesha.nfsd
	env:
	- name: CONTAINER_IMAGE
	value: quay.io/ceph/ceph:v17
	- name: POD_NAME
	valueFrom:
	fieldRef:
	apiVersion: v1
	fieldPath: metadata.name
	- name: POD_NAMESPACE
	valueFrom:
	fieldRef:
	apiVersion: v1
	fieldPath: metadata.namespace
	- name: NODE_NAME
	valueFrom:
	fieldRef:
	apiVersion: v1
	fieldPath: spec.nodeName
	- name: POD_MEMORY_LIMIT
	valueFrom:
	resourceFieldRef:
	divisor: "0"
	resource: limits.memory
	- name: POD_MEMORY_REQUEST
	valueFrom:
	resourceFieldRef:
	divisor: "0"
	resource: requests.memory
	- name: POD_CPU_LIMIT
	valueFrom:
	resourceFieldRef:
	divisor: "1"
	resource: limits.cpu
	- name: POD_CPU_REQUEST
	valueFrom:
	resourceFieldRef:
	divisor: "0"
	resource: requests.cpu
	- name: ROOK_CEPH_MON_HOST
	valueFrom:
	secretKeyRef:
	key: mon_host
	name: rook-ceph-config
	- name: ROOK_CEPH_MON_INITIAL_MEMBERS
	valueFrom:
	secretKeyRef:
	key: mon_initial_members
	name: rook-ceph-config
	image: quay.io/ceph/ceph:v17
	imagePullPolicy: IfNotPresent
	name: nfs-ganesha
	# BIG NOTE: ganesha container needs package 'sssd-client' package installed
	resources: {}
	securityContext:
	privileged: false
	terminationMessagePath: /dev/termination-log
	terminationMessagePolicy: File
	volumeMounts:
	- mountPath: /etc/ceph
	name: etc-ceph
	- mountPath: /etc/ceph/keyring-store/
	name: rook-ceph-nfs-my-nfs-a-keyring
	readOnly: true
	- mountPath: /etc/ganesha
	name: ganesha-config
	- mountPath: /run/dbus
	name: run-dbus
	- mountPath: /var/lib/sss/pipes
	name: sssd-sockets
	- mountPath: /etc/nsswitch.conf
	name: nsswitch-conf
	subPath: nsswitch.conf
	- args:
	- --nofork
	- --system
	- --nopidfile
	- --nosyslog
	command:
	- dbus-daemon
	env:
	- name: CONTAINER_IMAGE
	value: quay.io/ceph/ceph:v17
	- name: POD_NAME
	valueFrom:
	fieldRef:
	apiVersion: v1
	fieldPath: metadata.name
	- name: POD_NAMESPACE
	valueFrom:
	fieldRef:
	apiVersion: v1
	fieldPath: metadata.namespace
	- name: NODE_NAME
	valueFrom:
	fieldRef:
	apiVersion: v1
	fieldPath: spec.nodeName
	- name: POD_MEMORY_LIMIT
	valueFrom:
	resourceFieldRef:
	divisor: "0"
	resource: limits.memory
	- name: POD_MEMORY_REQUEST
	valueFrom:
	resourceFieldRef:
	divisor: "0"
	resource: requests.memory
	- name: POD_CPU_LIMIT
	valueFrom:
	resourceFieldRef:
	divisor: "1"
	resource: limits.cpu
	- name: POD_CPU_REQUEST
	valueFrom:
	resourceFieldRef:
	divisor: "0"
	resource: requests.cpu
	image: quay.io/ceph/ceph:v17
	imagePullPolicy: IfNotPresent
	name: dbus-daemon
	resources: {}
	terminationMessagePath: /dev/termination-log
	terminationMessagePolicy: File
	volumeMounts:
	- mountPath: /run/dbus
	name: run-dbus
	# vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv SSSD vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv #
	- name: sssd-daemon
	image: quay.io/brgardne/sssd:latest
	imagePullPolicy: Always
	resources: {}
	volumeMounts:
	# sssd uses dbus only for internal communications, so no shared dbus connection needed
	# mounting dbus causes failures for unknown reasons, and is only necessary to
	# commmunicate w/ sssd's dbus api (not used by nfs-ganesha)
	# - mountPath: /run/dbus
	# name: run-dbus
	- mountPath: /etc/sssd/conf.d
	name: sssd-config
	- mountPath: /var/lib/sss/pipes
	name: sssd-sockets
	# - mountPath: /etc/nsswitch.conf
	# name: nsswitch-conf
	# subPath: nsswitch.conf
	# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ SSSD ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #
	dnsPolicy: ClusterFirst
	initContainers:
	- command:
	- /bin/bash
	- -c
	- \|
	set -xEeuo pipefail

	cat << EOF > /etc/ceph/ceph.conf
	[global]
	mon_host = $(ROOK_CEPH_MON_HOST)

	[client.nfs-ganesha.my-nfs.a]
	keyring = /etc/ceph/keyring-store/keyring
	EOF

	chmod 444 /etc/ceph/ceph.conf

	cat /etc/ceph/ceph.conf
	env:
	- name: ROOK_CEPH_MON_HOST
	valueFrom:
	secretKeyRef:
	key: mon_host
	name: rook-ceph-config
	- name: ROOK_CEPH_MON_INITIAL_MEMBERS
	valueFrom:
	secretKeyRef:
	key: mon_initial_members
	name: rook-ceph-config
	image: quay.io/ceph/ceph:v17
	imagePullPolicy: IfNotPresent
	name: generate-minimal-ceph-conf
	resources: {}
	securityContext:
	privileged: false
	terminationMessagePath: /dev/termination-log
	terminationMessagePolicy: File
	volumeMounts:
	- mountPath: /etc/ceph
	name: etc-ceph
	- mountPath: /etc/ceph/keyring-store/
	name: rook-ceph-nfs-my-nfs-a-keyring
	readOnly: true
	# copy the sssd sockets (pipes) starting content to the dir where it will be shared between containers
	- name: copy-sssd-sockets
	image: quay.io/brgardne/sssd:latest
	command:
	- bash
	- -c
	- \|
	set -ex
	cp -a -v /var/lib/sss/pipes/* /tmp/var/lib/sss/pipes/.
	ls -alF -R /tmp/var/lib/sss/pipes
	volumeMounts:
	- mountPath: /tmp/var/lib/sss/pipes
	name: sssd-sockets
	restartPolicy: Always
	schedulerName: default-scheduler
	securityContext: {}
	terminationGracePeriodSeconds: 30
	tolerations:
	- effect: NoExecute
	key: node.kubernetes.io/unreachable
	operator: Exists
	tolerationSeconds: 5
	volumes:
	- emptyDir: {}
	name: etc-ceph
	- name: rook-ceph-nfs-my-nfs-a-keyring
	secret:
	defaultMode: 420
	secretName: rook-ceph-nfs-my-nfs-a-keyring
	- configMap:
	defaultMode: 420
	items:
	- key: config
	path: ganesha.conf
	name: rook-ceph-nfs-my-nfs-a
	name: ganesha-config
	- emptyDir: {}
	name: run-dbus
	- name: sssd-config
	configMap:
	name: sssd-config
	defaultMode: 0600 # required
	- name: sssd-sockets
	emptyDir: {}
	- name: nsswitch-conf
	configMap:
	name: nsswitch-conf

	---
	apiVersion: v1
	kind: ConfigMap
	metadata:
	name: sssd-config
	namespace: rook-ceph
	data:
	50-ldap.conf: \|
	[domain/default]
	id_provider = ldap
	#access_provider = simple
	access_provider = ldap
	ldap_access_filter = memberOf=cn=rook,ou=groups,dc=rook,dc=net
	#autofs_provider = ldap
	auth_provider = ldap
	#chpass_provider = ldap
	ldap_uri = ldap://ldap-test-server.default.svc.cluster.local/
	ldap_search_base = dc=rook,dc=net
	ldap_default_bind_dn = cn=admin,dc=rook,dc=net
	ldap_default_authtok_type = password
	ldap_default_authtok = admin
	# ldap_id_use_start_tls = true
	# cache_credentials = true
	# ldap_tls_cacertdir = /etc/openldap/certs
	ldap_tls_reqcert = allow
	ldap_user_search_base = ou=users,dc=rook,dc=net
	ldap_group_search_base = ou=groups,dc=rook,dc=net
	debug_level = 9
	# enumerate = true

	[sssd]
	services = nss
	domains = default
	config_file_version = 2
	enable_files_domain = false
	debug_level = 9

	[nss]
	local_negative_timeout = 0
	debug_level = 9
	filter_users = root
	---
	apiVersion: v1
	kind: ConfigMap
	metadata:
	name: nsswitch-conf
	namespace: rook-ceph
	data:
	nsswitch.conf: \|
	passwd: sss
	group: sss
	netgroup: sss