Are resources and placement working in devicesets?

cluster.yaml

---

apiVersion: ceph.rook.io/v1
kind: CephCluster
metadata:
  name: rook-ceph
  namespace: rook-ceph
spec:
  cephVersion:
    image: ceph/ceph:v14
    allowUnsupported: false
  # Even w/ PVs, this must be set or mons fail to start due to permission
  # problems
  dataDirHostPath: /var/lib/rook
  dashboard:
    enabled: false
  mon:
    count: 3
    allowMultiplePerNode: false
    volumeClaimTemplate:
      spec:
        storageClassName: gp2
        resources:
          requests:
            storage: 10Gi
  network:
    hostNetwork: false
  rbdMirroring:
    workers: 0
  annotations:
  resources:
    mds: # doesn't get passed?
      limits:
        cpu: 2000m
        memory: 4Gi
      requests:
        cpu: 100m
        memory: 4Gi
    mgr:
      limits:
        cpu: 2000m
        memory: 3Gi
      requests:
        cpu: 100m
        memory: 500Mi
    mon:
      limits:
        cpu: 2000m
        memory: 2Gi
      requests:
        cpu: 100m
        memory: 1Gi
  storage:
    storageClassDeviceSets:
      - name: east-1a
        count: 1
        placement:
          nodeAffinity:
            requiredDuringSchedulingIgnoredDuringExecution:
              nodeSelectorTerms:
                - matchExpressions:
                    - key: failure-domain.beta.kubernetes.io/zone
                      operator: In
                      values: [us-east-1a]
        resources:
          limits:
            cpu: 2000m
            memory: 4Gi
          requests:
            cpu: 100m
            memory: 4Gi
        volumeClaimTemplates:
          - metadata:
              name: data
            spec:
              resources:
                requests:
                  storage: 1Ti
              storageClassName: gp2
              volumeMode: Block
              accessModes:
                - ReadWriteOnce
      - name: east-1b
        count: 1
        placement:
          nodeAffinity:
            requiredDuringSchedulingIgnoredDuringExecution:
              nodeSelectorTerms:
                - matchExpressions:
                    - key: failure-domain.beta.kubernetes.io/zone
                      operator: In
                      values: [us-east-1b]
        resources:
          limits:
            cpu: 2000m
            memory: 4Gi
          requests:
            cpu: 100m
            memory: 4Gi
        volumeClaimTemplates:
          - metadata:
              name: data
            spec:
              resources:
                requests:
                  storage: 1Ti
              storageClassName: gp2
              volumeMode: Block
              accessModes:
                - ReadWriteOnce
      - name: east-1c
        count: 1
        placement:
          nodeAffinity:
            requiredDuringSchedulingIgnoredDuringExecution:
              nodeSelectorTerms:
                - matchExpressions:
                    - key: failure-domain.beta.kubernetes.io/zone
                      operator: In
                      values: [us-east-1c]
        resources:
          limits:
            cpu: 2000m
            memory: 4Gi
          requests:
            cpu: 100m
            memory: 4Gi
        volumeClaimTemplates:
          - metadata:
              name: data
            spec:
              resources:
                requests:
                  storage: 1Ti
              storageClassName: gp2
              volumeMode: Block
              accessModes:
                - ReadWriteOnce
    topologyAware: true

osd_pod.yaml

---

apiVersion: v1
kind: Pod
metadata:
  annotations:
    openshift.io/scc: rook-ceph
  creationTimestamp: "2019-08-14T00:47:55Z"
  generateName: rook-ceph-osd-0-9d4fd68ff-
  labels:
    app: rook-ceph-osd
    ceph-osd-id: "0"
    pod-template-hash: 9d4fd68ff
    rook_cluster: rook-ceph
  name: rook-ceph-osd-0-9d4fd68ff-cw2sx
  namespace: rook-ceph
  ownerReferences:
  - apiVersion: apps/v1
    blockOwnerDeletion: true
    controller: true
    kind: ReplicaSet
    name: rook-ceph-osd-0-9d4fd68ff
    uid: 27403a6a-be2d-11e9-af6a-02df3e5f3a82
  resourceVersion: "74367"
  selfLink: /api/v1/namespaces/rook-ceph/pods/rook-ceph-osd-0-9d4fd68ff-cw2sx
  uid: 2743b3d2-be2d-11e9-af6a-02df3e5f3a82
spec:
  affinity: {}
  containers:
  - args:
    - --
    - /rook/rook
    - ceph
    - osd
    - start
    - --
    - --foreground
    - --id
    - "0"
    - --osd-uuid
    - 4494f602-b2dc-4127-951c-584c53317c3f
    - --conf
    - /var/lib/rook/osd0/rook-ceph.config
    - --cluster
    - ceph
    - --setuser
    - ceph
    - --setgroup
    - ceph
    - --setuser-match-path
    - /var/lib/rook/osd0
    - --default-log-to-file
    - "false"
    - --ms-learn-addr-from-peer=false
    command:
    - /rook/tini
    env:
    - name: ROOK_NODE_NAME
      value: east-1a-0-data-p7qfz
    - name: ROOK_PRIVATE_IP
      valueFrom:
        fieldRef:
          apiVersion: v1
          fieldPath: status.podIP
    - name: ROOK_PUBLIC_IP
      valueFrom:
        fieldRef:
          apiVersion: v1
          fieldPath: status.podIP
    - name: TINI_SUBREAPER
    - name: CONTAINER_IMAGE
      value: ceph/ceph:v14
    - name: POD_NAME
      valueFrom:
        fieldRef:
          apiVersion: v1
          fieldPath: metadata.name
    - name: POD_NAMESPACE
      valueFrom:
        fieldRef:
          apiVersion: v1
          fieldPath: metadata.namespace
    - name: NODE_NAME
      valueFrom:
        fieldRef:
          apiVersion: v1
          fieldPath: spec.nodeName
    - name: POD_MEMORY_LIMIT
      valueFrom:
        resourceFieldRef:
          divisor: "0"
          resource: limits.memory
    - name: POD_MEMORY_REQUEST
      valueFrom:
        resourceFieldRef:
          divisor: "0"
          resource: requests.memory
    - name: POD_CPU_LIMIT
      valueFrom:
        resourceFieldRef:
          divisor: "1"
          resource: limits.cpu
    - name: POD_CPU_REQUEST
      valueFrom:
        resourceFieldRef:
          divisor: "0"
          resource: requests.cpu
    - name: ROOK_OSD_UUID
      value: 4494f602-b2dc-4127-951c-584c53317c3f
    - name: ROOK_OSD_ID
      value: "0"
    - name: ROOK_OSD_STORE_TYPE
      value: bluestore
    - name: ROOK_PVC_BACKED_OSD
      value: "true"
    image: ceph/ceph:v14
    imagePullPolicy: IfNotPresent
    lifecycle:
      postStart:
        exec:
          command:
          - chown
          - --recursive
          - --verbose
          - ceph:ceph
          - /var/log/ceph
          - /var/lib/rook/osd0
    name: osd
    resources: {}
    securityContext:
      privileged: true
      readOnlyRootFilesystem: false
      runAsUser: 0
    terminationMessagePath: /dev/termination-log
    terminationMessagePolicy: File
    volumeMounts:
    - mountPath: /var/lib/rook
      name: rook-data
    - mountPath: /etc/ceph
      name: ceph-default-config-dir
    - mountPath: /var/log/ceph
      name: rook-ceph-log
    - mountPath: /dev
      name: devices
    - mountPath: /rook
      name: rook-binaries
    - mountPath: /run/udev
      name: run-udev
    - mountPath: /mnt
      name: east-1a-0-data-p7qfz-bridge
    - mountPath: /var/run/secrets/kubernetes.io/serviceaccount
      name: rook-ceph-osd-token-nr4jq
      readOnly: true
  dnsPolicy: ClusterFirst
  enableServiceLinks: true
  hostPID: true
  imagePullSecrets:
  - name: rook-ceph-osd-dockercfg-2fd5k
  initContainers:
  - args:
    - ceph
    - osd
    - init
    env:
    - name: ROOK_NODE_NAME
      value: east-1a-0-data-p7qfz
    - name: ROOK_CLUSTER_ID
      value: 948e2f25-be2c-11e9-af6a-02df3e5f3a82
    - name: ROOK_PRIVATE_IP
      valueFrom:
        fieldRef:
          apiVersion: v1
          fieldPath: status.podIP
    - name: ROOK_PUBLIC_IP
      valueFrom:
        fieldRef:
          apiVersion: v1
          fieldPath: status.podIP
    - name: ROOK_CLUSTER_NAME
      value: rook-ceph
    - name: ROOK_MON_ENDPOINTS
      valueFrom:
        configMapKeyRef:
          key: data
          name: rook-ceph-mon-endpoints
    - name: ROOK_MON_SECRET
      valueFrom:
        secretKeyRef:
          key: mon-secret
          name: rook-ceph-mon
    - name: ROOK_ADMIN_SECRET
      valueFrom:
        secretKeyRef:
          key: admin-secret
          name: rook-ceph-mon
    - name: ROOK_CONFIG_DIR
      value: /var/lib/rook
    - name: ROOK_CEPH_CONFIG_OVERRIDE
      value: /etc/rook/config/override.conf
    - name: ROOK_FSID
      valueFrom:
        secretKeyRef:
          key: fsid
          name: rook-ceph-mon
    - name: TINI_SUBREAPER
    - name: ROOK_OSD_ID
      value: "0"
    - name: ROOK_CEPH_VERSION
      value: ceph version 14.2.2 nautilus
    - name: ROOK_IS_DEVICE
      value: "true"
    image: rook/ceph:master
    imagePullPolicy: IfNotPresent
    name: config-init
    resources: {}
    securityContext:
      privileged: true
      readOnlyRootFilesystem: false
      runAsUser: 0
    terminationMessagePath: /dev/termination-log
    terminationMessagePolicy: File
    volumeMounts:
    - mountPath: /var/lib/rook
      name: rook-data
    - mountPath: /etc/ceph
      name: ceph-default-config-dir
    - mountPath: /var/log/ceph
      name: rook-ceph-log
    - mountPath: /etc/rook/config
      name: rook-config-override
    - mountPath: /var/run/secrets/kubernetes.io/serviceaccount
      name: rook-ceph-osd-token-nr4jq
      readOnly: true
  - args:
    - copy-binaries
    - --copy-to-dir
    - /rook
    image: rook/ceph:master
    imagePullPolicy: IfNotPresent
    name: copy-bins
    resources: {}
    terminationMessagePath: /dev/termination-log
    terminationMessagePolicy: File
    volumeMounts:
    - mountPath: /rook
      name: rook-binaries
    - mountPath: /var/run/secrets/kubernetes.io/serviceaccount
      name: rook-ceph-osd-token-nr4jq
      readOnly: true
  - args:
    - cp
    - -a
    - /east-1a-0-data-p7qfz
    - /mnt/east-1a-0-data-p7qfz
    image: ceph/ceph:v14
    imagePullPolicy: IfNotPresent
    name: blkdevmapper
    resources: {}
    securityContext:
      privileged: true
    terminationMessagePath: /dev/termination-log
    terminationMessagePolicy: File
    volumeDevices:
    - devicePath: /east-1a-0-data-p7qfz
      name: east-1a-0-data-p7qfz
    volumeMounts:
    - mountPath: /mnt
      name: east-1a-0-data-p7qfz-bridge
    - mountPath: /var/run/secrets/kubernetes.io/serviceaccount
      name: rook-ceph-osd-token-nr4jq
      readOnly: true
  nodeName: ip-10-0-165-53.ec2.internal
  priority: 0
  restartPolicy: Always
  schedulerName: default-scheduler
  securityContext:
    fsGroup: 1000510000
    seLinuxOptions:
      level: s0:c23,c2
  serviceAccount: rook-ceph-osd
  serviceAccountName: rook-ceph-osd
  terminationGracePeriodSeconds: 30
  tolerations:
  - effect: NoExecute
    key: node.kubernetes.io/not-ready
    operator: Exists
    tolerationSeconds: 300
  - effect: NoExecute
    key: node.kubernetes.io/unreachable
    operator: Exists
    tolerationSeconds: 300
  volumes:
  - hostPath:
      path: /var/lib/rook
      type: ""
    name: rook-data
  - emptyDir: {}
    name: ceph-default-config-dir
  - configMap:
      defaultMode: 420
      items:
      - key: config
        path: override.conf
      name: rook-config-override
    name: rook-config-override
  - hostPath:
      path: /var/lib/rook/log/rook-ceph
      type: ""
    name: rook-ceph-log
  - hostPath:
      path: /dev
      type: ""
    name: devices
  - name: east-1a-0-data-p7qfz
    persistentVolumeClaim:
      claimName: east-1a-0-data-p7qfz
  - emptyDir:
      medium: Memory
    name: east-1a-0-data-p7qfz-bridge
  - emptyDir: {}
    name: rook-binaries
  - hostPath:
      path: /run/udev
      type: ""
    name: run-udev
  - name: rook-ceph-osd-token-nr4jq
    secret:
      defaultMode: 420
      secretName: rook-ceph-osd-token-nr4jq
status:
  conditions:
  - lastProbeTime: null
    lastTransitionTime: "2019-08-14T00:48:01Z"
    status: "True"
    type: Initialized
  - lastProbeTime: null
    lastTransitionTime: "2019-08-14T00:48:02Z"
    status: "True"
    type: Ready
  - lastProbeTime: null
    lastTransitionTime: "2019-08-14T00:48:02Z"
    status: "True"
    type: ContainersReady
  - lastProbeTime: null
    lastTransitionTime: "2019-08-14T00:47:55Z"
    status: "True"
    type: PodScheduled
  containerStatuses:
  - containerID: cri-o://b701a7edc5a204310f6c086c05b9ab79ac10ad0435e0234c51e4062d9c5bee58
    image: docker.io/ceph/ceph:v14
    imageID: docker.io/ceph/ceph@sha256:567fe78d90a63ead11deadc2cbf5a912e42bfcc6ef4b1d6154f4b4fea4019052
    lastState: {}
    name: osd
    ready: true
    restartCount: 0
    state:
      running:
        startedAt: "2019-08-14T00:48:01Z"
  hostIP: 10.0.165.53
  initContainerStatuses:
  - containerID: cri-o://9a9126183e320af2a03d1da97139b2ef6ac6abeab0786467ea61109036353968
    image: docker.io/rook/ceph:master
    imageID: docker.io/rook/ceph@sha256:876e1b283e3185f5e9efc59b608905a6cc973f325d060eeeb4594ca052ef91a4
    lastState: {}
    name: config-init
    ready: true
    restartCount: 0
    state:
      terminated:
        containerID: cri-o://9a9126183e320af2a03d1da97139b2ef6ac6abeab0786467ea61109036353968
        exitCode: 0
        finishedAt: "2019-08-14T00:47:58Z"
        reason: Completed
        startedAt: "2019-08-14T00:47:58Z"
  - containerID: cri-o://0cf0efaf82b6f7e73d46874a9f21908da555de1c0b6d13ce6ebf5e91e37e3d2c
    image: docker.io/rook/ceph:master
    imageID: docker.io/rook/ceph@sha256:876e1b283e3185f5e9efc59b608905a6cc973f325d060eeeb4594ca052ef91a4
    lastState: {}
    name: copy-bins
    ready: true
    restartCount: 0
    state:
      terminated:
        containerID: cri-o://0cf0efaf82b6f7e73d46874a9f21908da555de1c0b6d13ce6ebf5e91e37e3d2c
        exitCode: 0
        finishedAt: "2019-08-14T00:47:59Z"
        reason: Completed
        startedAt: "2019-08-14T00:47:59Z"
  - containerID: cri-o://fb0c15753623ea2f8e0c78f6f436dd1ad320aeb7db96191f16d3dc77ab7d5154
    image: docker.io/ceph/ceph:v14
    imageID: docker.io/ceph/ceph@sha256:567fe78d90a63ead11deadc2cbf5a912e42bfcc6ef4b1d6154f4b4fea4019052
    lastState: {}
    name: blkdevmapper
    ready: true
    restartCount: 0
    state:
      terminated:
        containerID: cri-o://fb0c15753623ea2f8e0c78f6f436dd1ad320aeb7db96191f16d3dc77ab7d5154
        exitCode: 0
        finishedAt: "2019-08-14T00:48:00Z"
        reason: Completed
        startedAt: "2019-08-14T00:48:00Z"
  phase: Running
  podIP: 10.131.0.29
  qosClass: BestEffort
  startTime: "2019-08-14T00:47:55Z"