etcd-backup-operator crashing

backup_cr.yaml

apiVersion: "etcd.database.coreos.com/v1beta2"
kind: "EtcdBackup"
metadata:
  name: example-etcd-cluster-backup
  namespace: etcd
spec:
  etcdEndpoints: [http://example-etcd-cluster-client-2:2379]
  storageType: S3
  BackupSource:
    S3:
      Path: http://domain.com:9000
      AWSSecret: aws
      ForcePathStyle: true
      #  s3:
      #    # The format of "path" must be: "<s3-bucket-name>/<path-to-backup-file>"
      #    # e.g: "mybucket/etcd.backup"
      #    path: <full-s3-path>
      #    awsSecret: <aws-secret>

etcd-backup-operator log

etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator time="2020-11-10T11:15:57Z" level=info msg="Event(v1.ObjectReference{Kind:\"Endpoints\", Namespace:\"etcd\", Name:\"etcd-backup-operator\", UID:\"8262a872-783f-43ed-a3b4-1f4740381dff\", APIVersion:\"v1\", ResourceVersion:\"2157081\", FieldPath:\"\"}): type: 'Normal' reason: 'LeaderElection' etcd-backup-operator-ffbc7cb76-hf8l8 became leader"
etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator time="2020-11-10T11:15:57Z" level=info msg="starting backup controller" pkg=controller
etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator E1110 11:16:28.320077       1 runtime.go:69] Observed a panic: "invalid memory address or nil pointer dereference" (runtime error: invalid memory address or nil pointer dereference)
etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator /Users/sam/go/src/github.com/coreos/etcd-operator/vendor/k8s.io/apimachinery/pkg/util/runtime/runtime.go:76
etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator /Users/sam/go/src/github.com/coreos/etcd-operator/vendor/k8s.io/apimachinery/pkg/util/runtime/runtime.go:65
etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator /Users/sam/go/src/github.com/coreos/etcd-operator/vendor/k8s.io/apimachinery/pkg/util/runtime/runtime.go:51
etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator /usr/local/go/src/runtime/asm_amd64.s:522
etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator /usr/local/go/src/runtime/panic.go:513
etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator /usr/local/go/src/runtime/panic.go:82
etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator /usr/local/go/src/runtime/signal_unix.go:390
etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator /Users/sam/go/src/github.com/coreos/etcd-operator/pkg/controller/backup-operator/s3_backup.go:35
etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator /Users/sam/go/src/github.com/coreos/etcd-operator/pkg/controller/backup-operator/sync.go:271
etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator /Users/sam/go/src/github.com/coreos/etcd-operator/pkg/controller/backup-operator/sync.go:110
etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator /Users/sam/go/src/github.com/coreos/etcd-operator/pkg/controller/backup-operator/sync.go:58
etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator /Users/sam/go/src/github.com/coreos/etcd-operator/pkg/controller/backup-operator/sync.go:44
etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator /Users/sam/go/src/github.com/coreos/etcd-operator/pkg/controller/backup-operator/controller.go:55
etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator /Users/sam/go/src/github.com/coreos/etcd-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133
etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator /Users/sam/go/src/github.com/coreos/etcd-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:134
etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator /Users/sam/go/src/github.com/coreos/etcd-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88
etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator /usr/local/go/src/runtime/asm_amd64.s:1333
etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator panic: runtime error: invalid memory address or nil pointer dereference [recovered]
etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator       panic: runtime error: invalid memory address or nil pointer dereference
etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator [signal SIGSEGV: segmentation violation code=0x1 addr=0x28 pc=0x12fe521]
etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator
etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator goroutine 46 [running]:
etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator github.com/coreos/etcd-operator/vendor/k8s.io/apimachinery/pkg/util/runtime.HandleCrash(0x0, 0x0, 0x0)
etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator       /Users/sam/go/src/github.com/coreos/etcd-operator/vendor/k8s.io/apimachinery/pkg/util/runtime/runtime.go:58 +0x108
etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator panic(0x148c180, 0x2568c00)
etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator       /usr/local/go/src/runtime/panic.go:513 +0x1b9
etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator github.com/coreos/etcd-operator/pkg/controller/backup-operator.handleS3(0x183ea80, 0xc0004d1380, 0x1861fe0, 0xc000128600, 0x0, 0xc000208480, 0x1, 0x1, 0x0, 0x0, ...)
etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator       /Users/sam/go/src/github.com/coreos/etcd-operator/pkg/controller/backup-operator/s3_backup.go:35 +0x51
etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator github.com/coreos/etcd-operator/pkg/controller/backup-operator.(*Backup).handleBackup(0xc000114160, 0x0, 0xc0001ba488, 0x162c400, 0x0, 0x0, 0x0)
etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator       /Users/sam/go/src/github.com/coreos/etcd-operator/pkg/controller/backup-operator/sync.go:271 +0x4dc
etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator github.com/coreos/etcd-operator/pkg/controller/backup-operator.(*Backup).processItem(0xc000114160, 0xc00029eee0, 0x20, 0x13f87a0, 0xc0002084a0)
etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator       /Users/sam/go/src/github.com/coreos/etcd-operator/pkg/controller/backup-operator/sync.go:110 +0x13a
etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator github.com/coreos/etcd-operator/pkg/controller/backup-operator.(*Backup).processNextItem(0xc000114160, 0xc0004a6700)
etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator       /Users/sam/go/src/github.com/coreos/etcd-operator/pkg/controller/backup-operator/sync.go:58 +0xd3
etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator github.com/coreos/etcd-operator/pkg/controller/backup-operator.(*Backup).runWorker(0xc000114160)
etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator       /Users/sam/go/src/github.com/coreos/etcd-operator/pkg/controller/backup-operator/sync.go:44 +0x2b
etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator github.com/coreos/etcd-operator/pkg/controller/backup-operator.(*Backup).runWorker-fm()
etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator       /Users/sam/go/src/github.com/coreos/etcd-operator/pkg/controller/backup-operator/controller.go:55 +0x2a
etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator github.com/coreos/etcd-operator/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc0002229a0)
etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator       /Users/sam/go/src/github.com/coreos/etcd-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133 +0x54
etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator github.com/coreos/etcd-operator/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0002229a0, 0x3b9aca00, 0x0, 0x1, 0xc0003c8120)
etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator       /Users/sam/go/src/github.com/coreos/etcd-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:134 +0xbe
etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator github.com/coreos/etcd-operator/vendor/k8s.io/apimachinery/pkg/util/wait.Until(0xc0002229a0, 0x3b9aca00, 0xc0003c8120)
etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator       /Users/sam/go/src/github.com/coreos/etcd-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88 +0x4d
etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator created by github.com/coreos/etcd-operator/pkg/controller/backup-operator.(*Backup).run
etcd-backup-operator-ffbc7cb76-hf8l8 etcd-backup-operator       /Users/sam/go/src/github.com/coreos/etcd-operator/pkg/controller/backup-operator/controller.go:55 +0x69c

etcd-backup-operator.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: etcd-backup-operator
  namespace: etcd
spec:
  replicas: 1
  selector:
    matchLabels:
      app: etcd-backup-operator
  template:
    metadata:
      labels:
        app: etcd-backup-operator
    spec:
      containers:
      - name: etcd-backup-operator
        image: quay.io/coreos/etcd-operator:v0.9.4
        command:
        - etcd-backup-operator
        env:
        - name: MY_POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        - name: MY_POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name

etcd-operator.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: etcd-operator
  namespace: etcd
spec:
  selector:
    matchLabels:
      app: etcd-operator
  replicas: 1
  template:
    metadata:
      labels:
        app: etcd-operator
    spec:
      containers:
      - name: etcd-operator
        image: quay.io/coreos/etcd-operator:v0.9.4
        command:
        - etcd-operator
        # Uncomment to act for resources in all namespaces. More information in doc/user/clusterwide.md
        #- -cluster-wide
        env:
        - name: MY_POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        - name: MY_POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name

example-etcd-cluster.yaml

apiVersion: "etcd.database.coreos.com/v1beta2"
kind: "EtcdCluster"
metadata:
  name: "example-etcd-cluster-2"
  namespace: etcd
  ## Adding this annotation make this cluster managed by clusterwide operators
  ## namespaced operators ignore it
  # annotations:
  #   etcd.database.coreos.com/scope: clusterwide
spec:
  size: 3
  version: "3.4.10"
  pod:
    persistentVolumeClaimSpec:
      storageClassName: rook-ceph-block
      accessModes:
      - ReadWriteOnce
      resources:
        requests:
          storage: 500Mi
    resources:
      limits:
        cpu: 200m
        memory: 100Mi
      requests:
        cpu: 50m
        memory: 50Mi
    affinity:
      podAntiAffinity:
        preferredDuringSchedulingIgnoredDuringExecution:
        - weight: 100
          podAffinityTerm:
            labelSelector:
              matchExpressions:
              - key: etcd_cluster
                operator: In
                values:
                - example-etcd-cluster
            topologyKey: kubernetes.io/hostname

s3-config

[default]
region = minio