confluent-tiered-storage.yaml

apiVersion: v1
kind: Service
metadata:
  name: confluent
  labels:
    app: confluent
spec:
  clusterIP: None
  ports:
  - name: kafka-port
    port: 9092
  selector:
    app: confluentkafka
---
kind: ConfigMap
apiVersion: v1
metadata:
  name: confluent-env-config
data:
  KAFKA_ZOOKEEPER_CONNECT: zk-cs:2181
  KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://:9092
  KAFKA_METRIC_REPORTERS: io.confluent.metrics.reporter.ConfluentMetricsReporter
  KAFKA_CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: confluent:9092
  KAFKA_COMPRESSION_TYPE: producer
  KAFKA_LOG_SEGMENT_BYTES: "104857600"
  KAFKA_CONFLUENT_TIER_ENABLE: "true"
  KAFKA_CONFLUENT_TIER_FEATURE: "true"
  KAFKA_CONFLUENT_TIER_BACKEND: S3
  KAFKA_CONFLUENT_TIER_S3_BUCKET: "my-kafka-bucket"
  KAFKA_CONFLUENT_TIER_S3_REGION: "us-west-2"
  KAFKA_CONFLUENT_TIER_S3_AWS_ENDPOINT_OVERRIDE: "http://FLASHBLADE_IP"
  KAFKA_CONFLUENT_TIER_LOCAL_HOTSET_MS: "0"
  KAFKA_CONFLUENT_TIER_ARCHIVER_NUM_THREADS: "8"
  KAFKA_CONFLUENT_TIER_FETCHER_NUM_THREADS: "16"
  KAFKA_CONFLUENT_TIER_TOPIC_DELETE_CHECK_INTERVAL_MS: "30000"
  KAFKA_CONFLUENT_TIER_METADATA_REPLICATION_FACTOR: "3"
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: confluentkafka
spec:
  serviceName: confluent
  replicas: 4
  selector:
    matchLabels:
      app: confluentkafka
  template:
    metadata:
      labels:
        app: confluentkafka
    spec:
      securityContext:
        fsGroup: 1000
      containers:
      - name: cp-server
        image: confluentinc/cp-server:6.0.0
        envFrom:
        - configMapRef:
            name: confluent-env-config
        env:
        - name: AWS_ACCESS_KEY_ID
          valueFrom:
            secretKeyRef:
              name: my-s3-keys
              key: access-key
        - name: AWS_SECRET_ACCESS_KEY
          valueFrom:
            secretKeyRef:
              name: my-s3-keys
              key: secret-key
        ports:
        - containerPort: 9092
          name: kafka-port
        imagePullPolicy: Always
        volumeMounts:
        - name: broker-data
          mountPath: /var/lib/kafka/data
        resources:
          requests:
            memory: 64Gi
            cpu: 24
          limits:
            memory: 64Gi
            cpu: 34
  volumeClaimTemplates:
  - metadata:
      name: broker-data
    spec:
      storageClassName: pure-block
      accessModes:
      - ReadWriteOnce
      resources:
        requests:
          storage: 1Ti