clusterfoo-thanos-store-gateway.yaml

# Deploy an additional thanos-store-gateway instance in to feed
# thanos-querier with TSDB from a different cluster S3 bucket
# (no direct connectivity between clusters)
# There `clusterfoo` is the "remote" cluster, running under
# a different AWS_ACCOUNT_NUM, using kiam to assume a role
# able to read the S3 bucket.
---
apiVersion: apps/v1beta2
kind: StatefulSet
metadata:
  labels:
    name: clusterfoo-thanos-store-gateway
  name: clusterfoo-thanos-store-gateway
  namespace: thanos
spec:
  # Total time "coverage" will be: DAYS_OFFSET + (replicas * DAYS_PER_POD)
  replicas: 5
  selector:
    matchLabels:
      name: clusterfoo-thanos-store-gateway
      thanos-store-api: "true"
  serviceName: clusterfoo-thanos-store-gateway
  template:
    metadata:
      annotations:
        # Using kiam for AWS authn, here assuming a role at a different AWS account
        iam.amazonaws.com/role: arn:aws:iam::<AWS_ACCOUNT_NUM>:role/k8s-prometheus-role
        prometheus.io/scrape: "true"
      labels:
        name: clusterfoo-thanos-store-gateway
        thanos-store-api: "true"
    spec:
      containers:
      - args:
        - --block-sync-concurrency=2
        - --chunk-pool-size=250MB
        - --data-dir=/data
        - --index-cache-size=512MB
        - --log.level=debug
        - --objstore.config-file=/config/thanos.yaml
        command:
        - /init/thanos-store.sh
        env:
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        image: quay.io/thanos/thanos:v0.10.1
        imagePullPolicy: IfNotPresent
        name: thanos-store-gateway
        ports:
        - containerPort: 10901
          name: grpc-nometrics
        - containerPort: 10902
          name: http
        resources:
          limits:
            cpu: "2"
            memory: 6Gi
          requests:
            cpu: 0.5
            memory: 3Gi
        stdin: false
        tty: false
        volumeMounts:
        - mountPath: /data
          name: data
        - mountPath: /init
          name: init
        - mountPath: /config
          name: s3-config
      imagePullSecrets: []
      initContainers:
      - args: []
        command:
        - bash
        - -c
        - |
          get_times() {
              # Use number after '-' as factor, e.g. 'foo-bar-0' -> '0'
              typeset -i factor=${1/*-/}
              echo --min-time=-$((DAYS_OFFSET+DAYS_PER_POD*(factor+1)))d --max-time=-$((DAYS_OFFSET+DAYS_PER_POD*(factor)))d
          }
          # Blow if any below envvar is unset
          : ${MY_POD_NAME:?} ${DAYS_PER_POD:?} ${DAYS_OFFSET:?}
          printf "#!/bin/sh\nexec thanos store $(get_times ${MY_POD_NAME}) \${@} \n" | install -m755 /dev/stdin /init/thanos-store.sh
          echo "INFO replica=${MY_POD_NAME} /init/thanos-store.sh:"
          cat /init/thanos-store.sh
        env:
        - name: DAYS_OFFSET
          value: "0"
        - name: DAYS_PER_POD
          value: "14"
        - name: MY_POD_NAME
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: metadata.name
        image: bitnami/minideb:buster
        imagePullPolicy: IfNotPresent
        name: thanos-init
        ports: []
        resources:
          limits:
            cpu: "0.1"
            memory: 200Mi
          requests:
            cpu: "0.1"
            memory: 200Mi
        stdin: false
        tty: false
        volumeMounts:
        - mountPath: /init
          name: init
      terminationGracePeriodSeconds: 30
      volumes:
      - emptyDir: {}
        name: data
      - emptyDir: {}
        name: init
      - configMap:
          name: clusterfoo-thanos-s3-config
        name: s3-config
  updateStrategy:
    type: RollingUpdate
  volumeClaimTemplates: []
---
apiVersion: v1
data:
  thanos.yaml: |
    type: S3
    config:
      bucket: thanos.clusterfoo.example.com
      endpoint: s3.amazonaws.com
kind: ConfigMap
metadata:
  labels:
    name: clusterfoo-thanos-s3-config
  name: clusterfoo-thanos-s3-config
  namespace: thanos