Spotinst Log Query Service

README.md

spotinst-log query service

Install this next to your spotinst-controller to print logs from the Spotinst Ocean. This service simply spits out logs to stdout in JSON format. Then your log aggregation service can use the container logs and publish them to your own logging platform.

Install

kubectl apply -f spotinst-log.yaml --namespace kube-system

The first query when first installing the service will look for logs back 1 hour and then will use the last log's timestamp for the next query's start timestamp.

This service assumes the spotinst-controller was installed via spotinst's helm chart with a basic configuration.

spotinst-log.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: spotinst-log
  namespace: kube-system
spec:
  replicas: 1
  selector:
    matchLabels:
      app: spotinst-log
  template:
    metadata:
      labels:
        app: spotinst-log
    spec:
      serviceAccountName: spotinst-log
      containers:
      - name: spotinst-log-cron
        image: isaaguilar/kubectl:latest
        command:
        - /bin/bash
        args:
        - /root/cmd/query.sh
        volumeMounts:
        - name: cmd
          mountPath: /root/cmd
        env:
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: metadata.namespace
        - name: SPOTINST_TOKEN
          valueFrom:
            configMapKeyRef:
              key: spotinst.token
              name: spotinst-kubernetes-cluster-controller-config
        - name: SPOTINST_ACCOUNT
          valueFrom:
            configMapKeyRef:
              key: spotinst.account
              name: spotinst-kubernetes-cluster-controller-config
        - name: CLUSTER_IDENTIFIER
          valueFrom:
            configMapKeyRef:
              key: spotinst.cluster-identifier
              name: spotinst-kubernetes-cluster-controller-config
      volumes:
      - name: cmd
        configMap:
          name: "spotinst-log"

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: spotinst-log
  namespace: kube-system
data:
  query.sh: |-
    #!/bin/bash
    if [ "$DEBUG" = "true" ];then set -x;fi
    ##
    ## Setup the spotinst-log-query ConfigMap to store the last_query_time
    ##
    kubectl get configmap --namespace ${POD_NAMESPACE} spotinst-log-query >/dev/null 2>err.log
    if [ -s err.log ] && (grep "not found" err.log >/dev/null 2>/dev/null); then
        kubectl create configmap --namespace ${POD_NAMESPACE} spotinst-log-query >/dev/null 2>err.log
        ##
        ## Exit if there is an error creating a configmap
        ##
        if [ -s err.log ];then
          >&2 cat err.log
          exit 1
        fi
    fi

    while true;do
      ##
      ## Find the last query time, this is the derived from the last log received
      ##
      last_query_time=$(kubectl get configmap --namespace ${POD_NAMESPACE} spotinst-log-query -ojsonpath='{.data.last_query_time}')
      if [ -z ${last_query_time} ];then
        # This should only happen the very first time the service is created
        last_query_time=$(date -d '1 hour ago' --iso-8601=seconds)
      fi

      ##
      ## Perform Spotinst Queries
      ##
      id=$(curl -s "https://api.spotinst.io/ocean/aws/k8s/cluster/?accountId=${SPOTINST_ACCOUNT}" -H "Authorization: Bearer ${SPOTINST_TOKEN}"|jq --arg name ${CLUSTER_IDENTIFIER} -r '.response.items[]|select(.name==$name)|.id')
      resp=$(curl -s "https://api.spotinst.io/aws/ec2/group/${id}/logs?fromDate=$(date "+%s000" -d "${last_query_time}")&toDate=$(date '+%s000')&severity=ALL&limit=1000&accountId=${SPOTINST_ACCOUNT}" -H "Authorization: Bearer ${SPOTINST_TOKEN}")

      ##
      ## Print to stdout
      ##
      jq -r '.response.items|reverse|.[]|@json' <<< $resp

      ##
      ## Finally, save the last query time so next job can pick up where this left off
      ##
      lastest_log=$(jq -r '.response.items[0].createdAt' <<< $resp)
      if [ "${lastest_log}" != "null" ];then
        kubectl patch configmap --namespace ${POD_NAMESPACE} spotinst-log-query --type merge -p '{"data":{"last_query_time":"'${lastest_log}'"}}' >/dev/null 2>/dev/null
      fi

      sleep 600
    done

---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: spotinst-log
  namespace: kube-system

---
# TODO find least-priv for this role
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: spotinst-log
  namespace: kube-system
rules:
- apiGroups:
  - ""
  resources:
  - configmaps
  verbs:
  - '*'

---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: spotinst-log
  namespace: kube-system
subjects:
- kind: ServiceAccount
  name: spotinst-log
  namespace: kube-system
roleRef:
  kind: Role
  name: spotinst-log
  apiGroup: rbac.authorization.k8s.io