keydb nomad job with vault password rotation

keydb.nomad.hcl

job "keydb-vanilla" {
  region      = "global"
  datacenters = ["dc1"]
  type        = "service"

  group "keydb" {
    count = 1

    vault {
      policies = ["nomad-keydb"]
    }

    network {
      mode = "host"
      port "redis" {
        static       = 6379
        to           = 6379
        host_network = "tailscale"
      }
    }

    update {
      min_healthy_time  = "10s"
      healthy_deadline  = "10m"
      progress_deadline = "20m"
      auto_revert       = true
    }

    service {
      name     = "keydb"
      port     = "redis"
      tags     = ["database", "cache"]
      provider = "nomad"
    }

    restart {
      attempts = 2
      interval = "30m"
      delay    = "15s"
      mode     = "fail"
    }

    task "keydb" {
      driver = "exec"
      user   = "keydb"

      config {
        command = "/usr/bin/keydb-server"
        args = [
          "/etc/keydb/keydb.conf",
        ]
      }

      resources {
        cpu    = 10000
        memory = 32 * 1024
      }

      template {
        data          = file("keydb.conf.tpl")
        destination   = "/etc/keydb/keydb.conf"
        change_mode   = "signal"
        change_signal = "SIGHUP"
        perms         = "0600"
        uid           = "106"
      }

      template {
        data          = <<EOH
        user default on -DEBUG +@all ~* >{{with secret "secret/data/keydb/default"}}{{.Data.data.password}}{{end}}
        EOH
        destination   = "${NOMAD_SECRETS_DIR}/users.acl"
        change_mode   = "signal"
        change_signal = "SIGHUP"
        perms         = "0600"
        uid           = "106"
      }


      volume_mount {
        volume      = "flash"
        destination = "/opt/keydb/flash"
        read_only   = false
      }

      volume_mount {
        volume      = "rdb"
        destination = "/opt/keydb/rdb"
        read_only   = false
      }
    }

    task "rotate-password" {
      driver = "exec"
      user   = "keydb"

      lifecycle {
        hook    = "poststart"
        sidecar = false
      }

      template {
        data          = <<EOH
        {
          "plugin_name": "redis-database-plugin",
          "allowed_roles": "access-keydb",
          "host": "{{ env "NOMAD_IP_redis" }}",
          "port": "{{ env "NOMAD_PORT_redis" }}",
          "username": "default",
          "password": "{{with secret "secret/data/keydb/default"}}{{.Data.data.password}}{{end}}"
        }
        EOH
        destination   = "${NOMAD_SECRETS_DIR}/keydb-database.json"
        change_mode   = "signal"
        change_signal = "SIGHUP"
        perms         = "0400"
        uid           = "106"
      }

      template {
        data = <<EOH
        #!/bin/bash 
        vault write keydb/config/pollux @${NOMAD_SECRETS_DIR}/keydb-database.json
        vault write -force keydb/rotate-root/pollux
        EOH
        destination   = "${NOMAD_ALLOC_DIR}/vault-keydb-config.sh"
        change_mode   = "signal"
        change_signal = "SIGHUP"
        perms         = "0555"
      }

      config {
        command = "/bin/sh"
        args = ["-c", "${NOMAD_ALLOC_DIR}/vault-keydb-config.sh"]
      }
    }

    volume "flash" {
      type      = "host"
      source    = "keydb-flash"
      read_only = false
    }
    volume "rdb" {
      type      = "host"
      source    = "keydb-rdb"
      read_only = false
    }
  }
}