README.md

Loki is a complex, unfriendly service that can sometimes store your logs. It does this via multiple services to handle the overhead of working with unstructured text inherent to working with logs. Its a massive set of microservices with the operational complexity inherent to the larger k8s ecosystem (read: upstream answers all questions in terms of kubernetes), but with some effort it can be made to run on Nomad.

loki.nomad

job "loki" {
  datacenters = ["minicluster"]
  type = "service"

  group "ingester" {
    count = 3

    spread {
      attribute = "${node.unique.id}"
    }

    network {
      mode = "host"
      port "http" {}
      port "grpc" {}
      port "memberlist" {}

      dns {
        servers = ["127.0.0.1"]
      }
    }

    service {
      name = "loki-memberlist"
      port = "memberlist"

      check {
        type = "tcp"
        port = "memberlist"
        address_mode = "host"
        interval = "5s"
        timeout = "2s"

        # This is so that boundary conditions during bootstrap are
        # met, so that servers can see each other as the ring boots
        # up.
        initial_status = "passing"
      }
    }

    task "ingester" {
      driver = "docker"
      kill_timeout = "5m"

      config {
        image = "grafana/loki:2.1.0"
        network_mode="host"

        args = [
          "-config.file=/local/loki.yml",
          "-target=ingester",
          "-server.http-listen-address=${NOMAD_IP_http}",
          "-server.http-listen-port=${NOMAD_PORT_http}",
          "-server.grpc-listen-address=${NOMAD_IP_grpc}",
          "-server.grpc-listen-port=${NOMAD_PORT_grpc}",
        ]
      }

      template {
        data = file("./loki.yml")
        destination = "local/loki.yml"
        perms = "0644"
        splay = "30s"
      }

      restart {
        attempts = 100
      }
    }
  }


  group "distributor" {
    count = 1

    spread {
      attribute = "${node.unique.id}"
    }

    network {
      mode = "host"
      port "http" { static = 3100 }
      port "grpc" {}

      dns {
        servers = ["127.0.0.1"]
      }
    }

    service {
      name = "loki"
      port = "http"
      tags = ["traefik.enable=true"]
    }

    task "ingester" {
      driver = "docker"
      kill_timeout = "5m"

      config {
        image = "grafana/loki:2.1.0"
        network_mode="host"

        args = [
          "-config.file=/local/loki.yml",
          "-target=distributor",
          "-server.http-listen-address=${NOMAD_IP_http}",
          "-server.http-listen-port=${NOMAD_PORT_http}",
          "-server.grpc-listen-address=${NOMAD_IP_grpc}",
          "-server.grpc-listen-port=${NOMAD_PORT_grpc}",
        ]
      }

      template {
        data = file("./loki.yml")
        destination = "local/loki.yml"
        perms = "0644"
        splay = "30s"
      }

      restart {
        attempts = 100
      }
    }
  }


  group "querier" {
    count = 3

    spread {
      attribute = "${node.unique.id}"
    }

    network {
      mode = "host"
      port "http" {}
      port "grpc" {}

      dns {
        servers = ["127.0.0.1"]
      }
    }

    service {
      name = "loki-querier"
      port = "grpc"

      check {
        type = "tcp"
        port = "grpc"
        address_mode = "host"
        interval = "5s"
        timeout = "2s"
      }
    }

    task "querier" {
      driver = "docker"
      kill_timeout = "5m"

      config {
        image = "grafana/loki:2.1.0"
        network_mode="host"

        args = [
          "-config.file=/local/loki.yml",
          "-target=querier",
          "-server.http-listen-address=${NOMAD_IP_http}",
          "-server.http-listen-port=${NOMAD_PORT_http}",
          "-server.grpc-listen-address=${NOMAD_IP_grpc}",
          "-server.grpc-listen-port=${NOMAD_PORT_grpc}",
        ]
      }

      template {
        data = file("./loki.yml")
        destination = "local/loki.yml"
        perms = "0644"
        splay = "30s"
      }

      restart {
        attempts = 100
      }
    }
  }

  group "query-frontend" {
    count = 2

    spread {
      attribute = "${node.unique.id}"
    }

    network {
      mode = "host"
      port "http" { static = 3100 }
      port "grpc" {}

      dns {
        servers = ["127.0.0.1"]
      }
    }

    service {
      name = "loki-queries"
      port = "http"
    }

    task "querier" {
      driver = "docker"
      kill_timeout = "5m"

      config {
        image = "grafana/loki:2.1.0"
        network_mode="host"

        args = [
          "-config.file=/local/loki.yml",
          "-target=querier",
          "-server.http-listen-address=${NOMAD_IP_http}",
          "-server.http-listen-port=${NOMAD_PORT_http}",
          "-server.grpc-listen-address=${NOMAD_IP_grpc}",
          "-server.grpc-listen-port=${NOMAD_PORT_grpc}",
        ]
      }

      template {
        data = file("./loki.yml")
        destination = "local/loki.yml"
        perms = "0644"
        splay = "30s"
      }

      restart {
        attempts = 100
      }
    }
  }

  group "compactor" {
    count = 1

    spread {
      attribute = "${node.unique.id}"
    }

    network {
      mode = "host"
      port "http" {}
      port "grpc" {}
      
      dns {
        servers = ["127.0.0.1"]
      }
    }

    task "compactor" {
      driver = "docker"
      kill_timeout = "5m"

      config {
        image = "grafana/loki:2.1.0"
        network_mode="host"

        args = [
          "-config.file=/local/loki.yml",
          "-target=compactor",
          "-server.http-listen-address=${NOMAD_IP_http}",
          "-server.http-listen-port=${NOMAD_PORT_http}",
          "-server.grpc-listen-address=${NOMAD_IP_grpc}",
          "-server.grpc-listen-port=${NOMAD_PORT_grpc}",
        ]
      }

      template {
        data = file("./loki.yml")
        destination = "local/loki.yml"
        perms = "0644"
        splay = "30s"
      }

      restart {
        attempts = 100
      }
    }
  }
}

loki.yml

---
auth_enabled: false

ingester:
  lifecycler:
    final_sleep:  "2s"
    ring:
      replication_factor: 1
      kvstore:
        store: memberlist
  chunk_idle_period: "1m"
  chunk_retain_period: "30s"

memberlist:
  abort_if_cluster_join_fails: false
  dead_node_reclaim_time: 30s
  randomize_node_name: true
  node_name: {{env "NOMAD_ALLOC_ID"}}

  bind_port: {{env "NOMAD_PORT_memberlist"}}

  join_members: ["dnssrv+_loki-memberlist._tcp.service.consul"]
  
  max_join_backoff: 1m
  max_join_retries: 10
  min_join_backoff: 1s

storage_config:
  boltdb_shipper:
    active_index_directory: "/loki/index"
    cache_location: "/loki/index_cache"
    resync_interval: "5s"
    shared_store: s3

  aws:
    bucketnames: loki
    endpoint: minio.service.consul:9000
    access_key_id: minioadmin
    secret_access_key: minioadmin
    s3forcepathstyle: true
    insecure: true

limits_config:
  enforce_metric_name: false
  reject_old_samples: true
  reject_old_samples_max_age: "168h"

compactor:
  working_directory: /alloc/data
  shared_store: s3

frontend:
    tail_proxy_url: "dnssrv+_loki-querier._tcp.service.consul"

schema_config:
  configs:
    - from: "2021-01-26"
      store: "boltdb-shipper"
      object_store: s3
      schema: "v11"
      index:
        prefix: "index_"
        period: "24h"