NFS tests in Nomad

01-nomad-nfs-tests.md

NFS volume on Nomad

NFS server

Host: nfs-server.local.example.com (10.0.10.1)

sudo apt install nfs-kernel-server

sudo systemctl status nfs-kernel-server
sudo systemctl start nfs-kernel-server

sudo mkdir -pv /srv/nfs/nomad

cat /etc/exports
# Output:
# /srv/nfs/nomad  10.0.10.0/24(rw,sync,no_subtree_check,no_root_squash)

sudo exportfs -a -v
sudo exportfs -s

NFS client

Just for testing the NFS server, not required for Nomad.

Host: nomad-client-01 (10.0.10.10)

sudo apt install nfs-common

sudo mkdir -pv /var/opt/nfs-tests

sudo mount -v nfs-server.local.example.com:/srv/nfs/nomad /var/opt/nfs-tests

sudo touch /var/opt/nfs-tests/test1.txt
sudo chown -v ubuntu: /var/opt/nfs-tests/test1.txt
sudo chown -v consul: /var/opt/nfs-tests/test1.txt

Nomad

Note

Docker plugin's allow_privileged option must be enabled in Nomad clients before proceeding.

Plugin controller and node:

nomad namespace apply 'plugins-csi-nfs'

nomad job plan controller.nomad
nomad job run controller.nomad

nomad job plan nodes.nomad
nomad job run nodes.nomad

nomad plugin status
nomad plugin status nfs

Volume:

nomad namespace apply 'nginx'

nomad volume register nginx-data.nomad
nomad volume status

NOMAD_NAMESPACE=nginx nomad volume deregister nginx-data

Job:

nomad job plan nginx.nomad
nomad job run nginx.nomad

References

• https://support.hashicorp.com/hc/en-us/articles/22557185128083-Nomad-NFS-CSI-Volume
• https://developer.hashicorp.com/nomad/docs/drivers/docker#allow_privileged
• https://developer.hashicorp.com/nomad/docs/job-specification/csi_plugin
• https://developer.hashicorp.com/nomad/docs/job-specification/volume

02-csi-nfs-controller.nomad

job "plugin-nfs-controller" {
  region      = "mga1"
  datacenters = ["lab1"]

  namespace = "plugins-csi-nfs"
  type      = "service"

  group "controller" {
    task "plugin" {
      driver = "docker"

      config {
        image = "mcr.microsoft.com/k8s/csi/nfs-csi:latest"
        args = [
          "--endpoint=unix://csi/csi.sock",
          "--nodeid=${attr.unique.hostname}",
          "--logtostderr",
          "-v=5",
        ]
      }

      csi_plugin {
        id        = "nfs"
        type      = "controller"
        mount_dir = "/csi"
      }

      resources {
        cpu    = 250
        memory = 128
      }
    }
  }
}

03-csi-nfs-nodes.nomad

job "plugin-nfs-nodes" {
  region      = "mga1"
  datacenters = ["lab1"]

  namespace = "plugins-csi-nfs"
  type      = "system"

  group "nodes" {
    task "plugin" {
      driver = "docker"

      config {
        image = "mcr.microsoft.com/k8s/csi/nfs-csi:latest"
        args = [
          "--endpoint=unix://csi/csi.sock",
          "--nodeid=${attr.unique.hostname}",
          "--logtostderr",
          "--v=5",
        ]
        privileged = true
      }

      csi_plugin {
        id        = "nfs"
        type      = "node"
        mount_dir = "/csi"
      }

      resources {
        cpu    = 250
        memory = 128
      }
    }
  }
}

04-volume-nginx-data.nomad

namespace = "nginx"
name      = "nginx-data"
id        = "nginx-data"

type      = "csi"
plugin_id = "nfs"

capability {
  access_mode     = "multi-node-multi-writer"
  attachment_mode = "file-system"
}

capability {
  access_mode     = "single-node-writer"
  attachment_mode = "file-system"
}

context {
  server = "nfs-server.local.example.com"
  share  = "/srv/nfs/nomad"
}

mount_options {
  fs_type = "nfs"
}

05-job-nginx.nomad

job "nginx" {
  region      = "mga1"
  datacenters = ["lab1"]

  namespace = "nginx"
  type      = "service"

  update {
    max_parallel = 1
    stagger      = "30s"
  }

  group "nginx" {
    count = 1

    volume "data-dir" {
      type            = "csi"
      source          = "nginx-data"
      read_only       = false
      access_mode     = "multi-node-multi-writer"
      attachment_mode = "file-system"
    }

    network {
      port "http" {
        to = 80
      }
    }

    service {
      name = "nginx"
      port = "http"

      tags = [
        "traefik.enable=true",
        "traefik.http.routers.nginx.entrypoints=apps",
        "traefik.http.routers.nginx.rule=Host(`nginx.local.example.com`)",
      ]

      check {
        type     = "http"
        port     = "http"
        method   = "HEAD"
        path     = "/"
        interval = "5s"
        timeout  = "2s"
      }
    }

    task "nginx" {
      driver = "docker"

      config {
        image      = "nginx:stable-alpine"
        force_pull = true
        ports      = ["http"]
      }

      volume_mount {
        volume      = "data-dir"
        destination = "/data"
      }

      template {
        data = <<-EOT
          {{ range nomadVarList -}}
          {{ with nomadVar .Path -}}
          {{ range $k, $v := . -}}
          {{ $k }}="{{ $v }}"
          {{ end -}}
          {{ end -}}
          {{ end -}}
        EOT

        destination = "${NOMAD_SECRETS_DIR}/.secrets"
        env         = true
      }

      resources {
        cpu    = 100
        memory = 128
      }
    }
  }
}