main.tf

provider "aws" {
  region = "ca-central-1"
}

provider "nomad" {
  address = data.http.nomad.url
}

variable "ssh_key_name" {
  type = string
}

data "aws_ami" "ubuntu" {
  filter {
    name   = "name"
    values = ["ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server-20240126"]
  }

  most_recent = true
  owners      = ["099720109477"] # Canonical
}

data "aws_vpc" "default" {
  default = true
}

data "aws_subnet" "default" {
  default_for_az    = true
  availability_zone = "ca-central-1a"
}

data "http" "my_ip" {
  url = "https://ipv4.icanhazip.com"
}

resource "aws_security_group" "nomad" {
  name   = "nomad-ebs-csi-test"
  vpc_id = data.aws_vpc.default.id

  ingress {
    from_port = 0
    to_port   = 0
    protocol  = "-1"
    self      = true
  }

  ingress {
    from_port   = 0
    to_port     = 0
    protocol    = "-1"
    cidr_blocks = ["${chomp(data.http.my_ip.response_body)}/32"]
  }

  ingress {
    from_port   = 4646
    to_port     = 4646
    protocol    = "tcp"
    cidr_blocks = ["${chomp(data.http.my_ip.response_body)}/32"]
  }

  ingress {
    from_port   = 22
    to_port     = 22
    protocol    = "tcp"
    cidr_blocks = ["${chomp(data.http.my_ip.response_body)}/32"]
  }

  egress {
    from_port        = 0
    to_port          = 0
    protocol         = "-1"
    cidr_blocks      = ["0.0.0.0/0"]
    ipv6_cidr_blocks = ["::/0"]
  }
}

resource "aws_instance" "nomad" {
  ami                         = data.aws_ami.ubuntu.id
  instance_type               = "t3.small"
  subnet_id                   = data.aws_subnet.default.id
  vpc_security_group_ids      = [aws_security_group.nomad.id]
  key_name                    = var.ssh_key_name
  iam_instance_profile        = aws_iam_instance_profile.nomad.id
  associate_public_ip_address = true

  user_data                   = file("${path.module}/nomad.sh")
  user_data_replace_on_change = true

  root_block_device {
    volume_size = 10
    volume_type = "gp3"
  }

  metadata_options {
    http_tokens                 = "required"
    http_put_response_hop_limit = "2"
  }
}

resource "aws_iam_instance_profile" "nomad" {
  name = "nomad-ebs-csi-test"
  role = aws_iam_role.nomad.name
}

resource "aws_iam_role" "nomad" {
  name               = "nomad-ebs-csi-test"
  assume_role_policy = data.aws_iam_policy_document.instance_assume_role_policy.json

  inline_policy {
    name = "nomad_ebc_csi_test"
    policy = jsonencode({
      Version = "2012-10-17"
      Statement = [
        {
          Action   = ["ec2:*"]
          Effect   = "Allow"
          Resource = "*"
        },
      ]
    })
  }

  path = "/"
}

data "aws_iam_policy_document" "instance_assume_role_policy" {
  statement {
    actions = ["sts:AssumeRole"]

    principals {
      type        = "Service"
      identifiers = ["ec2.amazonaws.com"]
    }
  }
}

data "http" "nomad" {
  url = "http://${aws_instance.nomad.public_ip}:4646"

  retry {
    attempts     = 300
    min_delay_ms = 1000
  }
}


resource "nomad_job" "plugin_aws_ebs_controller" {
  jobspec = file("${path.module}/plugin-aws-ebs-controller.nomad.hcl")
}

resource "nomad_job" "plugin_aws_ebs_nodes" {
  jobspec = file("${path.module}/plugin-aws-ebs-nodes.nomad.hcl")
}

data "nomad_plugin" "aws_ebs" {
  depends_on = [
    nomad_job.plugin_aws_ebs_controller,
    nomad_job.plugin_aws_ebs_nodes,
  ]

  plugin_id             = "aws-ebs"
  wait_for_healthy      = true
  wait_for_registration = true
}

resource "nomad_csi_volume" "mysql" {
  name         = "mysql"
  volume_id    = "mysql"
  capacity_min = "10G"
  plugin_id    = data.nomad_plugin.aws_ebs.plugin_id

  capability {
    access_mode     = "single-node-writer"
    attachment_mode = "file-system"
  }

  topology_request {
    required {
      topology {
        segments = {
          "topology.ebs.csi.aws.com/zone" = "ca-central-1a"
          "topology.kubernetes.io/zone"   = "ca-central-1a"
        }
      }
    }
  }
}

output "nomad_addr" {
  value = data.http.nomad.url
}

mysql.nomad.hcl

job "mysql" {
  group "mysql" {
    network {
      port "db" {
        static = 3306
      }
    }

    volume "mysql" {
      type            = "csi"
      source          = "mysql"
      access_mode     = "single-node-writer"
      attachment_mode = "file-system"
    }

    task "mysql" {
      driver = "docker"

      volume_mount {
        volume      = "mysql"
        destination = "/var/lib/mysql"
      }

      env {
        MYSQL_RANDOM_ROOT_PASSWORD = "true"
      }

      config {
        image = "mysql:8.0-debian"
        ports = ["db"]
      }

      resources {
        cpu    = 500
        memory = 1024
      }
    }
  }
}

nomad.sh

#!/bin/bash -xe

wget -O- https://apt.releases.hashicorp.com/gpg | gpg --dearmor | tee /usr/share/keyrings/hashicorp-archive-keyring.gpg
wget -O /etc/apt/keyrings/docker.asc https://download.docker.com/linux/ubuntu/gpg
chmod a+r /etc/apt/keyrings/docker.asc

echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/hashicorp.list
echo \
  "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
  $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
  tee /etc/apt/sources.list.d/docker.list > /dev/null

apt-get -qq update && apt-get -qq install -y --no-install-recommends \
    docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin \
    nomad \
    tzdata

cat <<'EOF' > /etc/nomad.d/nomad.hcl
data_dir  = "/opt/nomad/data"
bind_addr = "0.0.0.0"

server {
  enabled          = true
  bootstrap_expect = 1
}

client {
  enabled = true
}

plugin "docker" {
  config {
    allow_privileged = true
  }
}
EOF

systemctl daemon-reload
systemctl enable nomad
systemctl start nomad

plugin-aws-ebs-controller.nomad.hcl

job "plugin-aws-ebs-controller" {
  group "controller" {
    task "plugin" {
      driver = "docker"

      config {
        image = "public.ecr.aws/ebs-csi-driver/aws-ebs-csi-driver:v1.28.0"
        args = [
          "controller",
          "--endpoint=${CSI_ENDPOINT}",
          "--logtostderr",
          "--v=5",
        ]
      }

      csi_plugin {
        id        = "aws-ebs"
        type      = "controller"
        mount_dir = "/csi"
      }

      resources {
        cpu    = 500
        memory = 256
      }
    }
  }
}

plugin-aws-ebs-nodes.nomad.hcl

job "plugin-aws-ebs-nodes" {
  type = "system"

  group "nodes" {
    task "plugin" {
      driver = "docker"

      config {
        image      = "public.ecr.aws/ebs-csi-driver/aws-ebs-csi-driver:v1.28.0"
        privileged = true

        args = [
          "node",
          "--endpoint=${CSI_ENDPOINT}",
          "--logtostderr",
          "--v=5",
        ]
      }

      csi_plugin {
        id        = "aws-ebs"
        type      = "node"
        mount_dir = "/csi"
      }

      resources {
        cpu    = 500
        memory = 256
      }
    }
  }
}