Gitlab Spot Runners

config.toml

# https://gitlab.com/gitlab-org/gitlab-runner/issues/3021
# TODO: token cannot have an underscore
# https://aws.amazon.com/ec2/spot/pricing/
# TODO: calculate base spot pricing

concurrent = 10
check_interval = 0

[[runners]]
  name = "gitlab-aws-autoscaler-<service>" # TODO: modify here
  url = "https://gitlab.com/"
  token = "" # TODO: modify here
  executor = "docker+machine"
  limit = 20
  [runners.docker]
    image = "alpine:latest"
    privileged = true
    disable_cache = true
    volumes = ["/var/run/docker.sock:/var/run/docker.sock"]
  [runners.cache]
    Type = "s3"
    ServerAddress = "s3.amazonaws.com"
    AccessKey = "" # TODO: modify here
    SecretKey = "" # TODO: modify here
    BucketName = "-vendor-gitlab" # TODO: modify here
    BucketLocation = "us-west-2" # TODO: modify here
    Shared = true
  [runners.machine]
    IdleCount = 0
    IdleTime = 1800
    MaxBuilds = 100
    OffPeakPeriods = [
      "* * 0-8,18-23 * * mon-fri *",
      "* * * * * sat,sun *"
    ]
    OffPeakIdleCount = 0
    OffPeakIdleTime = 1200
    MachineDriver = "amazonec2"
    MachineName = "gitlab-docker-machine-%s"
    MachineOptions = [
      "amazonec2-access-key=", # TODO: modify here
      "amazonec2-secret-key=", # TODO: modify here
      "amazonec2-region=us-west-2", # TODO: modify here
      "amazonec2-vpc-id=vpc-", # TODO: modify here
      "amazonec2-subnet-id=subnet-", # TODO: modify here
      "amazonec2-use-private-address=true",
      "amazonec2-tags=runner-manager-name,gitlab-aws-autoscaler-<service>,gitlab,true,gitlab-runner-autoscale,true",
      "amazonec2-security-group=docker-machine-scaler",
      "amazonec2-instance-type=m5.xlarge",
      "amazonec2-request-spot-instance=true",
      "amazonec2-spot-price=", # TODO: modify here
      "amazonec2-block-duration-minutes=60"
    ]

ecs.tpl

Content-Type: multipart/mixed; boundary="==BOUNDARY=="
MIME-Version: 1.0

{* https://docs.gitlab.com/runner/configuration/runner_autoscale_aws/index.html *}

--==BOUNDARY==
Content-Type: text/x-shellscript; charset="us-ascii"
#!/bin/bash

curl -L https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh | sudo bash
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -

add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"

apt-cache policy docker-ce

apt-get update
apt-get install -y gitlab-runner docker-ce

curl -L https://github.com/docker/machine/releases/download/v${docker_machine_version}/docker-machine-`uname -s`-`uname -m` >/usr/local/bin/docker-machine && \
  chmod +x /usr/local/bin/docker-machine

mv docker-machine /usr/local/bin

--==BOUNDARY==--

keys.tf

# ssh-keygen -t rsa -b 4096 -C "your_email@example.com" at path defined below

variable "name" {}

resource "aws_key_pair" "key" {
  key_name = "${terraform.workspace}-${var.name}"
  public_key = "${file("../secrets/terraform/keys/${var.name}/${terraform.workspace}.pub")}"
}

# OUTPUT

output "name" {
  value = "${aws_key_pair.key.key_name}"
}

module.tf

variable "availability_zones" {
  type = "string"
  default = "us-west-2a,us-west-2b,us-west-2c" # TODO: modify if needed
}

module "ec2-key-bastion" {
  source = "./modules/aws/ec2/keys"
  name = "bastion"
}

module "security" {
  source = "./modules/aws/security"
  intra_subnets_cidr_blocks = "${join(",", module.networking.intra_subnets_cidr_blocks)}"
  private_subnets_cidr_blocks = "${join(",", module.networking.private_subnets_cidr_blocks)}"
  public_subnets_cidr_blocks = "${join(",", module.networking.public_subnets_cidr_blocks)}"
  vpc_id = "${module.networking.vpc_id}"
  vpn_eip = "${module.networking.openvpn_private_ip}"
}

module "networking" {
  source = "./modules/aws/vpc"
  zones = "${var.availability_zones}"
}

module "gitlab" {
  source = "./modules/aws/ec2/gitlab"
  instance_name = "gitab"
  instance_type = "t2.small"
  key_name = "${module.ec2-key-bastion.name}"
  security_ids = "${join(",", list(module.security.bastion_id, module.security.gitlab_id))}"
  subnet_id = "${element(module.networking.public_subnets, 0)}"
}

notes.md

https://docs.gitlab.com/runner/register/index.html#gnulinux

• Register the runner first
• registration should have resulted in config.toml, interpolate changes with next step

https://docs.gitlab.com/runner/configuration/runner_autoscale_aws/index.html

Place config.toml at /etc/gitlab-runner/config.toml and interpolate settings. Restart runner.

resource.tf

variable "instance_name" {}
variable "instance_type" {}
variable "key_name" {}
variable "security_ids" {}
variable "subnet_id" {}

data "aws_ami" "ubuntu" {
  most_recent = true

  filter {
      name   = "name"
      values = ["ubuntu/images/hvm-ssd/ubuntu-xenial-16.04-amd64-server-*"]
  }

  filter {
      name   = "virtualization-type"
      values = ["hvm"]
  }

  owners = ["099720109477"]
}

data "template_file" "user_data" {
  template = "${file("./modules/aws/ec2/gitlab/scripts/ecs.tpl")}"

  vars {
    docker_machine_version = "0.16.0"
  }
}

resource "aws_instance" "host" {
  ami = "${data.aws_ami.ubuntu.id}"
  subnet_id = "${var.subnet_id}"
  instance_type = "${var.instance_type}"
  key_name = "${var.key_name}"

  vpc_security_group_ids = ["${split(",", var.security_ids)}"]

  user_data = "${data.template_file.user_data.rendered}"
  monitoring = true

  associate_public_ip_address = true

  lifecycle {
    create_before_destroy = true
    ignore_changes = [
      "associate_public_ip_address",
      "user_data"
    ]
  }

  root_block_device {
    volume_type = "gp2"
    volume_size = 100
  }

  tags {
    Name = "${terraform.workspace}-${var.instance_name}"
  }
}

# OUTPUT

output "id" {
  value = "${aws_instance.host.id}"
}

security.tf

resource "aws_security_group" "gitlab" {
  name = "${terraform.workspace}-gitlab"
  description = "allow inbound access to gitlab"
  vpc_id = "${var.vpc_id}"

  ingress {
    protocol = "tcp"
    from_port = 2376
    to_port = 2376
    cidr_blocks = ["0.0.0.0/0"]
  }

  ingress {
    protocol = "tcp"
    from_port = 3376
    to_port = 3376
    cidr_blocks = ["0.0.0.0/0"]
  }

  egress {
    protocol = "-1"
    from_port = 0
    to_port = 0
    cidr_blocks = ["0.0.0.0/0"]
  }

  lifecycle {
    create_before_destroy = true
  }
}

resource "aws_security_group" "bastion" {
  name = "${terraform.workspace}-bastion"
  description = "allow inbound access to bastion"
  vpc_id = "${var.vpc_id}"

  ingress {
    protocol = "tcp"
    from_port = 22
    to_port = 22
    cidr_blocks = ["0.0.0.0/0"]
  }

  egress {
    protocol = "-1"
    from_port = 0
    to_port = 0
    cidr_blocks = ["0.0.0.0/0"]
  }

  lifecycle {
    create_before_destroy = true
  }
}

vpc.tf

# NOTE: 65.534k hosts
variable "cidr" {
  type = "map"
  default = {
    development = "172.16.0.0/16"
    test = "172.17.0.0/16"
    staging = "172.18.0.0/16"
    production = "172.19.0.0/16"
  }
}

# NOTE: NAT access
variable "private" {
  type = "map"
  default = {
    development = "172.16.1.0/24,172.16.2.0/24,172.16.3.0/24"
    test = "172.17.1.0/24,172.17.2.0/24,172.17.3.0/24"
    staging = "172.18.1.0/24,172.18.2.0/24,172.18.3.0/24"
    production = "172.19.1.0/24,172.19.2.0/24,172.19.3.0/24"
  }
}

# NOTE: VPN access
variable "public" {
  type = "map"
  default = {
    development = "172.16.101.0/24,172.16.102.0/24,172.16.103.0/24"
    test = "172.17.101.0/24,172.17.102.0/24,172.17.103.0/24"
    staging = "172.18.101.0/24,172.18.102.0/24,172.18.103.0/24"
    production = "172.19.101.0/24,172.19.102.0/24,172.19.103.0/24"
  }
}

# NOTE: AWS egress only
variable "intra" {
  type = "map"
  default = {
    development = "172.16.201.0/24,172.16.202.0/24,172.16.203.0/24"
    test = "172.17.201.0/24,172.17.202.0/24,172.17.203.0/24"
    staging = "172.18.201.0/24,172.18.202.0/24,172.18.203.0/24"
    production = "172.19.201.0/24,172.19.202.0/24,172.19.203.0/24"
  }
}

variable "zones" {}
variable "openvpn_instance_id" {}

resource "aws_eip" "nat" {
  count = 3
  vpc = true

  tags {
    Name = "${terraform.workspace}-nat"
  }
}

# START OPEN VPN
# resource "aws_eip" "openvpn" {
#   vpc = true

#   tags {
#     Name = "${terraform.workspace}-openvpn"
#   }
# }

# resource "aws_eip_association" "openvpn" {
#   instance_id = "${var.openvpn_instance_id}"
#   allocation_id = "${aws_eip.openvpn.id}"
# }
# END OPEN VPN

module "vpc" {
  source  = "terraform-aws-modules/vpc/aws"
  version = "1.40.0"

  cidr = "${var.cidr[terraform.workspace]}"
  azs = ["${split(",", var.zones)}"]

  enable_nat_gateway = true
  single_nat_gateway = false
  reuse_nat_ips = true
  one_nat_gateway_per_az = false

  external_nat_ip_ids = ["${aws_eip.nat.*.id}"]

  enable_vpn_gateway = true

  enable_dns_hostnames = true
  enable_dns_support = true

  enable_dhcp_options = true

  private_subnets = ["${split(",", var.private[terraform.workspace])}"]
  public_subnets = ["${split(",", var.public[terraform.workspace])}"]
  intra_subnets = ["${split(",", var.intra[terraform.workspace])}"]

  tags = { "env" = "${terraform.workspace}" }
}


# EXPORTS

output "intra_subnets" {
  value = "${module.vpc.intra_subnets}"
}

output "private_subnets" {
  value = "${module.vpc.private_subnets}"
}

output "public_subnets" {
  value = "${module.vpc.public_subnets}"
}

output "intra_subnets_cidr_blocks" {
  value = "${module.vpc.intra_subnets_cidr_blocks}"
}

output "private_subnets_cidr_blocks" {
  value = "${module.vpc.private_subnets_cidr_blocks}"
}

output "public_subnets_cidr_blocks" {
  value = "${module.vpc.public_subnets_cidr_blocks}"
}

output "vpc_id" {
  value = "${module.vpc.vpc_id}"
}

# output "openvpn_public_ip" {
#   value = "${aws_eip.openvpn.public_ip}"
# }

# output "openvpn_private_ip" {
#   value = "${aws_eip.openvpn.private_ip}"
# }