Skip to content

Instantly share code, notes, and snippets.

@zealot128
Last active January 6, 2023 20:08
Show Gist options
  • Save zealot128/987e794cda5ca893f6845ad4d3b82859 to your computer and use it in GitHub Desktop.
Save zealot128/987e794cda5ca893f6845ad4d3b82859 to your computer and use it in GitHub Desktop.
Gitlab Autoscaling Infrastructure on Hcloud with internal caching

Order and provision a Hetzner Cloud based Gitlab-Runner Docker-machine autoscaling infrastructure

See my blog for more information.

  • Adjust settings in vars.auto.tfvars.
  • Run with terraform init && terraform apply

Content:

  • order volume + smallest HCloud VM (~3 EUR/month)
  • formats and mounts the volume on first boot
  • volume is used for Docker registry proxy and S3-Proxy for shared caching.
  • installs docker, docker-machine, and hcloud plugin for docker-machine.
  • afterwards, registers Runner at your Gitlab instance with Off-Peak feature enabled (meaning in the night time it scales down.) See cloudinit.yml:112.
  • on destroy, unregister runner and delete all running docker machine minions.
#cloud-config
groups:
- docker
users:
- name: gitlab-runner
groups: docker
apt:
sources:
docker.list:
source: 'deb [arch=amd64] https://download.docker.com/linux/ubuntu $RELEASE stable'
keyid: 0EBFCD88 # GPG key ID published on a key server
gitlab.list:
source: 'deb https://packages.gitlab.com/runner/gitlab-runner/ubuntu/ $RELEASE main'
keyid: "0x14219A96E15E78F4"
keyserver: keyserver.ubuntu.com
gitlab2.list:
keyid: "0x3CFCF9BAF27EAB47"
keyserver: keyserver.ubuntu.com
package_upgrade: true
package_update: true
packages:
- debian-archive-keyring
- apt-transport-https
- ca-certificates
- software-properties-common
- htop
- docker-ce
- golang-go
- gitlab-runner
- fail2ban
- vim
- curl
- tar
- jq
write_files:
- owner: 'root:root'
path: /root/s3_cache.sh
content: |
volume=$(ls /dev/disk/by-id/scsi-0HC_Volume*)
yes | sudo mkfs.ext4 -F $volume
mkdir /export
mount -o discard,defaults $volume /export
echo "$volume /export ext4 discard,nofail,defaults 0 0" >> /etc/fstab
systemctl daemon-reload
mkdir /export/runner
docker run -it -d --restart always -p 9005:9000 \
-v /.minio:/root/.minio -v /export:/export \
--name minio \
minio/minio:latest server /export
sleep 40
echo "export MY_IP=$(ip addr show eth0 | grep -Po 'inet \K[\d.]+')" >> ~/.profile
echo "export CACHE_S3_SERVER_ADDRESS=$(ip addr show eth0 | grep -Po 'inet \K[\d.]+'):9005" >> ~/.profile
echo "export CACHE_TYPE=s3" >> ~/.profile
echo "export CACHE_SHARED=true" >> ~/.profile
echo "export CACHE_S3_BUCKET_NAME=runner" >> ~/.profile
echo "export CACHE_S3_INSECURE=true" >> ~/.profile
echo "export CACHE_S3_ACCESS_KEY=$(cat /export/.minio.sys/config/config.json | jq -r '.credential.accessKey')" >> ~/.profile
echo "export CACHE_S3_SECRET_KEY=$(cat /export/.minio.sys/config/config.json | jq -r '.credential.secretKey')" >> ~/.profile
- owner: 'root:root'
path: /root/docker_proxy.sh
content: |
docker run -d -p 6000:5000 \
-e REGISTRY_PROXY_REMOTEURL=https://registry-1.docker.io \
-v /export/registry:/var/lib/registry \
--restart always \
--name registry registry:2
- owner: 'root:root'
path: /root/download_driver.sh
content: |
echo "Downloading docker-machine"
url=$(curl --silent https://api.github.com/repos/docker/machine/releases | jq -r '. | first | .assets[] | select(.name|contains("Linux-x86_64")).browser_download_url')
wget -q $url -O /usr/bin/docker-machine
chmod +x /usr/bin/docker-machine
echo "Downloading docker-machine-driver-hetzner"
mkdir /build
cd /build
url=$(curl --silent https://api.github.com/repos/JonasProgrammer/docker-machine-driver-hetzner/releases | jq -r '. | first | .assets[] | select(.name|contains("linux_amd64")).browser_download_url')
curl -sL $url | tar xvz
chmod +x docker-machine-driver-hetzner
mv docker-machine-driver-hetzner /usr/local/bin/
- owner: 'root:root'
path: /root/register.sh
content: |
source ~/.profile
gitlab-runner register \
--executor docker+machine \
-u ${gitlab_url} \
--run-untagged \
--tag-list ${tag_list} \
--locked=false \
--non-interactive \
-r ${registration_token} \
--docker-privileged=true \
--docker-pull-policy=if-not-present --docker-shm-size=268435456 \
--docker-volumes='/cache' \
--docker-image="${default_image}" \
--machine-idle-nodes 1 \
--machine-idle-time 1800 \
--machine-max-builds 100 \
--machine-machine-driver "hetzner" \
--machine-machine-name "runner-%s" \
--machine-machine-options "hetzner-api-token=${hcloud_token}" \
--machine-machine-options "hetzner-image=${autoscaling_os}" \
--machine-machine-options "hetzner-server-type=${autoscaling_server}" \
--machine-machine-options "engine-registry-mirror=http://$MY_IP:6000" \
--machine-off-peak-periods '* * 0-8,18-23 * * mon-fri *' \
--machine-off-peak-periods '* * * * * sat,sun *' \
--machine-off-peak-idle-count '0' \
--machine-off-peak-timezone "Europe/Berlin" \
--docker-allowed-images '*' \
--docker-allowed-images '*/*' \
--docker-allowed-images '*/*/*' \
--docker-allowed-images '${allowed_images_extra}' \
--docker-allowed-services 'redis:*' \
--docker-allowed-services 'postgres:*' \
--docker-allowed-services 'mysql:*' \
--docker-allowed-services '${allowed_images_extra}'
sed -i 's/concurrent = [0-9]\+/concurrent = ${autoscaling_concurrency}/' /etc/gitlab-runner/config.toml
runcmd:
- [/bin/bash, /root/s3_cache.sh]
- [/bin/bash, /root/docker_proxy.sh]
- [/bin/bash, /root/download_driver.sh]
- [/bin/bash, /root/register.sh]
power_state:
delay: "now"
mode: reboot
message: First reboot
condition: True
swap:
filename: /var/swap.img
size: "auto" # or size in bytes
maxsize: 4294967296
# vim: ft=yaml
variable "autoscaling_server" {
default = "cx31"
}
variable "autoscaling_os" {
default = "ubuntu-18.04"
}
variable "autoscaling_concurrency" {
default = 3
}
variable "hcloud_token" {}
provider "hcloud" {
token = "${var.hcloud_token}"
}
resource "hcloud_ssh_key" "default" {
name = "Terraform Key"
public_key = "${file("keys/id_rsa.pub")}"
}
data "template_file" "broker_cloudinit" {
template = "${file("templates/cloudinit.yml")}"
vars = {
gitlab_url = "${var.gitlab_instance_url}"
registration_token = "${var.registration_token}"
allowed_images_extra = "${var.allowed_images_extra}"
default_image = "${var.default_image}"
hcloud_token = "${var.hcloud_token}"
tag_list = "ruby"
autoscaling_os = "${var.autoscaling_os}"
autoscaling_server = "${var.autoscaling_server}"
autoscaling_concurrency = "${var.autoscaling_concurrency}"
}
}
resource "hcloud_server" "autoscaling_boss" {
count = 1
name = "hcbroker-${count.index}"
image = "ubuntu-18.04"
server_type = "cx11"
location = "nbg1"
ssh_keys = ["${hcloud_ssh_key.default.id}"]
user_data = "${data.template_file.broker_cloudinit.rendered}"
provisioner "remote-exec" {
inline = [
"docker-machine kill `docker-machine ls -f '{{.Name}}'` || true",
"yes | docker-machine rm `docker-machine ls -f '{{.Name}}'` || true",
"gitlab-runner unregister --all-runners || true"
]
when = "destroy"
connection {
type = "ssh"
user = "root"
private_key = "${file("keys/id_rsa")}"
}
}
}
resource "hcloud_volume" "cache_volume" {
name = "gitlab-runner-cache"
size = 50
server_id = "${hcloud_server.autoscaling_boss.id}"
format = "ext4"
automount = true
}
output "broker_ssh_ip" {
value = "ssh -F ssh.config root@${join(" ", hcloud_server.autoscaling_boss.*.ipv4_address)}"
}
hcloud_token = "YOUR CLOUD API TOKEN"
autoscaling_server = "cx31"
autoscaling_os = "ubuntu-18.04"
autoscaling_concurrency = 3
gitlab_instance_url = "https://git.YOURCOMPANY.com"
allowed_images_extra = "registry.YOURCOMPANY.com/developers/docker-images/*"
default_image = "registry.YOURCOMPANY.com/developers/docker-images/base:latest"
registration_token = "YOUR GITLAB REGISTRATION TOKEN"
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment