Prefect on Nomad

README.md

Prefect

Prefect is a workflow engine that manages the directed acyclic graph as a generated artifact from its individual nodes. It is a highly performant system that can scale across an external Dask cluster for pretty insane throughput.

Prefect does not currently have a nomad provider, so this file uses the local agent running in docker instead.

prefect-docker.nomad

job "prefect-docker" {
  datacenters = ["minicluster"]
  type = "system"

  group "agent" {
    volume "docker_socket" {
      type = "host"
      source = "docker_socket"
    }

    network {
      mode = "bridge"
    }

    restart {
      attempts = 10
    }

    service {
      port = 1

      connect {
        sidecar_service {
          proxy {
            upstreams {
              destination_name = "prefect-apollo"
              local_bind_port = 4200
            }
          }
        }
      }
    }

    task "agent" {
      driver = "docker"

      config {
        image = "prefecthq/prefect:1.1.0"
        command = "prefect"
        args = ["agent", "docker", "start", "-l", "example", "--name", "${node.unique.name}"]
      }

      env {
        PREFECT__BACKEND="server"
      }

      volume_mount {
        volume = "docker_socket"
        destination = "/run/docker.sock"
      }
    }
  }
}

prefect.nomad

job "prefect" {
  type = "service"
  datacenters = ["minicluster"]

  group "apollo" {
    count = 1

    network {
      mode = "bridge"
    }

    restart {
      attempts = 100
      delay = "30s"
      interval = "60m"
    }

    service {
      name = "prefect-apollo"
      port = 4200
      tags = [
        "traefik.enable=true",
        "traefik.consulcatalog.connect=true",
      ]

      check {
        type = "http"
        path = "/.well-known/apollo/server-health"
        interval = "15s"
        timeout = "2s"
        expose = true
      }

      connect {
        sidecar_service {
          proxy {
            upstreams {
              local_bind_port = 4201
              destination_name = "prefect-graphql" 
            }
            upstreams {
              local_bind_port = 3000
              destination_name = "prefect-hasura"
            }
          }
        }
      }
    }

    task "apollo" {
      driver = "docker"

      config {
        image = "prefecthq/apollo:core-1.1.0"
        command = "bash"
        args = ["-c", "./post-start.sh && npm run serve"]
      }

      env {
        GRAPHQL_SERVICE_HOST="127.0.0.1"
        GRAPHQL_SERVICE_PORT="4201"
        HASURA_API_URL="http://127.0.0.1:3000/v1alpha1/graphql"
        PREFECT_API_HEALTH_URL="http://127.0.0.1:4201/health"
        PREFECT_SERVER__TELEMETRY__ENABLED="false"
        PREFECT_SERVER__SERVICES__HOST="127.0.0.1"
      }
    }
  }
  group "graphql" {
    count = 1
    network {
      mode = "bridge"
    }

    restart {
      attempts = 100
      delay = "30s"
      interval = "60m"
    }

    service {
      name = "prefect-graphql"
      port = 4201

      check {
        type = "http"
        path = "/health"
        interval = "15s"
        timeout = "2s"
        expose = true
      }

      connect {
        sidecar_service {
          proxy {
            upstreams {
              destination_name = "prefect-hasura"
              local_bind_port = 3000
            }
            upstreams {
              destination_name = "prefect-postgres"
              local_bind_port = 5432
            }
          }
        }
      }
    }

    task "graphql" {
      driver = "docker"

      config {
        image = "prefecthq/server:core-1.1.0"
        command = "bash"
        args = ["-c", "prefect-server database upgrade -y && python src/prefect_server/services/graphql/server.py"]
      }

      env {
        PREFECT_CORE_VERSION="1.1.0"
        PREFECT_SERVER_DB_CMD="prefect-server database upgrade -y"
        PREFECT_SERVER__DATABASE__CONNECTION_URL="postgresql://prefect:test-password@127.0.0.1:5432/prefect_server"
        PREFECT_SERVER__HASURA__ADMIN_SECRET="hasura-secret-admin-secret"
        PREFECT_SERVER__HASURA__HOST="127.0.0.1"
      }
    }
  }

  group "hasura" {
    count = 1

    network {
      mode = "bridge"
    }

    restart {
      attempts = 100
      delay = "30s"
      interval = "60m"
    }

    service {
      name = "prefect-hasura"
      port = 3000

      check {
        type = "http"
        path = "/healthz"
        interval = "15s"
        timeout = "2s"
        expose = true
      }

      connect {
        sidecar_service {
          proxy {
            upstreams {
              destination_name = "prefect-postgres"
              local_bind_port = 5432
            }
          }
        }
      }
    }
    
    task "hasura" {
      driver = "docker"

      config {
        image = "hasura/graphql-engine:v2.1.1"
        command = "graphql-engine"
        args = ["serve"]
      }

      env {
        HASURA_GRAPHQL_DATABASE_URL="postgresql://prefect:test-password@127.0.0.1:5432/prefect_server"
        HASURA_GRAPHQL_ENABLE_CONSOLE="true"
        HASURA_GRAPHQL_LOG_LEVEL="warn"
        HASURA_GRAPHQL_QUERY_PLAN_CACHE_SIZE="100"
        HASURA_GRAPHQL_SERVER_PORT="3000"
        HASURA_GRAPHQL_V1_BOOLEAN_NULL_COLLAPSE="true"
      }
    }
  }
  group "towel" {
    count = 1

    network {
      mode = "bridge"
    }

    service {
      name = "prefect-towel"
      port = 1

      connect {
        sidecar_service {
          proxy {
            upstreams {
              local_bind_port = 3000
              destination_name = "prefect-hasura"
            }
          }
        }
      }
    }
    
    task "towel" {
      driver = "docker"

      config {
        image = "prefecthq/server:core-1.1.0"
        command = "python"
        args = ["src/prefect_server/services/towel/__main__.py"]
      }

      env {
        PREFECT_SERVER__HASURA__ADMIN_SECRET="hasura-secret-admin-secret"
        PREFECT_SERVER__HASURA__HOST="127.0.0.1"
      }
    }
  }

  group "ui" {
    count = 1

    network {
      mode = "bridge"
    }

    service {
      name = "prefect-ui"
      port = 8080
      tags = [
        "traefik.enable=true",
        "traefik.consulcatalog.connect=true",
      ]

      check {
        type = "http"
        path = "/"
        interval = "30s"
        timeout = "2s"
        expose = true
      }

      connect {
        sidecar_service {
          proxy {
            upstreams {
              destination_name = "prefect-apollo"
              local_bind_port = 4200
            }
          }
        }
      }
    }
    
    task "ui" {
      driver = "docker"

      config {
        image = "prefecthq/ui:core-1.1.0"
        command = "/intercept.sh"
      }

      env {
        PREFECT_SERVER__APOLLO_URL="http://prefect-apollo.mc/graphql"
      }
    }
  }

  group "postgres" {
    count = 1

    network {
      mode = "bridge"
    }

    service {
      name = "prefect-postgres"
      port = 5432

      connect {
        sidecar_service {}
      }
    }
    
    task "postgres" {
      driver = "docker"

      config {
        image = "postgres:11"
        args = ["postgres", "-c", "max_connections=150"]
      }
      env {
        POSTGRES_DB="prefect_server"
        POSTGRES_PASSWORD="test-password"
        POSTGRES_USER="prefect"
      }
    }
  }
}