Apache Aurora .aurora file example for starting a Go fileserver that copy a remote swift container to a local filesystem. In this i was playing around with task combining, task concat and Sequential tasks

fileserver.aurora

# --- templates and variables ---

class Profile(Struct):
  web_version = Default(String, '0.0.5')
  rclone_version = Default(String, '0.0.9')
  parent_environment = Default(String, 'test')
  instances = Default(Integer, 2)
  cluster = Default(String, 'one')
  role = Default(String, 'test')
  environment = Required(String)
  production = Required(String)
  repository_url = Default(String, 'https://some_url/here/')
  swift_container = Default(String, 'repository')
  contact = Default(String, 'my@mail.com')
  file_server_dir = Required(String)


PROD_SWIFT_ONE = Profile(environment = 'prod', production = 'True', cluster = 'one', web_version = '0.4.1', )
TEST_SWIFT_ONE = Profile(environment = 'test', production = 'False', cluster = 'one', web_version = '0.4.2', swift_container = 'test')
DEV_SWIFT_ONE =  Profile(environment = 'dev', production = 'False', cluster = 'one', web_version = '0.4.5', swift_container = 'test')
PROD_SWIFT_TWO = Profile(environment = 'prod', production = 'True', cluster = 'two', web_version = '0.4.1')
TEST_SWIFT_TWO = Profile(environment = 'test', production = 'False', cluster = 'two', web_version = '0.4.2', swift_container = 'test')
DEV_SWIFT_TWO =  Profile(environment = 'dev', production = 'False', cluster = 'two', web_version = '0.4.5', swift_container = 'test')

# --- processes ---
fetch_http = Process(
  name = 'Get HTTP binary',
  cmdline = 'wget  https://{{profile.repository_url}}/gowebserver-{{profile.web_version}} && chmod 755 ./gowebserver-{{profile.web_version}}'
)

fetch_rclone = Process(
  name = 'Get rclone binary',
  cmdline = 'wget https://{{profile.repository_url}}/rclone-{{profile.rclone_version}} && chmod 755 ./rclone-{{profile.rclone_version}}'
)

config_rclone = Process(
  name = 'Get rclone configuration',
  cmdline = 'wget https://{{profile.repository_url}}/rcloneconf && mv rcloneconf ~/.rclone.conf && chmod 600 ~/.rclone.conf'
)

config_directory = Process(
  name = 'Setup directory',
  cmdline = 'mkdir ./{{profile.file_server_dir}} && chmod 644 ./{{profile.file_server_dir}}'
)

swift_http = Process(
  name = 'Webserver',
  cmdline = './gowebserver-{{profile.web_version}} -p {{thermos.[ports[http]}} -d {{profile.file_server_dir}}',
  daemon = True,
  logger = Logger(
    mode = LoggerMode('rotate'),
    rotate = RotatePolicy(log_size=10*MB, backups=4)
    )
)

s3_rclone = Process(
  name = 'Rclone copy S3 to local filesystem',
  cmdline = """
  while true
  do
    ./rclone --config=./.rclone.conf copy S3:{{profile.swift_container}} {{profile.file_server_dir}}
    sleep 360
  done""",
  logger = Logger(
    mode = LoggerMode('rotate'),
   rotate = RotatePolicy(log_size=10*MB, backups=4)
   )
)
# --- tasks ---

fetch_task = SequentialTask(
  processes = [fetch_http, fetch_rclone, config_rclone, config_directory],
  resources = Resources(cpu = 1.0, ram = 1024*MB, disk = 10*GB))

swiftweb_task = Task(
  processes = [swift_http],
  resources = Resources(cpu = 0.5, ram = 512*MB, disk = 1*GB))

rclone_task = Task(
  processes = [swift_rclone],
  resources = Resources(cpu = 1.0, ram = 1024*MB, disk = 250*GB)
)

combined_swiftweb = Tasks.combine(rclone_task, swiftweb_task)

complete_task = Tasks.concat(fetch_task, combined_swiftweb)

# --- Jobs ----- 

swift_template = Job(
  name = 'Swift',
  role = '{{profile.role}}',
  cluster = '{{profile.cluster}}',
  environment = '{{profile.environment}}',
  contact = '{{profile.contact}}',
  production = '{{profile.production}}',
  constraints={'host': 'limit:1'},
  service = 'True',
  instances = {{profile.instances}}, 
  task = complete_task)

# --- job ---

jobs = [
  swift_template.bind(profile = PROD_SWIFT_ONE),
  swift_template.bind(profile = PROD_SWIFT_TWO),
  swift_template.bind(profile = TEST_SWIFT_ONE),
  swift_template.bind(profile = TEST_SWIFT_TWO),
  swift_template.bind(profile = DEV_SWIFT_ONE),
  swift_template.bind(profile = DEV_SWIFT_TWO),
]