Example Cromwell / WDL usage at the MGI

application.conf

webservice {
  port = 8000
  interface = 0.0.0.0
  instance.name = "reference"
}
 
akka {
  loggers = ["akka.event.slf4j.Slf4jLogger"]
  actor {
    default-dispatcher {
      fork-join-executor {
        # Number of threads = min(parallelism-factor * cpus, parallelism-max)
        # Below are the default values set by Akka, uncomment to tune these
        #parallelism-factor = 3.0
        #parallelism-max = 64
      }
    }
  }
 
  dispatchers {
    # A dispatcher for actors performing blocking io operations
    # Prevents the whole system from being slowed down when waiting for responses from external resources for instance
    io-dispatcher {
      type = Dispatcher
      executor = "fork-join-executor"
      # Using the forkjoin defaults, this can be tuned if we wish
    }
 
    # A dispatcher for actors handling API operations
    # Keeps the API responsive regardless of the load of workflows being run
    api-dispatcher {
      type = Dispatcher
      executor = "fork-join-executor"
    }
 
    # A dispatcher for engine actors
    # Because backends behaviour is unpredictable (potentially blocking, slow) the engine runs
    # on its own dispatcher to prevent backends from affecting its performance.
    engine-dispatcher {
      type = Dispatcher
      executor = "fork-join-executor"
    }
    # A dispatcher used by supported backend actors
    backend-dispatcher {
      type = Dispatcher
      executor = "fork-join-executor"
    }
    # Note that without further configuration, all other actors run on the default dispatcher
  }
}
 
spray.can {
  server {
    request-timeout = 40s
  }
  client {
    request-timeout = 40s
    connecting-timeout = 40s
  }
}
 
system {
  // If 'true', a SIGINT will trigger Cromwell to attempt to abort all currently running jobs before exiting
  abort-jobs-on-terminate = false
 
  // Max number of retries per job that the engine will attempt in case of a retryable failure received from the backend
  max-retries = 10
 
  // If 'true' then when Cromwell starts up, it tries to restart incomplete workflows
  workflow-restart = true
 
  // Cromwell will cap the number of running workflows at N
  max-concurrent-workflows = 5000
 
  // Cromwell will launch up to N submitted workflows at a time, regardless of how many open workflow slots exist
  max-workflow-launch-count = 50
 
  // Number of seconds between workflow launches
  new-workflow-poll-rate = 20
 
  // Since the WorkflowLogCopyRouter is initialized in code, this is the number of workers
  number-of-workflow-log-copy-workers = 10
}
 
workflow-options {
  // These workflow options will be encrypted when stored in the database
  encrypted-fields: []
 
  // AES-256 key to use to encrypt the values in `encrypted-fields`
  base64-encryption-key: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
 
  // Directory where to write per workflow logs
  workflow-log-dir: "cromwell-workflow-logs"
 
  // When true, per workflow logs will be deleted after copying
  workflow-log-temporary: true
 
  // Workflow-failure-mode determines what happens to other calls when a call fails. Can be either ContinueWhilePossible or NoNewCalls.
  // Can also be overridden in workflow options. Defaults to NoNewCalls. Uncomment to change:
  //workflow-failure-mode: "ContinueWhilePossible"
}
 
// Optional call-caching configuration.
call-caching {
  enabled = false
}
 
engine {
  // This instructs the engine which filesystems are at its disposal to perform any IO operation that it might need.
  // For instance, WDL variables declared at the Workflow level will be evaluated using the filesystems declared here.
  // If you intend to be able to run workflows with this kind of declarations:
  // workflow {
  //    String str = read_string("gs://bucket/my-file.txt")
  // }
  // You will need to provide the engine with a gcs filesystem
  // Note that the default filesystem (local) is always available.
  //filesystems {
  //  gcs {
  //    auth = "application-default"
  //  }
  //}
}
 
backend {
  default = "LSF"
  providers {
    Local {
      actor-factory = "cromwell.backend.impl.sfs.config.ConfigBackendLifecycleActorFactory"
      config {
        run-in-background = true
        runtime-attributes = "String? docker"
        submit = "/bin/bash ${script}"
        submit-docker = "docker run --rm -v ${cwd}:${docker_cwd} -i ${docker} /bin/bash < ${script}"
 
        // Root directory where Cromwell writes job results.  This directory must be
        // visible and writeable by the Cromwell process as well as the jobs that Cromwell
        // launches.
        root: "cromwell-executions"
        filesystems {
          local {
            // Cromwell makes a link to your input files within <root>/<workflow UUID>/workflow-inputs
            // The following are strategies used to make those links.  They are ordered.  If one fails
            // The next one is tried:
            //
            // hard-link: attempt to create a hard-link to the file
            // copy: copy the file
            // soft-link: create a symbolic link to the file
            //
            // NOTE: soft-link will be skipped for Docker jobs
            localization: [
              "hard-link", "soft-link", "copy"
            ]
          }
        }
      }
    }
 
    LSF {
      actor-factory = "cromwell.backend.impl.sfs.config.ConfigBackendLifecycleActorFactory"
      config {
        runtime-attributes = """
        Int cpu = 1
        String? memory_gb
        String? queue
        String? project
        String? docker_image
        String? resource
        String? job_group
        """
        submit = """
        bsub \
        -J ${job_name} \
        -cwd ${cwd} \
        -o ${out} \
        -e ${err} \
        ${"-a \"docker(" + docker_image + ")\""} \
        ${"-P " + project} \
        ${"-q " + queue} \
        ${"-M " + memory_gb} \
        ${"-n " + cpu} \
        ${"-R \"" + resource + "\""} \
        ${"-g \"" + job_group + "\""} \
        /bin/bash ${script}
        """
 
        kill = "bkill ${job_id}"
        check-alive = "bjobs -noheader -o \"stat\" ${job_id} | /bin/grep 'PEND\\|RUN'"
        job-id-regex = "Job <(\\d+)>.*"
 
      }
    }
  }
}
 
services {
  KeyValue {
    class = "cromwell.services.keyvalue.impl.SqlKeyValueServiceActor"
  }
 
  MetadataService {
    class = "cromwell.services.metadata.impl.MetadataServiceActor"
  }
}
 
database {
  // This specifies which database to use
  config = main.mysql
 
  main {
    mysql {
      driver = "slick.driver.MySQLDriver$"
      db {
        driver = "com.mysql.jdbc.Driver"
        url = "jdbc:mysql://localhost:3306/cromwell?socket=/tmp/mysqld.sock"
        user = "cromwell"
        password = "test4cromwell"
        connectionTimeout = 5000
      }
    }
  }
}

example.wdl

task ps {
  command {
    ps
  }
  
  runtime {
          docker_image: "ubuntu:xenial"
          cpu: "1"
          memory_gb: "4"
          queue: "research-hpc"
          resource: "rusage[gtmp=10, mem=4000]"
          job_group: '/myjobgroup/'
  }
 
  output {
    File procs = stdout()
  }
}
 
task cgrep {
  String pattern
  File in_file
  
  command {
    grep '${pattern}' ${in_file} | wc -l
  }
  
  runtime {
          docker_image: "ubuntu:xenial"
          cpu: "1"
          memory_gb: "4"
          queue: "research-hpc"
          resource: "rusage[gtmp=10, mem=4000]"
          job_group: '/myjobgroup/'
  }
 
  output {
    Int count = read_int(stdout())
  }
  
}
 
task wc {
  File in_file
  
  command {
    cat ${in_file} | wc -l
  }
  
  runtime {
          docker_image: "ubuntu:xenial"
          cpu: "1"
          memory_gb: "4"
          queue: "research-hpc"
          resource: "rusage[gtmp=10, mem=4000]"
          job_group: '/myjobgroup/'
  }
 
  output {
    Int count = read_int(stdout())
  }
  
}
 
workflow three_step {
  call ps
  call cgrep {
    input: in_file=ps.procs
  }
  call wc {
    input: in_file=ps.procs
  }
}

my.cnf

[client]
port                = 3306
socket              = /tmp/mysqld.sock
 
[mysqld_safe]
socket              = /tmp/mysqld.sock
nice                = 0
 
[mysqld]
user                = mysql
pid-file            = /path/to/test-project/mysql-directory/run/mysqld/mysqld.pid
socket              = /tmp/mysqld.sock
port                = 3306
basedir             = /usr
datadir             = /path/to/test-project/mysql-directory/lib/mysql
tmpdir              = /tmp
skip-external-locking
bind-address        = 127.0.0.1
key_buffer          = 16M
max_allowed_packet  = 16M
thread_stack        = 192K
thread_cache_size   = 8
myisam-recover      = BACKUP
query_cache_limit   = 1M
query_cache_size    = 16M
log_error           = /path/to/test-project/mysql-directory/log/mysql/error.log
expire_logs_days    = 10
max_binlog_size     = 100M
 
[mysqldump]
quick
quote-names
max_allowed_packet  = 16M
 
[mysql]
 
[isamchk]
key_buffer          = 16M
 
!includedir /etc/mysql/conf.d/

workflow.sh

#!/bin/bash
 
set -e
err_report() {
  echo "Error on line $1, error code:$?"
}
trap 'err_report $LINENO' ERR
 
 
workflow_dir=/path/to/test-project
cd $workflow_dir
 
# if you need to use a mysql database
# mysql_dir=$workflow_dir/mysql-directory
# mkdir -p $mysql_dir/run/mysqld
# mkdir -p $mysql_dir/lib/mysql
# mkdir -p $mysql_dir/log/mysql
# mysql_install_db --user=$USER --basedir=/usr/ --ldata=$mysql_dir/lib/mysql/
# mysqld_safe --defaults-file=/$mysql_dir/my.cnf &
# for i in `seq 1 28`  ; do 
#     if [ ! -e "/tmp/mysqld.sock" ] ; then 
#         sleep $i; 
#     fi; 
# done

# echo "create database cromwell; create user 'cromwell'@'localhost' identified by 'test4cromwell'; grant all privileges on *.* to 'cromwell'@localhost;" \
# | mysql -u root --socket=/tmp/mysqld.sock
 
/usr/bin/java -Dconfig.file=$workflow_dir/application.conf -jar /cromwell/cromwell.jar run $workflow_dir/example.wdl -
 
# If you setup a mysql database, make sure you properly shut it down
# /usr/bin/mysqladmin -u root --socket /tmp/mysqld.sock shutdown