Skip to content

Instantly share code, notes, and snippets.

@dtrudg
Created May 2, 2019 17:05
Show Gist options
  • Save dtrudg/4458b8b9a9d3dc559caed64451f48cb5 to your computer and use it in GitHub Desktop.
Save dtrudg/4458b8b9a9d3dc559caed64451f48cb5 to your computer and use it in GitHub Desktop.
Nucleus nextflow ignite sbatch script
#!/bin/bash
#SBATCH --job-name=ignite-nf-wordcount
#SBATCH --output=nf_ignite_%j.log
# Create the ignite allocation across 4 32GB nodes
#SBATCH --partition=32GB
#SBATCH -N 4
# This creates a random cluster seed, so the ignite cluster can communicate
# with itself and not clash with other running ignite clusters on Nucleus.
export NXF_CLUSTER_SEED=$(shuf -i 0-16777216 -n 1)
module add nextflow
# We need to provide user writable locations for nextflow to initialize its
# Ignite dependencies etc. for this job.
#
# Nextflow is designed to bring down latest non-core dependencies on the fly,
# so it's better to do this than adding a fixed set into the BioHPC modules.
export NXF_ASSETS="$(pwd)/nf_ignite_${NXF_CLUSTER_SEED}/assets"
export NXF_LAUNCHER="$(pwd)/nf_ignite_${NXF_CLUSTER_SEED}/launcher"
export CAPSULE_CACHE_DIR="$(pwd)/nf_ignite_${NXF_CLUSTER_SEED}/capsule"
# Multicast cluster discovery may be unreliable with the multiple network
# interfaces and subnets in the Nucleus cluster, so we need to use a directory
# to co-ordinate the cluster.
export CLUSTER_STATE_DIR="$(pwd)/nf_ignite_${NXF_CLUSTER_SEED}/state"
mkdir -p "${CLUSTER_STATE_DIR}"
# Nextflow ignite implementation runs process at the master, as well as on the
# Ignite workers, so we need to run N-1 workers, and 1 master - rather than N
# workers and 1 master.
let NUM_WORKERS="$SLURM_JOB_NUM_NODES -1"
echo "Will run ${NUM_WORKERS} + 1 master across allocation of ${SLURM_JOB_NUM_NODES} nodes."
# Start up N-1 workers
srun --exclusive -n ${NUM_WORKERS} --cpus-per-task=${SLURM_CPUS_ON_NODE} --distribution=cyclic nextflow node -cluster.join path:${CLUSTER_STATE_DIR} &
# Start 1 master
srun --exclusive -n 1 --cpus-per-task=${SLURM_CPUS_ON_NODE} --distribution=cyclic nextflow run main.nf -process.executor ignite -cluster.join path:${CLUSTER_STATE_DIR} -with-trace -with-timeline
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment