Dask on Chandra slurm cluster

dask-on-slurm.py

from dask_jobqueue import SLURMCluster
from dask.distributed import Client
import dask
import dask.array as da
import subprocess
import time

# Configure the SLURM cluster
cluster = SLURMCluster(
    queue='debug',           # Replace with your Slurm queue name
    account='alankar',       # Replace with your project name if needed
    cores=16,                          # Number of cores per job
    memory='64GB',                     # Memory per job
    processes=1,                       # Number of processes per job
    walltime='01:00:00',               # Walltime per job
    # job_extra=['--constraint=your-constraint'],  # Any extra Slurm options
    interface='ib0'
)

# Scale the cluster to the desired number of jobs
cluster.scale(jobs=10)  # Adjust based on the number of nodes you want to use

# Connect to the cluster
client = Client(cluster)

# Print the dashboard URL
print("Dashboard URL:", client.dashboard_link)

array = da.random.random((10000, 10000), chunks=(1000, 1000))
result = array.mean()
print(result)
print(dask.compute(result))

dask.visualize(result, filename="computation.svg", engine="graphviz", optimize_graph=True)

client.close()