SCOOP test code for SLURM

get_slurm_nodelist.py

"""
   Copyright 2018 Islam Elnabarawy

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
"""
import argparse
import re

__author__ = "Islam Elnabarawy"


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('node_list')
    parser.add_argument('tasks_per_node')
    args = parser.parse_args()

    node_list = args.node_list
    tasks_per_node = args.tasks_per_node

    print('node_list:', node_list)
    print('tasks_per_node:', tasks_per_node)

    node_list = list(filter(None, node_list.split()))

    rep_spec = []
    for spec in filter(None, tasks_per_node.split(',')):
        if spec.isnumeric():
            rep_spec.append(int(spec))
        else:
            m = re.match('(\d+)\(x(\d+)\)', spec)
            assert m, "Invalid tasks_per_node specification."
            rep_spec.extend([int(m[1])] * int(m[2]))

    assert len(node_list) == len(rep_spec), "Number of nodes in node_list does not match the given tasks_per_node " \
                                            "specification. "
    hosts = list(zip(node_list, rep_spec))
    for host in hosts:
        print(host)


if __name__ == '__main__':
    main()

multitask.sub

#!/bin/bash
#SBATCH --job-name=test-multitasks
#SBATCH --ntasks=128
#SBATCH --mem=512
#SBATCH --time=1:0:0
#SBATCH --export=all
#SBATCH --out=test-multitasks-%j.out

export SLURM_JOB_HOSTLIST="$(scontrol show hostnames ${SLURM_JOB_NODELIST})"

env

scoop-sdp.submit

#!/bin/bash
#SBATCH --job-name=scoop
#SBATCH --ntasks=64
#SBATCH --mem=512
#SBATCH --time=1:0:0
#SBATCH --export=all
#SBATCH --out=scoop-%j.out

env > scoop-${SLURM_JOB_ID}.log

scontrol show JobID=${SLURM_JOB_ID}

python -m scoop -vvv --prolog $HOME/usesdp.sh scoop-test.py

scoop-test.py

from __future__ import print_function

import time

import scoop


def helloWorld(value):
    time.sleep(10)
    return "[{1}] Hello World from Future #{0}".format(value, scoop.worker)


if __name__ == "__main__":
    returnValues = list(scoop.futures.map(helloWorld, range(64)))
    print("\n".join(returnValues))