Asynchronous submission of jobs using addprocs_qrsh

example_qrsh.jl

#!/usr/bin/env julia

using ClusterManagers

function submit_task()
    # Complex task goes here
    function mytask(x)
        return sinint(x)
    end

    # Redirections prevent addprocs from bombaring us with qsub info
    # Must be *inside* this function (no clue why)
    redirect_stdout()
    redirect_stderr()

    # Non-blocking
    qrsh = QRSHManager(1, "");
    Distributed.cluster_mgmt_from_master_check()
    id = Distributed.addprocs_locked(qrsh; qsub_env="")
    
    # Blocking
    #id = addprocs_qrsh(1)

    # This executes "@everywhere using SpecialFunctions"
    eval(macroexpand(quote @everywhere using SpecialFunctions end))

    # pmap is a blocking call to run functions and retrieve their results
    # It returns an array of outputs for each processor in the list
    dat = pmap(x -> mytask(id[]), id)[1];

    return dat
end

println("Running jobs...");
old_stdout = STDOUT
old_stderr = STDERR
jobs = 2
results = []
for i=1:jobs
    @async push!(results, submit_task());
end

# Keep checking if all results are done
runtime = @elapsed while length(results) != jobs
    sleep(0.1) # This is essential (also don't know why)
end
redirect_stdout(old_stdout)
redirect_stderr(old_stderr)

runtime_fmt = Libc.strftime("%T", runtime + 86400 * 365.25 * 4)
println("Done! Results: $results")
println("      Runtime: $runtime_fmt")