Julia parallelism with JuliaCall

parallelism_with_juliacall.py

from juliacall import Main as jl
import time
jl.include("utils.jl")

def wait_for_spectrum(s):
    while True:
        time.sleep(0.1)
        if jl.isready(s):
            print("Worker is ready!")
            print(jl.fetch(s))
            break
        else:
            print("Worker is not ready yet...")

# start two worker processes 
s1 = jl.generate_spectrum_on_worker(5777.0, 4.44)
s2 = jl.generate_spectrum_on_worker(5777.0, 4.44)

wait_for_spectrum(s1)
wait_for_spectrum(s2)

utils.jl

using Distributed

# Start a worker process for each thread
addprocs(Sys.CPU_THREADS)

@everywhere begin
    using Korg

    function generate_spectrum(teff, logg)
        println("interpolating atmosphere")
        atm = Korg.interpolate_marcs(teff, logg)

        println("synthesizing")
        sol = synthesize(atm, [], format_A_X(), 5000, 5050)

        println("done!")
        # only return 10 pixels to keep stdout tidy
        sol.flux[1:10]
    end
end

function generate_spectrum_on_worker(teff, logg)
    @spawnat :any generate_spectrum(teff, logg)
end