shmem and mmap

mapshmem.jl

using Base.FS

# Below constants are vaid for Ubuntu. May be different for OSX
const PROT_READ  = 0x01
const PROT_WRITE = 0x02
const MAP_SHARED = 0x01

type ShmemVar
    v_name::Symbol
    v_type::Type
    v_dims::Union(Tuple, Int)
end

function map_shmem(map_list, shm_pfx)
    for i in 1:nprocs()
        remotecall_fetch(i, setup_shmem, map_list, shm_pfx)
    end
end

free_shmem(map_list::ShmemVar, shm_pfx) = unlink_shmem([map_list], shm_pfx)
free_shmem(map_list::Symbol, shm_pfx) = unlink_shmem([map_list], shm_pfx)
free_shmem(map_list, shm_pfx) = unlink_shmem(map_list, shm_pfx)

get_shmem_name(shm_pfx, v_name) = "/julia_" * shm_pfx * "_" * string(v_name)


function setup_shmem(map_list, shm_pfx)
    if isa(map_list, ShmemVar)
        map_list = [map_list]
    end
    for e in map_list
        v_name = e.v_name
        v_type = e.v_type
        v_dims = e.v_dims

        if isa(v_dims, Tuple)
            dims = prod(v_dims)
        else
            dims = v_dims
        end

        numb = dims * sizeof(v_type)

        fd_mem = ccall(:shm_open, Int, (Ptr{Uint8}, Int, Int), get_shmem_name(shm_pfx, v_name), JL_O_CREAT | JL_O_RDWR, S_IRUSR | S_IWUSR)
        if !(fd_mem > 0) error("shm_open() failed") end

        rc = ccall(:ftruncate, Int, (Int, Int), fd_mem, numb)
#        println(string(myid()) * "@ftruncate, rc : " * string(rc) * ", fd_mem : " * string(fd_mem))
        if !(rc == 0) error("ftruncate() failed") end

        x = ccall(:mmap, Ptr{Void}, (Ptr{Void}, Int, Int32, Int32, Int32, FileOffset), C_NULL, numb, PROT_READ | PROT_WRITE, MAP_SHARED, fd_mem, 0)
#        println(string(myid()) * "@mmap, x : " * string(x) * ", fd_mem : " * string(fd_mem)* ", numb : " * string(numb))
        if (x == C_NULL) error("mmap() failed") end

        eval(:(global $v_name; $v_name = pointer_to_array(pointer($v_type, $x), $v_dims)))

        ccall(:close, Int, (Int, ), fd_mem)
    end
end

function unlink_shmem(map_list, shm_pfx)
    for e in map_list
        if isa(e, Tuple)
            (v_n, _, _) = e
        else
            v_n = e
        end
        ccall(:shm_unlink, Int, (Ptr{Uint8}, ), get_shmem_name(shm_pfx, v_n))
    end
end

For example:

julia> require("mapshmem.jl")

# map tvar on all process to an Array of type Int32 .  
julia> map_shmem(ShmemVar(:tvar, Int32, 100000000), "pfx")

julia> tvar
100000000-element Int32 Array:
 0
 0
 0
 .
 .
 .

# The 'global tvar;' statement is important to be able to access the shared memory segment.
julia> @parallel for i=1:100000000
           global tvar; tvar[i] = i
        end


julia> remotecall_fetch(2, () -> tvar[67000000])
67000000

julia> remotecall_fetch(1, () -> tvar[67000000])
67000000

# This will unlink the created shared memory segment
julia> free_shmem(:tvar, "pfx")