parallel_iteration.jl

struct ChanWrap
    iter_chan::RemoteChannel
end
Base.next(d::ChanWrap) = take!(d.iter_chan)


    const rchan = RemoteChannel(() -> Channel{StructCompData}(args["max_channel_size"]))
    @everywhere function produce_data(iter_chan, dbfile, nworkers, batch_size,
                                      niters; kwargs...)
        total_iters = batch_size * (niters + 2)
        niters_per_process = ceil(Int, total_iters / nworkers)
        db = MNLSocial.SQLite.DB(dbfile) # create sql file / connect to it if it exists
        MNLSocial.SQLite.@register db rand01() = rand()
        sti = MNLSocial.StructIter(db=db, queue_size=niters_per_process; kwargs...)
        for i in 1:niters_per_process
            d = next(sti)
            put!(iter_chan, d)
        end
    end
    for p in workers()
        @async remote_do(produce_data, p, rchan, SQL_FILE,
                         args["nworkers"],
                         args["batch_size"], args["niters"];
                         iter_args...)
    end
    D = ChanWrap(rchan)

#=
then a function consumes ChanWrap and computes
=#