Iterating over a directory contents with Julia and libuv's readdir

dirmap.c

#include "uv.h"

typedef void (*callback)(const char *, int, void*);

void dirmap(const char path[], callback cb, void* thunk) {
  uv_fs_t readdir_req;

  uv_fs_opendir(NULL, &readdir_req, path, NULL);

  uv_dirent_t dirent;
  uv_dir_t* rdir = readdir_req.ptr;
  rdir->dirents = &dirent;
  rdir->nentries = 1;

  while(uv_fs_readdir(NULL, &readdir_req, readdir_req.ptr, NULL) > 0) {
    (*cb)(dirent.name, dirent.type, thunk);
  }

  uv_fs_req_cleanup(&readdir_req);
  uv_fs_closedir(NULL, &readdir_req, readdir_req.ptr, NULL);
}

readdircb.jl

using BenchmarkTools

struct MyStruct
    path::String
    chan::Channel{String}
end

function dirmap(path, p, c::Ref{MyStruct})
    ccall(("dirmap", "./dirmap.so"), Cvoid, (Cstring, Ptr{Cvoid}, Ref{MyStruct}), path, p, c)
end

function dirmap(path, p)
    ccall(("dirmap", "./dirmap.so"), Cvoid, (Cstring, Ptr{Cvoid}, Ptr{Cvoid}), path, p, C_NULL)
end

function fullnamescb(cname::Cstring, type::Cint, c::MyStruct)
    name = unsafe_string(cname)
    fullname = c.path*"/"*name
    if type==2
        p = @cfunction(fullnamescb, Cvoid, (Cstring, Cint, Ref{MyStruct}))
        dirmap(fullname, p, Ref(MyStruct(fullname, c.chan)))
    else
        put!(c.chan, fullname)
    end
    nothing
end

testpath = "/tmp/testdir"
# testpath = "/tmp/testdir/c4ca4238a0b923820dcc509a6f75849b"

## Create a Channel with the complete file names
mypaths = Channel{String}() do c
    p = @cfunction(fullnamescb, Cvoid, (Cstring, Cint, Ref{MyStruct}))
    dirmap(testpath, p, Ref(MyStruct(testpath, c)))
end

for pp in Iterators.take(mypaths, 22)
    println("File $pp")
end

## Just iterate over the directory counting files
filecount = 0
function countcb(cname::Cstring, type::Cint, c)
    if type==1
        global filecount += 1
    end
    nothing
end

q = @cfunction(countcb, Cvoid, (Cstring, Cint, Ref{MyStruct}))
dirmap(testpath, q)
@show filecount

## Testing with 640000 files
#@btime dirmap(testpath, q)
# 391.338 ms (640000 allocations: 9.77 MiB)
#@btime length(readdir(testpath))
# 853.210 ms (640021 allocations: 48.06 MiB)