run_with_timeout.jl

"""
    run_with_timeout(
        command; log=stdout, timeout = 5*60, name = "",
        wait_time = 1, verbose = true
    )
Runs `command` and pipes all output to `log`. The process will be terminated after
`timeout` seconds without any output. `name` describes the process in log messages,
and `verbose` determines whether meta-logs ("process started" etc.) will be printed.
"""
function run_with_timeout(
        command; log=stdout, timeout = 5*60, name = "",
        wait_time = 1, verbose = true
    )

    out_io = IOBuffer()
    err_io = IOBuffer()
    pipe = pipeline(command, stdout = out_io, stderr = err_io)
    process = run(pipe, wait = false)
    timeout_start = time()

task = @async begin
    logfallback = false
    io = try
        log isa String ? open(log, "w") : log
    catch err
        @error "Error opening logfile, falling back to stdout" error=err
        logfallback = true
        stdout
    end
    try
        tstart = time()
        verbose && @info("starting $name")
        while process_running(process)
            elapsed = (time() - timeout_start)
            if elapsed > timeout
                verbose && @info("killing $name")
                kill(process)
                break
            end

            errstr, outstr = String.(take!.((err_io, out_io)))
            is_silent = length(errstr) == 0 && length(outstr) == 0
            isempty(outstr) || println(io, outstr)
            isempty(errstr) || println(io, errstr)

            # if something printed reset timeout
            if !is_silent
                timeout_start = time()
            end

            sleep(wait_time)
        end

        verbose && @info("$name completed in $(round(time() - tstart, digits=1)) seconds")
    catch err
        @error "Error while running $(name) with timeout." error=err
    finally
        errstr, outstr = String.(take!.((err_io, out_io)))
        isempty(outstr) || println(io, outstr)
        isempty(errstr) || println(io, errstr)

        flush(io)
        if log isa String && !logfallback
            close(io)
        end
    end
end
return process, task
end