Minimal Julia Compilation

MinimalBuild.jl

## Better way for creating standalone EXE files using Julia,
## Taken from: https://github.com/JuliaComputing/static-julia
## Assumptions:
## 1. g++ / x86_64-w64-mingw32-gcc is available and is in path
## 2. patchelf is in the path
module MinimalBuild 

# Assumption: the application main module is in MyApplication.jl
# this module must export a function called julia_main that is ccallable
appname = "MyApplication"

# Optional: load the module, the only reason we do this is to get
# MyApplication.VERSION and use it to give a nice name to the directory
# to distribute
using MyApplication 

"""
pretty print commands as we are running them
"""
function run_with_echo(cmd)
 info("running: $cmd")
 run(cmd)
end

"""
creates shared object + exe in the current directory
"""
function compile(julia_program_file, julia_install_path,
                 julia_pkgdir=joinpath(Pkg.dir(), ".."))
    filename = split(julia_program_file, ".")[1]
    O_FILE = "$(filename).o"

    SYS_LIB = joinpath(julia_install_path, "lib", "julia", "sys.$(Libdl.dlext)")
    JULIA_EXE = joinpath(julia_install_path, "bin", "julia")
    LIB_PATH = joinpath(julia_install_path, "lib")
    IMAGE_FILE = "lib$(filename).$(Libdl.dlext)"
    SO_FILE = IMAGE_FILE
    EXE_FILE = is_windows() ? "$(filename).exe" : "$(filename)"
    if is_windows()
     julia_pkgdir = replace(julia_pkgdir, "\\", "\\\\")
     julia_program_file = replace(julia_program_file, "\\", "\\\\")
    end

    # the actual compilation of julia code happens here
    run_with_echo(`"$(Base.julia_cmd())" "-J$(SYS_LIB)" "-Ccore2" "--startup-file=no" "--output-o" "$(O_FILE)" "-e" "
         vers = \""v$(VERSION.major).$(VERSION.minor)"\"
         const DIR_NAME = \"".julia"\"
         push!(Base.LOAD_CACHE_PATH, abspath(\""$julia_pkgdir"\", \""lib"\", vers))
         include(\""$(julia_program_file)"\")
         empty!(Base.LOAD_CACHE_PATH)
         "`)

    # julia-config.jl gives automatically the C and LD flags for compilation
    cflags = Base.shell_split(readstring(`$(JULIA_EXE) $(joinpath(julia_install_path, "share", "julia", "julia-config.jl")) --cflags`))
    ldflags = Base.shell_split(readstring(`$(JULIA_EXE) $(joinpath(julia_install_path, "share", "julia", "julia-config.jl")) --ldflags`))
    ldlibs = Base.shell_split(readstring(`$(JULIA_EXE) $(joinpath(julia_install_path, "share", "julia", "julia-config.jl")) --ldlibs`))

    if is_windows()
     # we need to setup a sane developping environemnt.
     # take the one from msys2.org and do pacman -S mingw-w64-x86_64  
     # (see julia/README.md for more info)
     run_with_echo(`x86_64-w64-mingw32-gcc -m64 -fPIC -Wl,-export-all-symbols -shared -o $(SO_FILE) $(O_FILE) $(ldflags) $(ldlibs)`)
     run_with_echo(`x86_64-w64-mingw32-gcc -m64 -DIMAGE_FILE=\"$(IMAGE_FILE)\" program.c -o $(EXE_FILE) -L. -Wl,-rpath,. -l$(filename) $(cflags) $(ldflags) $(ldlibs) -lopenlibm -Wl,-rpath,\$ORIGIN`)
    else
     run_with_echo(`g++ -m64 -fPIC -shared -o $(SO_FILE) $(O_FILE) $(ldflags) $(ldlibs)`)
     run_with_echo(`gcc -m64 -DIMAGE_FILE=\"$(IMAGE_FILE)\" program.c -o $(EXE_FILE)  $(cflags) $(ldflags) -L. -Wl,-rpath,. -l$(filename) $(ldlibs) -lm -Wl,-rpath,\$ORIGIN`)
    end
end#function


"""
relocate shared object and dependencies to target directory
"""
function do_compile()
 tic()
 # figure out julia install path
 target = "$(appname)_$(MyApplication.VERSION)_$(Sys.MACHINE)"
 try mkdir(target) catch warn("directory $target exists") end

 # julia install dir
 install_path = joinpath(JULIA_HOME,"..")

 # julia package dir
 julia_pkgdir=joinpath(Pkg.dir(), "..")

 # do actual compilation
 compile("$(appname).jl",install_path,julia_pkgdir)
 
 # move files to right place
 info("Moving files ...")
 SO_FILE = "lib$(appname).$(Libdl.dlext)"
 EXE_FILE = is_windows() ? "$(appname).exe" : appname
 for f in [SO_FILE,EXE_FILE]
  mv(f,joinpath(target,f),remove_destination=true)
 end

 # Copy needed shared libraries to the target directory
 if is_windows()
  paths = [JULIA_HOME]
 end
 if is_unix()
  # libjulia is in a different directory than all the other libraries
  paths = [joinpath(JULIA_HOME,"../lib/julia"), joinpath(JULIA_HOME,"../lib")]
 end
 for path in paths
  # keep files of form *.so.* or *.dylib.* or *.dll
  shlibs = filter(Regex(".*\.$(Libdl.dlext).*"),readdir(path))
  # except sys.[so,dylib,dll]
  filter!(x->!ismatch(Regex("sys.*"),x),shlibs)
  for shlib in shlibs
   cp(joinpath(path, shlib), joinpath(target, shlib), remove_destination=true)
  end
 end

 # on unix systems the generated exe and .so need to be patched in order for the
 # binary to find the libdfasim library
 if is_unix()
  info("patching exe and .so to make them relocatable")
 end
 if is_linux()
  # use patchelf
  run_with_echo(`patchelf --set-rpath \$ORIGIN/ $(joinpath(target, EXE_FILE))`)
  run_with_echo(`patchelf --set-rpath \$ORIGIN/ $(joinpath(target, SO_FILE))`)
 end
 if is_apple()
  #See http://voidnoise.co.uk/blog/?p=299 for an explanation
  run_with_echo(`install_name_tool -id @executable_path/$(SO_FILE) $target/$(SO_FILE)`)
  run_with_echo(`install_name_tool -change $(SO_FILE) @executable_path/$(SO_FILE) $target/$(appname)`)
  # to check use: 
  #   otool -D $target/libdfasim.dylib 
  #   otool -L $target/dfasim
 end

 info("Binary package is in: $target. Compilation took $(toq()) seconds")

end


end#module

program.c

// This file is a part of Julia. License is MIT: http://julialang.org/license

// Standard headers
#include <string.h>
#include <stdint.h>

// Julia headers (for initialization and gc commands)
#include "uv.h"
#include "julia.h"

// Declare C prototype of a function defined in Julia
extern int julia_main(jl_array_t*);

// main function 
int main(int argc, char *argv[])
{
    int retcode;
    int i;
    uv_setup_args(argc, argv); // no-op on Windows

    // initialization
    libsupport_init();
    // jl_options.compile_enabled = JL_OPTIONS_COMPILE_OFF;
    jl_options.julia_home = NULL;
    jl_options.image_file = IMAGE_FILE;
    julia_init(JL_IMAGE_JULIA_HOME);

    // build arguments array: `String[ unsafe_string(argv[i]) for i in 1:argc ]`
    jl_array_t *ARGS = jl_alloc_array_1d(jl_apply_array_type((jl_value_t*)jl_string_type, 1), 0);
    JL_GC_PUSH1(&ARGS);
    jl_array_grow_end(ARGS, argc - 1);
    for (i = 1; i < argc; i++) {
        jl_value_t *s = (jl_value_t*)jl_cstr_to_string(argv[i]);
        jl_arrayset(ARGS, s, i - 1);
    }
    // call the work function, and get back a value
    retcode = julia_main(ARGS);
    JL_GC_POP();

    // Cleanup and gracefully exit
    jl_atexit_hook(retcode);
    return retcode;
}

This example combines the approach in BuildExecutable.jl with that of static-julia to compile a distributable package for your julia program. We use ArgParse.jl to handle arguments. This is a usage case where (unfortunately and as of writing this) precompilation does not speed up things.

Note: static-julia has changed a lot since the last time I checked, it now is a standalone CLI utility that compiles your code and I would say makes this example obsolete. However this example automatically packages the binary file in a directory and has been tried out in linux, mac and windows with Julia 0.6.0

Executing the CLI program directly, gives unnaturally slow startup times. (Note how the command line arguments for MyApplication are given after the double dash --).

$ time /Applications/Julia-0.6.app/Contents/Resources/julia/bin/julia -e 'include("MyApplication.jl");MyApplication.julia_main(ARGS)' -- -n 0 8.3 -5.4
This is version 0.1.0. Parsed args:
  proj  =>  +proj=longlat
  utm  =>  
  y-coord  =>  -5.4
  areamin  =>  0.0
  x-coord  =>  8.3
  areamax  =>  Inf
  dist  =>  10.0
4.094u 0.299s 0:04.38 100.0%	0+0k 573+6io 139pf+0w

To get the built version, make sure your ~/.juliarc.jl file contains push!(LOAD_PATH,"."). Then start an interactive session and write:

include("MinimalBuild.jl")
MinimalBuild.do_compile()

This takes a while and creates a directory named e.g. MyApplication_0.1.0_x86_64-apple-darwin13.4.0 with the binary in there. The directory name depends on MyApplication.VERSION and on the platform Sys.MACHINE. Using this binary gives much more reasonable startup times (though still not as fast as e.g. python or C...)

$ time MyApplication_0.1.0_x86_64-apple-darwin13.4.0/MyApplication -n 0 8.3 -5.4
This is version 0.1.0. Parsed args:
  proj  =>  +proj=longlat
  utm  =>  
  y-coord  =>  -5.4
  areamin  =>  0.0
  x-coord  =>  8.3
  areamax  =>  Inf
  dist  =>  10.0
0.679u 0.196s 0:00.70 122.8%	0+0k 44+4io 5pf+0w