Julia parser and writer for the IDX file format used by e.g. the MNIST dataset

idx_io.jl

const idx_types = [(UInt8, 0x08), (Int8, 0x09), (Int16, 0x0B), (Int32, 0x0C), (Float32, 0x0D), (Float64, 0x0E)]
const typefromcode = Dict(idx_types)
const codefromtype = Dict(map(reverse, idx_types))

function write_idx(stream::Union{IOStream,String}, data::Array{T,N}) where {T,N}
  dims = convert(Array{Int32}, collect(size(data)))
  write(stream, 0x00, 0x00, typefromcode[T], UInt8(N), hton.(dims), hton.(data))
end

read_idx(filename::String) = open(read_idx, filename)

function read_idx(stream::IOStream)
  header = read(stream, 4)
  dims = ntoh.(read!(stream, Array{Int32}(undef, header[4])))
  data = ntoh.(read!(stream, Array{codefromtype[header[3]]}(undef, dims...)))
end