Skip to content

Instantly share code, notes, and snippets.

@lshifr
Last active December 14, 2015 14:58
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save lshifr/5104039 to your computer and use it in GitHub Desktop.
Save lshifr/5104039 to your computer and use it in GitHub Desktop.
Java code for Mathematica to read entire file of floats
JCompileLoad@"
import java.io.*;
import java.nio.ByteBuffer;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.channels.FileChannel.MapMode;
import java.util.Arrays;
public class EntireTableReader{
public static float[] getFloatTable(String filename, int rowByteCount, int rowChunkSize)
throws FileNotFoundException, IOException{
File fl = new File(filename);
FileInputStream str = new FileInputStream(fl);
FileChannel ch = str.getChannel();
MappedByteBuffer mb = ch.map( \
FileChannel.MapMode.READ_ONLY, 0L, ch.size( ));
final int buffrows = rowChunkSize;
final int buffSize = buffrows * rowByteCount;
byte[] buffer = new byte[buffSize];
int rows = (int)(fl.length()/rowByteCount);
int ncols = (int)(rowByteCount/4);
float[] result = new float[rows*ncols];
byte[] intermediate = new byte[rowByteCount*rows];
int cycles = (int)(rows/buffrows);
int remaining = rows % buffrows;
byte[] remBuffer = new byte[remaining * rowByteCount];
int ctr=0;
try{
for(int j=0;j<cycles;j++){
int bctr = 0;
mb.get(buffer);
for(int i=0;i < buffrows;i++){
System.arraycopy(buffer, bctr,intermediate,ctr,rowByteCount);
ctr+=rowByteCount;
bctr+=rowByteCount;
}
}
int bctr = 0;
mb.get(remBuffer);
for(int i=0; i < remaining;i++){
System.arraycopy(remBuffer, bctr,intermediate,ctr,rowByteCount);
ctr+=rowByteCount;
bctr+=rowByteCount;
}
ByteBuffer buf2 = ByteBuffer.wrap(intermediate);
for(int i=0;i<rows*ncols;i++){
result[i]=buf2.getFloat();
}
} finally{
str.close();
}
return result;
}
}"
(* Create a file *)
fname = "C:\\Temp\\randomdataReal";
n = 10^6;
outputstr = OpenWrite[fname, BinaryFormat -> True]
lreals = RandomReal[100, {n, 10}];
BinaryWrite[outputstr, lreals, {"Real32", "Real32", "Real32",
"Real32", "Real32", "Real32", "Real32", "Real32", "Real32",
"Real32"}, ByteOrdering -> +1]
Close[outputstr]
(* Read the data *)
read =
EntireTableReader`getFloatTable["C:\\Temp\\randomdataReal", 10*4,
1000]; // AbsoluteTiming
(* {0.3789062, Null} *)
(* Need to partition back to columns manually *)
readpart = Partition[read, 10];
(* compare - note that the results are only approximately the same *)
Chop[readpart - lreals, 10^-5] // Total
(* {0, 0, 0, 0, 0, 0, 0, 0, 0, 0} *)
(* Compare *)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment