Last active
December 14, 2015 14:58
-
-
Save lshifr/5104039 to your computer and use it in GitHub Desktop.
Java code for Mathematica to read entire file of floats
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
JCompileLoad@" | |
import java.io.*; | |
import java.nio.ByteBuffer; | |
import java.nio.MappedByteBuffer; | |
import java.nio.channels.FileChannel; | |
import java.nio.channels.FileChannel.MapMode; | |
import java.util.Arrays; | |
public class EntireTableReader{ | |
public static float[] getFloatTable(String filename, int rowByteCount, int rowChunkSize) | |
throws FileNotFoundException, IOException{ | |
File fl = new File(filename); | |
FileInputStream str = new FileInputStream(fl); | |
FileChannel ch = str.getChannel(); | |
MappedByteBuffer mb = ch.map( \ | |
FileChannel.MapMode.READ_ONLY, 0L, ch.size( )); | |
final int buffrows = rowChunkSize; | |
final int buffSize = buffrows * rowByteCount; | |
byte[] buffer = new byte[buffSize]; | |
int rows = (int)(fl.length()/rowByteCount); | |
int ncols = (int)(rowByteCount/4); | |
float[] result = new float[rows*ncols]; | |
byte[] intermediate = new byte[rowByteCount*rows]; | |
int cycles = (int)(rows/buffrows); | |
int remaining = rows % buffrows; | |
byte[] remBuffer = new byte[remaining * rowByteCount]; | |
int ctr=0; | |
try{ | |
for(int j=0;j<cycles;j++){ | |
int bctr = 0; | |
mb.get(buffer); | |
for(int i=0;i < buffrows;i++){ | |
System.arraycopy(buffer, bctr,intermediate,ctr,rowByteCount); | |
ctr+=rowByteCount; | |
bctr+=rowByteCount; | |
} | |
} | |
int bctr = 0; | |
mb.get(remBuffer); | |
for(int i=0; i < remaining;i++){ | |
System.arraycopy(remBuffer, bctr,intermediate,ctr,rowByteCount); | |
ctr+=rowByteCount; | |
bctr+=rowByteCount; | |
} | |
ByteBuffer buf2 = ByteBuffer.wrap(intermediate); | |
for(int i=0;i<rows*ncols;i++){ | |
result[i]=buf2.getFloat(); | |
} | |
} finally{ | |
str.close(); | |
} | |
return result; | |
} | |
}" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
(* Create a file *) | |
fname = "C:\\Temp\\randomdataReal"; | |
n = 10^6; | |
outputstr = OpenWrite[fname, BinaryFormat -> True] | |
lreals = RandomReal[100, {n, 10}]; | |
BinaryWrite[outputstr, lreals, {"Real32", "Real32", "Real32", | |
"Real32", "Real32", "Real32", "Real32", "Real32", "Real32", | |
"Real32"}, ByteOrdering -> +1] | |
Close[outputstr] | |
(* Read the data *) | |
read = | |
EntireTableReader`getFloatTable["C:\\Temp\\randomdataReal", 10*4, | |
1000]; // AbsoluteTiming | |
(* {0.3789062, Null} *) | |
(* Need to partition back to columns manually *) | |
readpart = Partition[read, 10]; | |
(* compare - note that the results are only approximately the same *) | |
Chop[readpart - lreals, 10^-5] // Total | |
(* {0, 0, 0, 0, 0, 0, 0, 0, 0, 0} *) | |
(* Compare *) | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment