Skip to content

Instantly share code, notes, and snippets.

@jtheuer
Created April 15, 2016 15:53
Show Gist options
  • Save jtheuer/a709ef97aad4eb63bc87c57744cccc19 to your computer and use it in GitHub Desktop.
Save jtheuer/a709ef97aad4eb63bc87c57744cccc19 to your computer and use it in GitHub Desktop.
Graphhopper offheap storage with URL loader.
import com.google.common.base.Preconditions;
import com.graphhopper.storage.DAType;
import com.graphhopper.storage.DataAccess;
import com.graphhopper.util.NotThreadSafe;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import xerial.larray.LByteArray;
import xerial.larray.japi.LArrayJ;
import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
/**
* LArray-based create-read-only off-heap data access that can be created from an url rather than a file.
*
* @author jan
*/
public class OffheapUrlDataAccess implements DataAccess {
// reserve some space for downstream usage (in classes using/extending this)
private static final int HEADER_OFFSET = 20 * 4 + 20;
private static final int SEGMENT_SIZE_DEFAULT = 1 << 20;
private final URL url;
private LByteArray larray;
private int header[] = new int[(HEADER_OFFSET - 20) / 4];
private int segmentSizeInBytes = SEGMENT_SIZE_DEFAULT;
private boolean closed = false;
public OffheapUrlDataAccess(URL url) {
this.url = url;
}
@Override
public OffheapUrlDataAccess create(long bytes) {
Preconditions.checkState(larray == null, "already created");
larray = LArrayJ.newLByteArray(bytes);
return this;
}
@Override
public boolean ensureCapacity(long bytes) {
create(bytes);
return true;
}
@Override
public DataAccess copyTo(DataAccess da) {
throw new UnsupportedOperationException();
}
@Override
public boolean loadExisting() {
if(isClosed()) {
throw new IllegalStateException("already closed");
}
try(InputStream in = openStream(url)) {
byte[] headerb = new byte[HEADER_OFFSET];
int bytesRead = in.read(headerb);
Preconditions.checkState(bytesRead == HEADER_OFFSET, "unexpect amount of bytes read");
long byteCount;
try(DataInputStream ois = new DataInputStream(new ByteArrayInputStream(headerb))) {
String versionHint = ois.readUTF();
if(!"GH".equals(versionHint)) {
throw new IllegalArgumentException("Not a GraphHopper file! Expected 'GH' as file marker but was " + versionHint);
}
byteCount = ois.readLong();
setSegmentSize(ois.readInt());
for(int i = 0; i < header.length; i++) {
header[i] = ois.readInt();
}
}
if(byteCount < 0) {
return false;
}
ensureCapacity(byteCount);
long pos = 0;
int n = 0;
byte[] buffer = new byte[4096];
while(-1 != (n = in.read(buffer))) {
larray.readFromArray(buffer, 0, pos, n);
pos += n;
}
Preconditions.checkState(pos == byteCount, "Read " + pos + " bytes, but expected " + byteCount);
return true;
} catch(IOException ex) {
throw new RuntimeException("Problem while loading " + url.toExternalForm(), ex);
}
}
/**
* Overwrite this method if you need a special function to open a stream from the url
*
* @return input stream
* @throws IOException
*/
protected InputStream openStream(URL url) throws IOException {
if(logger.isDebugEnabled()) {
logger.debug("Opening " + url);
}
return url.openStream();
}
@Override
public void flush() {
}
@Override
public final void setInt(long bytePos, int value) {
throw new UnsupportedOperationException();
}
@Override
public final int getInt(long bytePos) {
return larray.getInt(bytePos);
}
@Override
public short getShort(long bytePos) {
return larray.getShort(bytePos);
}
@Override
public void setShort(long bytePos, short value) {
throw new UnsupportedOperationException();
}
@Override
public final void setBytes(long bytePos, byte[] values, int length) {
throw new UnsupportedOperationException();
}
@Override
public final void getBytes(long bytePos, byte[] values, int length) {
larray.writeToArray(bytePos, values, 0, length);
}
@Override
public final long getCapacity() {
return larray == null ? 0 : larray.length();
}
@Override
public final int getSegments() {
return (int) (getCapacity() / segmentSizeInBytes);
}
@Override
public final void trimTo(long bytes) {
throw new UnsupportedOperationException();
}
@Override
public DAType getType() {
return DAType.UNSAFE_STORE;
}
@Override
public String getName() {
return url.getFile();
}
protected String getFullName() {
return url.toExternalForm();
}
@Override
public void close() {
larray.free();
closed = true;
}
@Override
public boolean isClosed() {
return closed;
}
@Override
public void setHeader(int bytePos, int value) {
bytePos >>= 2;
header[bytePos] = value;
}
@Override
public int getHeader(int bytePos) {
bytePos >>= 2;
return header[bytePos];
}
@Override
public DataAccess setSegmentSize(int bytes) {
//ignore;
return this;
}
@Override
public int getSegmentSize() {
return segmentSizeInBytes;
}
@Override
public String toString() {
return getFullName();
}
@Override
public void rename(String newName) {
throw new UnsupportedOperationException();
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment