Graphhopper offheap storage with URL loader.

OffheapUrlDataAccess.java

import com.google.common.base.Preconditions;
import com.graphhopper.storage.DAType;
import com.graphhopper.storage.DataAccess;
import com.graphhopper.util.NotThreadSafe;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import xerial.larray.LByteArray;
import xerial.larray.japi.LArrayJ;

import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;

/**
 * LArray-based create-read-only off-heap data access that can be created from an url rather than a file.
 *
 * @author jan
 */
public class OffheapUrlDataAccess implements DataAccess {

	// reserve some space for downstream usage (in classes using/extending this)
	private static final int HEADER_OFFSET = 20 * 4 + 20;
	private static final int SEGMENT_SIZE_DEFAULT = 1 << 20;

	private final URL url;
	private LByteArray larray;

	private int header[] = new int[(HEADER_OFFSET - 20) / 4];
	private int segmentSizeInBytes = SEGMENT_SIZE_DEFAULT;
	private boolean closed = false;

	public OffheapUrlDataAccess(URL url) {
		this.url = url;
	}

	@Override
	public OffheapUrlDataAccess create(long bytes) {
		Preconditions.checkState(larray == null, "already created");
		larray = LArrayJ.newLByteArray(bytes);
		return this;
	}

	@Override
	public boolean ensureCapacity(long bytes) {
		create(bytes);
		return true;
	}

	@Override
	public DataAccess copyTo(DataAccess da) {
		throw new UnsupportedOperationException();
	}

	@Override
	public boolean loadExisting() {
		if(isClosed()) {
			throw new IllegalStateException("already closed");
		}

		try(InputStream in = openStream(url)) {
			byte[] headerb = new byte[HEADER_OFFSET];
			int bytesRead = in.read(headerb);
			Preconditions.checkState(bytesRead == HEADER_OFFSET, "unexpect amount of bytes read");

			long byteCount;
			try(DataInputStream ois = new DataInputStream(new ByteArrayInputStream(headerb))) {
				String versionHint = ois.readUTF();
				if(!"GH".equals(versionHint)) {
					throw new IllegalArgumentException("Not a GraphHopper file! Expected 'GH' as file marker but was " + versionHint);
				}

				byteCount = ois.readLong();
				setSegmentSize(ois.readInt());
				for(int i = 0; i < header.length; i++) {
					header[i] = ois.readInt();
				}
			}

			if(byteCount < 0) {
				return false;
			}
			ensureCapacity(byteCount);

			long pos = 0;
			int n = 0;
			byte[] buffer = new byte[4096];
			while(-1 != (n = in.read(buffer))) {
				larray.readFromArray(buffer, 0, pos, n);
				pos += n;
			}
			Preconditions.checkState(pos == byteCount, "Read " + pos + " bytes, but expected " + byteCount);

			return true;
		} catch(IOException ex) {
			throw new RuntimeException("Problem while loading " + url.toExternalForm(), ex);
		}
	}

	/**
	 * Overwrite this method if you need a special function to open a stream from the url
	 *
	 * @return input stream
	 * @throws IOException
	 */
	protected InputStream openStream(URL url) throws IOException {
		if(logger.isDebugEnabled()) {
			logger.debug("Opening " + url);
		}
		return url.openStream();
	}

	@Override
	public void flush() {

	}

	@Override
	public final void setInt(long bytePos, int value) {
		throw new UnsupportedOperationException();
	}

	@Override
	public final int getInt(long bytePos) {
		return larray.getInt(bytePos);
	}

	@Override
	public short getShort(long bytePos) {
		return larray.getShort(bytePos);
	}

	@Override
	public void setShort(long bytePos, short value) {
		throw new UnsupportedOperationException();
	}

	@Override
	public final void setBytes(long bytePos, byte[] values, int length) {
		throw new UnsupportedOperationException();
	}

	@Override
	public final void getBytes(long bytePos, byte[] values, int length) {
		larray.writeToArray(bytePos, values, 0, length);
	}

	@Override
	public final long getCapacity() {
		return larray == null ? 0 : larray.length();
	}

	@Override
	public final int getSegments() {
		return (int) (getCapacity() / segmentSizeInBytes);
	}

	@Override
	public final void trimTo(long bytes) {
		throw new UnsupportedOperationException();
	}

	@Override
	public DAType getType() {
		return DAType.UNSAFE_STORE;
	}

	@Override
	public String getName() {
		return url.getFile();
	}

	protected String getFullName() {
		return url.toExternalForm();
	}

	@Override
	public void close() {
		larray.free();
		closed = true;
	}

	@Override
	public boolean isClosed() {
		return closed;
	}

	@Override
	public void setHeader(int bytePos, int value) {
		bytePos >>= 2;
		header[bytePos] = value;
	}

	@Override
	public int getHeader(int bytePos) {
		bytePos >>= 2;
		return header[bytePos];
	}

	@Override
	public DataAccess setSegmentSize(int bytes) {
		//ignore;
		return this;
	}

	@Override
	public int getSegmentSize() {
		return segmentSizeInBytes;
	}

	@Override
	public String toString() {
		return getFullName();
	}

	@Override
	public void rename(String newName) {
		throw new UnsupportedOperationException();
	}
}