Skip to content

Instantly share code, notes, and snippets.

@vmarcinko
Created January 21, 2015 16:03
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save vmarcinko/217c36115c975c8c464e to your computer and use it in GitHub Desktop.
Save vmarcinko/217c36115c975c8c464e to your computer and use it in GitHub Desktop.
Nanocube DMP encoder in java
public class CategoryEnumInfo {
private final String label;
private final Byte encodedValue;
public CategoryEnumInfo(String label, Byte encodedValue) {
this.label = label;
this.encodedValue = encodedValue;
}
public String getLabel() {
return label;
}
public Byte getEncodedValue() {
return encodedValue;
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
CategoryEnumInfo that = (CategoryEnumInfo) o;
if (!label.equals(that.label)) {
return false;
}
if (!encodedValue.equals(that.encodedValue)) {
return false;
}
return true;
}
@Override
public int hashCode() {
int result = label.hashCode();
result = 31 * result + encodedValue.hashCode();
return result;
}
@Override
public String toString() {
final StringBuilder sb = new StringBuilder("CategoryEnumInfo{");
sb.append("label='").append(label).append('\'');
sb.append(", encodedValue=").append(encodedValue);
sb.append('}');
return sb.toString();
}
}
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.charset.Charset;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
public class NanocubeDmpEncoder {
private static final Charset charset = Charset.forName("UTF8");
private final String name;
private final int locationZoom;
private final Date timeOffset;
private final int timeBinInSecs;
private final int timeByteLength;
private final Map<String, Map<Object, CategoryEnumInfo>> categoryEnumerations;
private final int recordLength;
public NanocubeDmpEncoder(
String name,
int locationZoom,
Date timeOffset,
int timeBinInSecs,
int timeByteLength,
Map<String, Map<Object, CategoryEnumInfo>> categoryEnumerations
) {
if (name == null) {
throw new IllegalArgumentException("Name is null");
}
if (locationZoom < 1) {
throw new IllegalArgumentException("Location zoom cannot be less than 1, but is: " + locationZoom);
}
if (timeOffset == null) {
throw new IllegalArgumentException("TimeOffset is null");
}
if (timeBinInSecs < 1) {
throw new IllegalArgumentException("Time bin cannot be less than 1 second, but is: " + timeBinInSecs);
}
if (timeByteLength != 2 && timeByteLength != 4 && timeByteLength != 8) {
throw new IllegalArgumentException("Time byte length must be 2, 4 or 8, but is " + timeByteLength);
}
this.name = name;
this.locationZoom = locationZoom;
this.timeOffset = timeOffset;
this.timeBinInSecs = timeBinInSecs;
this.timeByteLength = timeByteLength;
this.categoryEnumerations = new HashMap<>(categoryEnumerations); // defensive copying
this.recordLength = calculateRecordLength(timeByteLength, categoryEnumerations.size());
}
private int calculateRecordLength(int timeByteLength, int categoryCount) {
return 12 + categoryCount + timeByteLength;
}
public int getRecordLength() {
return recordLength;
}
public byte[] encodeHeaders() throws IOException {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
encodeString(baos, "name: " + name);
encodeString(baos, "encoding: binary");
encodeString(baos, "metadata: location__origin degrees_mercator_quadtree" + locationZoom);
encodeString(baos, "field: location nc_dim_quadtree_" + locationZoom);
for (Map.Entry<String, Map<Object, CategoryEnumInfo>> entry : categoryEnumerations.entrySet()) {
String categoryName = entry.getKey();
Map<Object, CategoryEnumInfo> enumerations = entry.getValue();
encodeString(baos, "field: " + categoryName + " nc_dim_cat_1");
for (CategoryEnumInfo enumInfo : enumerations.values()) {
encodeString(baos, "valname: " + categoryName + " " + enumInfo.getEncodedValue() + " " + enumInfo.getLabel());
}
}
SimpleDateFormat timeOffsetFormat = new SimpleDateFormat("yyyy-MM-dd_HH:mm:ss");
String formattedTimeOffset = timeOffsetFormat.format(timeOffset);
encodeString(baos, "metadata: tbin " + formattedTimeOffset + "_" + timeBinInSecs + "s");
encodeString(baos, "field: Time nc_dim_time_" + timeByteLength);
encodeString(baos, "field: count nc_var_uint_4");
encodeString(baos, "");
return baos.toByteArray();
}
public byte[] encodeRecord(double latitude, double longitude, Date time, int count, Map<String, Object> categoryValues) throws IOException {
if (time == null || time.before(timeOffset)) {
throw new IllegalArgumentException("Time must be equal or after configured time offset (" + timeOffset + "), but is: " + time);
}
ByteBuffer byteBuffer = ByteBuffer.allocate(recordLength);
byteBuffer.order(ByteOrder.LITTLE_ENDIAN);
// location
int lonValue = lonToTileX(longitude, locationZoom);
byteBuffer.putInt(lonValue);
int latValue = latToTileY(latitude, locationZoom);
byteBuffer.putInt(latValue);
// category values
for (Map.Entry<String, Map<Object, CategoryEnumInfo>> entry : categoryEnumerations.entrySet()) {
String categoryName = entry.getKey();
Map<Object, CategoryEnumInfo> enumerations = entry.getValue();
Object categoryValue = categoryValues.get(categoryName);
CategoryEnumInfo enumInfo = enumerations.get(categoryValue);
byteBuffer.put(enumInfo.getEncodedValue());
}
// time
long binIndex = timeToBinIndex(time);
switch (timeByteLength) {
case 2:
byteBuffer.putShort((short) binIndex);
break;
case 4:
byteBuffer.putInt((int) binIndex);
break;
default:
byteBuffer.putLong(binIndex);
break;
}
// count
byteBuffer.putInt(count);
return byteBuffer.array();
}
private long timeToBinIndex(Date time) {
long ageSinceOffsetInMillis = time.getTime() - timeOffset.getTime();
long timeBinInMillis = 1000 * timeBinInSecs;
long ageInBins = (int) (ageSinceOffsetInMillis / timeBinInMillis);
long maxValue = getTimeMaxValue();
if (ageInBins > maxValue) {
throw new IllegalArgumentException("Invalid time: " + time + "; period since time offset (" + timeOffset + ") calculated in time bins (" + ageInBins + " of " + timeBinInSecs + "s bins) exceeds maximum " + timeByteLength + " byte integer value: " + maxValue);
}
return ageInBins;
}
private static void encodeString(ByteArrayOutputStream baos, String str) throws IOException {
String eolAppended = str + "\n";
baos.write(eolAppended.getBytes(charset));
}
private static int lonToTileX(double lonDeg, int zoom) {
lonDeg = Math.max(-180, lonDeg);
lonDeg = Math.min(180, lonDeg);
double n = Math.pow(2, zoom);
return (int) (n * ((lonDeg + 180) / 360));
}
private static int latToTileY(double latDeg, int zoom) {
latDeg = Math.max(-85.0511, latDeg);
latDeg = Math.min(85.0511, latDeg);
double latRad = latDeg / (double) 180 * Math.PI;
double n = Math.pow(2, zoom);
double yTile = n * (1 - (Math.log(Math.tan(latRad) + 1.0 / Math.cos(latRad)) / Math.PI)) / 2.0;
return (int) (n - 1 - yTile);
}
private long getTimeMaxValue() {
switch (timeByteLength) {
case 2:
return Short.MAX_VALUE;
case 4:
return Integer.MAX_VALUE;
default:
return Long.MAX_VALUE;
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment