Skip to content

Instantly share code, notes, and snippets.

@fteychene

fteychene/ProtobufMain.java

Created December 18, 2015 14:12
Show Gist options
  • Save fteychene/7a315b23033e12cb93a2 to your computer and use it in GitHub Desktop.
Save fteychene/7a315b23033e12cb93a2 to your computer and use it in GitHub Desktop.
Download ZIP
Test protobuf and Chronicle for events
Raw
ProtobufMain.java
package fte.tests.chronicle;
import com.cognitect.transit.Reader;
import com.cognitect.transit.TransitFactory;
import com.cognitect.transit.Writer;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.datatype.guava.GuavaModule;
import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
import net.openhft.chronicle.Chronicle;
import net.openhft.chronicle.ChronicleQueueBuilder;
import net.openhft.chronicle.ExcerptAppender;
import net.openhft.chronicle.ExcerptTailer;
import org.msgpack.MessagePack;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.LockSupport;
/**
* Created by fteychene on 12/18/15.
*/
public class ProtobufMain {
private static final int MAX_VALUE = 1000000;
public static void main(String[] args) throws IOException, InterruptedException {
AtomicBoolean pushFinish = new AtomicBoolean(false);
AtomicBoolean finish1 = new AtomicBoolean(false);
AtomicBoolean finish2 = new AtomicBoolean(false);
String basePath = System.getProperty("java.io.tmpdir") + "/getting-started";
Chronicle chronicle = ChronicleQueueBuilder.indexed(basePath).build();
ExcerptAppender appender = chronicle.createAppender();
ExcerptTailer reader1 = chronicle.createExcerpt();
ExcerptTailer reader2 = chronicle.createExcerpt();
Map<String, Object> event = new HashMap<String, Object>();
event.put("app_id", "widget");
event.put("platform", "web");
event.put("etl_tstamp", 1443130964305L);
event.put("collector_tstamp", 1443130964305L);
event.put("dvce_tstamp", 1442998019482L);
event.put("event", "pp");
event.put("event_id", "61cdefb9-8874-43f8-9a5b-a360f60bda22");
event.put("txn_id", null);
event.put("name_tracker", "cf");
event.put("v_tracker", "js-2.4.3");
event.put("v_collector", null);
event.put("v_etl", null);
event.put("user_id", "1ab8409898cf13bf");
event.put("user_ipaddress", "92.140.22.34");
event.put("user_fingerprint", "876697152");
event.put("domain_userid", "1ab8409898cf13bf");
event.put("domain_sessionidx", 1);
event.put("network_userid", "a1188773-0a93-4649-a132-3950c0c82aa5");
event.put("geo_country", "FR");
event.put("geo_region", null);
event.put("geo_city", null);
event.put("geo_zipcode", null);
event.put("geo_latitude", 48.86);
event.put("geo_longitude", 2.35);
event.put("geo_region_name", null);
event.put("ip_isp", null);
event.put("ip_organization", null);
event.put("ip_domain", null);
event.put("ip_netspeed", null);
event.put("page_url", "http,//unprogrammepourlafrance.com/");
event.put("page_title", "Un Programme pour la France");
event.put("page_referrer", null);
event.put("page_urlscheme", "http");
event.put("page_urlhost", "unprogrammepourlafrance.com");
event.put("page_urlport", -1);
event.put("page_urlpath", "/");
event.put("page_urlquery", null);
event.put("page_urlfragment", null);
event.put("refr_urlscheme", null);
event.put("refr_urlhost", null);
event.put("refr_urlport", -1);
event.put("refr_urlpath", "/");
event.put("refr_urlquery", null);
event.put("refr_urlfragment", null);
event.put("refr_medium", null);
event.put("refr_source", null);
event.put("refr_term", null);
event.put("contexts", null);
event.put("se_category", null);
event.put("se_action", null);
event.put("se_label", null);
event.put("se_property", null);
event.put("se_value", null);
event.put("unstruct_event", null);
event.put("pp_xoffset_min", 0);
event.put("pp_xoffset_max", 0);
event.put("pp_yoffset_min", 0);
event.put("pp_yoffset_max", 0);
event.put("useragent", null);
event.put("br_name", "Safari 8");
event.put("br_family", "Safari");
event.put("br_version", null);
event.put("br_type", "Browser");
event.put("br_renderengine", null);
event.put("br_lang", "fr-fr");
event.put("br_cookies", true);
event.put("br_colordepth", "24");
event.put("br_viewwidth", null);
event.put("br_viewheight", null);
event.put("os_name", "Mac OS X");
event.put("os_family", "Mac OS X");
event.put("os_manufacturer", "Apple Inc.");
event.put("os_timezone", "Europe/Berlin");
event.put("dvce_type", "Computer");
event.put("dvce_ismobile", null);
event.put("dvce_screenwidth", null);
event.put("dvce_screenheight", null);
event.put("doc_charset", "UTF-8");
event.put("doc_width", null);
event.put("doc_height", null);
event.put("geo_timezone", null);
event.put("mkt_clickid", null);
event.put("mkt_network", null);
event.put("etl_tags", null);
event.put("dvce_sent_tstamp", null);
event.put("refr_domain_userid", null);
event.put("refr_dvce_tstamp", null);
event.put("derived_contexts", null);
event.put("domain_sessionid", null);
event.put("derived_tstamp", null);
event.put("social_network", null);
event.put("tnetwork_userid", null);
event.put("ds", "1440x1115");
event.put("connection_type", null);
event.put("vp", "1440x815");
event.put("mac_address", null);
event.put("social_pagepath", null);
event.put("social_target", null);
event.put("res", "1440x900");
event.put("social_action", null);
event.put("account", "12ABF1209540U03U4");
event.put("question", "ABC098239FD871616D");
event.put("age_range", "13-15");
event.put("gender", "M");
// ForkJoinPool.commonPool().execute(() -> {
long pushTime = System.currentTimeMillis();
int pushCounter = 0;
while (pushCounter < MAX_VALUE) {
appender.startExcerpt();
toEvent(event).writeTo(appender.outputStream());
appender.finish();
pushCounter++;
}
long pushEndTime = System.currentTimeMillis();
System.out.println("Finished push " + pushCounter + " events in " + (pushEndTime - pushTime) + " ms");
pushFinish.set(true);
// });
ForkJoinPool.commonPool().execute(() -> {
long startTime = System.currentTimeMillis();
int localCounter = 0;
while (localCounter < MAX_VALUE) {
if (reader1.nextIndex()) {
try {
Events.Event test = Events.Event.parseFrom(reader1.inputStream());
} catch (IOException e) {
e.printStackTrace();
}
} else {
LockSupport.parkNanos(TimeUnit.MICROSECONDS.toNanos(1));
}
localCounter++;
}
long endTime = System.currentTimeMillis();
System.out.println("Finished process " + localCounter + " in consumer 1 events in " + (endTime - startTime) + " ms");
finish1.set(true);
});
ForkJoinPool.commonPool().execute(() -> {
long startTime = System.currentTimeMillis();
int localCounter = 0;
while (localCounter < MAX_VALUE) {
if (reader2.nextIndex()) {
try {
Events.Event test = Events.Event.parseFrom(reader2.inputStream());
} catch (IOException e) {
e.printStackTrace();
}
} else {
LockSupport.parkNanos(TimeUnit.MICROSECONDS.toNanos(1));
}
localCounter++;
}
long endTime = System.currentTimeMillis();
System.out.println("Finished process " + localCounter + " in consumer 2 events in " + (endTime - startTime) + " ms");
finish2.set(true);
});
while (!pushFinish.get() || !finish1.get() || !finish2.get()) {}
reader1.finish();
reader2.finish();
chronicle.clear();
}
public static Events.Event toEvent(Map<String, Object> map) {
Events.Event.Builder builder = Events.Event.newBuilder();
if (map.get("app_id") != null) { builder.setAppId((String) map.get("app_id"));}
if (map.get("platform") != null) { builder.setPlatform((String) map.get("platform"));}
if (map.get("etl_tstamp") != null) { builder.setEtlTstamp((Long) map.get("etl_tstamp"));}
if (map.get("collector_tstamp") != null) { builder.setCollectorTstamp((Long) map.get("collector_tstamp"));}
if (map.get("dvce_tstamp") != null) { builder.setDvceCreatedTstamp((Long) map.get("dvce_tstamp"));}
if (map.get("event") != null) { builder.setEvent((String) map.get("event"));}
if (map.get("event_id") != null) { builder.setEventId((String) map.get("event_id"));}
if (map.get("txn_id") != null) { builder.setTxnId((String) map.get("txn_id"));}
if (map.get("name_tracker") != null) { builder.setNameTracker((String) map.get("name_tracker"));}
if (map.get("v_tracker") != null) { builder.setVTracker((String) map.get("v_tracker"));}
if (map.get("v_collector") != null) { builder.setVCollector((String) map.get("v_collector"));}
if (map.get("v_etl") != null) { builder.setVEtl((String) map.get("v_etl"));}
if (map.get("user_id") != null) { builder.setUserId((String) map.get("user_id"));}
if (map.get("user_ipaddress") != null) { builder.setUserIpaddress((String) map.get("user_ipaddress"));}
if (map.get("user_fingerprint") != null) { builder.setUserFingerprint((String) map.get("user_fingerprint"));}
if (map.get("domain_userid") != null) { builder.setDomainUserid((String) map.get("domain_userid"));}
if (map.get("domain_sessionidx") != null) { builder.setDomainSessionidx((Integer) map.get("domain_sessionidx"));}
if (map.get("network_userid") != null) { builder.setNetworkUserid((String) map.get("network_userid"));}
if (map.get("geo_country") != null) { builder.setGeoCountry((String) map.get("geo_country"));}
if (map.get("geo_region") != null) { builder.setGeoRegion((String) map.get("geo_region"));}
if (map.get("geo_city") != null) { builder.setGeoCity((String) map.get("geo_city"));}
if (map.get("geo_zipcode") != null) { builder.setGeoZipcode((String) map.get("geo_zipcode"));}
if (map.get("geo_latitude") != null) { builder.setGeoLatitude((Double) map.get("geo_latitude"));}
if (map.get("geo_longitude") != null) { builder.setGeoLongitude((Double) map.get("geo_longitude"));}
if (map.get("geo_region_name") != null) { builder.setGeoRegionName((String) map.get("geo_region_name"));}
if (map.get("ip_isp") != null) { builder.setIpIsp((String) map.get("ip_isp"));}
if (map.get("ip_organization") != null) { builder.setIpOrganization((String) map.get("ip_organization"));}
if (map.get("ip_domain") != null) { builder.setIpDomain((String) map.get("ip_domain"));}
if (map.get("ip_netspeed") != null) { builder.setIpNetspeed((String) map.get("ip_netspeed"));}
if (map.get("page_url") != null) { builder.setPageUrl((String) map.get("page_url"));}
if (map.get("page_title") != null) { builder.setPageTitle((String) map.get("page_title"));}
if (map.get("page_referrer") != null) { builder.setPageReferrer((String) map.get("page_referrer"));}
if (map.get("page_urlscheme") != null) { builder.setPageUrlscheme((String) map.get("page_urlscheme"));}
if (map.get("page_urlhost") != null) { builder.setPageUrlhost((String) map.get("page_urlhost"));}
if (map.get("page_urlport") != null) { builder.setPageUrlport((Integer) map.get("page_urlport"));}
if (map.get("page_urlpath") != null) { builder.setPageUrlpath((String) map.get("page_urlpath"));}
if (map.get("page_urlquery") != null) { builder.setPageUrlquery((String) map.get("page_urlquery"));}
if (map.get("page_urlfragment") != null) { builder.setPageUrlfragment((String) map.get("page_urlfragment"));}
if (map.get("refr_urlscheme") != null) { builder.setRefrUrlscheme((String) map.get("refr_urlscheme"));}
if (map.get("refr_urlhost") != null) { builder.setRefrUrlhost((String) map.get("refr_urlhost"));}
if (map.get("refr_urlport") != null) { builder.setRefrUrlport((Integer) map.get("refr_urlport"));}
if (map.get("refr_urlpath") != null) { builder.setRefrUrlpath((String) map.get("refr_urlpath"));}
if (map.get("refr_urlquery") != null) { builder.setRefrUrlquery((String) map.get("refr_urlquery"));}
if (map.get("refr_urlfragment") != null) { builder.setRefrUrlfragment((String) map.get("refr_urlfragment"));}
if (map.get("refr_medium") != null) { builder.setRefrMedium((String) map.get("refr_medium"));}
if (map.get("refr_source") != null) { builder.setRefrSource((String) map.get("refr_source"));}
if (map.get("refr_term") != null) { builder.setRefrTerm((String) map.get("refr_term"));}
if (map.get("se_category") != null) { builder.setSeCategory((String) map.get("se_category"));}
if (map.get("se_action") != null) { builder.setSeAction((String) map.get("se_action"));}
if (map.get("se_label") != null) { builder.setSeLabel((String) map.get("se_label"));}
if (map.get("se_property") != null) { builder.setSeProperty((String) map.get("se_property"));}
if (map.get("se_value") != null) { builder.setSeValue((Double) map.get("se_value"));}
if (map.get("pp_xoffset_min") != null) { builder.setPpXoffsetMin((Integer) map.get("pp_xoffset_min"));}
if (map.get("pp_xoffset_max") != null) { builder.setPpXoffsetMax((Integer) map.get("pp_xoffset_max"));}
if (map.get("pp_yoffset_min") != null) { builder.setPpYoffsetMin((Integer) map.get("pp_yoffset_min"));}
if (map.get("pp_yoffset_max") != null) { builder.setPpYoffsetMax((Integer) map.get("pp_yoffset_max"));}
if (map.get("useragent") != null) { builder.setUseragent((String) map.get("useragent"));}
if (map.get("br_name") != null) { builder.setBrName((String) map.get("br_name"));}
if (map.get("br_family") != null) { builder.setBrFamily((String) map.get("br_family"));}
if (map.get("br_version") != null) { builder.setBrVersion((String) map.get("br_version"));}
if (map.get("br_type") != null) { builder.setBrType((String) map.get("br_type"));}
if (map.get("br_renderengine") != null) { builder.setBrRenderengine((String) map.get("br_renderengine"));}
if (map.get("br_lang") != null) { builder.setBrLang((String) map.get("br_lang"));}
if (map.get("br_cookies") != null) { builder.setBrCookies((Boolean) map.get("br_cookies"));}
if (map.get("br_colordepth") != null) { builder.setBrColordepth((String) map.get("br_colordepth"));}
if (map.get("br_viewwidth") != null) { builder.setBrViewwidth((Integer) map.get("br_viewwidth"));}
if (map.get("br_viewheight") != null) { builder.setBrViewheight((Integer) map.get("br_viewheight"));}
if (map.get("os_name") != null) { builder.setOsName((String) map.get("os_name"));}
if (map.get("os_family") != null) { builder.setOsFamily((String) map.get("os_family"));}
if (map.get("os_manufacturer") != null) { builder.setOsManufacturer((String) map.get("os_manufacturer"));}
if (map.get("os_timezone") != null) { builder.setOsTimezone((String) map.get("os_timezone"));}
if (map.get("dvce_type") != null) { builder.setDvceType((String) map.get("dvce_type"));}
if (map.get("dvce_ismobile") != null) { builder.setDvceIsmobile((Boolean) map.get("dvce_ismobile"));}
if (map.get("dvce_screenwidth") != null) { builder.setDvceScreenwidth((Integer) map.get("dvce_screenwidth"));}
if (map.get("dvce_screenheight") != null) { builder.setDvceScreenheight((Integer) map.get("dvce_screenheight"));}
if (map.get("doc_charset") != null) { builder.setDocCharset((String) map.get("doc_charset"));}
if (map.get("doc_width") != null) { builder.setDocWith((Integer) map.get("doc_width"));}
if (map.get("doc_height") != null) { builder.setDocHeight((Integer) map.get("doc_height"));}
if (map.get("geo_timezone") != null) { builder.setGeoTimezone((String) map.get("geo_timezone"));}
if (map.get("mkt_clickid") != null) { builder.setMktClickid((String) map.get("mkt_clickid"));}
if (map.get("mkt_network") != null) { builder.setMktNetwork((String) map.get("mkt_network"));}
if (map.get("etl_tags") != null) { builder.setEtlTags((String) map.get("etl_tags"));}
if (map.get("dvce_sent_tstamp") != null) { builder.setDvceSentTstamp((Long) map.get("dvce_sent_tstamp"));}
if (map.get("refr_domain_userid") != null) { builder.setRefrDomainUserid((String) map.get("refr_domain_userid"));}
if (map.get("refr_dvce_tstamp") != null) { builder.setRefrDvceTstamp((Long) map.get("refr_dvce_tstamp"));}
if (map.get("domain_sessionid") != null) { builder.setDomainSessionid((String) map.get("domain_sessionid"));}
if (map.get("derived_tstamp") != null) { builder.setDerivedTstamp((Long) map.get("derived_tstamp"));}
// event.put("social_network", null);
// event.put("tnetwork_userid", null);
// event.put("ds", "1440x1115");
// event.put("connection_type", null);
// event.put("vp", "1440x815");
// event.put("mac_address", null);
// event.put("social_pagepath", null);
// event.put("social_target", null);
// event.put("res", "1440x900");
// event.put("social_action", null);
// event.put("account", "12ABF1209540U03U4");
// event.put("question", "ABC098239FD871616D");
// event.put("age_range", "13-15");
// event.put("gender", "M");
return builder.build();
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment