abramsm

output.std-tree:{
    root:[
        {const:"root"}
        {branch:[
            [
                // record segment hits in case we want to do a quick sanity check
                {const:"seg_stats", filter:{from:"SRC", require:"SEG"}}
                {field:"DATE_YMD"}
                // TODO:  create a field for 'SEGMENTIDS and keep track of the unique count of UIDs
                // and the top 20 UIDs for each SEGMENTIDS                

abramsm

map.filterOut:[
    {time-to-date-ymd {}}
    // only the most wellformed of UIDs are counted.
    {from:"UID", require.match:["[0-9a-f]{16}"]}
    // filter segment data
    {safely:[
        {is-not-empty:"RED"}
        // filter segments based on the provided list
        {from:"SEGMENTIDS", list-apply.elementFilter.require:[%[segment-ids]%]}
        {is-not-empty:"SEGMENTIDS"}

abramsm

map.fields:[
    // fields common to all sources
    "TIME"
    "UID"
    // fields specific to segment data
    "SEGMENTIDS"
    // fields specific to usync data
    {from:"PUID_%[pid]%", to:"PUID"}
    // injected source names
    "RED"

abramsm

include "extra-filters"
%{user-filters}%

source:[
    {mesh2:{
        markDir:"pixelatorDataMarks"
        mesh:{
            startDate:"%[segment-start-date:{{now-1}}]%"
            endDate:"%[segment-end-date:{{now-1}}]%"
            dateFormat:"yyMMdd"

abramsm

        InstanceSpec spec = new InstanceSpec(null, -1, -1, -1, true, -1, 2000, 10);
        System.setProperty("zookeeper.serverCnxnFactory", "org.apache.zookeeper.server.NettyServerCnxnFactory");
        myKeeper = new TestingServer(spec);
        String keeperPort = String.valueOf(spec.getPort());
        System.setProperty("zk.servers", "localhost:" + keeperPort);
        zkClient = CuratorFrameworkFactory.builder()
                .sessionTimeoutMs(60000)
                .connectionTimeoutMs(10000)
                .connectString("localhost:" + keeperPort)
                .retryPolicy(new RetryOneTime(1000))

abramsm

$ vagrant init precise32 http://files.vagrantup.com/precise32.box
$ vagrant up
$ vagrant ssh
$ sudo apt-get update 
$ sudo apt-get install python-software-properties
$ sudo add-apt-repository ppa:webupd8team/java
$ sudo apt-get update
$ sudo apt-get install oracle-java7-installer
$ sudo apt-get install rabbitmq-server
$ sudo apt-get install maven

abramsm

{"op":"chain", "filter":[
  {"op":"field", "from":"UID"},

  {"op":"equals", "left":"FIELD_ONE", "right":"FIELD_TWO", "not":true},

  {"op":"concat", "in":["FOO", "BAR"], "out":"OUTPUT", "join":":"},

  {"op":"num", "columns":["END", "START", "WALL"], "define":"c0,c1,sub,
   v1000,ddiv,toint,v2,set"},
]}

abramsm

3.535, 5214d63bab95687d, 166.144.203.186, "the then good"
3.568, 5dbd9451948ad895, 88.120.153.226, "know boys"
4.206, 5dbd9451948ad895, 88.120.153.226, "to"
4.673, b967d99cad0b3e60, 88.120.153.226, "seven"
4.900, bd0d760fbb338955, 166.144.203.186, "did local if to"
6.166, ef909223e4873178, 166.144.203.186, "every to"
7.050, ff1fda5a8c6361fe, 166.144.203.186, "talking from wore"
8.114, 90fbf36695d3a2d, 176.205.174.108, "was i favorite papa"
8.732, 3ef5a81b79e149a6, 166.144.203.186, "us pile we it"
9.697, 8a9d23755e58f66, 88.120.153.226, "make to"

abramsm

    // -Dpagedb.kvstore.type=1
    // -Deps.gz.type=3
    // -Deps.cache.pages=100000
    // -Xmx2G
{
    "type":"map",
    taskthreads:2,
    
    "source":{
        "type":"mesh2",

abramsm

    "output":{
        "type":"tree",
        "root":{"path":"TREE"},
        
        "paths":{
            "TREE":[
                {"type":"const", "value":"root", "data":{
                    "topterms":{"type":"key.top","key":"TERMS","size":1000},
                    "topuids":{"type":"key.top", "key":"UID", "size":1000},
                    "topip":{"type":"key.top", "key":"IP", "size":1000},