rsumbaly

/*
 * Copyright 2010 LinkedIn, Inc
 * 
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software

rsumbaly

a) Hive SerDe for Binary JSON 
- http://code.google.com/p/hive-json-serde/source/browse/trunk/src/org/apache/hadoop/hive/contrib/serde2/JsonSerde.java
- http://wiki.apache.org/hadoop/Hive/StorageHandlers

b) Hive/Voldemort integration
- http://wiki.apache.org/hadoop/Hive/HBaseIntegration

c) Voldemort Pub/Sub
- Kafka?

rsumbaly

package voldemort.store.readwrite.mr;

import java.io.File;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Properties;

rsumbaly

   currentCluster = ServerTestUtils.getLocalCluster(3, new int[][] { { 0, 1, 2, 3 },
                { 4, 5, 6 }, { 7, 8, 9 } });

        targetCluster = ServerTestUtils.getLocalCluster(4, new int[][] { { 0, 4 }, { 1, 3, 5 },
                { 7, 8, 9 }, { 2, 6 } });

        RebalanceClusterPlan rebalancePlan = new RebalanceClusterPlan(currentCluster,
                                                                      targetCluster,
                                                                      storeDefList,
                                                                      new HashMap<String, Long>(),

rsumbaly

HOWTO
------------------------------------------------------------------------------
a) Get Voldemort
> mkdir voldemort-0.81 ; cd voldemort-0.81
> git clone git@github.com:voldemort/voldemort.git
> git checkout -b 81 origin/release-081
OR
> wget http://github.com/downloads/voldemort/voldemort/voldemort-0.81.tar.gz
> tar xvfz voldemort-0.81.tar.gz
> cd voldemort-0.81

rsumbaly

package voldemort.store.readonly.mr;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;

public class DummyMapper extends AbstractHadoopStoreBuilderMapper<LongWritable, Text> {

    @Override
    public Object makeKey(LongWritable key, Text value) {
        return value.toString().split(" ")[0];

rsumbaly

        nodes = new ArrayList<Node>();
            LinkedList<Integer> zoneProximityList = this.clientZone.getProximityList();
            if(pipeline.getOperation() != Operation.PUT) {
                // GET, GET_VERSIONS, DELETE

                // Add a node from every zone
                for(int index = 0; index < pipelineData.getZonesRequired(); index++) {
                    List<Node> zoneNodes = zoneIdToNode.get(zoneProximityList.get(index));
                    if(zoneNodes != null && zoneNodes.size() > 0) {
                        nodes.add(zoneNodes.remove(0));

rsumbaly

package voldemort.routing;

import java.util.HashMap;
import java.util.List;

import junit.framework.TestCase;
import voldemort.TestUtils;
import voldemort.cluster.Node;

import com.google.common.collect.Lists;

rsumbaly

package voldemort;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;

rsumbaly

        // Basic put with zone read = 0, zone write = 0
        Store<ByteArray, byte[], byte[]> s1 = getStore(cluster,
                                                       1,
                                                       1,
                                                       0,
                                                       0,
                                                       8,
                                                       null,
                                                       Sets.newHashSet(4, 5, 6, 7),
                                                       zoneReplicationFactor,