cworks

" syntax highlighting
syntax on
" tab width = 4 spaces
set tabstop=4
" indent width = 4
set shiftwidth=4
" expand tabs to spaces
set expandtab
"auto-indent code blocks
set autoindent

cworks

/**
 * Convert the ResultSet to a List of Maps, where each Map represents a row with columnNames and columValues
 * @param rs
 * @return
 * @throws SQLException
 */
private List<Map<String, Object>> resultSetToList(ResultSet rs) throws SQLException {
    ResultSetMetaData md = rs.getMetaData();
    int columns = md.getColumnCount();
    List<Map<String, Object>> rows = new ArrayList<Map<String, Object>>();

cworks

public static void printMap(Map mp) {
    Iterator it = mp.entrySet().iterator();
    while (it.hasNext()) {
        Map.Entry pairs = (Map.Entry)it.next();
        System.out.println(pairs.getKey() + " = " + pairs.getValue());
        it.remove(); // avoids a ConcurrentModificationException
    }
}

cworks

import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

public class PlayerBattingReducer extends Reducer<Text, Text, Text, Text> {

    /**
     * Reduce yearly player stats to career stats
     * @param key
     * @param values
     * @param context

cworks

corbett@corbett17:~/Downloads/datasets|⇒  unzip lahman2012-csv.zip -d lahman2012
Archive:  lahman2012-csv.zip
  inflating: lahman2012/TeamsHalf.csv
  inflating: lahman2012/AllstarFull.csv
  inflating: lahman2012/Appearances.csv
  inflating: lahman2012/AwardsManagers.csv
  inflating: lahman2012/AwardsPlayers.csv
  inflating: lahman2012/AwardsShareManagers.csv
  inflating: lahman2012/AwardsSharePlayers.csv
  inflating: lahman2012/Batting.csv

cworks

corbett@corbett17:~/Downloads/datasets|⇒  hadoop fs -put lahman2012 datasets/lahman2012
corbett@corbett17:~/Downloads/datasets|⇒  hadoop fs -ls datasets/lahman2012
Found 26 items
-rw-r--r--   3 corbett supergroup     198529 2013-02-28 22:23 /user/corbett/datasets/lahman2012/AllstarFull.csv
-rw-r--r--   3 corbett supergroup    5730747 2013-02-28 22:23 /user/corbett/datasets/lahman2012/Appearances.csv
-rw-r--r--   3 corbett supergroup       7304 2013-02-28 22:23 /user/corbett/datasets/lahman2012/AwardsManagers.csv
-rw-r--r--   3 corbett supergroup     240867 2013-02-28 22:23 /user/corbett/datasets/lahman2012/AwardsPlayers.csv
-rw-r--r--   3 corbett supergroup      16719 2013-02-28 22:23 /user/corbett/datasets/lahman2012/AwardsShareManagers.csv
-rw-r--r--   3 corbett supergroup     220135 2013-02-28 22:23 /user/corbett/datasets/lahman2012/AwardsSharePlayers.csv
-rw-r--r--   3 corbett supergroup    6488747 2013-02-28 22:23 /user/corbett/datasets/lahman2012/Batting.csv

cworks

corbett@corbett17:/tmp/sandbox|⇒  tree
|____hadoop-foo
| |____net.cworks.mrfun
| | |____pom.xml
| | |____README.md
| | |____src
| | | |____main
| | | | |____java
| | | | | |____PlayerBatting.java
| | | | | |____PlayerBattingMapper.java

cworks

/**
 * main method
 * @param args
 */
public static void main(String[] args) throws Exception {
    Logger logger = LoggerFactory.getLogger(PlayerBatting.class);
    logger.info("PlayerBatting starting up...");

    /*
     * The expected command-line arguments are the paths containing

cworks

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

public final class PlayerBattingMapper extends Mapper<LongWritable, Text, Text, Text> {

    /*
     * The map method runs once for each line of text in the input file. The method receives a key
     * of type LongWritable, a value of type Text, and a Context object.
     */

cworks

corbett@corbett17:~/dev/hadoop-foo/net.cworks.mrfun|master⚡ ⇒  mvn clean package
[INFO] Scanning for projects...
[INFO]
[INFO] ------------------------------------------------------------------------
[INFO] Building net.cworks.mrfun 0.0.1-SNAPSHOT
[INFO] ------------------------------------------------------------------------
...
...
...
[INFO] Building jar: /Users/corbett/dev/hadoop-foo/net.cworks.mrfun/target/net-cworks-mrfun.jar