jiminoc

/**
  * we're going to receive an injected output buffer back to our curried function so we can just write lines
  * to the output file, the resource will be flushed and closed for us at the end of the run so we don't have to
  * worry about resource management
  */
  @Test
  def workWithBufferedWriterTest() {
    val tmpDir = System.getProperty("java.io.tmpdir")
    val tmpFilename = tmpDir + "/scalatowntest.log"
    println("writing to file: " + tmpFilename)

jiminoc

/**
  * here we're going to test the loan pattern using something we do all the time, iterate over files and work with
  * the lines in those files. Java makes it a pain the arse to just read simple lines in a file, lots of set up
  * code and resource management. The nice part of this pattern is once you create that set up code once you can
  * use a nice clean API like the one seen below.
  * You can see here we're passing in a function to "withFileIterator that accepts a "line" string, we then can
  * work on that line string freely and the resource will be closed in a finally block behind the scenes.
  */
  @Test
  def loanPatternWithFileIterator() {

jiminoc

-- Set the default number of reducers for this script
set default_parallel 15;
set job.name unique_users

-- INCLUDE THE OFFICIAL SCHEMA, beacons will be a relation returned
IMPORT 'schema.pig';
beacons = grvschema('$input');

fb = FOREACH beacons GENERATE userGuid;

jiminoc

-- Data would be in the following format
-- 2011-01-01^pageview^http://somesite.com/url1^randomuserid123

DEFINE grvschema(inputdata) 
returns BEACONS {
  $BEACONS = LOAD '$inputdata' USING PigStorage('^') as (date:chararray, action:chararray, url:chararray, userId:chararray);
};

jiminoc

public void testThatDifferentRPCandHttpPortsAreOK() 
	throws IOException {

		Configuration conf = new HdfsConfiguration();
		FileSystem.setDefaultUri(conf, "hdfs://localhost:8000");
		conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, "127.0.0.1:9000");

		try {
			NameNode nameNode = new NameNode(conf); // should be OK!
		} catch (Exception e) {