sambos

public class AppContext {  
	  
    private static ApplicationContext ctx;    

    public static void setApplicationContext(ApplicationContext applicationContext) {  
        ctx = applicationContext;  
    }  
  
    public static ApplicationContext getApplicationContext() {  
        return ctx;  

sambos

Understanding AST

AST Tree

RooT
-sibling
 |-child
 |-sibling
 |-sibling
-Sibling

sambos

#!/usr/bin/env python

# curl equivalent:
#  curl --negotiate -u:userid "http://localhost:11000/oozie?job=x" -H 'Content-type:text/xml' 
import urllib2
import json
import pycurl

#req = urllib2.Request('http://localhost:11000/oozie/v1/jobs?jobtype=wf')
#response = urllib2.urlopen(req)

sambos

#!/usr/local/bin/python

from contextlib import contextmanager
from datetime import datetime
from itertools import islice
from random import choice, randint
from string import lowercase
import pycurl
import pprint
import json

sambos

cd your to your cloned project
-- make sure you pull all the code from the repo before you lose it
git pull origin master

rm -rf .git

-- recreate from the contents
git init
git add .
git commit -m"initial"

sambos

Common Spark Troubleshooting Tips

https://www.datastax.com/dev/blog/common-spark-troubleshooting

sambos

Useful HDFS Commands

Displayed in this format:
 +-------------------------------------------------------------------+ 
 | size  |  disk_space_consumed_with_all_replicas  |  full_path_name | 
 +-------------------------------------------------------------------+ 
 
-du [-s] [-h] ... : Show the amount of space, in bytes, used by the files that match the specified file pattern.

-s : Rather than showing the size of each individual file that matches the

sambos

Scala Implicits - nice explanation
https://www.theguardian.com/info/developer-blog/2016/dec/22/parental-advisory-implicit-content

sambos

aggregateByKey

A general form of reduceByKey, that can be used when the return value is different type than input. Takes an initial value of accumulator.

     val initialList = scala.collection.mutable.ListBuffer[Row]()
     val addToList = (acc:scala.collection.mutable.ListBuffer[Row], x:Row) => x +: acc
     val mergePartitionLists = (acc1: scala.collection.mutable.ListBuffer[Row],acc2: scala.collection.mutable.ListBuffer[Row]) => acc1 ++ acc2
    
     val gbyKey = rdd.map(x => (x.getAs[String]("xtransId"), x)).aggregateByKey(initialList)(addToList,         mergePartitionLists).map(x => (x._1, x._2.toList))
                          

sambos

SysLogWriter

package rsol.stream.demo;

import java.net.DatagramPacket;
import java.net.DatagramSocket;
import java.net.InetAddress;
import java.net.SocketException;
import java.net.UnknownHostException;