This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import itertools | |
from random import shuffle | |
def permutations(l): | |
result = [] | |
for i in xrange(len(l)+1): | |
result += list(itertools.permutations(l,i)) | |
return result | |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import os,sys,re,math,os.path | |
from collections import defaultdict | |
from itertools import groupby | |
from bz2 import BZ2File | |
from gzip import GzipFile as GZFile | |
import getopt | |
from json import loads as json_parse | |
from json import dumps as json_print | |
from md5 import md5 as md5_hash | |
from xml.dom.minidom import parse as xmlparse |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// import org.apache.commons.lang3.RandomUtils; | |
import org.apache.hadoop.conf.Configuration; | |
import org.apache.hadoop.fs.FSDataInputStream; | |
import org.apache.hadoop.fs.FileSystem; | |
import org.apache.hadoop.fs.Path; | |
import com.google.common.base.Stopwatch; | |
import java.io.IOException; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
divisor = 19; | |
print """ | |
drop table if exists postal_distribution; | |
CREATE TABLE if not exists postal_distribution( | |
zipcode varchar(6), lat double, lon double) | |
partitioned by (xcross int) | |
stored as orc; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
BEGIN {FS=":";} | |
/^physical id/ {phys=$2} | |
/^cpu cores/ {cores[phys]=$2;} | |
END { total = 0; for(i in cores) { total = total + cores[i]} print "Total cores = ", total} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import java.io.File; | |
import java.util.*; | |
import java.lang.management.BufferPoolMXBean; | |
import java.lang.management.ManagementFactory; | |
import javax.management.MBeanServerConnection; | |
import javax.management.ObjectName; | |
import javax.management.remote.*; | |
import com.sun.tools.attach.VirtualMachine; // Attach API |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import os, sys, re, math | |
from org.apache.pig.scripting import * | |
operation = Pig.compile(""" | |
set pig.splitCombination false; | |
set tez.grouping.min-size 52428800; | |
set tez.grouping.max-size 52428800; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import sys, re, os, math | |
import dpkt | |
import socket | |
from collections import defaultdict | |
def ip_str(ip): | |
return socket.inet_ntoa(ip) | |
class Connection(object): | |
def __init__(self): |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import yum | |
yb = yum.YumBase() | |
yb.setCacheDir() | |
pkgs=[p for p in yb.pkgSack.returnNewestByNameArch(patterns='*.rpm') if 'HDP' in p.repoid] | |
for p in pkgs: | |
print "wget -c ", p.remote_url |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import re, sys | |
NX = True | |
try: | |
import networkx as nx | |
except: | |
NX = False | |
sys.stderr.write("Could not import nx\npip install networkx, please\n") | |
plan39 = """ | |
Map 1 <- Map 5 (BROADCAST_EDGE), Map 6 (BROADCAST_EDGE), Map 8 (BROADCAST_EDGE), Reducer 7 (BROADCAST_EDGE), Reducer 9 (BROADCAST_EDGE) |