bartekdobija

# /etc/dhcp/hdcpd.conf

option domain-name "home.com";
default-lease-time 600;
max-lease-time 7200;
log-facility local7;
subnet 192.168.1.0 netmask 255.255.255.0 {
  range 192.168.1.10 192.168.1.100;
  option routers 192.168.1.1;
  allow booting;

bartekdobija

# ***** Set root logger level to DEBUG and its only appender to A.
log4j.logger.org.apache.pig=WARN, A
log4j.logger.org.apache.hadoop=ERROR, A

# ***** A is set to be a ConsoleAppender.
log4j.appender.A=org.apache.log4j.ConsoleAppender
# ***** A uses PatternLayout.
log4j.appender.A.layout=org.apache.log4j.PatternLayout
log4j.appender.A.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n

bartekdobija

#!/usr/bin/env bash

function process() {
    files=$(ls *.$1)
    for file in $files; do
        tail -n $2 $file > "$output_folder/$file"
    done
}

function show_help() {

bartekdobija

#!/usr/bin/env bash

# This script will download, unpack and register Hadoop 2.5.1 in your home directory. 
# No RPMs, no integration with the OS.
# The installer is created to automate and simplify the process of deployment of the local development Hadoop version.
# It is ideal for developers who begin their adventure with the cluster or for all you lazy programmers that make 90% of the community;)
# The simplest way to execute this script is to:
# bash -c "$(curl -fsSL https://goo.gl/x2VTbz)"

tmp_hadoop_location=/tmp/hadoop-2.5.1.tar.gz

bartekdobija

#!/usr/bin/env bash 

# In this case I have a Hadoop distro compiled from source:
#   MAVEN_OPTS="-Xms512m -Xmx1024m" mvn package -Pdist,native -DskipTests -Dtar
# verified with:
#   hadoop checknative -a
# with output:
#  Native library checking:
#  hadoop:  true /usr/local/hadoop-2.6.0/lib/native/libhadoop.so.1.0.0
#  zlib:    true /lib64/libz.so.1

bartekdobija

# JupyterHub start / stop / restart script
#
# description: JupyterHub start / stop / restart script
# chkconfig: 2345 20 80
# processname: jupyterhub
# pidfile: /var/run/jupyterhub.pid

JUPYTERHUB_HOME=/usr/local/bin
JUPYTERHUB_CONFIG=/etc/ipython/jupyterhub_config.py
JUPYTERHUB_LOG=/var/log/jupyterhub.log

bartekdobija

# Anaconda dependencies
$anaconda_deps = <<SCRIPT

  ANACONDA_INSTALLER=https://3230d63b5fc54e62148e-c95ac804525aac4b6dba79b00b39d1d3.ssl.cf1.rackcdn.com/Anaconda-2.3.0-Linux-x86_64.sh

  if [ ! -d "/usr/local/anaconda" ]; then
    echo "Anaconda installation..." \
      && echo "downloading binaries" \
      && wget ${ANACONDA_INSTALLER} -q -P /tmp/ \
      && echo "running installer" \

bartekdobija

# Basic OS configuration
$sysconfig = <<SCRIPT

  # disable IPv6
  echo "net.ipv6.conf.all.disable_ipv6=1" > /etc/sysctl.conf && sysctl -f /etc/sysctl.conf

  # this should be a persistent config
  ulimit -n 65536
  ulimit -s 10240

bartekdobija

# Basic OS configuration
$sysconfig = <<SCRIPT

  # disable IPv6
  echo "net.ipv6.conf.all.disable_ipv6=1" > /etc/sysctl.conf && sysctl -f /etc/sysctl.conf

  # this should be a persistent config
  ulimit -n 65536
  ulimit -s 10240

bartekdobija

MAVEN_OPTS="-Xmx5g -XX:MaxPermSize=1024m" /usr/local/apache-maven-3.0.4/bin/mvn clean install -DskipTests -Pvendor-repos -Dhadoop.profile=2 -Dhadoop.version=2.6.0-cdh5.10.0 -Dhbase.version=1.2.0-cdh5.10.0