bepcyc

#you'll need mencoder and x264 packages installed
#just add a video name like output.mp4 at the end
alias jpgtoh264="mencoder mf://*.jpg -nosound -of lavf -lavfopts format=mp4 -ovc x264 -x264encopts pass=1:bitrate=2000:crf=24 -mf type=jpg:fps=30 -o"

bepcyc

yarn node -list -all 2>>/dev/null|cut -f3|grep -v "Total Nodes"|grep -P "\:\d{2,}$"|cut -d':' -f1


hadoop fs -mkdir /tmp/${tmp_dir}
hadoop fs -put ${dest} /tmp/${tmp_dir}/
pdsh hadoop fs -get /tmp/${tmp_dir}/${dest}

bepcyc

# add these lines to .bashrc or to the other start script
export SEARCH_MR_JOB_JAR="/opt/cloudera/parcels/CDH/lib/solr/contrib/mr/search-mr-job.jar"
alias dfsFind="hadoop jar ${SEARCH_MR_JOB_JAR} org.apache.solr.hadoop.HdfsFindTool"
#alias MapReduceIndexerTool="hadoop jar ${SEARCH_MR_JOB_JAR} org.apache.solr.hadoop.MapReduceIndexerTool"

# use it like regular find:
# dfsFind / -name "*.snappy" | grep flume

bepcyc

# insert somewhere in function working with sc directly
sc.stop()
from pyspark import SparkContext
SparkContext.setSystemProperty('spark.executor.memory', '6g') # no sure which one works, use both
SparkContext.setSystemProperty('spark.python.worker.memory', '6g') # no sure which one works, use both
SparkContext.setSystemProperty('spark.shuffle.spill', 'false')
SparkContext.setSystemProperty('spark.driver.memory', '2g')
SparkContext.setSystemProperty('spark.io.compression.codec', 'snappy') # just to be sure
sc = SparkContext("local[8]", "Simple App") # set to your number of cores

bepcyc

# set 8 cores and 6GB of RAM
Vagrant.configure(2) do |config|
  config.vm.define "myvm" do |master|
    master.vm.provider :virtualbox do |v|
      v.customize ["modifyvm", :id, "--ioapic", "on"] # this one is important for setting cores
      v.customize ["modifyvm", :id, "--cpus", 8]
      v.customize ["modifyvm", :id, "--memory", 6144]
    end
  end
end

bepcyc

# -*- mode: ruby -*-
# vi: set ft=ruby :

ipythonPort = 8001                 # Ipython port to forward (also set in IPython notebook config)

Vagrant.configure(2) do |config|
  config.ssh.insert_key = true
  config.vm.define "sparkvm" do |master|
    master.vm.box = "sparkmooc/base"
    master.vm.box_download_insecure = true

bepcyc

package com.avira.ds.sparser.spark

import org.apache.hadoop.io.NullWritable
import org.apache.hadoop.mapred.lib.MultipleTextOutputFormat
import org.apache.spark.{SparkContext, SparkConf}
import scala.language.implicitConversions

sealed trait Event
case class ClickEvent(blaBla: String) extends Event
case class ViewEvent(blaBla: String) extends Event

bepcyc

#define in apt.conf

Acquire {
  Retries "0";
  HTTP {
    Proxy "http://username:password@proxy.mystarhub.com.sg:8080";
  };
};

bepcyc

    /**
     * The way to free memory arranged for Bitmaps in native memory or just not GC'ed
     * @source <a href="http://stackoverflow.com/a/7783511/918211">StackOverflow Thread</a>
     */
    public static void stripImageView(ImageView view) {
        if (view.getDrawable() instanceof BitmapDrawable) {
            ((BitmapDrawable) view.getDrawable()).getBitmap().recycle();
        }
        view.getDrawable().setCallback(null);
        view.setImageDrawable(null);

bepcyc

before backup:
dpkg --get-selections > packages.txt

while restoring:
dpkg --clear-selections
dpkg --set-selections < packages.txt
aptitude install