snoremac

./src/ruby/common_crawl_input.rb 2>/dev/null \
  | ./src/ruby/common_crawl_mapper.rb hello,world \
  | sort -t$'\t' -k1 \
  | ./src/ruby/common_crawl_reducer.rb

snoremac

# Bulk copy a crawl segment from S3 to the running cluster.

elastic-mapreduce -j j-2XP9O9IRLHHBU \
  --jar /home/hadoop/lib/emr-s3distcp-1.0.jar \
  --arg --src --arg s3://aws-publicdatasets/common-crawl/parse-output/segment/1346823845675 \
  --arg --srcPattern --arg '.*textData.*' \
  --arg --dest --arg hdfs:///common-crawl/parse-output/segment/1346823845675

snoremac

# Run the word count from local HDFS.

elastic-mapreduce -j j-2XP9O9IRLHHBU \
  --jar s3n://emr-examples.dius.com.au/java/emr-examples.jar \
  --main-class au.com.dius.emr.CommonCrawlTool \
  --arg -D --arg target.words=hello,world \
  --arg -D --arg base.uri=hdfs:///common-crawl \
  --arg -D --arg max.segments=1

snoremac

elastic-mapreduce --list

snoremac

#!/bin/bash -eu

# Example: sudo symlink-java jdk1.8.0_40.jdk jdk1.8.0_45.jdk

OLD_JDK=$1
NEW_JDK=$2

SYMLINK_DIR=/usr/bin
TARGET_DIR=/Library/Java/JavaVirtualMachines/$NEW_JDK/Contents/Home/bin