Skip to content

Instantly share code, notes, and snippets.

;; Logical / unsigned shift right in Clojure (>>> operator in Java)
;;
;; Algorithm from http://www.sitepoint.com/forums/php-34/unsigned-right-bitwise-shift-449434.html
(defn logical-shift-right [n s]
(if (neg? n)
(bit-or (bit-shift-right (bit-and n 0x7fffffff) s)
(bit-shift-right 0x40000000 (dec s)))
(bit-shift-right n s)))
ERROR in (test-parse-line-op) (FlowStepJob.java:173)
Uncaught exception, not in assertion.
expected: nil
actual: cascading.flow.FlowException: step failed: (1/1) ...QDWpEp0jBsowT4Bo4U+++TI/-Tmp-/sink/0ba51806-c2a2-41bf-96cf-06bfe61b86bc"]"], with job id: job_local_0001, please see cluster logs for failure messages
at cascading.flow.FlowStepJob.blockOnJob (FlowStepJob.java:173)
cascading.flow.FlowStepJob.start (FlowStepJob.java:138)
cascading.flow.FlowStepJob.call (FlowStepJob.java:127)
cascading.flow.FlowStepJob.call (FlowStepJob.java:39)
java.util.concurrent.FutureTask$Sync.innerRun (FutureTask.java:303)
java.util.concurrent.FutureTask.run (FutureTask.java:138)
$ lein test
Testing cascalog-weather.test.weather
11/06/29 22:46:49 INFO hadoop.Hadoop18TapUtil: setting up task: 'attempt_002147483647_0000_m_000000_0' - file:/var/folders/YZ/YZO0QDWpEp0jBsowT4Bo4U+++TI/-Tmp-/tap57/4abafe2e-a136-4415-b7bc-09fd51acc301/_temporary/_attempt_002147483647_0000_m_000000_0
11/06/29 22:46:49 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
11/06/29 22:46:49 INFO hadoop.TapCollector: closing tap collector for: /var/folders/YZ/YZO0QDWpEp0jBsowT4Bo4U+++TI/-Tmp-/tap57/4abafe2e-a136-4415-b7bc-09fd51acc301/part-00000
11/06/29 22:46:49 INFO hadoop.Hadoop18TapUtil: committing task: 'attempt_002147483647_0000_m_000000_0' - file:/var/folders/YZ/YZO0QDWpEp0jBsowT4Bo4U+++TI/-Tmp-/tap57/4abafe2e-a136-4415-b7bc-09fd51acc301/_temporary/_attempt_002147483647_0000_m_000000_0
11/06/29 22:46:49 INFO hadoop.Hadoop18TapUtil: saved output of task 'attempt_002147483647_0000_m_000000_0' to file:/var/folders/YZ/YZO0QDWpEp0jBsow
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.RuntimeException: java.lang.NullPointerException
LazySeq.java:47 clojure.lang.LazySeq.sval
LazySeq.java:56 clojure.lang.LazySeq.seq
Cons.java:39 clojure.lang.Cons.next
RT.java:1178 clojure.lang.RT.length
RT.java:1157 clojure.lang.RT.seqToArray
LazySeq.java:126 clojure.lang.LazySeq.toArray
RT.java:1135 clojure.lang.RT.toArray
core.clj:300 clojure.core/to-array
@SuppressWarnings("serial")
@PlatformRunner.Platform({ LocalPlatform.class, HadoopPlatform.class})
public class SortTest extends PlatformTestCase {
private static final inputFileSort = "src/test/data/sort.txt";
public SortTest() {
super(false);
}
@jorisbontje
jorisbontje / positweets.hive
Created May 15, 2012 10:53
Twitter sentiment analysis using Apache Hive
drop table if exists raw_tweets;
drop table if exists tweets;
drop table if exists positive_hashtags_per_day;
drop table if exists count_positive_hashtags_per_day;
drop table if exists top5_positive_hashtags_per_day;
create table raw_tweets (json string);
load data local inpath 'sample.json' into table raw_tweets;
create table tweets as
@jorisbontje
jorisbontje / export-scm-config.sh
Created May 27, 2012 12:04
Export the Cloudera Manager configuration
#!/bin/bash
USERNAME=admin
PASSWORD=admin
SCM_URL=http://localhost:7180
COOKIES_FILE=cookies.txt
EXPORT_FILE=export.txt
wget -q --post-data="j_username=${USERNAME}&j_password=${PASSWORD}" --save-cookies ${COOKIES_FILE} --keep-session-cookies -O /dev/null ${SCM_URL}/j_spring_security_check
wget -q -O ${EXPORT_FILE} --load-cookies ${COOKIES_FILE} ${SCM_URL}/cmf/exportCLI
@jorisbontje
jorisbontje / heroku_dynos.sh
Created November 14, 2012 10:22
Heroku total number of dynos
#!/bin/sh
# Return total number of Heroku dynos for an account
#
# Uses:
# jutil <https://github.com/misterfifths/jutil.git>
# underscore.js <http://underscorejs.org/>
API_KEY="<your Heroku API key>"
curl -s -H "Accept: application/json" -u :$API_KEY https://api.heroku.com/apps | jselect 'dynos' | jutil 'return _.reduce($, function(memo, num){ return memo + num; }, 0);'
@jorisbontje
jorisbontje / WebHDFS.txt
Last active December 13, 2015 23:49
WebHDFS
curl -O http://python-distribute.org/distribute_setup.py
sudo python distribute_setup.py
curl -O https://raw.github.com/pypa/pip/master/contrib/get-pip.py
sudo python get-pip.py
sudo pip install webhdfs
cp /usr/lib/python2.6/site-packages/webhdfs/example.py .
@jorisbontje
jorisbontje / gist:5056544
Created February 28, 2013 12:57
Hive Avro.txt
0) Download avro-tools jar file from avro.apache.org
1) Extract Avro schema using avro-tools.jar
java -jar avro-tools*.jar getschema file.avro > file.avsc
2) Upload Avro schema to hdfs
hadoop fs -cp file.avsc /use/training/file.avsc