View gist:879473
;; Logical / unsigned shift right in Clojure (>>> operator in Java) | |
;; | |
;; Algorithm from http://www.sitepoint.com/forums/php-34/unsigned-right-bitwise-shift-449434.html | |
(defn logical-shift-right [n s] | |
(if (neg? n) | |
(bit-or (bit-shift-right (bit-and n 0x7fffffff) s) | |
(bit-shift-right 0x40000000 (dec s))) | |
(bit-shift-right n s))) |
View gist:1053586
ERROR in (test-parse-line-op) (FlowStepJob.java:173) | |
Uncaught exception, not in assertion. | |
expected: nil | |
actual: cascading.flow.FlowException: step failed: (1/1) ...QDWpEp0jBsowT4Bo4U+++TI/-Tmp-/sink/0ba51806-c2a2-41bf-96cf-06bfe61b86bc"]"], with job id: job_local_0001, please see cluster logs for failure messages | |
at cascading.flow.FlowStepJob.blockOnJob (FlowStepJob.java:173) | |
cascading.flow.FlowStepJob.start (FlowStepJob.java:138) | |
cascading.flow.FlowStepJob.call (FlowStepJob.java:127) | |
cascading.flow.FlowStepJob.call (FlowStepJob.java:39) | |
java.util.concurrent.FutureTask$Sync.innerRun (FutureTask.java:303) | |
java.util.concurrent.FutureTask.run (FutureTask.java:138) |
View gist:1054917
$ lein test | |
Testing cascalog-weather.test.weather | |
11/06/29 22:46:49 INFO hadoop.Hadoop18TapUtil: setting up task: 'attempt_002147483647_0000_m_000000_0' - file:/var/folders/YZ/YZO0QDWpEp0jBsowT4Bo4U+++TI/-Tmp-/tap57/4abafe2e-a136-4415-b7bc-09fd51acc301/_temporary/_attempt_002147483647_0000_m_000000_0 | |
11/06/29 22:46:49 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable | |
11/06/29 22:46:49 INFO hadoop.TapCollector: closing tap collector for: /var/folders/YZ/YZO0QDWpEp0jBsowT4Bo4U+++TI/-Tmp-/tap57/4abafe2e-a136-4415-b7bc-09fd51acc301/part-00000 | |
11/06/29 22:46:49 INFO hadoop.Hadoop18TapUtil: committing task: 'attempt_002147483647_0000_m_000000_0' - file:/var/folders/YZ/YZO0QDWpEp0jBsowT4Bo4U+++TI/-Tmp-/tap57/4abafe2e-a136-4415-b7bc-09fd51acc301/_temporary/_attempt_002147483647_0000_m_000000_0 | |
11/06/29 22:46:49 INFO hadoop.Hadoop18TapUtil: saved output of task 'attempt_002147483647_0000_m_000000_0' to file:/var/folders/YZ/YZO0QDWpEp0jBsow |
View gist:1074930
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.RuntimeException: java.lang.NullPointerException | |
LazySeq.java:47 clojure.lang.LazySeq.sval | |
LazySeq.java:56 clojure.lang.LazySeq.seq | |
Cons.java:39 clojure.lang.Cons.next | |
RT.java:1178 clojure.lang.RT.length | |
RT.java:1157 clojure.lang.RT.seqToArray | |
LazySeq.java:126 clojure.lang.LazySeq.toArray | |
RT.java:1135 clojure.lang.RT.toArray | |
core.clj:300 clojure.core/to-array |
View SortTest.java
@SuppressWarnings("serial") | |
@PlatformRunner.Platform({ LocalPlatform.class, HadoopPlatform.class}) | |
public class SortTest extends PlatformTestCase { | |
private static final inputFileSort = "src/test/data/sort.txt"; | |
public SortTest() { | |
super(false); | |
} |
View positweets.hive
drop table if exists raw_tweets; | |
drop table if exists tweets; | |
drop table if exists positive_hashtags_per_day; | |
drop table if exists count_positive_hashtags_per_day; | |
drop table if exists top5_positive_hashtags_per_day; | |
create table raw_tweets (json string); | |
load data local inpath 'sample.json' into table raw_tweets; | |
create table tweets as |
View export-scm-config.sh
#!/bin/bash | |
USERNAME=admin | |
PASSWORD=admin | |
SCM_URL=http://localhost:7180 | |
COOKIES_FILE=cookies.txt | |
EXPORT_FILE=export.txt | |
wget -q --post-data="j_username=${USERNAME}&j_password=${PASSWORD}" --save-cookies ${COOKIES_FILE} --keep-session-cookies -O /dev/null ${SCM_URL}/j_spring_security_check | |
wget -q -O ${EXPORT_FILE} --load-cookies ${COOKIES_FILE} ${SCM_URL}/cmf/exportCLI |
View heroku_dynos.sh
#!/bin/sh | |
# Return total number of Heroku dynos for an account | |
# | |
# Uses: | |
# jutil <https://github.com/misterfifths/jutil.git> | |
# underscore.js <http://underscorejs.org/> | |
API_KEY="<your Heroku API key>" | |
curl -s -H "Accept: application/json" -u :$API_KEY https://api.heroku.com/apps | jselect 'dynos' | jutil 'return _.reduce($, function(memo, num){ return memo + num; }, 0);' |
View WebHDFS.txt
curl -O http://python-distribute.org/distribute_setup.py | |
sudo python distribute_setup.py | |
curl -O https://raw.github.com/pypa/pip/master/contrib/get-pip.py | |
sudo python get-pip.py | |
sudo pip install webhdfs | |
cp /usr/lib/python2.6/site-packages/webhdfs/example.py . |
View gist:5056544
0) Download avro-tools jar file from avro.apache.org | |
1) Extract Avro schema using avro-tools.jar | |
java -jar avro-tools*.jar getschema file.avro > file.avsc | |
2) Upload Avro schema to hdfs | |
hadoop fs -cp file.avsc /use/training/file.avsc |
OlderNewer