Skip to content

Instantly share code, notes, and snippets.

@eshioji
eshioji / GroupedIterator.scala
Last active October 15, 2017 18:07
GroupedIterator (very useful to use with Spark's repartitionAndSortWithinPartitions)
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@eshioji
eshioji / wip
Created June 3, 2015 09:52
wip
[root@strm02l1 ~]# storm jar /tmp/storm_perf_test-1.0.0-SNAPSHOT-jar-with-dependencies.jar com.yahoo.storm.perftest.Main --spoutParallel 4 --boltParallel 8 --maxSpoutPending 1000 --ackEnabled
Running: /usr/java/latest/bin/java -client -Dstorm.options= -Dstorm.home=/usr/share/storm-0.9.4 -Dstorm.log.dir=/var/log/storm -Djava.library.path=/usr/bin -Dstorm.conf.file= -cp /usr/share/storm-0.9.4/lib/asm-4.0.jar:/usr/share/storm-0.9.4/lib/carbonite-1.4.0.jar:/usr/share/storm-0.9.4/lib/chill-java-0.3.5.jar:/usr/share/storm-0.9.4/lib/clj-stacktrace-0.2.2.jar:/usr/share/storm-0.9.4/lib/clj-time-0.4.1.jar:/usr/share/storm-0.9.4/lib/clojure-1.5.1.jar:/usr/share/storm-0.9.4/lib/clout-1.0.1.jar:/usr/share/storm-0.9.4/lib/commons-codec-1.6.jar:/usr/share/storm-0.9.4/lib/commons-exec-1.1.jar:/usr/share/storm-0.9.4/lib/commons-fileupload-1.2.1.jar:/usr/share/storm-0.9.4/lib/commons-io-2.4.jar:/usr/share/storm-0.9.4/lib/commons-lang-2.5.jar:/usr/share/storm-0.9.4/lib/commons-logging-1.1.3.jar:/usr/share/storm-0.9.4/lib/compoj
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.Timer;
import com.google.common.base.Throwables;
import com.google.common.util.concurrent.AbstractService;
import kafka.javaapi.producer.Producer;
import kafka.producer.KeyedMessage;
import kafka.producer.ProducerConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@eshioji
eshioji / gist:3868903
Created October 10, 2012 22:26
generated template sample
#Configuration file template for blahblahblah.properties generated on Wed Oct 10 15:24:18 BST 2012
# Uncomment and provide values to overwrite default value
# Name:ACCESS_KEY Type:String Default value=No default value
# ACCESS_KEY=null
# Name:MAX_RETRY Type:Integer Default value=10
# MAX_RETRY=10
# Name:JOB_MANAGER_THREAD_NUM Type:Integer Default value=4 Description:Number of threads used by the job manager.
@eshioji
eshioji / gist:3868631
Created October 10, 2012 21:40
Usage example
public MyService(Config config) {
HttpClient.Builder builder = new HttpClient.Builder();
builder
.setUserAgent(config.value(Prop.USER_AGENT, String.class))
.setRequestTimeout(config.value(Prop.REQ_TIMEOUT,Integer.class))
// etc.etc….
.setCompression(config.value(Prop.COMPRESSION_ENABLED, Boolean.class));
if(config.value(Prop.PROXY_ENABLED,Boolean.class)){
Proxy proxy = config.value(Prop.PROXY,Proxy.class);
builder.setProxy(proxy);
@eshioji
eshioji / gist:3868359
Created October 10, 2012 20:57
My config code
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import com.google.common.io.Closeables;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.annotation.concurrent.ThreadSafe;
import java.io.*;