Amit Mor amimimor
amimimor
/ http_dispatcher_and_workers.go
Created
March 9, 2016 07:48
A template for an HTTP Request dispatcher and worker 'pool' (sharing channels with the Dispatcher)
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| package main | |
| import ( | |
| // "flag" | |
| "fmt" | |
| "log" | |
| "net/http" | |
| "os" | |
| "strings" |
amimimor
/ kafka producer and consumer
Last active
August 29, 2015 14:24
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/python | |
| from kafka import SimpleProducer, KafkaClient, RoundRobinPartitioner, KeyedProducer | |
| import os, time | |
| broker = os.environ["broker"] | |
| topic = os.environ["topic"] | |
| parts = int(os.environ["parts"]) | |
| # To send messages synchronously | |
| kafka = KafkaClient(str.format('{}:9092', broker)) |
amimimor
/ Delimited.scala
Created
May 24, 2012 13:39
Scalding DelimitedScheme2 - with optional skipHeader and writeHeader TextDelimited
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import cascading.tuple.Fields | |
| import cascading.scheme.Scheme | |
| import cascading.scheme.local.{TextLine => CLTextLine, TextDelimited => CLTextDelimited} | |
| import cascading.scheme.hadoop.{TextLine => CHTextLine, TextDelimited => CHTextDelimited, SequenceFile => CHSequenceFile} | |
| import org.apache.hadoop.mapred.{JobConf, RecordReader, OutputCollector } | |
| import com.twitter.scalding.{Args, FixedPathSource, Job, Source} | |
| trait DelimitedScheme2 extends Source { | |
| //override these as needed: |