Skip to content

Instantly share code, notes, and snippets.

View koen-dejonghe's full-sized avatar

Koen Dejonghe koen-dejonghe

33 followers · 5 following
View GitHub Profile
@koen-dejonghe
koen-dejonghe / PooledSocketStreamPublisher.scala
Created June 3, 2015 07:50
Implementation of a connection pool for use with spark streaming. See http://stackoverflow.com/questions/30450763/spark-streaming-and-connection-pool-implementation
package net.atos.sparti.pub
import java.io.PrintStream
import java.net.Socket
import org.apache.commons.pool2.impl.{DefaultPooledObject, GenericObjectPool}
import org.apache.commons.pool2.{ObjectPool, PooledObject, BasePooledObjectFactory}
import org.apache.spark.streaming.dstream.DStream
class PooledSocketStreamPublisher[T](host: String, port: Int)
@koen-dejonghe
koen-dejonghe / RabbitReceiver.scala
Last active March 1, 2016 17:58
package botkop.sparti.receiver
import com.rabbitmq.client._
import org.apache.spark.Logging
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.StreamingContext
import org.apache.spark.streaming.dstream.ReceiverInputDStream
import org.apache.spark.streaming.receiver.Receiver
import scala.reflect.ClassTag
@koen-dejonghe
koen-dejonghe / LinearIterativeLearning.scala
Last active January 9, 2017 07:00
Toy example from lecture 3 in Geoffrey Hintons' course Neural Networks for Machine Learning
import scala.language.postfixOps
import scala.util.Random
object LinearIterativeLearning extends App {
case class LabeledData(numFish: Int, numChips: Int, numKetchup: Int, price: Double = 0.0)
case class Weights(fish: Double, chips: Double, ketchup: Double)
@koen-dejonghe
koen-dejonghe / SigmoidIterativeLearning.scala
Created January 10, 2017 13:19
package botkop.nn.lecture3
import scala.language.postfixOps
import scala.util.Random
object SigmoidIterativeLearning extends App {
case class LabeledData(numFish: Double, numChips: Double, numKetchup: Double, price: Double = 0.0)
case class Weights(fish: Double, chips: Double, ketchup: Double)
val trueWeights = Weights(0.150, 0.050, 0.100)
@koen-dejonghe
koen-dejonghe / SlidingWindow.scala
Last active August 19, 2022 06:52
case class SomeEvent(value: Long)
val events = Source
.tick(0 seconds, 250 millis, "")
.zipWithIndex
.map { case (_, l) =>
SomeEvent(l)
}
val group = Flow[SomeEvent].groupedWithin(100, 500 millis) // +/- 2 events per group
@koen-dejonghe
koen-dejonghe / gist:9e72a918edb2486804d9d31b9be3744f
Created December 11, 2017 05:00
min_char_rnn.out
This file has been truncated, but you can view the full file.
ap Love whof wamp m rove mak waizor in! shenven wor gpear I d selve for trave sp
----
iter 3326900, loss 40.139576923598945
----
ouc
O vow sppa'g nour love?
FhCVaxt let ard tave wak! por uppit. onoum--prroudix.
For tomn olip toor houk on trarid
Are I!
Wort deve wa?!
@koen-dejonghe
koen-dejonghe / min_char_rnn.out
Created December 11, 2017 05:01
This file has been truncated, but you can view the full file.
ap Love whof wamp m rove mak waizor in! shenven wor gpear I d selve for trave sp
----
iter 3326900, loss 40.139576923598945
----
ouc
O vow sppa'g nour love?
FhCVaxt let ard tave wak! por uppit. onoum--prroudix.
For tomn olip toor houk on trarid
Are I!
Wort deve wa?!
@koen-dejonghe
koen-dejonghe / AutoGradSpec.scala
Last active January 18, 2018 07:58
package scorch
import scorch.autograd._
import botkop.numsca.Tensor
import botkop.{numsca => ns}
import org.scalactic.{Equality, TolerantNumerics}
import org.scalatest.{FlatSpec, Matchers}
class AutoGradSpec extends FlatSpec with Matchers {
@koen-dejonghe
koen-dejonghe / DinosaurIslandCharRnn.scala
Last active February 20, 2018 08:07
Dinosaurus Island -- Character level language model
package scorch.examples
import botkop.numsca.Tensor
import botkop.{numsca => ns}
import scorch.autograd._
import scorch.nn.rnn.RecurrentModule
import scorch.nn.Optimizer
import scala.annotation.tailrec
import scala.io.Source
@koen-dejonghe
koen-dejonghe / NaiveMemoryManagement.scala
Last active May 27, 2018 08:12
Using the JVM's garbage collector to control off heap memory
package jtorch.cpu
import java.util.concurrent.atomic.AtomicLong
import com.typesafe.scalalogging.LazyLogging
import scala.concurrent.{Await, Future}
import scala.concurrent.duration._
import scala.language.postfixOps