Skip to content

Instantly share code, notes, and snippets.

@avibryant
Created January 22, 2015 17:49
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save avibryant/f158a2e4abe977dc15f3 to your computer and use it in GitHub Desktop.
Save avibryant/f158a2e4abe977dc15f3 to your computer and use it in GitHub Desktop.
case class SetSizeAggregator(hllBits: Int, maxSetSize: Int = 10)
extends MonoidAggregator[Array[Byte], Either[HLL, Set[Array[Byte]]], Long] {
def prepare(in: Array[Byte]) = Right(Set(in))
def present(sum: Either[HLL, Set[Array[Byte]]]) = {
sum match {
case Right(set) => set.size
case Left(hll) => hll.approximateSize.estimate
}
}
implicit val hllMonoid = new HyperLogLogMonoid(hllBits)
val monoid = new EventuallyMonoid[HLL, Set[Array[Byte]]](hllMonoid.batchCreate(_))(_.size > maxSetSize)
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment