Skip to content

Instantly share code, notes, and snippets.

@bigsnarfdude
Created August 3, 2014 02:54
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save bigsnarfdude/55b3b6c394f923bcbe66 to your computer and use it in GitHub Desktop.
Save bigsnarfdude/55b3b6c394f923bcbe66 to your computer and use it in GitHub Desktop.
problem with algebird 0.7.0 jar from maven repo
antigensAir:bin antigen$ ./scala -cp /Users/antigen/Downloads/algebird-core_2.10-0.7.0.jar
Welcome to Scala version 2.10.4 (Java HotSpot(TM) 64-Bit Server VM, Java 1.7.0_60).
Type in expressions to have them evaluated.
Type :help for more information.
scala> import com.twitter.algebird._
import com.twitter.algebird._
scala> import HyperLogLog._
import HyperLogLog._
scala> import com.twitter.algebird.Monoid
import com.twitter.algebird.Monoid
scala> import com.twitter.algebird.DecayedValue
import com.twitter.algebird.DecayedValue
scala> import com.twitter.algebird.Operators._
import com.twitter.algebird.Operators._
scala>
scala>
scala>
scala>
scala> val hll = new HyperLogLogMonoid(4)
hll: com.twitter.algebird.HyperLogLogMonoid = com.twitter.algebird.HyperLogLogMonoid@7eba4a46
scala> val data = List(1, 1, 2, 2, 3, 3, 4, 4, 5, 5)
data: List[Int] = List(1, 1, 2, 2, 3, 3, 4, 4, 5, 5)
scala> val seqHll = data.map { hll(_) }
seqHll: List[com.twitter.algebird.HLL] = List(SparseHLL(4,Map(2 -> Max(1))), SparseHLL(4,Map(2 -> Max(1))), SparseHLL(4,Map(2 -> Max(2))), SparseHLL(4,Map(2 -> Max(2))), SparseHLL(4,Map(13 -> Max(1))), SparseHLL(4,Map(13 -> Max(1))), SparseHLL(4,Map(4 -> Max(1))), SparseHLL(4,Map(4 -> Max(1))), SparseHLL(4,Map(12 -> Max(1))), SparseHLL(4,Map(12 -> Max(1))))
scala> val sumHll = hll.sum(seqHll)
sumHll: com.twitter.algebird.HLL = DenseHLL(4,Array(0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0))
scala> val approxSizeOf = hll.sizeOf(sumHll)
approxSizeOf: com.twitter.algebird.Approximate[Long] = Approximate(1,4,9,0.9972)
scala> val actualSize = data.toSet.size
actualSize: Int = 5
scala> val estimate = approxSizeOf.estimate
estimate: Long = 4
scala> val NUM_HASHES = 6
NUM_HASHES: Int = 6
scala> val WIDTH = 32
WIDTH: Int = 32
scala> val SEED = 1
SEED: Int = 1
scala> val bfMonoid = new BloomFilterMonoid(NUM_HASHES, WIDTH, SEED)
bfMonoid: com.twitter.algebird.BloomFilterMonoid = BloomFilterMonoid(6,32,1)
scala> val bf = bfMonoid.create("1", "2", "3", "4", "100")
java.lang.NoClassDefFoundError: com/googlecode/javaewah/EWAHCompressedBitmap
at com.twitter.algebird.RichCBitSet$.apply(BloomFilter.scala:26)
at com.twitter.algebird.BFItem.$plus$plus(BloomFilter.scala:137)
at com.twitter.algebird.BloomFilterMonoid.plus(BloomFilter.scala:79)
at com.twitter.algebird.BloomFilterMonoid$$anonfun$create$1.apply(BloomFilter.scala:90)
at com.twitter.algebird.BloomFilterMonoid$$anonfun$create$1.apply(BloomFilter.scala:90)
at scala.collection.IndexedSeqOptimized$class.foldl(IndexedSeqOptimized.scala:51)
at scala.collection.IndexedSeqOptimized$class.foldLeft(IndexedSeqOptimized.scala:60)
at scala.collection.mutable.WrappedArray.foldLeft(WrappedArray.scala:34)
at com.twitter.algebird.BloomFilterMonoid.create(BloomFilter.scala:90)
at .<init>(<console>:22)
at .<clinit>(<console>)
at .<init>(<console>:7)
at .<clinit>(<console>)
at $print(<console>)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at scala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:734)
at scala.tools.nsc.interpreter.IMain$Request.loadAndRun(IMain.scala:983)
at scala.tools.nsc.interpreter.IMain.loadAndRunReq$1(IMain.scala:573)
at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:604)
at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:568)
at scala.tools.nsc.interpreter.ILoop.reallyInterpret$1(ILoop.scala:760)
at scala.tools.nsc.interpreter.ILoop.interpretStartingWith(ILoop.scala:805)
at scala.tools.nsc.interpreter.ILoop.command(ILoop.scala:717)
at scala.tools.nsc.interpreter.ILoop.processLine$1(ILoop.scala:581)
at scala.tools.nsc.interpreter.ILoop.innerLoop$1(ILoop.scala:588)
at scala.tools.nsc.interpreter.ILoop.loop(ILoop.scala:591)
at scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply$mcZ$sp(ILoop.scala:882)
at scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply(ILoop.scala:837)
at scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply(ILoop.scala:837)
at scala.tools.nsc.util.ScalaClassLoader$.savingContextLoader(ScalaClassLoader.scala:135)
at scala.tools.nsc.interpreter.ILoop.process(ILoop.scala:837)
at scala.tools.nsc.MainGenericRunner.runTarget$1(MainGenericRunner.scala:83)
at scala.tools.nsc.MainGenericRunner.process(MainGenericRunner.scala:96)
at scala.tools.nsc.MainGenericRunner$.main(MainGenericRunner.scala:105)
at scala.tools.nsc.MainGenericRunner.main(MainGenericRunner.scala)
Caused by: java.lang.ClassNotFoundException: com.googlecode.javaewah.EWAHCompressedBitmap
at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
... 38 more
scala> val approxBool = bf.contains("1")
<console>:18: error: not found: value bf
val approxBool = bf.contains("1")
^
scala> val res = approxBool.isTrue
<console>:18: error: not found: value approxBool
val res = approxBool.isTrue
^
scala> val DELTA = 1E-10
DELTA: Double = 1.0E-10
scala> val EPS = 0.001
EPS: Double = 0.001
scala> val SEED = 1
SEED: Int = 1
scala> val CMS_MONOID = new CountMinSketchMonoid(EPS, DELTA, SEED)
CMS_MONOID: com.twitter.algebird.CountMinSketchMonoid = com.twitter.algebird.CountMinSketchMonoid@4c36c09a
scala> val data = List(1L, 1L, 3L, 4L, 5L)
data: List[Long] = List(1, 1, 3, 4, 5)
scala> val cms = CMS_MONOID.create(data)
cms: com.twitter.algebird.CMS = CMSInstance(CMSCountsTable(Vector(Vector(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0...
scala> cms.totalCount
res0: Long = 5
scala> cms.frequency(1L).estimate
res1: Long = 2
scala> cms.frequency(2L).estimate
res2: Long = 0
scala> cms.frequency(3L).estimate
res3: Long = 1
scala> val data = List("1", "2", "3", "4", "5")
data: List[String] = List(1, 2, 3, 4, 5)
scala>
scala> val data = {
| val rnd = new scala.util.Random
| (1 to 100).map { _ => rnd.nextInt(1000).toDouble }.toSeq
| }
data: scala.collection.immutable.Seq[Double] = Vector(453.0, 959.0, 969.0, 737.0, 320.0, 668.0, 17.0, 39.0, 548.0, 259.0, 883.0, 622.0, 555.0, 608.0, 638.0, 236.0, 213.0, 796.0, 609.0, 486.0, 85.0, 379.0, 187.0, 254.0, 621.0, 606.0, 616.0, 222.0, 821.0, 520.0, 655.0, 53.0, 641.0, 271.0, 344.0, 149.0, 576.0, 128.0, 259.0, 141.0, 120.0, 87.0, 740.0, 490.0, 687.0, 834.0, 754.0, 237.0, 941.0, 265.0, 958.0, 769.0, 717.0, 813.0, 761.0, 252.0, 714.0, 362.0, 311.0, 232.0, 359.0, 754.0, 655.0, 476.0, 998.0, 119.0, 884.0, 971.0, 497.0, 364.0, 54.0, 265.0, 695.0, 64.0, 625.0, 862.0, 71.0, 785.0, 640.0, 944.0, 184.0, 713.0, 906.0, 41.0, 65.0, 944.0, 242.0, 418.0, 924.0, 77.0, 868.0, 242.0, 533.0, 506.0, 387.0, 200.0, 504.0, 275.0, 664.0, 793.0)
scala>
scala> val HalfLife = 10.0
HalfLife: Double = 10.0
scala> val normalization = HalfLife / math.log(2)
normalization: Double = 14.426950408889635
scala>
scala> implicit val monoid = DecayedValue.monoidWithEpsilon(1e-3)
monoid: com.twitter.algebird.Monoid[com.twitter.algebird.DecayedValue] = DecayedValueMonoid(0.001)
scala>
scala> data.zipWithIndex.scanLeft(Monoid.zero[DecayedValue]) { (previous, data) =>
| val (value, time) = data
| val decayed = Monoid.plus(previous, DecayedValue.build(value, time, HalfLife))
| println("At %d: decayed=%f".format(time, (decayed.value / normalization)))
| decayed
| }
At 0: decayed=31.399567
At 1: decayed=95.769647
At 2: decayed=156.522202
At 3: decayed=197.125325
At 4: decayed=206.105142
At 5: decayed=238.605129
At 6: decayed=223.804807
At 7: decayed=211.520543
At 8: decayed=235.340110
At 9: decayed=237.532599
At 10: decayed=282.830648
At 11: decayed=307.004080
At 12: decayed=324.914604
At 13: decayed=345.299393
At 14: decayed=366.398516
At 15: decayed=358.220177
At 16: decayed=348.995278
At 17: decayed=380.798624
At 18: decayed=397.510343
At 19: decayed=404.577217
At 20: decayed=383.375642
At 21: decayed=383.972401
At 22: decayed=371.220770
At 23: decayed=363.967164
At 24: decayed=382.637812
At 25: decayed=399.018421
At 26: decayed=414.995217
At 27: decayed=402.592097
At 28: decayed=432.539092
At 29: decayed=439.616896
At 30: decayed=455.578208
At 31: decayed=428.743178
At 32: decayed=444.462265
At 33: decayed=433.482245
At 34: decayed=428.297499
At 35: decayed=409.943590
At 36: decayed=422.416171
At 37: decayed=403.000508
At 38: decayed=393.965282
At 39: decayed=377.355980
At 40: decayed=360.403345
At 41: decayed=342.298592
At 42: decayed=370.668771
At 43: decayed=379.810404
At 44: decayed=401.994849
At 45: decayed=432.882931
At 46: decayed=456.157354
At 47: decayed=442.037448
At 48: decayed=477.660673
At 49: decayed=464.041566
At 50: decayed=499.369591
At 51: decayed=519.231321
At 52: decayed=534.158606
At 53: decayed=554.740468
At 54: decayed=570.339659
At 55: decayed=549.613027
At 56: decayed=562.297795
At 57: decayed=549.734322
At 58: decayed=534.477136
At 59: decayed=514.765816
At 60: decayed=505.177473
At 61: decayed=523.610546
At 62: decayed=533.947055
At 63: decayed=531.184024
At 64: decayed=564.788307
At 65: decayed=535.214575
At 66: decayed=560.647067
At 67: decayed=590.406801
At 68: decayed=585.318439
At 69: decayed=571.351972
At 70: decayed=536.833234
At 71: decayed=519.251519
At 72: decayed=532.652527
At 73: decayed=501.418523
At 74: decayed=511.161723
At 75: decayed=536.680038
At 76: decayed=505.661527
At 77: decayed=526.210941
At 78: decayed=535.333588
At 79: decayed=564.916993
At 80: decayed=539.840100
At 81: decayed=553.110017
At 82: decayed=578.869029
At 83: decayed=542.945805
At 84: decayed=511.091805
At 85: decayed=542.298610
At 86: decayed=522.756656
At 87: decayed=516.722759
At 88: decayed=546.166181
At 89: decayed=514.928299
At 90: decayed=540.610266
At 91: decayed=521.181376
At 92: decayed=523.224163
At 93: decayed=523.258653
At 94: decayed=515.042383
At 95: decayed=494.414479
At 96: decayed=496.239638
At 97: decayed=482.069501
At 98: decayed=495.811722
At 99: decayed=517.575265
res4: scala.collection.immutable.Seq[com.twitter.algebird.DecayedValue] = Vector(DecayedValue(0.0,-Infinity), DecayedValue(453.0,0.0), DecayedValue(1381.6639451661738,0.06931471805599453), DecayedValue(2258.138044056943,0.13862943611198905), DecayedValue(2843.9172945495243,0.20794415416798356), DecayedValue(2973.4686610168064,0.2772588722239781), DecayedValue(3442.344360029456,0.34657359027997264), DecayedValue(3228.82085613814,0.4158883083359671), DecayedValue(3051.596382539004,0.48520302639196167), DecayedValue(3395.2401017632665,0.5545177444479562), DecayedValue(3426.871029133915,0.6238324625039507), DecayedValue(4080.3837279236345,0.6931471805599453), DecayedValue(4429.1326362826985,0.7624618986159398), DecayedValue(4687.526873544153,0.8317766166719343), DecayedValue(4981.6172217320...
scala>
scala> val DELTA = 1E-8
DELTA: Double = 1.0E-8
scala> val EPS = 0.001
EPS: Double = 0.001
scala> val SEED = 1
SEED: Int = 1
scala> val HEAVY_HITTERS_COUNT = 10
HEAVY_HITTERS_COUNT: Int = 10
scala>
scala> implicit def string2Bytes(i : String) = i.toCharArray.map(_.toByte)
warning: there were 1 feature warning(s); re-run with -feature for details
string2Bytes: (i: String)Array[Byte]
scala>
scala>
scala> val PARAMS = SketchMapParams[String](SEED, EPS, DELTA, HEAVY_HITTERS_COUNT)
PARAMS: com.twitter.algebird.SketchMapParams[String] = SketchMapParams(1,2719,19,10)
scala> val MONOID = SketchMap.monoid[String, Long](PARAMS)
MONOID: com.twitter.algebird.SketchMapMonoid[String,Long] = com.twitter.algebird.SketchMapMonoid@1494fa59
scala> val data = List( ("1", 1L), ("3", 2L), ("4", 1L), ("5", 1L) )
data: List[(String, Long)] = List((1,1), (3,2), (4,1), (5,1))
scala> val sm = MONOID.create(data)
sm: com.twitter.algebird.SketchMap[String,Long] =
SketchMap(Row: 19, Cols: 2719. Dense elements:
List((282,1), (763,2), (2172,1), (2523,1))
List((112,1), (364,2), (923,1), (2433,1))
List((1615,1), (1652,1), (1794,2), (2357,1))
List((497,2), (1687,1), (2288,1), (2395,1))
List((1115,2), (1252,1), (1441,1), (1798,1))
List((309,1), (556,2), (944,1), (2580,1))
List((474,1), (570,2), (1073,1), (2271,1))
List((194,1), (1101,2), (1420,1), (2296,1))
List((505,1), (1943,1), (1949,1), (2047,2))
List((1062,1), (1646,1), (1840,1), (2139,2))
List((549,1), (820,1), (1468,1), (1547,2))
List((837,2), (1064,1), (1941,1), (2305,1))
List((338,1), (482,1), (1331,2), (2285,1))
List((158,1), (1010,2), (1244,1), (1725,1))
List((90,1), (606,1), (1464,1), (1474,2))
List((143,1), (768,1), (935,2), (1340,1))
List...
scala> sm.totalValue
res5: Long = 5
scala> MONOID.frequency(sm, "1")
res6: Long = 1
scala> MONOID.frequency(sm, "2")
res7: Long = 0
scala> MONOID.frequency(sm, "3")
res8: Long = 2
scala>
scala> Max(10) + Max(30) + Max(20)
res9: com.twitter.algebird.Max[Int] = Max(30)
scala> case class TwitterUser(val name: String, val numFollowers: Int) extends Ordered[TwitterUser] {
| def compare(that: TwitterUser): Int = {
| val c = this.numFollowers - that.numFollowers
| if (c == 0) this.name.compareTo(that.name) else c
| }
| }
defined class TwitterUser
scala>
scala> val barackobama = TwitterUser("BarackObama", 40267391)
barackobama: TwitterUser = TwitterUser(BarackObama,40267391)
scala> val katyperry = TwitterUser("katyperry", 48013573)
katyperry: TwitterUser = TwitterUser(katyperry,48013573)
scala> val ladygaga = TwitterUser("ladygaga", 40756470)
ladygaga: TwitterUser = TwitterUser(ladygaga,40756470)
scala> val miguno = TwitterUser("miguno", 731)
miguno: TwitterUser = TwitterUser(miguno,731)
scala> val taylorswift = TwitterUser("taylorswift13", 37125055)
taylorswift: TwitterUser = TwitterUser(taylorswift13,37125055)
scala> val winner: Max[TwitterUser] = Max(barackobama) + Max(katyperry) + Max(ladygaga) + Max(miguno) + Max(taylorswift)
winner: com.twitter.algebird.Max[TwitterUser] = Max(TwitterUser(katyperry,48013573))
scala> Min(10) + Min(20) + Min(30)
res10: com.twitter.algebird.Min[Int] = Min(10)
scala> val data2 = Map(1 -> 1, 2 -> 1)
data2: scala.collection.immutable.Map[Int,Int] = Map(1 -> 1, 2 -> 1)
scala> val data1 = Map(1 -> 3, 2 -> 5, 3 -> 7, 5 -> 1)
data1: scala.collection.immutable.Map[Int,Int] = Map(1 -> 3, 2 -> 5, 3 -> 7, 5 -> 1)
scala> data1 + data2
res11: scala.collection.immutable.Map[Int,Int] = Map(1 -> 4, 2 -> 6, 3 -> 7, 5 -> 1)
scala> Set(1,2,3) + Set(3,4,5)
res12: scala.collection.immutable.Set[Int] = Set(5, 1, 2, 3, 4)
scala> List(1,2,3) + List(3,4,5)
res13: List[Int] = List(1, 2, 3, 3, 4, 5)
scala> Map(1 -> 3, 2 -> 4, 3 -> 1) * Map(2 -> 2)
res14: scala.collection.immutable.Map[Int,Int] = Map(2 -> 8)
scala> Map(1 -> Set(2,3), 2 -> Set(1)) + Map(2 -> Set(2,3))
res15: scala.collection.immutable.Map[Int,scala.collection.immutable.Set[Int]] = Map(1 -> Set(2, 3), 2 -> Set(1, 2, 3))
scala>
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment