Skip to content

Instantly share code, notes, and snippets.

View geoHeil's full-sized avatar

geoHeil geoHeil

View GitHub Profile
@geoHeil
geoHeil / exception.scala
Created November 27, 2016 10:04
spark null exception
import org.apache.log4j.{Level, Logger}
import org.apache.spark.SparkConf
import org.apache.spark.sql.functions._
import org.apache.spark.sql.{Dataset, SparkSession}
case class FooBar(city: String, postcode: String)
object Foo extends App {
Logger.getLogger("org").setLevel(Level.WARN)
@geoHeil
geoHeil / estim2trans.scala
Last active November 28, 2016 15:02
Spark convert estimator to transformer
// Copyright (C) 2016 Georg Heiler
// master thesis to detect fraud of never paying customers
package org.apache.spark.ml.feature
import org.apache.log4j.{Level, Logger}
import org.apache.spark.SparkConf
import org.apache.spark.ml.param.{Param, ParamMap, Params}
import org.apache.spark.ml.util._
import org.apache.spark.ml.{Estimator, Model, Transformer}
@geoHeil
geoHeil / estimatorProblem.scala
Created November 28, 2016 15:08
estimator problem
// Copyright (C) 2016 Georg Heiler
// master thesis to detect fraud of never paying customers
package org.apache.spark.ml.feature
import org.apache.log4j.{ Level, Logger }
import org.apache.spark.SparkConf
import org.apache.spark.ml.param.{ Param, ParamMap, Params }
import org.apache.spark.ml.util._
import org.apache.spark.ml.{ Estimator, Model, Transformer }
// Copyright (C) 2016 Georg Heiler
// master thesis to detect fraud of never paying customers
package org.apache.spark.ml.feature
import org.apache.log4j.{ Level, Logger }
import org.apache.spark.SparkConf
import org.apache.spark.ml.param.{ Param, ParamMap, Params }
import org.apache.spark.ml.util._
import org.apache.spark.ml.{ Estimator, Model, Transformer }
@geoHeil
geoHeil / gist:fe82d7c1384953b7d2eeb851a95f17b0
Last active November 29, 2016 08:26
estimator only using default value
package org.apache.spark.ml.feature
import org.apache.log4j.{ Level, Logger }
import org.apache.spark.SparkConf
import org.apache.spark.ml.param.{ Param, ParamMap, Params }
import org.apache.spark.ml.util._
import org.apache.spark.ml.{ Estimator, Model, Transformer }
import org.apache.spark.sql.functions._
import org.apache.spark.sql.types._
import org.apache.spark.sql.{ DataFrame, Dataset, SparkSession }
@geoHeil
geoHeil / xgboost.scala
Last active January 11, 2017 05:40
xgboost 4j scala tracker problem
import ml.dmlc.xgboost4j.scala.spark.{ TrackerConf, XGBoostEstimator }
import org.apache.log4j.{ Level, Logger }
import org.apache.spark.SparkConf
import org.apache.spark.ml.Pipeline
import org.apache.spark.ml.evaluation.BinaryClassificationEvaluator
import org.apache.spark.ml.feature.VectorAssembler
import org.apache.spark.ml.tuning.{ CrossValidator, ParamGridBuilder }
import org.apache.spark.sql.SparkSession
import scala.concurrent.duration._
@geoHeil
geoHeil / scala.scala
Created January 11, 2017 14:32
summary statistics
case class MeasureUnit(name: String, value: Double)
val measureSeq = Seq(MeasureUnit("metric1", 0.04), MeasureUnit("metric1", 0.09),
MeasureUnit("metric2", 0.64), MeasureUnit("metric2", 0.34), MeasureUnit("metric2", 0.84))
type Name = String
// "metric1" -> Seq(0.04, 0.09), "metric2" -> Seq(0.64, 0.34, 0.84)
val groupedMeasures: Map[Name, Seq[Double]] =
measureSeq
FORMAT: 1A
HOST: http://polls.apiblueprint.org/
# test
Polls is a simple API allowing consumers to view polls and vote in them.
## Questions Collection [/questions]
### List All Questions [GET]
@geoHeil
geoHeil / igraph.R
Last active January 18, 2017 07:20
Igraph calculate custom metrics
library(igraph)
id = c("a", "b", "c", "d", "e", "f", "g")
name = c("Alice", "Bob", "Charlie", "David", "Esther", "Fanny", "Gaby")
directConnectionToTrump = c(TRUE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE)
verticeData <- data.frame(id, name, directConnectionToTrump)
verticeData
src <- c("a", "b", "c", "f", "e", "e", "d", "a")
dst <- c("b", "c", "b", "c", "f", "d", "a", "e")
relationship <-c("A", "B", "B", "B", "B", "A", "A", "A")
/etc/nginx # ls
conf.d fastcgi.conf.default fastcgi_params.default koi-win mime.types.default nginx.conf scgi_params uwsgi_params win-utf
fastcgi.conf fastcgi_params koi-utf mime.types modules nginx.conf.default scgi_params.default uwsgi_params.default
/etc/nginx # ls /etc/nginx/conf.d/
default.conf sites-enabled.conf
/etc/nginx # cat conf.d/sites-enabled.conf
server {
listen 8080;
server_name application;