Skip to content

Instantly share code, notes, and snippets.

Mark M markmo

  • Australia
Block or report user

Report or block markmo

Hide content and notifications from this user.

Learn more about blocking users

Contact Support about this user’s behavior.

Learn more about reporting abuse

Report abuse
View GitHub Profile
View gist:2a6e3e22266ca51a6aef
import com.typesafe.config.ConfigFactory
import star.io.{JdbcReader, Reader}
import star.{Loader, StarConfig}
/**
* Created by markmo on 12/03/2016.
*/
class StarSpec extends UnitSpec {
val starConf = new StarConfig(ConfigFactory.load("star.conf"))
@markmo
markmo / gist:7653ba15bd054887286e
Created Mar 12, 2016
Transformation DSL Example
View gist:7653ba15bd054887286e
/* WIP - NOT READY
"DSL notation" should "execute a pipeline" in {
import diamond.transformation._
// import helper functions such as fieldLocator
import RowTransformation._
// create a new context object that can pass state between transformations
val ctx = new TransformationContext
View gist:b057e2537b377e23005c
import java.util.Calendar
import diamond.io.{CSVSink, CSVSource}
import diamond.models.{AttributeType, Event, Feature}
import diamond.store.{FeatureStore, FeatureStoreRepository}
import diamond.transform.TransformationContext
import diamond.transform.row.{AppendColumnRowTransformation, RowTransformation}
import diamond.transform.sql.NamedSQLTransformation
import diamond.transform.table.RowTransformationPipeline
import diamond.utility.dateFunctions._
View gist:2c775674307bff284f2b
import java.util.Calendar
import diamond.models.Event
import diamond.transform.eventFunctions._
import diamond.utility.dateFunctions._
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.DataFrame
import org.apache.spark.streaming.Duration
/**
View gist:84c4fa00a7e03f673d62
/**
* Created by markmo on 20/01/2016.
*/
class DemoSpec extends UnitSpec {
"Data" should "be processed" in {
val demo = sqlContext.read.load(s"$BASE_URI/$LAYER_RAW/Customer_Demographics.parquet")
val tx = sqlContext.read.load(s"$BASE_URI/$LAYER_RAW/Customer_Transactions.parquet")
View gist:71fc668207b2f9cb2e4d
/*
* Alternative implementation using a Custom Interpolator.
*
* @param sc StringContext
*/
implicit class TemplateHelper(val sc: StringContext) extends AnyVal {
def t(args: Any*): String = {
val strings = sc.parts.iterator
val expressions = args.iterator
View gist:87e2a11bce0fadf8e773
package diamond.transform
import org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructType, TimestampType}
/**
* Created by markmo on 12/12/2015.
*/
object schemas {
val inputSchema = StructType(Seq(
@markmo
markmo / gist:2f79e46397567bca7acf
Created Mar 12, 2016
Spark Transformation DSL
View gist:2f79e46397567bca7acf
package diamond.transform
import org.apache.spark.sql.{Row, DataFrame}
import org.apache.spark.sql.types.DataType
import scala.language.implicitConversions
/**
* All this is WIP - NOT READY
*
View gist:c8ebb803b06b649ad48d
trait FeatureType[V] {
val name: String
}
trait FeatureX[V] extends FeatureType[V] {
val value: V
}
case class FeatureCollection(id: Int,
View gist:2493df23cc96aca8f851
package diamond.models
import java.util.Date
/**
* @param entity String entity id (usually hashed)
* @param eventType String attribute name
* @param ts Date event timestamp
* @param namespace String logical grouping
* @param session Option[String] session id
You can’t perform that action at this time.