This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import com.typesafe.config.ConfigFactory | |
import star.io.{JdbcReader, Reader} | |
import star.{Loader, StarConfig} | |
/** | |
* Created by markmo on 12/03/2016. | |
*/ | |
class StarSpec extends UnitSpec { | |
val starConf = new StarConfig(ConfigFactory.load("star.conf")) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
/* WIP - NOT READY | |
"DSL notation" should "execute a pipeline" in { | |
import diamond.transformation._ | |
// import helper functions such as fieldLocator | |
import RowTransformation._ | |
// create a new context object that can pass state between transformations | |
val ctx = new TransformationContext |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import java.util.Calendar | |
import diamond.io.{CSVSink, CSVSource} | |
import diamond.models.{AttributeType, Event, Feature} | |
import diamond.store.{FeatureStore, FeatureStoreRepository} | |
import diamond.transform.TransformationContext | |
import diamond.transform.row.{AppendColumnRowTransformation, RowTransformation} | |
import diamond.transform.sql.NamedSQLTransformation | |
import diamond.transform.table.RowTransformationPipeline | |
import diamond.utility.dateFunctions._ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import java.util.Calendar | |
import diamond.models.Event | |
import diamond.transform.eventFunctions._ | |
import diamond.utility.dateFunctions._ | |
import org.apache.spark.rdd.RDD | |
import org.apache.spark.sql.DataFrame | |
import org.apache.spark.streaming.Duration | |
/** |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
/** | |
* Created by markmo on 20/01/2016. | |
*/ | |
class DemoSpec extends UnitSpec { | |
"Data" should "be processed" in { | |
val demo = sqlContext.read.load(s"$BASE_URI/$LAYER_RAW/Customer_Demographics.parquet") | |
val tx = sqlContext.read.load(s"$BASE_URI/$LAYER_RAW/Customer_Transactions.parquet") |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
/* | |
* Alternative implementation using a Custom Interpolator. | |
* | |
* @param sc StringContext | |
*/ | |
implicit class TemplateHelper(val sc: StringContext) extends AnyVal { | |
def t(args: Any*): String = { | |
val strings = sc.parts.iterator | |
val expressions = args.iterator |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package diamond.transform | |
import org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructType, TimestampType} | |
/** | |
* Created by markmo on 12/12/2015. | |
*/ | |
object schemas { | |
val inputSchema = StructType(Seq( |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package diamond.transform | |
import org.apache.spark.sql.{Row, DataFrame} | |
import org.apache.spark.sql.types.DataType | |
import scala.language.implicitConversions | |
/** | |
* All this is WIP - NOT READY | |
* |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
trait FeatureType[V] { | |
val name: String | |
} | |
trait FeatureX[V] extends FeatureType[V] { | |
val value: V | |
} | |
case class FeatureCollection(id: Int, |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package diamond.models | |
import java.util.Date | |
/** | |
* @param entity String entity id (usually hashed) | |
* @param eventType String attribute name | |
* @param ts Date event timestamp | |
* @param namespace String logical grouping | |
* @param session Option[String] session id |
NewerOlder