Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Combine enrich submodules into a single SBT project
From fe136f1c24a2d3510f654348fa4522f48cdc6efb Mon Sep 17 00:00:00 2001
From: Anton Parkhomenko <mailbox@chuwy.me>
Date: Sat, 13 Jun 2020 14:52:47 +0700
Subject: [PATCH] Unify
---
beam.sbt | 88 ---------
build.sbt | 172 ++++++++++++++++++
common.sbt | 68 -------
...mmonSettings.scala => BuildSettings.scala} | 68 ++++++-
...nDependencies.scala => Dependencies.scala} | 106 +++++++----
project/StreamDependencies.scala | 78 --------
project/StreamSettings.scala | 42 -----
project/plugins.sbt | 3 +-
stream.sbt | 123 -------------
9 files changed, 300 insertions(+), 448 deletions(-)
delete mode 100644 beam.sbt
create mode 100644 build.sbt
delete mode 100644 common.sbt
rename project/{CommonSettings.scala => BuildSettings.scala} (51%)
rename project/{CommonDependencies.scala => Dependencies.scala} (53%)
delete mode 100644 project/StreamDependencies.scala
delete mode 100644 project/StreamSettings.scala
delete mode 100644 stream.sbt
diff --git a/beam.sbt b/beam.sbt
deleted file mode 100644
index c7e6c28d..00000000
--- a/beam.sbt
+++ /dev/null
@@ -1,88 +0,0 @@
-import sbt._
-import Keys._
-
-lazy val resolutionRepos = Seq(
- // For Snowplow
- "Snowplow Analytics Maven releases repo" at "http://maven.snplow.com/releases/",
- // For ua-parser
- "user-agent-parser repo" at "https://clojars.org/repo/"
-)
-
-// we fork a JVM per test in order to not reuse enrichment registries
-import Tests._
-{
- def oneJVMPerTest(tests: Seq[TestDefinition]) =
- tests.map(t => new Group(t.name, Seq(t), SubProcess(ForkOptions()))).toSeq
- testGrouping in Test := oneJVMPerTest((definedTests in Test).value)
-}
-
-lazy val commonSettings = Defaults.coreDefaultSettings ++ Seq(
- organization := "com.snowplowanalytics",
- version := "1.0.0",
- scalaVersion := "2.12.10",
- resolvers ++= resolutionRepos
-)
-
-lazy val paradiseDependency =
- "org.scalamacros" % "paradise" % scalaMacrosVersion cross CrossVersion.full
-lazy val macroSettings = Seq(
- libraryDependencies += "org.scala-lang" % "scala-reflect" % scalaVersion.value,
- addCompilerPlugin(paradiseDependency)
-)
-
-lazy val noPublishSettings = Seq(
- publish := {},
- publishLocal := {},
- publishArtifact := false
-)
-
-import com.typesafe.sbt.packager.docker._
-packageName in Docker := "snowplow/beam-enrich"
-maintainer in Docker := "Snowplow Analytics Ltd. <support@snowplowanalytics.com>"
-dockerBaseImage := "snowplow-docker-registry.bintray.io/snowplow/base-debian:0.1.0"
-daemonUser in Docker := "snowplow"
-dockerUpdateLatest := true
-
-lazy val scioVersion = "0.7.4"
-lazy val beamVersion = "2.11.0"
-lazy val sceVersion = "1.0.0"
-lazy val scalaMacrosVersion = "2.1.1"
-lazy val slf4jVersion = "1.7.25"
-lazy val circeVersion = "0.11.1"
-lazy val scalatestVersion = "3.0.8"
-
-lazy val root: Project = Project(
- "beam-enrich",
- file(".")
-).settings(
- commonSettings ++ macroSettings ++ noPublishSettings,
- description := "Streaming enrich job written using SCIO",
- buildInfoKeys := Seq[BuildInfoKey](organization, name, version, "sceVersion" -> sceVersion),
- buildInfoPackage := "com.snowplowanalytics.snowplow.enrich.beam.generated",
- scalafmtConfig := file(".scalafmt.conf"),
- scalafmtOnCompile := true,
- libraryDependencies ++= Seq(
- "com.spotify" %% "scio-core" % scioVersion,
- "org.apache.beam" % "beam-runners-google-cloud-dataflow-java" % beamVersion,
- "com.snowplowanalytics" %% "snowplow-common-enrich" % sceVersion,
- "org.slf4j" % "slf4j-simple" % slf4jVersion
- ) ++ Seq(
- "com.spotify" %% "scio-test" % scioVersion,
- "org.scalatest" %% "scalatest" % scalatestVersion,
- "io.circe" %% "circe-literal" % circeVersion
- ).map(_ % "test")
-).enablePlugins(JavaAppPackaging, DockerPlugin, BuildInfoPlugin)
-
-lazy val repl: Project = Project(
- "repl",
- file(".repl")
-).settings(
- commonSettings ++ macroSettings ++ noPublishSettings,
- description := "Scio REPL for beam-enrich",
- libraryDependencies ++= Seq(
- "com.spotify" %% "scio-repl" % scioVersion
- ),
- mainClass in Compile := Some("com.spotify.scio.repl.ScioShell")
-).dependsOn(
- root
-)
diff --git a/build.sbt b/build.sbt
new file mode 100644
index 00000000..c2f05f05
--- /dev/null
+++ b/build.sbt
@@ -0,0 +1,172 @@
+/*
+ * Copyright (c) 2012-2020 Snowplow Analytics Ltd. All rights reserved.
+ *
+ * This program is licensed to you under the Apache License Version 2.0, and
+ * you may not use this file except in compliance with the Apache License
+ * Version 2.0. You may obtain a copy of the Apache License Version 2.0 at
+ * http://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the Apache License Version 2.0 is distributed on an "AS
+ * IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the Apache License Version 2.0 for the specific language
+ * governing permissions and limitations there under.
+ */
+// =======================================================
+// scalafmt: {align.tokens = [":="]}
+// =======================================================
+
+/** Metaproject with aggregated modules */
+lazy val root = project
+ .in(file("."))
+ .settings(name := "enrich")
+ .settings(BuildSettings.basicSettings)
+ .aggregate(common, beam, stream, kinesis, kafka, nsq, stdin, integrationTests)
+
+/** Snowplow Common Enrich - core of ETL, published as Maven Artifact */
+lazy val common = project
+ .in(file("modules/common"))
+ .settings(
+ name := "snowplow-common-enrich",
+ description := "Common functionality for enriching raw Snowplow events"
+ )
+ .settings(BuildSettings.formatting)
+ .settings(BuildSettings.basicSettings)
+ .settings(BuildSettings.scalifySettings)
+ .settings(BuildSettings.publishSettings)
+ .settings(parallelExecution in Test := false)
+ .settings(
+ libraryDependencies ++= Seq(
+ Dependencies.Libraries.jodaTime,
+ Dependencies.Libraries.commonsCodec,
+ Dependencies.Libraries.useragent,
+ Dependencies.Libraries.jacksonDatabind,
+ Dependencies.Libraries.uaParser,
+ Dependencies.Libraries.postgresDriver,
+ Dependencies.Libraries.mysqlConnector,
+ Dependencies.Libraries.jaywayJsonpath,
+ Dependencies.Libraries.iabClient,
+ Dependencies.Libraries.yauaa,
+ Dependencies.Libraries.rhino,
+ Dependencies.Libraries.guava,
+ Dependencies.Libraries.circeOptics,
+ Dependencies.Libraries.circeJackson,
+ Dependencies.Libraries.refererParser,
+ Dependencies.Libraries.maxmindIplookups,
+ Dependencies.Libraries.scalaUri,
+ Dependencies.Libraries.scalaForex,
+ Dependencies.Libraries.scalaWeather,
+ Dependencies.Libraries.gatlingJsonpath,
+ Dependencies.Libraries.scalaLruMap,
+ Dependencies.Libraries.badRows,
+ Dependencies.Libraries.snowplowRawEvent,
+ Dependencies.Libraries.collectorPayload,
+ Dependencies.Libraries.schemaSniffer,
+ Dependencies.Libraries.specs2,
+ Dependencies.Libraries.specs2Cats,
+ Dependencies.Libraries.specs2Scalacheck,
+ Dependencies.Libraries.specs2Mock,
+ Dependencies.Libraries.circeLiteral % Test
+ )
+ )
+
+/** Common logic for Stream Enrich artifacts, not published */
+lazy val stream = project
+ .in(file("modules/stream"))
+ .settings(BuildSettings.allStreamSettings)
+ .settings(moduleName := "snowplow-stream-enrich")
+ .enablePlugins(BuildInfoPlugin)
+ .settings(
+ buildInfoKeys := Seq[BuildInfoKey](organization, name, version, "commonEnrichVersion" -> version.value),
+ buildInfoPackage := "com.snowplowanalytics.snowplow.enrich.stream.generated"
+ )
+ .dependsOn(common)
+
+/** Kinesis Stream Enrich, Docker/JAR */
+lazy val kinesis = project
+ .in(file("modules/kinesis"))
+ .settings(BuildSettings.allStreamSettings)
+ .settings(moduleName := "snowplow-stream-enrich-kinesis")
+ .settings(packageName in Docker := "snowplow/stream-enrich-kinesis")
+ .settings(libraryDependencies ++= Seq(
+ Dependencies.Libraries.kinesisClient,
+ Dependencies.Libraries.kinesisSdk,
+ Dependencies.Libraries.dynamodbSdk,
+ Dependencies.Libraries.jacksonCbor
+ ))
+ .enablePlugins(JavaAppPackaging, DockerPlugin)
+ .dependsOn(stream)
+
+/** Kafka Stream Enrich, Docker/JAR */
+lazy val kafka = project
+ .in(file("modules/kafka"))
+ .settings(moduleName := "snowplow-stream-enrich-kafka")
+ .settings(packageName in Docker := "snowplow/stream-enrich-kafka")
+ .settings(BuildSettings.allStreamSettings)
+ .settings(libraryDependencies ++= Seq(
+ Dependencies.Libraries.kafkaClients
+ ))
+ .enablePlugins(JavaAppPackaging, DockerPlugin)
+ .dependsOn(stream)
+
+/** NSQ Stream Enrich, Docker/JAR */
+lazy val nsq = project
+ .in(file("modules/nsq"))
+ .settings(moduleName := "snowplow-stream-enrich-nsq")
+ .settings(BuildSettings.allStreamSettings)
+ .settings(packageName in Docker := "snowplow/stream-enrich-nsq")
+ .settings(libraryDependencies ++= Seq(Dependencies.Libraries.nsqClient))
+ .enablePlugins(JavaAppPackaging, DockerPlugin)
+ .dependsOn(stream)
+
+/** Stdin Stream Enrich, JAR */
+lazy val stdin = project
+ .in(file("modules/stdin"))
+ .settings(moduleName := "snowplow-stream-enrich-stdin")
+ .settings(BuildSettings.allStreamSettings)
+ .dependsOn(stream)
+
+/** Beam Enrich, published as Docker image */
+lazy val beam =
+ project
+ .in(file("modules/beam"))
+ .dependsOn(common)
+ .settings(BuildSettings.basicSettings)
+ .settings(BuildSettings.formatting)
+ .settings(
+ description := "Streaming enrich job written using SCIO",
+ buildInfoKeys := Seq[BuildInfoKey](organization, name, version, "sceVersion" -> version.value),
+ buildInfoPackage := "com.snowplowanalytics.snowplow.enrich.beam.generated",
+ libraryDependencies ++= Seq(
+ Dependencies.Libraries.scio,
+ Dependencies.Libraries.beam,
+ Dependencies.Libraries.slf4j,
+ Dependencies.Libraries.scioTest,
+ Dependencies.Libraries.scalaTest,
+ Dependencies.Libraries.circeLiteral % Test,
+ ),
+ packageName in Docker := "snowplow/beam-enrich"
+ )
+ .settings(
+ libraryDependencies += "org.scala-lang" % "scala-reflect" % scalaVersion.value,
+ addCompilerPlugin("org.scalamacros" % "paradise" % "2.1.1" cross CrossVersion.full)
+ )
+ .settings(
+ publish := {},
+ publishLocal := {},
+ publishArtifact := false,
+ testGrouping in Test := BuildSettings.oneJVMPerTest((definedTests in Test).value)
+ )
+ .enablePlugins(JavaAppPackaging, DockerPlugin, BuildInfoPlugin)
+
+/** Metaproject for testing Stream Enrich */
+lazy val integrationTests = project
+ .in(file("modules/integration-tests"))
+ .settings(moduleName := "integration-tests")
+ .settings(BuildSettings.allStreamSettings)
+ .settings(BuildSettings.addExampleConfToTestCp)
+ .settings(libraryDependencies ++= Seq(
+ Dependencies.Libraries.kafka,
+ Dependencies.Libraries.jinJava
+ ))
+ .dependsOn(stream % "test->test", kafka % "test->compile")
diff --git a/common.sbt b/common.sbt
deleted file mode 100644
index 651dbc6c..00000000
--- a/common.sbt
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Copyright (c) 2012-2020 Snowplow Analytics Ltd. All rights reserved.
- *
- * This program is licensed to you under the Apache License Version 2.0, and
- * you may not use this file except in compliance with the Apache License
- * Version 2.0. You may obtain a copy of the Apache License Version 2.0 at
- * http://www.apache.org/licenses/LICENSE-2.0.
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the Apache License Version 2.0 is distributed on an "AS
- * IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- * implied. See the Apache License Version 2.0 for the specific language
- * governing permissions and limitations there under.
- */
-// =======================================================
-// scalafmt: {align.tokens = [":="]}
-// =======================================================
-lazy val root = project
- .in(file("."))
- .settings(
- name := "snowplow-common-enrich",
- version := "1.1.0",
- description := "Common functionality for enriching raw Snowplow events"
- )
- .settings(BuildSettings.formatting)
- .settings(BuildSettings.buildSettings)
- .settings(BuildSettings.publishSettings)
- .settings(parallelExecution in Test := false)
- .settings(
- libraryDependencies ++= Seq(
- // Java
- Dependencies.Libraries.jodaTime,
- Dependencies.Libraries.commonsCodec,
- Dependencies.Libraries.useragent,
- Dependencies.Libraries.jacksonDatabind,
- Dependencies.Libraries.uaParser,
- Dependencies.Libraries.postgresDriver,
- Dependencies.Libraries.mysqlConnector,
- Dependencies.Libraries.jaywayJsonpath,
- Dependencies.Libraries.iabClient,
- Dependencies.Libraries.yauaa,
- Dependencies.Libraries.kryo,
- Dependencies.Libraries.rhino,
- Dependencies.Libraries.guava,
- // Scala
- Dependencies.Libraries.cats,
- Dependencies.Libraries.circeOptics,
- Dependencies.Libraries.circeJackson,
- Dependencies.Libraries.refererParser,
- Dependencies.Libraries.maxmindIplookups,
- Dependencies.Libraries.scalaUri,
- Dependencies.Libraries.scalaForex,
- Dependencies.Libraries.scalaWeather,
- Dependencies.Libraries.scalaj,
- Dependencies.Libraries.gatlingJsonpath,
- Dependencies.Libraries.scalaLruMap,
- Dependencies.Libraries.badRows,
- // Thrift schemas
- Dependencies.Libraries.snowplowRawEvent,
- Dependencies.Libraries.collectorPayload,
- Dependencies.Libraries.schemaSniffer,
- // Scala (test only)
- Dependencies.Libraries.specs2,
- Dependencies.Libraries.specs2Cats,
- Dependencies.Libraries.specs2Scalacheck,
- Dependencies.Libraries.specs2Mock
- ) ++ Dependencies.Libraries.circeDeps
- )
diff --git a/project/CommonSettings.scala b/project/BuildSettings.scala
similarity index 51%
rename from project/CommonSettings.scala
rename to project/BuildSettings.scala
index d0aef8ec..1b6491d9 100644
--- a/project/CommonSettings.scala
+++ b/project/BuildSettings.scala
@@ -20,24 +20,27 @@ import Keys._
import bintray.BintrayPlugin._
import bintray.BintrayKeys._
+
+import com.typesafe.sbt.SbtNativePackager.autoImport._
+import com.typesafe.sbt.packager.linux.LinuxPlugin.autoImport._
+import com.typesafe.sbt.packager.docker.DockerPlugin.autoImport._
+
import org.scalafmt.sbt.ScalafmtPlugin.autoImport._
+import sbtassembly.AssemblyPlugin.autoImport._
+
object BuildSettings {
- // Basic settings for our app
+ /** Common settings for all modules */
lazy val basicSettings = Seq(
organization := "com.snowplowanalytics",
scalaVersion := "2.12.10",
- javacOptions := javaCompilerOptions,
+ version := "1.3.0",
+ javacOptions := Seq("-source", "1.8", "-target", "1.8"),
resolvers ++= Dependencies.resolutionRepos
)
- lazy val javaCompilerOptions = Seq(
- "-source", "1.8",
- "-target", "1.8"
- )
-
- // Makes our SBT app settings available from within the ETL
+ /** Custom BuildInfo replacement, used only by SCE now */
lazy val scalifySettings = Seq(
sourceGenerators in Compile += Def.task {
val file = (sourceManaged in Compile).value / "settings.scala"
@@ -53,8 +56,7 @@ object BuildSettings {
}.taskValue
)
- lazy val buildSettings = basicSettings ++ scalifySettings
-
+ /** Bintray Maven settings, used only by SCE now */
lazy val publishSettings = bintraySettings ++ Seq(
publishMavenStyle := true,
publishArtifact := true,
@@ -81,4 +83,50 @@ object BuildSettings {
scalafmtConfig := file(".scalafmt.conf"),
scalafmtOnCompile := true
)
+
+ /** Fork a JVM per test in order to not reuse enrichment registries */
+ def oneJVMPerTest(tests: Seq[TestDefinition]): Seq[Tests.Group] =
+ tests.map(t => Tests.Group(t.name, Seq(t), Tests.SubProcess(ForkOptions())))
+
+ /** Building fat jar, used only by SE now */
+ lazy val sbtAssemblySettings = Seq(
+ assemblyJarName in assembly := { s"${moduleName.value}-${version.value}.jar" },
+ assemblyMergeStrategy in assembly := {
+ case x if x.endsWith("ProjectSettings$.class") => MergeStrategy.first
+ case x if x.endsWith("module-info.class") => MergeStrategy.first
+ case x =>
+ val oldStrategy = (assemblyMergeStrategy in assembly).value
+ oldStrategy(x)
+ }
+ )
+
+ /** Add sample configuration for integration tests */
+ lazy val addExampleConfToTestCp = Seq(
+ unmanagedClasspath in Test += baseDirectory.value.getParentFile / "config"
+ )
+
+ /** Common Docker settings, used by both SE and BE */
+ lazy val dockerSettings = Seq(
+ maintainer in Docker := "Snowplow Analytics Ltd. <support@snowplowanalytics.com>",
+ dockerBaseImage := "snowplow-docker-registry.bintray.io/snowplow/base-debian:0.1.0",
+ daemonUser in Docker := "snowplow",
+ daemonUserUid in Docker := None,
+ defaultLinuxInstallLocation in Docker := "/home/snowplow", // must be home directory of daemonUser
+ dockerUpdateLatest := true
+ )
+
+ /** Common configuration for all Stream Enrich modules */
+ lazy val allStreamSettings = basicSettings ++ sbtAssemblySettings ++ dockerSettings ++ formatting ++
+ Seq(libraryDependencies := Seq(
+ Dependencies.Libraries.config,
+ Dependencies.Libraries.slf4j,
+ Dependencies.Libraries.log4jOverSlf4j,
+ Dependencies.Libraries.s3Sdk,
+ Dependencies.Libraries.gsSdk,
+ Dependencies.Libraries.scopt,
+ Dependencies.Libraries.pureconfig,
+ Dependencies.Libraries.snowplowTracker,
+ Dependencies.Libraries.specs2,
+ Dependencies.Libraries.scalacheck
+ ))
}
diff --git a/project/CommonDependencies.scala b/project/Dependencies.scala
similarity index 53%
rename from project/CommonDependencies.scala
rename to project/Dependencies.scala
index cf97ce11..afe4ca5e 100644
--- a/project/CommonDependencies.scala
+++ b/project/Dependencies.scala
@@ -18,16 +18,14 @@ import sbt._
object Dependencies {
val resolutionRepos = Seq(
- // For some misc Scalding and Twitter libs
- "Concurrent Maven Repo" at "http://conjars.org/repo",
+ // For some Twitter libs and uaParser utils
+ "Concurrent Maven Repo" at "https://conjars.org/repo",
// For Twitter's util functions
- "Twitter Maven Repo" at "http://maven.twttr.com/",
- // For Snowplow libs
- "Snowplow Analytics Maven repo" at "http://maven.snplow.com/releases/",
- "Snowplow Analytics Maven snapshot repo" at "http://maven.snplow.com/snapshots/",
- "Snowplow Bintray" at "https://snowplow.bintray.com/snowplow-maven/",
- // For uaParser utils
- "user-agent-parser repo" at "https://clojars.org/repo/"
+ "Twitter Maven Repo" at "https://maven.twttr.com/",
+ // For modern Snowplow libs
+ "Snowplow Bintray" at "https://snowplow.bintray.com/snowplow-maven/",
+ // For legacy Snowplow libs
+ ("Snowplow Analytics Maven repo" at "http://maven.snplow.com/releases/").withAllowInsecureProtocol(true)
)
object V {
@@ -35,18 +33,16 @@ object Dependencies {
val commonsCodec = "1.12"
val jodaTime = "2.10.1"
val useragent = "1.21"
- val jacksonDatabind = "2.9.8"
val uaParser = "1.4.3"
val postgresDriver = "42.2.5"
val mysqlConnector = "8.0.16"
val jaywayJsonpath = "2.4.0"
val iabClient = "0.1.0"
val yauaa = "5.8"
- val kryo = "2.24.0"
val guava = "28.1-jre" //used only for InetAddress (because it doesn't do dns lookup)
val rhino = "1.7.10"
- // Scala
- val cats = "1.6.0"
+ val slf4j = "1.7.26"
+
val refererParser = "1.0.0"
val maxmindIplookups = "0.6.1"
val circe = "0.11.1"
@@ -54,63 +50,97 @@ object Dependencies {
val circeJackson = "0.11.1"
val scalaForex = "0.7.0"
val scalaWeather = "0.5.0"
- val scalaj = "2.4.1"
val gatlingJsonpath = "0.6.14"
val scalaUri = "1.4.5"
val scalaLruMap = "0.3.0"
val badRows = "1.0.0"
- // Thrift schemas
+
val snowplowRawEvent = "0.1.0"
val collectorPayload = "0.0.0"
val schemaSniffer = "0.0.0"
- // Scala (test only)
+
+ val awsSdk = "1.11.566"
+ val gcpSdk = "1.106.0"
+ val kinesisClient = "1.10.0"
+ val kafka = "2.2.1"
+ val nsqClient = "1.2.0"
+ val jackson = "2.9.9"
+ val config = "1.3.4"
+
+ val scopt = "3.7.1"
+ val pureconfig = "0.11.0"
+ val snowplowTracker = "0.6.1"
+
val specs2 = "4.5.1"
+ val scalacheck = "1.14.0"
+ val jinJava = "2.5.0"
+
+ val scio = "0.7.4"
+ val beam = "2.11.0"
+ val macros = "2.1.1"
+ val scalaTest = "3.0.8"
}
object Libraries {
- // Java
val commonsCodec = "commons-codec" % "commons-codec" % V.commonsCodec
val jodaTime = "joda-time" % "joda-time" % V.jodaTime
val useragent = "eu.bitwalker" % "UserAgentUtils" % V.useragent
- val jacksonDatabind = "com.fasterxml.jackson.core" % "jackson-databind" % V.jacksonDatabind
+ val jacksonDatabind = "com.fasterxml.jackson.core" % "jackson-databind" % V.jackson
val uaParser = "com.github.ua-parser" % "uap-java" % V.uaParser
val postgresDriver = "org.postgresql" % "postgresql" % V.postgresDriver
val mysqlConnector = "mysql" % "mysql-connector-java" % V.mysqlConnector
val jaywayJsonpath = "com.jayway.jsonpath" % "json-path" % V.jaywayJsonpath
- val iabClient = "com.snowplowanalytics" % "iab-spiders-and-robots-client" % V.iabClient
val yauaa = "nl.basjes.parse.useragent" % "yauaa" % V.yauaa
- val kryo = "com.esotericsoftware.kryo" % "kryo" % V.kryo
val guava = "com.google.guava" % "guava" % V.guava
val rhino = "org.mozilla" % "rhino" % V.rhino
- // Scala
- val circeDeps = List(
- "circe-core",
- "circe-generic",
- "circe-parser",
- "circe-literal",
- "circe-java8"
- ).map("io.circe" %% _ % V.circe)
- val cats = "org.typelevel" %% "cats-core" % V.cats
+ val circeGeneric = "io.circe" %% "circe-generic" % V.circe
+ val circeParser = "io.circe" %% "circe-parser" % V.circe
+ val circeLiteral = "io.circe" %% "circe-literal" % V.circe
+ val circeJava8 = "io.circe" %% "circe-java8" % V.circe
val circeOptics = "io.circe" %% "circe-optics" % V.circeOptics
val circeJackson = "io.circe" %% "circe-jackson29" % V.circeJackson
+ val scalaUri = "io.lemonlabs" %% "scala-uri" % V.scalaUri
+ val gatlingJsonpath = "io.gatling" %% "jsonpath" % V.gatlingJsonpath
val scalaForex = "com.snowplowanalytics" %% "scala-forex" % V.scalaForex
val refererParser = "com.snowplowanalytics" %% "scala-referer-parser" % V.refererParser
val maxmindIplookups = "com.snowplowanalytics" %% "scala-maxmind-iplookups" % V.maxmindIplookups
- val scalaUri = "io.lemonlabs" %% "scala-uri" % V.scalaUri
val scalaWeather = "com.snowplowanalytics" %% "scala-weather" % V.scalaWeather
- val scalaj = "org.scalaj" %% "scalaj-http" % V.scalaj
- val gatlingJsonpath = "io.gatling" %% "jsonpath" % V.gatlingJsonpath
val scalaLruMap = "com.snowplowanalytics" %% "scala-lru-map" % V.scalaLruMap
val badRows = "com.snowplowanalytics" %% "snowplow-badrows" % V.badRows
- // Thrift schemas
val snowplowRawEvent = "com.snowplowanalytics" % "snowplow-thrift-raw-event" % V.snowplowRawEvent
val collectorPayload = "com.snowplowanalytics" % "collector-payload-1" % V.collectorPayload
val schemaSniffer = "com.snowplowanalytics" % "schema-sniffer-1" % V.schemaSniffer
- // Scala (test only)
- val specs2 = "org.specs2" %% "specs2-core" % V.specs2 % "test"
- val specs2Cats = "org.specs2" %% "specs2-cats" % V.specs2 % "test"
- val specs2Scalacheck = "org.specs2" %% "specs2-scalacheck" % V.specs2 % "test"
- val specs2Mock = "org.specs2" %% "specs2-mock" % V.specs2 % "test"
+ val iabClient = "com.snowplowanalytics" % "iab-spiders-and-robots-client" % V.iabClient
+
+ val specs2 = "org.specs2" %% "specs2-core" % V.specs2 % Test
+ val specs2Cats = "org.specs2" %% "specs2-cats" % V.specs2 % Test
+ val specs2Scalacheck = "org.specs2" %% "specs2-scalacheck" % V.specs2 % Test
+ val specs2Mock = "org.specs2" %% "specs2-mock" % V.specs2 % Test
+
+ // Stream
+ val slf4j = "org.slf4j" % "slf4j-simple" % V.slf4j
+ val kinesisSdk = "com.amazonaws" % "aws-java-sdk-kinesis" % V.awsSdk
+ val dynamodbSdk = "com.amazonaws" % "aws-java-sdk-dynamodb" % V.awsSdk
+ val s3Sdk = "com.amazonaws" % "aws-java-sdk-s3" % V.awsSdk
+ val kinesisClient = "com.amazonaws" % "amazon-kinesis-client" % V.kinesisClient
+ val gsSdk = "com.google.cloud" % "google-cloud-storage" % V.gcpSdk
+ val kafkaClients = "org.apache.kafka" % "kafka-clients" % V.kafka
+ val jacksonCbor = "com.fasterxml.jackson.dataformat" % "jackson-dataformat-cbor" % V.jackson
+ val config = "com.typesafe" % "config" % V.config
+ val log4jOverSlf4j = "org.slf4j" % "log4j-over-slf4j" % V.slf4j
+ val scopt = "com.github.scopt" %% "scopt" % V.scopt
+ val pureconfig = "com.github.pureconfig" %% "pureconfig" % V.pureconfig
+ val nsqClient = "com.snowplowanalytics" % "nsq-java-client" % V.nsqClient
+ val snowplowTracker = "com.snowplowanalytics" %% "snowplow-scala-tracker-emitter-id" % V.snowplowTracker
+ val scalacheck = "org.scalacheck" %% "scalacheck" % V.scalacheck % Test
+ val kafka = "org.apache.kafka" %% "kafka" % V.kafka % Test
+ val jinJava = "com.hubspot.jinjava" % "jinjava" % V.jinJava % Test
+
+ // Beam
+ val scio = "com.spotify" %% "scio-core" % V.scio
+ val beam = "org.apache.beam" % "beam-runners-google-cloud-dataflow-java" % V.beam
+ val scioTest = "com.spotify" %% "scio-test" % V.scio % Test
+ val scalaTest = "org.scalatest" %% "scalatest" % V.scalaTest % Test
}
}
diff --git a/project/StreamDependencies.scala b/project/StreamDependencies.scala
deleted file mode 100644
index 41db63e3..00000000
--- a/project/StreamDependencies.scala
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Copyright (c) 2013-2020 Snowplow Analytics Ltd. All rights reserved.
- *
- * This program is licensed to you under the Apache License Version 2.0,
- * and you may not use this file except in compliance with the Apache License Version 2.0.
- * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the Apache License Version 2.0 is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
- */
-import sbt._
-
-object Dependencies {
-
- val resolutionRepos = Seq(
- // For Snowplow
- "Snowplow Analytics Maven releases repo" at "http://maven.snplow.com/releases/",
- "Snowplow Analytics Maven snapshot repo" at "http://maven.snplow.com/snapshots/",
- "snowplow" at "https://snowplow.bintray.com/snowplow-maven",
- // For uaParser utils
- "user-agent-parser repo" at "https://clojars.org/repo/",
- // For user-agent-utils
- "user-agent-utils repo" at "https://raw.github.com/HaraldWalker/user-agent-utils/mvn-repo/"
- )
-
- object V {
- // Java
- val awsSdk = "1.11.566"
- val kinesisClient = "1.10.0"
- val kafka = "2.2.1"
- val nsqClient = "1.2.0"
- val jackson = "2.9.9"
- val config = "1.3.4"
- val slf4j = "1.7.26"
- val gcpSdk = "1.106.0"
- // Scala
- val scopt = "3.7.1"
- val pureconfig = "0.11.0"
- val snowplowRawEvent = "0.1.0"
- val snowplowCommonEnrich = "1.1.0"
- val igluClient = "0.5.0"
- val snowplowTracker = "0.6.1"
- // Scala (test only)
- val specs2 = "4.5.1"
- val scalacheck = "1.14.0"
- val jinJava = "2.5.0"
- }
-
- object Libraries {
- // Java
- val kinesisSdk = "com.amazonaws" % "aws-java-sdk-kinesis" % V.awsSdk
- val dynamodbSdk = "com.amazonaws" % "aws-java-sdk-dynamodb" % V.awsSdk
- val s3Sdk = "com.amazonaws" % "aws-java-sdk-s3" % V.awsSdk
- val kinesisClient = "com.amazonaws" % "amazon-kinesis-client" % V.kinesisClient
- val kafkaClients = "org.apache.kafka" % "kafka-clients" % V.kafka
- val nsqClient = "com.snowplowanalytics" % "nsq-java-client" % V.nsqClient
- val jacksonCbor = "com.fasterxml.jackson.dataformat" % "jackson-dataformat-cbor" % V.jackson
- val jacksonDatabind = "com.fasterxml.jackson.core" % "jackson-databind" % V.jackson
- val config = "com.typesafe" % "config" % V.config
- val slf4j = "org.slf4j" % "slf4j-simple" % V.slf4j
- val log4jOverSlf4j = "org.slf4j" % "log4j-over-slf4j" % V.slf4j
- val gsSdk = "com.google.cloud" % "google-cloud-storage" % V.gcpSdk
- // Scala
- val scopt = "com.github.scopt" %% "scopt" % V.scopt
- val pureconfig = "com.github.pureconfig" %% "pureconfig" % V.pureconfig
- val snowplowRawEvent = "com.snowplowanalytics" % "snowplow-thrift-raw-event" % V.snowplowRawEvent
- val snowplowCommonEnrich = "com.snowplowanalytics" %% "snowplow-common-enrich" % V.snowplowCommonEnrich
- val snowplowTracker = "com.snowplowanalytics" %% "snowplow-scala-tracker-emitter-id" % V.snowplowTracker
- // Scala (test only)
- val specs2 = "org.specs2" %% "specs2-core" % V.specs2 % "test"
- val scalacheck = "org.scalacheck" %% "scalacheck" % V.scalacheck % "test"
- val kafka = "org.apache.kafka" %% "kafka" % V.kafka % "test"
- // Java (test only)
- val jinJava = "com.hubspot.jinjava" % "jinjava" % V.jinJava % "test"
- }
-}
diff --git a/project/StreamSettings.scala b/project/StreamSettings.scala
deleted file mode 100644
index 58c5869e..00000000
--- a/project/StreamSettings.scala
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Copyright (c) 2013-2020 Snowplow Analytics Ltd. All rights reserved.
- *
- * This program is licensed to you under the Apache License Version 2.0,
- * and you may not use this file except in compliance with the Apache License Version 2.0.
- * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the Apache License Version 2.0 is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
- */
-
-// SBT
-import sbt._
-import Keys._
-
-object BuildSettings {
- // sbt-assembly settings for building a fat jar
- import sbtassembly.AssemblyPlugin.autoImport._
- lazy val sbtAssemblySettings = Seq(
- assemblyJarName in assembly := { s"${moduleName.value}-${version.value}.jar" },
- assemblyMergeStrategy in assembly := {
- case x if x.endsWith("ProjectSettings$.class") => MergeStrategy.first
- case x if x.endsWith("module-info.class") => MergeStrategy.first
- case x =>
- val oldStrategy = (assemblyMergeStrategy in assembly).value
- oldStrategy(x)
- }
- )
-
- // Scalafmt plugin
- import org.scalafmt.sbt.ScalafmtPlugin.autoImport._
- lazy val formatting = Seq(
- scalafmtConfig := file(".scalafmt.conf"),
- scalafmtOnCompile := true
- )
-
- lazy val addExampleConfToTestCp = Seq(
- unmanagedClasspath in Test += baseDirectory.value.getParentFile / "examples"
- )
-}
diff --git a/project/plugins.sbt b/project/plugins.sbt
index 93da04cc..e64ace2d 100644
--- a/project/plugins.sbt
+++ b/project/plugins.sbt
@@ -2,4 +2,5 @@ addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.9")
addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.9.0")
addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.3.25")
addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.0.0")
-addSbtPlugin("io.github.davidgregory084" % "sbt-tpolecat" % "0.1.6")
\ No newline at end of file
+addSbtPlugin("io.github.davidgregory084" % "sbt-tpolecat" % "0.1.6")
+addSbtPlugin("org.foundweekends" % "sbt-bintray" % "0.5.4")
diff --git a/stream.sbt b/stream.sbt
deleted file mode 100644
index a5b4f9e1..00000000
--- a/stream.sbt
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Copyright (c) 2012-2020 Snowplow Analytics Ltd. All rights reserved.
- *
- * This program is licensed to you under the Apache License Version 2.0,
- * and you may not use this file except in compliance with the Apache License Version 2.0.
- * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the Apache License Version 2.0 is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
- */
-import com.typesafe.sbt.packager.docker._
-
-lazy val commonDependencies = Seq(
- // Java
- Dependencies.Libraries.config,
- Dependencies.Libraries.slf4j,
- Dependencies.Libraries.log4jOverSlf4j,
- Dependencies.Libraries.jacksonDatabind,
- Dependencies.Libraries.s3Sdk,
- Dependencies.Libraries.gsSdk,
- // Scala
- Dependencies.Libraries.scopt,
- Dependencies.Libraries.pureconfig,
- Dependencies.Libraries.snowplowRawEvent,
- Dependencies.Libraries.snowplowCommonEnrich,
- Dependencies.Libraries.snowplowTracker,
- // Test
- Dependencies.Libraries.specs2,
- Dependencies.Libraries.scalacheck
-)
-
-lazy val buildSettings = Seq(
- organization := "com.snowplowanalytics",
- name := "snowplow-stream-enrich",
- version := "1.1.0",
- description := "The streaming Snowplow Enrichment process",
- scalaVersion := "2.12.10",
- resolvers ++= Dependencies.resolutionRepos
-)
-
-lazy val dockerSettings = Seq(
- maintainer in Docker := "Snowplow Analytics Ltd. <support@snowplowanalytics.com>",
- dockerBaseImage := "snowplow-docker-registry.bintray.io/snowplow/base-debian:0.1.0",
- daemonUser in Docker := "snowplow",
- daemonUserUid in Docker := None,
- defaultLinuxInstallLocation in Docker := "/home/snowplow", // must be home directory of daemonUser
- dockerUpdateLatest := true
-)
-
-lazy val allSettings = buildSettings ++
- BuildSettings.sbtAssemblySettings ++
- Seq(libraryDependencies ++= commonDependencies) ++
- dockerSettings
-
-lazy val root = project.in(file("."))
- .settings(buildSettings)
- .aggregate(core, kinesis, kafka, nsq, stdin, integrationTests)
-
-lazy val core = project
- .settings(moduleName := "snowplow-stream-enrich")
- .settings(buildSettings)
- .settings(libraryDependencies ++= commonDependencies)
- .settings(BuildSettings.formatting)
- .enablePlugins(BuildInfoPlugin)
- .settings(
- buildInfoKeys := Seq[BuildInfoKey](organization, name, version,
- "commonEnrichVersion" -> Dependencies.V.snowplowCommonEnrich),
- buildInfoPackage := "com.snowplowanalytics.snowplow.enrich.stream.generated"
- )
-
-lazy val kinesis = project
- .settings(moduleName := "snowplow-stream-enrich-kinesis")
- .settings(allSettings)
- .settings(packageName in Docker := "snowplow/stream-enrich-kinesis")
- .settings(BuildSettings.formatting)
- .settings(libraryDependencies ++= Seq(
- Dependencies.Libraries.kinesisClient,
- Dependencies.Libraries.kinesisSdk,
- Dependencies.Libraries.dynamodbSdk,
- Dependencies.Libraries.jacksonCbor
- ))
- .enablePlugins(JavaAppPackaging, DockerPlugin)
- .dependsOn(core)
-
-lazy val kafka = project
- .settings(moduleName := "snowplow-stream-enrich-kafka")
- .settings(allSettings)
- .settings(packageName in Docker := "snowplow/stream-enrich-kafka")
- .settings(BuildSettings.formatting)
- .settings(libraryDependencies ++= Seq(
- Dependencies.Libraries.kafkaClients
- ))
- .enablePlugins(JavaAppPackaging, DockerPlugin)
- .dependsOn(core)
-
-lazy val nsq = project
- .settings(moduleName := "snowplow-stream-enrich-nsq")
- .settings(allSettings)
- .settings(packageName in Docker := "snowplow/stream-enrich-nsq")
- .settings(BuildSettings.formatting)
- .settings(libraryDependencies ++= Seq(Dependencies.Libraries.nsqClient))
- .enablePlugins(JavaAppPackaging, DockerPlugin)
- .dependsOn(core)
-
-lazy val stdin = project
- .settings(moduleName := "snowplow-stream-enrich-stdin")
- .settings(allSettings)
- .settings(BuildSettings.formatting)
- .dependsOn(core)
-
-lazy val integrationTests = project.in(file("./integration-tests"))
- .settings(moduleName := "integration-tests")
- .settings(allSettings)
- .settings(BuildSettings.formatting)
- .settings(BuildSettings.addExampleConfToTestCp)
- .settings(libraryDependencies ++= Seq(
- // Test
- Dependencies.Libraries.kafka,
- Dependencies.Libraries.jinJava
- ))
- .dependsOn(core % "test->test", kafka % "test->compile")
--
2.26.2
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment