Created
June 13, 2020 11:06
-
-
Save chuwy/e65a9d12d94f7f5104967e845b116bf4 to your computer and use it in GitHub Desktop.
Combine enrich submodules into a single SBT project
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| From fe136f1c24a2d3510f654348fa4522f48cdc6efb Mon Sep 17 00:00:00 2001 | |
| From: Anton Parkhomenko <mailbox@chuwy.me> | |
| Date: Sat, 13 Jun 2020 14:52:47 +0700 | |
| Subject: [PATCH] Unify | |
| --- | |
| beam.sbt | 88 --------- | |
| build.sbt | 172 ++++++++++++++++++ | |
| common.sbt | 68 ------- | |
| ...mmonSettings.scala => BuildSettings.scala} | 68 ++++++- | |
| ...nDependencies.scala => Dependencies.scala} | 106 +++++++---- | |
| project/StreamDependencies.scala | 78 -------- | |
| project/StreamSettings.scala | 42 ----- | |
| project/plugins.sbt | 3 +- | |
| stream.sbt | 123 ------------- | |
| 9 files changed, 300 insertions(+), 448 deletions(-) | |
| delete mode 100644 beam.sbt | |
| create mode 100644 build.sbt | |
| delete mode 100644 common.sbt | |
| rename project/{CommonSettings.scala => BuildSettings.scala} (51%) | |
| rename project/{CommonDependencies.scala => Dependencies.scala} (53%) | |
| delete mode 100644 project/StreamDependencies.scala | |
| delete mode 100644 project/StreamSettings.scala | |
| delete mode 100644 stream.sbt | |
| diff --git a/beam.sbt b/beam.sbt | |
| deleted file mode 100644 | |
| index c7e6c28d..00000000 | |
| --- a/beam.sbt | |
| +++ /dev/null | |
| @@ -1,88 +0,0 @@ | |
| -import sbt._ | |
| -import Keys._ | |
| - | |
| -lazy val resolutionRepos = Seq( | |
| - // For Snowplow | |
| - "Snowplow Analytics Maven releases repo" at "http://maven.snplow.com/releases/", | |
| - // For ua-parser | |
| - "user-agent-parser repo" at "https://clojars.org/repo/" | |
| -) | |
| - | |
| -// we fork a JVM per test in order to not reuse enrichment registries | |
| -import Tests._ | |
| -{ | |
| - def oneJVMPerTest(tests: Seq[TestDefinition]) = | |
| - tests.map(t => new Group(t.name, Seq(t), SubProcess(ForkOptions()))).toSeq | |
| - testGrouping in Test := oneJVMPerTest((definedTests in Test).value) | |
| -} | |
| - | |
| -lazy val commonSettings = Defaults.coreDefaultSettings ++ Seq( | |
| - organization := "com.snowplowanalytics", | |
| - version := "1.0.0", | |
| - scalaVersion := "2.12.10", | |
| - resolvers ++= resolutionRepos | |
| -) | |
| - | |
| -lazy val paradiseDependency = | |
| - "org.scalamacros" % "paradise" % scalaMacrosVersion cross CrossVersion.full | |
| -lazy val macroSettings = Seq( | |
| - libraryDependencies += "org.scala-lang" % "scala-reflect" % scalaVersion.value, | |
| - addCompilerPlugin(paradiseDependency) | |
| -) | |
| - | |
| -lazy val noPublishSettings = Seq( | |
| - publish := {}, | |
| - publishLocal := {}, | |
| - publishArtifact := false | |
| -) | |
| - | |
| -import com.typesafe.sbt.packager.docker._ | |
| -packageName in Docker := "snowplow/beam-enrich" | |
| -maintainer in Docker := "Snowplow Analytics Ltd. <support@snowplowanalytics.com>" | |
| -dockerBaseImage := "snowplow-docker-registry.bintray.io/snowplow/base-debian:0.1.0" | |
| -daemonUser in Docker := "snowplow" | |
| -dockerUpdateLatest := true | |
| - | |
| -lazy val scioVersion = "0.7.4" | |
| -lazy val beamVersion = "2.11.0" | |
| -lazy val sceVersion = "1.0.0" | |
| -lazy val scalaMacrosVersion = "2.1.1" | |
| -lazy val slf4jVersion = "1.7.25" | |
| -lazy val circeVersion = "0.11.1" | |
| -lazy val scalatestVersion = "3.0.8" | |
| - | |
| -lazy val root: Project = Project( | |
| - "beam-enrich", | |
| - file(".") | |
| -).settings( | |
| - commonSettings ++ macroSettings ++ noPublishSettings, | |
| - description := "Streaming enrich job written using SCIO", | |
| - buildInfoKeys := Seq[BuildInfoKey](organization, name, version, "sceVersion" -> sceVersion), | |
| - buildInfoPackage := "com.snowplowanalytics.snowplow.enrich.beam.generated", | |
| - scalafmtConfig := file(".scalafmt.conf"), | |
| - scalafmtOnCompile := true, | |
| - libraryDependencies ++= Seq( | |
| - "com.spotify" %% "scio-core" % scioVersion, | |
| - "org.apache.beam" % "beam-runners-google-cloud-dataflow-java" % beamVersion, | |
| - "com.snowplowanalytics" %% "snowplow-common-enrich" % sceVersion, | |
| - "org.slf4j" % "slf4j-simple" % slf4jVersion | |
| - ) ++ Seq( | |
| - "com.spotify" %% "scio-test" % scioVersion, | |
| - "org.scalatest" %% "scalatest" % scalatestVersion, | |
| - "io.circe" %% "circe-literal" % circeVersion | |
| - ).map(_ % "test") | |
| -).enablePlugins(JavaAppPackaging, DockerPlugin, BuildInfoPlugin) | |
| - | |
| -lazy val repl: Project = Project( | |
| - "repl", | |
| - file(".repl") | |
| -).settings( | |
| - commonSettings ++ macroSettings ++ noPublishSettings, | |
| - description := "Scio REPL for beam-enrich", | |
| - libraryDependencies ++= Seq( | |
| - "com.spotify" %% "scio-repl" % scioVersion | |
| - ), | |
| - mainClass in Compile := Some("com.spotify.scio.repl.ScioShell") | |
| -).dependsOn( | |
| - root | |
| -) | |
| diff --git a/build.sbt b/build.sbt | |
| new file mode 100644 | |
| index 00000000..c2f05f05 | |
| --- /dev/null | |
| +++ b/build.sbt | |
| @@ -0,0 +1,172 @@ | |
| +/* | |
| + * Copyright (c) 2012-2020 Snowplow Analytics Ltd. All rights reserved. | |
| + * | |
| + * This program is licensed to you under the Apache License Version 2.0, and | |
| + * you may not use this file except in compliance with the Apache License | |
| + * Version 2.0. You may obtain a copy of the Apache License Version 2.0 at | |
| + * http://www.apache.org/licenses/LICENSE-2.0. | |
| + * | |
| + * Unless required by applicable law or agreed to in writing, software | |
| + * distributed under the Apache License Version 2.0 is distributed on an "AS | |
| + * IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or | |
| + * implied. See the Apache License Version 2.0 for the specific language | |
| + * governing permissions and limitations there under. | |
| + */ | |
| +// ======================================================= | |
| +// scalafmt: {align.tokens = [":="]} | |
| +// ======================================================= | |
| + | |
| +/** Metaproject with aggregated modules */ | |
| +lazy val root = project | |
| + .in(file(".")) | |
| + .settings(name := "enrich") | |
| + .settings(BuildSettings.basicSettings) | |
| + .aggregate(common, beam, stream, kinesis, kafka, nsq, stdin, integrationTests) | |
| + | |
| +/** Snowplow Common Enrich - core of ETL, published as Maven Artifact */ | |
| +lazy val common = project | |
| + .in(file("modules/common")) | |
| + .settings( | |
| + name := "snowplow-common-enrich", | |
| + description := "Common functionality for enriching raw Snowplow events" | |
| + ) | |
| + .settings(BuildSettings.formatting) | |
| + .settings(BuildSettings.basicSettings) | |
| + .settings(BuildSettings.scalifySettings) | |
| + .settings(BuildSettings.publishSettings) | |
| + .settings(parallelExecution in Test := false) | |
| + .settings( | |
| + libraryDependencies ++= Seq( | |
| + Dependencies.Libraries.jodaTime, | |
| + Dependencies.Libraries.commonsCodec, | |
| + Dependencies.Libraries.useragent, | |
| + Dependencies.Libraries.jacksonDatabind, | |
| + Dependencies.Libraries.uaParser, | |
| + Dependencies.Libraries.postgresDriver, | |
| + Dependencies.Libraries.mysqlConnector, | |
| + Dependencies.Libraries.jaywayJsonpath, | |
| + Dependencies.Libraries.iabClient, | |
| + Dependencies.Libraries.yauaa, | |
| + Dependencies.Libraries.rhino, | |
| + Dependencies.Libraries.guava, | |
| + Dependencies.Libraries.circeOptics, | |
| + Dependencies.Libraries.circeJackson, | |
| + Dependencies.Libraries.refererParser, | |
| + Dependencies.Libraries.maxmindIplookups, | |
| + Dependencies.Libraries.scalaUri, | |
| + Dependencies.Libraries.scalaForex, | |
| + Dependencies.Libraries.scalaWeather, | |
| + Dependencies.Libraries.gatlingJsonpath, | |
| + Dependencies.Libraries.scalaLruMap, | |
| + Dependencies.Libraries.badRows, | |
| + Dependencies.Libraries.snowplowRawEvent, | |
| + Dependencies.Libraries.collectorPayload, | |
| + Dependencies.Libraries.schemaSniffer, | |
| + Dependencies.Libraries.specs2, | |
| + Dependencies.Libraries.specs2Cats, | |
| + Dependencies.Libraries.specs2Scalacheck, | |
| + Dependencies.Libraries.specs2Mock, | |
| + Dependencies.Libraries.circeLiteral % Test | |
| + ) | |
| + ) | |
| + | |
| +/** Common logic for Stream Enrich artifacts, not published */ | |
| +lazy val stream = project | |
| + .in(file("modules/stream")) | |
| + .settings(BuildSettings.allStreamSettings) | |
| + .settings(moduleName := "snowplow-stream-enrich") | |
| + .enablePlugins(BuildInfoPlugin) | |
| + .settings( | |
| + buildInfoKeys := Seq[BuildInfoKey](organization, name, version, "commonEnrichVersion" -> version.value), | |
| + buildInfoPackage := "com.snowplowanalytics.snowplow.enrich.stream.generated" | |
| + ) | |
| + .dependsOn(common) | |
| + | |
| +/** Kinesis Stream Enrich, Docker/JAR */ | |
| +lazy val kinesis = project | |
| + .in(file("modules/kinesis")) | |
| + .settings(BuildSettings.allStreamSettings) | |
| + .settings(moduleName := "snowplow-stream-enrich-kinesis") | |
| + .settings(packageName in Docker := "snowplow/stream-enrich-kinesis") | |
| + .settings(libraryDependencies ++= Seq( | |
| + Dependencies.Libraries.kinesisClient, | |
| + Dependencies.Libraries.kinesisSdk, | |
| + Dependencies.Libraries.dynamodbSdk, | |
| + Dependencies.Libraries.jacksonCbor | |
| + )) | |
| + .enablePlugins(JavaAppPackaging, DockerPlugin) | |
| + .dependsOn(stream) | |
| + | |
| +/** Kafka Stream Enrich, Docker/JAR */ | |
| +lazy val kafka = project | |
| + .in(file("modules/kafka")) | |
| + .settings(moduleName := "snowplow-stream-enrich-kafka") | |
| + .settings(packageName in Docker := "snowplow/stream-enrich-kafka") | |
| + .settings(BuildSettings.allStreamSettings) | |
| + .settings(libraryDependencies ++= Seq( | |
| + Dependencies.Libraries.kafkaClients | |
| + )) | |
| + .enablePlugins(JavaAppPackaging, DockerPlugin) | |
| + .dependsOn(stream) | |
| + | |
| +/** NSQ Stream Enrich, Docker/JAR */ | |
| +lazy val nsq = project | |
| + .in(file("modules/nsq")) | |
| + .settings(moduleName := "snowplow-stream-enrich-nsq") | |
| + .settings(BuildSettings.allStreamSettings) | |
| + .settings(packageName in Docker := "snowplow/stream-enrich-nsq") | |
| + .settings(libraryDependencies ++= Seq(Dependencies.Libraries.nsqClient)) | |
| + .enablePlugins(JavaAppPackaging, DockerPlugin) | |
| + .dependsOn(stream) | |
| + | |
| +/** Stdin Stream Enrich, JAR */ | |
| +lazy val stdin = project | |
| + .in(file("modules/stdin")) | |
| + .settings(moduleName := "snowplow-stream-enrich-stdin") | |
| + .settings(BuildSettings.allStreamSettings) | |
| + .dependsOn(stream) | |
| + | |
| +/** Beam Enrich, published as Docker image */ | |
| +lazy val beam = | |
| + project | |
| + .in(file("modules/beam")) | |
| + .dependsOn(common) | |
| + .settings(BuildSettings.basicSettings) | |
| + .settings(BuildSettings.formatting) | |
| + .settings( | |
| + description := "Streaming enrich job written using SCIO", | |
| + buildInfoKeys := Seq[BuildInfoKey](organization, name, version, "sceVersion" -> version.value), | |
| + buildInfoPackage := "com.snowplowanalytics.snowplow.enrich.beam.generated", | |
| + libraryDependencies ++= Seq( | |
| + Dependencies.Libraries.scio, | |
| + Dependencies.Libraries.beam, | |
| + Dependencies.Libraries.slf4j, | |
| + Dependencies.Libraries.scioTest, | |
| + Dependencies.Libraries.scalaTest, | |
| + Dependencies.Libraries.circeLiteral % Test, | |
| + ), | |
| + packageName in Docker := "snowplow/beam-enrich" | |
| + ) | |
| + .settings( | |
| + libraryDependencies += "org.scala-lang" % "scala-reflect" % scalaVersion.value, | |
| + addCompilerPlugin("org.scalamacros" % "paradise" % "2.1.1" cross CrossVersion.full) | |
| + ) | |
| + .settings( | |
| + publish := {}, | |
| + publishLocal := {}, | |
| + publishArtifact := false, | |
| + testGrouping in Test := BuildSettings.oneJVMPerTest((definedTests in Test).value) | |
| + ) | |
| + .enablePlugins(JavaAppPackaging, DockerPlugin, BuildInfoPlugin) | |
| + | |
| +/** Metaproject for testing Stream Enrich */ | |
| +lazy val integrationTests = project | |
| + .in(file("modules/integration-tests")) | |
| + .settings(moduleName := "integration-tests") | |
| + .settings(BuildSettings.allStreamSettings) | |
| + .settings(BuildSettings.addExampleConfToTestCp) | |
| + .settings(libraryDependencies ++= Seq( | |
| + Dependencies.Libraries.kafka, | |
| + Dependencies.Libraries.jinJava | |
| + )) | |
| + .dependsOn(stream % "test->test", kafka % "test->compile") | |
| diff --git a/common.sbt b/common.sbt | |
| deleted file mode 100644 | |
| index 651dbc6c..00000000 | |
| --- a/common.sbt | |
| +++ /dev/null | |
| @@ -1,68 +0,0 @@ | |
| -/* | |
| - * Copyright (c) 2012-2020 Snowplow Analytics Ltd. All rights reserved. | |
| - * | |
| - * This program is licensed to you under the Apache License Version 2.0, and | |
| - * you may not use this file except in compliance with the Apache License | |
| - * Version 2.0. You may obtain a copy of the Apache License Version 2.0 at | |
| - * http://www.apache.org/licenses/LICENSE-2.0. | |
| - * | |
| - * Unless required by applicable law or agreed to in writing, software | |
| - * distributed under the Apache License Version 2.0 is distributed on an "AS | |
| - * IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or | |
| - * implied. See the Apache License Version 2.0 for the specific language | |
| - * governing permissions and limitations there under. | |
| - */ | |
| -// ======================================================= | |
| -// scalafmt: {align.tokens = [":="]} | |
| -// ======================================================= | |
| -lazy val root = project | |
| - .in(file(".")) | |
| - .settings( | |
| - name := "snowplow-common-enrich", | |
| - version := "1.1.0", | |
| - description := "Common functionality for enriching raw Snowplow events" | |
| - ) | |
| - .settings(BuildSettings.formatting) | |
| - .settings(BuildSettings.buildSettings) | |
| - .settings(BuildSettings.publishSettings) | |
| - .settings(parallelExecution in Test := false) | |
| - .settings( | |
| - libraryDependencies ++= Seq( | |
| - // Java | |
| - Dependencies.Libraries.jodaTime, | |
| - Dependencies.Libraries.commonsCodec, | |
| - Dependencies.Libraries.useragent, | |
| - Dependencies.Libraries.jacksonDatabind, | |
| - Dependencies.Libraries.uaParser, | |
| - Dependencies.Libraries.postgresDriver, | |
| - Dependencies.Libraries.mysqlConnector, | |
| - Dependencies.Libraries.jaywayJsonpath, | |
| - Dependencies.Libraries.iabClient, | |
| - Dependencies.Libraries.yauaa, | |
| - Dependencies.Libraries.kryo, | |
| - Dependencies.Libraries.rhino, | |
| - Dependencies.Libraries.guava, | |
| - // Scala | |
| - Dependencies.Libraries.cats, | |
| - Dependencies.Libraries.circeOptics, | |
| - Dependencies.Libraries.circeJackson, | |
| - Dependencies.Libraries.refererParser, | |
| - Dependencies.Libraries.maxmindIplookups, | |
| - Dependencies.Libraries.scalaUri, | |
| - Dependencies.Libraries.scalaForex, | |
| - Dependencies.Libraries.scalaWeather, | |
| - Dependencies.Libraries.scalaj, | |
| - Dependencies.Libraries.gatlingJsonpath, | |
| - Dependencies.Libraries.scalaLruMap, | |
| - Dependencies.Libraries.badRows, | |
| - // Thrift schemas | |
| - Dependencies.Libraries.snowplowRawEvent, | |
| - Dependencies.Libraries.collectorPayload, | |
| - Dependencies.Libraries.schemaSniffer, | |
| - // Scala (test only) | |
| - Dependencies.Libraries.specs2, | |
| - Dependencies.Libraries.specs2Cats, | |
| - Dependencies.Libraries.specs2Scalacheck, | |
| - Dependencies.Libraries.specs2Mock | |
| - ) ++ Dependencies.Libraries.circeDeps | |
| - ) | |
| diff --git a/project/CommonSettings.scala b/project/BuildSettings.scala | |
| similarity index 51% | |
| rename from project/CommonSettings.scala | |
| rename to project/BuildSettings.scala | |
| index d0aef8ec..1b6491d9 100644 | |
| --- a/project/CommonSettings.scala | |
| +++ b/project/BuildSettings.scala | |
| @@ -20,24 +20,27 @@ import Keys._ | |
| import bintray.BintrayPlugin._ | |
| import bintray.BintrayKeys._ | |
| + | |
| +import com.typesafe.sbt.SbtNativePackager.autoImport._ | |
| +import com.typesafe.sbt.packager.linux.LinuxPlugin.autoImport._ | |
| +import com.typesafe.sbt.packager.docker.DockerPlugin.autoImport._ | |
| + | |
| import org.scalafmt.sbt.ScalafmtPlugin.autoImport._ | |
| +import sbtassembly.AssemblyPlugin.autoImport._ | |
| + | |
| object BuildSettings { | |
| - // Basic settings for our app | |
| + /** Common settings for all modules */ | |
| lazy val basicSettings = Seq( | |
| organization := "com.snowplowanalytics", | |
| scalaVersion := "2.12.10", | |
| - javacOptions := javaCompilerOptions, | |
| + version := "1.3.0", | |
| + javacOptions := Seq("-source", "1.8", "-target", "1.8"), | |
| resolvers ++= Dependencies.resolutionRepos | |
| ) | |
| - lazy val javaCompilerOptions = Seq( | |
| - "-source", "1.8", | |
| - "-target", "1.8" | |
| - ) | |
| - | |
| - // Makes our SBT app settings available from within the ETL | |
| + /** Custom BuildInfo replacement, used only by SCE now */ | |
| lazy val scalifySettings = Seq( | |
| sourceGenerators in Compile += Def.task { | |
| val file = (sourceManaged in Compile).value / "settings.scala" | |
| @@ -53,8 +56,7 @@ object BuildSettings { | |
| }.taskValue | |
| ) | |
| - lazy val buildSettings = basicSettings ++ scalifySettings | |
| - | |
| + /** Bintray Maven settings, used only by SCE now */ | |
| lazy val publishSettings = bintraySettings ++ Seq( | |
| publishMavenStyle := true, | |
| publishArtifact := true, | |
| @@ -81,4 +83,50 @@ object BuildSettings { | |
| scalafmtConfig := file(".scalafmt.conf"), | |
| scalafmtOnCompile := true | |
| ) | |
| + | |
| + /** Fork a JVM per test in order to not reuse enrichment registries */ | |
| + def oneJVMPerTest(tests: Seq[TestDefinition]): Seq[Tests.Group] = | |
| + tests.map(t => Tests.Group(t.name, Seq(t), Tests.SubProcess(ForkOptions()))) | |
| + | |
| + /** Building fat jar, used only by SE now */ | |
| + lazy val sbtAssemblySettings = Seq( | |
| + assemblyJarName in assembly := { s"${moduleName.value}-${version.value}.jar" }, | |
| + assemblyMergeStrategy in assembly := { | |
| + case x if x.endsWith("ProjectSettings$.class") => MergeStrategy.first | |
| + case x if x.endsWith("module-info.class") => MergeStrategy.first | |
| + case x => | |
| + val oldStrategy = (assemblyMergeStrategy in assembly).value | |
| + oldStrategy(x) | |
| + } | |
| + ) | |
| + | |
| + /** Add sample configuration for integration tests */ | |
| + lazy val addExampleConfToTestCp = Seq( | |
| + unmanagedClasspath in Test += baseDirectory.value.getParentFile / "config" | |
| + ) | |
| + | |
| + /** Common Docker settings, used by both SE and BE */ | |
| + lazy val dockerSettings = Seq( | |
| + maintainer in Docker := "Snowplow Analytics Ltd. <support@snowplowanalytics.com>", | |
| + dockerBaseImage := "snowplow-docker-registry.bintray.io/snowplow/base-debian:0.1.0", | |
| + daemonUser in Docker := "snowplow", | |
| + daemonUserUid in Docker := None, | |
| + defaultLinuxInstallLocation in Docker := "/home/snowplow", // must be home directory of daemonUser | |
| + dockerUpdateLatest := true | |
| + ) | |
| + | |
| + /** Common configuration for all Stream Enrich modules */ | |
| + lazy val allStreamSettings = basicSettings ++ sbtAssemblySettings ++ dockerSettings ++ formatting ++ | |
| + Seq(libraryDependencies := Seq( | |
| + Dependencies.Libraries.config, | |
| + Dependencies.Libraries.slf4j, | |
| + Dependencies.Libraries.log4jOverSlf4j, | |
| + Dependencies.Libraries.s3Sdk, | |
| + Dependencies.Libraries.gsSdk, | |
| + Dependencies.Libraries.scopt, | |
| + Dependencies.Libraries.pureconfig, | |
| + Dependencies.Libraries.snowplowTracker, | |
| + Dependencies.Libraries.specs2, | |
| + Dependencies.Libraries.scalacheck | |
| + )) | |
| } | |
| diff --git a/project/CommonDependencies.scala b/project/Dependencies.scala | |
| similarity index 53% | |
| rename from project/CommonDependencies.scala | |
| rename to project/Dependencies.scala | |
| index cf97ce11..afe4ca5e 100644 | |
| --- a/project/CommonDependencies.scala | |
| +++ b/project/Dependencies.scala | |
| @@ -18,16 +18,14 @@ import sbt._ | |
| object Dependencies { | |
| val resolutionRepos = Seq( | |
| - // For some misc Scalding and Twitter libs | |
| - "Concurrent Maven Repo" at "http://conjars.org/repo", | |
| + // For some Twitter libs and uaParser utils | |
| + "Concurrent Maven Repo" at "https://conjars.org/repo", | |
| // For Twitter's util functions | |
| - "Twitter Maven Repo" at "http://maven.twttr.com/", | |
| - // For Snowplow libs | |
| - "Snowplow Analytics Maven repo" at "http://maven.snplow.com/releases/", | |
| - "Snowplow Analytics Maven snapshot repo" at "http://maven.snplow.com/snapshots/", | |
| - "Snowplow Bintray" at "https://snowplow.bintray.com/snowplow-maven/", | |
| - // For uaParser utils | |
| - "user-agent-parser repo" at "https://clojars.org/repo/" | |
| + "Twitter Maven Repo" at "https://maven.twttr.com/", | |
| + // For modern Snowplow libs | |
| + "Snowplow Bintray" at "https://snowplow.bintray.com/snowplow-maven/", | |
| + // For legacy Snowplow libs | |
| + ("Snowplow Analytics Maven repo" at "http://maven.snplow.com/releases/").withAllowInsecureProtocol(true) | |
| ) | |
| object V { | |
| @@ -35,18 +33,16 @@ object Dependencies { | |
| val commonsCodec = "1.12" | |
| val jodaTime = "2.10.1" | |
| val useragent = "1.21" | |
| - val jacksonDatabind = "2.9.8" | |
| val uaParser = "1.4.3" | |
| val postgresDriver = "42.2.5" | |
| val mysqlConnector = "8.0.16" | |
| val jaywayJsonpath = "2.4.0" | |
| val iabClient = "0.1.0" | |
| val yauaa = "5.8" | |
| - val kryo = "2.24.0" | |
| val guava = "28.1-jre" //used only for InetAddress (because it doesn't do dns lookup) | |
| val rhino = "1.7.10" | |
| - // Scala | |
| - val cats = "1.6.0" | |
| + val slf4j = "1.7.26" | |
| + | |
| val refererParser = "1.0.0" | |
| val maxmindIplookups = "0.6.1" | |
| val circe = "0.11.1" | |
| @@ -54,63 +50,97 @@ object Dependencies { | |
| val circeJackson = "0.11.1" | |
| val scalaForex = "0.7.0" | |
| val scalaWeather = "0.5.0" | |
| - val scalaj = "2.4.1" | |
| val gatlingJsonpath = "0.6.14" | |
| val scalaUri = "1.4.5" | |
| val scalaLruMap = "0.3.0" | |
| val badRows = "1.0.0" | |
| - // Thrift schemas | |
| + | |
| val snowplowRawEvent = "0.1.0" | |
| val collectorPayload = "0.0.0" | |
| val schemaSniffer = "0.0.0" | |
| - // Scala (test only) | |
| + | |
| + val awsSdk = "1.11.566" | |
| + val gcpSdk = "1.106.0" | |
| + val kinesisClient = "1.10.0" | |
| + val kafka = "2.2.1" | |
| + val nsqClient = "1.2.0" | |
| + val jackson = "2.9.9" | |
| + val config = "1.3.4" | |
| + | |
| + val scopt = "3.7.1" | |
| + val pureconfig = "0.11.0" | |
| + val snowplowTracker = "0.6.1" | |
| + | |
| val specs2 = "4.5.1" | |
| + val scalacheck = "1.14.0" | |
| + val jinJava = "2.5.0" | |
| + | |
| + val scio = "0.7.4" | |
| + val beam = "2.11.0" | |
| + val macros = "2.1.1" | |
| + val scalaTest = "3.0.8" | |
| } | |
| object Libraries { | |
| - // Java | |
| val commonsCodec = "commons-codec" % "commons-codec" % V.commonsCodec | |
| val jodaTime = "joda-time" % "joda-time" % V.jodaTime | |
| val useragent = "eu.bitwalker" % "UserAgentUtils" % V.useragent | |
| - val jacksonDatabind = "com.fasterxml.jackson.core" % "jackson-databind" % V.jacksonDatabind | |
| + val jacksonDatabind = "com.fasterxml.jackson.core" % "jackson-databind" % V.jackson | |
| val uaParser = "com.github.ua-parser" % "uap-java" % V.uaParser | |
| val postgresDriver = "org.postgresql" % "postgresql" % V.postgresDriver | |
| val mysqlConnector = "mysql" % "mysql-connector-java" % V.mysqlConnector | |
| val jaywayJsonpath = "com.jayway.jsonpath" % "json-path" % V.jaywayJsonpath | |
| - val iabClient = "com.snowplowanalytics" % "iab-spiders-and-robots-client" % V.iabClient | |
| val yauaa = "nl.basjes.parse.useragent" % "yauaa" % V.yauaa | |
| - val kryo = "com.esotericsoftware.kryo" % "kryo" % V.kryo | |
| val guava = "com.google.guava" % "guava" % V.guava | |
| val rhino = "org.mozilla" % "rhino" % V.rhino | |
| - // Scala | |
| - val circeDeps = List( | |
| - "circe-core", | |
| - "circe-generic", | |
| - "circe-parser", | |
| - "circe-literal", | |
| - "circe-java8" | |
| - ).map("io.circe" %% _ % V.circe) | |
| - val cats = "org.typelevel" %% "cats-core" % V.cats | |
| + val circeGeneric = "io.circe" %% "circe-generic" % V.circe | |
| + val circeParser = "io.circe" %% "circe-parser" % V.circe | |
| + val circeLiteral = "io.circe" %% "circe-literal" % V.circe | |
| + val circeJava8 = "io.circe" %% "circe-java8" % V.circe | |
| val circeOptics = "io.circe" %% "circe-optics" % V.circeOptics | |
| val circeJackson = "io.circe" %% "circe-jackson29" % V.circeJackson | |
| + val scalaUri = "io.lemonlabs" %% "scala-uri" % V.scalaUri | |
| + val gatlingJsonpath = "io.gatling" %% "jsonpath" % V.gatlingJsonpath | |
| val scalaForex = "com.snowplowanalytics" %% "scala-forex" % V.scalaForex | |
| val refererParser = "com.snowplowanalytics" %% "scala-referer-parser" % V.refererParser | |
| val maxmindIplookups = "com.snowplowanalytics" %% "scala-maxmind-iplookups" % V.maxmindIplookups | |
| - val scalaUri = "io.lemonlabs" %% "scala-uri" % V.scalaUri | |
| val scalaWeather = "com.snowplowanalytics" %% "scala-weather" % V.scalaWeather | |
| - val scalaj = "org.scalaj" %% "scalaj-http" % V.scalaj | |
| - val gatlingJsonpath = "io.gatling" %% "jsonpath" % V.gatlingJsonpath | |
| val scalaLruMap = "com.snowplowanalytics" %% "scala-lru-map" % V.scalaLruMap | |
| val badRows = "com.snowplowanalytics" %% "snowplow-badrows" % V.badRows | |
| - // Thrift schemas | |
| val snowplowRawEvent = "com.snowplowanalytics" % "snowplow-thrift-raw-event" % V.snowplowRawEvent | |
| val collectorPayload = "com.snowplowanalytics" % "collector-payload-1" % V.collectorPayload | |
| val schemaSniffer = "com.snowplowanalytics" % "schema-sniffer-1" % V.schemaSniffer | |
| - // Scala (test only) | |
| - val specs2 = "org.specs2" %% "specs2-core" % V.specs2 % "test" | |
| - val specs2Cats = "org.specs2" %% "specs2-cats" % V.specs2 % "test" | |
| - val specs2Scalacheck = "org.specs2" %% "specs2-scalacheck" % V.specs2 % "test" | |
| - val specs2Mock = "org.specs2" %% "specs2-mock" % V.specs2 % "test" | |
| + val iabClient = "com.snowplowanalytics" % "iab-spiders-and-robots-client" % V.iabClient | |
| + | |
| + val specs2 = "org.specs2" %% "specs2-core" % V.specs2 % Test | |
| + val specs2Cats = "org.specs2" %% "specs2-cats" % V.specs2 % Test | |
| + val specs2Scalacheck = "org.specs2" %% "specs2-scalacheck" % V.specs2 % Test | |
| + val specs2Mock = "org.specs2" %% "specs2-mock" % V.specs2 % Test | |
| + | |
| + // Stream | |
| + val slf4j = "org.slf4j" % "slf4j-simple" % V.slf4j | |
| + val kinesisSdk = "com.amazonaws" % "aws-java-sdk-kinesis" % V.awsSdk | |
| + val dynamodbSdk = "com.amazonaws" % "aws-java-sdk-dynamodb" % V.awsSdk | |
| + val s3Sdk = "com.amazonaws" % "aws-java-sdk-s3" % V.awsSdk | |
| + val kinesisClient = "com.amazonaws" % "amazon-kinesis-client" % V.kinesisClient | |
| + val gsSdk = "com.google.cloud" % "google-cloud-storage" % V.gcpSdk | |
| + val kafkaClients = "org.apache.kafka" % "kafka-clients" % V.kafka | |
| + val jacksonCbor = "com.fasterxml.jackson.dataformat" % "jackson-dataformat-cbor" % V.jackson | |
| + val config = "com.typesafe" % "config" % V.config | |
| + val log4jOverSlf4j = "org.slf4j" % "log4j-over-slf4j" % V.slf4j | |
| + val scopt = "com.github.scopt" %% "scopt" % V.scopt | |
| + val pureconfig = "com.github.pureconfig" %% "pureconfig" % V.pureconfig | |
| + val nsqClient = "com.snowplowanalytics" % "nsq-java-client" % V.nsqClient | |
| + val snowplowTracker = "com.snowplowanalytics" %% "snowplow-scala-tracker-emitter-id" % V.snowplowTracker | |
| + val scalacheck = "org.scalacheck" %% "scalacheck" % V.scalacheck % Test | |
| + val kafka = "org.apache.kafka" %% "kafka" % V.kafka % Test | |
| + val jinJava = "com.hubspot.jinjava" % "jinjava" % V.jinJava % Test | |
| + | |
| + // Beam | |
| + val scio = "com.spotify" %% "scio-core" % V.scio | |
| + val beam = "org.apache.beam" % "beam-runners-google-cloud-dataflow-java" % V.beam | |
| + val scioTest = "com.spotify" %% "scio-test" % V.scio % Test | |
| + val scalaTest = "org.scalatest" %% "scalatest" % V.scalaTest % Test | |
| } | |
| } | |
| diff --git a/project/StreamDependencies.scala b/project/StreamDependencies.scala | |
| deleted file mode 100644 | |
| index 41db63e3..00000000 | |
| --- a/project/StreamDependencies.scala | |
| +++ /dev/null | |
| @@ -1,78 +0,0 @@ | |
| -/* | |
| - * Copyright (c) 2013-2020 Snowplow Analytics Ltd. All rights reserved. | |
| - * | |
| - * This program is licensed to you under the Apache License Version 2.0, | |
| - * and you may not use this file except in compliance with the Apache License Version 2.0. | |
| - * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. | |
| - * | |
| - * Unless required by applicable law or agreed to in writing, | |
| - * software distributed under the Apache License Version 2.0 is distributed on an | |
| - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| - * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. | |
| - */ | |
| -import sbt._ | |
| - | |
| -object Dependencies { | |
| - | |
| - val resolutionRepos = Seq( | |
| - // For Snowplow | |
| - "Snowplow Analytics Maven releases repo" at "http://maven.snplow.com/releases/", | |
| - "Snowplow Analytics Maven snapshot repo" at "http://maven.snplow.com/snapshots/", | |
| - "snowplow" at "https://snowplow.bintray.com/snowplow-maven", | |
| - // For uaParser utils | |
| - "user-agent-parser repo" at "https://clojars.org/repo/", | |
| - // For user-agent-utils | |
| - "user-agent-utils repo" at "https://raw.github.com/HaraldWalker/user-agent-utils/mvn-repo/" | |
| - ) | |
| - | |
| - object V { | |
| - // Java | |
| - val awsSdk = "1.11.566" | |
| - val kinesisClient = "1.10.0" | |
| - val kafka = "2.2.1" | |
| - val nsqClient = "1.2.0" | |
| - val jackson = "2.9.9" | |
| - val config = "1.3.4" | |
| - val slf4j = "1.7.26" | |
| - val gcpSdk = "1.106.0" | |
| - // Scala | |
| - val scopt = "3.7.1" | |
| - val pureconfig = "0.11.0" | |
| - val snowplowRawEvent = "0.1.0" | |
| - val snowplowCommonEnrich = "1.1.0" | |
| - val igluClient = "0.5.0" | |
| - val snowplowTracker = "0.6.1" | |
| - // Scala (test only) | |
| - val specs2 = "4.5.1" | |
| - val scalacheck = "1.14.0" | |
| - val jinJava = "2.5.0" | |
| - } | |
| - | |
| - object Libraries { | |
| - // Java | |
| - val kinesisSdk = "com.amazonaws" % "aws-java-sdk-kinesis" % V.awsSdk | |
| - val dynamodbSdk = "com.amazonaws" % "aws-java-sdk-dynamodb" % V.awsSdk | |
| - val s3Sdk = "com.amazonaws" % "aws-java-sdk-s3" % V.awsSdk | |
| - val kinesisClient = "com.amazonaws" % "amazon-kinesis-client" % V.kinesisClient | |
| - val kafkaClients = "org.apache.kafka" % "kafka-clients" % V.kafka | |
| - val nsqClient = "com.snowplowanalytics" % "nsq-java-client" % V.nsqClient | |
| - val jacksonCbor = "com.fasterxml.jackson.dataformat" % "jackson-dataformat-cbor" % V.jackson | |
| - val jacksonDatabind = "com.fasterxml.jackson.core" % "jackson-databind" % V.jackson | |
| - val config = "com.typesafe" % "config" % V.config | |
| - val slf4j = "org.slf4j" % "slf4j-simple" % V.slf4j | |
| - val log4jOverSlf4j = "org.slf4j" % "log4j-over-slf4j" % V.slf4j | |
| - val gsSdk = "com.google.cloud" % "google-cloud-storage" % V.gcpSdk | |
| - // Scala | |
| - val scopt = "com.github.scopt" %% "scopt" % V.scopt | |
| - val pureconfig = "com.github.pureconfig" %% "pureconfig" % V.pureconfig | |
| - val snowplowRawEvent = "com.snowplowanalytics" % "snowplow-thrift-raw-event" % V.snowplowRawEvent | |
| - val snowplowCommonEnrich = "com.snowplowanalytics" %% "snowplow-common-enrich" % V.snowplowCommonEnrich | |
| - val snowplowTracker = "com.snowplowanalytics" %% "snowplow-scala-tracker-emitter-id" % V.snowplowTracker | |
| - // Scala (test only) | |
| - val specs2 = "org.specs2" %% "specs2-core" % V.specs2 % "test" | |
| - val scalacheck = "org.scalacheck" %% "scalacheck" % V.scalacheck % "test" | |
| - val kafka = "org.apache.kafka" %% "kafka" % V.kafka % "test" | |
| - // Java (test only) | |
| - val jinJava = "com.hubspot.jinjava" % "jinjava" % V.jinJava % "test" | |
| - } | |
| -} | |
| diff --git a/project/StreamSettings.scala b/project/StreamSettings.scala | |
| deleted file mode 100644 | |
| index 58c5869e..00000000 | |
| --- a/project/StreamSettings.scala | |
| +++ /dev/null | |
| @@ -1,42 +0,0 @@ | |
| -/* | |
| - * Copyright (c) 2013-2020 Snowplow Analytics Ltd. All rights reserved. | |
| - * | |
| - * This program is licensed to you under the Apache License Version 2.0, | |
| - * and you may not use this file except in compliance with the Apache License Version 2.0. | |
| - * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. | |
| - * | |
| - * Unless required by applicable law or agreed to in writing, | |
| - * software distributed under the Apache License Version 2.0 is distributed on an | |
| - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| - * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. | |
| - */ | |
| - | |
| -// SBT | |
| -import sbt._ | |
| -import Keys._ | |
| - | |
| -object BuildSettings { | |
| - // sbt-assembly settings for building a fat jar | |
| - import sbtassembly.AssemblyPlugin.autoImport._ | |
| - lazy val sbtAssemblySettings = Seq( | |
| - assemblyJarName in assembly := { s"${moduleName.value}-${version.value}.jar" }, | |
| - assemblyMergeStrategy in assembly := { | |
| - case x if x.endsWith("ProjectSettings$.class") => MergeStrategy.first | |
| - case x if x.endsWith("module-info.class") => MergeStrategy.first | |
| - case x => | |
| - val oldStrategy = (assemblyMergeStrategy in assembly).value | |
| - oldStrategy(x) | |
| - } | |
| - ) | |
| - | |
| - // Scalafmt plugin | |
| - import org.scalafmt.sbt.ScalafmtPlugin.autoImport._ | |
| - lazy val formatting = Seq( | |
| - scalafmtConfig := file(".scalafmt.conf"), | |
| - scalafmtOnCompile := true | |
| - ) | |
| - | |
| - lazy val addExampleConfToTestCp = Seq( | |
| - unmanagedClasspath in Test += baseDirectory.value.getParentFile / "examples" | |
| - ) | |
| -} | |
| diff --git a/project/plugins.sbt b/project/plugins.sbt | |
| index 93da04cc..e64ace2d 100644 | |
| --- a/project/plugins.sbt | |
| +++ b/project/plugins.sbt | |
| @@ -2,4 +2,5 @@ addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.9") | |
| addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.9.0") | |
| addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.3.25") | |
| addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.0.0") | |
| -addSbtPlugin("io.github.davidgregory084" % "sbt-tpolecat" % "0.1.6") | |
| \ No newline at end of file | |
| +addSbtPlugin("io.github.davidgregory084" % "sbt-tpolecat" % "0.1.6") | |
| +addSbtPlugin("org.foundweekends" % "sbt-bintray" % "0.5.4") | |
| diff --git a/stream.sbt b/stream.sbt | |
| deleted file mode 100644 | |
| index a5b4f9e1..00000000 | |
| --- a/stream.sbt | |
| +++ /dev/null | |
| @@ -1,123 +0,0 @@ | |
| -/* | |
| - * Copyright (c) 2012-2020 Snowplow Analytics Ltd. All rights reserved. | |
| - * | |
| - * This program is licensed to you under the Apache License Version 2.0, | |
| - * and you may not use this file except in compliance with the Apache License Version 2.0. | |
| - * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. | |
| - * | |
| - * Unless required by applicable law or agreed to in writing, | |
| - * software distributed under the Apache License Version 2.0 is distributed on an | |
| - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| - * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. | |
| - */ | |
| -import com.typesafe.sbt.packager.docker._ | |
| - | |
| -lazy val commonDependencies = Seq( | |
| - // Java | |
| - Dependencies.Libraries.config, | |
| - Dependencies.Libraries.slf4j, | |
| - Dependencies.Libraries.log4jOverSlf4j, | |
| - Dependencies.Libraries.jacksonDatabind, | |
| - Dependencies.Libraries.s3Sdk, | |
| - Dependencies.Libraries.gsSdk, | |
| - // Scala | |
| - Dependencies.Libraries.scopt, | |
| - Dependencies.Libraries.pureconfig, | |
| - Dependencies.Libraries.snowplowRawEvent, | |
| - Dependencies.Libraries.snowplowCommonEnrich, | |
| - Dependencies.Libraries.snowplowTracker, | |
| - // Test | |
| - Dependencies.Libraries.specs2, | |
| - Dependencies.Libraries.scalacheck | |
| -) | |
| - | |
| -lazy val buildSettings = Seq( | |
| - organization := "com.snowplowanalytics", | |
| - name := "snowplow-stream-enrich", | |
| - version := "1.1.0", | |
| - description := "The streaming Snowplow Enrichment process", | |
| - scalaVersion := "2.12.10", | |
| - resolvers ++= Dependencies.resolutionRepos | |
| -) | |
| - | |
| -lazy val dockerSettings = Seq( | |
| - maintainer in Docker := "Snowplow Analytics Ltd. <support@snowplowanalytics.com>", | |
| - dockerBaseImage := "snowplow-docker-registry.bintray.io/snowplow/base-debian:0.1.0", | |
| - daemonUser in Docker := "snowplow", | |
| - daemonUserUid in Docker := None, | |
| - defaultLinuxInstallLocation in Docker := "/home/snowplow", // must be home directory of daemonUser | |
| - dockerUpdateLatest := true | |
| -) | |
| - | |
| -lazy val allSettings = buildSettings ++ | |
| - BuildSettings.sbtAssemblySettings ++ | |
| - Seq(libraryDependencies ++= commonDependencies) ++ | |
| - dockerSettings | |
| - | |
| -lazy val root = project.in(file(".")) | |
| - .settings(buildSettings) | |
| - .aggregate(core, kinesis, kafka, nsq, stdin, integrationTests) | |
| - | |
| -lazy val core = project | |
| - .settings(moduleName := "snowplow-stream-enrich") | |
| - .settings(buildSettings) | |
| - .settings(libraryDependencies ++= commonDependencies) | |
| - .settings(BuildSettings.formatting) | |
| - .enablePlugins(BuildInfoPlugin) | |
| - .settings( | |
| - buildInfoKeys := Seq[BuildInfoKey](organization, name, version, | |
| - "commonEnrichVersion" -> Dependencies.V.snowplowCommonEnrich), | |
| - buildInfoPackage := "com.snowplowanalytics.snowplow.enrich.stream.generated" | |
| - ) | |
| - | |
| -lazy val kinesis = project | |
| - .settings(moduleName := "snowplow-stream-enrich-kinesis") | |
| - .settings(allSettings) | |
| - .settings(packageName in Docker := "snowplow/stream-enrich-kinesis") | |
| - .settings(BuildSettings.formatting) | |
| - .settings(libraryDependencies ++= Seq( | |
| - Dependencies.Libraries.kinesisClient, | |
| - Dependencies.Libraries.kinesisSdk, | |
| - Dependencies.Libraries.dynamodbSdk, | |
| - Dependencies.Libraries.jacksonCbor | |
| - )) | |
| - .enablePlugins(JavaAppPackaging, DockerPlugin) | |
| - .dependsOn(core) | |
| - | |
| -lazy val kafka = project | |
| - .settings(moduleName := "snowplow-stream-enrich-kafka") | |
| - .settings(allSettings) | |
| - .settings(packageName in Docker := "snowplow/stream-enrich-kafka") | |
| - .settings(BuildSettings.formatting) | |
| - .settings(libraryDependencies ++= Seq( | |
| - Dependencies.Libraries.kafkaClients | |
| - )) | |
| - .enablePlugins(JavaAppPackaging, DockerPlugin) | |
| - .dependsOn(core) | |
| - | |
| -lazy val nsq = project | |
| - .settings(moduleName := "snowplow-stream-enrich-nsq") | |
| - .settings(allSettings) | |
| - .settings(packageName in Docker := "snowplow/stream-enrich-nsq") | |
| - .settings(BuildSettings.formatting) | |
| - .settings(libraryDependencies ++= Seq(Dependencies.Libraries.nsqClient)) | |
| - .enablePlugins(JavaAppPackaging, DockerPlugin) | |
| - .dependsOn(core) | |
| - | |
| -lazy val stdin = project | |
| - .settings(moduleName := "snowplow-stream-enrich-stdin") | |
| - .settings(allSettings) | |
| - .settings(BuildSettings.formatting) | |
| - .dependsOn(core) | |
| - | |
| -lazy val integrationTests = project.in(file("./integration-tests")) | |
| - .settings(moduleName := "integration-tests") | |
| - .settings(allSettings) | |
| - .settings(BuildSettings.formatting) | |
| - .settings(BuildSettings.addExampleConfToTestCp) | |
| - .settings(libraryDependencies ++= Seq( | |
| - // Test | |
| - Dependencies.Libraries.kafka, | |
| - Dependencies.Libraries.jinJava | |
| - )) | |
| - .dependsOn(core % "test->test", kafka % "test->compile") | |
| -- | |
| 2.26.2 | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment