renardeinside

package net.renarde.demos.common

import org.apache.spark.internal.Logging

trait GenericApp extends App with Logging {
  val appName: String
  lazy val config = ConfigProvider.getGlobalConfig().getConfig(appName).resolve()
}

renardeinside

package net.renarde.demos

import org.apache.spark.sql.SparkSession

package object apps {
  implicit val spark: SparkSession = SparkSession.builder().getOrCreate()
}

renardeinside

package net.renarde.demos.utils

import org.apache.spark.sql.SparkSession

import java.nio.file.Files

trait SparkSupport {

  implicit val spark: SparkSession = SparkSession
    .builder()

renardeinside

package net.renarde.demos.utils

import com.dimafeng.testcontainers.KafkaContainer
import org.apache.spark.internal.Logging
import org.scalatest.{BeforeAndAfterAll, Suite}

trait KafkaSupport extends BeforeAndAfterAll with Logging {
  self: Suite =>

  val kafka = new KafkaContainer()

renardeinside

common {
  "kafka.bootstrap.servers" = ${KAFKA_BOOTSTRAP_SERVERS}
  database = "default"
}

reader {
  inputs {
    kafka {
      "kafka.bootstrap.servers" = ${common."kafka.bootstrap.servers"}
      "topic" = "some-topic-name"

renardeinside

package net.renarde.demos.apps
import com.typesafe.config.ConfigFactory

object SampleAppWithConfig extends App {
  val conf = ConfigFactory.load()
  println("Running!")
  println(conf.getConfig("reader"))
}

renardeinside

common {
  "kafka.bootstrap.servers" = ${KAFKA_BOOTSTRAP_SERVERS}
}

reader {
  inputs {
    kafka {
      "kafka.bootstrap.servers" = ${common."kafka.bootstrap.servers"}
      "topic" = "some-topic-name"
    }

renardeinside

echo """[global]
index-url=https://${AZ_DEVOPS_FEED_NAME}:${AZ_DEVOPS_TOKEN}@pkgs.dev.azure.com/${AZ_DEVOPS_ORG_NAME}/${AZ_DEVOPS_PROJECT_NAME}/_packaging/${AZ_DEVOPS_FEED_NAME}/pypi/simple/
""" > /etc/pip.conf

renardeinside

stages:
- stage: onPush
  # omitted for readability

- stage: onRelease
  condition: startsWith(variables['Build.SourceBranch'], 'refs/tags/v')
  jobs:
  - job: onReleaseJob
    pool:
      vmImage: 'ubuntu-18.04'

renardeinside

stages:
- stage: onPush
  condition: not(startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
  jobs:
  - job: onPushJob
    pool:
      vmImage: 'ubuntu-18.04'

    steps:
    - script: env | sort