tzachz

@task
def nightly_test(ctx):
    """--->  Run Component Tests (Behave) - nightly scenarios """
    ctx.run("behave tests/features --no-capture --no-capture-stderr --tags=nightly")


@task
def release_test(ctx):
    """--->  Run Component Tests (Behave) - release scenarios"""
    ctx.run("behave tests/features --no-capture --no-capture-stderr --tags=release")

tzachz

Feature: test source amazon_brazil_v2

  @release
  Scenario: release test
    When I apply source amazon_br_v2.json to cached htmls amazon_br_v2.response.json for the following queries
      | queries    |
      | B079FX76R9 |
    Then I expect response to match the scraped items in amazon_br_v2.response.json

  @nightly

tzachz

import org.apache.spark.SparkContext
import org.apache.spark.sql.expressions.{MutableAggregationBuffer, UserDefinedAggregateFunction}
import org.apache.spark.sql.types._
import org.apache.spark.sql.{Column, Row, SQLContext}

/***
  * UDAF combining maps, overriding any duplicate key with "latest" value
  * @param keyType DataType of Map key
  * @param valueType DataType of Value key
  * @param merge function to merge values of identical keys

tzachz

apiVersion: apps/v1
kind: Deployment
metadata:
 name: publisher-report-fetcher-prod-amazon
spec:
 replicas: 3
 template:
   metadata:
     labels:
       app: publisher-report-fetcher-prod

tzachz

guice.modules: ${GUICE_MODULES:-[
   "io.skai.publisherreportfetcher.amazon.AmazonModule",
   "io.skai.publisherreportfetcher.google.GoogleModule",
   # … all other modules
]}

tzachz

@Singleton
class PublisherRegistry @Inject()(publishers: Seq[Publisher]) {

 def getPublisher(publisherType: PublisherType): Publisher = publishers
   .find(_.publisherType == publisherType)
   .getOrElse(failGracefully(publisherType))

}

tzachz

class GoogleModule extends AbstractModule {
 override def configure(): Unit = {
   // Register your implementation of Publisher by adding it to the set:
   Multibinder.newSetBinder(binder, classOf[Publisher])
       .addBinding()
       .to(classOf[GooglePublisher])
 }
}

tzachz

package com.kenshoo.kripke.core

import com.yammer.metrics.Metrics
import com.yammer.metrics.core.{MetricName, Counter}
import org.apache.spark.Accumulator
import org.apache.spark.rdd.RDD

import scala.reflect.ClassTag

object CounterBackedAccumulatorUtil {

tzachz

import com.codahale.metrics.MetricRegistry;
import io.dropwizard.db.DataSourceFactory;
import io.dropwizard.jackson.Jackson;
import io.dropwizard.jdbi.DBIFactory;
import io.dropwizard.setup.Environment;
import liquibase.Liquibase;
import liquibase.database.jvm.JdbcConnection;
import liquibase.exception.LiquibaseException;
import liquibase.resource.ClassLoaderResourceAccessor;
import org.junit.rules.ExternalResource;

tzachz

package com.tzachz.retry

import akka.actor.ActorSystem
import akka.pattern.Patterns.after

import scala.concurrent.duration._
import scala.concurrent.{ExecutionContext, Future}
import scala.reflect.ClassTag

/**