IndexTask.scala

  val exclusions = List( // removed cid c-geo:c3
    "timestamp","d_memberID","gwid","diid","c-geo:st","c-geo:ac","c-geo:c2","c-geo:cn","c-geo:ct","c-geo:dc","c-geo:la","c-geo:lo","c-geo:pc","c-geo:rc","c-geo:sp",
    "ua","ua-cp","ua-os","accept","accept-charset","accept-encoding","accept-language","connection","from","x-wap-profile","x-att-deviceid","via","x-forwarded-for","x-uidh","forwarded","referrer","c-ip","ce","count","ctr","logic","max","miid","msg","npid","cs-uri-stem"
  )
  val segmentGrn = Granularity.DAY

  private def hadoopTask(interval: String) =
    IndexTask(
      IndexTask.hadoopType,
      IngestionSpec(
        DataSchema(
          "gwiq",
          Parser.hadoopyString(
            ParseSpec.json(
              TimestampSpec("timestamp"), DimensionsSpec(List.empty, exclusions, List.empty)
            )
          ),
          List(Aggregation.count("count"), Aggregation.hll("gwid", "gwid"), Aggregation.hll("d_memberID", "d_memberID")),
          GranularitySpec.uniform(List(interval), Some(segmentGrn.toString), Some(Granularity.DAY.toString))
        ),
        IoConfig.hadoop(InputSpec.granularity(Granularity.DAY.toString, s"s3n://${sys.env("AWS_ACCESS_KEY_ID")}:${sys.env("AWS_SECRET_ACCESS_KEY")}@gwiq-views-p/gwiq/base-json-gz", ".*", Some("yyyy/MM/dd"))),
        Some(TuningConfig(TuningConfig.hadoopType))
      )
    )