onastavchuk

fun main(args: Array<String>) {
  PropertyConfigurator.configure("src/main/resources/log4j.properties")
  System.setProperty("hadoop.home.dir", "c:/winutil/")
  Class.forName("com.dremio.jdbc.Driver")

onastavchuk

 val spark = SparkSession.builder()
     .appName("Dremio-Spark example")
     .master("local[*]")
     .getOrCreate()

onastavchuk

val dfRaw = spark.read()
        .format("jdbc")
        .option("url", "jdbc:dremio:direct=127.0.0.1:31010")
        .option("dbtable", """"@admin"."logs"""")
        .option("user", "admin")
        .option("password", "admin123qwe")
        .load()

onastavchuk

fun castDf(df: Dataset<Row>) =
    df.withColumn("_tmp", split(col("request"), " ")).select(
        col("host"),
        unix_timestamp(
            col("request_time"), "dd/MMM/yyyy:HH:mm:ss"
        ).cast("timestamp").alias("time"),
        col("_tmp").getItem(0).alias("verb"),
        col("_tmp").getItem(1).alias("resource"),
        col("status").cast("short"),
        col("bytes")

onastavchuk

val df = castDf(dfRaw)
df.printSchema()

onastavchuk

df.describe("bytes").show()

onastavchuk

df.groupBy("status")
    .count()
    .sort("status")
    .show()

onastavchuk

df.groupBy("host")
    .count()
    .filter(col("count").gt(700))
    .sort(col("count").desc())
    .show()

onastavchuk

df.filter(col("status").equalTo(404))
    .groupBy("resource")
    .count()
    .sort(desc("count"))
    .show(10)

onastavchuk

df.select("host")
    .distinct()
    .count()