parquet-query

package com.emasphere.poc.parquetspark.sample;

import com.emasphere.poc.parquetspark.ParquetCsvImporter;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SQLContext;
import org.apache.spark.sql.functions;

import java.math.BigDecimal;

/**
 * @author Sebastien Gerard
 */
public class Sample01 {

    public static BigDecimal compute(JavaSparkContext context) {
        final Dataset<Row> agg = SQLContext
                .getOrCreate(context.sc())
                .read()
                .parquet(ParquetCsvImporter.LOCATION)
                .select(
                        "montant", "code_vendeur", "datefact"
                )
                .filter(
                        "code_vendeur like '%JVA%' and datefact >=  date '2018-01-01' and datefact <= date '2018-06-30'"
                )
                .agg(
                        functions.sum("montant")
                );

        agg.explain();

        return agg
                .first()
                .getDecimal(0);
    }

}