BatchProcess.java

import com.tdunning.math.stats.TDigest;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.VoidFunction;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SparkSession;

public class BatchProcess {

    public static void main(String[] args) {

        String master = "local";
        String appName = "java_client";
        startProcess(master, appName);
    }

    public static void startProcess(String master, String appName) {


        SparkSession session = SparkSession.builder().master(master).appName(appName).getOrCreate();

        Dataset<Row> dataSet = session.read().option("header", "true").option("inferSchema", "true").csv("J:\\csv_path\\T_EN");

        JavaRDD<Integer> rdd = dataSet.javaRDD().map(
                new Function<Row, Integer>() {
                    public Integer call(Row r) {
                        return (Integer) r.getAs("Duration");
                    }
                });

        TDigest totalDigest = TDigest.createDigest(100);

        rdd.foreach(new VoidFunction<Integer>() {
            public void call(Integer value) {
                System.out.println(value);
                totalDigest.add(value);
            }
        });

        System.out.println("tdigest size " + totalDigest.size());

    }


}