alfredrichards

// returns a CompletableFuture which completes only when all the completable futures completes
CompletableFuture.allOf(completableFuture1, completableFuture2, ...);

// returns a CompletableFuture which completes when any one of the completable future completes
CompletableFuture.anyOf(completableFuture1, completableFuture2, ...);

alfredrichards

// callback is executed after the completion of 
// completableFuture1, completableFuture2

completableFuture1.thenCombine(completableFuture2, biFunction);

completableFuture1.thenAcceptBoth(completableFuture2, biConsumer);

completableFuture1.runAfterBoth(completableFuture2, runnable);

alfredrichards

CompletableFuture.supplyAsync(supplier)
        .thenApply(function)
        .thenAccept(consumer)
        .thenRun(runnableTask);

alfredrichards

ExecutorService executorService = Executors.newSingleThreadExecutor();

CompletableFuture.runAsync(task1, executorService)
            .thenRunAsync(task2, executorService)
            .join();

executorService.shutdown();

alfredrichards

Supplier<List<Integer>> supplier = () -> {
    try {
        Thread.sleep(1000);
    } catch (InterruptedException e) {
        e.printStackTrace();
    }
    return Arrays.asList(1, 2, 3);
};

Consumer<List<Integer>> consumer = input -> {

alfredrichards

ExecutorService executorService = Executors.newSingleThreadExecutor();

Runnable runnableTask = () -> System.out.println("This is a Runnable task");
Future<?> future1 = executorService.submit(runnableTask);
future1.wait();

Callable<String> callableTask = () -> "This is a Callable task";
Future<String> future2 = executorService.submit(callableTask);
String result = future2.get();

alfredrichards

export SPARK_HOME="/path/to/spark-home"
export PATH="$SPARK_HOME/bin:$PATH"

export PYSPARK_SUBMIT_ARGS="pyspark-shell"
export PYSPARK_DRIVER_PYTHON=ipython
export PYSPARK_DRIVER_PYTHON_OPTS='notebook' pyspark

alfredrichards

from pyspark.sql import SparkSession
from pyspark.sql.functions import col, lower
import sys


def main(argv):
    spark = SparkSession.builder\
                        .appName('spark-movies-app')\
                        .getOrCreate()