Play 2.5 Java - Download a set of images, described in a JSON file with a list of URLs, using Akka stream.

java

package services;

import akka.actor.ActorSystem;
import akka.stream.ActorMaterializer;
import akka.stream.Materializer;
import akka.stream.javadsl.FileIO;
import com.fasterxml.jackson.databind.JsonNode;
import play.api.Play;
import play.libs.Json;
import play.libs.ws.StreamedResponse;
import play.libs.ws.WSClient;

import javax.inject.Inject;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.util.List;
import java.util.concurrent.CompletionStage;

/**
 * This is a Play simple service, that uses Play WSClient and Akka Stream to load a JSON file with a list of URLS.
 * Then each URL is downloaded and saved to a local folder using Akka Stream
 *
 * @author Nicolas Martignole Lunatech
 */
public class ImageDownloaderService {
    private final WSClient ws;
    private final ActorSystem system;
    private final Materializer materializer;

    // Try to use private field and constructor injection
    // See Guice doc here https://github.com/google/guice/wiki/MinimizeMutability
    @Inject
    public ImageDownloaderService(final WSClient ws, final ActorSystem system) {
        this.ws = ws;
        this.system = system;
        materializer = ActorMaterializer.create(system);
    }

    /**
     * Load the specified JSON file, extract a list of URLs (in fact a list of images) then download all images.
     *
     * @param dataFile
     */
    public void processDataFile(String dataFile) {

        File jsonFile = new File(dataFile);

        if (!jsonFile.exists()) {
            play.Logger.error("File " + jsonFile.getAbsolutePath() + " does not exist");
            return;
        }
        if (!jsonFile.canRead()) {
            play.Logger.error("File " + jsonFile.getAbsolutePath() + " cannot be read");
            return;
        }

        try {
            JsonNode json = Json.parse(new FileInputStream(jsonFile));
            json.withArray("urls").forEach(node -> downloadImageFile(node.asText()));
        } catch (FileNotFoundException e) {
            play.Logger.error("Cannot proceed, the input JSON file was not found or could not be read", e);
        }
    }

    // Play 2.5 Akka Stream -> Download an image
    protected void downloadImageFile(String imageURL) {
        play.Logger.debug("Downloading " + imageURL);

        CompletionStage<StreamedResponse> futureResponse = ws.url(imageURL).setFollowRedirects(true).stream();

        // Version 1
//        futureResponse.thenApply(response -> {
//            response.getBody().runWith(FileIO.toPath(Paths.get("factorials.txt")), materializer);
//            System.out.println("In the future Response");
//            return "ok";
//        });

        // Version 2
        futureResponse.whenComplete((response, throwable) -> {
            if (throwable != null) {
                play.Logger.error("Could not download an image, url " + imageURL, throwable);
            } else {
                downloadFile(response, imageURL);
            }
        });


    }

    protected void downloadFile(StreamedResponse streamedResponse, final String url) {
        if (streamedResponse.getHeaders().getStatus() != 200) {
            play.Logger.warn("Tried to download an image but got a HTTP Status error code != 200", url);
            return;
        }
        List<String> allContentType = streamedResponse.getHeaders().getHeaders().get("Content-Type");
        String contentType;
        if (allContentType.isEmpty()) {
            contentType = "application/octet-stream";
        } else {
            contentType = allContentType.get(0);
        }
        play.Logger.debug("ImageDownloaderService identified content-type " + contentType);

        if (contentType.equalsIgnoreCase("text/html")) {
            play.Logger.warn("Received text/html instead of image. Could not download " + url);
            return;
        }

        File imagesFolder = Play.current().getFile("public/images");
        if (!imagesFolder.exists() || !imagesFolder.canRead() || !imagesFolder.canWrite()) {
            play.Logger.error("The folder public/images does not exist or cannot be read");
            return;
        }

        String extension = extractExtension(url);
        if (extension == null) {
            play.Logger.error("Could not find image type from url " + url);
            play.Logger.error("However we had content-type => " + contentType);
        } else {

            File newImage = new File(imagesFolder, url.hashCode() + extension);
            streamedResponse.getBody().runWith(FileIO.toPath(newImage.toPath()), materializer);
            play.Logger.debug("Saved new image " + newImage.getAbsolutePath());
        }
    }

    protected String extractExtension(String url) {
        if (url == null) return null;
        if (url.toLowerCase().endsWith(".jpg")) return ".jpg";
        if (url.toLowerCase().endsWith(".png")) return ".png";
        if (url.toLowerCase().contains(".jpg")) return ".jpg";
        if (url.toLowerCase().contains(".png")) return ".png";
        return null;
    }

}