Skip to content

Instantly share code, notes, and snippets.

@nicmarti
Created Jul 21, 2017
Embed
What would you like to do?
Play 2.5 Java - Download a set of images, described in a JSON file with a list of URLs, using Akka stream.
package services;
import akka.actor.ActorSystem;
import akka.stream.ActorMaterializer;
import akka.stream.Materializer;
import akka.stream.javadsl.FileIO;
import com.fasterxml.jackson.databind.JsonNode;
import play.api.Play;
import play.libs.Json;
import play.libs.ws.StreamedResponse;
import play.libs.ws.WSClient;
import javax.inject.Inject;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.util.List;
import java.util.concurrent.CompletionStage;
/**
* This is a Play simple service, that uses Play WSClient and Akka Stream to load a JSON file with a list of URLS.
* Then each URL is downloaded and saved to a local folder using Akka Stream
*
* @author Nicolas Martignole Lunatech
*/
public class ImageDownloaderService {
private final WSClient ws;
private final ActorSystem system;
private final Materializer materializer;
// Try to use private field and constructor injection
// See Guice doc here https://github.com/google/guice/wiki/MinimizeMutability
@Inject
public ImageDownloaderService(final WSClient ws, final ActorSystem system) {
this.ws = ws;
this.system = system;
materializer = ActorMaterializer.create(system);
}
/**
* Load the specified JSON file, extract a list of URLs (in fact a list of images) then download all images.
*
* @param dataFile
*/
public void processDataFile(String dataFile) {
File jsonFile = new File(dataFile);
if (!jsonFile.exists()) {
play.Logger.error("File " + jsonFile.getAbsolutePath() + " does not exist");
return;
}
if (!jsonFile.canRead()) {
play.Logger.error("File " + jsonFile.getAbsolutePath() + " cannot be read");
return;
}
try {
JsonNode json = Json.parse(new FileInputStream(jsonFile));
json.withArray("urls").forEach(node -> downloadImageFile(node.asText()));
} catch (FileNotFoundException e) {
play.Logger.error("Cannot proceed, the input JSON file was not found or could not be read", e);
}
}
// Play 2.5 Akka Stream -> Download an image
protected void downloadImageFile(String imageURL) {
play.Logger.debug("Downloading " + imageURL);
CompletionStage<StreamedResponse> futureResponse = ws.url(imageURL).setFollowRedirects(true).stream();
// Version 1
// futureResponse.thenApply(response -> {
// response.getBody().runWith(FileIO.toPath(Paths.get("factorials.txt")), materializer);
// System.out.println("In the future Response");
// return "ok";
// });
// Version 2
futureResponse.whenComplete((response, throwable) -> {
if (throwable != null) {
play.Logger.error("Could not download an image, url " + imageURL, throwable);
} else {
downloadFile(response, imageURL);
}
});
}
protected void downloadFile(StreamedResponse streamedResponse, final String url) {
if (streamedResponse.getHeaders().getStatus() != 200) {
play.Logger.warn("Tried to download an image but got a HTTP Status error code != 200", url);
return;
}
List<String> allContentType = streamedResponse.getHeaders().getHeaders().get("Content-Type");
String contentType;
if (allContentType.isEmpty()) {
contentType = "application/octet-stream";
} else {
contentType = allContentType.get(0);
}
play.Logger.debug("ImageDownloaderService identified content-type " + contentType);
if (contentType.equalsIgnoreCase("text/html")) {
play.Logger.warn("Received text/html instead of image. Could not download " + url);
return;
}
File imagesFolder = Play.current().getFile("public/images");
if (!imagesFolder.exists() || !imagesFolder.canRead() || !imagesFolder.canWrite()) {
play.Logger.error("The folder public/images does not exist or cannot be read");
return;
}
String extension = extractExtension(url);
if (extension == null) {
play.Logger.error("Could not find image type from url " + url);
play.Logger.error("However we had content-type => " + contentType);
} else {
File newImage = new File(imagesFolder, url.hashCode() + extension);
streamedResponse.getBody().runWith(FileIO.toPath(newImage.toPath()), materializer);
play.Logger.debug("Saved new image " + newImage.getAbsolutePath());
}
}
protected String extractExtension(String url) {
if (url == null) return null;
if (url.toLowerCase().endsWith(".jpg")) return ".jpg";
if (url.toLowerCase().endsWith(".png")) return ".png";
if (url.toLowerCase().contains(".jpg")) return ".jpg";
if (url.toLowerCase().contains(".png")) return ".png";
return null;
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment