Skip to content

Instantly share code, notes, and snippets.

@willprice76
Last active August 27, 2020 10:58
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save willprice76/79950586bcb4f09d45f16f4919fa2c66 to your computer and use it in GitHub Desktop.
Save willprice76/79950586bcb4f09d45f16f4919fa2c66 to your computer and use it in GitHub Desktop.
Adding scoring to the Spark NLP controller
package org.example.sparknlp;
import com.johnsnowlabs.nlp.LightPipeline;
import com.johnsnowlabs.nlp.SparkNLP;
import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline;
import org.apache.spark.sql.SparkSession;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RestController;
import scala.collection.JavaConversions;
import scala.collection.Seq;
import scala.collection.immutable.Map;
import java.time.Duration;
import java.time.Instant;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
@RestController
public class SparkNlpController {
private static final Logger LOG = LoggerFactory.getLogger(SparkNlpController.class);
private LightPipeline scoringPipeline;
private SparkSession spark;
public SparkNlpController() {
spark = SparkNLP.start(false, false);
scoringPipeline = new PretrainedPipeline("analyze_sentiment", "en").lightModel();
}
@PostMapping("/sentiment/score")
public List<String> score(@RequestBody String[] inputData) {
Instant start = Instant.now();
LOG.debug("Analyzing {} rows of text data...", inputData.length);
//Run test text through the pipeline
Map<String, Seq<String>>[] annotations = scoringPipeline.annotate(inputData);
//Mangle the sentiment data out
List<String> output = Arrays.stream(annotations).map(annotation -> {
List<String> sentiment = JavaConversions.seqAsJavaList(annotation.get("sentiment").getOrElse(null));
return sentiment.get(0);
}).collect(Collectors.toList());
LOG.debug("Analysis completed in {} milliseconds", Duration.between(start, Instant.now()).toMillis());
return output;
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment