Skip to content

Instantly share code, notes, and snippets.

@willprice76
Created February 6, 2020 17:09
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save willprice76/ceec2026c0935d90967d0be80205c4ed to your computer and use it in GitHub Desktop.
Save willprice76/ceec2026c0935d90967d0be80205c4ed to your computer and use it in GitHub Desktop.
Test class for loading Spark NLP pipeline from local FS
package test;
import org.apache.spark.ml.PipelineModel;
import org.apache.spark.sql.SparkSession;
import org.junit.jupiter.api.Test;
import java.nio.file.Paths;
public class LoadPipelineTest {
@Test
public void testSparkNLP() {
//Set up Spark session - ignore Hadoop/jetty/netty errors
SparkSession spark = SparkSession
.builder()
.appName("Spark NLP")
.master("local[*]")
.config("spark.master", "local")
.config("spark.driver.memory", "6G")
.config("spark.serializer", "org.apache.spark.serializer" +
".KryoSerializer")
.config("spark.jars.packages", "com.johnsnowlabs.nlp:spark-nlp_2.11:2.4.0")
.getOrCreate();
//Load local FS pretrained pipeline
String path = "analyze_sentiment_en_2.1.0_2.4_1563204637489";
String localPipelineLocation = Paths.get(path).toAbsolutePath().toString();
PipelineModel pipeline = PipelineModel.load(localPipelineLocation);
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment