Spark NLP SentenceDetector

SentenceEmbeddings.scala

  override def annotate(annotations: Seq[Annotation]): Seq[Annotation] = {
    val sentences = SentenceSplit.unpack(annotations)

    val embeddingsSentences = WordpieceEmbeddingsSentence.unpack(annotations)

    val embeddings = embeddingsSentences.map {
      case (tokenEmbedding) =>
        val allEmbeddings = tokenEmbedding.tokens.map { token =>
          token.embeddings
        }
        calculateSentenceEmbeddings(allEmbeddings)
    }.toArray

    val tup = sentences zip embeddings

    tup.map {
      (element: (Sentence, Array[Float])) => {
        Annotation(
          annotatorType = outputAnnotatorType,
          begin = element._1.start,
          end = element._1.end,
          result = element._1.content,
          metadata = Map("sentence" -> element._1.index.toString,
            "token" -> element._1.content,
            "pieceId" -> "-1",
            "isWordStart" -> "true"
          ),
          embeddings = element._2)
      }
    }
  }