Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
import org.apache.spark.streaming.dstream.DStream
import org.apache.spark.sql.Row
trait StreamTransformer[S, T] {
def parse(input: DStream[S]): DStream[T]
}
class ProxyRowTransformer extends StreamTransformer[String, Row] {
def parse(stream: DStream[String]): DStream[Row] = {
stream
.transform(rdd => {
rdd.map(line => {
// Transform lines of text into SQL records, include failures
})
})
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.