Skip to content

Instantly share code, notes, and snippets.

@heinrichreimer
Created April 1, 2020 22:37
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save heinrichreimer/28b1bcfe7b151bb7ebf56b4bf0821f0d to your computer and use it in GitHub Desktop.
Save heinrichreimer/28b1bcfe7b151bb7ebf56b4bf0821f0d to your computer and use it in GitHub Desktop.
Elasticsearch Spark extension functions for Kotlin.
package de.webis.webisstud.thesis.reimer.clueweb.graph
import org.apache.spark.SparkConf
import org.apache.spark.api.java.JavaPairRDD
import org.apache.spark.api.java.JavaRDD
import org.apache.spark.api.java.JavaSparkContext
import org.elasticsearch.hadoop.cfg.ConfigurationOptions.*
import org.elasticsearch.spark.rdd.api.java.JavaEsSpark
fun JavaSparkContext.esRDD(): JavaPairRDD<String, Map<String, Any>> =
JavaEsSpark.esRDD(this)
fun JavaSparkContext.esRDD(resource: String): JavaPairRDD<String, Map<String, Any>> =
JavaEsSpark.esRDD(this, resource)
fun JavaSparkContext.esRDD(resource: String, query: String): JavaPairRDD<String, Map<String, Any>> =
JavaEsSpark.esRDD(this, resource, query)
fun JavaSparkContext.esRDD(cfg: Map<String, String>) =
JavaEsSpark.esRDD(this, cfg)
fun JavaSparkContext.esJsonRDD(): JavaPairRDD<String, String> =
JavaEsSpark.esJsonRDD(this)
fun JavaSparkContext.esJsonRDD(resource: String): JavaPairRDD<String, String> =
JavaEsSpark.esJsonRDD(this, resource)
fun JavaSparkContext.esJsonRDD(resource: String, query: String): JavaPairRDD<String, String> =
JavaEsSpark.esJsonRDD(this, resource, query)
fun JavaSparkContext.esJsonRDD(cfg: Map<String, String>): JavaPairRDD<String, String> =
JavaEsSpark.esJsonRDD(this, cfg)
fun JavaRDD<*>.saveToEs(resource: String) =
JavaEsSpark.saveToEs(this, resource)
fun JavaRDD<*>.saveToEs(resource: String, cfg: Map<String, String>) =
JavaEsSpark.saveToEs(this, resource, cfg)
fun JavaRDD<*>.saveToEs(cfg: Map<String, String>) =
JavaEsSpark.saveToEs(this, cfg)
fun <K, V> JavaPairRDD<K, V>.saveToEsWithMeta(resource: String) =
JavaEsSpark.saveToEsWithMeta(this, resource)
fun <K, V> JavaPairRDD<K, V>.saveToEsWithMeta(resource: String, cfg: Map<String, String>) =
JavaEsSpark.saveToEsWithMeta(this, resource, cfg)
fun <K, V> JavaPairRDD<K, V>.saveToEsWithMeta(cfg: Map<String, String>) =
JavaEsSpark.saveToEsWithMeta(this, cfg)
fun JavaRDD<String>.saveJsonToEs(resource: String) =
JavaEsSpark.saveJsonToEs(this, resource)
fun JavaRDD<String>.saveJsonToEs(resource: String, cfg: Map<String, String>) =
JavaEsSpark.saveJsonToEs(this, resource, cfg)
fun JavaRDD<String>.saveJsonToEs(cfg: Map<String, String>) =
JavaEsSpark.saveJsonToEs(this, cfg)
fun JavaRDD<ByteArray>.saveJsonByteArrayToEs(resource: String) =
JavaEsSpark.saveJsonByteArrayToEs(this, resource)
fun JavaRDD<ByteArray>.saveJsonByteArrayToEs(resource: String, cfg: Map<String, String>) =
JavaEsSpark.saveJsonByteArrayToEs(this, resource, cfg)
fun JavaRDD<ByteArray>.saveJsonByteArrayToEs(cfg: Map<String, String>) =
JavaEsSpark.saveJsonByteArrayToEs(this, cfg)
var SparkConf.esResource: String
get() = get(ES_RESOURCE)
set(value) {
set(ES_RESOURCE, value)
}
var SparkConf.esResourceRead: String
get() = get(ES_RESOURCE_READ)
set(value) {
set(ES_RESOURCE_READ, value)
}
var SparkConf.esResourceWrite: String
get() = get(ES_RESOURCE_WRITE)
set(value) {
set(ES_RESOURCE_WRITE, value)
}
var SparkConf.esNodes: String
get() = get(ES_NODES)
set(value) {
set(ES_NODES, value)
}
var SparkConf.esPort: Int
get() = getInt(ES_PORT, ES_PORT_DEFAULT.toInt())
set(value) {
set(ES_PORT, value.toString())
}
var SparkConf.esNodesWanOnly: Boolean
get() = getBoolean(ES_NODES_WAN_ONLY, ES_NODES_WAN_ONLY_DEFAULT.toBoolean())
set(value) {
set(ES_NODES_WAN_ONLY, value.toString())
}
var SparkConf.esNodesResolveHostname: Boolean
get() = getBoolean(ES_NODES_RESOLVE_HOST_NAME, !esNodesWanOnly)
set(value) {
set(ES_NODES_RESOLVE_HOST_NAME, value.toString())
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment