Scetch of TransmartRDD
class TransmartRDD extends RDD { | |
private Closure getRDD | |
private Closure getLocalFakeRDD | |
@Lazy | |
RDD rdd = getRDD() // getRDD also constructs the RDD lazily | |
@Lazy | |
LocalRDD localRDD = getLocalFakeRDD() | |
TransmartRDD(getRDD, getLocalFakeRDD) {this.getRDD = getRDD; this.getLocalFakeRDD = getLocalFakeRDD} | |
LocalRDD local() {return localRDD} | |
RDD map(...) { return rdd.map(...) } | |
RDD reduce(...) { return rdd.reduce(...) } | |
RDD filter(...) {return rdd.filter(...) } | |
RDD coalesce(...) { return rdd.coalesce(...) } | |
// etc | |
} | |
// in a different file | |
class LocalRDD extends RDD { | |
LocalRDD map(...) ... | |
LocalRDD reduce(...) ... | |
LocalRDD filter(...) ... | |
... | |
//partition related methods | |
RDD coalesce(...) { throw NotImplementedException(...) } | |
... | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment