Skip to content

Instantly share code, notes, and snippets.

Dana Groce danared

  • MongoDB
Block or report user

Report or block danared

Hide content and notifications from this user.

Learn more about blocking users

Contact Support about this user’s behavior.

Learn more about reporting abuse

Report abuse
View GitHub Profile
View mongospark
def main(args: Array[String]): Unit = {
// Set up configurations
val sc = getSparkContext()
val sqlContext = SQLContext.getOrCreate(sc)
val readConfig = ReadConfig(Map("uri" -> "mongodb://127.0.0.1/movies.movie_ratings?readPreference=primaryPreferred"))
val writeConfig = WriteConfig(Map("uri" -> "mongodb://127.0.0.1/movies.user_recommendations"))
val userId = 0
View mongospark
/**
* Gets or creates the Spark Context
*/
def getSparkContext(): SparkContext = {
val conf = new SparkConf()
.setMaster("local[*]")
.setAppName("MovieRatings")
val sc = SparkContext.getOrCreate(conf)
sc.setCheckpointDir("/tmp/checkpoint/")
View mongospark
package example
import org.apache.log4j.{Level, Logger}
import org.apache.spark.ml.evaluation.RegressionEvaluator
import org.apache.spark.ml.recommendation.ALS
import org.apache.spark.ml.tuning.{ParamGridBuilder, TrainValidationSplit}
import org.apache.spark.sql.SQLContext
import org.apache.spark.{SparkConf, SparkContext}
import com.mongodb.spark.MongoSpark
View gist:60eac629c1b55d88c29d
function foo() {
someAsyncFunction(params, function(err, results) {
console.log(“one”);
});
console.log(“two”);
}
View gist:e3bd05a074e3608cb862
{
"_id" : ObjectId("523b2fcc054b1b8c579bdb82"),
"read" : true,
"user_id" : ObjectId("54ad6c3ae764de42070b27b1"),
"feed_entry_id" : ObjectId("523b1153a2aa6a3233a91412"),
"feed_id" : ObjectId("523b1153a2aa6a3233a913f8")
}
View randomdocs
number_of_documents = 100
load_data(collection, number_of_documents)
winner = [ d for d in collection.aggregate([{'$sample': {'size': 1 }}])][0]
print "AND THE WINNER IS ..... " + winner['name']
View randomdocs
number_of_documents = 100
load_data(collection, number_of_documents )
query = {'i': random.randint(0, 10 ) }
docs = [x for x in collection.find(query)]
winner = random.sample(docs, 1)[0]
View randomdocs
def load_data(collection, n=100):
#fixed number of marks
max_i = 10
for j,d in load_data_file(n):
d['i'] = random.randint(0, max_i)
collection.insert( d )
View randomdocs
def load_data(collection, n=100):
#let's skip some elements
skiplist = [10, 12, 231 , 2 , 4]
for i,d in load_data_file(n):
d['i'] = i
if i in skiplist:
continue
collection.insert( d )
load_data(collection, 100)
View randomdocs
number_of_documents = collection.count()
You can’t perform that action at this time.