Skip to content

Instantly share code, notes, and snippets.

@Fokko
Last active May 16, 2018 08:05
Show Gist options
  • Save Fokko/fd99e0d487c8a616e614446f173c8616 to your computer and use it in GitHub Desktop.
Save Fokko/fd99e0d487c8a616e614446f173c8616 to your computer and use it in GitHub Desktop.
import requests as rq
import argparse
import random
from pyspark.sql import SparkSession
spark = SparkSession.builder.appName('Set Vincent\'s cluster on fire').getOrCreate()
def f(_):
current_max = {'score': 0.0};
url = 'http://pydata2018.godatadriven.com'
min_lat, max_lat = -90.0, 90.0
min_lng, max_lng = -180.0, 180.0
payload = {'name': 'Fokko',
'moonlat': random.random()*(max_lat-min_lat)+min_lat,
'moonlng': random.random()*(max_lng-min_lng)+min_lng}
resp = rq.post(url, json=payload).json()
if 'score' not in resp.keys():
raise Exception("Could not find key in {}", str(resp))
if resp['score'] >= current_max['score']:
current_max = resp;
return current_max;
topn = spark.sparkContext.parallelize(range(1, 10000000), 10000).map(f).takeOrdered(20, key=lambda x: x['score'] * -1)
print(topn)
spark.stop()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment