-
-
Save anonymous/26c592dbf7a801d73353 to your computer and use it in GitHub Desktop.
If multiple requests come in at same time to `/next_url`, can I expect only 1 unique URL to be sent to each request?
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
offset = 0 | |
LIMIT = 100 | |
SAVE_AMOUNT = 50 | |
saves = 0 | |
outstanding_urls = {} | |
pending_results = [] | |
def getMoreURLs(): | |
global offset | |
global LIMIT | |
print "getMoreURLS() - OFFSET: %d, LIMIT: %d" % (offset, LIMIT) | |
# create SQL query | |
cur.execute("SELECT id, url FROM recipes ORDER BY id ASC OFFSET %s LIMIT %s", [str(offset), str(LIMIT)]) | |
# update offset variable | |
offset += LIMIT | |
# return query | |
return cur.fetchall() | |
urls = getMoreURLs() # first 100 URLs | |
@app.route('/api/v1/next_url', methods=['GET']) | |
def get_next(): | |
global urls | |
if len(urls) > 0: | |
url = urls.pop(0) | |
else: | |
urls = getMoreURLs() | |
# Check for any remaining URLs | |
if len(urls) == 0: | |
return jsonify({'error': 'No more URLs remaining'}) | |
# Create Object | |
response_obj = {'id': url[0], 'url': url[1], 'fetched_at': getTime()} | |
# Add URL to tracking dict | |
outstanding_urls[str(url[0])] = response_obj | |
return jsonify(response_obj) | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment