Skip to content

Instantly share code, notes, and snippets.

@ccp-zoetrope
Last active June 30, 2022 06:28
Show Gist options
  • Save ccp-zoetrope/6dc7bab3e8097ba5dad7629abbe31382 to your computer and use it in GitHub Desktop.
Save ccp-zoetrope/6dc7bab3e8097ba5dad7629abbe31382 to your computer and use it in GitHub Desktop.
Concurrently get all market orders
import grequests
import time
from requests.exceptions import HTTPError
MARKET_URL = "https://esi.tech.ccp.is/v1/markets/10000042/orders/"
def concurrent_requests(pages):
reqs = []
start_time = time.time()
for page in range(2, pages + 1):
req = grequests.get(MARKET_URL, params={'page': page})
reqs.append(req)
responses = grequests.map(reqs)
end_time = time.time()
elapsed = end_time - start_time
print('Elapsed time for {} requests was: {}'.format(len(responses), elapsed))
return responses
if __name__ == '__main__':
all_orders = []
req = grequests.get(MARKET_URL).send()
res = req.response
res.raise_for_status()
all_orders.extend(res.json())
pages = int(res.headers['X-Pages'])
responses = concurrent_requests(pages)
for response in responses:
try:
response.raise_for_status()
except HTTPError:
print('Received status code {} from {}'.format(response.status_code, response.url))
continue
data = response.json()
all_orders.extend(data)
print('Got {:,d} orders.'.format(len(all_orders)))
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment