Skip to content

Instantly share code, notes, and snippets.

@ccp-zoetrope

ccp-zoetrope/get_concurrent.py

Last active June 30, 2022 06:28
Download ZIP

Revisions

  1. ccp-zoetrope renamed this gist Sep 7, 2017. 1 changed file with 2 additions and 2 deletions.
    4 changes: 2 additions & 2 deletions get_async.py → get_concurrent.py
    View file
    Original file line number Diff line number Diff line change
    @@ -6,7 +6,7 @@
    MARKET_URL = "https://esi.tech.ccp.is/v1/markets/10000042/orders/"


    def async_requests(pages):
    def concurrent_requests(pages):
    reqs = []
    start_time = time.time()

    @@ -32,7 +32,7 @@ def async_requests(pages):

    all_orders.extend(res.json())
    pages = int(res.headers['X-Pages'])
    responses = async_requests(pages)
    responses = concurrent_requests(pages)

    for response in responses:
    try:
  2. ccp-zoetrope revised this gist Sep 7, 2017. 1 changed file with 1 addition and 2 deletions.
    3 changes: 1 addition & 2 deletions get_async.py
    View file
    Original file line number Diff line number Diff line change
    @@ -8,13 +8,12 @@

    def async_requests(pages):
    reqs = []
    start_time = time.time()

    for page in range(2, pages + 1):
    req = grequests.get(MARKET_URL, params={'page': page})
    reqs.append(req)

    start_time = time.time()

    responses = grequests.map(reqs)

    end_time = time.time()
  3. ccp-zoetrope created this gist Sep 7, 2017.
    47 changes: 47 additions & 0 deletions get_async.py
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,47 @@
    import grequests
    import time

    from requests.exceptions import HTTPError

    MARKET_URL = "https://esi.tech.ccp.is/v1/markets/10000042/orders/"


    def async_requests(pages):
    reqs = []

    for page in range(2, pages + 1):
    req = grequests.get(MARKET_URL, params={'page': page})
    reqs.append(req)

    start_time = time.time()

    responses = grequests.map(reqs)

    end_time = time.time()
    elapsed = end_time - start_time
    print('Elapsed time for {} requests was: {}'.format(len(responses), elapsed))

    return responses


    if __name__ == '__main__':
    all_orders = []
    req = grequests.get(MARKET_URL).send()
    res = req.response

    res.raise_for_status()

    all_orders.extend(res.json())
    pages = int(res.headers['X-Pages'])
    responses = async_requests(pages)

    for response in responses:
    try:
    response.raise_for_status()
    except HTTPError:
    print('Received status code {} from {}'.format(response.status_code, response.url))
    continue

    data = response.json()
    all_orders.extend(data)
    print('Got {:,d} orders.'.format(len(all_orders)))