nga

ngacrawl.py

```python
import asyncio
from aiohttp import ClientSession, CookieJar
import os
import json
import pickle
import http.cookiejar

cj = http.cookiejar.MozillaCookieJar('cookies.txt')
cj.load()
cookies = {}
for each in cj:
    cookies[each.name] = each.value

async def fetch(url, session):
    async with session.get(url) as response:
        j = await response.text(encoding="gbk")
        j = json.loads(j.split("window.script_muti_get_var_store=")[1])
        return j

async def bound_fetch(sem, url, session):
    # Getter function with semaphore.
    async with sem:
        return await fetch(url, session)

async def run(r):
    url = "http://bbs.ngacn.cc/thread.php?fid=459&lite=js&page={}"
    tasks = []
    # create instance of Semaphore
    sem = asyncio.Semaphore(4)

    # Create client session that will ensure we dont open new connection
    # per each request.
    async with ClientSession(cookies=cookies) as session:
        for i in range(1, r + 1):
            # pass Semaphore and session to every GET request
            task = asyncio.ensure_future(bound_fetch(sem, url.format(i), session))
            tasks.append(task)

        responses = await asyncio.gather(*tasks)

        pickle.dump(responses, open("data.pickle", "wb"))

number = 4
loop = asyncio.get_event_loop()

future = asyncio.ensure_future(run(number))
loop.run_until_complete(future)
```