Skip to content

Instantly share code, notes, and snippets.

Created September 6, 2014 23:17
Show Gist options
  • Save anonymous/20608afe2d168dc872e3 to your computer and use it in GitHub Desktop.
Save anonymous/20608afe2d168dc872e3 to your computer and use it in GitHub Desktop.
async getter
import asyncdispatch, httpclient, os, strutils
# ~/nimbb/nim/bin/nim c --threadAnalysis:off -d:ssl -r tcrawl3.nim 10 300 500
var numRunning = 0
var good = 0
var bad = 0
# not used....
var concurrency = 3
if paramCount() > 0:
concurrency = paramStr(1).parseInt()
var sleepTime = 300
if paramCount() > 1:
sleepTime = paramStr(2).parseInt()
var numToRun = 30
if paramCount() > 2:
numToRun = paramStr(3).parseInt()
var totGot = 0
proc createRequest(url: string) {.async.} =
numRunning += 1
echo "numRunning: ", numRunning
var client = newAsyncHttpClient()
try:
let resp = await client.request(url)
echo("Got resp: ", resp.status)
good += 1
echo "good: ", good
except:
bad += 1
echo "bad: ", bad
echo("Exception!")
numRunning -= 1
totGot += 1
echo "got: ", totGot
# 1000 domains
var domurl = "http://goo.gl/yzlQ5n"
var doms = getContent(domurl).split("\n")
echo doms
#quit()
proc main() {.async.} =
for x in low(doms)..numToRun:
asyncCheck createRequest("http://www." & doms[x])
echo "all running"
while numRunning > 0:
echo "sleeping: ", sleepTime, " running: ", numRunning
await sleepAsync(sleepTime)
when isMainModule:
waitFor main()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment