Created
September 5, 2012 16:42
-
-
Save grapeot/3639640 to your computer and use it in GitHub Desktop.
An example using semaphore to manage async downloading requests
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
var semaphor = new Semaphore(50, 50); // We allow at most 50 threads for crawling | |
var resultPins = new List<Pin>(); // Results stored here | |
foreach (var pin in new HashSet<string>(pinIdList)) | |
{ | |
semaphor.WaitOne(); | |
Console.Write(">"); | |
var pinClient = new WebClient(); | |
pinClient.DownloadStringCompleted += (sender, ex) => | |
{ | |
var html = ex.Result.Replace("\n", ""); | |
pinClient.Dispose(); | |
lock (pinIdList) | |
{ | |
// do some post-processing and write back the results | |
} | |
Console.Write("<"); | |
semaphor.Release(); | |
}; | |
pinClient.DownloadStringAsync(new Uri(string.Format("http://pinterest.com/pin/{0}/", pin))); | |
} | |
for (int i = 0; i < 50; i++) semaphor.WaitOne(); // Wait until the last thread ends. | |
semaphor.Dispose(); | |
Console.WriteLine(); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment