Skip to content

Instantly share code, notes, and snippets.

@owickstrom
Created July 31, 2012 16:47
Show Gist options
  • Star 2 You must be signed in to star a gist
  • Fork 3 You must be signed in to fork a gist
  • Save owickstrom/3218376 to your computer and use it in GitHub Desktop.
Save owickstrom/3218376 to your computer and use it in GitHub Desktop.
C++11 Async cURL
#include <iostream>
#include <curl/curl.h>
#include <math.h>
#include <vector>
#include <future>
#include <boost/format.hpp>
#define logbase(a, b) (log(a)/log(b))
using namespace std;
using boost::format;
const char *fileSizes[] = {"byte", "K", "MB", "GB", "TB"};
const int numFileSizes = 5;
// adds chunk size to sum
size_t addChunkSize(char *ptr, size_t size, size_t nmemb, void *data)
{
auto acc = (long *)data;
auto chunkSize = size * nmemb;
*acc += chunkSize;
return chunkSize;
}
// returns a formatted string (number of bytes, K, MB, GB or TB)
string getSizeString(long fsize)
{
int fmtIndex;
format fmter;
if(fsize == 0 || (fmtIndex = (int)floor(logbase(fsize, 1024))) == 0)
{
fmter = format("%ld %s") % fsize % fileSizes[0];
}
else
{
fmtIndex = (fmtIndex >= numFileSizes) ? numFileSizes : fmtIndex;
fmter = format("%.2f %s")
% (fsize / pow(1024, fmtIndex))
% fileSizes[fmtIndex];
}
return fmter.str();
}
void usage(string name)
{
cout << "Usage: " << name << " url1 url2 url3 ...\n";
}
int main(int argc, const char * argv[])
{
long acc = 0L;
auto results = vector<future<long>>();
// if no urls
if(argc == 1)
{
usage(string(argv[0]));
exit(1);
}
for(int i = 1; i < argc; i++)
{
const char *url = argv[i];
// create future using a lambda and push it to the result vector
results.push_back(async(launch::async,
[url]() -> long
{
CURL *curl;
CURLcode res;
long siteSize = 0;
curl = curl_easy_init();
if(curl) {
// setup
curl_easy_setopt(curl, CURLOPT_URL, url);
// pass the siteSize pointer along (received by addChunkSize)
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &siteSize);
// callback for downloaded chunks
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, addChunkSize);
// make the request
res = curl_easy_perform(curl);
// something failed
if(res != CURLE_OK)
{
cerr << "\rWebsite size fetch failed for " << url << ": "
<< curl_easy_strerror(res) << "\n";
siteSize = 0;
}
// successful
else
{
cerr << "\rSize of " << url << ": "
<< getSizeString(siteSize) << "\n";
}
// cleanup
curl_easy_cleanup(curl);
}
return siteSize;
}));
}
// gather results from futures
for(auto &result : results)
{
acc += result.get();
}
cerr << "\nTotal size of websites: " << getSizeString(acc) << "\n";
return 0;
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment