Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
2 parallel streams, using select to remove unnecessary loop cycles
Aim: download total 100 files, 2 files multiplexed at a time
Code:
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#ifdef _WIN32
#define SHORT_SLEEP Sleep(100)
#else
#define SHORT_SLEEP usleep(100000)
#endif
/* curl stuff */
#include <curl/curl.h>
struct MemoryStruct {
char *memory;
size_t size;
};
static size_t header_callback(char *buffer, size_t size,
size_t nitems, void *userdata)
{
/* received header is nitems * size long in 'buffer' NOT ZERO TERMINATED */
/* 'userdata' is set with CURLOPT_HEADERDATA */
return nitems * size;
}
static size_t
WriteMemoryCallback(void *contents, size_t size, size_t nmemb, void *userp)
{
size_t realsize = size * nmemb;
struct MemoryStruct *mem = (struct MemoryStruct *)userp;
mem->memory = realloc(mem->memory, mem->size + realsize + 1);
/* out of memory! */
printf("not enough memory (realloc returned NULL)\n");
return 0;
}
memcpy(&(mem->memory[mem->size]), contents, realsize);
mem->size += realsize;
mem->memory[mem->size] = 0;
// printf("\nWriting: %d Bytes in memory",mem->size);
return realsize;
}
int main(){
//check nghttp2 supprt
const curl_version_info_data *data = curl_version_info(CURLVERSION_NOW);
if(data->features & CURL_VERSION_HTTP2){
fprintf(stdout, "This libcurl DOES have HTTP2 support!\n");
}
CURLcode res;
struct MemoryStruct chunk;
// Create multi handle with multiplex over a single connection
CURLM *multi_handle = curl_multi_init();
curl_multi_setopt(multi_handle, CURLMOPT_MAX_HOST_CONNECTIONS, (long) 1L);
curl_multi_setopt(multi_handle, CURLMOPT_PIPELINING, CURLPIPE_HTTP1 | CURLPIPE_MULTIPLEX);
//fd
struct timeval timeout;
int rc;
fd_set fdread;
CURLMcode mc;
fd_set fdwrite;
fd_set fdexcep;
int maxfd = -1;
long curl_timeo;
curl_multi_timeout(multi_handle, &curl_timeo);
if(curl_timeo < 0)
curl_timeo = 1000;
timeout.tv_sec = curl_timeo / 1000;
timeout.tv_usec = (curl_timeo % 1000) * 1000;
FD_ZERO(&fdread);
FD_ZERO(&fdwrite);
FD_ZERO(&fdexcep);
/* get file descriptors from the transfers */
mc = curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd);
// Add some requests
//int NUM_HANDLES = 1;
int NUM_HANDLES = 2;
CURL *easy[NUM_HANDLES];
for (int i = 0; i < NUM_HANDLES; i++) {
//easy[i] = curl_easy_init();
} /* curl_easy_setopt(easy[i], CURLOPT_VERBOSE, 1L);
curl_easy_setopt(easy[i], CURLOPT_SSLCERTTYPE, "PEM");
curl_easy_setopt(easy[i], CURLOPT_CAINFO, "cert.pem");
curl_easy_setopt(easy[i], CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_2);
curl_easy_setopt(easy[i], CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
}*/
// for (int j = 1; j < 149; j++) {
for (int j = 0; j < 2; j++) {
chunk.memory = malloc(1); /* will be grown as needed by the realloc above */
chunk.size = 0; /* no data at this point */
for (int i = 0; i < NUM_HANDLES; i++) {
char url[1024];
snprintf(url, 1024, "https://10.10.3.2/www-itec.uni-klu.ac.at/ftp/datasets/DASHDataset2014/Bi\
gBuckBunny/2sec/bunny_4219897bps/BigBuckBunny_2s%d.m4s",((j*2)+(i+1)));
//snprintf(url, 1024, "https://http2.akamai.com/demo");
easy[i] = curl_easy_init();
curl_easy_setopt(easy[i], CURLOPT_VERBOSE, 1L);
printf("\nurl:%s",url);
curl_easy_setopt(easy[i], CURLOPT_URL, url);
curl_easy_setopt(easy[i], CURLOPT_SSLCERTTYPE, "PEM");
curl_easy_setopt(easy[i], CURLOPT_CAINFO, "cert.pem");
curl_easy_setopt(easy[i], CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_2);
curl_easy_setopt(easy[i], CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
/* send all data to this function */
curl_easy_setopt(easy[i], CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
/* we pass our 'chunk' struct to the callback function */
curl_easy_setopt(easy[i], CURLOPT_WRITEDATA, (void *)&chunk);
curl_multi_add_handle(multi_handle, easy[i]);
}
/* we start some action by calling perform right away */
int still_running;
curl_multi_perform(multi_handle, &still_running);
do {
struct timeval timeout;
int rc; /* select() return code */
CURLMcode mc; /* curl_multi_fdset() return code */
fd_set fdread;
fd_set fdwrite;
fd_set fdexcep;
int maxfd = -1;
long curl_timeo = -1;
FD_ZERO(&fdread);
FD_ZERO(&fdwrite);
FD_ZERO(&fdexcep);
/* set a suitable timeout to play around with */
timeout.tv_sec = 1;
timeout.tv_usec = 0;
curl_multi_timeout(multi_handle, &curl_timeo);
if(curl_timeo >= 0) {
timeout.tv_sec = curl_timeo / 1000;
if(timeout.tv_sec > 1)
timeout.tv_sec = 1;
else
timeout.tv_usec = (curl_timeo % 1000) * 1000;
}
/* get file descriptors from the transfers */
mc = curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd);
if(mc != CURLM_OK) {
fprintf(stderr, "curl_multi_fdset() failed, code %d.\n", mc);
break;
}
//fprintf(stderr, "curl_multi_fdset() failed, code %d.\n", mc);
/* On success the value of maxfd is guaranteed to be >= -1. We call
select(maxfd + 1, ...); specially in case of (maxfd == -1) there are
no fds ready yet so we call select(0, ...) --or Sleep() on Windows--
to sleep 100ms, which is the minimum suggested value in the
curl_multi_fdset() doc. */
if(maxfd == -1) {
#ifdef _WIN32
Sleep(100);
rc = 0;
#else
/* Portable sleep for platforms other than Windows. */
struct timeval wait = { 0, 100 * 1000 }; /* 100ms */
rc = select(0, NULL, NULL, NULL, &wait);
#endif
}
else {
/* Note that on some platforms 'timeout' may be modified by select().
If you need access to the original value save a copy beforehand. */
rc = select(maxfd + 1, &fdread, &fdwrite, &fdexcep, &timeout);
}
switch(rc) {
case -1:
/* select error */
break;
case 0:
default:
/* timeout or readable/writable sockets */
curl_multi_perform(multi_handle, &still_running);
break;
}
} while(still_running);
free(chunk.memory); // essentially data from parallel streams is stored in memory
// and cleared when we move to next set of parallel stream downloads
}
//cleanups
for(int i = 0; i < NUM_HANDLES; i++)
curl_easy_cleanup(easy[i]);
curl_multi_cleanup(multi_handle);
exit(0);
}
Compile:
gcc libcurl-multi-stream-client-app.c -o libcurl-multi-stream-client-app.o -std=c99 -lcurl
./libcurl-multi-stream-client-app.o
g++ libcurl-http2-final-slave-func.c -o libcurl-http2-final-slave-func.o --std=c++11 -lcurl
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment