2 parallel streams, using select to remove unnecessary loop cycles

multi-stream-libcurl

Aim: download total 100 files, 2 files multiplexed at a time

Code:
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#ifdef _WIN32
#define SHORT_SLEEP Sleep(100)
#else
#define SHORT_SLEEP usleep(100000)
#endif

/* curl stuff */
#include <curl/curl.h>

struct MemoryStruct {
  char *memory;
  size_t size;
};

static size_t header_callback(char *buffer, size_t size,
                              size_t nitems, void *userdata)
{
  /* received header is nitems * size long in 'buffer' NOT ZERO TERMINATED */
  /* 'userdata' is set with CURLOPT_HEADERDATA */
  return nitems * size;
}

static size_t
WriteMemoryCallback(void *contents, size_t size, size_t nmemb, void *userp)
{
  size_t realsize = size * nmemb;
  struct MemoryStruct *mem = (struct MemoryStruct *)userp;

  mem->memory = realloc(mem->memory, mem->size + realsize + 1);
    /* out of memory! */
    printf("not enough memory (realloc returned NULL)\n");
    return 0;
  }

  memcpy(&(mem->memory[mem->size]), contents, realsize);
  mem->size += realsize;
  mem->memory[mem->size] = 0;

  //  printf("\nWriting: %d Bytes in memory",mem->size);                                            

  return realsize;
}

int main(){
  //check nghttp2 supprt                                                                            
  const curl_version_info_data *data = curl_version_info(CURLVERSION_NOW);
  if(data->features & CURL_VERSION_HTTP2){
    fprintf(stdout, "This libcurl DOES have HTTP2 support!\n");
  }

  CURLcode res;

  struct MemoryStruct chunk;

  // Create multi handle with multiplex over a single connection                                    
  CURLM *multi_handle = curl_multi_init();
  curl_multi_setopt(multi_handle, CURLMOPT_MAX_HOST_CONNECTIONS, (long) 1L);
  curl_multi_setopt(multi_handle, CURLMOPT_PIPELINING, CURLPIPE_HTTP1 | CURLPIPE_MULTIPLEX);

  //fd                                                                                              
  struct timeval timeout;
  int rc;
  fd_set fdread;
  CURLMcode mc;
  fd_set fdwrite;
  fd_set fdexcep;
  int maxfd = -1;

  long curl_timeo;

  curl_multi_timeout(multi_handle, &curl_timeo);
  if(curl_timeo < 0)
    curl_timeo = 1000;

  timeout.tv_sec = curl_timeo / 1000;
  timeout.tv_usec = (curl_timeo % 1000) * 1000;

  FD_ZERO(&fdread);
  FD_ZERO(&fdwrite);
  FD_ZERO(&fdexcep);

  /* get file descriptors from the transfers */
  mc = curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd);

  // Add some requests                                                                              
  //int NUM_HANDLES = 1;
    int NUM_HANDLES = 2;
  CURL *easy[NUM_HANDLES];
  for (int i = 0; i < NUM_HANDLES; i++) {
    //easy[i] = curl_easy_init();                                                                   
  } /*  curl_easy_setopt(easy[i], CURLOPT_VERBOSE, 1L);                                             
    curl_easy_setopt(easy[i], CURLOPT_SSLCERTTYPE, "PEM");                                          
    curl_easy_setopt(easy[i], CURLOPT_CAINFO, "cert.pem");                                          
    curl_easy_setopt(easy[i], CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_2);                         
    curl_easy_setopt(easy[i], CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);                         
  }*/

  //  for (int j = 1; j < 149; j++) {                                                               
  for (int j = 0; j < 2; j++) {

    chunk.memory = malloc(1);  /* will be grown as needed by the realloc above */
    chunk.size = 0;    /* no data at this point */

    for (int i = 0; i < NUM_HANDLES; i++) {
      char url[1024];
      snprintf(url, 1024, "https://10.10.3.2/www-itec.uni-klu.ac.at/ftp/datasets/DASHDataset2014/Bi\
gBuckBunny/2sec/bunny_4219897bps/BigBuckBunny_2s%d.m4s",((j*2)+(i+1)));
      //snprintf(url, 1024, "https://http2.akamai.com/demo");                                       
      easy[i] = curl_easy_init();
      curl_easy_setopt(easy[i], CURLOPT_VERBOSE, 1L);
      printf("\nurl:%s",url);
      curl_easy_setopt(easy[i], CURLOPT_URL, url);
      curl_easy_setopt(easy[i], CURLOPT_SSLCERTTYPE, "PEM");
      curl_easy_setopt(easy[i], CURLOPT_CAINFO, "cert.pem");
      curl_easy_setopt(easy[i], CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_2);
      curl_easy_setopt(easy[i], CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
      /* send all data to this function  */
      curl_easy_setopt(easy[i], CURLOPT_WRITEFUNCTION, WriteMemoryCallback);

      /* we pass our 'chunk' struct to the callback function */
      curl_easy_setopt(easy[i], CURLOPT_WRITEDATA, (void *)&chunk);

      curl_multi_add_handle(multi_handle, easy[i]);
    }

    /* we start some action by calling perform right away */
    int still_running;
    curl_multi_perform(multi_handle, &still_running);
    do {
      struct timeval timeout;
      int rc; /* select() return code */
      CURLMcode mc; /* curl_multi_fdset() return code */

      fd_set fdread;
      fd_set fdwrite;
      fd_set fdexcep;
      int maxfd = -1;

      long curl_timeo = -1;

      FD_ZERO(&fdread);
      FD_ZERO(&fdwrite);
      FD_ZERO(&fdexcep);
      /* set a suitable timeout to play around with */
      timeout.tv_sec = 1;
      timeout.tv_usec = 0;

      curl_multi_timeout(multi_handle, &curl_timeo);
      if(curl_timeo >= 0) {
        timeout.tv_sec = curl_timeo / 1000;
        if(timeout.tv_sec > 1)
          timeout.tv_sec = 1;
        else
          timeout.tv_usec = (curl_timeo % 1000) * 1000;
      }

      /* get file descriptors from the transfers */
      mc = curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd);

      if(mc != CURLM_OK) {
        fprintf(stderr, "curl_multi_fdset() failed, code %d.\n", mc);
        break;
      }
//fprintf(stderr, "curl_multi_fdset() failed, code %d.\n", mc);                               

      /* On success the value of maxfd is guaranteed to be >= -1. We call                           
       select(maxfd + 1, ...); specially in case of (maxfd == -1) there are                         
       no fds ready yet so we call select(0, ...) --or Sleep() on Windows--                         
       to sleep 100ms, which is the minimum suggested value in the                                  
       curl_multi_fdset() doc. */

      if(maxfd == -1) {
#ifdef _WIN32
        Sleep(100);
        rc = 0;
#else
        /* Portable sleep for platforms other than Windows. */
        struct timeval wait = { 0, 100 * 1000 }; /* 100ms */
        rc = select(0, NULL, NULL, NULL, &wait);
#endif
      }
      else {
        /* Note that on some platforms 'timeout' may be modified by select().      
           If you need access to the original value save a copy beforehand. */
        rc = select(maxfd + 1, &fdread, &fdwrite, &fdexcep, &timeout);
      }

      switch(rc) {
      case -1:
        /* select error */
        break;
      case 0:
      default:
        /* timeout or readable/writable sockets */
        curl_multi_perform(multi_handle, &still_running);
        break;
      }
    } while(still_running);
    free(chunk.memory); // essentially data from parallel streams is stored in memory               
                        // and cleared when we move to next set of parallel stream downloads        

  }
    //cleanups                                                                                      
    for(int i = 0; i < NUM_HANDLES; i++)
      curl_easy_cleanup(easy[i]);
    curl_multi_cleanup(multi_handle);

  exit(0);
}


Compile:
gcc libcurl-multi-stream-client-app.c -o libcurl-multi-stream-client-app.o -std=c99 -lcurl
./libcurl-multi-stream-client-app.o

g++ libcurl-http2-final-slave-func.c -o libcurl-http2-final-slave-func.o --std=c++11 -lcurl