Curl example with a background thread, a mutex, and a shared buffer

threadmutex.c

/*
 * Perform a fetch in a background worker thread, and use a mutex to safely grab chunks from the buffer.
 *
 * Usage:
 * cc threadmutex.c -lcurl -lpthread -o threadmutex
 * ./threadmutex
 *
 * Or, if curl will be using OpenSSL define "USE_OPENSSL":
 * cc threadmutex.c -lcurl -lpthread -lcrypto -DUSE_OPENSSL -o threadmutex
 * ./threadmutex
 *
 */
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <unistd.h>
#include <pthread.h>
#include <curl/curl.h>

#ifdef USE_OPENSSL
#include <openssl/crypto.h>
#endif

#define NUMT 4

struct WorkerContext {
  pthread_mutex_t mutex;
  char *url;
  char *buffer;
  size_t size;
  int finished;
};

static size_t WriteMemoryCallback(void *ptr, size_t size, size_t nmemb, void *context) {
  struct WorkerContext *wc = (struct WorkerContext *)context;
  pthread_mutex_lock(&wc->mutex);
  size_t bytec = size * nmemb;
  wc->buffer = realloc(wc->buffer, wc->size + bytec + 1);
  if(wc->buffer == NULL) {
    fprintf(stderr, "not enough memory (realloc returned NULL)\n");
    pthread_mutex_unlock(&wc->mutex);
    return 0;
  }
  memcpy(&(wc->buffer[wc->size]), ptr, bytec);
  wc->size += bytec;
  wc->buffer[wc->size] = 0;
  pthread_mutex_unlock(&wc->mutex);
  return nmemb;
}

static void *WorkerThread(void *context) {
  struct WorkerContext *wc = (struct WorkerContext *)context;
  CURL *curl = curl_easy_init();
  curl_easy_setopt(curl, CURLOPT_URL, wc->url);
  curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
  curl_easy_setopt(curl, CURLOPT_WRITEDATA, wc);
  curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
  curl_easy_perform(curl); /* ignores error */
  curl_easy_cleanup(curl);
  wc->finished = 1;
  return NULL;
}

#ifdef USE_OPENSSL
static void pthread_mutex_openssl(int mode, int type, const char *file, int line) {
  static pthread_mutex_t *locks = NULL;
  if(locks == NULL) {
    int i;
    locks = OPENSSL_malloc(CRYPTO_num_locks() * sizeof(pthread_mutex_t));
    for(i = 0; i < CRYPTO_num_locks(); i++)
      pthread_mutex_init(&(locks[i]), NULL);
  }

  if(mode & CRYPTO_LOCK)
    pthread_mutex_lock(&(locks[type]));
  else
    pthread_mutex_unlock(&(locks[type]));
}
#endif

int main(int argc, char **argv) {

  /* Must initialize libcurl before any threads are started */
  curl_global_init(CURL_GLOBAL_ALL);

  #ifdef USE_OPENSSL
  /* If using OpenSSL, need to set up a couple things for pthreads... */
  /* (NSS, GnuTLS, and SecureTransport do not require extra setup for pthreads) */
  CRYPTO_set_id_callback(pthread_self);
  CRYPTO_set_locking_callback(pthread_mutex_openssl);
  #endif

  /* initialize the WorkerContect structure */
  pthread_t tid;
  struct WorkerContext wc;
  pthread_mutex_init(&(wc.mutex), NULL);
  wc.url = "https://www.ameriwether.com/cgi-bin/nph-comet.pl";
  wc.buffer = malloc(1);
  wc.size = 0;
  wc.buffer[wc.size] = 0;
  wc.finished = 0;

  /* start the worker thread */
  int error = pthread_create(&tid, NULL, WorkerThread, &wc);
  if(0 != error)
    fprintf(stderr, "Couldn't run worker thread, errno %d\n", error);
  else
    fprintf(stderr, "Worker thread started to fetch %s\n", wc.url);
  pthread_detach(tid);

  /* check the buffer contents every so often, and retrieve any completed lines */
  do {
    char *n;

    /* look for a newline in the buffer */
    pthread_mutex_lock(&wc.mutex);
    if((n = strchr(wc.buffer, '\n')) != NULL) {

      /* the buffer has a newline in it, so copy everything up to that */
      size_t linesize = n - wc.buffer + 1;
      char *line = malloc(linesize + 1);
      if(line == NULL) {
        fprintf(stderr, "not enough memory (malloc returned NULL)\n");
        return 1;
      }
      memcpy(line, wc.buffer, linesize);
      line[linesize] = '\0';

      /* move the remainder of the buffer to the origin, and realloc */
      wc.size = wc.size - linesize;
      memmove(wc.buffer, n + 1, wc.size + 1);
      wc.buffer = realloc(wc.buffer, wc.size + 1);
      if(wc.buffer == NULL) {
        fprintf(stderr, "not enough memory (realloc returned NULL)\n");
        return 1;
      }

      /* do something with the line that was extracted from the buffer */
      printf("Line: %s", line);
      free(line);

    }
    pthread_mutex_unlock(&wc.mutex);

    /* if the buffer didn't have a newline in it yet, pause for a little while before checking again */
    if(n == NULL) usleep(100000);

  } while(!wc.finished);

  return 0;
}

Here is the "https://www.ameriwether.com/cgi-bin/nph-comet.pl" test URL source:
https://gist.github.com/p120ph37/c2fb59ead8983cdbd130