mbedtls pthread server + client for reproducing multi-thread error

ssl_pthread_server.c

/*
 *  SSL server demonstration program using pthread for handling multiple
 *  clients.
 *
 *  Copyright The Mbed TLS Contributors
 *  SPDX-License-Identifier: Apache-2.0
 *
 *  Licensed under the Apache License, Version 2.0 (the "License"); you may
 *  not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 *  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 */

#include "mbedtls/build_info.h"

#include "mbedtls/platform.h"
#include "mbedtls/debug.h"

#if !defined(MBEDTLS_BIGNUM_C)  || !defined(MBEDTLS_ENTROPY_C) ||         \
    !defined(MBEDTLS_SSL_TLS_C) || !defined(MBEDTLS_SSL_SRV_C) ||         \
    !defined(MBEDTLS_NET_C) || !defined(MBEDTLS_RSA_C) ||                 \
    !defined(MBEDTLS_CTR_DRBG_C) || !defined(MBEDTLS_X509_CRT_PARSE_C) || \
    !defined(MBEDTLS_FS_IO) || !defined(MBEDTLS_THREADING_C) ||           \
    !defined(MBEDTLS_THREADING_PTHREAD) || !defined(MBEDTLS_PEM_PARSE_C)
int main(void)
{
    mbedtls_printf("MBEDTLS_BIGNUM_C and/or MBEDTLS_ENTROPY_C "
                   "and/or MBEDTLS_SSL_TLS_C and/or MBEDTLS_SSL_SRV_C and/or "
                   "MBEDTLS_NET_C and/or MBEDTLS_RSA_C and/or "
                   "MBEDTLS_CTR_DRBG_C and/or MBEDTLS_X509_CRT_PARSE_C and/or "
                   "MBEDTLS_THREADING_C and/or MBEDTLS_THREADING_PTHREAD "
                   "and/or MBEDTLS_PEM_PARSE_C not defined.\n");
    mbedtls_exit(0);
}
#else

#include <stdlib.h>
#include <string.h>

#if defined(_WIN32)
#include <windows.h>
#endif

#if !defined(_WIN32)
#include <unistd.h>
#endif

#include "mbedtls/entropy.h"
#include "mbedtls/ctr_drbg.h"
#include "mbedtls/x509.h"
#include "mbedtls/ssl.h"
#include "mbedtls/net_sockets.h"
#include "mbedtls/error.h"
#include "test/certs.h"

#if defined(MBEDTLS_SSL_CACHE_C)
#include "mbedtls/ssl_cache.h"
#endif

#if defined(MBEDTLS_MEMORY_BUFFER_ALLOC_C)
#include "mbedtls/memory_buffer_alloc.h"
#endif


#define HTTP_RESPONSE \
    "HTTP/1.0 200 OK\r\nContent-Type: text/html\r\n\r\n" \
    "<h2>mbed TLS Test Server</h2>\r\n" \
    "<p>Successful connection using: %s</p>\r\n"

#define SERVER_PORT "4433"
#define SERVER_NAME "localhost"
#define GET_REQUEST "GET / HTTP/1.0\r\n\r\n"

#define DEBUG_LEVEL 0

#define SERVER_MAX_NUM_THREADS 5
#define CLIENT_MAX_NUM_THREADS 5

mbedtls_threading_mutex_t debug_mutex;

static void server_mutexed_debug(void *ctx, int level,
                             const char *file, int line,
                             const char *str)
{
    long int thread_id = (long int) pthread_self();

    mbedtls_mutex_lock(&debug_mutex);

    ((void) level);
    mbedtls_fprintf((FILE *) ctx, "[server] %s:%04d: [ #%ld ] %s",
                    file, line, thread_id, str);
    fflush((FILE *) ctx);

    mbedtls_mutex_unlock(&debug_mutex);
}

static void client_mutexed_debug(void *ctx, int level,
                             const char *file, int line,
                             const char *str)
{
    long int thread_id = (long int) pthread_self();

    mbedtls_mutex_lock(&debug_mutex);

    ((void) level);
    mbedtls_fprintf((FILE *) ctx, "[client] %s:%04d: [ #%ld ] %s",
                    file, line, thread_id, str);
    fflush((FILE *) ctx);

    mbedtls_mutex_unlock(&debug_mutex);
}

typedef struct {
    mbedtls_net_context client_fd;
    int thread_complete;
    const mbedtls_ssl_config *config;
} thread_info_t;

typedef struct {
    int active;
    thread_info_t   data;
    pthread_t       thread;
} pthread_info_t;

static thread_info_t    base_info;
static pthread_info_t   server_threads[SERVER_MAX_NUM_THREADS];

static void *server_handle_ssl_connection(void *data)
{
    int ret, len;
    thread_info_t *thread_info = (thread_info_t *) data;
    mbedtls_net_context *client_fd = &thread_info->client_fd;
    long int thread_id = (long int) pthread_self();
    unsigned char buf[1024];
    mbedtls_ssl_context ssl;

    /* Make sure memory references are valid */
    mbedtls_ssl_init(&ssl);

    mbedtls_printf("[server]  [ #%ld ]  Setting up SSL/TLS data\n", thread_id);

    /*
     * 4. Get the SSL context ready
     */
    if ((ret = mbedtls_ssl_setup(&ssl, thread_info->config)) != 0) {
        mbedtls_printf("[server]  [ #%ld ]  failed: mbedtls_ssl_setup returned -0x%04x\n",
                       thread_id, (unsigned int) -ret);
        goto thread_exit;
    }

    mbedtls_ssl_set_bio(&ssl, client_fd, mbedtls_net_send, mbedtls_net_recv, NULL);

    /*
     * 5. Handshake
     */
    mbedtls_printf("[server]  [ #%ld ]  Performing the SSL/TLS handshake\n", thread_id);

    while ((ret = mbedtls_ssl_handshake(&ssl)) != 0) {
        if (ret != MBEDTLS_ERR_SSL_WANT_READ && ret != MBEDTLS_ERR_SSL_WANT_WRITE) {
            mbedtls_printf("[server]  [ #%ld ]  failed: mbedtls_ssl_handshake returned -0x%04x\n",
                           thread_id, (unsigned int) -ret);
            goto thread_exit;
        }
    }

    mbedtls_printf("  ok\n");

    /*
     * 6. Read the HTTP Request
     */
    mbedtls_printf("[server]  [ #%ld ]  < Read from client\n", thread_id);

    do {
        len = sizeof(buf) - 1;
        memset(buf, 0, sizeof(buf));
        ret = mbedtls_ssl_read(&ssl, buf, len);

        if (ret == MBEDTLS_ERR_SSL_WANT_READ || ret == MBEDTLS_ERR_SSL_WANT_WRITE) {
            continue;
        }

        if (ret <= 0) {
            switch (ret) {
                case MBEDTLS_ERR_SSL_PEER_CLOSE_NOTIFY:
                    mbedtls_printf("[server]  [ #%ld ]  connection was closed gracefully\n",
                                   thread_id);
                    goto thread_exit;

                case MBEDTLS_ERR_NET_CONN_RESET:
                    mbedtls_printf("[server]  [ #%ld ]  connection was reset by peer\n",
                                   thread_id);
                    goto thread_exit;

                default:
                    mbedtls_printf("[server]  [ #%ld ]  mbedtls_ssl_read returned -0x%04x\n",
                                   thread_id, (unsigned int) -ret);
                    goto thread_exit;
            }
        }

        len = ret;
        // mbedtls_printf("[server]  [ #%ld ]  %d bytes read\n=====\n%s\n=====\n",
        //                thread_id, len, (char *) buf);

        if (ret > 0) {
            break;
        }
    } while (1);

    /*
     * 7. Write the 200 Response
     */
    mbedtls_printf("[server]  [ #%ld ]  > Write to client:\n", thread_id);

    len = sprintf((char *) buf, HTTP_RESPONSE,
                  mbedtls_ssl_get_ciphersuite(&ssl));

    while ((ret = mbedtls_ssl_write(&ssl, buf, len)) <= 0) {
        if (ret == MBEDTLS_ERR_NET_CONN_RESET) {
            mbedtls_printf("[server]  [ #%ld ]  failed: peer closed the connection\n",
                           thread_id);
            goto thread_exit;
        }

        if (ret != MBEDTLS_ERR_SSL_WANT_READ && ret != MBEDTLS_ERR_SSL_WANT_WRITE) {
            mbedtls_printf("[server]  [ #%ld ]  failed: mbedtls_ssl_write returned -0x%04x\n",
                           thread_id, (unsigned int) ret);
            goto thread_exit;
        }
    }

    len = ret;
    // mbedtls_printf("[server]  [ #%ld ]  %d bytes written\n=====\n%s\n=====\n",
    //                thread_id, len, (char *) buf);

    mbedtls_printf("[server]  [ #%ld ]  . Closing the connection...", thread_id);

    while ((ret = mbedtls_ssl_close_notify(&ssl)) < 0) {
        if (ret != MBEDTLS_ERR_SSL_WANT_READ &&
            ret != MBEDTLS_ERR_SSL_WANT_WRITE) {
            mbedtls_printf("[server]  [ #%ld ]  failed: mbedtls_ssl_close_notify returned -0x%04x\n",
                           thread_id, (unsigned int) ret);
            goto thread_exit;
        }
    }

    mbedtls_printf(" ok\n");

    ret = 0;

thread_exit:

#ifdef MBEDTLS_ERROR_C
    if (ret != 0) {
        char error_buf[100];
        mbedtls_strerror(ret, error_buf, 100);
        mbedtls_printf("[server]  [ #%ld ]  Last error was: -0x%04x - %s\n\n",
                       thread_id, (unsigned int) -ret, error_buf);
    }
#endif

    mbedtls_net_free(client_fd);
    mbedtls_ssl_free(&ssl);

    thread_info->thread_complete = 1;

    return NULL;
}

static int server_thread_create(mbedtls_net_context *client_fd)
{
    int ret, i;

    /*
     * Find in-active or finished thread slot
     */
    for (i = 0; i < SERVER_MAX_NUM_THREADS; i++) {
        if (server_threads[i].active == 0) {
            break;
        }

        if (server_threads[i].data.thread_complete == 1) {
            mbedtls_printf("[server]    Cleaning up thread %d\n", i);
            pthread_join(server_threads[i].thread, NULL);
            memset(&server_threads[i], 0, sizeof(pthread_info_t));
            break;
        }
    }

    if (i == SERVER_MAX_NUM_THREADS) {
        return -1;
    }

    /*
     * Fill thread-info for thread
     */
    memcpy(&server_threads[i].data, &base_info, sizeof(base_info));
    server_threads[i].active = 1;
    memcpy(&server_threads[i].data.client_fd, client_fd, sizeof(mbedtls_net_context));

    if ((ret = pthread_create(&server_threads[i].thread, NULL, server_handle_ssl_connection,
                              &server_threads[i].data)) != 0) {
        return ret;
    }

    return 0;
}

void *server(void *arg)
{
    (void)arg;
    int ret;
    mbedtls_net_context listen_fd, client_fd;
    const char pers[] = "ssl_pthread_server";

    mbedtls_entropy_context entropy;
    mbedtls_ctr_drbg_context ctr_drbg;
    mbedtls_ssl_config conf;
    mbedtls_x509_crt srvcert;
    mbedtls_x509_crt cachain;
    mbedtls_pk_context pkey;
#if defined(MBEDTLS_MEMORY_BUFFER_ALLOC_C)
    unsigned char alloc_buf[100000];
#endif
#if defined(MBEDTLS_SSL_CACHE_C)
    mbedtls_ssl_cache_context cache;
#endif

#if defined(MBEDTLS_MEMORY_BUFFER_ALLOC_C)
    mbedtls_memory_buffer_alloc_init(alloc_buf, sizeof(alloc_buf));
#endif

#if defined(MBEDTLS_SSL_CACHE_C)
    mbedtls_ssl_cache_init(&cache);
#endif

    mbedtls_x509_crt_init(&srvcert);
    mbedtls_x509_crt_init(&cachain);

    mbedtls_ssl_config_init(&conf);
    mbedtls_ctr_drbg_init(&ctr_drbg);
    memset(server_threads, 0, sizeof(server_threads));
    mbedtls_net_init(&listen_fd);
    mbedtls_net_init(&client_fd);

    mbedtls_mutex_init(&debug_mutex);

    base_info.config = &conf;

    /*
     * We use only a single entropy source that is used in all the threads.
     */
    mbedtls_entropy_init(&entropy);

    /*
     * 1a. Seed the random number generator
     */
    mbedtls_printf("[server]  . Seeding the random number generator...");

    if ((ret = mbedtls_ctr_drbg_seed(&ctr_drbg, mbedtls_entropy_func, &entropy,
                                     (const unsigned char *) pers,
                                     strlen(pers))) != 0) {
        mbedtls_printf(" failed: mbedtls_ctr_drbg_seed returned -0x%04x\n",
                       (unsigned int) -ret);
        goto server_exit;
    }

    mbedtls_printf(" ok\n");

    /*
     * 1b. Load the certificates and private RSA key
     */
    mbedtls_printf("\n[server]  . Loading the server cert. and key...");
    fflush(stdout);

    /*
     * This demonstration program uses embedded test certificates.
     * Instead, you may want to use mbedtls_x509_crt_parse_file() to read the
     * server and CA certificates, as well as mbedtls_pk_parse_keyfile().
     */
    ret = mbedtls_x509_crt_parse(&srvcert, (const unsigned char *) mbedtls_test_srv_crt,
                                 mbedtls_test_srv_crt_len);
    if (ret != 0) {
        mbedtls_printf(" failed\n[server]  !  mbedtls_x509_crt_parse returned %d\n\n", ret);
        goto server_exit;
    }

    ret = mbedtls_x509_crt_parse(&cachain, (const unsigned char *) mbedtls_test_cas_pem,
                                 mbedtls_test_cas_pem_len);
    if (ret != 0) {
        mbedtls_printf(" failed\n[server]  !  mbedtls_x509_crt_parse returned %d\n\n", ret);
        goto server_exit;
    }

    mbedtls_pk_init(&pkey);
    ret =  mbedtls_pk_parse_key(&pkey, (const unsigned char *) mbedtls_test_srv_key,
                                mbedtls_test_srv_key_len, NULL, 0,
                                mbedtls_ctr_drbg_random, &ctr_drbg);
    if (ret != 0) {
        mbedtls_printf(" failed\n[server]  !  mbedtls_pk_parse_key returned %d\n\n", ret);
        goto server_exit;
    }

    mbedtls_printf(" ok\n");

    /*
     * 1c. Prepare SSL configuration
     */
    mbedtls_printf("[server]  . Setting up the SSL data....");

    if ((ret = mbedtls_ssl_config_defaults(&conf,
                                           MBEDTLS_SSL_IS_SERVER,
                                           MBEDTLS_SSL_TRANSPORT_STREAM,
                                           MBEDTLS_SSL_PRESET_DEFAULT)) != 0) {
        mbedtls_printf("[server] failed: mbedtls_ssl_config_defaults returned -0x%04x\n",
                       (unsigned int) -ret);
        goto server_exit;
    }
#if defined(MBEDTLS_SSL_PROTO_TLS1_3)
    mbedtls_ssl_conf_max_tls_version(&conf, MBEDTLS_SSL_VERSION_TLS1_3);
    mbedtls_ssl_conf_min_tls_version(&conf, MBEDTLS_SSL_VERSION_TLS1_3);
#endif  /* MBEDTLS_SSL_PROTO_TLS1_3 */
    mbedtls_ssl_conf_rng(&conf, mbedtls_ctr_drbg_random, &ctr_drbg);
    mbedtls_ssl_conf_dbg(&conf, server_mutexed_debug, stdout);

    /* mbedtls_ssl_cache_get() and mbedtls_ssl_cache_set() are thread-safe if
     * MBEDTLS_THREADING_C is set.
     */
#if defined(MBEDTLS_SSL_CACHE_C)
    mbedtls_ssl_conf_session_cache(&conf, &cache,
                                   mbedtls_ssl_cache_get,
                                   mbedtls_ssl_cache_set);
#endif

    mbedtls_ssl_conf_ca_chain(&conf, &cachain, NULL);
    if ((ret = mbedtls_ssl_conf_own_cert(&conf, &srvcert, &pkey)) != 0) {
        mbedtls_printf(" failed\n[server]  ! mbedtls_ssl_conf_own_cert returned %d\n\n", ret);
        goto server_exit;
    }

    mbedtls_printf("[server] ok\n");

    /*
     * 2. Setup the listening TCP socket
     */
    mbedtls_printf("[server]  . Bind on https://localhost:4433/ ...");
    fflush(stdout);

    if ((ret = mbedtls_net_bind(&listen_fd, NULL, "4433", MBEDTLS_NET_PROTO_TCP)) != 0) {
        mbedtls_printf(" failed\n[server]  ! mbedtls_net_bind returned %d\n\n", ret);
        goto server_exit;
    }

    mbedtls_printf("[server] ok\n");

reset:
#ifdef MBEDTLS_ERROR_C
    if (ret != 0) {
        char error_buf[100];
        mbedtls_strerror(ret, error_buf, 100);
        mbedtls_printf("[server]    Last error was: -0x%04x - %s\n", (unsigned int) -ret,
                       error_buf);
    }
#endif

    /*
     * 3. Wait until a client connects
     */
    mbedtls_printf("[server]    Waiting for a remote connection\n");

    if ((ret = mbedtls_net_accept(&listen_fd, &client_fd,
                                  NULL, 0, NULL)) != 0) {
        mbedtls_printf("[server]   failed: mbedtls_net_accept returned -0x%04x\n",
                       (unsigned int) ret);
        goto server_exit;
    }

    mbedtls_printf("[server]    ok\n");
    mbedtls_printf("[server]    Creating a new thread\n");

    if ((ret = server_thread_create(&client_fd)) != 0) {
        mbedtls_printf("[server]    failed: server_thread_create returned %d\n", ret);
        mbedtls_net_free(&client_fd);
        goto reset;
    }

    ret = 0;
    goto reset;

server_exit:
    mbedtls_x509_crt_free(&srvcert);
    mbedtls_pk_free(&pkey);
#if defined(MBEDTLS_SSL_CACHE_C)
    mbedtls_ssl_cache_free(&cache);
#endif
    mbedtls_ctr_drbg_free(&ctr_drbg);
    mbedtls_entropy_free(&entropy);
    mbedtls_ssl_config_free(&conf);
    mbedtls_net_free(&listen_fd);
    mbedtls_mutex_free(&debug_mutex);
    size_t p_ret = ret;
    return (void *)p_ret;
}

void *client(void *arg)
{
    (void)arg;
    long int thread_id = (long int) pthread_self();
    int ret = 1, len;
    int exit_code = MBEDTLS_EXIT_FAILURE;
    mbedtls_net_context server_fd;
    uint32_t flags;
    unsigned char buf[1024];
    const char *pers = "ssl_client1";

    mbedtls_entropy_context entropy;
    mbedtls_ctr_drbg_context ctr_drbg;
    mbedtls_ssl_context ssl;
    mbedtls_ssl_config conf;
    mbedtls_x509_crt cacert;

    /*
     * 0. Initialize the RNG and the session data
     */
    mbedtls_net_init(&server_fd);
    mbedtls_ssl_init(&ssl);
    mbedtls_ssl_config_init(&conf);
    mbedtls_x509_crt_init(&cacert);
    mbedtls_ctr_drbg_init(&ctr_drbg);
    mbedtls_entropy_init(&entropy);

#if defined(MBEDTLS_USE_PSA_CRYPTO)
    psa_status_t status = psa_crypto_init();
    if (status != PSA_SUCCESS) {
        mbedtls_fprintf(stderr, "Failed to initialize PSA Crypto implementation: %d\n",
                        (int) status);
        goto client_exit;
    }
#endif /* MBEDTLS_USE_PSA_CRYPTO */

    mbedtls_printf("\n[client]  [ #%ld ]  . Seeding the random number generator...", thread_id);
    fflush(stdout);


    if ((ret = mbedtls_ctr_drbg_seed(&ctr_drbg, mbedtls_entropy_func, &entropy,
                                     (const unsigned char *) pers,
                                     strlen(pers))) != 0) {
        mbedtls_printf(" failed\n[client]  [ #%ld ]  ! mbedtls_ctr_drbg_seed returned %d\n", thread_id, ret);
        goto client_exit;
    }

    mbedtls_printf(" ok\n");

    /*
     * 0. Initialize certificates
     */
    mbedtls_printf("[client]  [ #%ld ]  . Loading the CA root certificate ...", thread_id);
    fflush(stdout);

    ret = mbedtls_x509_crt_parse(&cacert, (const unsigned char *) mbedtls_test_cas_pem,
                                 mbedtls_test_cas_pem_len);
    if (ret < 0) {
        mbedtls_printf(" failed\n[client]  [ #%ld ]  !  mbedtls_x509_crt_parse returned -0x%x\n\n",
                       thread_id, (unsigned int) -ret);
        goto client_exit;
    }

    mbedtls_printf(" ok (%d skipped)\n", ret);

    /*
     * 1. Start the connection
     */
    mbedtls_printf("[client]  [ #%ld ]  . Connecting to tcp/%s/%s...", thread_id ,SERVER_NAME, SERVER_PORT);
    fflush(stdout);

    if ((ret = mbedtls_net_connect(&server_fd, SERVER_NAME,
                                   SERVER_PORT, MBEDTLS_NET_PROTO_TCP)) != 0) {
        mbedtls_printf(" failed\n[client]  [ #%ld ]  ! mbedtls_net_connect returned %d\n\n", thread_id ,ret);
        goto client_exit;
    }

    mbedtls_printf(" ok\n");

    /*
     * 2. Setup stuff
     */
    mbedtls_printf("[client]  [ #%ld ]  . Setting up the SSL/TLS structure...", thread_id);
    fflush(stdout);

    if ((ret = mbedtls_ssl_config_defaults(&conf,
                                           MBEDTLS_SSL_IS_CLIENT,
                                           MBEDTLS_SSL_TRANSPORT_STREAM,
                                           MBEDTLS_SSL_PRESET_DEFAULT)) != 0) {
        mbedtls_printf(" failed\n[client]  [ #%ld ]  ! mbedtls_ssl_config_defaults returned %d\n\n", thread_id, ret);
        goto client_exit;
    }

    mbedtls_printf(" ok\n");

    /* OPTIONAL is not optimal for security,
     * but makes interop easier in this simplified example */
    mbedtls_ssl_conf_authmode(&conf, MBEDTLS_SSL_VERIFY_OPTIONAL);
    mbedtls_ssl_conf_ca_chain(&conf, &cacert, NULL);
    mbedtls_ssl_conf_rng(&conf, mbedtls_ctr_drbg_random, &ctr_drbg);
    mbedtls_ssl_conf_dbg(&conf, client_mutexed_debug, stdout);
#if defined(MBEDTLS_SSL_PROTO_TLS1_3)
    mbedtls_ssl_conf_max_tls_version(&conf, MBEDTLS_SSL_VERSION_TLS1_3);
#endif  /* MBEDTLS_SSL_PROTO_TLS1_3 */

    if ((ret = mbedtls_ssl_setup(&ssl, &conf)) != 0) {
        mbedtls_printf(" failed\n[client]  [ #%ld ]  ! mbedtls_ssl_setup returned %d\n\n", thread_id, ret);
        goto client_exit;
    }

    if ((ret = mbedtls_ssl_set_hostname(&ssl, SERVER_NAME)) != 0) {
        mbedtls_printf(" failed\n[client]  [ #%ld ]  ! mbedtls_ssl_set_hostname returned %d\n\n", thread_id, ret);
        goto client_exit;
    }

    mbedtls_ssl_set_bio(&ssl, &server_fd, mbedtls_net_send, mbedtls_net_recv, NULL);

    /*
     * 4. Handshake
     */
    mbedtls_printf("[client]  [ #%ld ]  . Performing the SSL/TLS handshake...", thread_id);
    fflush(stdout);

    while ((ret = mbedtls_ssl_handshake(&ssl)) != 0) {
        if (ret != MBEDTLS_ERR_SSL_WANT_READ && ret != MBEDTLS_ERR_SSL_WANT_WRITE) {
            mbedtls_printf(" failed\n[client]  [ #%ld ]  ! mbedtls_ssl_handshake returned -0x%x\n\n", thread_id,
                           (unsigned int) -ret);
            goto client_exit;
        }
    }

    mbedtls_printf("[client]  [ #%ld ] ok\n", thread_id);

    /*
     * 5. Verify the server certificate
     */
    mbedtls_printf("[client]  [ #%ld ]  . Verifying peer X.509 certificate...", thread_id);

    /* In real life, we probably want to bail out when ret != 0 */
    if ((flags = mbedtls_ssl_get_verify_result(&ssl)) != 0) {
#if !defined(MBEDTLS_X509_REMOVE_INFO)
        char vrfy_buf[512];
#endif

        mbedtls_printf(" failed\n");

#if !defined(MBEDTLS_X509_REMOVE_INFO)
        mbedtls_x509_crt_verify_info(vrfy_buf, sizeof(vrfy_buf), "  ! ", flags);

        mbedtls_printf("[client]  [ #%ld ]%s\n", thread_id, vrfy_buf);
#endif
    } else {
        mbedtls_printf("[client]  [ #%ld ] ok\n", thread_id);
    }

    /*
     * 3. Write the GET request
     */
    mbedtls_printf("[client]  [ #%ld ]  > Write to server:", thread_id);
    fflush(stdout);

    len = sprintf((char *) buf, GET_REQUEST);

    while ((ret = mbedtls_ssl_write(&ssl, buf, len)) <= 0) {
        if (ret != MBEDTLS_ERR_SSL_WANT_READ && ret != MBEDTLS_ERR_SSL_WANT_WRITE) {
            mbedtls_printf(" failed\n[client]  [ #%ld ]  ! mbedtls_ssl_write returned %d\n\n", thread_id, ret);
            goto client_exit;
        }
    }

    len = ret;
    // mbedtls_printf("[client]  [ #%ld ] %d bytes written\n\n%s", thread_id, len, (char *) buf);

    /*
     * 7. Read the HTTP response
     */
    mbedtls_printf("[client]  [ #%ld ]  < Read from server:", thread_id);
    fflush(stdout);

    do {
        len = sizeof(buf) - 1;
        memset(buf, 0, sizeof(buf));
        ret = mbedtls_ssl_read(&ssl, buf, len);

        if (ret == MBEDTLS_ERR_SSL_WANT_READ || ret == MBEDTLS_ERR_SSL_WANT_WRITE) {
            continue;
        }

        if (ret == MBEDTLS_ERR_SSL_PEER_CLOSE_NOTIFY) {
            break;
        }

        if (ret < 0) {
            mbedtls_printf("failed\n[client]  [ #%ld ]  ! mbedtls_ssl_read returned %d\n\n", thread_id, ret);
            break;
        }

        if (ret == 0) {
            mbedtls_printf("\n\n[client]  [ #%ld ]EOF\n\n", thread_id);
            break;
        }

        len = ret;
        // mbedtls_printf("[client]  [ #%ld ] %d bytes read\n\n%s", thread_id, len, (char *) buf);
    } while (1);

    mbedtls_ssl_close_notify(&ssl);

    exit_code = MBEDTLS_EXIT_SUCCESS;

client_exit:

#ifdef MBEDTLS_ERROR_C
    if (exit_code != MBEDTLS_EXIT_SUCCESS) {
        char error_buf[100];
        mbedtls_strerror(ret, error_buf, 100);
        mbedtls_printf("[client]  [ #%ld ]Last error was: %d - %s\n\n", thread_id, ret, error_buf);
    }
#endif

    mbedtls_net_free(&server_fd);
    mbedtls_x509_crt_free(&cacert);
    mbedtls_ssl_free(&ssl);
    mbedtls_ssl_config_free(&conf);
    mbedtls_ctr_drbg_free(&ctr_drbg);
    mbedtls_entropy_free(&entropy);

    size_t p_ret = ret;
    return (void *)p_ret;
}


int main(void)
{
    int ret = 0;
#if defined(MBEDTLS_DEBUG_C)
    mbedtls_debug_set_threshold(DEBUG_LEVEL);
#endif
#if defined(MBEDTLS_USE_PSA_CRYPTO) || defined(MBEDTLS_SSL_PROTO_TLS1_3)
    psa_status_t status;
    status = psa_crypto_init();
    if (status != PSA_SUCCESS) {
        mbedtls_printf("[main] Failed to initialize PSA Crypto implementation: %d\n",
                        (int) status);
        ret = MBEDTLS_ERR_SSL_HW_ACCEL_FAILED;
        goto main_exit;
    }
#endif  /* MBEDTLS_USE_PSA_CRYPTO || MBEDTLS_SSL_PROTO_TLS1_3 */


    pthread_t server_main_thread;
    pthread_t client_threads[CLIENT_MAX_NUM_THREADS];
    memset(client_threads, 0, sizeof(client_threads));

    // Create the thread with pthread_create
    ret = pthread_create(&server_main_thread, NULL, server, NULL);
    if (ret != 0) {
        perror("[main] pthread_create server main thread");
        return 1;
    }
    // sleep 100ms to wait server up
    mbedtls_net_usleep(100000);

    for (int i = 0; i < CLIENT_MAX_NUM_THREADS; i++) {
        if ((ret = pthread_create(&client_threads[i], NULL, client, NULL)) != 0) {
            mbedtls_printf("[main] Failed to pthread_create PSA client thread %d: %d\n", i, ret);
            perror("[main] pthread_create client thread");
            return ret;
        }
    }

    for (int i = 0; i < CLIENT_MAX_NUM_THREADS; i++) {
        if ((ret = pthread_join(client_threads[i], NULL)) != 0) {
            mbedtls_printf("[main] Failed to pthread_join PSA client thread %d: %d\n", i, ret);
            perror("[main] pthread_join client thread");
            return ret;
        }
    }


main_exit:
#if defined(MBEDTLS_MEMORY_BUFFER_ALLOC_C)
    mbedtls_memory_buffer_alloc_free();
#endif
#if defined(MBEDTLS_USE_PSA_CRYPTO)
    mbedtls_psa_crypto_free();
#endif /* MBEDTLS_USE_PSA_CRYPTO */

    return ret;
}

#endif /* MBEDTLS_BIGNUM_C && MBEDTLS_ENTROPY_C &&
          MBEDTLS_SSL_TLS_C && MBEDTLS_SSL_SRV_C && MBEDTLS_NET_C &&
          MBEDTLS_RSA_C && MBEDTLS_CTR_DRBG_C && MBEDTLS_THREADING_C &&
          MBEDTLS_THREADING_PTHREAD && MBEDTLS_PEM_PARSE_C */