LibreSSL BN_nist_mod_384 undefined behavior

libressl.sh

#!/bin/bash

set -e

export CFLAGS="-fsanitize=object-size -fno-sanitize-recover=object-size"
export CC=clang

git clone --depth 1 https://github.com/libressl-portable/portable libressl
cd libressl
./update.sh
mkdir build/
cd build/
cmake -DCMAKE_C_COMPILER=$CC -DCMAKE_C_FLAGS="$CFLAGS" -DENABLE_ASM=OFF ..
make crypto -j$(nproc)

cat <<EOT >> poc.c
#include <openssl/bn.h>
#include <openssl/ec.h>
#include <openssl/obj_mac.h>

#define CF_CHECK_EQ(expr, res) if ( (expr) != (res) ) { goto end; }
#define CF_CHECK_NE(expr, res) if ( (expr) == (res) ) { goto end; }

int main(void)
{
    EC_KEY* key = NULL;
    BIGNUM* prv = NULL;
    BIGNUM* pub_x = NULL;
    BIGNUM* pub_y = NULL;
    EC_POINT* pub = NULL;
    EC_GROUP* group = NULL;
    char* pub_x_str = NULL;
    char* pub_y_str = NULL;
    BN_CTX* ctx = BN_CTX_new();

    CF_CHECK_NE(key = EC_KEY_new(), NULL);
    CF_CHECK_NE(prv = BN_new(), NULL);
    CF_CHECK_NE(pub_x = BN_new(), NULL);
    CF_CHECK_NE(pub_y = BN_new(), NULL);

    CF_CHECK_NE(BN_dec2bn(&prv, "5222222"), 0);
    CF_CHECK_NE(group = EC_GROUP_new_by_curve_name(NID_secp384r1), NULL);
    CF_CHECK_EQ(EC_KEY_set_group(key, group), 1);
    CF_CHECK_EQ(EC_KEY_set_private_key(key, prv), 1);
    CF_CHECK_NE(pub = EC_POINT_new(group), NULL);
    CF_CHECK_EQ(EC_POINT_mul(group, pub, prv, NULL, NULL, NULL), 1);
    printf("%s\n", EC_POINT_point2hex(group, pub,
POINT_CONVERSION_UNCOMPRESSED, ctx));
end:
    return 0;
}
EOT

$CC $CFLAGS -I ../include/ poc.c crypto/libcrypto.a -lpthread

# Run it a few times and observe the output difference
./a.out
./a.out
./a.out
./a.out
./a.out
./a.out
./a.out
./a.out