cross-platform / cross-compiler standalone endianness conversion

endianness.h

/** 
* @file   endianness.h
* @brief  Convert Endianness of shorts, longs, long longs, regardless of architecture/OS
*
* Defines (without pulling in platform-specific network include headers):
* bswap16, bswap32, bswap64, ntoh16, hton16, ntoh32 hton32, ntoh64, hton64
*
* Should support linux / macos / solaris / windows.
* Supports GCC (on any platform, including embedded), MSVC2015, and clang,
* and should support intel, solaris, and ibm compilers as well.
*
* Copyright 2020 github user jtbr, Released under MIT license
*
* SPDX-License-Identifier: MIT OR Apache-2.0
*/

#ifndef ENDIANNESS_H_
#define ENDIANNESS_H_

#include <stdlib.h>
#include <stdint.h>
#ifdef __cplusplus
#include <cstring> // for memcpy
#endif

/* Detect platform endianness at compile time */

// If boost were available on all platforms, could use this instead to detect endianness
// #include <boost/predef/endian.h>

// When available, these headers can improve platform endianness detection
#ifdef __has_include // C++17, supported as extension to C++11 in clang, GCC 5+, vs2015
#  if __has_include(<endian.h>)
#    include <endian.h> // gnu libc normally provides, linux
#  elif __has_include(<machine/endian.h>)
#    include <machine/endian.h> //open bsd, macos
#  elif __has_include(<sys/param.h>)
#    include <sys/param.h> // mingw, some bsd (not open/macos)
#  elif __has_include(<sys/isadefs.h>)
#    include <sys/isadefs.h> // solaris
#  endif
#endif

#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__)
#  if (defined(__BYTE_ORDER__)  && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) || \
     (defined(__BYTE_ORDER) && __BYTE_ORDER == __BIG_ENDIAN) || \
	 (defined(_BYTE_ORDER) && _BYTE_ORDER == _BIG_ENDIAN) || \
	 (defined(BYTE_ORDER) && BYTE_ORDER == BIG_ENDIAN) || \
     (defined(__sun) && defined(__SVR4) && defined(_BIG_ENDIAN)) || \
     defined(__ARMEB__) || defined(__THUMBEB__) || defined(__AARCH64EB__) || \
     defined(_MIBSEB) || defined(__MIBSEB) || defined(__MIBSEB__) || \
     defined(_M_PPC)
#    define __BIG_ENDIAN__
#  elif (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) || /* gcc */\
     (defined(__BYTE_ORDER) && __BYTE_ORDER == __LITTLE_ENDIAN) /* linux header */ || \
	 (defined(_BYTE_ORDER) && _BYTE_ORDER == _LITTLE_ENDIAN) || \
	 (defined(BYTE_ORDER) && BYTE_ORDER == LITTLE_ENDIAN) /* mingw header */ ||  \
     (defined(__sun) && defined(__SVR4) && defined(_LITTLE_ENDIAN)) || /* solaris */ \
     defined(__ARMEL__) || defined(__THUMBEL__) || defined(__AARCH64EL__) || \
     defined(_MIPSEL) || defined(__MIPSEL) || defined(__MIPSEL__) || \
     defined(_M_IX86) || defined(_M_X64) || defined(_M_IA64) || /* msvc for intel processors */ \
     defined(_M_ARM) /* msvc code on arm executes in little endian mode */
#    define __LITTLE_ENDIAN__
#  endif
#endif

#if !defined(__LITTLE_ENDIAN__) & !defined(__BIG_ENDIAN__)
#  error "UNKNOWN Platform / endianness. Configure endianness checks for this platform or set explicitly."
#endif

#if defined(bswap16) || defined(bswap32) || defined(bswap64) || defined(bswapf) || defined(bswapd)
#  error "unexpected define!" // freebsd may define these; probably just need to undefine them
#endif

/* Define byte-swap functions, using fast processor-native built-ins where possible */
#if defined(_MSC_VER) // needs to be first because msvc doesn't short-circuit after failing defined(__has_builtin)
#  define bswap16(x)     _byteswap_ushort((x))
#  define bswap32(x)     _byteswap_ulong((x))
#  define bswap64(x)     _byteswap_uint64((x))
#elif (__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)
#  define bswap16(x)     __builtin_bswap16((x))
#  define bswap32(x)     __builtin_bswap32((x))
#  define bswap64(x)     __builtin_bswap64((x))
#elif defined(__has_builtin) && __has_builtin(__builtin_bswap64)  /* for clang; gcc 5 fails on this and && shortcircuit fails; must be after GCC check */
#  define bswap16(x)     __builtin_bswap16((x))
#  define bswap32(x)     __builtin_bswap32((x))
#  define bswap64(x)     __builtin_bswap64((x))
#else
    /* even in this case, compilers often optimize by using native instructions */
    static inline uint16_t bswap16(uint16_t x) {
		return ((( x  >> 8 ) & 0xffu ) | (( x  & 0xffu ) << 8 ));
	}
    static inline uint32_t bswap32(uint32_t x) {
        return ((( x & 0xff000000u ) >> 24 ) |
                (( x & 0x00ff0000u ) >> 8  ) |
                (( x & 0x0000ff00u ) << 8  ) |
                (( x & 0x000000ffu ) << 24 ));
    }
    static inline uint64_t bswap64(uint64_t x) {
        return ((( x & 0xff00000000000000ull ) >> 56 ) |
                (( x & 0x00ff000000000000ull ) >> 40 ) |
                (( x & 0x0000ff0000000000ull ) >> 24 ) |
                (( x & 0x000000ff00000000ull ) >> 8  ) |
                (( x & 0x00000000ff000000ull ) << 8  ) |
                (( x & 0x0000000000ff0000ull ) << 24 ) |
                (( x & 0x000000000000ff00ull ) << 40 ) |
                (( x & 0x00000000000000ffull ) << 56 ));
    }
#endif

//! Byte-swap 32-bit float
static inline float bswapf(float f) {
#ifdef __cplusplus
    static_assert(sizeof(float) == sizeof(uint32_t), "Unexpected float format");
    /* Problem: de-referencing float pointer as uint32_t breaks strict-aliasing rules for C++ and C, even if it normally works
     *   uint32_t val = bswap32(*(reinterpret_cast<const uint32_t *>(&f)));
     *   return *(reinterpret_cast<float *>(&val));
     */
    // memcpy approach is guaranteed to work in C & C++ and fn calls should be optimized out:
    uint32_t asInt;
    std::memcpy(&asInt, reinterpret_cast<const void *>(&f), sizeof(uint32_t));
    asInt = bswap32(asInt);
    std::memcpy(&f, reinterpret_cast<void *>(&asInt), sizeof(float));
    return f;
#else
    _Static_assert(sizeof(float) == sizeof(uint32_t), "Unexpected float format");
    // union approach is guaranteed to work in C99 and later (but not in C++, though in practice it normally will):
    union { uint32_t asInt; float asFloat; } conversion_union;
    conversion_union.asFloat = f;
    conversion_union.asInt = bswap32(conversion_union.asInt);
    return conversion_union.asFloat;
#endif
}

//! Byte-swap 64-bit double
static inline double bswapd(double d) {
#ifdef __cplusplus
    static_assert(sizeof(double) == sizeof(uint64_t), "Unexpected double format");
    uint64_t asInt;
    std::memcpy(&asInt, reinterpret_cast<const void *>(&d), sizeof(uint64_t));
    asInt = bswap64(asInt);
    std::memcpy(&d, reinterpret_cast<void *>(&asInt), sizeof(double));
    return d;
#else
    _Static_assert(sizeof(double) == sizeof(uint64_t), "Unexpected double format");
    union { uint64_t asInt; double asDouble; } conversion_union;
    conversion_union.asDouble = d;
    conversion_union.asInt = bswap64(conversion_union.asInt);
    return conversion_union.asDouble;
#endif
}


/* Define network - host byte swaps as needed depending upon platform endianness */
// (note that network order is big endian)

#if defined(__LITTLE_ENDIAN__)
#  define ntoh16(x)     bswap16((x))
#  define hton16(x)     bswap16((x))
#  define ntoh32(x)     bswap32((x))
#  define hton32(x)     bswap32((x))
#  define ntoh64(x)     bswap64((x))
#  define hton64(x)     bswap64((x))
#  define ntohf(x)      bswapf((x))
#  define htonf(x)      bswapf((x))
#  define ntohd(x)      bswapd((x))
#  define htond(x)      bswapd((x))
#elif defined(__BIG_ENDIAN__)
#  define ntoh16(x)     (x)
#  define hton16(x)     (x)
#  define ntoh32(x)     (x)
#  define hton32(x)     (x)
#  define ntoh64(x)     (x)
#  define hton64(x)     (x)
#  define ntohf(x)      (x)
#  define htonf(x)      (x)
#  define ntohd(x)      (x)
#  define htond(x)      (x)
#  else
#    warning "UNKNOWN Platform / endianness; network / host byte swaps not defined."
#endif

#endif //ENDIANNESS_H_

Like your endianess.h, but had an issue with the __has_builtin

/endianness.h:65:47: error: missing binary operator before token "("
 #elif (defined(__has_builtin) && __has_builtin(bswap16)) || /* for clang or future gcc */ \

Found this solution: https://gitlab.gnome.org/GNOME/glib/commit/e2bd6a6a8f2b9f22e7092578f920e2b3057f4156 and implemented in https://gist.github.com/raidoz/4163b8ec6672aabb0656b96692af5e33

It also seems endianness detection fails, because at least in my env, something from __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ || __BYTE_ORDER == __BIG_ENDIAN || _BYTE_ORDER == _BIG_ENDIAN || BYTE_ORDER == BIG_ENDIAN will always match - both elements in at least one of the cases are not defined and the == is evaluated to true. Fixed it by first checking if the respective BYTE_ORDER variant has been defined.

@raidoz Thanks for your feedback. I'm sorry that I had made revisions to fix these issues but somehow the edits didn't get committed to this gist. Your fix for the defined(__has_builtin) issue is better, but I avoided it by simply checking for GCC 4 or greater first (as GCC seems to be the compiler that exhibits this error in my testing). And you're also right about the cause of the byte order defines check failing.

These changes are in now. Would be curious to know if they work for your platform (and what that is).

I'll point out that this breaks on Windows fairly easily because Winsock2.h defines the host to network / network to host functions as well with conflicting return / parameter types (i.e. __inline unsigned __int32 htonf ( float Value ) vs static inline float htonf(float f). To work around this, I'll probably name the actual function _htonf and then #define htonf(x) _htonf((x)) since the Windows header checks for an htonf preprocessor definition.

The latest version avoids potential issues from breaking strict-aliasing rules, in C and C++.
@borrrden I updated it to make htonf/ntohf/htond/ntohd be macros, and I've tested while including winsock2. Thanks.
It also avoids unnecessarily calling a function for htonf on big endian machines as it did before (even if it did nothing, as it still does).

@jtbr, thank you for a thorough investigation of macros defined by various compilers for differently-endianed targets! Our project suffers from incomplete (basically, lacking and used ad-hoc) compile-time endiannes detection; also, your bswapNN() implementations are indeed magically compile into a single instruction by every modern compiler I checked at Godbolt's! I would like to pull a good 1/3 if not 1/2 of the file. But here's the rub: we're using Apache 2.0 throughout. Would you not please object to dual-licensing this snippet, to simplify the ingestion? IANAL, and it's hard for me to figure out how to properly do that legally when licenses mix. Indeed, you will be credited as a contributor and a joint copyright owner: that's our common and rigorously followed policy. In acadaemia a failure to attribute is looked at as if it were a felony...

I'm linking to our historic byte swapping bogosity; a few lines above is a runtime endiannes check that we only fully trust, generally, lacking a compile-time one. And it cannot be turned into a constexpr function, unfortunately. These have long been on the list of things to get rid of and to add, respectively. :)

In case you aren't aware, SPDX formal expressions are coming into fashion, and are even on track for ISO standardisation. Although not used in Kaldi, a conservative project with a long history, here's a usage example in another Kaldi-related project. Basically, the "disjunctive OR" refers to a choice of any of the SPDX-listed license IDs, as in this example, which the user can select from, so all it takes is putting a line into the file header comment block (hint, hint! :)) like:

SPDX-License-Identifier: MIT OR Apache-2.0

Cheers!

@kkm000, no problem: done. I’m glad to hear you’ll be able to use it in your project :)

@jtbr, thanks so much!!!