Generate localized SNMPv3 keys from password and engine id

generate_localized_key.c

// Needs to be linked with BSD message digest library libmd (-lmd)
#include <stdio.h>
#include <string.h>
#include <ctype.h>
#include <sys/types.h>
#include <sysexits.h>
#include <sha1.h>
#include <md5.h>

typedef unsigned char u_char;
typedef unsigned int u_int;
typedef unsigned long u_long;

// From https://datatracker.ietf.org/doc/html/rfc2574#appendix-A.2.1
void password_to_key_md5(
   u_char *password,    /* IN */
   u_int   passwordlen, /* IN */
   u_char *engineID,    /* IN  - pointer to snmpEngineID  */
   u_int   engineLength,/* IN  - length of snmpEngineID */
   u_char *key)         /* OUT - pointer to caller 16-octet buffer */
{
   MD5_CTX     MD;
   u_char     *cp, password_buf[64];
   u_long      password_index = 0;
   u_long      count = 0, i;

   MD5Init (&MD);   /* initialize MD5 */

   /**********************************************/
   /* Use while loop until we've done 1 Megabyte */
   /**********************************************/
   while (count < 1048576) {
      cp = password_buf;
      for (i = 0; i < 64; i++) {
          /*************************************************/
          /* Take the next octet of the password, wrapping */
          /* to the beginning of the password as necessary.*/
          /*************************************************/
          *cp++ = password[password_index++ % passwordlen];
      }
      MD5Update (&MD, password_buf, 64);
      count += 64;
   }
   MD5Final (key, &MD);          /* tell MD5 we're done */

   /*****************************************************/
   /* Now localize the key with the engineID and pass   */
   /* through MD5 to produce final key                  */
   /* May want to ensure that engineLength <= 32,       */
   /* otherwise need to use a buffer larger than 64     */
   /*****************************************************/
   memcpy(password_buf, key, 16);
   memcpy(password_buf+16, engineID, engineLength);
   memcpy(password_buf+16+engineLength, key, 16);

   MD5Init(&MD);
   MD5Update(&MD, password_buf, 32+engineLength);
   MD5Final(key, &MD);
   return;
}

// https://datatracker.ietf.org/doc/html/rfc2574#appendix-A.2.2
void password_to_key_sha1(
   u_char *password,    /* IN */
   u_int   passwordlen, /* IN */
   u_char *engineID,    /* IN  - pointer to snmpEngineID  */
   u_int   engineLength,/* IN  - length of snmpEngineID */
   u_char *key)         /* OUT - pointer to caller 20-octet buffer */
{
   SHA1_CTX     SH;
   u_char     *cp, password_buf[72];
   u_long      password_index = 0;
   u_long      count = 0, i;

   SHA1Init (&SH);   /* initialize SHA */

   /**********************************************/
   /* Use while loop until we've done 1 Megabyte */
   /**********************************************/
   while (count < 1048576) {
      cp = password_buf;
      for (i = 0; i < 64; i++) {
          /*************************************************/
          /* Take the next octet of the password, wrapping */
          /* to the beginning of the password as necessary.*/
          /*************************************************/
          *cp++ = password[password_index++ % passwordlen];
      }
      SHA1Update (&SH, password_buf, 64);
      count += 64;
   }
   SHA1Final (key, &SH);          /* tell SHA we're done */

   /*****************************************************/
   /* Now localize the key with the engineID and pass   */
   /* through SHA to produce final key                  */
   /* May want to ensure that engineLength <= 32,       */
   /* otherwise need to use a buffer larger than 72     */
   /*****************************************************/
   memcpy(password_buf, key, 20);
   memcpy(password_buf+20, engineID, engineLength);
   memcpy(password_buf+20+engineLength, key, 20);

   SHA1Init(&SH);
   SHA1Update(&SH, password_buf, 40+engineLength);
   SHA1Final(key, &SH);
   return;
}

static int decode_hex_char(const unsigned char hex) {
    if (hex >= '0' && hex <= '9') {
        return hex - '0';
    } else if (hex >= 'A' && hex <= 'F') {
        return hex - 'A' + 10;
    } else if (hex >= 'a' && hex <= 'f') {
        return hex - 'a' + 10;
    } else {
        return -1;
    }
}

static ssize_t decode_hex(const char *hex_string, unsigned char *out)
{
    ssize_t j = 0;
    for (int i = 0; hex_string[i];) {
        // Delimiters
        if (isspace(hex_string[i]) || strchr(":-.", hex_string[i])) {
            i++;
            continue;
        }

        const int b1 = decode_hex_char(hex_string[i++]);
        const int b2 = decode_hex_char(hex_string[i++]);

        if (b1 < 0 || b2 < 0) {
            return -1;
        }

        out[j++] = b1 << 4 | b2;
    }

    return j;
}

void print_hex(const unsigned char *bytes, size_t length) {
    for (int i = 0; i < length; i++) {
        printf("%02hhx", bytes[i]);
    }
    puts("\n");
}

int main(int argc, const char *argv[]) {
    if (argc != 3) {
        fputs("Usage: key ENGINE-ID PASSWORD\n", stderr);
        return EX_USAGE;
    }
    u_char engineId[strlen(argv[1]) / 2];
    ssize_t engineIdLength = decode_hex(argv[1], engineId);
    if (engineIdLength < 0) {
        fprintf(stderr, "Not a valid hex string: %s\n", argv[1]);
    }
    const char *password = argv[2];

    // Second round according to
    // https://datatracker.ietf.org/doc/html/draft-reeder-snmpv3-usm-3desede-00#section-2.1

    u_char sha1_key[20];
    puts("SHA1 (1st round): ");
    password_to_key_sha1((u_char *)password, strlen(password), engineId, engineIdLength, sha1_key);
    print_hex(sha1_key, sizeof(sha1_key));
    puts("SHA1 (2nd round): ");
    password_to_key_sha1(sha1_key, sizeof(sha1_key), engineId, engineIdLength, sha1_key);
    print_hex(sha1_key, sizeof(sha1_key));
    u_char md5_key[16];
    puts("MD5 (1st round): ");
    password_to_key_md5((u_char *)password, strlen(password), engineId, engineIdLength, md5_key);
    print_hex(md5_key, sizeof(md5_key));
    puts("MD5 (2nd round): ");
    password_to_key_md5(md5_key, sizeof(md5_key), engineId, engineIdLength, md5_key);
    print_hex(md5_key, sizeof(md5_key));

    return 0;
}