A simple O(M)-space and O(N+M)-time algorithm for generating M unique pseudorandom numbers in the range [0, N) in C.

uniq_rand.c

#include <stdint.h>
#include <stdlib.h>

uint64_t *uniq_rand(uint64_t m, uint64_t n) {
  uint64_t *vs, im, in, rm, rn, rim, tmp;

  vs = malloc(m * sizeof(uint64_t));

  // Knuth algorithm
  for (im = 0, in = 0; im < m && in < n; in++) {
    rm = m - im;
    rn = n - in;
    if (rand() % rn < rm) {
      vs[im] = in;
      im++;
    }
  }

  // Fisher-Yates algorithm
  for (im = 0; im < m; im++) {
    rim = rand() % m;
    tmp = vs[im];
    vs[im] = vs[rim];
    vs[rim] = tmp;
  }

  return vs;
}