factorial.asm

bits 64
global factorial
; uint64_t factorial(uint n [edi],bool *valid [rsi])
factorial:
  mov [rsi], dword 1 ; assume no overflow until proven otherwise
  cmp edi, 1
  jbe .ret1
  
  cmp edi, 21
  jae .overflow ; result will not fit in a uint64
  
  mov eax, edi
  sub edi, 1

.loop:
  mul rdi
  
  sub edi, 1
  jnz .loop
  
  ret ; happy ending
  
.overflow:
  mov [rsi], dword 0 ; rax contains garbage now : mark as invalid
  ret

.ret1:
  mov eax, 1
  ret

factorial.c

#include <assert.h>
#include <limits.h>
#include <stdint.h>

uint64_t factorial(unsigned int, int*);

int main(void)
{
  int valid;
  assert(factorial(0, &valid) == 1 && valid == 1);
  assert(factorial(1, &valid) == 1 && valid == 1);
  assert(factorial(5, &valid) == 120 && valid == 1);
  assert(factorial(15, &valid) == 1307674368000 && valid == 1);
  
  factorial(UINT_MAX, &valid);
  assert(valid == 0);
  
  return 0;
}