further optimized prime counter with reordered code to avoid register stall (actually working :O) and check and compare square root of current number

prime64_M_rs.asm

format PE64 console 3.1

entry @f

include 'win64a.inc'

MAX_ITERATIONS = 100000000

section '' import code data \
			readable writeable executable
	library msvcrt,'MSVCRT.DLL'
	import msvcrt,\
		printf,'printf'

	align 10h
	@@:
		mov		rdi	,	rdi
		inc		rdi
		mov		rsi	,	5
	@:
		xor		rbp	,	rbp
	@@:
		mov		rcx	,	[p + rbp * 8]
		xor		rdx	,	rdx
		mov		rax	,	rsi
		inc		rbp
		div		rcx
		test	rdx	,	rdx
		jz		@f

		cmp		rbp	,	rdi
		jb		@b

		mov		[p + rdi * 8],	rsi
		invoke	printf,	fmt,	rsi
		inc		rdi
	@@:
		add		rsi	,	2
		;cmp		rsi	,	-3
		;jb		@
		jmp		@

		;ret
	fmt	db	'%u',13,0
	
	align	10h
	p	dq	3
		rq	MAX_ITERATIONS

prime64_M_rs_sqrt.asm

; 8 tabs suck, github please store the indent setting for files or something

format PE64 console 3.1

entry @f

include 'win64a.inc'

MAX_ITERATIONS = 100000000

macro alignd boundary,value:? { db (boundary-1)-($+boundary-1) mod boundary dup value }

section '' import code data \
			readable writeable executable
	library msvcrt,'MSVCRT.DLL'
	import msvcrt,\
		printf,'printf'

	alignd 20h
	; rdi = current index at which to store the next possible prime, increments if found
	; rbp = current index of prime numbers to test against rsi
	; rsi = current number to test as prime
	; r8  = upper limit of numbers to check, square root of current number
	; rcx = dividend, any of the prime numbers to test against rdx
	; rdx = remainder after rdx:rax / rcx, if 0 skip current value that rsi is
	@@:
		mov	rdi	,	1			;	j = 1	;	number of primes stored
		mov	rsi	,	5			;	X = 5;
		fld1
	align 10h
	main:
		push	rsi
		fild	qword		[rsp]			;	A = X;
		fsqrt						;	sqrt(A);
		xor	rbp	,	rbp			;	i = 0;
		fld	st
		fistp	qword		[rsp]
		fprem						;	A %= 1.0;
		ftst
		fnstsw	ax
		pop	r8					;	C = floor(X);
		test	ah	,	40h			;	if (A == 0.0)
		jnz	next					;		skip		---.
	align 10h						;				   |
	@@:							;				   |	<--.
		mov	rcx	,	[p + rbp * 8]		;	A = p[i++];		   |	   |
		xor	rdx	,	rdx			;				   |
		mov	rax	,	rsi			;				   |
		inc	rbp					;				   |
								;				   |
		div	rcx					;	B = X % A;		   |	   |
		test	rdx	,	rdx			;	if (B == 0)		   |	   |
		jz	next					;		skip		---|	   |
								;				   |	   |
		cmp	rcx	,	r8			;	if (A < C)		   |	   |
		jb	@b					;		continue	   |	---'
								;				   |
		mov	[p + rdi * 8],	rsi			;	p[j] = X;		   |
		invoke	printf,	fmt,	rsi			;				   |
		inc	rdi					;	j++;			   |
	next:							;				<--'
		add	rsi	,	2			;	X += 2	;	next odd number
		fxch
		ffree	st1
		jmp	main
		;cmp	rsi	,	-3
		;jb	@

		;ret
	fmt	db	'%u',13,0
	
	alignd	10h
	p	dq	3
		rq	MAX_ITERATIONS