LZ3 decompressor for pokemon crystal, faster but larger and i plan on rewriting parts

decompress.asm

MACRO ld_a_dei
	ld a, [de]
	inc de
ENDM

MACRO ld_hli_dei
	ld a, [de]
	ld [hli], a
	inc de
ENDM

MACRO ld_hli_ded
	ld a, [de]
	ld [hli], a
	dec de
ENDM

Decompress::
; Pokemon GSC uses an lz variant (lz3) for compression.
; This is mainly (but not necessarily) used for graphics.

; This function decompresses lz-compressed data from hl to de.

DEF LZ_END EQU $ff ; Compressed data is terminated with $ff.

; A typical control command consists of:

DEF LZ_CMD EQU %11100000 ; command id (bits 5-7)
DEF LZ_LEN EQU %00011111 ; length n   (bits 0-4)

; Additional parameters are read during command execution.

; Commands:

DEF LZ_LITERAL   EQU 0 << 5 ; Read literal data for n bytes.
DEF LZ_ITERATE   EQU 1 << 5 ; Write the same byte for n bytes.
DEF LZ_ALTERNATE EQU 2 << 5 ; Alternate two bytes for n bytes.
DEF LZ_ZERO      EQU 3 << 5 ; Write 0 for n bytes.

; Another class of commands reuses data from the decompressed output.
DEF LZ_RW        EQU 2 + 5 ; bit

; These commands take a signed offset to start copying from.
; Wraparound is simulated.
; Positive offsets (15-bit) are added to the start address.
; Negative offsets (7-bit) are subtracted from the current position.

DEF LZ_REPEAT    EQU 4 << 5 ; Repeat n bytes from the offset.
DEF LZ_FLIP      EQU 5 << 5 ; Repeat n bitflipped bytes.
DEF LZ_REVERSE   EQU 6 << 5 ; Repeat n bytes in reverse.

; If the value in the count needs to be larger than 5 bits,
; LZ_LONG can be used to expand the count to 10 bits.
DEF LZ_LONG      EQU 7 << 5

; A new control command is read in bits 2-4.
; The top two bits of the length are bits 0-1.
; Another byte is read containing the bottom 8 bits.
DEF LZ_LONG_CMD EQU %00011100
DEF LZ_LONG_HI  EQU %00000011

; In other words, the structure of the command becomes
; 111xxxyy yyyyyyyy
; x: the new control command
; y: the length

	; swap hl and de
	push de
	ld d, h
	ld e, l
	pop hl

	; Save the output address
	; for rewrite commands.
	push hl
	call .Main
	; pop output address
	add sp, 2

	; swap back
	push de
	ld d, h
	ld e, l
	pop hl
	ret

.lzdata
	srl b
	rr c
	jr nc, :+
	ld_hli_dei

:	srl b
	rr c
	jr .lzdata_2_x

.lzzero
	xor a
.fill
	srl b
	rr c
	jr nc, :+
	ld [hli], a

:	srl b
	rr c
	jr nc, :+
	ld [hli], a
	ld [hli], a

:	ld [hli], a
	jr z, .Main
:	ld [hli], a
	ld [hli], a
	ld [hli], a
	ld [hli], a
	dec c
	jr nz, :-
	jr .Main

.lzdata_short
	srl c
	jr nc, .lzdata_2
	ld_hli_dei

.lzdata_2
	srl c
.lzdata_2_x
	jr nc, .lzdata_start
	ld_hli_dei
	ld_hli_dei

.lzdata_start
	ld_hli_dei
	jr z, .Main
:	ld_hli_dei
	ld_hli_dei
	ld_hli_dei
	ld_hli_dei
	dec c
	jr nz, :-
	; Fallthrough

; Where the magic starts
.Main
	ld_a_dei
	ld c, a
	cp LZ_LEN + 1
	; For lzdata no more work is needed
	jr c, .lzdata_short
	cp LZ_LONG
	jr nc, .long

	ld b, a
	; Mask length bits for c
	and LZ_LEN
	ld c, a
	; Get the command bits back
	xor b
	ld b, a

	; lzdata has already been handled
	bit LZ_RW, a
	jr nz, .rewrite

	cp LZ_ALTERNATE
	jr z, .lzalt
	jr nc, .lzzero

;.lziterate
	ld_a_dei
	jr .fill

.long
	inc a
	ret z ; LZ_END
	dec a
	ld b, c

	; low length byte
	ld_a_dei
	ld c, a

	ld a, b
	and LZ_LONG_CMD
	jr z, .lzdata
	bit LZ_RW - 3, a
	jr nz, .rewrite

	cp LZ_ALTERNATE >> 3
	jr z, .lzalt
	jr nc, .lzzero

;.lziterate
	ld_a_dei
	jr .fill

.lzalt
	; Mask the command bits from b
	xor b
	rrca
	ld b, a
	rr c
	inc b
	inc c

	ld_a_dei
	push de
	; Use output to stash d
	ld [hl], a
	ld a, [de]
	ld d, [hl]
	ld e, a

; An lzalt of length 1 is valid, and nonsense, but wont break this
	jr nc, .odd

.even_loop
	ld a, d
	ld [hli], a
	ld a, e
	ld [hli], a
	dec c
	jr nz, .even_loop
	dec b
	jr nz, .even_loop
	pop de
	inc de
	jr .Main

.odd_loop
	ld a, e
	ld [hli], a
.odd	ld a, d
	ld [hli], a
	dec c
	jr nz, .odd_loop
	dec b
	jr nz, .odd_loop
	pop de
	inc de
	jr .Main

; this is here so it can reach .Main with a jr
.flip
	inc c
	; Mask b leaving the length bits in a
	xor b
	inc a

:	push af
:	ld_a_dei

	ld b, a
	rlca
	rlca
	xor b
	and $aa
	xor b
	ld b, a
	swap b
	xor b
	and $33
	xor b
	rrca

	ld [hli], a
	dec c
	jr nz, :-

	pop af
	dec a
	jr nz, :--

	pop de
	jp .Main

.rewrite
	ld_a_dei
	bit 7, a
	jr nz, .relative

	; 15 bit absolute
	ld [hl], a
	ld_a_dei
	push de
	push hl
	ld e, a
	ld d, [hl]

	; lzaddr from the stack
	ld hl, sp+6
	ld a, [hli]
	ld h, [hl]
	ld l, a

	jr .got_offset

.relative
	push de
	push hl
	res 7, a
	cpl
	ld e, a
	ld d, $ff

.got_offset
	add hl, de
	ld d, h
	ld e, l
	pop hl

	ld a, b
	and LZ_CMD
	cp LZ_FLIP
	jr z, .flip
	jr nc, .reverse

.repeat
	srl b
	rr c
	jr nc, :+
	ld_hli_dei

:	srl b
	rr c
	jr nc, :+
	ld_hli_dei
	ld_hli_dei

:	ld_hli_dei
	jr z, .repeat_done
:	ld_hli_dei
	ld_hli_dei
	ld_hli_dei
	ld_hli_dei

	dec c
	jr nz, :-

.repeat_done
	pop de
	jp .Main

.reverse
	srl b
	rr c
	jr nc, :+
	ld_hli_ded

:	srl b
	rr c
	jr nc, :+
	ld_hli_ded
	ld_hli_ded

:	ld_hli_ded
	jr z, .reverse_done
:	ld_hli_ded
	ld_hli_ded
	ld_hli_ded
	ld_hli_ded
	dec c
	jr nz, :-

.reverse_done
	pop de
	jp .Main