Fade out and in for GG and SMS (GBDK2020)

fadeinout.c

#if defined(MASTERSYSTEM)
#define COL_DEPTH_BITS 2
#else
#define COL_DEPTH_BITS 4
#endif

#define COL_PER_PALETTE 16

#define R_MASK ((1<<COL_DEPTH_BITS)-1)
#define G_MASK (R_MASK << COL_DEPTH_BITS)
#define B_MASK (G_MASK << COL_DEPTH_BITS)


#define R_ADD 1
#define G_ADD (R_ADD << COL_DEPTH_BITS)
#define B_ADD (G_ADD << COL_DEPTH_BITS)


palette_color_t fade_palettes_cache[(1 << COL_DEPTH_BITS)][COL_PER_PALETTE];

void calculate_fade_palettes(palette_color_t fade_palettes[(1 << COL_DEPTH_BITS)][COL_PER_PALETTE], palette_color_t to[COL_PER_PALETTE]){
    uint8_t i, j;
    for(i=0; i < COL_PER_PALETTE; i++){
        fade_palettes[0][i] = 0;
    }
    palette_color_t color, colorto;
    for(i = 1; i < (1 << COL_DEPTH_BITS); i++) {
        for(j = 0; j < COL_PER_PALETTE; j++) {
                color = fade_palettes[i-1][j];
                colorto = to[j];
                if((color & B_MASK) != (colorto & B_MASK)) color += B_ADD;
                if((color & G_MASK) != (colorto & G_MASK)) color += G_ADD;
                if((color & R_MASK) != (colorto & R_MASK)) color += R_ADD;
                fade_palettes[i][j] = color;
        }
    }
}

// wait for line 216  - if you change palette before there there would be artifacts on frame
void wait_for_safe_pallete_change(void)
{
__asm
	push af
1$:
	in a, (0x7e)
	sub a, #216
	jp C, 1$
	pop af
__endasm;
}

void wait_frames(uint8_t frames) {
    for(uint8_t i = 0; i < frames; i++) {
        wait_vbl_done();
    }
}

void fadein(palette_color_t fade_palettes[(1 << COL_DEPTH_BITS)][COL_PER_PALETTE]) {
    for(uint8_t i = 0; i < (1 << COL_DEPTH_BITS); i++) {
        wait_for_safe_pallete_change();
        set_palette(0, 1, fade_palettes[i]);
        set_palette(1, 1, fade_palettes[i]); //sprites color on game gear it also change mode 0 colors
        wait_frames(32 / (1 << COL_DEPTH_BITS));
    }
}
 
void fadeout(palette_color_t fade_palettes[(1 << COL_DEPTH_BITS)][COL_PER_PALETTE]) {
    for(uint8_t i = (1 << COL_DEPTH_BITS); i > 0; i--) {
        wait_for_safe_pallete_change();
        set_palette(0, 1, fade_palettes[i-1]);
        set_palette(1, 1, fade_palettes[i-1]); //sprites color on game gear it also change mode 0 colors
        wait_frames(32 / (1 << COL_DEPTH_BITS));
    }
}


////////usage/////////////
// for palette
palette_color_t bigmap_palettes[COL_PER_PALETTE];
// define cache
palette_color_t fade_palettes_cache[(1 << COL_DEPTH_BITS)][COL_PER_PALETTE];

//then calculate fade step pallets

calculate_fade_palettes(fade_palettes_cache, bigmap_palettes);

//fade in
fadein(fade_palettes_cache);

//fade out
fadeout(fade_palettes_cache);

v2.c

#if defined(MASTERSYSTEM)
#define COL_DEPTH_BITS 2
#else
#define COL_DEPTH_BITS 4
#endif

#define COL_PER_PALETTE 16

#define R_MASK ((1<<COL_DEPTH_BITS)-1)
#define G_MASK (R_MASK << COL_DEPTH_BITS)
#define B_MASK (G_MASK << COL_DEPTH_BITS)


#define R_ADD 1
#define G_ADD (R_ADD << COL_DEPTH_BITS)
#define B_ADD (G_ADD << COL_DEPTH_BITS)

// wait for line 216  - if you change palette before there there would be artifacts on frame
void wait_for_safe_palette_change(void)
{
__asm
	push af
1$:
	in a, (0x7e)
	sub a, #216
	jp C, 1$
	pop af
__endasm;
}

void wait_frames(uint8_t frames) {
    for(uint8_t i = 0; i < frames; i++) {
        vsync();
    }
}

//https://discord.com/channels/790342889318252555/791053087958892564/1212435850219225098
//    V2 is 16, 15, 14, 13, 12, 11, 10,  9,  8,  7,  6,  5,  4,  3,  2,  1,  0,  0,  0,  0,  0,  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
//not V2 is 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
#define V2

void fadein(palette_color_t *orginal_palette) {
    palette_color_t fade_palettes_cache[COL_PER_PALETTE] = {0};
    palette_color_t color, colorto;
    uint8_t i, j;
    for(i = (1 << COL_DEPTH_BITS) - 1; i != 255 ; i--) {
        for(j = 0; j < COL_PER_PALETTE; j++) {
                color = fade_palettes_cache[j];
                colorto = orginal_palette[j];
#ifdef V2                
                if((colorto & B_MASK) > (i << (2*COL_DEPTH_BITS))) color += B_ADD;
                if((colorto & G_MASK) > (i << COL_DEPTH_BITS)) color += G_ADD;
                if((colorto & R_MASK) > i) color += R_ADD;
#else
                if((color & B_MASK) != (colorto & B_MASK)) color += B_ADD;
                if((color & G_MASK) != (colorto & G_MASK)) color += G_ADD;
                if((color & R_MASK) != (colorto & R_MASK)) color += R_ADD;
#endif
                fade_palettes_cache[j] = color;
        }
        vsync();
        wait_for_safe_palette_change();
        set_palette(0, 1, fade_palettes_cache);
        set_palette(1, 1, fade_palettes_cache); //sprites color on game gear it also change mode 0 colors
        wait_frames(32 / (1 << COL_DEPTH_BITS));
    }
}
 
void fadeout(palette_color_t *orginal_palette) {
    palette_color_t fade_palettes_cache[COL_PER_PALETTE];    
    palette_color_t color;
    uint8_t i, j;
    for (i = 0; i < COL_PER_PALETTE; i++)
    {
        fade_palettes_cache[i] = orginal_palette[i];
    }
    for(i = (1 << COL_DEPTH_BITS) - 1; i != 255 ; i--) {
        for(j = 0; j < COL_PER_PALETTE; j++) {
                color = fade_palettes_cache[j];
#ifdef V2
                if((color & B_MASK) > 0) color -= B_ADD;
                if((color & G_MASK) > 0) color -= G_ADD;
                if((color & R_MASK) > 0) color -= R_ADD;
#else
                if((color & B_MASK) > ((i << (2*COL_DEPTH_BITS)))) color -= B_ADD;
                if((color & G_MASK) > ((i << COL_DEPTH_BITS))) color -= G_ADD;
                if((color & R_MASK) > (i)) color -= R_ADD;
#endif
                fade_palettes_cache[j] = color;
        }
        vsync();
        wait_for_safe_palette_change();
        set_palette(0, 1, fade_palettes_cache);
        set_palette(1, 1, fade_palettes_cache); //sprites color on game gear it also change mode 0 colors
        wait_frames(32 / (1 << COL_DEPTH_BITS));
    }
}