too-much-optimization.c

too-much-optimization.c

// gcc -c -O2 too-much-optimization.c
//
// objdump -d too-much-optimization.o
//
// the overflow check in gdk_pixbuf_new_reduced function disappears under Fedora 24 GCC 6.1.1
//
// http://c-faq.com/misc/intovf.html
// http://c-faq.com/misc/sd26.html

#include <stdio.h>
#include <stdlib.h>
#include <limits.h>

void *gdk_pixbuf_new_reduced(int has_alpha, int bits_per_sample, int width, int height)
{
        void *buf = NULL;
        int channels;
        int rowstride;

        if (width < 0)
                return NULL;
        if (height < 0)
                return 0;

        channels = has_alpha ? 4 : 3;
        rowstride = width * channels;
        if (rowstride / channels != width) /* overflow check - gets eliminated! */
                return NULL;

        buf = calloc(height, rowstride);
        if (!buf)
                return NULL;

        return buf;
}