Arduino sketch to find the memory that was never used (HighWaterMark of the Stack) for a AVR microcontroller.

HighWaterMark.ino

// -----------------------------------------
// HighWaterMark of the stack
// -----------------------------------------
// Calculate the high water mark for a Arduino with AVR microcontroller.
// It is the amount of memory that was never used since the Arduino was started.
//
// The __init() or .init1 up to .init9 was used in the past.
// It was used to run assembly code to fill the unused ram with a pattern.
// With Arduino 1.8.0 that did no longer work.
//
// This sketch provides an alternative.
// A function is used that fills ram with a pattern
// between the top of the heap and the bottom of the stack.
//
//
// First version, December 2016, by Koepel, Public Domain.
//    Tested with Arduino Uno and Arduino IDE 1.8.0
//    Not fully working yet. There is a bug that makes the
//    number of found unused ram a low number (for example 10)
//    when MemoryNotUsed() is called for the first time.
// Version 2.0
//    december 2016, by Koepel, Public Domain.
//    Added to search upward and downward for the pattern.
//    That seems to help.
//    Changed pattern to 4 bytes instead of just 1 byte.
//


// The _end address is the same as the __heap_start address.
// The __stack address is the same as RAMEND;
// The __brkval is zero when the heap is not used yet,
// otherwise it is the top of the heap.
// The SP is the 16-bit register of the stack pointer.

extern int __heap_start;     // The __heap_start is a linker label.
extern int *__brkval;        // The __brkval is a value

const uint32_t memory_pattern = 0xC5AA550F;   // you may choose your own pattern


void setup()
{
  // Fill the unused memory with a pattern.
  // This function could be called before the setup() function, 
  // with a global variable that is filled with the return value 
  // of the function. In that case, function prototyping will be needed.
  int fill_size = MemoryFillPattern();
  
  Serial.begin(9600);
  while (!Serial);   // wait for serial port to connect for ATmega32U4 based boards.

  Serial.println(F( "HighWaterMark test sketch"));
  Serial.print(F( "Filled "));
  Serial.print( fill_size);
  Serial.println(F( " bytes with a pattern"));
}


void loop()
{
  int highwater = MemoryNotUsed();
  
  Serial.print(F( "MemoryNotUsed is "));
  Serial.print( highwater);
  Serial.println(F( " bytes"));

  // Add your own code here.


  delay(5000);
}


// ----------------------------
// MemoryNotUsed()
// ----------------------------
// Search for the pattern in ram between the heap and the stack. 
// Returns the number of bytes that are found, accurate to 4 bytes.
//
// This function might fail.
// Sometimes buffers are declared on the stack or sometimes heap
// is allocated and it is used only partially and released. 
// Then this function thinks that it found the pattern, 
// while the big chunk of unused ram is elsewhere.
// To avoid most problems, the pattern consists of 4 bytes, and the
// memory is searched both upwards and downwards.
// 
int MemoryNotUsed()
{
  byte *p;              // pointer to 1 byte.
  uint32_t *p4;         // pointer to 4 bytes.
  int countUp = 0;
  int countDown = 0;
  byte *pBottom = (byte *) &__heap_start;

  // When the heap is in use, the __brkval is no longer zero,
  // and the value is set to above the used part of the heap.
  if( __brkval != 0)
    pBottom = (byte *) __brkval;

  byte *pTop = (byte *) SP;    // current stack pointer

  // ------------------------
  // Upwards
  // ------------------------
  // Search for the 4 byte pattern, search memory per byte.

  p = pBottom;
  while( *(uint32_t *)p != memory_pattern && p < pTop)
  {
    p++;
  }

  // The pattern is found. 
  // Count for how long the pattern is in the memory.
  // The pointer is increased 4 bytes at a time.
  p4 = (uint32_t *) p;
  while( *p4 == memory_pattern && p4 < pTop)
  {
    p4++;
    countUp += 4;
  }

  // ------------------------
  // Downwards
  // ------------------------
  // Search for the 4 byte pattern, search memory per byte
  
  p = pTop;
  while( *(uint32_t *)p != memory_pattern && p > pBottom)
  {
    p--;
  }

  // The pattern is found. 
  // Count for how long the pattern is in the memory.
  // The pointer is lowered 4 bytes at a time.
  p4 = (uint32_t *) p;
  while( *p4 == memory_pattern && p4 >= pBottom)
  {
    p4--;
    countDown += 4;
  }

  // If both the countUp and countDown are the same,
  // then it is sure that the number is correct.
  // When they differ, they both could be wrong.
  // I think that the 'countDown' is more accurate,
  // since the heap is used in an other way than the stack.
// #define DEBUG_HIGHWATERMARK
#ifdef DEBUG_HIGHWATERMARK
  if( countUp != countDown)
  {
    Serial.println(F( "Warning: HighWaterMark might not be accurate"));
    Serial.println( countUp);
    Serial.println( countDown);
  }
#endif

  return( max( countUp, countDown));
} 


// ----------------------------
// MemoryFillPattern()
// ----------------------------
// Fill memory between heap en stack with a pattern.
// Returns the number of bytes that are filled (accurate to 4 bytes).
//
int MemoryFillPattern()
{
  int count = 0;
  uint32_t *p4 = (uint32_t *) &__heap_start;   // top of normal variables is bottom of heap.

  // When the heap is in use, then the __brkval is no longer zero,
  // and the value is set to above the used part of the heap.
  if( __brkval != 0)
    p4 = (uint32_t *) __brkval;

  byte *pTop = (byte *) SP;  // SP is the 16 bits combination of SPL and SPH registers.
  pTop -= 8;         // For safety, be sure not to overwrite own stack variables.

  while( p4 < pTop)
  {
    *p4++ = memory_pattern;  // first fill 4 bytes, then increase pointer.
    count += 4;              // 4 bytes at a time.
  }

  return( count);
}


// ----------------------------
// DumpRam()
// ----------------------------
// Function used during development to dump the whole ram.
// The actual ram data starts at 0x100 !
//
void DumpRam()
{
  for( int i=0; i<=RAMEND; i++)
  {
    if( i%16 == 0)
    {
      if( i < 16)
        Serial.print( "0");
      if( i < 256)
        Serial.print( "0");
      if( i < 4096)
        Serial.print( "0");
      Serial.print( i, HEX);
      Serial.print( " ");
    }

    // Make 'i' a pointer to memory and get the contents of the that memory.
    byte data = *(byte *)i;
    Serial.print( " ");
    if( data < 16)
      Serial.print( "0");
    Serial.print( data, HEX);
    if( (i + 1) %16 == 0)
      Serial.println();
  }
  Serial.println();
}

In May 2017 I found that something like this already existed. It is called StackPaint and StackCount. I have tested my sketch with an Arduino Uno, Leonardo and Mega 2560 board, and I don't know if the StackPaint code will work on all three.

The StackPaint uses a single byte. I tried that, but it was more reliable with a pattern of 4 bytes.
My sketch scans up and down, because scanning just one way did not always work.

StackPaint mentioned on StackExchange: https://arduino.stackexchange.com/questions/763/im-using-too-much-ram-how-can-this-be-measured
StackPaint at avrfreaks: http://www.avrfreaks.net/forum/soft-c-avrgcc-monitoring-stack-usage?name=PNphpBB2&file=viewtopic&t=52249

There are some warnings, I have to cast p4 to byte * when comparing to a byte *.