Reproducing an apparent memory leak under JRuby FFI

ffi_leak.rb

#
#  this gist reproduces an apparent memory leak when passing callbacks
#  to an FFI attached function. the result shown here is that
#  FooStruct, a FFI managed struct, is not being released reliably.
#
#  to witness this first-hand, run the script with the following
#  constant set to _true_, and again set to _false_.
#
#  this was reproduced on jruby 1.3.1.
#
#  note that you'll have to run "make" to build the test shared library.
#
#  also please note that this code does not leak under ruby-ffi (MRI).
#

YES_I_WOULD_LIKE_JRUBY_FFI_TO_LEAK_SOME_MEMORY_PLEASE = true

# originally reported in http://github.com/tenderlove/nokogiri/issues#issue/103

require 'rubygems'
require 'ffi'

STATUS_EVERY = 1000
STDOUT.sync = true

if YES_I_WOULD_LIKE_JRUBY_FFI_TO_LEAK_SOME_MEMORY_PLEASE
  puts "=> You've asked for memory to be leaked under JRuby."
else
  puts "=> We're not going to leak memory. Watch."
end

def create_an_object
  if YES_I_WOULD_LIKE_JRUBY_FFI_TO_LEAK_SOME_MEMORY_PLEASE
    FooLib.testClosure(Proc.new {}) # this line causes a memory leak
  else
    FooLib.testClosure(nil)         # this line does not cause a memory leak
  end
  FooStruct.new(FooLib.malloc(100000))
end

def create_lots_of_objects
  n_allocated = 0
  while n_allocated += 1 do
    create_an_object
    puts "allocated #{n_allocated} objects" if (n_allocated % STATUS_EVERY) == 0
  end
end

module FooLib
  extend FFI::Library
  ffi_lib 'foo'
  callback        :test_closure, [],              :void
  attach_function :malloc,       [:int],          :pointer
  attach_function :free,         [:pointer],      :void
  attach_function :testClosure,  [:test_closure], :void
end

class FooStruct < FFI::ManagedStruct # :nodoc:
  layout :dummy, :int
  @n_released = 0

  def self.release ptr
    FooLib.free(ptr)
    @n_released += 1
    puts "freed     #{@n_released} objects" if (@n_released % STATUS_EVERY) == 0
  end
end

create_lots_of_objects

foo.c

void testClosure(void (*closure)(void))
{
}

jruby-leak.log

=> You've asked for memory to be leaked under JRuby.
allocated 1000 objects
freed     1000 objects
allocated 2000 objects
allocated 3000 objects
freed     2000 objects
allocated 4000 objects
allocated 5000 objects
freed     3000 objects
freed     4000 objects
allocated 6000 objects
freed     5000 objects
allocated 7000 objects
freed     6000 objects
allocated 8000 objects
allocated 9000 objects
allocated 10000 objects
allocated 11000 objects
allocated 12000 objects
freed     7000 objects
freed     8000 objects
freed     9000 objects
allocated 13000 objects
freed     10000 objects
freed     11000 objects
freed     12000 objects
allocated 14000 objects
allocated 15000 objects
allocated 16000 objects
allocated 17000 objects
allocated 18000 objects
allocated 19000 objects
allocated 20000 objects
allocated 21000 objects
freed     13000 objects
freed     14000 objects
freed     15000 objects
freed     16000 objects
freed     17000 objects
allocated 22000 objects
freed     18000 objects
freed     19000 objects
freed     20000 objects
freed     21000 objects
allocated 23000 objects
allocated 24000 objects
allocated 25000 objects
allocated 26000 objects
freed     22000 objects
allocated 27000 objects
allocated 28000 objects
allocated 29000 objects
allocated 30000 objects
allocated 31000 objects
allocated 32000 objects
allocated 33000 objects
allocated 34000 objects
allocated 35000 objects
freed     23000 objects
freed     24000 objects
freed     25000 objects
allocated 36000 objects
freed     26000 objects
freed     27000 objects
freed     28000 objects
freed     29000 objects
freed     30000 objects
freed     31000 objects
freed     32000 objects
freed     33000 objects
freed     34000 objects
freed     35000 objects
freed     36000 objects
allocated 37000 objects
allocated 38000 objects
allocated 39000 objects
allocated 40000 objects
allocated 41000 objects
allocated 42000 objects
allocated 43000 objects
allocated 44000 objects
allocated 45000 objects
allocated 46000 objects
allocated 47000 objects
allocated 48000 objects
allocated 49000 objects
freed     37000 objects
freed     38000 objects
allocated 50000 objects
allocated 51000 objects
allocated 52000 objects
allocated 53000 objects
allocated 54000 objects
allocated 55000 objects
allocated 56000 objects
allocated 57000 objects
freed     39000 objects
freed     40000 objects
allocated 58000 objects
freed     41000 objects
freed     42000 objects
freed     43000 objects
freed     44000 objects
freed     45000 objects
freed     46000 objects
freed     47000 objects
freed     48000 objects
freed     49000 objects
freed     50000 objects
allocated 59000 objects
freed     51000 objects
freed     52000 objects
freed     53000 objects
freed     54000 objects
freed     55000 objects
freed     56000 objects
freed     57000 objects
allocated 60000 objects
freed     58000 objects
allocated 61000 objects
allocated 62000 objects
allocated 63000 objects
allocated 64000 objects
allocated 65000 objects
allocated 66000 objects
allocated 67000 objects
allocated 68000 objects
allocated 69000 objects
allocated 70000 objects
allocated 71000 objects
allocated 72000 objects
allocated 73000 objects
allocated 74000 objects
allocated 75000 objects
allocated 76000 objects
allocated 77000 objects
allocated 78000 objects
allocated 79000 objects
allocated 80000 objects
allocated 81000 objects
allocated 82000 objects
allocated 83000 objects
allocated 84000 objects
Foreign.java:-2:in `newClosure': java.lang.NoClassDefFoundError: java/lang/IndexOutOfMemoryError
	from ClosureManager.java:42:in `newClosure'
	from CallbackManager.java:117:in `getCallback'
	from CallbackMarshaller.java:64:in `marshalParam'
	from CallbackMarshaller.java:57:in `marshal'
	from CallbackMarshaller.java:50:in `marshal'
	from CallbackMethodWithBlock.java:46:in `call'
	from DynamicMethod.java:168:in `call'
	from DynamicMethod.java:164:in `call'
	from CachingCallSite.java:115:in `call'
	from ffi_leak.rb:34:in `method__0$RUBY$create_an_object'
	from ffi_leak#create_an_object:-1:in `call'
	from ffi_leak#create_an_object:-1:in `call'
	from CachingCallSite.java:75:in `call'
	from ffi_leak.rb:44:in `method__1$RUBY$create_lots_of_objects'
	from ffi_leak#create_lots_of_objects:-1:in `call'
	from ffi_leak#create_lots_of_objects:-1:in `call'
	from CachingCallSite.java:258:in `cacheAndCall'
	from CachingCallSite.java:77:in `call'
	from ffi_leak.rb:69:in `__file__'
	from ffi_leak.rb:-1:in `load'
	from Ruby.java:592:in `runScript'
	from Ruby.java:514:in `runNormally'
	from Ruby.java:360:in `runFromMain'
	from Main.java:268:in `run'
	from Main.java:113:in `run'
	from Main.java:97:in `main'

jruby-no-leak.log

=> We're not going to leak memory. Watch.
allocated 1000 objects
freed     1000 objects
allocated 2000 objects
freed     2000 objects
allocated 3000 objects
freed     3000 objects
allocated 4000 objects
freed     4000 objects
allocated 5000 objects
freed     5000 objects
allocated 6000 objects
freed     6000 objects
allocated 7000 objects
freed     7000 objects
allocated 8000 objects
freed     8000 objects
allocated 9000 objects
freed     9000 objects
allocated 10000 objects
allocated 11000 objects
freed     10000 objects
freed     11000 objects
allocated 12000 objects
freed     12000 objects
allocated 13000 objects
freed     13000 objects
allocated 14000 objects
freed     14000 objects
allocated 15000 objects
freed     15000 objects
allocated 16000 objects
freed     16000 objects
allocated 17000 objects
freed     17000 objects
allocated 18000 objects
freed     18000 objects
allocated 19000 objects
freed     19000 objects
allocated 20000 objects
freed     20000 objects
allocated 21000 objects
freed     21000 objects
allocated 22000 objects
freed     22000 objects
allocated 23000 objects
freed     23000 objects
allocated 24000 objects
freed     24000 objects
allocated 25000 objects
freed     25000 objects
allocated 26000 objects
freed     26000 objects
allocated 27000 objects
freed     27000 objects
allocated 28000 objects
freed     28000 objects
allocated 29000 objects
freed     29000 objects
allocated 30000 objects
freed     30000 objects
allocated 31000 objects
freed     31000 objects
allocated 32000 objects
freed     32000 objects
allocated 33000 objects
freed     33000 objects
allocated 34000 objects
freed     34000 objects
allocated 35000 objects
freed     35000 objects
allocated 36000 objects
freed     36000 objects
allocated 37000 objects
freed     37000 objects
allocated 38000 objects
freed     38000 objects
allocated 39000 objects
freed     39000 objects
allocated 40000 objects
freed     40000 objects
allocated 41000 objects
freed     41000 objects
allocated 42000 objects
freed     42000 objects
allocated 43000 objects
freed     43000 objects
allocated 44000 objects
freed     44000 objects
allocated 45000 objects
freed     45000 objects
allocated 46000 objects
freed     46000 objects
allocated 47000 objects
freed     47000 objects
allocated 48000 objects
freed     48000 objects
allocated 49000 objects
freed     49000 objects
allocated 50000 objects
freed     50000 objects
allocated 51000 objects
freed     51000 objects
allocated 52000 objects
freed     52000 objects
allocated 53000 objects
freed     53000 objects
allocated 54000 objects
freed     54000 objects
allocated 55000 objects
freed     55000 objects
allocated 56000 objects
freed     56000 objects
allocated 57000 objects
freed     57000 objects
allocated 58000 objects
freed     58000 objects
allocated 59000 objects
freed     59000 objects
allocated 60000 objects
freed     60000 objects
allocated 61000 objects
freed     61000 objects
allocated 62000 objects
freed     62000 objects
allocated 63000 objects
freed     63000 objects
allocated 64000 objects
freed     64000 objects
allocated 65000 objects
freed     65000 objects
allocated 66000 objects
freed     66000 objects
allocated 67000 objects
freed     67000 objects
allocated 68000 objects
freed     68000 objects
allocated 69000 objects
freed     69000 objects
allocated 70000 objects
freed     70000 objects
allocated 71000 objects
freed     71000 objects
allocated 72000 objects
freed     72000 objects
allocated 73000 objects
freed     73000 objects
allocated 74000 objects
freed     74000 objects
allocated 75000 objects
freed     75000 objects
allocated 76000 objects
freed     76000 objects
allocated 77000 objects
freed     77000 objects
allocated 78000 objects
freed     78000 objects
allocated 79000 objects
freed     79000 objects
allocated 80000 objects
freed     80000 objects
allocated 81000 objects
freed     81000 objects
allocated 82000 objects
freed     82000 objects
allocated 83000 objects
freed     83000 objects
allocated 84000 objects
freed     84000 objects
allocated 85000 objects
freed     85000 objects
allocated 86000 objects
freed     86000 objects
allocated 87000 objects
allocated 88000 objects
allocated 89000 objects
freed     87000 objects
freed     88000 objects
allocated 90000 objects
freed     89000 objects
freed     90000 objects
allocated 91000 objects
freed     91000 objects
allocated 92000 objects
freed     92000 objects
allocated 93000 objects
freed     93000 objects
allocated 94000 objects
freed     94000 objects
allocated 95000 objects
freed     95000 objects
allocated 96000 objects
freed     96000 objects
allocated 97000 objects
freed     97000 objects
allocated 98000 objects
freed     98000 objects
allocated 99000 objects
freed     99000 objects
allocated 100000 objects
freed     100000 objects
allocated 101000 objects
freed     101000 objects
allocated 102000 objects
freed     102000 objects
allocated 103000 objects
freed     103000 objects
allocated 104000 objects
freed     104000 objects
allocated 105000 objects
freed     105000 objects
allocated 106000 objects
freed     106000 objects
allocated 107000 objects
freed     107000 objects
allocated 108000 objects
freed     108000 objects
allocated 109000 objects
freed     109000 objects
allocated 110000 objects
freed     110000 objects
allocated 111000 objects
freed     111000 objects
allocated 112000 objects
freed     112000 objects
allocated 113000 objects
freed     113000 objects
allocated 114000 objects
freed     114000 objects
allocated 115000 objects
freed     115000 objects
allocated 116000 objects
freed     116000 objects
allocated 117000 objects
freed     117000 objects
allocated 118000 objects
freed     118000 objects
allocated 119000 objects
freed     119000 objects
allocated 120000 objects
freed     120000 objects
allocated 121000 objects
freed     121000 objects
allocated 122000 objects
freed     122000 objects
allocated 123000 objects
freed     123000 objects
allocated 124000 objects
freed     124000 objects
allocated 125000 objects
freed     125000 objects
allocated 126000 objects
freed     126000 objects
allocated 127000 objects
freed     127000 objects
allocated 128000 objects
freed     128000 objects
allocated 129000 objects
freed     129000 objects
allocated 130000 objects
freed     130000 objects
allocated 131000 objects
freed     131000 objects
allocated 132000 objects
freed     132000 objects
allocated 133000 objects
freed     133000 objects
allocated 134000 objects
freed     134000 objects
allocated 135000 objects
freed     135000 objects
allocated 136000 objects
freed     136000 objects
allocated 137000 objects
freed     137000 objects
allocated 138000 objects
freed     138000 objects
allocated 139000 objects
freed     139000 objects
allocated 140000 objects
freed     140000 objects
allocated 141000 objects
freed     141000 objects
allocated 142000 objects
freed     142000 objects
allocated 143000 objects
freed     143000 objects
allocated 144000 objects
freed     144000 objects
allocated 145000 objects
freed     145000 objects
allocated 146000 objects
freed     146000 objects
allocated 147000 objects
freed     147000 objects
allocated 148000 objects
freed     148000 objects
allocated 149000 objects
freed     149000 objects
allocated 150000 objects
freed     150000 objects
allocated 151000 objects
freed     151000 objects
allocated 152000 objects
freed     152000 objects
allocated 153000 objects
freed     153000 objects
allocated 154000 objects
freed     154000 objects
allocated 155000 objects
freed     155000 objects
allocated 156000 objects
freed     156000 objects
allocated 157000 objects
freed     157000 objects
allocated 158000 objects
allocated 159000 objects
allocated 160000 objects
freed     158000 objects
freed     159000 objects
freed     160000 objects
allocated 161000 objects
freed     161000 objects
allocated 162000 objects
freed     162000 objects
allocated 163000 objects
freed     163000 objects
allocated 164000 objects
freed     164000 objects
allocated 165000 objects
freed     165000 objects
allocated 166000 objects
freed     166000 objects
allocated 167000 objects
freed     167000 objects
allocated 168000 objects
freed     168000 objects
allocated 169000 objects
freed     169000 objects
allocated 170000 objects
freed     170000 objects
allocated 171000 objects
freed     171000 objects
allocated 172000 objects
freed     172000 objects
allocated 173000 objects
freed     173000 objects
allocated 174000 objects
freed     174000 objects
allocated 175000 objects
freed     175000 objects
allocated 176000 objects
freed     176000 objects
allocated 177000 objects
freed     177000 objects
allocated 178000 objects
freed     178000 objects
allocated 179000 objects
freed     179000 objects
allocated 180000 objects
freed     180000 objects
allocated 181000 objects
freed     181000 objects
allocated 182000 objects
freed     182000 objects
allocated 183000 objects
freed     183000 objects
allocated 184000 objects
freed     184000 objects
allocated 185000 objects
freed     185000 objects
allocated 186000 objects
freed     186000 objects
allocated 187000 objects
freed     187000 objects
allocated 188000 objects
freed     188000 objects
allocated 189000 objects
freed     189000 objects
allocated 190000 objects
freed     190000 objects
allocated 191000 objects
freed     191000 objects
allocated 192000 objects
freed     192000 objects
allocated 193000 objects
freed     193000 objects
allocated 194000 objects
freed     194000 objects
allocated 195000 objects
freed     195000 objects
allocated 196000 objects
freed     196000 objects
allocated 197000 objects
freed     197000 objects
allocated 198000 objects
freed     198000 objects
allocated 199000 objects
freed     199000 objects
allocated 200000 objects
freed     200000 objects
allocated 201000 objects
freed     201000 objects
allocated 202000 objects
freed     202000 objects
allocated 203000 objects
freed     203000 objects
allocated 204000 objects
freed     204000 objects
allocated 205000 objects
freed     205000 objects
allocated 206000 objects
freed     206000 objects
allocated 207000 objects
freed     207000 objects
allocated 208000 objects
freed     208000 objects
allocated 209000 objects
freed     209000 objects
allocated 210000 objects
freed     210000 objects
allocated 211000 objects
freed     211000 objects
^C

Makefile

libfoo.so:
	$(CC) -shared -o libfoo.so foo.c