Inotify watch starvation and recovery

graph.yaml

---
graph: mygraph
comment: hello world example
resources:
  file:
  - name: dir1
    path: "/tmp/removeme/"
    state: exists
  - name: dir2
    path: "/tmp/removeme2/"
    state: exists
edges: []

preparation.sh

# create a tree with almost 8192 subdirectories
for i in `seq 1 8073` ; do HEX=`printf %04x $i` ; BASE=${HEX%??} ; mkdir -p /tmp/removeme/$BASE/$^CX ; done

recovery.sh

# however, removing each created subdirectory prior to making the next one will not deplete inotify watches
for i in `seq 1 10` ; do mkdir /tmp/removeme2/sub$i ; rmdir /tmp/removeme2/sub$i ; sleep .5 ; done

# alternatively, mgmt can take care of the removal itself (see following graph)
# then the loop from the previous example works as well and mgmt does not crash

safer-graph.yaml

---
graph: mygraph
comment: hello world example
resources:
  file:
  - name: dir1
    path: "/tmp/removeme/"
    state: exists
  - name: dir2
    path: "/tmp/removeme2/"
    state: exists
    force: true
edges: []

starvation.sh

# create some directories in /tmp/removeme2
for i in `seq 1 10` ; do mkdir /tmp/removeme2/sub$i ; sleep .5 ; done

# this kills mgmt like this:
# 00:12:11 file.go:143: File[dir2]: Watching: /tmp/removeme2
# 00:12:11 file.go:297: File[dir2]: adding /tmp/removeme2 to watch
# 00:12:11 file.go:297: File[dir2]: adding /tmp/removeme2/sub1 to watch
# 00:12:11 file.go:297: File[dir2]: adding /tmp/removeme2/sub2 to watch
# 00:12:11 file.go:297: File[dir2]: adding /tmp/removeme2/sub3 to watch
# 00:12:11 file.go:297: File[dir2]: adding /tmp/removeme2/sub4 to watch
# 00:12:11 file.go:154: File[dir2]: watcher.Add(/tmp/removeme2): Error: no space left on device
# 00:12:11 file.go:161: File[dir2]: Out of inotify watches!
# 00:12:11 file.go:162: no space left on device