salt-master memory debugging

memdebug.diff

diff -Nur /root/salt/cli/daemons.py /usr/lib/python2.7/site-packages/salt/cli/daemons.py
--- /root/salt/cli/daemons.py	2016-03-26 17:30:02.823000000 -0400
+++ /usr/lib/python2.7/site-packages/salt/cli/daemons.py	2016-03-26 18:36:16.112000000 -0400
@@ -43,6 +43,9 @@
 from salt.utils import migrations
 from salt.utils import kinds
 
+from pympler import muppy
+from pympler import summary
+
 try:
     from salt.utils import parsers, ip_bracket
     from salt.utils.verify import check_user, verify_env, verify_socket
@@ -153,7 +156,15 @@
         '''
         self.prepare()
         if check_user(self.config['user']):
-            logger.info('The salt master is starting up')
+            logger.info('The salt master is starting up (memory_debug=%s)',
+                        self.config['memory_debug'])
+            if self.config['memory_debug']:
+                all_objects = muppy.get_objects()
+                objcount = len(all_objects)
+                logger.debug('Initial object count: %s', objcount)
+                memsum = summary.summarize(all_objects)
+                summary.print_(memsum)
+
             self.master.start()
 
     def shutdown(self):
diff -Nur /root/salt/config.py /usr/lib/python2.7/site-packages/salt/config.py
--- /root/salt/config.py	2016-03-26 17:30:02.541000000 -0400
+++ /usr/lib/python2.7/site-packages/salt/config.py	2016-03-26 18:29:02.625000000 -0400
@@ -766,6 +766,9 @@
 
     # Delay in seconds before executing bootstrap (salt cloud)
     'bootstrap_delay': int,
+
+    # Enable memory debugging
+    'memory_debug': bool,
 }
 
 # default configurations
@@ -971,6 +974,7 @@
     # ZMQ HWM for EventPublisher pub socket - different for minion vs. master
     'event_publisher_pub_hwm': 1000,
     'event_match_type': 'startswith',
+    'memory_debug': False,
 }
 
 DEFAULT_MASTER_OPTS = {
@@ -1206,6 +1210,7 @@
     'dummy_pub': False,
     'http_request_timeout': 1 * 60 * 60.0,  # 1 hour
     'http_max_body': 100 * 1024 * 1024 * 1024,  # 100GB
+    'memory_debug': False,
 }
 
 
@@ -1386,11 +1391,17 @@
     return '{0[id]}.{0[append_domain]}'.format(opts)
 
 
+#import traceback
+#import StringIO
 def _read_conf_file(path):
     '''
     Read in a config file from a given path and process it into a dictionary
     '''
-    log.debug('Reading configuration from {0}'.format(path))
+    pid = os.getpid()
+    log.debug('[{}] Reading configuration from {}'.format(pid, path))
+    #trace = StringIO.StringIO()
+    #traceback.print_stack(file=trace)
+    #log.debug(trace.getvalue())
     with salt.utils.fopen(path, 'r') as conf_file:
         try:
             conf_opts = yaml.safe_load(conf_file.read()) or {}
diff -Nur /root/salt/utils/verify.py /usr/lib/python2.7/site-packages/salt/utils/verify.py
--- /root/salt/utils/verify.py	2016-03-26 17:30:02.433000000 -0400
+++ /usr/lib/python2.7/site-packages/salt/utils/verify.py	2016-03-26 18:47:16.817000000 -0400
@@ -15,6 +15,10 @@
 import socket
 import logging
 
+from pympler import muppy
+from pympler import summary
+from pympler import tracker
+
 # Import third party libs
 if sys.platform.startswith('win'):
     import win32file
@@ -400,6 +404,7 @@
             raise SaltClientError(msg)
 
 
+memtracker = None
 def check_max_open_files(opts):
     '''
     Check the number of max allowed open files and adjust if needed
@@ -421,6 +426,17 @@
             accepted_count
         )
     )
+    if opts['memory_debug']:
+        global memtracker
+        if not memtracker:
+            memtracker = tracker.SummaryTracker()
+        all_objects = muppy.get_objects()
+        objcount = len(all_objects)
+        log.debug('Current object count: %s', objcount)
+        sys.stdout.write('Current object count: {}\n'.format(objcount))
+        memtracker.print_diff()
+        sys.stdout.write('\n')
+        sys.stdout.flush()
 
     level = logging.INFO