test.py

import gc
import os
import pandas as pd
import psutil
import time
import sys

from random import seed
from random import random
from datetime import datetime

seed(1)

def humamReadableSize(size, decimal_places=3):
  for unit in ['B','KiB','MiB','GiB','TiB']:
    if size < 1024.0:
        break
    size /= 1024.0

  return f"{size:.{decimal_places}f}{unit}"

def totalMemUsage():
  process = psutil.Process(os.getpid())
  mem = psutil.Process(os.getpid()).memory_info().rss
  return humamReadableSize(mem)

class Test:
  def __init__(self):
    self.__points = pd.DataFrame()

  def flush(self):
    # Should free memory here
    if (self.__points.shape[0] == 5):
      print(">> FLUSHING")
      print("   GC Threshold:", gc.get_threshold())
      print("   GC Count Before:", gc.get_count())

      # We need to release resources here to prevent memory
      # from keep growing.
      self.__points = pd.DataFrame()

      # Collect right away to prevent memory grow
      gc.collect()
      print("   GC Count After:", gc.get_count())
  
  def handlePoint(self, point):
    self.__points = pd.concat([self.__points, pd.DataFrame([point])], ignore_index=True)
    print(f"MEM USAGE: {totalMemUsage()}")
    self.flush()


test = Test()

while True:
  time.sleep(1)

  test.handlePoint({
    "time": datetime.now().isoformat(),
    "val-1": random(),
    "val-2": random(),
    "val-3": random(),
    "val-4": random(),
    "val-5": random(),
    "text-val-1": "hello",
    "text-val-2": "foo"
  })