demonstrating "too many open files" issue when using pubsub

pubsub_too_many_open_files.py

import psutil
import gc
from google.cloud import pubsub_v1


topic_project_id = 'PROJECT-ID' # todo: replace with yours
topic_name = 'TOPIC-NAME' # todo: replace with yours

data_str = 'just some text'
p = psutil.Process()


def publish_func(i, publisher):
  topic_path = publisher.topic_path(topic_project_id, topic_name)
  data = '{}-{}'.format(data_str, i).encode('utf-8')

  future = publisher.publish(topic_path, data)
  future.result()


def publish_func_1(i):
  publisher = pubsub_v1.PublisherClient()
  topic_path = publisher.topic_path(topic_project_id, topic_name)
  data = '{}-{}'.format(data_str, i).encode('utf-8')

  future = publisher.publish(topic_path, data)
  future.result()


def publish_func_2(i):
  publisher = pubsub_v1.PublisherClient()
  topic_path = publisher.topic_path(topic_project_id, topic_name)
  data = '{}-{}'.format(data_str, i).encode('utf-8')

  future = publisher.publish(topic_path, data)
  future.result()

  publisher.api.transport._channel.close()


class MyPublisher(pubsub_v1.PublisherClient):

  def __enter__(self):
    return self

  def __exit__(self, exc_type, exc_val, exc_tb):
    self.api.transport._channel.close()


def publish_func_3(i):
  data = '{}-{}'.format(data_str, i).encode('utf-8')

  with MyPublisher() as publisher:

    topic_path = publisher.topic_path(topic_project_id, topic_name)
    future = publisher.publish(topic_path, data)
    future.result()


def naive():
  for i in range(10):
    publish_func_1(i)
    print('{} connections: {}'.format(i, len(p.connections())))

def with_close():
  for i in range(10):
    publish_func_2(i)
    print('{} connections: {}'.format(i, len(p.connections())))

def with_gc():
  for i in range(10):
    publish_func_1(i)
    n = gc.collect()
    print('{} connections: {}'.format(i, len(p.connections())))
    print('Unreachable objects: {}'.format(n))

def with_with():
  for i in range(10):
    publish_func_3(i)
    print('{} connections: {}'.format(i, len(p.connections())))

def reuse_publisher():
  publisher = pubsub_v1.PublisherClient()
  for i in range(10):
    publish_func(i, publisher)
    print('{} connections: {}'.format(i, len(p.connections())))


print('\nwith_close:')
with_close()

print('\nreuse_publisher:')
reuse_publisher()

print('\nnaive:')
naive()

print('\nwith_gc:')
with_gc()

print('\nwith_with:')
with_with()

requirements.txt

google-cloud-pubsub==0.39.1
futures==3.1.1
psutil

z_outcome.txt

$ python pubsub_too_many_open_files.py 

with_close:
0 connections: 0
1 connections: 0
2 connections: 0
3 connections: 0
4 connections: 0
5 connections: 0
6 connections: 0
7 connections: 0
8 connections: 0
9 connections: 0

reuse_publisher:
0 connections: 2
1 connections: 2
2 connections: 2
3 connections: 2
4 connections: 2
5 connections: 2
6 connections: 2
7 connections: 2
8 connections: 2
9 connections: 2

naive:
0 connections: 4
1 connections: 6
2 connections: 8
3 connections: 10
4 connections: 12
5 connections: 14
6 connections: 16
7 connections: 18
8 connections: 20
9 connections: 22

with_gc:
0 connections: 24
Unreachable objects: 1126
1 connections: 26
Unreachable objects: 222
2 connections: 28
Unreachable objects: 222
3 connections: 30
Unreachable objects: 222
4 connections: 32
Unreachable objects: 222
5 connections: 34
Unreachable objects: 222
6 connections: 36
Unreachable objects: 222
7 connections: 38
Unreachable objects: 222
8 connections: 40
Unreachable objects: 222
9 connections: 42
Unreachable objects: 222

with_with:
0 connections: 42
1 connections: 42
2 connections: 42
3 connections: 42
4 connections: 42
5 connections: 42
6 connections: 42
7 connections: 42
8 connections: 42
9 connections: 42