Buffer vs. Array join

array.py

import resource

def max_rss():
    return resource.getrusage(
        resource.RUSAGE_SELF,
    ).ru_maxrss

prev_result = None
def mem_delta(prefix=None):
    global prev_result

    if prefix is not None:
        prefix += ': '
    else:
        prefix = ''

    new_result = max_rss()
    message = prefix + str(new_result - prev_result)
    prev_result = new_result

    return message

def generate_string():
    return 'a' * 1024


assert generate_string() is not generate_string(), \
    "generate_string must generate non-interned strings for this to be a " \
    "decent benchmark."

buffer_ = []

# Before all the munging. Baseline.
prev_result = max_rss()

for i in xrange(1024 ** 2):
    buffer_.append(generate_string())
print mem_delta('After filling the buffer')

result = b''.join(buffer_)
print mem_delta('After getting value')

del buffer_
print mem_delta('After releasing the buffer')

buffer.py

import resource
from io import BytesIO

def max_rss():
    return resource.getrusage(
        resource.RUSAGE_SELF,
    ).ru_maxrss

prev_result = None
def mem_delta(prefix=None):
    global prev_result

    if prefix is not None:
        prefix += ': '
    else:
        prefix = ''

    new_result = max_rss()
    message = prefix + str(new_result - prev_result)
    prev_result = new_result

    return message

def generate_string():
    return 'a' * 1024


assert generate_string() is not generate_string(), \
    "generate_string must generate non-interned strings for this to be a " \
    "decent benchmark."

buffer_ = BytesIO()

# Before all the munging. Baseline.
prev_result = max_rss()

for i in xrange(1024 ** 2):
    buffer_.write(generate_string())
print mem_delta('After filling the buffer')

result = buffer_.getvalue()
print mem_delta('After getting value')

del buffer_
print mem_delta('After releasing the buffer')

results.txt

# For array.py
# 0
After filling the buffer: 1105908
After getting value: 1048480
After releasing the buffer: 216
# 1
After filling the buffer: 1105908
After getting value: 1048612
After releasing the buffer: 84
# 2
After filling the buffer: 1105908
After getting value: 1048444
After releasing the buffer: 252
# 3
After filling the buffer: 1105908
After getting value: 1048444
After releasing the buffer: 252

# For buffer.py
# 0
After filling the buffer: 1048812
After getting value: 1048492
After releasing the buffer: 124
# 1
After filling the buffer: 1049516
After getting value: 1048584
After releasing the buffer: 0
# 2
After filling the buffer: 1049604
After getting value: 1048588
After releasing the buffer: 0
# 3
After filling the buffer: 1050328
After getting value: 1048488
After releasing the buffer: 0

pypy results are worse...:

# array.py
#0
After filling the buffer: 73916416
After getting value: 1074282496
After releasing the buffer: 0
#1
After filling the buffer: 73916416
After getting value: 1074270208
After releasing the buffer: 0
#2
After filling the buffer: 74911744
After getting value: 1074278400
After releasing the buffer: 0

# buffer.py
#0
After filling the buffer: 1074376704
After getting value: 1073778688
After releasing the buffer: 0
#1
After filling the buffer: 1074388992
After getting value: 1073766400
After releasing the buffer: 0
#2
After filling the buffer: 1074405376
After getting value: 1073766400
After releasing the buffer: 0