How to take the sum of very small numbers, a + b + c + ..., when you only have log(a), log(b), log(c), and exp(log(a)) = 0.0 due to lack of precision. What we get in return is log(a + b + c + ...)

smallNumbers.py

from pylab import *

# Some Log rules:
# a*b = log(a) + log(b)
# a/b = log(a) - log(b)
# a**b = b*log(a)
# exp(log(x)) = log(exp(x)) = x
#
# The Trick:
# log(a + b) = log(a + b) - log(c) + log(c)
#            = log(a/c + b/c) + log(c)
#            = log(exp(log(a) - log(c)) + exp(log(b) - log(c))) + log(c)
#
# log(c) = max(log(a), log(b))
# this makes sure that we arent taking a log(0)

# dealing with really small numbers
a = array([1e-900] * 200)
# a = [0.0, 0.0, ...]
print sum(a)
# 0.0

# so we do everything with logs
aLogs = array([-900] * 200)


def logOfSumGivenLogs(aLogs):
    logC = max(aLogs)
    return log(sum([exp(logA - logC) for logA in aLogs])) + logC

print logOfSumGivenLogs(aLogs)