Exploding activations from repeated matrix multiplications.

exploding_activations.py

import torch

# experiment settings
d = 5
nlayers = 100
normalize = False # set True to use normalization

# create vector with random entries between [-1, 1]
input_vector = (torch.rand(d) - 0.5) * 2.0

# create matrix with random entries between [-1, 1]
# by which we can repeatedly multiply the input vector
weight_matrix = (torch.rand(d, d) - 0.5) * 2.0

output = input_vector
for i in range(nlayers):
    # optionally perform normalization
    if normalize:
        output = (output - torch.mean(output)) / torch.std(output)

    # repeatedly multiply the vector by the matrix
    output = weight_matrix @ output

# observe output values
print(output)