see https://github.com/andrewschreiber/numpy-saliency

save_vanilla_gradient(network, image, data).py

# Create a saliency map for each data point
for i, image in enumerate(data):
    # Forward pass on image
    # Note: the activations are saved on each layer
    output = image
    for l in range(len(network.layers)):
        output = network.layers[l].forward(output)

    # Backprop to get gradient
    label_one_hot = labels[i]
    dy = np.array(label_one_hot)
    for l in range(len(network.layers)-1, -1, -1):
        dout = network.layers[l].backward(dy)
        dy = dout
    raw_saliency_map = dout