Comparison results of monotone and nonmonotone output on Hopfield Network.

hopfield_network.py

#!/usr/bin/env python
# encoding: utf-8

import numpy as np
from matplotlib import pyplot as plt
import matplotlib.cm as cm

class HopfieldNetwork(object):
    def __init__(self, nonmonotone=False):
        self.__nonmonotone = nonmonotone

    def fit(self, patterns):
        """Hebbian Rule"""
        num_neurons = len(patterns[0])

        self.weight = np.zeros((num_neurons, num_neurons))
        for pattern in patterns:
            self.weight += np.outer(pattern, pattern)
        for i in range(num_neurons):
            self.weight[i, i] = 0.0
        self.weight /= num_neurons

    def predict(self, pattern, loop=100):
        return self.__predict(pattern, loop=loop)

    def __predict(self, xr, loop=10):
        last_energy = self.__energy(xr)
        for i in range(loop):
            if self.__nonmonotone:
                xr = self.__nonmonotone_output(xr)
            else:
                xr = self.__output(xr)

            energy = self.__energy(xr)
            if last_energy == energy:
                break
            last_energy = energy
        return np.sign(xr)

    def __output(self, x):
        return np.sign(self.weight.dot(x))

    def __nonmonotone_output(self, x):
        """ http://volga.esys.tsukuba.ac.jp/~mor/paper/mor1993.pdf """
        h = 1.8   # TODO: not optimized
        lam = 2.7
        psi = np.zeros(len(x))
        uu = self.weight.dot(x)
        psi[uu < -h] = -1
        psi[uu > h] = 1
        return np.sign(self.weight.dot(x - lam*psi))

    def __energy(self, xr):
        return - xr.dot(self.weight).dot(xr)

# utils

def get_corrupted_input(input, corruption_level):
    corrupted = np.copy(input)
    inv = np.random.binomial(n=1, p=corruption_level, size=len(input))
    for i, v in enumerate(input):
        if inv[i]:
            corrupted[i] = -1 * v
    return corrupted

if __name__ == "__main__":
    np.random.seed(123)
    plt.figure(figsize=(10, 5))

    for ni, nonmonotone in enumerate([False, True]):
        plt.subplot(1, 2, ni+1)

        for num_neurons in (25, 50, 100, 400, 800, 1200):
            line = []
            ratios = np.arange(0.1, 0.31, 0.01)
            for ratio in ratios:
                print ni, num_neurons, ratio

                num_patterns = int(num_neurons * ratio)
                patterns = np.sign(np.random.rand(num_patterns, num_neurons) * 2 - 1).tolist()  # +-1

                hn = HopfieldNetwork(nonmonotone=nonmonotone)
                hn.fit(patterns)

                sum_error_bits = 0
                for pattern in patterns:
                    test = get_corrupted_input(pattern, 0.1)
                    predicted = hn.predict(test)
                    sum_error_bits += len(np.where(predicted != pattern)[0])
                line.append(float(sum_error_bits) / len(patterns) / num_neurons)
            plt.plot(ratios, line, 'x-', label='neurons={n}'.format(n=num_neurons), alpha=0.5)

        plt.legend(loc='upper left')
        plt.xlabel('memory ratio')
        plt.ylabel('average error bits ratio')
        plt.xlim([0.1, 0.3])
        plt.ylim([-0.01, 1.0])
        plt.grid()
        plt.title('nonmonotone={n}'.format(n=nonmonotone))

    plt.savefig('result.png', dpi=150)

Experiment for the following discussion.
https://www.reddit.com/r/MachineLearning/comments/5j2ts7/d_questionproblem_with_largen_hopfield_network

result image