Nikolaj-K/network1.py

## network1.py
"""
A compact feedforward AI. Script explained here:

https://youtu.be/z2aq21lMw40

In this video we're implementing a feed forward neutral network that is able to discern
handwritten digits in vanilla Python.

References:
http://static.latexstudio.net/article/2018/0912/neuralnetworksanddeeplearning.pdf  # deep learning book
https://en.wikipedia.org/wiki/MNIST_database  # mnist data set
https://en.wikipedia.org/wiki/Stochastic_del_descent  # SGD
https://en.wikipedia.org/wiki/Feedforward_neural_network  # NN
http://neuralnetworksanddeeplearning.com/images/tikz12.png  # image of the network architecture used in the video
https://www.youtube.com/results?search_query=3blue1brown+deep+learning+chapter
https://youtu.be/X1mo7Uwvzn8?t=145 # on sigmoid curves (Nikolaj-K)

Notions present in the book but not touched upon in this script:
    - improved initialization of network weights
    - improved cost function
    - regularization
    - convolution
    - max pooling
    - other activation functions than sigmoid
    - rectified linear units
    - softmax
    - external library speedup
"""

import gzip
import numpy as np
import pickle
import time

import warnings; warnings.filterwarnings("ignore", category=np.VisibleDeprecationWarning)   # np.array cast non-niceness


class IOPair:
    """
    4 functions that ought to be implemented
    E.g. if the data type is an image together with a label, _get_input might
    return the pixels and _correct might validate correct classifications by the network
    """
    def get_input(self):
        return self._get_input()

    def output_ground_truth(self):
        return self._output_ground_truth()

    def show(self, activation_prediction) -> None:
        self._show(activation_prediction)

    def correct(self, activation_prediction) -> bool:
        gt = self._output_ground_truth()
        ap_formatted = self._to_output_format(activation_prediction)
        return ap_formatted == gt


class Math:
    def sigmoid(z): # Smooth step.
        return (1 + np.exp(-z)) ** -1

    def dsigmoid(z): # Derivative of a smooth step.
        s = Math.sigmoid(z)
        return s * (1 - s)

    def zeros(np_array):
        return [np.zeros(x.shape) for x in np_array]


class NeuralNetwork:
    """
    Feedforward neural
    """
    def __init__(self, input_layer_size, hidden_layer_sizes, output_layer_size):
        """
        Initializing a network archetecture.
        1. input layer
        2. output layer according to this functions parameter
        3. outputer layer
        Weights are connections. Compute them by zipping over the dimensions.
        """
        self.__output_layer_size = output_layer_size
        in_layers_sizes = [input_layer_size] + hidden_layer_sizes
        out_layers_sizes = hidden_layer_sizes + [output_layer_size]

        self.__weights = [np.random.randn(*sizes) for sizes in zip(out_layers_sizes, in_layers_sizes)]
        self.__biases = [np.random.randn(size, 1) for size in out_layers_sizes] # sampling from a normal distirbution
        # The elements are pairs of sizes of connected layers (number of outgoing resp. ingoing neurons.)


    def __feed_forward(self, data):
        """
        Propagate the activations, from data all through to the prediction.
        """
        a = data  # Activation

        as_ = [a]  # Layer by layer
        zs = []  # List to store all the z vectors, layer by layer
        for w, b in zip(self.__weights, self.__biases):
            z = np.dot(w, a) + b
            a = Math.sigmoid(z)
            zs.append(z)
            as_.append(a)  # Derivable from data and zs

        return zs, as_


    def predicts_correctly(self, in_out_pair, show=False):
        """
        Wrap the feed forward call to return a prediction & success pair.
        """
        _zs, _as = self.__feed_forward(in_out_pair.get_input())
        activation_prediction = _as[-1]
        if show:
            in_out_pair.show(activation_prediction)

        return in_out_pair.correct(activation_prediction)


    def run_sgd(self, epochs, batch_size, eta, training_data):
        """
        Run the stochastic gradient descent algorithm:
        In epochs (iterations), put the training data set in random batches,
        and update the weights and biases for the purpose of improving the
        delta between predicted and the ground turth classifications.
        See formulas in any deep learning book!
        Note: Makes use of the normalized mini batch gradient.
        """
        data = list(training_data) # local copy
        np.random.shuffle(data)

        for epoch in range(epochs):
            for idx in range(0, len(data), batch_size):
                batch = data[idx : idx + batch_size]
                del_w, del_b = self.__mean_gradient(batch)  # Call subroutine
                self.__weights -= eta * del_w  # Normalize
                self.__biases -= eta * del_b

            self.__log_score(data)  # Optional


    def __gradient(self, in_out_pair):
        """
        Details:
        The cost C is (a neuron sum over neurons j at the last layer)
            (output_j - a_j)^2
        For every layer l:
            a_{l-1} = \sigma(z_{l-1})
        is the "activation" from the previous layer and
            z_l = w_l * a_{l-1} + b_l

        The derivative of C w.r.t. a_j is a linear difference,
        although we're interested in the derivative's w.r.t. z.

        For further details for why dw and db are what they are, see the chap2 here

        http://neuralnetworksanddeeplearning.com/chap2.html
        """
        zs, as_ = self.__feed_forward(in_out_pair.get_input())

        # Note: as_[-1] holds activation of final output layer.
        l = -1  # -1 is index of the last layer
        activation_prediction = as_[l]

        cost_derivative = activation_prediction - in_out_pair.output_ground_truth()  # d cost / d a at the final layer

        # Backward pass
        dw = Math.zeros(self.__weights)
        db = Math.zeros(self.__biases)
        activation = Math.dsigmoid(zs[l])
        delta = cost_derivative * activation  # d cost_l / d z_l
        # Here: Last activation minus the ground truth

        dw[l] = np.dot(delta, as_[l - 1].transpose())
        db[l] = delta
        for l in range(l - 1, l-len(self.__biases), -1):
            activation = Math.dsigmoid(zs[l])

            delta = np.dot(self.__weights[l + 1].transpose(), delta) * activation
            dw[l] = np.dot(delta, as_[l - 1].transpose())
            db[l] = delta

        return np.array(dw), np.array(db)


    def __mean_gradient(self, in_out_pairs):
        """
        Compute the gradient for the batch of images.
        Note: Makes use of the image gradient.
        """
        del_w = Math.zeros(self.__weights)
        del_b = Math.zeros(self.__biases)
        for in_out_pair in in_out_pairs:
            dw, db = self.__gradient(in_out_pair)
            del_w += np.array(dw)
            del_b += np.array(db)

        n = len(in_out_pairs)
        return del_w / n, del_b / n


    def __log_score(self, test_data):
        score = sum(map(self.predicts_correctly, test_data)) # count the successes
        total = len(test_data)
        percentage = round(score * 100.0 / total, 2)
        print(f"[log_score] Test data score: {score}/{total}. ({percentage}%)")


class Config:
    # Script
    # Git clone this (or download just the .gz file): https://github.com/MichalDanielDobrzanski/DeepLearningPython
    DATA_DIRPATH = f"/Users/amoogle/Documents/Git/neural_network/DeepLearningPython/"
    DATA_FILENAME = "mnist.pkl.gz"
    DATA_FILEPATH = f"{DATA_DIRPATH}/{DATA_FILENAME}"

    # Learning and Network
    EPOCHS = 4
    BATCH_SIZE = 10
    ETA = 3.0
    HIDDEN_LAYER_SIZES = [10, 20, 10] # array of ints

    # Misc for logging and evaluation
    SOME_INDEX = 98


class Image(IOPair):
    IMAGE_WIDTH = 28
    NUM_PIXELS = IMAGE_WIDTH ** 2
    NUMBER_BASE = 10

    def __init__(self, image_and_gt_pair):
        """
        Class holding an array of grayscale values (=the networks input layer
        format) as well as the ground truth value digit displayed on that image
        """
        self.__pixels = image_and_gt_pair[0]
        self.__gt_label = image_and_gt_pair[1]  # Classification/Label

    def __to_prediction_digit(self, activation_prediction):
        """
        Activation_prediction is an arraw of floats while the return value is an index into that list.
        E.g. [[9.89989735e-03] [1.08126136e-02] [6.17035645e-04] [1.14060381e-02] [5.98185829e-03]
              [7.31337324e-03] [1.61992979e-05] [9.24512441e-01] [9.88453143e-05] [8.47158033e-02]]
        \mapsto 7
        as '9.24512441e-01' is the biggest value
        """
        return np.argmax(activation_prediction)

    def __digit_to_output_format(self, d_in):
        """
        E.g.
        7 \mapsto [0, 0, 0, 0, 0, 0, 0, 1, 0, 0]
        """
        return [[int(d_in==d)] for d in range(self.NUMBER_BASE)]

    def _get_input(self):
        return self.__pixels

    def _output_ground_truth(self):
        return self.__digit_to_output_format(self.__gt_label)

    def _to_output_format(self, activation_prediction):
        pd = self.__to_prediction_digit(activation_prediction)
        return self.__digit_to_output_format(pd)

    def _show(self, activation_prediction) -> None:
        """
        Print the grayscale image as ascii image.
        And optionally compare the prediction to a ground turth value.
        """
        GRAYSCALE_ASCII = " .',_-+*$#"
        HORIZONTAL_LINE: str = 2 * self.IMAGE_WIDTH * "_"

        # Draw image
        print(HORIZONTAL_LINE) # top of bounding box
        for idx, pixel in enumerate(self.__pixels):
            end_of_row = (idx + 1) % self.IMAGE_WIDTH == 0
            char = GRAYSCALE_ASCII[int(10 * pixel)]
            print(char, end=" |\n" if end_of_row else " ")
        print(HORIZONTAL_LINE + "|") # bottom of bounding box

        print(f"[show] Ground truth value is {self.__gt_label}.")
        if self.correct(activation_prediction):
            print("[show] And this was predicted correctly. :)")
        else:
            print(f"[show] And this was wrongly predicted as predicted as {activation_prediction}. :(")


def load_MNIST_data__and_cast():
    """
    Load the pairs of grayscale image array and ground truth for training and test data. The files are found here:
    https://github.com/MichalDanielDobrzanski/DeepLearningPython35/blob/master/mnist.pkl.gz
    This holds 3 data sets, but we won't use the validation one.
    :return: Images.
    """
    print(f"[load_MNIST_data] Trying to load data from <{Config.DATA_FILEPATH}>")
    with gzip.open(Config.DATA_FILEPATH, 'rb') as in_file:
        training_data_gz, _validation_data_gz, test_data_gz = pickle.load(in_file, encoding="latin1") # Load the "raw" data sets
    print(f"[load_MNIST_data] Data loading successful. len(training_data)={len(training_data_gz)}. len(test_data)={len(test_data_gz)}")

    def reshape_and_zip(all_images__and__all_gt_labels):
        all_images = all_images__and__all_gt_labels[0]
        assert all(len(img)==Image.NUM_PIXELS for img in all_images)

        all_gt_labels = all_images__and__all_gt_labels[1]
        assert all(num in range(10) for num in all_gt_labels)
        assert len(all_images)==len(all_gt_labels)

        SHAPE = (Image.NUM_PIXELS, 1)
        all_images_ = (np.reshape(img, SHAPE) for img in all_images)
        return zip(all_images_, all_gt_labels)

    def cast_all_to_image(all_images__and__all_gt_labels):
        return [Image(image_and_gt_pair) for image_and_gt_pair in reshape_and_zip(all_images__and__all_gt_labels)]

    return cast_all_to_image(training_data_gz), cast_all_to_image(test_data_gz)


def run_main():
    """
    Unpack data, train and do some examples.
    """
    # Set up network
    INPUT_LAYER_SIZE = Image.NUM_PIXELS
    OUTPUT_LAYER_SIZE = Image.NUMBER_BASE
    net = NeuralNetwork(
        INPUT_LAYER_SIZE,
        Config.HIDDEN_LAYER_SIZES,
        OUTPUT_LAYER_SIZE
    )

    # Train
    training_data, test_data = load_MNIST_data__and_cast()
    print(f"\n[run_main] Starting the training with {Config.EPOCHS} epochs...")
    net.run_sgd(
        Config.EPOCHS,
        Config.BATCH_SIZE,
        Config.ETA,
        training_data
    )

    # Evaluate
    print("[run_main] " + 40 * "*" + " Let's predict some example_imgs.")
    for some_other_index in [Config.SOME_INDEX * n for n in range(1, 8)]:
        img = test_data[some_other_index]
        _is_correct: bool = net.predicts_correctly(img, show=True)
        time.sleep(1)  # Just for clarity


if __name__=='__main__':
    run_main()
	"""
	A compact feedforward AI. Script explained here:

	https://youtu.be/z2aq21lMw40

	In this video we're implementing a feed forward neutral network that is able to discern
	handwritten digits in vanilla Python.

	References:
	http://static.latexstudio.net/article/2018/0912/neuralnetworksanddeeplearning.pdf # deep learning book
	https://en.wikipedia.org/wiki/MNIST_database # mnist data set
	https://en.wikipedia.org/wiki/Stochastic_del_descent # SGD
	https://en.wikipedia.org/wiki/Feedforward_neural_network # NN
	http://neuralnetworksanddeeplearning.com/images/tikz12.png # image of the network architecture used in the video
	https://www.youtube.com/results?search_query=3blue1brown+deep+learning+chapter
	https://youtu.be/X1mo7Uwvzn8?t=145 # on sigmoid curves (Nikolaj-K)

	Notions present in the book but not touched upon in this script:
	- improved initialization of network weights
	- improved cost function
	- regularization
	- convolution
	- max pooling
	- other activation functions than sigmoid
	- rectified linear units
	- softmax
	- external library speedup
	"""

	import gzip
	import numpy as np
	import pickle
	import time

	import warnings; warnings.filterwarnings("ignore", category=np.VisibleDeprecationWarning) # np.array cast non-niceness


	class IOPair:
	"""
	4 functions that ought to be implemented
	E.g. if the data type is an image together with a label, _get_input might
	return the pixels and _correct might validate correct classifications by the network
	"""
	def get_input(self):
	return self._get_input()

	def output_ground_truth(self):
	return self._output_ground_truth()

	def show(self, activation_prediction) -> None:
	self._show(activation_prediction)

	def correct(self, activation_prediction) -> bool:
	gt = self._output_ground_truth()
	ap_formatted = self._to_output_format(activation_prediction)
	return ap_formatted == gt


	class Math:
	def sigmoid(z): # Smooth step.
	return (1 + np.exp(-z)) ** -1

	def dsigmoid(z): # Derivative of a smooth step.
	s = Math.sigmoid(z)
	return s * (1 - s)

	def zeros(np_array):
	return [np.zeros(x.shape) for x in np_array]


	class NeuralNetwork:
	"""
	Feedforward neural
	"""
	def __init__(self, input_layer_size, hidden_layer_sizes, output_layer_size):
	"""
	Initializing a network archetecture.
	1. input layer
	2. output layer according to this functions parameter
	3. outputer layer
	Weights are connections. Compute them by zipping over the dimensions.
	"""
	self.__output_layer_size = output_layer_size
	in_layers_sizes = [input_layer_size] + hidden_layer_sizes
	out_layers_sizes = hidden_layer_sizes + [output_layer_size]

	self.__weights = [np.random.randn(*sizes) for sizes in zip(out_layers_sizes, in_layers_sizes)]
	self.__biases = [np.random.randn(size, 1) for size in out_layers_sizes] # sampling from a normal distirbution
	# The elements are pairs of sizes of connected layers (number of outgoing resp. ingoing neurons.)


	def __feed_forward(self, data):
	"""
	Propagate the activations, from data all through to the prediction.
	"""
	a = data # Activation

	as_ = [a] # Layer by layer
	zs = [] # List to store all the z vectors, layer by layer
	for w, b in zip(self.__weights, self.__biases):
	z = np.dot(w, a) + b
	a = Math.sigmoid(z)
	zs.append(z)
	as_.append(a) # Derivable from data and zs

	return zs, as_


	def predicts_correctly(self, in_out_pair, show=False):
	"""
	Wrap the feed forward call to return a prediction & success pair.
	"""
	_zs, _as = self.__feed_forward(in_out_pair.get_input())
	activation_prediction = _as[-1]
	if show:
	in_out_pair.show(activation_prediction)

	return in_out_pair.correct(activation_prediction)


	def run_sgd(self, epochs, batch_size, eta, training_data):
	"""
	Run the stochastic gradient descent algorithm:
	In epochs (iterations), put the training data set in random batches,
	and update the weights and biases for the purpose of improving the
	delta between predicted and the ground turth classifications.
	See formulas in any deep learning book!
	Note: Makes use of the normalized mini batch gradient.
	"""
	data = list(training_data) # local copy
	np.random.shuffle(data)

	for epoch in range(epochs):
	for idx in range(0, len(data), batch_size):
	batch = data[idx : idx + batch_size]
	del_w, del_b = self.__mean_gradient(batch) # Call subroutine
	self.__weights -= eta * del_w # Normalize
	self.__biases -= eta * del_b

	self.__log_score(data) # Optional


	def __gradient(self, in_out_pair):
	"""
	Details:
	The cost C is (a neuron sum over neurons j at the last layer)
	(output_j - a_j)^2
	For every layer l:
	a_{l-1} = \sigma(z_{l-1})
	is the "activation" from the previous layer and
	z_l = w_l * a_{l-1} + b_l

	The derivative of C w.r.t. a_j is a linear difference,
	although we're interested in the derivative's w.r.t. z.

	For further details for why dw and db are what they are, see the chap2 here

	http://neuralnetworksanddeeplearning.com/chap2.html
	"""
	zs, as_ = self.__feed_forward(in_out_pair.get_input())

	# Note: as_[-1] holds activation of final output layer.
	l = -1 # -1 is index of the last layer
	activation_prediction = as_[l]

	cost_derivative = activation_prediction - in_out_pair.output_ground_truth() # d cost / d a at the final layer

	# Backward pass
	dw = Math.zeros(self.__weights)
	db = Math.zeros(self.__biases)
	activation = Math.dsigmoid(zs[l])
	delta = cost_derivative * activation # d cost_l / d z_l
	# Here: Last activation minus the ground truth

	dw[l] = np.dot(delta, as_[l - 1].transpose())
	db[l] = delta
	for l in range(l - 1, l-len(self.__biases), -1):
	activation = Math.dsigmoid(zs[l])

	delta = np.dot(self.__weights[l + 1].transpose(), delta) * activation
	dw[l] = np.dot(delta, as_[l - 1].transpose())
	db[l] = delta

	return np.array(dw), np.array(db)


	def __mean_gradient(self, in_out_pairs):
	"""
	Compute the gradient for the batch of images.
	Note: Makes use of the image gradient.
	"""
	del_w = Math.zeros(self.__weights)
	del_b = Math.zeros(self.__biases)
	for in_out_pair in in_out_pairs:
	dw, db = self.__gradient(in_out_pair)
	del_w += np.array(dw)
	del_b += np.array(db)

	n = len(in_out_pairs)
	return del_w / n, del_b / n


	def __log_score(self, test_data):
	score = sum(map(self.predicts_correctly, test_data)) # count the successes
	total = len(test_data)
	percentage = round(score * 100.0 / total, 2)
	print(f"[log_score] Test data score: {score}/{total}. ({percentage}%)")


	class Config:
	# Script
	# Git clone this (or download just the .gz file): https://github.com/MichalDanielDobrzanski/DeepLearningPython
	DATA_DIRPATH = f"/Users/amoogle/Documents/Git/neural_network/DeepLearningPython/"
	DATA_FILENAME = "mnist.pkl.gz"
	DATA_FILEPATH = f"{DATA_DIRPATH}/{DATA_FILENAME}"

	# Learning and Network
	EPOCHS = 4
	BATCH_SIZE = 10
	ETA = 3.0
	HIDDEN_LAYER_SIZES = [10, 20, 10] # array of ints

	# Misc for logging and evaluation
	SOME_INDEX = 98


	class Image(IOPair):
	IMAGE_WIDTH = 28
	NUM_PIXELS = IMAGE_WIDTH ** 2
	NUMBER_BASE = 10

	def __init__(self, image_and_gt_pair):
	"""
	Class holding an array of grayscale values (=the networks input layer
	format) as well as the ground truth value digit displayed on that image
	"""
	self.__pixels = image_and_gt_pair[0]
	self.__gt_label = image_and_gt_pair[1] # Classification/Label

	def __to_prediction_digit(self, activation_prediction):
	"""
	Activation_prediction is an arraw of floats while the return value is an index into that list.
	E.g. [[9.89989735e-03] [1.08126136e-02] [6.17035645e-04] [1.14060381e-02] [5.98185829e-03]
	[7.31337324e-03] [1.61992979e-05] [9.24512441e-01] [9.88453143e-05] [8.47158033e-02]]
	\mapsto 7
	as '9.24512441e-01' is the biggest value
	"""
	return np.argmax(activation_prediction)

	def __digit_to_output_format(self, d_in):
	"""
	E.g.
	7 \mapsto [0, 0, 0, 0, 0, 0, 0, 1, 0, 0]
	"""
	return [[int(d_in==d)] for d in range(self.NUMBER_BASE)]

	def _get_input(self):
	return self.__pixels

	def _output_ground_truth(self):
	return self.__digit_to_output_format(self.__gt_label)

	def _to_output_format(self, activation_prediction):
	pd = self.__to_prediction_digit(activation_prediction)
	return self.__digit_to_output_format(pd)

	def _show(self, activation_prediction) -> None:
	"""
	Print the grayscale image as ascii image.
	And optionally compare the prediction to a ground turth value.
	"""
	GRAYSCALE_ASCII = " .',_-+*$#"
	HORIZONTAL_LINE: str = 2 * self.IMAGE_WIDTH * "_"

	# Draw image
	print(HORIZONTAL_LINE) # top of bounding box
	for idx, pixel in enumerate(self.__pixels):
	end_of_row = (idx + 1) % self.IMAGE_WIDTH == 0
	char = GRAYSCALE_ASCII[int(10 * pixel)]
	print(char, end=" \|\n" if end_of_row else " ")
	print(HORIZONTAL_LINE + "\|") # bottom of bounding box

	print(f"[show] Ground truth value is {self.__gt_label}.")
	if self.correct(activation_prediction):
	print("[show] And this was predicted correctly. :)")
	else:
	print(f"[show] And this was wrongly predicted as predicted as {activation_prediction}. :(")


	def load_MNIST_data__and_cast():
	"""
	Load the pairs of grayscale image array and ground truth for training and test data. The files are found here:
	https://github.com/MichalDanielDobrzanski/DeepLearningPython35/blob/master/mnist.pkl.gz
	This holds 3 data sets, but we won't use the validation one.
	:return: Images.
	"""
	print(f"[load_MNIST_data] Trying to load data from <{Config.DATA_FILEPATH}>")
	with gzip.open(Config.DATA_FILEPATH, 'rb') as in_file:
	training_data_gz, _validation_data_gz, test_data_gz = pickle.load(in_file, encoding="latin1") # Load the "raw" data sets
	print(f"[load_MNIST_data] Data loading successful. len(training_data)={len(training_data_gz)}. len(test_data)={len(test_data_gz)}")

	def reshape_and_zip(all_images__and__all_gt_labels):
	all_images = all_images__and__all_gt_labels[0]
	assert all(len(img)==Image.NUM_PIXELS for img in all_images)

	all_gt_labels = all_images__and__all_gt_labels[1]
	assert all(num in range(10) for num in all_gt_labels)
	assert len(all_images)==len(all_gt_labels)

	SHAPE = (Image.NUM_PIXELS, 1)
	all_images_ = (np.reshape(img, SHAPE) for img in all_images)
	return zip(all_images_, all_gt_labels)

	def cast_all_to_image(all_images__and__all_gt_labels):
	return [Image(image_and_gt_pair) for image_and_gt_pair in reshape_and_zip(all_images__and__all_gt_labels)]

	return cast_all_to_image(training_data_gz), cast_all_to_image(test_data_gz)


	def run_main():
	"""
	Unpack data, train and do some examples.
	"""
	# Set up network
	INPUT_LAYER_SIZE = Image.NUM_PIXELS
	OUTPUT_LAYER_SIZE = Image.NUMBER_BASE
	net = NeuralNetwork(
	INPUT_LAYER_SIZE,
	Config.HIDDEN_LAYER_SIZES,
	OUTPUT_LAYER_SIZE
	)

	# Train
	training_data, test_data = load_MNIST_data__and_cast()
	print(f"\n[run_main] Starting the training with {Config.EPOCHS} epochs...")
	net.run_sgd(
	Config.EPOCHS,
	Config.BATCH_SIZE,
	Config.ETA,
	training_data
	)

	# Evaluate
	print("[run_main] " + 40 * "*" + " Let's predict some example_imgs.")
	for some_other_index in [Config.SOME_INDEX * n for n in range(1, 8)]:
	img = test_data[some_other_index]
	_is_correct: bool = net.predicts_correctly(img, show=True)
	time.sleep(1) # Just for clarity


	if __name__=='__main__':
	run_main()