tensorflow_nn.py

import tensorflow as tf
import numpy as np
import pandas as pd

class ReluDense(tf.Module):
    def __init__(self, in_features, out_features, name=None):
        super().__init__(name=name)
        self.w = tf.Variable(
            tf.random.normal([out_features, out_features]), name='w'
        )
        self.b = tf.Variable(
            tf.zeros([out_features]), name='b'
        )
    
    def __call__(self, x):
        y = tf.matmul(x, self.w) + self.b
        return tf.nn.relu(y)

class Dense(tf.Module):
    def __init__(self, in_features, out_features, name=None):
        super().__init__(name=name)
        self.w = tf.Variable(
            tf.random.normal([out_features, out_features]), name='w'
        )
        self.b = tf.Variable(
            tf.zeros([out_features]), name='b'
        )
    
    def __call__(self, x):
        return tf.matmul(x, self.w) + self.b
    
class NeuralNet(Model):

    def __init__(self, X_in, X_out, optimizer):
        super(NeuralNet, self).__init__()
        self.layer_one = ReluDense(X_in, X_out)
        self.layer_two = Dense(X_in, X_out)
        self.layer_three = Dense(X_in, X_out)
        self.out = Dense(X_in, X_out)
        self.optimizer = optimizer

    def call(self, x, is_training=False):
        x = self.layer_one(x)
        x = self.out(x)
        return tf.reduce_mean(x, axis=1)

    def step(self, x, y):
        with tf.GradientTape() as tape:
            pred = neural_net(x, is_training=True)
            loss = loss_func(pred, y)

        gradients = tape.gradient(loss, neural_net.trainable_variables)
        self.optimizer.apply_gradients(zip(gradients, neural_net.trainable_variables))

def loss_func(x, y):
    x = tf.cast(x, tf.float64)
    y = tf.cast(y, tf.float64)
    return tf.metrics.MSE(x, y)

def run_optimization(x, y):

    with tf.GradientTape() as tape:
        pred = neural_net(x, is_training=True)
        loss = loss_func(pred, y)

    gradients = tape.gradient(loss, neural_net.trainable_variables)
    optimizer.apply_gradients(zip(gradients, neural_net.trainable_variables))

if __name__ == '__main__':
    
    X = pd.read_csv("X.csv")
    y = np.load("y.npy")
    X = X.values
    X_val = pd.read_csv("X_val.csv")
    X_val = X_val.values
    y_val = np.load("y_val.npy")
    #X = np.random.normal(0, 10, size=1000).reshape(100, 10)
    #y = np.random.normal(10, 100, size=100)
    final_losses = []
    for _ in range(1000):
        optimizer = tf.optimizers.Adam(learning_rate=0.5)
        neural_net = NeuralNet(X.shape[0], X.shape[1], optimizer)

        for step in range(1000):
            neural_net.step(X, y)

            pred = neural_net(X_val, is_training=True)
            loss = loss_func(pred, y_val)

        final_losses.append(loss.numpy())
    print(min(final_losses))