Training loss increases instead of decrease with epochs-CodePudding

I am developing from scratch my first feed-forward fully-connected ANN based on batch learning mode on a toy training set. I am using back-propagation for calculating the gradient of the loss function with respect to weights and biases and using the gradient descent method as a learning rule. But when I print the training loss, it gets bigger as the epoch increases:

E(0) on TrS is: [[7.83898769]]
E(1) on TrS is: [[10.00738465]]
E(2) on TrS is: [[10.76653098]]
E(3) on TrS is: [[15.94001008]]
E(4) on TrS is: [[23.80650667]]
E(5) on TrS is: [[28.65805023]]
E(6) on TrS is: [[29.56550719]]
E(7) on TrS is: [[30.5424694]]
E(8) on TrS is: [[34.26980112]]
E(9) on TrS is: [[39.9948856]]

This is my loss_functions.py file:

import numpy as np

def sum_of_squares(c, t, y, derivative=False):
    ret = 0
    for k in range(c):
        ret  = np.square(y - t)
    ret = 1 / 2 * ret
    if derivative:
        return y - t
    return ret

this is my activation_functions.py file:

import numpy as np


def sigmoid(a, derivative=False):
    f_a = 1 / (1   np.exp(-a))
    df_a = np.multiply(f_a, (1 - f_a)) 
    if derivative:
        return df_a
    return f_a


def identity(a, derivative=False):
    f = a
    df = np.ones(np.shape(a))
    if derivative:
        return df
    return f

and this is the main.py file:

from activation_functions import *
from loss_functions import *


class NeuralNetwork:

    def _init_(self):
        self.layers = []

    def add_layer(self, layer):
        self.layers.append(layer)

    def create(self):
        for i, layer in enumerate(self.layers):
            if i == 0:
                layer.type = "input"
            else:
                if i == len(self.layers) - 1:
                    layer.type = "output"
                else:
                    layer.type = "hidden"
                layer.configure(self.layers[i - 1].neurons)

    def train(self, X, targets):
        MAX_EPOCHS = 10
        loss_function = sum_of_squares
        E = 0  # errore sull'intero DS
        for epoch in range(MAX_EPOCHS):
            for i, x in enumerate(X):
                target = targets[i]
                prediction = self.forward_prop(x.T)
                E_n = loss_function(c, target, prediction)
                E  = E_n  
                self.back_prop(target, local_loss=sum_of_squares)
            print("E(%d) on TrS is:" % epoch, E)  # increasing!!!
            self.learning_rule(l_rate=0.05)  

    def forward_prop(self, z):
        for layer in self.layers:
            z = layer.forward_prop_step(z)
        return z

    def back_prop(self, target, local_loss):
        for i, layer in enumerate(self.layers[:0:-1]):
            next_layer = self.layers[-i]  
            prev_layer = self.layers[-i - 2] 
            layer.back_prop_step(next_layer, prev_layer, target, local_loss)

    def learning_rule(self, l_rate):
        # GD
        for layer in self.layers:
            if layer.type != "input":
                layer.weight -= l_rate * layer.dE_dW
                layer.bias -= l_rate * layer.dE_db


class Layer:

    def _init_(self, neurons, type=None, activation=None):
        self.dE_dW = 0 
        self.dE_db = 0
        self.dEn_db = None  # based on the n-th item
        self.dEn_dW = None  # based on the n-th item
        self.dact_a = None  
        self.out = None
        self.weight = None  
        self.bias = None 
        self.w_sum = None  
        self.neurons = neurons 
        self.type = type  
        self.activation = activation 
        self.deltas = None  

    def configure(self, prev_layer_neurons):
        self.weight = np.asmatrix(np.random.normal(0, 0.5, (self.neurons, prev_layer_neurons)))
        self.bias = np.asmatrix(np.random.normal(0, 0.5, self.neurons)).T 
        if self.activation is None:
            if self.type == "hidden":
                self.activation = sigmoid
            elif self.type == "output":
                self.activation = identity

    def forward_prop_step(self, z):
        if self.type == "input":
            self.out = z
        else:
            self.w_sum = np.dot(self.weight, z)   self.bias
            self.out = self.activation(self.w_sum)
        return self.out

    def back_prop_step(self, next_layer, prev_layer, target, local_loss):
        if self.type == "input":
            pass
        elif self.type == "output":
            self.dact_a = self.activation(self.w_sum, derivative=True) 
            self.deltas = np.multiply(self.dact_a, local_loss(c, target, self.out, derivative=True)) 
        else:
            self.dact_a = self.activation(self.w_sum, derivative=True)  
            self.deltas = np.multiply(self.dact_a, np.dot(next_layer.weight.T, next_layer.deltas))

        self.dEn_dW = np.dot(self.deltas, prev_layer.out.T)

        self.dEn_db = self.deltas

        self.dE_dW  = self.dEn_dW

        self.dE_db  = self.dEn_db


if _name_ == '_main_':
    net = NeuralNetwork() 

    for m in (2, 4, 4, 1):
        net.add_layer(Layer(m))

    net.create()

    X = np.asmatrix([
        [1, 0],
        [1, 1],
        [0, 1],
        [0, 0]
    ])

    targets = np.asarray([1, 0, 0, 0])

    net.train(X, targets)

What I did for trying to fix the problem is:

Check for any bug
Decrease the learning rate (l_rate)
Increase MAX_EPOCHS value
Replace - symbol to in GD formula

Unfortunately none of these worked. There must be a hidden bug somewhere in the code...

How can I resolve the issue?

CodePudding user response：

Your task is about classification meanwhile you have chose the sum-of-squares loss function which works properly for regression tasks. You should use the cross-entropy loss function, instead.