# Classification - Backpropagation, Softmax

from ctypes.wintypes import RGB
from unittest import skip
import numpy as np
import matplotlib.pyplot as plt
from math import pi
from math import sqrt
from math import sin
from math import cos
import random

POINT_N = 300

def make_data(N):
    d = 0.10
    x = []
    y = []
    for n in range(N):
        n = random.randint(0,3)
        if n==0:
            y.append([1,0,0])
            f = random.uniform(0.0, pi)
            x.append([-0.25+0.5*cos(f)+random.gauss(mu=0.0, sigma=d), 0.0+0.5*sin(f)+random.gauss(mu=0.0, sigma=d)])
        elif n==1:
            y.append([0,1,0])
            f = random.uniform(pi, 2*pi)
            x.append([0.25+0.5*cos(f)+random.gauss(mu=0.0, sigma=d), 0.0+0.5*sin(f)+random.gauss(mu=0.0, sigma=d)])
        else:
            y.append([0,0,1])
            f = random.uniform(0.0, 2*pi)
            x.append([0.0+1.25*cos(f)+random.gauss(mu=0.0, sigma=d), 0.0+1.0*sin(f)+random.gauss(mu=0.0, sigma=d)])
    return x,y

train_x,train_y = make_data(POINT_N)
test_x,test_y = make_data(POINT_N)

plt.figure(figsize=(9,9))
for n in range(POINT_N):
    plt.scatter(test_x[n][0], test_x[n][1], color = [test_y[n][0], test_y[n][1], test_y[n][2]], marker='o')
plt.show()

DIM_N = 2
LAYER_N = 2
NEURON_N = [7, 3]           # neurons for layer

A = 3.0
def Sigma(x):
    return 1.0 / (1.0 + np.exp(-A*x))

A_delta = 0.5
def Delta(x):
    return np.exp(-A_delta*x*x)

A_soft = 10.0
def Soft(x):
    max_x = np.max(x)
    return np.exp(A_soft*(x-max_x))

np.random.seed(3)
class LAYER:
    def __init__(self, n_neurons, n_input):
        self.n_neurons = n_neurons
        self.n_input = n_input
        self.w = np.random.rand(self.n_neurons,self.n_input)
        self.dw = np.zeros([self.n_neurons,self.n_input], dtype=float)
        self.b = np.zeros(self.n_neurons, dtype=float)
        self.db = np.zeros(self.n_neurons, dtype=float)
    def forward(self, inputs):
        return np.dot(self.w, inputs) + (self.b).T
    def forward_add(self, inputs):
        return np.dot(self.w+self.dw, inputs) + (self.b+self.db).T

Layer = [LAYER(NEURON_N[0], DIM_N)]
for l in range(1, LAYER_N):
    Layer.append(LAYER(NEURON_N[l], NEURON_N[l-1]))

xx = [np.empty(DIM_N, dtype=float)]
xx.extend([np.empty(Layer[l].n_neurons, dtype=float) for l in range(LAYER_N)])

U = []
def Run(data):
    xx[0] = data
    for l in range(1,LAYER_N):
        xx[l] = Sigma(Layer[l-1].forward(xx[l-1]))
    xx[LAYER_N] = Soft(Layer[LAYER_N-1].forward(xx[LAYER_N-1]))
    norm = 0.0
    for n in range(Layer[LAYER_N-1].n_neurons):
        norm += xx[LAYER_N][n]
    xx[LAYER_N] = xx[LAYER_N]/norm

    U.clear()
    for l in range(LAYER_N-1):
        u = np.empty([Layer[l].n_neurons,Layer[l].n_input], dtype=float)
        for i in range(Layer[l].n_neurons):
            for j in range(Layer[l].n_input):
                u[i][j] = Layer[l].w[i][j]*A*xx[l][j]*(1-xx[l][j])
        U.append(u)
    l = LAYER_N-1
    u = np.empty([Layer[l].n_neurons,Layer[l].n_input], dtype=float)
    for i in range(Layer[l].n_neurons):
        for j in range(Layer[l].n_input):
            u[i][j] = Layer[l].w[i][j]*A_soft*xx[l][j]*(1-xx[l][j])
    U.append(u)

# Backward propagation
def Backward(t):
#    n = 0
    for n in range(Layer[LAYER_N-1].n_neurons):
        Layer[LAYER_N-1].db[n] += (xx[LAYER_N][n]-t[n])*A_delta*Delta(xx[LAYER_N][n]-t[n])*A*xx[LAYER_N][n]*(1.0-xx[LAYER_N][n])
        for m in range(Layer[LAYER_N-1].n_input):
            Layer[LAYER_N-1].dw[n][m] += Layer[LAYER_N-1].db[n]*xx[LAYER_N-1][m]

        for l in range(1,LAYER_N):
            for k in range(Layer[LAYER_N-l-1].n_neurons):
                Layer[LAYER_N-l-1].db[k] += np.dot(Layer[LAYER_N-l].db,U[LAYER_N-l])[k]
                for m in range(Layer[LAYER_N-l-1].n_input):
                    Layer[LAYER_N-l-1].dw[k][m] += Layer[LAYER_N-l-1].db[k]*xx[LAYER_N-l-1][m]

def Slope(h):
    for l in range(LAYER_N):
        for i in range(Layer[l].n_neurons):
            Layer[l].b[i] -= h*Layer[l].db[i]
            for j in range(Layer[l].n_input):
                Layer[l].w[i][j] -= h*Layer[l].dw[i][j]


eps = 0.01
F_ERR = 1.0e+99
H = 0.1
iter = 0
while iter<1000:
    iter += 1
    err = 0
    for n in range(POINT_N):
        Run(train_x[n])
        for l in range(LAYER_N):
            Layer[l].db.fill(0.0)
            Layer[l].dw.fill(0.0)
        Backward(train_y[n])
        Slope(H)
        for i in range(Layer[LAYER_N-1].n_neurons):
            err += Delta(xx[LAYER_N][i]-train_y[n][i])
    F_ERR = err/POINT_N/Layer[LAYER_N-1].n_neurons
    print("*** ", 1.0-F_ERR)
    if 1.0-F_ERR < eps:
        break

for i in range(POINT_N):
    Run(test_x[i])
    print(test_y[i], ' - ', "%5.3F" % xx[LAYER_N][0], "%5.3F" % xx[LAYER_N][1], "%5.3F" % xx[LAYER_N][2])

err = 0.0
plt.figure(figsize=(9,9))
for n in range(POINT_N):
    Run(test_x[n])
    for i in range(Layer[LAYER_N-1].n_neurons):
        err += Delta(xx[LAYER_N][i]-test_y[n][i])
    plt.scatter(test_x[n][0], test_x[n][1], color = [xx[LAYER_N][0], xx[LAYER_N][1], xx[LAYER_N][2]], marker='o')
plt.show()

print("iter=", iter)
err = err/POINT_N/Layer[LAYER_N-1].n_neurons
print("err=", "%5.3F"%(1.0-err))