# approximation function - Backpropagation

from unittest import skip
import numpy as np
import matplotlib.pyplot as plt
#import pandas as pd
from sklearn import datasets
#from sklearn import preprocessing
#from math import exp
from math import sqrt
import random
#from math import sin

POINT_N = 300
data,target = datasets.make_moons (POINT_N, noise=0.12, random_state=np.random.seed(1))
test,y = datasets.make_moons (POINT_N, noise=0.12, random_state=np.random.seed(2))
DIM_N = 2

LAYER_N = 3
NEURON_N = [5, 4, 1]           # neurons for layer

##LAYER_N = 2
#NEURON_N = [6, 1]           # neurons for layer

plt.figure(figsize=(9,9))
for n in range(POINT_N):
    plt.scatter(data[n,0], data[n,1], c = ('blue' if target[n] else 'green'), marker='o')
plt.show()

#print (data)

A = 3.0
def Sigma(x):
    return 1.0 / (1.0 + np.exp(-A*x))

np.random.seed(3)
class LAYER:
    def __init__(self, n_neurons, n_input):
        self.n_neurons = n_neurons
        self.n_input = n_input
        self.w = np.random.rand(self.n_neurons,self.n_input) - 0.5
        self.dw = np.zeros([self.n_neurons,self.n_input], dtype=float)
        self.b = np.zeros(self.n_neurons, dtype=float)
        self.db = np.zeros(self.n_neurons, dtype=float)
    def forward(self, inputs):
        return np.dot(self.w, inputs) + (self.b).T
    def forward_add(self, inputs):
        return np.dot(self.w+self.dw, inputs) + (self.b+self.db).T

Layer = [LAYER(NEURON_N[0], DIM_N)]
for l in range(1, LAYER_N):
    Layer.append(LAYER(NEURON_N[l], NEURON_N[l-1]))

xx = [np.empty(DIM_N, dtype=float)]
xx.extend([np.empty(Layer[l].n_neurons, dtype=float) for l in range(LAYER_N)])

U = []
def Run(data):
    xx[0] = data
    for l in range(1,LAYER_N+1):
        xx[l] = Sigma(Layer[l-1].forward(xx[l-1]))
    U.clear()
    for l in range(LAYER_N):
        u = np.empty([Layer[l].n_neurons,Layer[l].n_input], dtype=float)
        for i in range(Layer[l].n_neurons):
            for j in range(Layer[l].n_input):
                u[i][j] = Layer[l].w[i][j]*A*xx[l][j]*(1-xx[l][j])
        U.append(u)

# Backward propagation
def Backward(t):
    Layer[LAYER_N-1].db[0] = (xx[LAYER_N][0]-t)*A*xx[LAYER_N][0]*(1.0-xx[LAYER_N][0])
    for m in range(Layer[LAYER_N-1].n_input):
        Layer[LAYER_N-1].dw[0][m] = Layer[LAYER_N-1].db[0]*xx[LAYER_N-1][m]

    for l in range(1,LAYER_N):
        for k in range(Layer[LAYER_N-l-1].n_neurons):
            Layer[LAYER_N-l-1].db[k] = np.dot(Layer[LAYER_N-l].db,U[LAYER_N-l])[k]
            for m in range(Layer[LAYER_N-l-1].n_input):
                Layer[LAYER_N-l-1].dw[k][m] = Layer[LAYER_N-l-1].db[k]*xx[LAYER_N-l-1][m]

def Slope(h):
    for l in range(LAYER_N):
        for i in range(Layer[l].n_neurons):
            Layer[l].b[i] -= h*Layer[l].db[i]
            for j in range(Layer[l].n_input):
                Layer[l].w[i][j] -= h*Layer[l].dw[i][j]


eps = 0.01
F_ACC = 0.0
H = 0.3
iter = 0
while iter<300:
    iter += 1
    acc = 0
    for n in range(POINT_N):
        Run(data[n])
        Backward(target[n])
        Slope(H)
        acc += (target[n] == ( 1 if xx[LAYER_N][0] > 0.5 else 0))
    F_ACC = acc/POINT_N
    print("*** ", F_ACC)
    if F_ACC > 1.0-eps:
        break

for i in range(POINT_N):
    Run(data[i])
    print("%5.3F" % target[i], ' - ', ' - ', "%5.3F" % xx[LAYER_N][0])

acc = 0.0
plt.figure(figsize=(9,9))
for n in range(POINT_N):
    Run(test[n])
    res = (0 if xx[LAYER_N][0]<0.5 else 1)
    acc += (res==y[n])
    plt.scatter(test[n,0], test[n,1], c = ('blue' if res else 'green'), marker='o')
plt.show()

print("iter=", iter)
acc /= POINT_N
print("acc=", acc)