"""G13: one-hidden-layer tanh network, flat parameters, analytic backpropagation."""
import math
import random
try:
from .ml import validate_xy, sigmoid, logit_loss
except ImportError:
from ml import validate_xy, sigmoid, logit_loss
def initialize(d=2, h=2, seed=7):
if type(d) is not int or type(h) is not int or not 1 <= d <= 64 or not 1 <= h <= 32:
raise ValueError("bounded positive layer dimensions required")
rng = random.Random(seed)
W = [rng.uniform(-1,1)/math.sqrt(d) for _ in range(h*d)]
return W + [0.]*h + [rng.uniform(-1,1)/math.sqrt(h) for _ in range(h)] + [0.]
def loss_grad(theta, X, y, h=2, lam=0.):
n,d = validate_xy(X,y)
if type(h) is not int or not 1 <= h <= 32 or len(theta) != h*d+2*h+1:
raise ValueError("parameter vector does not match network shape")
if not all(math.isfinite(v) for v in [*theta,lam]) or lam < 0:
raise ValueError("finite parameters and nonnegative regularization required")
bstart, vstart = h*d, h*d+h
grad, loss = [0.]*len(theta),0.
for x,target in zip(X,y):
a = [math.tanh(math.fsum(theta[j*d+k]*x[k] for k in range(d))
+theta[bstart+j]) for j in range(h)]
z = math.fsum(theta[vstart+j]*a[j] for j in range(h))+theta[-1]
loss += logit_loss(z,target)/n
dz = (sigmoid(z)-target)/n
grad[-1] += dz
for j in range(h):
grad[vstart+j] += dz*a[j]
du = dz*theta[vstart+j]*(1-a[j]*a[j])
grad[bstart+j] += du
for k in range(d):
grad[j*d+k] += du*x[k]
for i in [*range(h*d), *range(vstart,vstart+h)]:
loss += .5*lam*theta[i]*theta[i]
grad[i] += lam*theta[i]
if not all(math.isfinite(v) for v in [loss,*grad]):
raise ValueError("nonfinite network calculation")
return loss,grad
def gradient_check(theta, X, y, h=2, lam=.03, step=1e-6):
if not math.isfinite(step) or step <= 0:
raise ValueError("positive finite difference step required")
_,g = loss_grad(theta,X,y,h,lam)
errors = []
for i in range(len(theta)):
a,b = list(theta),list(theta)
a[i] += step
b[i] -= step
numerical = (loss_grad(a,X,y,h,lam)[0]-loss_grad(b,X,y,h,lam)[0])/(2*step)
errors.append(abs(numerical-g[i])/max(1.,abs(numerical),abs(g[i])))
return max(errors)
if __name__ == "__main__":
X,y = [[0.,0.],[0.,1.],[1.,0.],[1.,1.]],[0,1,1,0]
theta = initialize()
print("scaled gradient discrepancy",gradient_check(theta,X,y))
before = loss_grad(theta,X,y)[0]
for _ in range(2500):
_,g = loss_grad(theta,X,y)
theta = [v-.2*dv for v,dv in zip(theta,g)]
print("XOR training loss", before, loss_grad(theta,X,y)[0])