ai-lab/m01_autograd/nn.py

"""AI01 lab, step 7: neurons, layers, and a multilayer perceptron built on your engine.

Everything here is ordinary Python on top of Value. There is no new calculus:
your engine already knows how to differentiate whatever these classes compute.
"""
import random

from engine import Value


class Module:
    """Base class for anything that has trainable parameters."""

    def parameters(self):
        """Return a list of every trainable Value. The base class has none."""
        raise NotImplementedError("Step 7: Module.parameters")

    def zero_grad(self):
        """Set the grad of every parameter back to 0.0."""
        raise NotImplementedError("Step 7: Module.zero_grad")


class Neuron(Module):
    """One neuron: a weighted sum of its inputs, plus a bias, then (optionally) tanh.

    Neuron(nin, nonlin=True)
      nin     number of inputs.
      nonlin  True applies tanh to the sum; False returns the sum itself (a
              linear neuron, used in the network's last layer).

    Start each weight at random.uniform(-1, 1) and the bias at 0.0.
    """

    def __init__(self, nin, nonlin=True):
        raise NotImplementedError("Step 7: Neuron.__init__")

    def __call__(self, x):
        """x is a list of nin numbers (floats or Values). Return one Value:
        tanh(w0*x0 + w1*x1 + ... + b), or the plain sum when nonlin is False."""
        raise NotImplementedError("Step 7: Neuron.__call__")

    def parameters(self):
        """Return [w0, w1, ..., w_{nin-1}, b], the live Value objects, in that order."""
        raise NotImplementedError("Step 7: Neuron.parameters")


class Layer(Module):
    """nout neurons that all read the same inputs.

    Layer(nin, nout, **kwargs). Pass **kwargs (such as nonlin=False) on to each Neuron.
    """

    def __init__(self, nin, nout, **kwargs):
        raise NotImplementedError("Step 7: Layer.__init__")

    def __call__(self, x):
        """Return a list with one output per neuron, or a single Value when nout == 1."""
        raise NotImplementedError("Step 7: Layer.__call__")

    def parameters(self):
        """Every neuron's parameters, neuron by neuron, in order."""
        raise NotImplementedError("Step 7: Layer.parameters")


class MLP(Module):
    """A multilayer perceptron: layers applied one after another.

    MLP(nin, nouts), where nouts lists each layer's width. MLP(2, [4, 4, 1])
    reads 2 numbers and has hidden layers of 4 and 4 neurons, then 1 output.
    Hidden layers use tanh; the LAST layer is linear (nonlin=False), so the
    output can be any real number.
    """

    def __init__(self, nin, nouts):
        raise NotImplementedError("Step 7: MLP.__init__")

    def __call__(self, x):
        """Feed x through every layer in order and return the last layer's output.

        Careful: a hidden layer of width 1 returns a single Value, but the next
        layer expects a list of inputs.
        """
        raise NotImplementedError("Step 7: MLP.__call__")

    def parameters(self):
        """Every layer's parameters, layer by layer, in order."""
        raise NotImplementedError("Step 7: MLP.parameters")