basic-autograd.md

Basic Autograd

This example covers the core automatic differentiation API.

Scalar Gradients

import numpy as np
from MiniTorch.core.variable import Variable

a = Variable(np.array(2.0))
b = Variable(np.array(3.0))

# f(a, b) = a^2 + a*b
# df/da = 2a + b = 7
# df/db = a = 2
f = a ** 2 + a * b
f.backward()

print(a.grad.data)  # 7.0
print(b.grad.data)  # 2.0

Vector Gradients

x = Variable(np.array([1.0, 2.0, 3.0]))
y = (x ** 2).sum()   # y = sum(x_i^2),  dy/dx_i = 2*x_i
y.backward()

print(x.grad.data)  # [2. 4. 6.]

Matrix Operations

X = Variable(np.random.randn(4, 3))
W = Variable(np.random.randn(3, 2))

Z = X @ W          # (4, 2)
loss = (Z ** 2).sum()
loss.backward()

print(X.grad.shape)  # (4, 3)
print(W.grad.shape)  # (3, 2)

Chain Rule Through Multiple Ops

from MiniTorch.ops.relu import relu
from MiniTorch.ops.sigmoid import sigmoid

x = Variable(np.linspace(-2, 2, 5))

h = relu(x)
y = sigmoid(h)
loss = y.sum()
loss.backward()

print(x.grad.data)

Verifying Gradients Numerically

from MiniTorch import numerical_diff
from MiniTorch.ops.relu import relu

x = Variable(np.random.randn(3, 3))
approximate = numerical_diff(relu, x)

Inference Mode (no_grad)

from MiniTorch.core.config import no_grad

with no_grad():
    # No graph is built — faster and lower memory
    out = (x ** 2).sum()

print(out.creator)  # None