What is Backpropagation?

import numpy as np
import wandb 

wandb.init(project="backprop", entity="byyoung3")

# Inputs (4 samples, 3 features each)
X = np.array([[0.1, 0.2, 0.7],
              [0.3, 0.7, 0.2],
              [0.6, 0.9, 0.1],
              [0.5, 0.3, 0.8]])

# Expected outputs (4 samples, 1 output each)
y = np.array([[1], [0], [1], [0]])

# Define the sigmoid activation function and its derivative
def sigmoid(x):
    return 1 / (1 + np.exp(-x))

def sigmoid_prime(x):
    return x * (1 - x)

# Initialize weights for the 2-layer network
inputSize = 3 # Number of input neurons (features)
hiddenSize = 4 # Number of hidden neurons
outputSize = 1 # Number of output neurons

# Weights for input to hidden layer
W1 = np.random.uniform(size=(inputSize, hiddenSize))
# Weights for hidden to output layer
W2 = np.random.uniform(size=(hiddenSize, outputSize))

# Training through forward and backward passes
for epoch in range(1000000): # Number of training iterations
    ##### Forward pass
    v1 = np.dot(inputs, W1)
    y1 = sigmoid(v1)

    v2 = np.dot(y1, W2)
    y2 = sigmoid(v2)

    ##### Backward pass
    # Error in output
    output_error = targets - y2
    output_delta = output_error * sigmoid_prime(y2)
    dEdW2 = y1.T.dot(output_delta)

    hidden_error = output_delta.dot(W2.T) # recursive calculation of hidden error using error from output layer 
    hidden_delta = hidden_error * sigmoid_prime(y1)
    dEdW1 = inputs.T.dot(hidden_delta)

    # Weight updates
    W2 += dEdW2 * learning_rate
    W1 += dEdW1 * learning_rate

    # Print loss metric
    print(0.5 * sum(np.square(output_error)) / len(output_error))
    if not epoch % 10: 
    # Log to Weights & Biases
        wandb.log({'loss': sum(np.square(output_error))/len(output_error)},step=epoch)
wandb.finish()

import numpy as np
import torch
import torch.nn as nn
import wandb

# Initialize Weights and Biases logging
wandb.init(project="bp_verification", entity="byyoung3")

# Sample inputs and expected outputs
inputs = np.array([[0.1, 0.2, 0.7],
                   [0.3, 0.7, 0.2],
                   [0.6, 0.9, 0.1],
                   [0.5, 0.3, 0.8]])
targets = np.array([[1], [0], [1], [0]])

# Sigmoid and its derivative
def sigmoid(z):
    return 1 / (1 + np.exp(-z))

def sigmoid_prime(a):
    return a * (1 - a)

# Network architecture
num_inputs = 3
num_hidden = 4
num_outputs = 1

# Weight initialization
W1 = np.random.uniform(size=(num_inputs, num_hidden))
W2 = np.random.uniform(size=(num_hidden, num_outputs))

# Convert numpy arrays to torch tensors
inputs_torch = torch.tensor(inputs, dtype=torch.float32)
targets_torch = torch.tensor(targets, dtype=torch.float32)
W1_torch = torch.tensor(W1, dtype=torch.float32, requires_grad=True)
W2_torch = torch.tensor(W2, dtype=torch.float32, requires_grad=True)

# Neural Network in PyTorch
class NeuralNetwork(nn.Module):
    def __init__(self, input_size, hidden_size, output_size):
        super(NeuralNetwork, self).__init__()
        self.hidden = nn.Linear(input_size, hidden_size)
        self.output = nn.Linear(hidden_size, output_size)
        self.sigmoid = nn.Sigmoid()
        
    def forward(self, x):
        x = self.hidden(x)
        x = self.sigmoid(x)
        x = self.output(x)
        x = self.sigmoid(x)
        return x

# Initialize the neural network
net = NeuralNetwork(num_inputs, num_hidden, num_outputs)

# Copy weights from numpy to torch model
net.hidden.weight.data = torch.tensor(W1.T, dtype=torch.float32)
net.hidden.bias.data = torch.zeros(num_hidden, dtype=torch.float32)
net.output.weight.data = torch.tensor(W2.T, dtype=torch.float32)
net.output.bias.data = torch.zeros(num_outputs, dtype=torch.float32)

# Loss and optimizer
criterion = nn.MSELoss()
optimizer = torch.optim.SGD(net.parameters(), lr=0.1)

# Training loop for NumPy network
for epoch in range(100): # Reduced number of iterations for example
    # Forward pass
    v1 = np.dot(inputs, W1)
    y1 = sigmoid(v1)
    v2 = np.dot(y1, W2)
    y2 = sigmoid(v2)

    # Compute error
    output_error_np = targets - y2
    # loss_np = 0.5 * np.mean(np.square(output_error_np))
    # we drop the .5 to remain consisent with torch implementation
    # this is fine in practice, since it's a constant scalar value 
    loss_np = np.mean(np.square(output_error_np)) 

    # Backward pass
    output_delta = output_error_np * sigmoid_prime(y2)
    dEdW2 = np.dot(y1.T, output_delta)
    hidden_error = np.dot(output_delta, W2.T)
    hidden_delta = hidden_error * sigmoid_prime(y1)
    dEdW1 = np.dot(inputs.T, hidden_delta)

    # Update weights
    W1 += dEdW1 * 0.1
    W2 += dEdW2 * 0.1
    if not epoch % 10: 
    # Log to Weights & Biases
        wandb.log({'loss_np': loss_np}, step=epoch)

wandb.finish()

wandb.init(project="bp_verification", entity="byyoung3")

# Training loop for PyTorch network
for epoch in range(100): # Reduced number of iterations for example
    # Forward pass
    outputs_torch = net(inputs_torch)
    
    # Compute loss
    loss_torch = criterion(outputs_torch, targets_torch)

    # Backward pass and optimization
    optimizer.zero_grad()
    loss_torch.backward()
    optimizer.step()

    # Log to Weights & Biases
    
    if not epoch % 10: 
    # Log to Weights & Biases
        wandb.log({'loss_torch': loss_torch.item()},step=epoch)


# Finish Weights & Biases logging session
wandb.finish()