Decoding Backpropagation and Its Role in Neural Network Learning

import wandb
wandb.login()

import numpy as np
from sklearn import datasets
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import accuracy_score
import matplotlib.pyplot as plt

iris = datasets.load_iris()
X = iris.data[:100]  # We only take the first two features.
y = iris.target[:100]  # We only take two classes.

# Create a scatter plot
plt.scatter(X[:, 0], X[:, 1], c=y)
plt.xlabel('Feature 1')
plt.ylabel('Feature 2')
plt.title('Iris Dataset (2 classes)')
plt.show()

# Data shape
X.shape, y.shape
# unique class in y
y_list = y.tolist()
unique = set(y_list)
print(unique)


# Split the dataset into a training set and a test set
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
# Initialize weights and biases
weights = np.random.rand(X_train.shape[1])
bias = np.random.rand(1)
learning_rate = 0.01

# Initialize weights and biases
weights = np.random.rand(X_train.shape[1])
bias = np.random.rand(1)
learning_rate = 0.01

# Sigmoid activation function
def sigmoid(x):
   return 1 / (1 + np.exp(-x))


# Derivative of the sigmoid function
def sigmoid_derivative(x):
   return sigmoid(x) * (1 - sigmoid(x))


# Backward pass
def backward_pass(a, X, y):
   dz = a - y
   dw = np.dot(X.T, dz) / X.shape[0]
   db = np.sum(dz) / X.shape[0]
   return dw, db

# Forward pass
def forward_pass(X, weights, bias):
   z = np.dot(X, weights) + bias
   a = sigmoid(z)
   return a

# Update rule
def update_parameters(weights, bias, dw, db, learning_rate):
   weights = weights - learning_rate * dw
   bias = bias - learning_rate * db
   return weights, bias


# Initialize list to store some important variables
losses = []
accuracies = []
weights_over_time = []
biases_over_time = []

epochs = 1000
lr = 0.001


run = wandb.init(
   # Set the project where this run will be logged
   project="first-project",
   # Track hyperparameters and run metadata
   config={
       "learning_rate": lr,
       "epochs": epochs,
   },
)




# Initialize weights and biases
weights = np.random.rand(X_train.shape[1])
bias = np.random.rand(1)


# Training loop
for i in range(epochs):
   a = forward_pass(X_train, weights, bias)
   dw, db = backward_pass(a, X_train, y_train)
   weights, bias = update_parameters(weights, bias, dw, db, lr)


   # Append weights and biases to lists
   weights_over_time.append(weights.copy())
   biases_over_time.append(bias.copy())


   # Calculate loss
   loss = -np.mean(y_train * np.log(a) + (1 - y_train) * np.log(1 - a))
   losses.append(loss)


   # Calculate accuracy
   y_pred = [1 if p >= 0.5 else 0 for p in a]
   accuracy = accuracy_score(y_train, y_pred)
   accuracies.append(accuracy)
   #
   wandb.log({"accuracy": accuracy, "loss": loss})

# Evaluate the model
y_pred = forward_pass(X_test, weights, bias)
y_pred = [1 if p >= 0.5 else 0 for p in y_pred]
print('Accuracy:', accuracy_score(y_test, y_pred))