#PyTorch Tutorial: How to Develop Deep Learning Models with Python
#https://machinelearningmastery.com/pytorch-tutorial-develop-deep-learning-models/

#The focus of this tutorial is on using the PyTorch API for common deep learning model development task.
#After completing this tutorial, you will know:
#   The difference between Torch and PyTorch and how to install and confirm PyTorch is working.
#   The five-step life-cycle of PyTorch models and how to define, fit, and evaluate models.
#   How to develop PyTorch deep learning models for regression, classification, and predictive modeling tasks.

# The PyTorch API is simple and flexible, making it a favorite for academics and 
# researchers in the development of new deep learning models and applications. 
# The extensive use has led to many extensions for specific applications (such as 
# text, computer vision, and audio data), and may pre-trained models that can be 
# used directly. As such, it may be the most popular library used by academics.
#
# The flexibility of PyTorch comes at the cost of ease of use, especially for 
# beginners, as compared to simpler interfaces like Keras. The choice to use 
# PyTorch instead of Keras gives up some ease of use, a slightly steeper 
# learning curve, and more code for more flexibility, and perhaps a more vibrant 
# academic community.


# 1. Install Pytorch using pip:
    
# Open Anaconda PowerShell and type:
# pip install torch
# Alternative way, using conda:
# If you don't have GPU in the system, set CUDA as None or CPU:
# conda install pytorch-cpu torchvision-cpu -c pytorch

# check pytorch version
import torch
print(torch.__version__)
# In my case is: 2.7.0+cpu


#2. PyTorch Deep Learning Model Life-Cycle

#The five steps in the life-cycle are as follows:
#Step 1: Prepare the Data
#  PyTorch provides the Dataset class that you can extend and customize to load your dataset.
#Step 2: Define the Model
#  The idiom for defining a model in PyTorch involves defining a class that extends the Module class.
#Step 3: Train the Model
#  The training process requires that you define a loss function and an optimization algorithm.
#Step 4: Evaluate the model
#  This can be achieved by using the DataLoader for the test dataset and collecting 
#  the predictions for the test set, then comparing the predictions to the expected 
#  values of the test set and calculating a performance metric.
#Step 5: Make predictions
#  This requires that you wrap the data in a PyTorch Tensor data structure.


#3. How to Develop PyTorch Deep Learning Models

# A Multilayer Perceptron model, or MLP for short, is a standard fully connected neural network model.

# We will use the Ionosphere binary (two class) classification dataset to 
# demonstrate an MLP for binary classification.

# This dataset involves predicting whether there is a structure in the atmosphere 
# or not given radar returns.

# We will use a LabelEncoder to encode the string labels to integer values 0 and 1. 
# The model will be fit on 67 percent of the data, and the remaining 33 percent 
# will be used for evaluation, split using the train_test_split() function.

# It is a good practice to use ‘relu‘ activation with a ‘He Uniform‘ weight 
# initialization. This combination goes a long way to overcome the problem of 
# vanishing gradients when training deep neural network models. For more on ReLU, 
# see the tutorial:
#https://machinelearningmastery.com/rectified-linear-activation-function-for-deep-learning-neural-networks/

# The model predicts the probability of class 1 and uses the sigmoid activation 
# function. The model is optimized using stochastic gradient descent and seeks 
# to minimize the binary cross-entropy loss.


# pytorch MLP for binary classification
from numpy import vstack
from pandas import read_csv
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import accuracy_score
from torch.utils.data import Dataset
from torch.utils.data import DataLoader
from torch.utils.data import random_split
from torch import Tensor
from torch.nn import Linear
from torch.nn import ReLU
from torch.nn import Sigmoid
from torch.nn import Module
from torch.optim import SGD
from torch.nn import BCELoss
from torch.nn.init import kaiming_uniform_
from torch.nn.init import xavier_uniform_

# About the dataset
# https://raw.githubusercontent.com/jbrownlee/Datasets/master/ionosphere.names
# ...
# 5. Number of Instances: 351
#
# 6. Number of Attributes: 34 plus the class attribute
#    -- All 34 predictor attributes are continuous

# 7. Attribute Information:     
#    -- All 34 are continuous, as described above
#    -- The 35th attribute is either "good" or "bad" according to the definition
#       summarized above.  This is a binary classification task.

# dataset definition
class CSVDataset(Dataset):
    # load the dataset
    def __init__(self, path):
        # load the csv file as a dataframe
        df = read_csv(path, header=None)
        # store the inputs and outputs
        self.X = df.values[:, :-1]
        self.y = df.values[:, -1]
        # ensure input data is floats
        self.X = self.X.astype('float32')
        # label encode target and ensure the values are floats
        self.y = LabelEncoder().fit_transform(self.y)
        self.y = self.y.astype('float32')
        self.y = self.y.reshape((len(self.y), 1))

    # number of rows in the dataset
    def __len__(self):
        return len(self.X)

    # get a row at an index
    def __getitem__(self, idx):
        return [self.X[idx], self.y[idx]]

    # get indexes for train and test rows
    def get_splits(self, n_test=0.33):
        # determine sizes
        test_size = round(n_test * len(self.X))
        train_size = len(self.X) - test_size
        # calculate the split
        return random_split(self, [train_size, test_size])

# model definition
class MLP(Module):
    # define model elements
    def __init__(self, n_inputs):
        super(MLP, self).__init__()
        # input to first hidden layer
        self.hidden1 = Linear(n_inputs, 10)
        kaiming_uniform_(self.hidden1.weight, nonlinearity='relu')
        self.act1 = ReLU()
        # second hidden layer
        self.hidden2 = Linear(10, 8)
        kaiming_uniform_(self.hidden2.weight, nonlinearity='relu')
        self.act2 = ReLU()
        # third hidden layer and output
        self.hidden3 = Linear(8, 1)
        xavier_uniform_(self.hidden3.weight)
        self.act3 = Sigmoid()

    # forward propagate input
    def forward(self, X):
        # input to first hidden layer
        X = self.hidden1(X)
        X = self.act1(X)
         # second hidden layer
        X = self.hidden2(X)
        X = self.act2(X)
        # third hidden layer and output
        X = self.hidden3(X)
        X = self.act3(X)
        return X

# prepare the dataset
def prepare_data(path):
    # load the dataset
    dataset = CSVDataset(path)
    # calculate split
    train, test = dataset.get_splits()
    # prepare data loaders
    train_dl = DataLoader(train, batch_size=32, shuffle=True)
    test_dl = DataLoader(test, batch_size=1024, shuffle=False)
    return train_dl, test_dl

# train the model
def train_model(train_dl, model):
    # define the optimization
    criterion = BCELoss()
    optimizer = SGD(model.parameters(), lr=0.01, momentum=0.9)
    # enumerate epochs
    for epoch in range(100):
        # enumerate mini batches
        for i, (inputs, targets) in enumerate(train_dl):
            # clear the gradients
            optimizer.zero_grad()
            # compute the model output
            yhat = model(inputs)
            # calculate loss
            loss = criterion(yhat, targets)
            # credit assignment
            loss.backward()
            # update model weights
            optimizer.step()

# evaluate the model
def evaluate_model(test_dl, model):
    predictions, actuals = list(), list()
    for i, (inputs, targets) in enumerate(test_dl):
        # evaluate the model on the test set
        yhat = model(inputs)
        # retrieve numpy array
        yhat = yhat.detach().numpy()
        actual = targets.numpy()
        actual = actual.reshape((len(actual), 1))
        # round to class values
        yhat = yhat.round()
        # store
        predictions.append(yhat)
        actuals.append(actual)
    predictions, actuals = vstack(predictions), vstack(actuals)
    # calculate accuracy
    acc = accuracy_score(actuals, predictions)
    return acc

# make a class prediction for one row of data
def predict(row, model):
    # convert row to data
    row = Tensor([row])
    # make prediction
    yhat = model(row)
    # retrieve numpy array
    yhat = yhat.detach().numpy()
    return yhat


# prepare the data
path = 'https://raw.githubusercontent.com/jbrownlee/Datasets/master/ionosphere.csv'
train_dl, test_dl = prepare_data(path)
print(len(train_dl.dataset), len(test_dl.dataset))

# define the network
model = MLP(34)

# train the model
train_model(train_dl, model)

# evaluate the model
acc = evaluate_model(test_dl, model)
print('Accuracy: %.3f' % acc)

# make a single prediction (expect class=1)
row = [1,0,0.99539,-0.05889,0.85243,0.02306,0.83398,-0.37708,1,0.03760,0.85243,-0.17755,0.59755,-0.44945,0.60536,-0.38223,0.84356,-0.38542,0.58212,-0.32192,0.56971,-0.29674,0.36946,-0.47357,0.56811,-0.51171,0.41078,-0.46168,0.21266,-0.34090,0.42267,-0.54487,0.18641,-0.45300]
yhat = predict(row, model) 
# get the value because yhat is array([[0.9945492]], dtype=float32)
# read about NumPy arrays at:
# https://numpy.org/devdocs//user/absolute_beginners.html
yhat_val = yhat[0,0] # could also use yhat.flat[0] 
print('Predicted: %.3f (class=%d)' % (yhat_val, yhat_val.round()))
