Logistic Regression Algorithm - Multiclass features

Oke_Matthew_Olawale/Logistic Regression ALgorithm for Binary Classification and Multiclass Classification.ipynb

@@ -0,0 +1,267 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "\n",
+    "class LogisticRegression:\n",
+    "    def __init__(self,lr=0.1, n_iters=1000, multi=False):\n",
+    "        self.lr=lr\n",
+    "        self.n_iters = n_iters\n",
+    "        self.weights = None\n",
+    "        self.bias = None\n",
+    "        self.multi = multi\n",
+    "        #weight and bias for each binary classification during multiclass classification(OVR)\n",
+    "        self.w = []\n",
+    "        self.b = []\n",
+    "        \n",
+    "    def fit(self,X,y):\n",
+    "        n_samples,n_features = X.shape\n",
+    "        \n",
+    "        #training for multi-classification\n",
+    "        if self.multi == True:\n",
+    "            for i in np.unique(y):\n",
+    "                \n",
+    "                #initialize wieghts\n",
+    "                self.weights = np.zeros(n_features)\n",
+    "                self.bias = 0\n",
+    "                \n",
+    "                #Gradient descent algo for each bianry classification during multi_class classification\n",
+    "                for _ in range(self.n_iters):\n",
+    "                    y_ = [1 if j == i else 0 for j in y]\n",
+    "                    linear_model = np.dot(X,self.weights)+self.bias\n",
+    "                    y_predicted = self._sigmoid(linear_model)\n",
+    "                    \n",
+    "                    dw = (1/n_samples)*np.dot(X.T,(y_predicted-y_))#- (2 *1* self.weights)\n",
+    "                    db = (2/n_samples)*np.sum(y_predicted-y_)\n",
+    "                    \n",
+    "                    #update weights\n",
+    "                    self.weights -=self.lr*dw\n",
+    "                    self.bias -=self.lr*db\n",
+    "                    \n",
+    "                #store binary weights which will be used during prediction(Multi-class)\n",
+    "                self.w.append(self.weights)\n",
+    "                self.b.append(self.bias)\n",
+    "        else:\n",
+    "            #Binary classification training\n",
+    "            self.weights = np.zeros(n_features)\n",
+    "            self.bias = 0\n",
+    "            \n",
+    "            #gradient descent optimzation\n",
+    "            for _ in range(self.n_iters):\n",
+    "                linear_model = np.dot(X,self.weights)+self.bias\n",
+    "                y_predicted = self._sigmoid(linear_model)\n",
+    "                \n",
+    "                #Calculate Error rates\n",
+    "                dw = (1/n_samples)*np.dot(X.T,(y_predicted-y))#- (2 *1* self.weights)\n",
+    "                db = (2/n_samples)*np.sum(y_predicted-y)\n",
+    "                \n",
+    "                #update weights and biases\n",
+    "                self.weights -=self.lr*dw\n",
+    "                self.bias -=self.lr*db\n",
+    "\n",
+    "    def predict(self,X):\n",
+    "        #Multi-class predictions\n",
+    "        if self.multi == True:\n",
+    "            predicts = []\n",
+    "            for i in range(len(self.b)):\n",
+    "                linear_model = np.dot(X,self.w[i])+self.b[i]\n",
+    "                y_predicted = self._sigmoid(linear_model)\n",
+    "                \n",
+    "                #store prediction for each binary-class predictions\n",
+    "                predicts.append(y_predicted)\n",
+    "            predicts = np.array(predicts)\n",
+    "            \n",
+    "            #collate the prediction for each data point\n",
+    "            predicts = [predicts[:,i] for i in range(len(y_predicted))]\n",
+    "            \n",
+    "            #return softmax to sum probabilities of predictions to 1\n",
+    "            return self.softmax(predicts)\n",
+    "        \n",
+    "        else:\n",
+    "            #binary predictions\n",
+    "            linear_model = np.dot(X,self.weights)+self.bias\n",
+    "            y_predicted = self._sigmoid(linear_model)\n",
+    "            y_predicted_cls = [1 if i > 0.5 else 0 for i in y_predicted]\n",
+    "            return y_predicted_cls\n",
+    "            \n",
+    "\n",
+    "    def _sigmoid(self,x):\n",
+    "        return 1/(1+np.exp(-x))\n",
+    "    \n",
+    "    def softmax(self,x):\n",
+    "        return np.exp(x)/ np.sum(np.exp(x),axis=1).reshape(-1,1)\n",
+    "    \n",
+    "    #simple accuracy calculation based on amount of model's right predictions compared to label\n",
+    "    def accuracy(self,y_true, y_pred):\n",
+    "        if self.multi == True:\n",
+    "            y_pred = np.argmax(y_pred,axis=1)\n",
+    "        accuracy = np.sum(y_true==y_pred)/len(y_true)\n",
+    "        return accuracy"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Binary classification"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Testing model on the breast cancer dataset provided  by sklearn\n",
+    "from sklearn.model_selection import train_test_split\n",
+    "from sklearn import datasets\n",
+    "import pandas as pd\n",
+    "bc = datasets.load_breast_cancer()\n",
+    "X,y = bc.data, bc.target\n",
+    "X_train,X_test,y_train,y_test = train_test_split(X,y, test_size = 0.2, random_state=1234)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "model1 = LogisticRegression(lr = 0.0001, n_iters=1000)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#fit training data\n",
+    "model1.fit(X_train,y_train)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#make predictions\n",
+    "predictions = model1.predict(X_test)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "0.9298245614035088"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "#calculate accuracy\n",
+    "model1.accuracy(y_test, predictions)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Multiclass Classification"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Testing model on the iris dataset with three different classes\n",
+    "from sklearn.model_selection import train_test_split\n",
+    "from sklearn import datasets\n",
+    "import pandas as pd\n",
+    "bc = datasets.load_iris()\n",
+    "X,y = bc.data, bc.target\n",
+    "X_train,X_test,y_train,y_test = train_test_split(X,y, test_size = 0.4, random_state=1234)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "0.9833333333333333"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "model1 = LogisticRegression(multi=True)\n",
+    "model1.fit(X_train,y_train)\n",
+    "predictions = model1.predict(X_test)\n",
+    "model1.accuracy(y_test, predictions)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

Oke_Matthew_Olawale/Readme.md

@@ -0,0 +1,43 @@
+This is a logistic regression algorithm built from scratch with the use of the numpy library only.
+
+Logistic regression most times only have the capability for binary classification but this particular can also solve for multiclass classification using the OVR(One VS Rest Method.)
+
+SUMMARY OF THE LOGISTIC REGRESSION ALGORITHM
+- Initialize weights with its column as the number of features present in dataset
+
+- Find the dot product of the weight and the data added with a particular bias which gives us our linear model.
+                    (input_data * weights) + bias. 
+
+- The sigmoid computation is then applied to this result to form probabilities(0 to 1) between predicted classes. 
+                sigmoid_function  = 1/1(exp(-x)
+
+- This helps find correlation between the input data and its weights from which error will be calculated and weigthts contributing to this error will be penalized by calculating the rate of change of error with weights and biases. 
+            dw = (1 / number of input_data)*(input_data * error)
+            db = (1 / number of input_data)*error
+            where, error = predictions - true_label
+
+- From here, weights are updated little by little by multiplying this rate of change with a particular fraction known as the learning rate and subtracting it from previous weights and biases
+
+- This goes for a couple of iterations defaultly set "n_iters = 1000". This procees is called the gradient descent algorithm.
+
+
+
+Multiclass Classification
+- This is done by making binary classification on one of the classes against other classes repeatedly for each class.
+
+- Sigmoid probabilties of their linear-model are computed.
+
+- Then each sigmoid probabilities of these binary classifiers are extracted for each datapoint and passed through a softmax activation function. This also helps to put all values directly in probabilties between 0s and 1s making them sum up to 1.
+            softmaxc_function = exp(xi)/ sum(exp(xi) , where xi = probability for each class for a particular input_data prediction
+
+- Then the class with the final highest probability is chosen as the prediction.
+
+
+For implementation:
+model = LogisticRegression()    #This is implements for binary classification and automatically sets the multi parameter to False.
+model.fit(X,y)
+model.predict(X)
+model.accuracy(y, y_predict)
+
+------------------------------------------------------------------------------------------------------------------
+model = LogisticRegression(multi=True)           #This is used for multiclass classification and "multi" is needed to be set "True".