Logistic Regression with Softmax.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logistic Regression with Softmax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['setosa', 'versicolor', 'virginica']\n"
     ]
    }
   ],
   "source": [
    "from sklearn.datasets import load_iris\n",
    "dataset = load_iris()\n",
    "X = dataset.data\n",
    "y = dataset.target\n",
    "\n",
    "target_names = list(dataset.target_names)\n",
    "print(target_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SoftmaxReg:\n",
    "    \"\"\"\n",
    "    Softmax Regression for multiclass.\n",
    "    \"\"\"\n",
    "    def __init__(self, \n",
    "                 epochs=100, \n",
    "                 tolerance = 1e-10, \n",
    "                 alpha=0.001, \n",
    "                 lambd=0, \n",
    "                 threshold=0.5, \n",
    "                 verbose=False,\n",
    "                 minibatch_size=30,\n",
    "                ):\n",
    "        self.epochs = epochs\n",
    "        self.alpha = alpha # Learning rate\n",
    "        self.lambd = lambd # Regularization parameter\n",
    "        self.tolerance = tolerance\n",
    "        self.threshold = threshold\n",
    "        self.verbose = verbose\n",
    "        self.minibatch_size = minibatch_size\n",
    "        \n",
    "    def add_ones(self, X):\n",
    "        return np.concatenate((np.ones((len(X),1)), X), axis = 1)\n",
    "    \n",
    "    def softmax(self, X, theta):\n",
    "        z = X@theta\n",
    "        z -= np.max(z) # to prevent overflow\n",
    "        return np.exp(z) / np.sum(np.exp(z), axis=1, keepdims=True)\n",
    "    \n",
    "    def compute_cost(self, X, y_true):\n",
    "        n, d = X.shape\n",
    "        y_hat = self.softmax(X, self.theta)\n",
    "        \n",
    "        temp_theta = self.theta[:, 1:].copy()\n",
    "        \n",
    "        if self.no_classes > 2:  \n",
    "            temp_theta = self.theta.copy()\n",
    "            temp_theta[0, :] = 0 # Not including bias in regularization\n",
    "        \n",
    "        Cost = (-1 / n) * np.sum(y_true * np.log(y_hat)) + (self.lambd/2)*np.sum(temp_theta*temp_theta)\n",
    "        \n",
    "        return Cost\n",
    "    \n",
    "    def compute_grad(self, X, y):\n",
    "        n, _ = X.shape \n",
    "        theta_wo_bias = self.theta.copy()\n",
    "        theta_wo_bias[0, :] = 0\n",
    "        grad = (-1 / n) * X.T@(y - self.softmax(X, self.theta)) + self.lambd * theta_wo_bias\n",
    "        if self.no_classes <= 2:\n",
    "            grad= grad.reshape(-1, 1)\n",
    "        return grad\n",
    "    \n",
    "    def get_minibatch(self, X, y,  minibatch):\n",
    "        X_mb = X[minibatch*self.minibatch_size: (minibatch+1)*self.minibatch_size]\n",
    "        y_mb = y[minibatch*self.minibatch_size: (minibatch+1)*self.minibatch_size]\n",
    "        return X_mb, y_mb\n",
    "    \n",
    "    def one_hot_encode(self, y):\n",
    "        n = len(y)\n",
    "        if self.no_classes > 2:\n",
    "            # Turn y into one-hot-labels if number of classes is greater than 2\n",
    "            y_encode = np.zeros((n, self.no_classes))\n",
    "            y_encode[range(n), y] = 1 #numpy advanced indexing\n",
    "            y = y_encode\n",
    "        else:\n",
    "            y = y.reshape(-1, 1)\n",
    "        return y\n",
    "    \n",
    "    def fit(self, X, y):\n",
    "        X = X.copy()\n",
    "        X = self.add_ones(X)\n",
    "        \n",
    "        n, d = X.shape\n",
    "        \n",
    "        self.classes = np.unique(y)\n",
    "        self.no_classes = len(self.classes)\n",
    "        \n",
    "        y = self.one_hot_encode(y)\n",
    "        \n",
    "        if self.no_classes > 2:\n",
    "            self.theta = np.zeros((d, self.no_classes))\n",
    "        else:\n",
    "            self.theta = np.zeros((d, 1))\n",
    "            \n",
    "        current_epoch = 1\n",
    "        norm = 1\n",
    "\n",
    "        no_of_minibatch = int(n/self.minibatch_size)\n",
    "        \n",
    "        while (norm >= self.tolerance and current_epoch < self.epochs):\n",
    "            # Shuffle X for minibatch gradient descent\n",
    "            shuffled_index = np.random.permutation(n)\n",
    "            X_shuffled = X[shuffled_index]\n",
    "            y_shuffled = y[shuffled_index]\n",
    "            \n",
    "            old_theta = self.theta.copy()        \n",
    "            \n",
    "            for mb in range(no_of_minibatch):\n",
    "                X_mb, y_mb = self.get_minibatch(X, y, mb)\n",
    "                \n",
    "                theta_wo_bias = self.theta[:, 1:].copy()\n",
    "                if self.no_classes > 2:\n",
    "                    grad = self.compute_grad(X_mb, y_mb)\n",
    "            \n",
    "                self.theta = self.theta - self.alpha*grad\n",
    "            \n",
    "            if self.verbose and (current_epoch%10 == 0):\n",
    "                print(f'cost for {current_epoch} epoch : {self.compute_cost(X, y)}')\n",
    "            norm = np.linalg.norm(old_theta - self.theta)\n",
    "            current_epoch += 1\n",
    "            \n",
    "        return self.theta\n",
    "    \n",
    "    def evaluate(self, X, y):\n",
    "        \"\"\"\n",
    "        Returns mse loss for a dataset evaluated on the hypothesis\n",
    "        \"\"\"\n",
    "        X = self.add_ones(X)\n",
    "        y = self.one_hot_encode(y)\n",
    "        return self.compute_cost(X, y)\n",
    "    \n",
    "    def predict(self, X):\n",
    "        proba = self.predict_proba(X)\n",
    "        if self.no_classes > 2:\n",
    "            # Multiclass classification\n",
    "            y_hat = np.argmax(proba, axis=1)\n",
    "        elif self.no_classes == 2:\n",
    "            # Binary classification\n",
    "            y_hat = (proba >= self.threshold).astype(int)\n",
    "        return y_hat\n",
    "        \n",
    "    def predict_proba(self, X):\n",
    "        \"\"\"\n",
    "        Returns probability of predictions.\n",
    "        \"\"\"\n",
    "        X = self.add_ones(X)\n",
    "        \n",
    "        return self.softmax(X, self.theta)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "logreg = SoftmaxReg(epochs=9000, \n",
    "                alpha=0.001, \n",
    "                verbose=True, \n",
    "                minibatch_size=20,\n",
    "               )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cost for 10 epoch : 1.0405393269354362\n",
      "cost for 20 epoch : 0.9971981162708727\n",
      "cost for 30 epoch : 0.9587096726587906\n",
      "cost for 40 epoch : 0.923934111226926\n",
      "cost for 50 epoch : 0.89242308351885\n",
      "cost for 60 epoch : 0.8638280402482029\n",
      "cost for 70 epoch : 0.8378377100145463\n",
      "cost for 80 epoch : 0.8141697597635079\n",
      "cost for 90 epoch : 0.7925694843009948\n",
      "cost for 100 epoch : 0.7728089104045688\n",
      "cost for 110 epoch : 0.7546853551472168\n",
      "cost for 120 epoch : 0.7380195087848012\n",
      "cost for 130 epoch : 0.7226532665277198\n",
      "cost for 140 epoch : 0.7084474996450751\n",
      "cost for 150 epoch : 0.6952798942623154\n",
      "cost for 160 epoch : 0.6830429333145113\n",
      "cost for 170 epoch : 0.6716420592393022\n",
      "cost for 180 epoch : 0.6609940300949891\n",
      "cost for 190 epoch : 0.6510254664892866\n",
      "cost for 200 epoch : 0.6416715780106838\n",
      "cost for 210 epoch : 0.6328750534977968\n",
      "cost for 220 epoch : 0.624585097856329\n",
      "cost for 230 epoch : 0.616756598147818\n",
      "cost for 240 epoch : 0.6093494026119763\n",
      "cost for 250 epoch : 0.6023276976868681\n",
      "cost for 260 epoch : 0.5956594696748159\n",
      "cost for 270 epoch : 0.5893160392982852\n",
      "cost for 280 epoch : 0.5832716589047583\n",
      "cost for 290 epoch : 0.5775031634652456\n",
      "cost for 300 epoch : 0.5719896677486922\n",
      "cost for 310 epoch : 0.5667123031421517\n",
      "cost for 320 epoch : 0.5616539885316237\n",
      "cost for 330 epoch : 0.55679923047311\n",
      "cost for 340 epoch : 0.5521339485817509\n",
      "cost for 350 epoch : 0.547645322663213\n",
      "cost for 360 epoch : 0.5433216586193358\n",
      "cost for 370 epoch : 0.5391522705918865\n",
      "cost for 380 epoch : 0.5351273771751188\n",
      "cost for 390 epoch : 0.531238009839419\n",
      "cost for 400 epoch : 0.5274759319729772\n",
      "cost for 410 epoch : 0.523833567173356\n",
      "cost for 420 epoch : 0.5203039356121432\n",
      "cost for 430 epoch : 0.5168805974587584\n",
      "cost for 440 epoch : 0.5135576024883439\n",
      "cost for 450 epoch : 0.5103294451171804\n",
      "cost for 460 epoch : 0.5071910242103863\n",
      "cost for 470 epoch : 0.5041376070933927\n",
      "cost for 480 epoch : 0.5011647972730505\n",
      "cost for 490 epoch : 0.49826850543811324\n",
      "cost for 500 epoch : 0.49544492336378115\n",
      "cost for 510 epoch : 0.4926905003923556\n",
      "cost for 520 epoch : 0.490001922202931\n",
      "cost for 530 epoch : 0.4873760916183973\n",
      "cost for 540 epoch : 0.48481011122864937\n",
      "cost for 550 epoch : 0.48230126763546727\n",
      "cost for 560 epoch : 0.4798470171476199\n",
      "cost for 570 epoch : 0.477444972774847\n",
      "cost for 580 epoch : 0.47509289238691277\n",
      "cost for 590 epoch : 0.47278866791922747\n",
      "cost for 600 epoch : 0.47053031551994123\n",
      "cost for 610 epoch : 0.46831596654514784\n",
      "cost for 620 epoch : 0.46614385931914876\n",
      "cost for 630 epoch : 0.46401233158578004\n",
      "cost for 640 epoch : 0.46191981358478756\n",
      "cost for 650 epoch : 0.4598648216942648\n",
      "cost for 660 epoch : 0.4578459525863804\n",
      "cost for 670 epoch : 0.45586187784911575\n",
      "cost for 680 epoch : 0.4539113390315971\n",
      "cost for 690 epoch : 0.4519931430749212\n",
      "cost for 700 epoch : 0.4501061580942055\n",
      "cost for 710 epoch : 0.4482493094810023\n",
      "cost for 720 epoch : 0.44642157629825174\n",
      "cost for 730 epoch : 0.4446219879426512\n",
      "cost for 740 epoch : 0.4428496210517391\n",
      "cost for 750 epoch : 0.44110359663515153\n",
      "cost for 760 epoch : 0.4393830774114357\n",
      "cost for 770 epoch : 0.4376872653335494\n",
      "cost for 780 epoch : 0.4360153992877188\n",
      "cost for 790 epoch : 0.4343667529517317\n",
      "cost for 800 epoch : 0.43274063279999325\n",
      "cost for 810 epoch : 0.4311363762438071\n",
      "cost for 820 epoch : 0.4295533498963622\n",
      "cost for 830 epoch : 0.42799094795282244\n",
      "cost for 840 epoch : 0.4264485906767537\n",
      "cost for 850 epoch : 0.4249257229848674\n",
      "cost for 860 epoch : 0.4234218131227443\n",
      "cost for 870 epoch : 0.4219363514248173\n",
      "cost for 880 epoch : 0.42046884915244914\n",
      "cost for 890 epoch : 0.4190188374044519\n",
      "cost for 900 epoch : 0.41758586609485293\n",
      "cost for 910 epoch : 0.4161695029931333\n",
      "cost for 920 epoch : 0.41476933282254663\n",
      "cost for 930 epoch : 0.41338495641247364\n",
      "cost for 940 epoch : 0.412015989901086\n",
      "cost for 950 epoch : 0.4106620639848817\n",
      "cost for 960 epoch : 0.4093228232119205\n",
      "cost for 970 epoch : 0.4079979253158294\n",
      "cost for 980 epoch : 0.4066870405878692\n",
      "cost for 990 epoch : 0.40538985128455757\n",
      "cost for 1000 epoch : 0.40410605106852904\n",
      "cost for 1010 epoch : 0.402835344480484\n",
      "cost for 1020 epoch : 0.40157744644023496\n",
      "cost for 1030 epoch : 0.400332081775002\n",
      "cost for 1040 epoch : 0.39909898477324307\n",
      "cost for 1050 epoch : 0.3978778987624269\n",
      "cost for 1060 epoch : 0.3966685757092651\n",
      "cost for 1070 epoch : 0.39547077584103024\n",
      "cost for 1080 epoch : 0.3942842672866733\n",
      "cost for 1090 epoch : 0.3931088257365513\n",
      "cost for 1100 epoch : 0.3919442341196498\n",
      "cost for 1110 epoch : 0.390790282297264\n",
      "cost for 1120 epoch : 0.3896467667721708\n",
      "cost for 1130 epoch : 0.3885134904123888\n",
      "cost for 1140 epoch : 0.38739026218868133\n",
      "cost for 1150 epoch : 0.38627689692501566\n",
      "cost for 1160 epoch : 0.38517321506123836\n",
      "cost for 1170 epoch : 0.38407904242728136\n",
      "cost for 1180 epoch : 0.38299421002824663\n",
      "cost for 1190 epoch : 0.38191855383977097\n",
      "cost for 1200 epoch : 0.3808519146130988\n",
      "cost for 1210 epoch : 0.3797941376893351\n",
      "cost for 1220 epoch : 0.37874507282237735\n",
      "cost for 1230 epoch : 0.37770457401006113\n",
      "cost for 1240 epoch : 0.37667249933307645\n",
      "cost for 1250 epoch : 0.3756487108012452\n",
      "cost for 1260 epoch : 0.3746330742067711\n",
      "cost for 1270 epoch : 0.3736254589840952\n",
      "cost for 1280 epoch : 0.37262573807601573\n",
      "cost for 1290 epoch : 0.3716337878057495\n",
      "cost for 1300 epoch : 0.3706494877546282\n",
      "cost for 1310 epoch : 0.36967272064514584\n",
      "cost for 1320 epoch : 0.368703372229085\n",
      "cost for 1330 epoch : 0.3677413311804685\n",
      "cost for 1340 epoch : 0.36678648899309624\n",
      "cost for 1350 epoch : 0.36583873988243854\n",
      "cost for 1360 epoch : 0.36489798069167556\n",
      "cost for 1370 epoch : 0.36396411080167645\n",
      "cost for 1380 epoch : 0.3630370320447318\n",
      "cost for 1390 epoch : 0.36211664862185283\n",
      "cost for 1400 epoch : 0.36120286702347426\n",
      "cost for 1410 epoch : 0.3602955959533922\n",
      "cost for 1420 epoch : 0.35939474625578816\n",
      "cost for 1430 epoch : 0.35850023084519494\n",
      "cost for 1440 epoch : 0.3576119646392642\n",
      "cost for 1450 epoch : 0.35672986449420907\n",
      "cost for 1460 epoch : 0.35585384914279733\n",
      "cost for 1470 epoch : 0.3549838391347794\n",
      "cost for 1480 epoch : 0.354119756779639\n",
      "cost for 1490 epoch : 0.3532615260915633\n",
      "cost for 1500 epoch : 0.35240907273653216\n",
      "cost for 1510 epoch : 0.3515623239814323\n",
      "cost for 1520 epoch : 0.35072120864510686\n",
      "cost for 1530 epoch : 0.349885657051253\n",
      "cost for 1540 epoch : 0.34905560098309085\n",
      "cost for 1550 epoch : 0.34823097363972205\n",
      "cost for 1560 epoch : 0.34741170959410844\n",
      "cost for 1570 epoch : 0.34659774475259836\n",
      "cost for 1580 epoch : 0.3457890163159365\n",
      "cost for 1590 epoch : 0.34498546274169334\n",
      "cost for 1600 epoch : 0.3441870237080538\n",
      "cost for 1610 epoch : 0.3433936400789087\n",
      "cost for 1620 epoch : 0.34260525387019347\n",
      "cost for 1630 epoch : 0.341821808217425\n",
      "cost for 1640 epoch : 0.3410432473443824\n",
      "cost for 1650 epoch : 0.34026951653288934\n",
      "cost for 1660 epoch : 0.3395005620936505\n",
      "cost for 1670 epoch : 0.33873633133809933\n",
      "cost for 1680 epoch : 0.33797677255121766\n",
      "cost for 1690 epoch : 0.3372218349652857\n",
      "cost for 1700 epoch : 0.33647146873452766\n",
      "cost for 1710 epoch : 0.33572562491061625\n",
      "cost for 1720 epoch : 0.3349842554190018\n",
      "cost for 1730 epoch : 0.33424731303603383\n",
      "cost for 1740 epoch : 0.3335147513668444\n",
      "cost for 1750 epoch : 0.3327865248239634\n",
      "cost for 1760 epoch : 0.33206258860663723\n",
      "cost for 1770 epoch : 0.33134289868082334\n",
      "cost for 1780 epoch : 0.33062741175983607\n",
      "cost for 1790 epoch : 0.329916085285617\n",
      "cost for 1800 epoch : 0.3292088774106091\n",
      "cost for 1810 epoch : 0.3285057469802081\n",
      "cost for 1820 epoch : 0.3278066535157722\n",
      "cost for 1830 epoch : 0.3271115571981682\n",
      "cost for 1840 epoch : 0.3264204188518349\n",
      "cost for 1850 epoch : 0.32573319992934285\n",
      "cost for 1860 epoch : 0.32504986249643464\n",
      "cost for 1870 epoch : 0.3243703692175254\n",
      "cost for 1880 epoch : 0.32369468334165025\n",
      "cost for 1890 epoch : 0.32302276868883806\n",
      "cost for 1900 epoch : 0.3223545896369004\n",
      "cost for 1910 epoch : 0.3216901111086176\n",
      "cost for 1920 epoch : 0.3210292985593105\n",
      "cost for 1930 epoch : 0.32037211796478077\n",
      "cost for 1940 epoch : 0.3197185358096103\n",
      "cost for 1950 epoch : 0.319068519075805\n",
      "cost for 1960 epoch : 0.31842203523177076\n",
      "cost for 1970 epoch : 0.3177790522216109\n",
      "cost for 1980 epoch : 0.3171395384547344\n",
      "cost for 1990 epoch : 0.31650346279576236\n",
      "cost for 2000 epoch : 0.3158707945547237\n",
      "cost for 2010 epoch : 0.3152415034775306\n",
      "cost for 2020 epoch : 0.31461555973672284\n",
      "cost for 2030 epoch : 0.31399293392247296\n",
      "cost for 2040 epoch : 0.31337359703384343\n",
      "cost for 2050 epoch : 0.312757520470287\n",
      "cost for 2060 epoch : 0.31214467602338053\n",
      "cost for 2070 epoch : 0.31153503586878944\n",
      "cost for 2080 epoch : 0.3109285725584492\n",
      "cost for 2090 epoch : 0.31032525901296004\n",
      "cost for 2100 epoch : 0.30972506851418785\n",
      "cost for 2110 epoch : 0.30912797469806397\n",
      "cost for 2120 epoch : 0.30853395154757596\n",
      "cost for 2130 epoch : 0.30794297338594745\n",
      "cost for 2140 epoch : 0.3073550148699942\n",
      "cost for 2150 epoch : 0.30677005098365895\n",
      "cost for 2160 epoch : 0.30618805703171165\n",
      "cost for 2170 epoch : 0.30560900863361523\n",
      "cost for 2180 epoch : 0.3050328817175492\n",
      "cost for 2190 epoch : 0.3044596525145863\n",
      "cost for 2200 epoch : 0.30388929755301836\n",
      "cost for 2210 epoch : 0.30332179365282513\n",
      "cost for 2220 epoch : 0.30275711792028437\n",
      "cost for 2230 epoch : 0.302195247742715\n",
      "cost for 2240 epoch : 0.30163616078335337\n",
      "cost for 2250 epoch : 0.3010798349763544\n",
      "cost for 2260 epoch : 0.3005262485219181\n",
      "cost for 2270 epoch : 0.2999753798815343\n",
      "cost for 2280 epoch : 0.2994272077733427\n",
      "cost for 2290 epoch : 0.2988817111676073\n",
      "cost for 2300 epoch : 0.2983388692822978\n",
      "cost for 2310 epoch : 0.29779866157877755\n",
      "cost for 2320 epoch : 0.29726106775759453\n",
      "cost for 2330 epoch : 0.2967260677543715\n",
      "cost for 2340 epoch : 0.296193641735793\n",
      "cost for 2350 epoch : 0.2956637700956859\n",
      "cost for 2360 epoch : 0.2951364334511922\n",
      "cost for 2370 epoch : 0.2946116126390289\n",
      "cost for 2380 epoch : 0.29408928871183465\n",
      "cost for 2390 epoch : 0.2935694429346\n",
      "cost for 2400 epoch : 0.29305205678117857\n",
      "cost for 2410 epoch : 0.29253711193087706\n",
      "cost for 2420 epoch : 0.2920245902651221\n",
      "cost for 2430 epoch : 0.29151447386420065\n",
      "cost for 2440 epoch : 0.2910067450040746\n",
      "cost for 2450 epoch : 0.2905013861532633\n",
      "cost for 2460 epoch : 0.2899983799697972\n",
      "cost for 2470 epoch : 0.28949770929823543\n",
      "cost for 2480 epoch : 0.2889993571667502\n",
      "cost for 2490 epoch : 0.2885033067842726\n",
      "cost for 2500 epoch : 0.28800954153770036\n",
      "cost for 2510 epoch : 0.2875180449891653\n",
      "cost for 2520 epoch : 0.28702880087335797\n",
      "cost for 2530 epoch : 0.28654179309490996\n",
      "cost for 2540 epoch : 0.28605700572582954\n",
      "cost for 2550 epoch : 0.2855744230029917\n",
      "cost for 2560 epoch : 0.2850940293256806\n",
      "cost for 2570 epoch : 0.28461580925318175\n",
      "cost for 2580 epoch : 0.2841397475024235\n",
      "cost for 2590 epoch : 0.2836658289456677\n",
      "cost for 2600 epoch : 0.2831940386082462\n",
      "cost for 2610 epoch : 0.28272436166634285\n",
      "cost for 2620 epoch : 0.2822567834448205\n",
      "cost for 2630 epoch : 0.28179128941509124\n",
      "cost for 2640 epoch : 0.28132786519302805\n",
      "cost for 2650 epoch : 0.28086649653691864\n",
      "cost for 2660 epoch : 0.2804071693454582\n",
      "cost for 2670 epoch : 0.279949869655783\n",
      "cost for 2680 epoch : 0.2794945836415395\n",
      "cost for 2690 epoch : 0.27904129761099344\n",
      "cost for 2700 epoch : 0.2785899980051727\n",
      "cost for 2710 epoch : 0.27814067139604676\n",
      "cost for 2720 epoch : 0.277693304484741\n",
      "cost for 2730 epoch : 0.2772478840997836\n",
      "cost for 2740 epoch : 0.2768043971953862\n",
      "cost for 2750 epoch : 0.276362830849755\n",
      "cost for 2760 epoch : 0.27592317226343543\n",
      "cost for 2770 epoch : 0.2754854087576855\n",
      "cost for 2780 epoch : 0.2750495277728799\n",
      "cost for 2790 epoch : 0.2746155168669421\n",
      "cost for 2800 epoch : 0.27418336371380675\n",
      "cost for 2810 epoch : 0.27375305610190714\n",
      "cost for 2820 epoch : 0.2733245819326923\n",
      "cost for 2830 epoch : 0.27289792921916933\n",
      "cost for 2840 epoch : 0.2724730860844708\n",
      "cost for 2850 epoch : 0.2720500407604498\n",
      "cost for 2860 epoch : 0.2716287815862964\n",
      "cost for 2870 epoch : 0.27120929700718127\n",
      "cost for 2880 epoch : 0.27079157557292066\n",
      "cost for 2890 epoch : 0.2703756059366659\n",
      "cost for 2900 epoch : 0.2699613768536143\n",
      "cost for 2910 epoch : 0.26954887717974285\n",
      "cost for 2920 epoch : 0.2691380958705624\n",
      "cost for 2930 epoch : 0.2687290219798948\n",
      "cost for 2940 epoch : 0.2683216446586679\n",
      "cost for 2950 epoch : 0.26791595315373357\n",
      "cost for 2960 epoch : 0.267511936806703\n",
      "cost for 2970 epoch : 0.26710958505280236\n",
      "cost for 2980 epoch : 0.26670888741974763\n",
      "cost for 2990 epoch : 0.266309833526637\n",
      "cost for 3000 epoch : 0.2659124130828622\n",
      "cost for 3010 epoch : 0.265516615887036\n",
      "cost for 3020 epoch : 0.2651224318259393\n",
      "cost for 3030 epoch : 0.2647298508734827\n",
      "cost for 3040 epoch : 0.2643388630896866\n",
      "cost for 3050 epoch : 0.26394945861967606\n",
      "cost for 3060 epoch : 0.26356162769269237\n",
      "cost for 3070 epoch : 0.26317536062112035\n",
      "cost for 3080 epoch : 0.2627906477995293\n",
      "cost for 3090 epoch : 0.26240747970373174\n",
      "cost for 3100 epoch : 0.26202584688985375\n",
      "cost for 3110 epoch : 0.26164573999342133\n",
      "cost for 3120 epoch : 0.2612671497284606\n",
      "cost for 3130 epoch : 0.26089006688661115\n",
      "cost for 3140 epoch : 0.26051448233625374\n",
      "cost for 3150 epoch : 0.2601403870216505\n",
      "cost for 3160 epoch : 0.2597677719620977\n",
      "cost for 3170 epoch : 0.2593966282510931\n",
      "cost for 3180 epoch : 0.2590269470555128\n",
      "cost for 3190 epoch : 0.25865871961480397\n",
      "cost for 3200 epoch : 0.25829193724018523\n",
      "cost for 3210 epoch : 0.25792659131386236\n",
      "cost for 3220 epoch : 0.25756267328825394\n",
      "cost for 3230 epoch : 0.2572001746852284\n",
      "cost for 3240 epoch : 0.25683908709535197\n",
      "cost for 3250 epoch : 0.2564794021771481\n",
      "cost for 3260 epoch : 0.2561211116563669\n",
      "cost for 3270 epoch : 0.2557642073252656\n",
      "cost for 3280 epoch : 0.25540868104189834\n",
      "cost for 3290 epoch : 0.25505452472941736\n",
      "cost for 3300 epoch : 0.2547017303753827\n",
      "cost for 3310 epoch : 0.25435029003108245\n",
      "cost for 3320 epoch : 0.2540001958108624\n",
      "cost for 3330 epoch : 0.25365143989146477\n",
      "cost for 3340 epoch : 0.2533040145113764\n",
      "cost for 3350 epoch : 0.2529579119701854\n",
      "cost for 3360 epoch : 0.2526131246279477\n",
      "cost for 3370 epoch : 0.2522696449045609\n",
      "cost for 3380 epoch : 0.2519274652791478\n",
      "cost for 3390 epoch : 0.25158657828944775\n",
      "cost for 3400 epoch : 0.25124697653121647\n",
      "cost for 3410 epoch : 0.25090865265763374\n",
      "cost for 3420 epoch : 0.25057159937871887\n",
      "cost for 3430 epoch : 0.2502358094607546\n",
      "cost for 3440 epoch : 0.24990127572571813\n",
      "cost for 3450 epoch : 0.24956799105072\n",
      "cost for 3460 epoch : 0.2492359483674499\n",
      "cost for 3470 epoch : 0.24890514066163008\n",
      "cost for 3480 epoch : 0.2485755609724763\n",
      "cost for 3490 epoch : 0.24824720239216505\n",
      "cost for 3500 epoch : 0.2479200580653082\n",
      "cost for 3510 epoch : 0.24759412118843416\n",
      "cost for 3520 epoch : 0.24726938500947604\n",
      "cost for 3530 epoch : 0.24694584282726606\n",
      "cost for 3540 epoch : 0.24662348799103628\n",
      "cost for 3550 epoch : 0.24630231389992654\n",
      "cost for 3560 epoch : 0.24598231400249798\n",
      "cost for 3570 epoch : 0.2456634817962518\n",
      "cost for 3580 epoch : 0.24534581082715654\n",
      "cost for 3590 epoch : 0.24502929468917853\n",
      "cost for 3600 epoch : 0.2447139270238202\n",
      "cost for 3610 epoch : 0.24439970151966342\n",
      "cost for 3620 epoch : 0.24408661191191794\n",
      "cost for 3630 epoch : 0.24377465198197665\n",
      "cost for 3640 epoch : 0.24346381555697502\n",
      "cost for 3650 epoch : 0.24315409650935726\n",
      "cost for 3660 epoch : 0.2428454887564461\n",
      "cost for 3670 epoch : 0.24253798626002007\n",
      "cost for 3680 epoch : 0.24223158302589326\n",
      "cost for 3690 epoch : 0.24192627310350268\n",
      "cost for 3700 epoch : 0.24162205058549852\n",
      "cost for 3710 epoch : 0.24131890960734093\n",
      "cost for 3720 epoch : 0.24101684434690115\n",
      "cost for 3730 epoch : 0.24071584902406581\n",
      "cost for 3740 epoch : 0.24041591790034925\n",
      "cost for 3750 epoch : 0.2401170452785068\n",
      "cost for 3760 epoch : 0.2398192255021555\n",
      "cost for 3770 epoch : 0.23952245295539737\n",
      "cost for 3780 epoch : 0.2392267220624484\n",
      "cost for 3790 epoch : 0.2389320272872712\n",
      "cost for 3800 epoch : 0.23863836313321107\n",
      "cost for 3810 epoch : 0.23834572414263913\n",
      "cost for 3820 epoch : 0.23805410489659629\n",
      "cost for 3830 epoch : 0.23776350001444316\n",
      "cost for 3840 epoch : 0.23747390415351363\n",
      "cost for 3850 epoch : 0.23718531200877205\n",
      "cost for 3860 epoch : 0.23689771831247575\n",
      "cost for 3870 epoch : 0.23661111783383879\n",
      "cost for 3880 epoch : 0.23632550537870212\n",
      "cost for 3890 epoch : 0.2360408757892059\n",
      "cost for 3900 epoch : 0.23575722394346665\n",
      "cost for 3910 epoch : 0.23547454475525637\n",
      "cost for 3920 epoch : 0.235192833173687\n",
      "cost for 3930 epoch : 0.23491208418289783\n",
      "cost for 3940 epoch : 0.23463229280174602\n",
      "cost for 3950 epoch : 0.23435345408350047\n",
      "cost for 3960 epoch : 0.2340755631155402\n",
      "cost for 3970 epoch : 0.23379861501905463\n",
      "cost for 3980 epoch : 0.233522604948748\n",
      "cost for 3990 epoch : 0.23324752809254715\n",
      "cost for 4000 epoch : 0.23297337967131163\n",
      "cost for 4010 epoch : 0.2327001549385481\n",
      "cost for 4020 epoch : 0.23242784918012727\n",
      "cost for 4030 epoch : 0.2321564577140034\n",
      "cost for 4040 epoch : 0.23188597588993803\n",
      "cost for 4050 epoch : 0.23161639908922554\n",
      "cost for 4060 epoch : 0.23134772272442236\n",
      "cost for 4070 epoch : 0.2310799422390792\n",
      "cost for 4080 epoch : 0.23081305310747535\n",
      "cost for 4090 epoch : 0.2305470508343567\n",
      "cost for 4100 epoch : 0.23028193095467583\n",
      "cost for 4110 epoch : 0.2300176890333352\n",
      "cost for 4120 epoch : 0.22975432066493384\n",
      "cost for 4130 epoch : 0.22949182147351463\n",
      "cost for 4140 epoch : 0.22923018711231674\n",
      "cost for 4150 epoch : 0.2289694132635286\n",
      "cost for 4160 epoch : 0.22870949563804507\n",
      "cost for 4170 epoch : 0.22845042997522574\n",
      "cost for 4180 epoch : 0.228192212042657\n",
      "cost for 4190 epoch : 0.2279348376359156\n",
      "cost for 4200 epoch : 0.2276783025783352\n",
      "cost for 4210 epoch : 0.22742260272077547\n",
      "cost for 4220 epoch : 0.22716773394139306\n",
      "cost for 4230 epoch : 0.22691369214541515\n",
      "cost for 4240 epoch : 0.22666047326491548\n",
      "cost for 4250 epoch : 0.22640807325859238\n",
      "cost for 4260 epoch : 0.22615648811154976\n",
      "cost for 4270 epoch : 0.22590571383507893\n",
      "cost for 4280 epoch : 0.22565574646644443\n",
      "cost for 4290 epoch : 0.22540658206867076\n",
      "cost for 4300 epoch : 0.22515821673033126\n",
      "cost for 4310 epoch : 0.22491064656534043\n",
      "cost for 4320 epoch : 0.2246638677127461\n",
      "cost for 4330 epoch : 0.22441787633652638\n",
      "cost for 4340 epoch : 0.22417266862538646\n",
      "cost for 4350 epoch : 0.2239282407925582\n",
      "cost for 4360 epoch : 0.22368458907560212\n",
      "cost for 4370 epoch : 0.2234417097362111\n",
      "cost for 4380 epoch : 0.2231995990600153\n",
      "cost for 4390 epoch : 0.22295825335639047\n",
      "cost for 4400 epoch : 0.2227176689582665\n",
      "cost for 4410 epoch : 0.22247784222193964\n",
      "cost for 4420 epoch : 0.22223876952688473\n",
      "cost for 4430 epoch : 0.22200044727557064\n",
      "cost for 4440 epoch : 0.22176287189327695\n",
      "cost for 4450 epoch : 0.22152603982791258\n",
      "cost for 4460 epoch : 0.221289947549836\n",
      "cost for 4470 epoch : 0.22105459155167775\n",
      "cost for 4480 epoch : 0.22081996834816306\n",
      "cost for 4490 epoch : 0.220586074475939\n",
      "cost for 4500 epoch : 0.22035290649340053\n",
      "cost for 4510 epoch : 0.22012046098051988\n",
      "cost for 4520 epoch : 0.2198887345386766\n",
      "cost for 4530 epoch : 0.21965772379049053\n",
      "cost for 4540 epoch : 0.21942742537965473\n",
      "cost for 4550 epoch : 0.2191978359707713\n",
      "cost for 4560 epoch : 0.218968952249188\n",
      "cost for 4570 epoch : 0.21874077092083696\n",
      "cost for 4580 epoch : 0.21851328871207493\n",
      "cost for 4590 epoch : 0.2182865023695239\n",
      "cost for 4600 epoch : 0.21806040865991536\n",
      "cost for 4610 epoch : 0.21783500436993367\n",
      "cost for 4620 epoch : 0.21761028630606283\n",
      "cost for 4630 epoch : 0.21738625129443384\n",
      "cost for 4640 epoch : 0.2171628961806726\n",
      "cost for 4650 epoch : 0.21694021782975176\n",
      "cost for 4660 epoch : 0.21671821312584125\n",
      "cost for 4670 epoch : 0.21649687897216147\n",
      "cost for 4680 epoch : 0.21627621229083802\n",
      "cost for 4690 epoch : 0.21605621002275752\n",
      "cost for 4700 epoch : 0.2158368691274246\n",
      "cost for 4710 epoch : 0.21561818658282003\n",
      "cost for 4720 epoch : 0.21540015938526103\n",
      "cost for 4730 epoch : 0.21518278454926204\n",
      "cost for 4740 epoch : 0.21496605910739708\n",
      "cost for 4750 epoch : 0.21474998011016302\n",
      "cost for 4760 epoch : 0.21453454462584517\n",
      "cost for 4770 epoch : 0.21431974974038276\n",
      "cost for 4780 epoch : 0.21410559255723619\n",
      "cost for 4790 epoch : 0.2138920701972561\n",
      "cost for 4800 epoch : 0.21367917979855242\n",
      "cost for 4810 epoch : 0.2134669185163658\n",
      "cost for 4820 epoch : 0.21325528352293943\n",
      "cost for 4830 epoch : 0.2130442720073921\n",
      "cost for 4840 epoch : 0.21283388117559296\n",
      "cost for 4850 epoch : 0.21262410825003675\n",
      "cost for 4860 epoch : 0.21241495046972045\n",
      "cost for 4870 epoch : 0.2122064050900212\n",
      "cost for 4880 epoch : 0.21199846938257472\n",
      "cost for 4890 epoch : 0.21179114063515575\n",
      "cost for 4900 epoch : 0.2115844161515587\n",
      "cost for 4910 epoch : 0.21137829325147942\n",
      "cost for 4920 epoch : 0.2111727692703992\n",
      "cost for 4930 epoch : 0.21096784155946804\n",
      "cost for 4940 epoch : 0.2107635074853901\n",
      "cost for 4950 epoch : 0.21055976443031005\n",
      "cost for 4960 epoch : 0.21035660979169996\n",
      "cost for 4970 epoch : 0.21015404098224785\n",
      "cost for 4980 epoch : 0.20995205542974646\n",
      "cost for 4990 epoch : 0.2097506505769837\n",
      "cost for 5000 epoch : 0.20954982388163357\n",
      "cost for 5010 epoch : 0.20934957281614833\n",
      "cost for 5020 epoch : 0.20914989486765156\n",
      "cost for 5030 epoch : 0.20895078753783208\n",
      "cost for 5040 epoch : 0.20875224834283843\n",
      "cost for 5050 epoch : 0.2085542748131754\n",
      "cost for 5060 epoch : 0.20835686449359983\n",
      "cost for 5070 epoch : 0.20816001494301953\n",
      "cost for 5080 epoch : 0.20796372373438954\n",
      "cost for 5090 epoch : 0.20776798845461447\n",
      "cost for 5100 epoch : 0.20757280670444597\n",
      "cost for 5110 epoch : 0.20737817609838496\n",
      "cost for 5120 epoch : 0.20718409426458415\n",
      "cost for 5130 epoch : 0.20699055884475004\n",
      "cost for 5140 epoch : 0.206797567494046\n",
      "cost for 5150 epoch : 0.20660511788099842\n",
      "cost for 5160 epoch : 0.20641320768740026\n",
      "cost for 5170 epoch : 0.20622183460821814\n",
      "cost for 5180 epoch : 0.20603099635149838\n",
      "cost for 5190 epoch : 0.20584069063827568\n",
      "cost for 5200 epoch : 0.20565091520248066\n",
      "cost for 5210 epoch : 0.20546166779084962\n",
      "cost for 5220 epoch : 0.20527294616283417\n",
      "cost for 5230 epoch : 0.20508474809051208\n",
      "cost for 5240 epoch : 0.2048970713584994\n",
      "cost for 5250 epoch : 0.20470991376386105\n",
      "cost for 5260 epoch : 0.2045232731160258\n",
      "cost for 5270 epoch : 0.20433714723669869\n",
      "cost for 5280 epoch : 0.2041515339597757\n",
      "cost for 5290 epoch : 0.20396643113125956\n",
      "cost for 5300 epoch : 0.20378183660917465\n",
      "cost for 5310 epoch : 0.20359774826348395\n",
      "cost for 5320 epoch : 0.20341416397600692\n",
      "cost for 5330 epoch : 0.2032310816403365\n",
      "cost for 5340 epoch : 0.20304849916175785\n",
      "cost for 5350 epoch : 0.20286641445716844\n",
      "cost for 5360 epoch : 0.20268482545499708\n",
      "cost for 5370 epoch : 0.20250373009512587\n",
      "cost for 5380 epoch : 0.20232312632880944\n",
      "cost for 5390 epoch : 0.20214301211859867\n",
      "cost for 5400 epoch : 0.20196338543826317\n",
      "cost for 5410 epoch : 0.20178424427271321\n",
      "cost for 5420 epoch : 0.20160558661792446\n",
      "cost for 5430 epoch : 0.201427410480863\n",
      "cost for 5440 epoch : 0.20124971387940943\n",
      "cost for 5450 epoch : 0.2010724948422847\n",
      "cost for 5460 epoch : 0.2008957514089771\n",
      "cost for 5470 epoch : 0.20071948162966854\n",
      "cost for 5480 epoch : 0.20054368356516222\n",
      "cost for 5490 epoch : 0.20036835528681093\n",
      "cost for 5500 epoch : 0.20019349487644533\n",
      "cost for 5510 epoch : 0.20001910042630386\n",
      "cost for 5520 epoch : 0.1998451700389618\n",
      "cost for 5530 epoch : 0.1996717018272622\n",
      "cost for 5540 epoch : 0.1994986939142464\n",
      "cost for 5550 epoch : 0.19932614443308647\n",
      "cost for 5560 epoch : 0.19915405152701604\n",
      "cost for 5570 epoch : 0.19898241334926378\n",
      "cost for 5580 epoch : 0.19881122806298604\n",
      "cost for 5590 epoch : 0.19864049384120072\n",
      "cost for 5600 epoch : 0.19847020886672226\n",
      "cost for 5610 epoch : 0.19830037133209494\n",
      "cost for 5620 epoch : 0.19813097943952898\n",
      "cost for 5630 epoch : 0.1979620314008364\n",
      "cost for 5640 epoch : 0.19779352543736742\n",
      "cost for 5650 epoch : 0.19762545977994606\n",
      "cost for 5660 epoch : 0.19745783266880934\n",
      "cost for 5670 epoch : 0.1972906423535436\n",
      "cost for 5680 epoch : 0.19712388709302334\n",
      "cost for 5690 epoch : 0.19695756515534996\n",
      "cost for 5700 epoch : 0.19679167481779072\n",
      "cost for 5710 epoch : 0.19662621436671945\n",
      "cost for 5720 epoch : 0.19646118209755517\n",
      "cost for 5730 epoch : 0.19629657631470437\n",
      "cost for 5740 epoch : 0.19613239533150098\n",
      "cost for 5750 epoch : 0.19596863747014887\n",
      "cost for 5760 epoch : 0.19580530106166305\n",
      "cost for 5770 epoch : 0.19564238444581275\n",
      "cost for 5780 epoch : 0.19547988597106453\n",
      "cost for 5790 epoch : 0.19531780399452517\n",
      "cost for 5800 epoch : 0.19515613688188588\n",
      "cost for 5810 epoch : 0.19499488300736614\n",
      "cost for 5820 epoch : 0.19483404075365918\n",
      "cost for 5830 epoch : 0.19467360851187676\n",
      "cost for 5840 epoch : 0.1945135846814943\n",
      "cost for 5850 epoch : 0.19435396767029717\n",
      "cost for 5860 epoch : 0.19419475589432775\n",
      "cost for 5870 epoch : 0.19403594777783117\n",
      "cost for 5880 epoch : 0.19387754175320354\n",
      "cost for 5890 epoch : 0.19371953626093877\n",
      "cost for 5900 epoch : 0.19356192974957667\n",
      "cost for 5910 epoch : 0.19340472067565223\n",
      "cost for 5920 epoch : 0.19324790750364376\n",
      "cost for 5930 epoch : 0.19309148870592183\n",
      "cost for 5940 epoch : 0.19293546276269966\n",
      "cost for 5950 epoch : 0.19277982816198283\n",
      "cost for 5960 epoch : 0.1926245833995195\n",
      "cost for 5970 epoch : 0.1924697269787508\n",
      "cost for 5980 epoch : 0.19231525741076308\n",
      "cost for 5990 epoch : 0.1921611732142384\n",
      "cost for 6000 epoch : 0.19200747291540646\n",
      "cost for 6010 epoch : 0.19185415504799752\n",
      "cost for 6020 epoch : 0.19170121815319444\n",
      "cost for 6030 epoch : 0.19154866077958568\n",
      "cost for 6040 epoch : 0.1913964814831187\n",
      "cost for 6050 epoch : 0.19124467882705384\n",
      "cost for 6060 epoch : 0.19109325138191766\n",
      "cost for 6070 epoch : 0.190942197725458\n",
      "cost for 6080 epoch : 0.1907915164425986\n",
      "cost for 6090 epoch : 0.19064120612539343\n",
      "cost for 6100 epoch : 0.19049126537298272\n",
      "cost for 6110 epoch : 0.1903416927915487\n",
      "cost for 6120 epoch : 0.1901924869942714\n",
      "cost for 6130 epoch : 0.19004364660128484\n",
      "cost for 6140 epoch : 0.18989517023963418\n",
      "cost for 6150 epoch : 0.18974705654323207\n",
      "cost for 6160 epoch : 0.18959930415281692\n",
      "cost for 6170 epoch : 0.1894519117159093\n",
      "cost for 6180 epoch : 0.18930487788677142\n",
      "cost for 6190 epoch : 0.18915820132636407\n",
      "cost for 6200 epoch : 0.18901188070230548\n",
      "cost for 6210 epoch : 0.18886591468883118\n",
      "cost for 6220 epoch : 0.1887203019667518\n",
      "cost for 6230 epoch : 0.1885750412234135\n",
      "cost for 6240 epoch : 0.1884301311526571\n",
      "cost for 6250 epoch : 0.18828557045477884\n",
      "cost for 6260 epoch : 0.1881413578364902\n",
      "cost for 6270 epoch : 0.18799749201087892\n",
      "cost for 6280 epoch : 0.18785397169737003\n",
      "cost for 6290 epoch : 0.18771079562168647\n",
      "cost for 6300 epoch : 0.1875679625158112\n",
      "cost for 6310 epoch : 0.1874254711179485\n",
      "cost for 6320 epoch : 0.18728332017248706\n",
      "cost for 6330 epoch : 0.1871415084299607\n",
      "cost for 6340 epoch : 0.18700003464701265\n",
      "cost for 6350 epoch : 0.1868588975863575\n",
      "cost for 6360 epoch : 0.18671809601674463\n",
      "cost for 6370 epoch : 0.18657762871292122\n",
      "cost for 6380 epoch : 0.186437494455597\n",
      "cost for 6390 epoch : 0.18629769203140692\n",
      "cost for 6400 epoch : 0.18615822023287637\n",
      "cost for 6410 epoch : 0.1860190778583855\n",
      "cost for 6420 epoch : 0.18588026371213295\n",
      "cost for 6430 epoch : 0.18574177660410224\n",
      "cost for 6440 epoch : 0.18560361535002615\n",
      "cost for 6450 epoch : 0.1854657787713525\n",
      "cost for 6460 epoch : 0.18532826569521\n",
      "cost for 6470 epoch : 0.18519107495437415\n",
      "cost for 6480 epoch : 0.18505420538723252\n",
      "cost for 6490 epoch : 0.18491765583775316\n",
      "cost for 6500 epoch : 0.18478142515544968\n",
      "cost for 6510 epoch : 0.1846455121953485\n",
      "cost for 6520 epoch : 0.1845099158179569\n",
      "cost for 6530 epoch : 0.18437463488922903\n",
      "cost for 6540 epoch : 0.18423966828053487\n",
      "cost for 6550 epoch : 0.18410501486862724\n",
      "cost for 6560 epoch : 0.18397067353561028\n",
      "cost for 6570 epoch : 0.1838366431689074\n",
      "cost for 6580 epoch : 0.18370292266123056\n",
      "cost for 6590 epoch : 0.18356951091054827\n",
      "cost for 6600 epoch : 0.18343640682005488\n",
      "cost for 6610 epoch : 0.18330360929813969\n",
      "cost for 6620 epoch : 0.18317111725835677\n",
      "cost for 6630 epoch : 0.18303892961939436\n",
      "cost for 6640 epoch : 0.18290704530504379\n",
      "cost for 6650 epoch : 0.18277546324417143\n",
      "cost for 6660 epoch : 0.1826441823706875\n",
      "cost for 6670 epoch : 0.18251320162351672\n",
      "cost for 6680 epoch : 0.18238251994656984\n",
      "cost for 6690 epoch : 0.18225213628871317\n",
      "cost for 6700 epoch : 0.18212204960374107\n",
      "cost for 6710 epoch : 0.18199225885034648\n",
      "cost for 6720 epoch : 0.18186276299209253\n",
      "cost for 6730 epoch : 0.18173356099738391\n",
      "cost for 6740 epoch : 0.18160465183943938\n",
      "cost for 6750 epoch : 0.18147603449626376\n",
      "cost for 6760 epoch : 0.18134770795061936\n",
      "cost for 6770 epoch : 0.18121967118999988\n",
      "cost for 6780 epoch : 0.18109192320660167\n",
      "cost for 6790 epoch : 0.18096446299729702\n",
      "cost for 6800 epoch : 0.18083728956360803\n",
      "cost for 6810 epoch : 0.18071040191167873\n",
      "cost for 6820 epoch : 0.18058379905224883\n",
      "cost for 6830 epoch : 0.18045748000062758\n",
      "cost for 6840 epoch : 0.180331443776667\n",
      "cost for 6850 epoch : 0.1802056894047363\n",
      "cost for 6860 epoch : 0.18008021591369588\n",
      "cost for 6870 epoch : 0.1799550223368717\n",
      "cost for 6880 epoch : 0.17983010771202929\n",
      "cost for 6890 epoch : 0.17970547108134874\n",
      "cost for 6900 epoch : 0.17958111149139996\n",
      "cost for 6910 epoch : 0.179457027993117\n",
      "cost for 6920 epoch : 0.17933321964177315\n",
      "cost for 6930 epoch : 0.17920968549695693\n",
      "cost for 6940 epoch : 0.17908642462254734\n",
      "cost for 6950 epoch : 0.17896343608668855\n",
      "cost for 6960 epoch : 0.17884071896176773\n",
      "cost for 6970 epoch : 0.17871827232438922\n",
      "cost for 6980 epoch : 0.17859609525535156\n",
      "cost for 6990 epoch : 0.1784741868396235\n",
      "cost for 7000 epoch : 0.1783525461663202\n",
      "cost for 7010 epoch : 0.17823117232868102\n",
      "cost for 7020 epoch : 0.17811006442404506\n",
      "cost for 7030 epoch : 0.1779892215538284\n",
      "cost for 7040 epoch : 0.17786864282350198\n",
      "cost for 7050 epoch : 0.17774832734256735\n",
      "cost for 7060 epoch : 0.17762827422453578\n",
      "cost for 7070 epoch : 0.1775084825869044\n",
      "cost for 7080 epoch : 0.17738895155113482\n",
      "cost for 7090 epoch : 0.1772696802426306\n",
      "cost for 7100 epoch : 0.17715066779071528\n",
      "cost for 7110 epoch : 0.1770319133286106\n",
      "cost for 7120 epoch : 0.17691341599341479\n",
      "cost for 7130 epoch : 0.17679517492608063\n",
      "cost for 7140 epoch : 0.17667718927139495\n",
      "cost for 7150 epoch : 0.17655945817795615\n",
      "cost for 7160 epoch : 0.17644198079815418\n",
      "cost for 7170 epoch : 0.1763247562881489\n",
      "cost for 7180 epoch : 0.17620778380784927\n",
      "cost for 7190 epoch : 0.17609106252089274\n",
      "cost for 7200 epoch : 0.17597459159462467\n",
      "cost for 7210 epoch : 0.17585837020007808\n",
      "cost for 7220 epoch : 0.1757423975119526\n",
      "cost for 7230 epoch : 0.1756266727085955\n",
      "cost for 7240 epoch : 0.1755111949719802\n",
      "cost for 7250 epoch : 0.17539596348768788\n",
      "cost for 7260 epoch : 0.1752809774448862\n",
      "cost for 7270 epoch : 0.17516623603631085\n",
      "cost for 7280 epoch : 0.17505173845824543\n",
      "cost for 7290 epoch : 0.1749374839105018\n",
      "cost for 7300 epoch : 0.1748234715964017\n",
      "cost for 7310 epoch : 0.17470970072275654\n",
      "cost for 7320 epoch : 0.1745961704998493\n",
      "cost for 7330 epoch : 0.17448288014141491\n",
      "cost for 7340 epoch : 0.174369828864622\n",
      "cost for 7350 epoch : 0.17425701589005385\n",
      "cost for 7360 epoch : 0.17414444044169025\n",
      "cost for 7370 epoch : 0.1740321017468885\n",
      "cost for 7380 epoch : 0.1739199990363659\n",
      "cost for 7390 epoch : 0.1738081315441809\n",
      "cost for 7400 epoch : 0.17369649850771504\n",
      "cost for 7410 epoch : 0.1735850991676559\n",
      "cost for 7420 epoch : 0.1734739327679779\n",
      "cost for 7430 epoch : 0.17336299855592607\n",
      "cost for 7440 epoch : 0.17325229578199702\n",
      "cost for 7450 epoch : 0.17314182369992237\n",
      "cost for 7460 epoch : 0.1730315815666511\n",
      "cost for 7470 epoch : 0.17292156864233216\n",
      "cost for 7480 epoch : 0.17281178419029797\n",
      "cost for 7490 epoch : 0.17270222747704628\n",
      "cost for 7500 epoch : 0.1725928977722242\n",
      "cost for 7510 epoch : 0.17248379434861086\n",
      "cost for 7520 epoch : 0.17237491648210082\n",
      "cost for 7530 epoch : 0.1722662634516873\n",
      "cost for 7540 epoch : 0.1721578345394462\n",
      "cost for 7550 epoch : 0.17204962903051907\n",
      "cost for 7560 epoch : 0.1719416462130972\n",
      "cost for 7570 epoch : 0.17183388537840535\n",
      "cost for 7580 epoch : 0.17172634582068536\n",
      "cost for 7590 epoch : 0.17161902683718036\n",
      "cost for 7600 epoch : 0.17151192772811943\n",
      "cost for 7610 epoch : 0.17140504779670107\n",
      "cost for 7620 epoch : 0.17129838634907724\n",
      "cost for 7630 epoch : 0.1711919426943391\n",
      "cost for 7640 epoch : 0.17108571614450005\n",
      "cost for 7650 epoch : 0.1709797060144812\n",
      "cost for 7660 epoch : 0.170873911622096\n",
      "cost for 7670 epoch : 0.1707683322880346\n",
      "cost for 7680 epoch : 0.17066296733584949\n",
      "cost for 7690 epoch : 0.1705578160919393\n",
      "cost for 7700 epoch : 0.17045287788553523\n",
      "cost for 7710 epoch : 0.1703481520486854\n",
      "cost for 7720 epoch : 0.1702436379162399\n",
      "cost for 7730 epoch : 0.1701393348258374\n",
      "cost for 7740 epoch : 0.17003524211788926\n",
      "cost for 7750 epoch : 0.16993135913556548\n",
      "cost for 7760 epoch : 0.16982768522478067\n",
      "cost for 7770 epoch : 0.16972421973417948\n",
      "cost for 7780 epoch : 0.1696209620151222\n",
      "cost for 7790 epoch : 0.16951791142167147\n",
      "cost for 7800 epoch : 0.16941506731057712\n",
      "cost for 7810 epoch : 0.16931242904126295\n",
      "cost for 7820 epoch : 0.16920999597581307\n",
      "cost for 7830 epoch : 0.16910776747895767\n",
      "cost for 7840 epoch : 0.1690057429180593\n",
      "cost for 7850 epoch : 0.16890392166309962\n",
      "cost for 7860 epoch : 0.16880230308666602\n",
      "cost for 7870 epoch : 0.16870088656393767\n",
      "cost for 7880 epoch : 0.1685996714726726\n",
      "cost for 7890 epoch : 0.16849865719319407\n",
      "cost for 7900 epoch : 0.16839784310837785\n",
      "cost for 7910 epoch : 0.1682972286036389\n",
      "cost for 7920 epoch : 0.16819681306691808\n",
      "cost for 7930 epoch : 0.16809659588867007\n",
      "cost for 7940 epoch : 0.16799657646184907\n",
      "cost for 7950 epoch : 0.16789675418189764\n",
      "cost for 7960 epoch : 0.16779712844673297\n",
      "cost for 7970 epoch : 0.16769769865673442\n",
      "cost for 7980 epoch : 0.1675984642147312\n",
      "cost for 7990 epoch : 0.16749942452598984\n",
      "cost for 8000 epoch : 0.16740057899820163\n",
      "cost for 8010 epoch : 0.16730192704147048\n",
      "cost for 8020 epoch : 0.16720346806830064\n",
      "cost for 8030 epoch : 0.16710520149358418\n",
      "cost for 8040 epoch : 0.16700712673458962\n",
      "cost for 8050 epoch : 0.1669092432109494\n",
      "cost for 8060 epoch : 0.1668115503446477\n",
      "cost for 8070 epoch : 0.1667140475600096\n",
      "cost for 8080 epoch : 0.16661673428368778\n",
      "cost for 8090 epoch : 0.16651960994465187\n",
      "cost for 8100 epoch : 0.16642267397417673\n",
      "cost for 8110 epoch : 0.1663259258058303\n",
      "cost for 8120 epoch : 0.16622936487546258\n",
      "cost for 8130 epoch : 0.16613299062119358\n",
      "cost for 8140 epoch : 0.16603680248340288\n",
      "cost for 8150 epoch : 0.16594079990471722\n",
      "cost for 8160 epoch : 0.1658449823300003\n",
      "cost for 8170 epoch : 0.16574934920634055\n",
      "cost for 8180 epoch : 0.16565389998304103\n",
      "cost for 8190 epoch : 0.16555863411160773\n",
      "cost for 8200 epoch : 0.1654635510457384\n",
      "cost for 8210 epoch : 0.16536865024131223\n",
      "cost for 8220 epoch : 0.16527393115637928\n",
      "cost for 8230 epoch : 0.16517939325114825\n",
      "cost for 8240 epoch : 0.165085035987977\n",
      "cost for 8250 epoch : 0.16499085883136172\n",
      "cost for 8260 epoch : 0.16489686124792585\n",
      "cost for 8270 epoch : 0.16480304270641016\n",
      "cost for 8280 epoch : 0.16470940267766163\n",
      "cost for 8290 epoch : 0.16461594063462356\n",
      "cost for 8300 epoch : 0.16452265605232502\n",
      "cost for 8310 epoch : 0.16442954840787022\n",
      "cost for 8320 epoch : 0.16433661718042927\n",
      "cost for 8330 epoch : 0.16424386185122686\n",
      "cost for 8340 epoch : 0.1641512819035328\n",
      "cost for 8350 epoch : 0.16405887682265188\n",
      "cost for 8360 epoch : 0.16396664609591352\n",
      "cost for 8370 epoch : 0.1638745892126622\n",
      "cost for 8380 epoch : 0.16378270566424807\n",
      "cost for 8390 epoch : 0.16369099494401596\n",
      "cost for 8400 epoch : 0.16359945654729638\n",
      "cost for 8410 epoch : 0.16350808997139585\n",
      "cost for 8420 epoch : 0.16341689471558693\n",
      "cost for 8430 epoch : 0.16332587028109863\n",
      "cost for 8440 epoch : 0.16323501617110733\n",
      "cost for 8450 epoch : 0.1631443318907271\n",
      "cost for 8460 epoch : 0.16305381694699958\n",
      "cost for 8470 epoch : 0.16296347084888585\n",
      "cost for 8480 epoch : 0.16287329310725607\n",
      "cost for 8490 epoch : 0.16278328323488078\n",
      "cost for 8500 epoch : 0.16269344074642128\n",
      "cost for 8510 epoch : 0.1626037651584212\n",
      "cost for 8520 epoch : 0.16251425598929617\n",
      "cost for 8530 epoch : 0.16242491275932672\n",
      "cost for 8540 epoch : 0.16233573499064682\n",
      "cost for 8550 epoch : 0.16224672220723713\n",
      "cost for 8560 epoch : 0.16215787393491463\n",
      "cost for 8570 epoch : 0.16206918970132442\n",
      "cost for 8580 epoch : 0.16198066903593122\n",
      "cost for 8590 epoch : 0.16189231147000982\n",
      "cost for 8600 epoch : 0.1618041165366369\n",
      "cost for 8610 epoch : 0.16171608377068258\n",
      "cost for 8620 epoch : 0.16162821270880134\n",
      "cost for 8630 epoch : 0.16154050288942348\n",
      "cost for 8640 epoch : 0.16145295385274727\n",
      "cost for 8650 epoch : 0.16136556514072997\n",
      "cost for 8660 epoch : 0.16127833629707938\n",
      "cost for 8670 epoch : 0.16119126686724564\n",
      "cost for 8680 epoch : 0.16110435639841328\n",
      "cost for 8690 epoch : 0.16101760443949223\n",
      "cost for 8700 epoch : 0.1609310105411103\n",
      "cost for 8710 epoch : 0.16084457425560464\n",
      "cost for 8720 epoch : 0.16075829513701348\n",
      "cost for 8730 epoch : 0.1606721727410687\n",
      "cost for 8740 epoch : 0.16058620662518744\n",
      "cost for 8750 epoch : 0.16050039634846355\n",
      "cost for 8760 epoch : 0.16041474147166065\n",
      "cost for 8770 epoch : 0.16032924155720368\n",
      "cost for 8780 epoch : 0.1602438961691714\n",
      "cost for 8790 epoch : 0.16015870487328784\n",
      "cost for 8800 epoch : 0.1600736672369155\n",
      "cost for 8810 epoch : 0.15998878282904713\n",
      "cost for 8820 epoch : 0.1599040512202981\n",
      "cost for 8830 epoch : 0.1598194719828989\n",
      "cost for 8840 epoch : 0.1597350446906871\n",
      "cost for 8850 epoch : 0.15965076891910082\n",
      "cost for 8860 epoch : 0.15956664424517003\n",
      "cost for 8870 epoch : 0.15948267024751006\n",
      "cost for 8880 epoch : 0.15939884650631367\n",
      "cost for 8890 epoch : 0.15931517260334366\n",
      "cost for 8900 epoch : 0.1592316481219259\n",
      "cost for 8910 epoch : 0.15914827264694167\n",
      "cost for 8920 epoch : 0.15906504576482083\n",
      "cost for 8930 epoch : 0.15898196706353382\n",
      "cost for 8940 epoch : 0.1588990361325854\n",
      "cost for 8950 epoch : 0.15881625256300724\n",
      "cost for 8960 epoch : 0.15873361594735058\n",
      "cost for 8970 epoch : 0.15865112587967903\n",
      "cost for 8980 epoch : 0.15856878195556234\n",
      "cost for 8990 epoch : 0.1584865837720688\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[ 0.36043894,  0.49106346, -0.8515024 ],\n",
       "       [ 0.74121438,  0.62128861, -1.36250298],\n",
       "       [ 1.77970306, -0.24955262, -1.53015043],\n",
       "       [-2.44103432, -0.12703503,  2.56806935],\n",
       "       [-1.11607952, -0.69511384,  1.81119335]])"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logreg.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = logreg.predict(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9866666666666667"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sum(y == predictions.squeeze()) / len(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "       2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "       2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2])"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions.squeeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}