SGD.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import tensorflow as tf\n",
    "\n",
    "import os\n",
    "for dirname, _, filenames in os.walk('/kaggle/input'):\n",
    "    for filename in filenames:\n",
    "        print(os.path.join(dirname, filename))\n",
    "\n",
    "# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n",
    "# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [],
   "source": [
    "x_train = np.load('x_train.npy')\n",
    "y_train = np.load('y_train.npy')\n",
    "x_test = np.load('x_test.npy')\n",
    "y_test = np.load('y_test.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_classes = 10\n",
    "n1 = 100\n",
    "no_users = 10\n",
    "y_train = tf.keras.utils.to_categorical(y_train, num_classes)\n",
    "y_test = tf.keras.utils.to_categorical(y_test, num_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = np.reshape(x_train, (x_train.shape[0], -1))/255\n",
    "x_test = np.reshape(x_test, (x_test.shape[0], -1))/255"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Network parameters.\n",
    "n_hidden_1 = 128 # 1st layer number of neurons.\n",
    "n_hidden_2 = 64 # 2nd layer number of neurons."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "W1 = tf.Variable(tf.random.truncated_normal([784,n_hidden_1], stddev=0.1))\n",
    "W2 = tf.Variable(tf.random.truncated_normal([n_hidden_1,n_hidden_2], stddev=0.1))\n",
    "W3 = tf.Variable(tf.random.truncated_normal([n_hidden_2,10], stddev=0.1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "class User:\n",
    "    def __init__(self):       \n",
    "        self.GW1 = tf.Variable(tf.random.truncated_normal([784,n_hidden_1], stddev=0.1))\n",
    "        self.GW2 = tf.Variable(tf.random.truncated_normal([n_hidden_1,n_hidden_2], stddev=0.1))\n",
    "        self.GW3 = tf.Variable(tf.random.truncated_normal([n_hidden_2,10], stddev=0.1))\n",
    "        self.H = 1.0#np.random.normal()   \n",
    "        self.abs_H_square = self.H**2\n",
    "        \n",
    "\n",
    "    def neural_net(self, x):\n",
    "        y1 = tf.nn.relu(tf.matmul(x, W1))\n",
    "        y2 = tf.nn.relu(tf.matmul(y1, W2))\n",
    "        ylogits = tf.matmul(y2, W3)\n",
    "        return tf.nn.softmax(ylogits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# mini-batch loss function.\n",
    "def mini_batches(X, Y, mb_size = 100):\n",
    "\n",
    "    m = X.shape[0]\n",
    "\n",
    "    perm = list(np.random.permutation(m))\n",
    "    #perm = perm_init[0:100]\n",
    "    X_temp = X[perm,:]\n",
    "    Y_temp = Y[perm,:].reshape((m, Y.shape[1]))\n",
    "    \n",
    "    X_r = tf.convert_to_tensor(X_temp[0:mb_size,:], dtype=np.float32)\n",
    "    Y_r = tf.convert_to_tensor(Y_temp[0:mb_size,:], dtype=np.float32)\n",
    "    return X_r,Y_r"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cross-Entropy loss function.\n",
    "def cross_entropy(y_pred, y_true):\n",
    "    # Clip prediction values to avoid log(0) error.\n",
    "    y_pred = tf.clip_by_value(y_pred, 1e-9, 1.)\n",
    "    # Compute cross-entropy.\n",
    "    return -tf.reduce_sum(y_true * tf.math.log(y_pred))#tf.reduce_sum(tf.math.square(y_true-y_pred))#-tf.reduce_sum(y_true * tf.math.log(y_pred))\n",
    "    #tf.reduce_mean(-tf.reduce_sum(y_true * tf.math.log(y_pred)))\n",
    "    #return tf.reduce_sum(-tf.reduce_mean(y_true * tf.math.log(y_pred)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Accuracy metric.\n",
    "def accuracy(y_pred, y_true):\n",
    "    # Predicted class is the index of highest score in prediction vector (i.e. argmax).\n",
    "    correct_prediction = tf.equal(tf.argmax(y_pred, 1), tf.argmax(y_true, 1))\n",
    "    return tf.reduce_mean(tf.cast(correct_prediction, tf.float32), axis=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "users = [User() for i in range(no_users)]\n",
    "optimizer = tf.optimizers.Adam(learning_rate=0.001)\n",
    "central_modal = [tf.Variable(tf.random.truncated_normal([784*n_hidden_1,1], stddev=0.1)), tf.Variable(tf.random.truncated_normal([n_hidden_1*n_hidden_2,1], stddev=0.1)), tf.Variable(tf.random.truncated_normal([10*n_hidden_2,1], stddev=0.1))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Optimization process. \n",
    "def run_optimization(x, y, user, W1, W2, W3, rho):\n",
    "    # Wrap computation inside a GradientTape for automatic differentiation.\n",
    "    \n",
    "    with tf.GradientTape() as g:\n",
    "        g.watch([W1,W2,W3])\n",
    "        pred = user.neural_net(x)\n",
    "        loss = cross_entropy(pred, y) \n",
    "        \n",
    "    \n",
    "    # Variables to update, i.e. trainable variables.\n",
    "    trainable_variables = [W1, W2 ,W3]\n",
    "    \n",
    "    # Compute gradients.\n",
    "    gradients1,gradients2,gradients3  = g.gradient(loss, trainable_variables)\n",
    "    #print(gradients2)\n",
    "    return gradients1,gradients2,gradients3, loss\n",
    "    #\n",
    "    \n",
    "    # Update W following gradients.\n",
    "    # optimizer.apply_gradients(zip(gradients, trainable_variables))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train_k = []\n",
    "y_train_k = []\n",
    "data_per_worker = int(x_train.shape[0]/no_users)\n",
    "for i in range(no_users):\n",
    "    first = i*data_per_worker\n",
    "    last = first + data_per_worker\n",
    "    x_train_k.append(x_train[first:last])\n",
    "    y_train_k.append(y_train[first:last])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_test = tf.convert_to_tensor(x_test, dtype=np.float32)\n",
    "y_test = tf.convert_to_tensor(y_test, dtype=np.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "[233.88618164]\n",
      "0.0923\n",
      "1\n",
      "[214.98000488]\n",
      "0.34170002\n",
      "2\n",
      "[198.16281738]\n",
      "0.5168\n",
      "3\n",
      "[165.91937256]\n",
      "0.61289996\n",
      "4\n",
      "[153.41501465]\n",
      "0.47279996\n",
      "5\n",
      "[371.6394043]\n",
      "0.1927\n",
      "6\n",
      "[255.50915527]\n",
      "0.3132\n",
      "7\n",
      "[217.38422852]\n",
      "0.1545\n",
      "8\n",
      "[202.81519775]\n",
      "0.3934\n",
      "9\n",
      "[170.07593994]\n",
      "0.4739\n",
      "10\n",
      "[152.20153809]\n",
      "0.48929995\n",
      "11\n",
      "[144.68491211]\n",
      "0.49359998\n",
      "12\n",
      "[220.55014648]\n",
      "0.2874\n",
      "13\n",
      "[197.79482422]\n",
      "0.40399998\n",
      "14\n",
      "[192.56936035]\n",
      "0.3239\n",
      "15\n",
      "[168.24342041]\n",
      "0.44579998\n",
      "16\n",
      "[139.62453613]\n",
      "0.51420003\n",
      "17\n",
      "[126.59660645]\n",
      "0.58169997\n",
      "18\n",
      "[114.67213135]\n",
      "0.6271\n",
      "19\n",
      "[123.82705078]\n",
      "0.5770999\n",
      "20\n",
      "[122.93376465]\n",
      "0.5684\n",
      "21\n",
      "[129.88619385]\n",
      "0.5779\n",
      "22\n",
      "[105.56350098]\n",
      "0.6447\n",
      "23\n",
      "[89.31483154]\n",
      "0.737\n",
      "24\n",
      "[92.26032104]\n",
      "0.67889994\n",
      "25\n",
      "[113.00563965]\n",
      "0.67920005\n",
      "26\n",
      "[96.08599854]\n",
      "0.66620004\n",
      "27\n",
      "[82.13078613]\n",
      "0.706\n",
      "28\n",
      "[94.98851318]\n",
      "0.6715001\n",
      "29\n",
      "[77.96811523]\n",
      "0.7464\n",
      "30\n",
      "[91.40199585]\n",
      "0.7189001\n",
      "31\n",
      "[79.50991821]\n",
      "0.7304\n",
      "32\n",
      "[69.25390625]\n",
      "0.80599993\n",
      "33\n",
      "[57.74441528]\n",
      "0.83280003\n",
      "34\n",
      "[47.51669922]\n",
      "0.83199996\n",
      "35\n",
      "[52.61737061]\n",
      "0.83140004\n",
      "36\n",
      "[67.17804565]\n",
      "0.7879\n",
      "37\n",
      "[77.11916504]\n",
      "0.74249995\n",
      "38\n",
      "[82.90761719]\n",
      "0.7508999\n",
      "39\n",
      "[68.01687622]\n",
      "0.7703\n",
      "40\n",
      "[57.76150513]\n",
      "0.8335999\n",
      "41\n",
      "[54.16195679]\n",
      "0.85550004\n",
      "42\n",
      "[51.40889282]\n",
      "0.82440007\n",
      "43\n",
      "[57.51361694]\n",
      "0.8009\n",
      "44\n",
      "[51.51304321]\n",
      "0.8256\n",
      "45\n",
      "[44.21025696]\n",
      "0.8653\n",
      "46\n",
      "[45.52014465]\n",
      "0.8642\n",
      "47\n",
      "[44.74914551]\n",
      "0.859\n",
      "48\n",
      "[42.71485596]\n",
      "0.89049995\n",
      "49\n",
      "[33.6874176]\n",
      "0.89530003\n",
      "50\n",
      "[32.76934509]\n",
      "0.8976\n",
      "51\n",
      "[37.28675842]\n",
      "0.90009993\n",
      "52\n",
      "[35.00209656]\n",
      "0.8991\n",
      "53\n",
      "[32.00873718]\n",
      "0.90279996\n",
      "54\n",
      "[33.79648437]\n",
      "0.9005\n",
      "55\n",
      "[34.84750671]\n",
      "0.89070004\n",
      "56\n",
      "[36.59139404]\n",
      "0.8960999\n",
      "57\n",
      "[33.07339783]\n",
      "0.9002\n",
      "58\n",
      "[38.72156067]\n",
      "0.9025\n",
      "59\n",
      "[40.3610321]\n",
      "0.8826\n",
      "60\n",
      "[51.743396]\n",
      "0.8346\n",
      "61\n",
      "[54.67161255]\n",
      "0.837\n",
      "62\n",
      "[49.39229431]\n",
      "0.8468\n",
      "63\n",
      "[36.67745056]\n",
      "0.88839996\n",
      "64\n",
      "[33.84798279]\n",
      "0.9099\n",
      "65\n",
      "[35.61949463]\n",
      "0.9115\n",
      "66\n",
      "[29.52372742]\n",
      "0.9078\n",
      "67\n",
      "[32.91260681]\n",
      "0.9102\n",
      "68\n",
      "[32.78168335]\n",
      "0.904\n",
      "69\n",
      "[33.83984985]\n",
      "0.8999999\n",
      "70\n",
      "[37.80060425]\n",
      "0.8807\n",
      "71\n",
      "[64.16912231]\n",
      "0.7953\n",
      "72\n",
      "[59.3725708]\n",
      "0.8026999\n",
      "73\n",
      "[56.45714722]\n",
      "0.7968\n",
      "74\n",
      "[44.52730713]\n",
      "0.8949\n",
      "75\n",
      "[39.63650818]\n",
      "0.90620005\n",
      "76\n",
      "[36.25291443]\n",
      "0.89399993\n",
      "77\n",
      "[29.66371765]\n",
      "0.917\n",
      "78\n",
      "[34.52924194]\n",
      "0.91730005\n",
      "79\n",
      "[29.30963135]\n",
      "0.917\n",
      "80\n",
      "[26.5979187]\n",
      "0.92069995\n",
      "81\n",
      "[24.94953156]\n",
      "0.92419994\n",
      "82\n",
      "[26.69433899]\n",
      "0.9290999\n",
      "83\n",
      "[21.6535553]\n",
      "0.91920006\n",
      "84\n",
      "[24.32187958]\n",
      "0.92319995\n",
      "85\n",
      "[30.0217926]\n",
      "0.9226\n",
      "86\n",
      "[33.11250305]\n",
      "0.9129\n",
      "87\n",
      "[26.10226135]\n",
      "0.9251\n",
      "88\n",
      "[26.35067139]\n",
      "0.91950005\n",
      "89\n",
      "[21.12319641]\n",
      "0.92519987\n",
      "90\n",
      "[23.69012146]\n",
      "0.92329997\n",
      "91\n",
      "[22.26813354]\n",
      "0.9251\n",
      "92\n",
      "[24.21194916]\n",
      "0.9313\n",
      "93\n",
      "[23.87413788]\n",
      "0.93029994\n",
      "94\n",
      "[23.78052673]\n",
      "0.92889994\n",
      "95\n",
      "[26.49723511]\n",
      "0.9304\n",
      "96\n",
      "[23.044487]\n",
      "0.92669994\n",
      "97\n",
      "[25.68246765]\n",
      "0.92459995\n",
      "98\n",
      "[26.24157104]\n",
      "0.92519987\n",
      "99\n",
      "[21.72705841]\n",
      "0.93\n",
      "100\n",
      "[22.38204956]\n",
      "0.9279\n",
      "101\n",
      "[21.9574585]\n",
      "0.93029994\n",
      "102\n",
      "[18.07964783]\n",
      "0.9292\n",
      "103\n",
      "[23.89588623]\n",
      "0.93169993\n",
      "104\n",
      "[19.08408508]\n",
      "0.9321\n",
      "105\n",
      "[23.90220337]\n",
      "0.9325\n",
      "106\n",
      "[19.45461426]\n",
      "0.9347\n",
      "107\n",
      "[20.64928894]\n",
      "0.9292\n",
      "108\n",
      "[22.09169617]\n",
      "0.9330999\n",
      "109\n",
      "[19.17837067]\n",
      "0.9342\n",
      "110\n",
      "[22.92601776]\n",
      "0.93509996\n",
      "111\n",
      "[20.25813446]\n",
      "0.93579996\n",
      "112\n",
      "[17.73648682]\n",
      "0.9355\n",
      "113\n",
      "[22.85075378]\n",
      "0.9351999\n",
      "114\n",
      "[27.32354431]\n",
      "0.9293\n",
      "115\n",
      "[24.48785706]\n",
      "0.92559993\n",
      "116\n",
      "[25.26592255]\n",
      "0.9257\n",
      "117\n",
      "[22.85463409]\n",
      "0.9285\n",
      "118\n",
      "[17.54357605]\n",
      "0.939\n",
      "119\n",
      "[17.42571259]\n",
      "0.943\n",
      "120\n",
      "[20.65670319]\n",
      "0.9423\n",
      "121\n",
      "[23.70417786]\n",
      "0.93990004\n",
      "122\n",
      "[21.74433441]\n",
      "0.9397\n",
      "123\n",
      "[23.5891983]\n",
      "0.9420999\n",
      "124\n",
      "[21.96424255]\n",
      "0.9408\n",
      "125\n",
      "[19.83377686]\n",
      "0.93920004\n",
      "126\n",
      "[23.65282135]\n",
      "0.93789995\n",
      "127\n",
      "[24.52144775]\n",
      "0.9297999\n",
      "128\n",
      "[20.02746429]\n",
      "0.92589986\n",
      "129\n",
      "[25.86256714]\n",
      "0.91730005\n",
      "130\n",
      "[25.85318298]\n",
      "0.91859996\n",
      "131\n",
      "[24.74356537]\n",
      "0.8992999\n",
      "132\n",
      "[28.10649109]\n",
      "0.9110001\n",
      "133\n",
      "[27.21092529]\n",
      "0.92200005\n",
      "134\n",
      "[21.78802185]\n",
      "0.9304999\n",
      "135\n",
      "[17.72246704]\n",
      "0.9429\n",
      "136\n",
      "[19.82916718]\n",
      "0.9437\n",
      "137\n",
      "[17.38136597]\n",
      "0.9431\n",
      "138\n",
      "[20.02658081]\n",
      "0.9452001\n",
      "139\n",
      "[22.35248718]\n",
      "0.9441001\n",
      "140\n",
      "[21.12239685]\n",
      "0.9417\n",
      "141\n",
      "[19.02099609]\n",
      "0.94130003\n",
      "142\n",
      "[17.30567474]\n",
      "0.9455001\n",
      "143\n",
      "[16.97142487]\n",
      "0.9471\n",
      "144\n",
      "[18.57231293]\n",
      "0.9438\n",
      "145\n",
      "[17.98648834]\n",
      "0.946\n",
      "146\n",
      "[14.36942444]\n",
      "0.9448001\n",
      "147\n",
      "[19.26452026]\n",
      "0.94700015\n",
      "148\n",
      "[17.68533478]\n",
      "0.9445001\n",
      "149\n",
      "[18.79015045]\n",
      "0.9419001\n",
      "150\n",
      "[18.58576508]\n",
      "0.94130003\n",
      "151\n",
      "[16.06457367]\n",
      "0.9453\n",
      "152\n",
      "[17.77783356]\n",
      "0.9465\n",
      "153\n",
      "[16.57757416]\n",
      "0.94760007\n",
      "154\n",
      "[15.92661133]\n",
      "0.9480001\n",
      "155\n",
      "[18.8313446]\n",
      "0.94910014\n",
      "156\n",
      "[13.9247879]\n",
      "0.9480001\n",
      "157\n",
      "[12.57947922]\n",
      "0.9474\n",
      "158\n",
      "[19.60766602]\n",
      "0.94820005\n",
      "159\n",
      "[16.6804184]\n",
      "0.9465\n",
      "160\n",
      "[21.55581818]\n",
      "0.95050013\n",
      "161\n",
      "[18.47348175]\n",
      "0.9473001\n",
      "162\n",
      "[16.58234558]\n",
      "0.9464\n",
      "163\n",
      "[18.80357361]\n",
      "0.9448999\n",
      "164\n",
      "[13.34017944]\n",
      "0.95040005\n",
      "165\n",
      "[13.1223465]\n",
      "0.94960004\n",
      "166\n",
      "[16.27261353]\n",
      "0.9486\n",
      "167\n",
      "[17.11811523]\n",
      "0.9478\n",
      "168\n",
      "[18.52843018]\n",
      "0.9448999\n",
      "169\n",
      "[14.77229309]\n",
      "0.9458\n",
      "170\n",
      "[17.30438538]\n",
      "0.9481\n",
      "171\n",
      "[18.37578888]\n",
      "0.94830006\n",
      "172\n",
      "[19.19131775]\n",
      "0.94840014\n",
      "173\n",
      "[15.06756592]\n",
      "0.94910014\n",
      "174\n",
      "[15.63102417]\n",
      "0.95019996\n",
      "175\n",
      "[16.68217621]\n",
      "0.945\n",
      "176\n",
      "[18.42115784]\n",
      "0.95100003\n",
      "177\n",
      "[14.90700378]\n",
      "0.95100003\n",
      "178\n",
      "[13.80279236]\n",
      "0.95040005\n",
      "179\n",
      "[17.0150116]\n",
      "0.9533001\n",
      "180\n",
      "[14.93966675]\n",
      "0.95100003\n",
      "181\n",
      "[19.31453857]\n",
      "0.94970006\n",
      "182\n",
      "[13.86994019]\n",
      "0.95040005\n",
      "183\n",
      "[16.13982697]\n",
      "0.95040005\n",
      "184\n",
      "[15.34678345]\n",
      "0.95089996\n",
      "185\n",
      "[15.13011932]\n",
      "0.95110005\n",
      "186\n",
      "[11.1951622]\n",
      "0.95019996\n",
      "187\n",
      "[17.81716614]\n",
      "0.95129997\n",
      "188\n",
      "[13.90062714]\n",
      "0.9514\n",
      "189\n",
      "[13.45416412]\n",
      "0.9538\n",
      "190\n",
      "[12.49069977]\n",
      "0.9538\n",
      "191\n",
      "[13.56180267]\n",
      "0.9535\n",
      "192\n",
      "[16.91916199]\n",
      "0.9535\n",
      "193\n",
      "[12.15191727]\n",
      "0.95129997\n",
      "194\n",
      "[14.8188446]\n",
      "0.94949996\n",
      "195\n",
      "[14.99619751]\n",
      "0.9499\n",
      "196\n",
      "[15.35630951]\n",
      "0.9441001\n",
      "197\n",
      "[17.91070099]\n",
      "0.9437\n",
      "198\n",
      "[15.00709686]\n",
      "0.94900006\n",
      "199\n",
      "[16.65108948]\n",
      "0.94630015\n",
      "200\n",
      "[20.00622101]\n",
      "0.9471\n",
      "201\n",
      "[15.74063263]\n",
      "0.9478\n",
      "202\n",
      "[16.98294373]\n",
      "0.94610006\n",
      "203\n",
      "[15.05049438]\n",
      "0.94610006\n",
      "204\n",
      "[16.7313446]\n",
      "0.95220006\n",
      "205\n",
      "[12.40358887]\n",
      "0.9521\n",
      "206\n",
      "[13.71652985]\n",
      "0.94960004\n",
      "207\n",
      "[16.78766479]\n",
      "0.94949996\n",
      "208\n",
      "[20.98197327]\n",
      "0.94900006\n",
      "209\n",
      "[15.94586945]\n",
      "0.95089996\n",
      "210\n",
      "[17.70294189]\n",
      "0.9535\n",
      "211\n",
      "[15.27630615]\n",
      "0.95419997\n",
      "212\n",
      "[11.5577652]\n",
      "0.9554001\n",
      "213\n",
      "[16.31827087]\n",
      "0.9533001\n",
      "214\n",
      "[10.96184082]\n",
      "0.95570004\n",
      "215\n",
      "[14.86826019]\n",
      "0.95549995\n",
      "216\n",
      "[18.23631592]\n",
      "0.94960004\n",
      "217\n",
      "[16.8658905]\n",
      "0.9439\n",
      "218\n",
      "[14.25201111]\n",
      "0.9486\n",
      "219\n",
      "[15.25275879]\n",
      "0.95699996\n",
      "220\n",
      "[10.57557755]\n",
      "0.95570004\n",
      "221\n",
      "[15.71900787]\n",
      "0.95559996\n",
      "222\n",
      "[13.06508636]\n",
      "0.95559996\n",
      "223\n",
      "[15.26266327]\n",
      "0.95629996\n",
      "224\n",
      "[15.12688904]\n",
      "0.9535\n",
      "225\n",
      "[15.40360565]\n",
      "0.9533001\n",
      "226\n",
      "[16.02177887]\n",
      "0.95269996\n",
      "227\n",
      "[13.21685181]\n",
      "0.95269996\n",
      "228\n",
      "[12.62574997]\n",
      "0.9552\n",
      "229\n",
      "[12.59870453]\n",
      "0.95180005\n",
      "230\n",
      "[11.64574356]\n",
      "0.95559996\n",
      "231\n",
      "[11.87482147]\n",
      "0.9574\n",
      "232\n",
      "[15.23928986]\n",
      "0.9540001\n",
      "233\n",
      "[15.25919189]\n",
      "0.95559996\n",
      "234\n",
      "[11.27142944]\n",
      "0.95570004\n",
      "235\n",
      "[16.88712769]\n",
      "0.95629996\n",
      "236\n",
      "[11.83954163]\n",
      "0.95489997\n",
      "237\n",
      "[12.88813629]\n",
      "0.95689994\n",
      "238\n",
      "[12.20308762]\n",
      "0.95579994\n",
      "239\n",
      "[15.3441391]\n",
      "0.94700015\n",
      "240\n",
      "[15.70653534]\n",
      "0.9519001\n",
      "241\n",
      "[16.90675507]\n",
      "0.95\n",
      "242\n",
      "[16.96562653]\n",
      "0.93649995\n",
      "243\n",
      "[18.07368317]\n",
      "0.9427001\n",
      "244\n",
      "[16.72968597]\n",
      "0.9451\n",
      "245\n",
      "[18.85174713]\n",
      "0.94069993\n",
      "246\n",
      "[16.3435791]\n",
      "0.9337999\n",
      "247\n",
      "[18.16613007]\n",
      "0.9427001\n",
      "248\n",
      "[16.33377991]\n",
      "0.9547001\n",
      "249\n",
      "[12.38187485]\n",
      "0.9585\n",
      "250\n",
      "[13.12574921]\n",
      "0.9589001\n",
      "251\n",
      "[11.79943161]\n",
      "0.9578999\n",
      "252\n",
      "[12.76274414]\n",
      "0.9575001\n",
      "253\n",
      "[10.21219406]\n",
      "0.9588\n",
      "254\n",
      "[11.6785614]\n",
      "0.95759994\n",
      "255\n",
      "[14.27465363]\n",
      "0.9588\n",
      "256\n",
      "[11.63887939]\n",
      "0.95710003\n",
      "257\n",
      "[12.54883423]\n",
      "0.9589001\n",
      "258\n",
      "[11.25333099]\n",
      "0.9581\n",
      "259\n",
      "[10.76090927]\n",
      "0.9587\n",
      "260\n",
      "[11.63195801]\n",
      "0.9587\n",
      "261\n",
      "[9.46658554]\n",
      "0.9595\n",
      "262\n",
      "[11.1362442]\n",
      "0.9610001\n",
      "263\n",
      "[10.08612213]\n",
      "0.95839995\n",
      "264\n",
      "[11.48879547]\n",
      "0.9610999\n",
      "265\n",
      "[12.64810867]\n",
      "0.9588\n",
      "266\n",
      "[13.24568481]\n",
      "0.9552\n",
      "267\n",
      "[14.57969971]\n",
      "0.9459001\n",
      "268\n",
      "[19.43094025]\n",
      "0.9451\n",
      "269\n",
      "[33.28409729]\n",
      "0.9045\n",
      "270\n",
      "[48.87897339]\n",
      "0.8625\n",
      "271\n",
      "[76.03153076]\n",
      "0.78639996\n",
      "272\n",
      "[119.92504883]\n",
      "0.7489\n",
      "273\n",
      "[99.68797607]\n",
      "0.69610006\n",
      "274\n",
      "[97.4154541]\n",
      "0.672\n",
      "275\n",
      "[74.61859131]\n",
      "0.7682\n",
      "276\n",
      "[55.55952148]\n",
      "0.8216001\n",
      "277\n",
      "[33.34538574]\n",
      "0.90760005\n",
      "278\n",
      "[24.2871109]\n",
      "0.9322001\n",
      "279\n",
      "[21.05373688]\n",
      "0.94139993\n",
      "280\n",
      "[18.3487442]\n",
      "0.9451\n",
      "281\n",
      "[19.21525116]\n",
      "0.9455999\n",
      "282\n",
      "[16.42952881]\n",
      "0.95030004\n",
      "283\n",
      "[16.56851501]\n",
      "0.9466001\n",
      "284\n",
      "[18.29153137]\n",
      "0.94760007\n",
      "285\n",
      "[16.60609131]\n",
      "0.9467\n",
      "286\n",
      "[16.30762329]\n",
      "0.95129997\n",
      "287\n",
      "[14.00298309]\n",
      "0.9507\n",
      "288\n",
      "[13.82110138]\n",
      "0.95170003\n",
      "289\n",
      "[12.19229889]\n",
      "0.94840014\n",
      "290\n",
      "[15.41594849]\n",
      "0.95299995\n",
      "291\n",
      "[14.30749054]\n",
      "0.95419997\n",
      "292\n",
      "[15.03039246]\n",
      "0.95509994\n",
      "293\n",
      "[17.19985352]\n",
      "0.9535\n",
      "294\n",
      "[11.58469467]\n",
      "0.94980013\n",
      "295\n",
      "[13.4015686]\n",
      "0.95460004\n",
      "296\n",
      "[12.43005981]\n",
      "0.95409995\n",
      "297\n",
      "[11.59113159]\n",
      "0.95500004\n",
      "298\n",
      "[12.30867996]\n",
      "0.95500004\n",
      "299\n",
      "[14.93526917]\n",
      "0.95419997\n",
      "300\n",
      "[16.80596313]\n",
      "0.95290005\n",
      "301\n",
      "[11.20822372]\n",
      "0.9528\n",
      "302\n",
      "[14.78443298]\n",
      "0.9545\n",
      "303\n",
      "[13.91057129]\n",
      "0.9540001\n",
      "304\n",
      "[14.39008789]\n",
      "0.9512001\n",
      "305\n",
      "[15.02006378]\n",
      "0.95369995\n",
      "306\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[11.29508362]\n",
      "0.95530003\n",
      "307\n",
      "[11.07876129]\n",
      "0.95629996\n",
      "308\n",
      "[13.62836151]\n",
      "0.95509994\n",
      "309\n",
      "[12.32988739]\n",
      "0.95640004\n",
      "310\n",
      "[11.13529739]\n",
      "0.95699996\n",
      "311\n",
      "[10.90436554]\n",
      "0.95649993\n",
      "312\n",
      "[12.32327728]\n",
      "0.9578999\n",
      "313\n",
      "[9.40004959]\n",
      "0.95710003\n",
      "314\n",
      "[14.36432495]\n",
      "0.95979995\n",
      "315\n",
      "[14.26242065]\n",
      "0.9592\n",
      "316\n",
      "[9.09278412]\n",
      "0.95629996\n",
      "317\n",
      "[10.88929214]\n",
      "0.9595\n",
      "318\n",
      "[9.55150833]\n",
      "0.95780003\n",
      "319\n",
      "[9.20744247]\n",
      "0.9608\n",
      "320\n",
      "[13.98526154]\n",
      "0.95780003\n",
      "321\n",
      "[9.10741882]\n",
      "0.9561001\n",
      "322\n",
      "[10.83208618]\n",
      "0.9561001\n",
      "323\n",
      "[12.32502213]\n",
      "0.95699996\n",
      "324\n",
      "[14.1322464]\n",
      "0.95649993\n",
      "325\n",
      "[12.57257385]\n",
      "0.95699996\n",
      "326\n",
      "[10.2576828]\n",
      "0.95710003\n",
      "327\n",
      "[10.78593674]\n",
      "0.95979995\n",
      "328\n",
      "[13.12767181]\n",
      "0.95979995\n",
      "329\n",
      "[10.77056503]\n",
      "0.9588\n",
      "330\n",
      "[9.95399094]\n",
      "0.9617999\n",
      "331\n",
      "[10.60417786]\n",
      "0.96259993\n",
      "332\n",
      "[10.90625305]\n",
      "0.9596001\n",
      "333\n",
      "[10.42366867]\n",
      "0.9601\n",
      "334\n",
      "[11.7083374]\n",
      "0.9601\n",
      "335\n",
      "[12.76949387]\n",
      "0.9595\n",
      "336\n",
      "[13.96759796]\n",
      "0.9582001\n",
      "337\n",
      "[16.35631104]\n",
      "0.95100003\n",
      "338\n",
      "[15.04008331]\n",
      "0.9481\n",
      "339\n",
      "[12.1263176]\n",
      "0.9526001\n",
      "340\n",
      "[9.28737488]\n",
      "0.9599999\n",
      "341\n",
      "[9.29443665]\n",
      "0.9613999\n",
      "342\n",
      "[10.99956512]\n",
      "0.9601\n",
      "343\n",
      "[11.86368027]\n",
      "0.9609\n",
      "344\n",
      "[13.90306854]\n",
      "0.9602\n",
      "345\n",
      "[9.59140472]\n",
      "0.958\n",
      "346\n",
      "[12.1600769]\n",
      "0.9609\n",
      "347\n",
      "[10.94238586]\n",
      "0.9587\n",
      "348\n",
      "[11.34573059]\n",
      "0.9622\n",
      "349\n",
      "[11.17151718]\n",
      "0.9575001\n",
      "350\n",
      "[16.12511902]\n",
      "0.95290005\n",
      "351\n",
      "[14.3640625]\n",
      "0.9466001\n",
      "352\n",
      "[16.82789612]\n",
      "0.9427999\n",
      "353\n",
      "[13.14602661]\n",
      "0.94980013\n",
      "354\n",
      "[12.8092041]\n",
      "0.9606999\n",
      "355\n",
      "[14.03343811]\n",
      "0.9596001\n",
      "356\n",
      "[9.37899017]\n",
      "0.9596999\n",
      "357\n",
      "[10.23575134]\n",
      "0.9634\n",
      "358\n",
      "[11.44647141]\n",
      "0.96329993\n",
      "359\n",
      "[9.83012009]\n",
      "0.9622\n",
      "360\n",
      "[8.90882034]\n",
      "0.9624999\n",
      "361\n",
      "[10.53687286]\n",
      "0.9627\n",
      "362\n",
      "[10.66947327]\n",
      "0.9637\n",
      "363\n",
      "[9.00208893]\n",
      "0.9622\n",
      "364\n",
      "[10.55136566]\n",
      "0.96329993\n",
      "365\n",
      "[12.40161743]\n",
      "0.95979995\n",
      "366\n",
      "[9.05621262]\n",
      "0.9592999\n",
      "367\n",
      "[9.8242981]\n",
      "0.9624999\n",
      "368\n",
      "[11.73083496]\n",
      "0.9631001\n",
      "369\n",
      "[8.72461548]\n",
      "0.9617001\n",
      "370\n",
      "[13.7806778]\n",
      "0.9644\n",
      "371\n",
      "[8.25377274]\n",
      "0.9642\n",
      "372\n",
      "[7.47879944]\n",
      "0.9635\n",
      "373\n",
      "[8.79694366]\n",
      "0.9629\n",
      "374\n",
      "[11.34312744]\n",
      "0.9634\n",
      "375\n",
      "[8.92840347]\n",
      "0.9636\n",
      "376\n",
      "[11.373246]\n",
      "0.9631001\n",
      "377\n",
      "[10.8272316]\n",
      "0.9642\n",
      "378\n",
      "[9.02256851]\n",
      "0.9631999\n",
      "379\n",
      "[9.14519501]\n",
      "0.9635\n",
      "380\n",
      "[11.26914749]\n",
      "0.9610999\n",
      "381\n",
      "[9.79601974]\n",
      "0.963\n",
      "382\n",
      "[10.91223373]\n",
      "0.965\n",
      "383\n",
      "[9.4157547]\n",
      "0.9646999\n",
      "384\n",
      "[13.03242188]\n",
      "0.9649\n",
      "385\n",
      "[8.67070618]\n",
      "0.9617999\n",
      "386\n",
      "[10.16361084]\n",
      "0.9658\n",
      "387\n",
      "[8.75907135]\n",
      "0.9637\n",
      "388\n",
      "[7.36803665]\n",
      "0.9638001\n",
      "389\n",
      "[9.72985382]\n",
      "0.9649\n",
      "390\n",
      "[8.5368454]\n",
      "0.96459997\n",
      "391\n",
      "[12.58366699]\n",
      "0.9648\n",
      "392\n",
      "[8.05457153]\n",
      "0.9653999\n",
      "393\n",
      "[12.58600693]\n",
      "0.96450007\n",
      "394\n",
      "[7.784198]\n",
      "0.96389997\n",
      "395\n",
      "[7.96865387]\n",
      "0.96590006\n",
      "396\n",
      "[8.28495941]\n",
      "0.9663\n",
      "397\n",
      "[9.60204544]\n",
      "0.9643\n",
      "398\n",
      "[7.44249878]\n",
      "0.9635\n",
      "399\n",
      "[7.83001862]\n",
      "0.9643\n",
      "400\n",
      "[9.10413437]\n",
      "0.96590006\n",
      "401\n",
      "[7.4909256]\n",
      "0.9642\n",
      "402\n",
      "[10.7547348]\n",
      "0.9637\n",
      "403\n",
      "[9.5998436]\n",
      "0.9585999\n",
      "404\n",
      "[10.25720215]\n",
      "0.9651\n",
      "405\n",
      "[9.0156723]\n",
      "0.9631001\n",
      "406\n",
      "[10.03272781]\n",
      "0.9610999\n",
      "407\n",
      "[8.89387665]\n",
      "0.9648\n",
      "408\n",
      "[9.4164978]\n",
      "0.96450007\n",
      "409\n",
      "[13.0732605]\n",
      "0.9636\n",
      "410\n",
      "[10.14836502]\n",
      "0.9631001\n",
      "411\n",
      "[9.48661728]\n",
      "0.9603001\n",
      "412\n",
      "[8.38889465]\n",
      "0.9623\n",
      "413\n",
      "[9.78038101]\n",
      "0.9656\n",
      "414\n",
      "[7.21726685]\n",
      "0.9665\n",
      "415\n",
      "[8.63667297]\n",
      "0.96730006\n",
      "416\n",
      "[8.73068695]\n",
      "0.96599996\n",
      "417\n",
      "[8.23952179]\n",
      "0.9672\n",
      "418\n",
      "[11.1569519]\n",
      "0.9655\n",
      "419\n",
      "[8.69501038]\n",
      "0.9653999\n",
      "420\n",
      "[7.80752411]\n",
      "0.9658\n",
      "421\n",
      "[11.57970734]\n",
      "0.9658\n",
      "422\n",
      "[8.07034531]\n",
      "0.95839995\n",
      "423\n",
      "[11.31823959]\n",
      "0.96590006\n",
      "424\n",
      "[10.51663513]\n",
      "0.9616\n",
      "425\n",
      "[8.98954697]\n",
      "0.9627\n",
      "426\n",
      "[8.43582993]\n",
      "0.9637\n",
      "427\n",
      "[10.23273849]\n",
      "0.9656\n",
      "428\n",
      "[5.86988373]\n",
      "0.96450007\n",
      "429\n",
      "[8.54732742]\n",
      "0.9653999\n",
      "430\n",
      "[6.58718567]\n",
      "0.96669996\n",
      "431\n",
      "[10.94920654]\n",
      "0.96599996\n",
      "432\n",
      "[8.77997208]\n",
      "0.9641\n",
      "433\n",
      "[7.91380844]\n",
      "0.965\n",
      "434\n",
      "[10.58006516]\n",
      "0.9658\n",
      "435\n",
      "[8.14230652]\n",
      "0.96529996\n",
      "436\n",
      "[9.54922867]\n",
      "0.96730006\n",
      "437\n",
      "[10.3049675]\n",
      "0.9628\n",
      "438\n",
      "[9.13750687]\n",
      "0.9637\n",
      "439\n",
      "[10.48434448]\n",
      "0.9653999\n",
      "440\n",
      "[7.21531601]\n",
      "0.9663\n",
      "441\n",
      "[6.74720612]\n",
      "0.96590006\n",
      "442\n",
      "[8.66024628]\n",
      "0.96739995\n",
      "443\n",
      "[9.9608429]\n",
      "0.9662\n",
      "444\n",
      "[8.70830154]\n",
      "0.9655\n",
      "445\n",
      "[7.53335876]\n",
      "0.9610999\n",
      "446\n",
      "[8.90770874]\n",
      "0.9655\n",
      "447\n",
      "[7.30413818]\n",
      "0.9667999\n",
      "448\n",
      "[6.72954559]\n",
      "0.9660999\n",
      "449\n",
      "[11.20168228]\n",
      "0.967\n",
      "450\n",
      "[8.21693268]\n",
      "0.9635\n",
      "451\n",
      "[8.89199371]\n",
      "0.96520007\n",
      "452\n",
      "[9.02561569]\n",
      "0.96459997\n",
      "453\n",
      "[9.26019974]\n",
      "0.96660006\n",
      "454\n",
      "[7.95336533]\n",
      "0.9672\n",
      "455\n",
      "[8.43603821]\n",
      "0.96870005\n",
      "456\n",
      "[8.2984726]\n",
      "0.96590006\n",
      "457\n",
      "[7.61274643]\n",
      "0.9682\n",
      "458\n",
      "[10.10674744]\n",
      "0.9653999\n",
      "459\n",
      "[8.94614868]\n",
      "0.96660006\n",
      "460\n",
      "[7.08323441]\n",
      "0.9660999\n",
      "461\n",
      "[9.36405182]\n",
      "0.96809995\n",
      "462\n",
      "[11.4432106]\n",
      "0.9634\n",
      "463\n",
      "[9.37087402]\n",
      "0.9669\n",
      "464\n",
      "[9.57471313]\n",
      "0.9657\n",
      "465\n",
      "[8.16843338]\n",
      "0.9664\n",
      "466\n",
      "[8.33589783]\n",
      "0.9682\n",
      "467\n",
      "[7.35476761]\n",
      "0.9644\n",
      "468\n",
      "[11.51395035]\n",
      "0.9656\n",
      "469\n",
      "[6.10572472]\n",
      "0.9660999\n",
      "470\n",
      "[7.03906555]\n",
      "0.9705\n",
      "471\n",
      "[8.67101212]\n",
      "0.9697\n",
      "472\n",
      "[6.95400391]\n",
      "0.967\n",
      "473\n",
      "[6.07928696]\n",
      "0.9683\n",
      "474\n",
      "[7.93747025]\n",
      "0.96739995\n",
      "475\n",
      "[9.22471313]\n",
      "0.9667999\n",
      "476\n",
      "[9.20097961]\n",
      "0.9643\n",
      "477\n",
      "[8.0800705]\n",
      "0.9662\n",
      "478\n",
      "[8.25551834]\n",
      "0.9669\n",
      "479\n",
      "[9.44951324]\n",
      "0.9683\n",
      "480\n",
      "[5.32468529]\n",
      "0.96730006\n",
      "481\n",
      "[9.21593018]\n",
      "0.97010005\n",
      "482\n",
      "[6.55662766]\n",
      "0.9672\n",
      "483\n",
      "[8.95389252]\n",
      "0.9699\n",
      "484\n",
      "[10.33107834]\n",
      "0.967\n",
      "485\n",
      "[7.82827454]\n",
      "0.9672\n",
      "486\n",
      "[10.00362091]\n",
      "0.96809995\n",
      "487\n",
      "[10.23638458]\n",
      "0.9682\n",
      "488\n",
      "[7.97363434]\n",
      "0.9664\n",
      "489\n",
      "[7.75533752]\n",
      "0.9683\n",
      "490\n",
      "[9.02347565]\n",
      "0.9677\n",
      "491\n",
      "[7.82082825]\n",
      "0.9689\n",
      "492\n",
      "[5.43103294]\n",
      "0.9682\n",
      "493\n",
      "[8.24961014]\n",
      "0.97019994\n",
      "494\n",
      "[8.99573441]\n",
      "0.9684\n",
      "495\n",
      "[8.98819275]\n",
      "0.9691\n",
      "496\n",
      "[7.20225296]\n",
      "0.9677\n",
      "497\n",
      "[9.33947144]\n",
      "0.9660999\n",
      "498\n",
      "[8.63032074]\n",
      "0.965\n",
      "499\n",
      "[5.36480675]\n",
      "0.9637\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "abs_weights_diff = []\n",
    "abs_biases_diff = []\n",
    "Train_Acc = []\n",
    "Test_Acc = []\n",
    "CrE_Train = []\n",
    "CrE_Test = []\n",
    "\n",
    "lr = 10**(-2)\n",
    "\n",
    "mb_size = 100\n",
    "n_epochs = 500\n",
    "\n",
    "acc_train = np.zeros([n_epochs,1])\n",
    "acc_test = np.zeros([n_epochs,1])\n",
    "total_loss = np.zeros([n_epochs,1])\n",
    "H_matrix = np.zeros([int(n_epochs/10)])\n",
    "eps = 10**(-6)\n",
    "\n",
    "ii = 0\n",
    "\n",
    "for k in range(n_epochs):\n",
    "        print(k)\n",
    "        \n",
    "        batch_x = []\n",
    "        batch_y = []\n",
    "        for i in range(no_users):\n",
    "            batch_xx , batch_yy = mini_batches(x_train_k[i],y_train_k[i],  mb_size)\n",
    "            batch_x.append(batch_xx)\n",
    "            batch_y.append(batch_yy)\n",
    "        \n",
    "        \n",
    "        W1 = tf.reshape(central_modal[0],[784, n_hidden_1])\n",
    "        W2 = tf.reshape(central_modal[1],[n_hidden_1, n_hidden_2])\n",
    "        W3 = tf.reshape(central_modal[2],[n_hidden_2, 10])\n",
    "        \n",
    "        # Run the optimization to update W  \n",
    "        for i in range(no_users):\n",
    "            gradients1, gradients2, gradients3, loss = run_optimization(batch_x[i], batch_y[i], users[i], W1, W2, W3, lr)\n",
    "            users[i].GW1 = gradients1\n",
    "            users[i].GW2 = gradients2\n",
    "            users[i].GW3 = gradients3\n",
    "            total_loss[k] = total_loss[k] + loss\n",
    "        \n",
    "        total_loss[k] = total_loss[k]/no_users\n",
    "        print(total_loss[k])\n",
    "            \n",
    "        sum_gradient_1 = 0\n",
    "        sum_gradient_2 = 0\n",
    "        sum_gradient_3 = 0\n",
    "        for i in range(no_users):\n",
    "            sum_gradient_1 = sum_gradient_1 + (users[i].GW1)\n",
    "            sum_gradient_2 = sum_gradient_2 + (users[i].GW2)\n",
    "            sum_gradient_3 = sum_gradient_3 + (users[i].GW3)\n",
    "                \n",
    "\n",
    "        sum_gradient_1 = tf.reshape(sum_gradient_1,[784*n_hidden_1, 1])\n",
    "        sum_gradient_2 = tf.reshape(sum_gradient_2,[n_hidden_1*n_hidden_2, 1])\n",
    "        sum_gradient_3 = tf.reshape(sum_gradient_3,[10*n_hidden_2, 1])\n",
    "        \n",
    "        central_modal[0] = central_modal[0] - lr*sum_gradient_1/no_users\n",
    "        central_modal[1] = central_modal[1] - lr*sum_gradient_2/no_users\n",
    "        central_modal[2] = central_modal[2] - lr*sum_gradient_3/no_users\n",
    "        \n",
    "        \n",
    "        train_acc = []\n",
    "        test_acc = []\n",
    "        for j in range(no_users):  \n",
    "            train_pred = users[j].neural_net(batch_x[j])\n",
    "            train_acc.append(accuracy(train_pred , batch_y[j]))\n",
    "            test_pred = users[j].neural_net(x_test)\n",
    "            test_acc.append(accuracy(test_pred , y_test))\n",
    "            \n",
    "        avgAcc_Train = np.mean(train_acc)\n",
    "        avgAcc_Test = np.mean(test_acc) \n",
    "        #print(avgAcc_Train)\n",
    "        print(avgAcc_Test)\n",
    "        acc_train[k] = avgAcc_Train\n",
    "        acc_test[k] = avgAcc_Test\n",
    "\n",
    "#np.save('loss_gd_40960_5.npy', total_loss)\n",
    "#np.save('acc_train_gd_40960_5.npy', acc_train)\n",
    "#np.save('acc_test_gd_40960_5.npy', acc_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('acc_test_gd.npy', acc_test)\n",
    "#np.save('nb_ts_gd_lr001.npy', nb_ts)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}