GADMM_deepNN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import tensorflow as tf\n",
    "import time\n",
    "\n",
    "import os\n",
    "for dirname, _, filenames in os.walk('/kaggle/input'):\n",
    "    for filename in filenames:\n",
    "        print(os.path.join(dirname, filename))\n",
    "\n",
    "# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n",
    "# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = np.load('x_train.npy')\n",
    "y_train = np.load('y_train.npy')\n",
    "x_test = np.load('x_test.npy')\n",
    "y_test = np.load('y_test.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_classes = 10\n",
    "n1 = 10\n",
    "no_users = 10\n",
    "y_train = tf.keras.utils.to_categorical(y_train, num_classes)\n",
    "y_test = tf.keras.utils.to_categorical(y_test, num_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = np.reshape(x_train, (x_train.shape[0], -1))/255\n",
    "x_test = np.reshape(x_test, (x_test.shape[0], -1))/255"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Network parameters.\n",
    "n_hidden_1 = 128 # 1st layer number of neurons.\n",
    "n_hidden_2 = 64 # 2nd layer number of neurons."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class User:\n",
    "    def __init__(self):\n",
    "        self.weights = {\n",
    "            'W1': tf.Variable(tf.random.truncated_normal([784,n_hidden_1], stddev=0.1)),\n",
    "            'W2': tf.Variable(tf.random.truncated_normal([n_hidden_1,n_hidden_2], stddev=0.1)),\n",
    "            'W3': tf.Variable(tf.random.truncated_normal([n_hidden_2,10], stddev=0.1)),\n",
    "        }\n",
    "\n",
    "        self.lambda_W1_l = tf.Variable(tf.zeros([784*n_hidden_1,1]))\n",
    "        self.lambda_W2_l = tf.Variable(tf.zeros([n_hidden_1*n_hidden_2,1]))\n",
    "        self.lambda_W3_l = tf.Variable(tf.zeros([n_hidden_2*10,1]))\n",
    "        \n",
    "        self.lambda_W1_r = tf.Variable(tf.zeros([784*n_hidden_1,1]))\n",
    "        self.lambda_W2_r = tf.Variable(tf.zeros([n_hidden_1*n_hidden_2,1]))\n",
    "        self.lambda_W3_r = tf.Variable(tf.zeros([n_hidden_2*10,1]))\n",
    "        \n",
    "        \n",
    "\n",
    "    def neural_net(self, x):\n",
    "        y1 = tf.nn.relu(tf.matmul(x, self.weights['W1']))\n",
    "        y2 = tf.nn.relu(tf.matmul(y1, self.weights['W2']))\n",
    "        ylogits = tf.matmul(y2, self.weights['W3'])\n",
    "        return tf.nn.softmax(ylogits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# mini-batch loss function.\n",
    "def mini_batches(X, Y, mb_size = 100):\n",
    "\n",
    "    m = X.shape[0]\n",
    "\n",
    "    perm = list(np.random.permutation(m))\n",
    "    #perm = perm_init[0:100]\n",
    "    X_temp = X[perm,:]\n",
    "    Y_temp = Y[perm,:].reshape((m, Y.shape[1]))\n",
    "    \n",
    "    X_r = tf.convert_to_tensor(X_temp[0:mb_size,:], dtype=np.float32)\n",
    "    Y_r = tf.convert_to_tensor(Y_temp[0:mb_size,:], dtype=np.float32)\n",
    "    return X_r,Y_r"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cross-Entropy loss function.\n",
    "def cross_entropy(y_pred, y_true):\n",
    "    # Clip prediction values to avoid log(0) error.\n",
    "    y_pred = tf.clip_by_value(y_pred, 1e-9, 1.)\n",
    "    # Compute cross-entropy.\n",
    "    return -tf.reduce_sum(y_true * tf.math.log(y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Accuracy metric.\n",
    "def accuracy(y_pred, y_true):\n",
    "    # Predicted class is the index of highest score in prediction vector (i.e. argmax).\n",
    "    correct_prediction = tf.equal(tf.argmax(y_pred, 1), tf.argmax(y_true, 1))\n",
    "    return tf.reduce_mean(tf.cast(correct_prediction, tf.float32), axis=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Optimization process. \n",
    "def run_optimization(x, y, user, user_l, user_r, rho, flag):\n",
    "    # Wrap computation inside a GradientTape for automatic differentiation.\n",
    "    #temp1_r = tf.reshape(user_r.weights['W1'],[784*n_hidden_1, 1])\n",
    "    #temp2_r = tf.reshape(user_r.weights['W2'],[n_hidden_1*n_hidden_2, 1])\n",
    "    #temp3_r = tf.reshape(user_r.weights['W3'],[10*n_hidden_2, 1])\n",
    "    \n",
    "    #temp1_l = tf.reshape(user_l.weights['W1'],[784*n_hidden_1, 1])\n",
    "    #temp2_l = tf.reshape(user_l.weights['W2'],[n_hidden_1*n_hidden_2, 1])\n",
    "    #temp3_l = tf.reshape(user_l.weights['W3'],[10*n_hidden_2, 1])\n",
    "    \n",
    "    with tf.GradientTape() as g:\n",
    "        #g.watch(x)\n",
    "        pred = user.neural_net(x)\n",
    "        if (flag == 0):\n",
    "            loss = cross_entropy(pred, y)\\\n",
    "            + rho/2 * tf.reduce_sum(tf.math.square(tf.reshape(user.weights['W1'],[784*n_hidden_1, 1])-tf.reshape(user_r.weights['W1'],[784*n_hidden_1, 1])))\\\n",
    "            + tf.reduce_sum(tf.math.multiply(user.lambda_W1_r,tf.reshape(user.weights['W1'],[784*n_hidden_1, 1])-tf.reshape(user_r.weights['W1'],[784*n_hidden_1, 1])))\\\n",
    "            + rho/2 * tf.reduce_sum(tf.math.square(tf.reshape(user.weights['W2'],[n_hidden_1*n_hidden_2, 1])-tf.reshape(user_r.weights['W2'],[n_hidden_1*n_hidden_2, 1]))) \\\n",
    "            + tf.reduce_sum(tf.math.multiply(user.lambda_W2_r,tf.reshape(user.weights['W2'],[n_hidden_1*n_hidden_2, 1])-tf.reshape(user_r.weights['W2'],[n_hidden_1*n_hidden_2, 1])))\\\n",
    "            + rho/2 * tf.reduce_sum(tf.math.square(tf.reshape(user.weights['W3'],[10*n_hidden_2, 1])-tf.reshape(user_r.weights['W3'],[10*n_hidden_2, 1]))) \\\n",
    "            + tf.reduce_sum(tf.math.multiply(user.lambda_W3_r,tf.reshape(user.weights['W3'],[10*n_hidden_2, 1])-tf.reshape(user_r.weights['W3'],[10*n_hidden_2, 1])))\n",
    "\n",
    "        elif (flag == 1):\n",
    "            loss = cross_entropy(pred, y)\\\n",
    "            + rho/2 * tf.reduce_sum(tf.math.square(tf.reshape(user.weights['W1'],[784*n_hidden_1, 1])-tf.reshape(user_l.weights['W1'],[784*n_hidden_1, 1])))\\\n",
    "            + tf.reduce_sum(tf.math.multiply(user.lambda_W1_l,-tf.reshape(user.weights['W1'],[784*n_hidden_1, 1])+tf.reshape(user_l.weights['W1'],[784*n_hidden_1, 1])))\\\n",
    "            + rho/2 * tf.reduce_sum(tf.math.square(tf.reshape(user.weights['W2'],[n_hidden_1*n_hidden_2, 1])-tf.reshape(user_l.weights['W2'],[n_hidden_1*n_hidden_2, 1]))) \\\n",
    "            + tf.reduce_sum(tf.math.multiply(user.lambda_W2_l,-tf.reshape(user.weights['W2'],[n_hidden_1*n_hidden_2, 1])+tf.reshape(user_l.weights['W2'],[n_hidden_1*n_hidden_2, 1])))\\\n",
    "            + rho/2 * tf.reduce_sum(tf.math.square(tf.reshape(user.weights['W3'],[10*n_hidden_2, 1])-tf.reshape(user_l.weights['W3'],[10*n_hidden_2, 1]))) \\\n",
    "            + tf.reduce_sum(tf.math.multiply(user.lambda_W3_l,-tf.reshape(user.weights['W3'],[10*n_hidden_2, 1])+tf.reshape(user_l.weights['W3'],[10*n_hidden_2, 1])))\n",
    "\n",
    "        else:\n",
    "            loss = cross_entropy(pred, y)\\\n",
    "            + rho/2 * tf.reduce_sum(tf.math.square(tf.reshape(user.weights['W1'],[784*n_hidden_1, 1])-tf.reshape(user_r.weights['W1'],[784*n_hidden_1, 1])))\\\n",
    "            + tf.reduce_sum(tf.math.multiply(user.lambda_W1_r,tf.reshape(user.weights['W1'],[784*n_hidden_1, 1])-tf.reshape(user_r.weights['W1'],[784*n_hidden_1, 1])))\\\n",
    "            + rho/2 * tf.reduce_sum(tf.math.square(tf.reshape(user.weights['W2'],[n_hidden_1*n_hidden_2, 1])-tf.reshape(user_r.weights['W2'],[n_hidden_1*n_hidden_2, 1]))) \\\n",
    "            + tf.reduce_sum(tf.math.multiply(user.lambda_W2_r,tf.reshape(user.weights['W2'],[n_hidden_1*n_hidden_2, 1])-tf.reshape(user_r.weights['W2'],[n_hidden_1*n_hidden_2, 1])))\\\n",
    "            + rho/2 * tf.reduce_sum(tf.math.square(tf.reshape(user.weights['W3'],[10*n_hidden_2, 1])-tf.reshape(user_r.weights['W3'],[10*n_hidden_2, 1]))) \\\n",
    "            + tf.reduce_sum(tf.math.multiply(user.lambda_W3_r,tf.reshape(user.weights['W3'],[10*n_hidden_2, 1])-tf.reshape(user_r.weights['W3'],[10*n_hidden_2, 1])))\n",
    "            + rho/2 * tf.reduce_sum(tf.math.square(tf.reshape(user.weights['W1'],[784*n_hidden_1, 1])-tf.reshape(user_l.weights['W1'],[784*n_hidden_1, 1])))\\\n",
    "            + tf.reduce_sum(tf.math.multiply(user.lambda_W1_l,-tf.reshape(user.weights['W1'],[784*n_hidden_1, 1])+tf.reshape(user_l.weights['W1'],[784*n_hidden_1, 1])))\\\n",
    "            + rho/2 * tf.reduce_sum(tf.math.square(tf.reshape(user.weights['W2'],[n_hidden_1*n_hidden_2, 1])-tf.reshape(user_l.weights['W2'],[n_hidden_1*n_hidden_2, 1]))) \\\n",
    "            + tf.reduce_sum(tf.math.multiply(user.lambda_W2_l,-tf.reshape(user.weights['W2'],[n_hidden_1*n_hidden_2, 1])+tf.reshape(user_l.weights['W2'],[n_hidden_1*n_hidden_2, 1])))\\\n",
    "            + rho/2 * tf.reduce_sum(tf.math.square(tf.reshape(user.weights['W3'],[10*n_hidden_2, 1])-tf.reshape(user_l.weights['W3'],[10*n_hidden_2, 1]))) \\\n",
    "            + tf.reduce_sum(tf.math.multiply(user.lambda_W3_l,-tf.reshape(user.weights['W3'],[10*n_hidden_2, 1])+tf.reshape(user_l.weights['W3'],[10*n_hidden_2, 1])))\n",
    "\n",
    "        \n",
    "    \n",
    "    # Variables to update, i.e. trainable variables.\n",
    "    trainable_variables = user.weights.values()\n",
    "    \n",
    "    # Compute gradients.\n",
    "    gradients = g.gradient(loss, trainable_variables)\n",
    "    #print(gradients)\n",
    "    \n",
    "    # Update W following gradients.\n",
    "    optimizer.apply_gradients(zip(gradients, trainable_variables))\n",
    "    return cross_entropy(pred, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "users = [User() for i in range(no_users)]\n",
    "rho = 20\n",
    "optimizer = tf.optimizers.Adam(learning_rate=0.001)#0.001\n",
    "#optimizer = tf.optimizers.SGD(learning_rate=0.001)\n",
    "#central_modal = [tf.Variable(tf.random.truncated_normal([784*n_hidden_1,1], stddev=0.1)), tf.Variable(tf.random.truncated_normal([n_hidden_1*n_hidden_2,1], stddev=0.1)), tf.Variable(tf.random.truncated_normal([10*n_hidden_2,1], stddev=0.1))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train_k = []\n",
    "y_train_k = []\n",
    "data_per_worker = int(x_train.shape[0]/no_users)\n",
    "for i in range(no_users):\n",
    "    first = i*data_per_worker\n",
    "    last = first + data_per_worker\n",
    "    x_train_k.append(x_train[first:last])\n",
    "    y_train_k.append(y_train[first:last])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_test = tf.convert_to_tensor(x_test, dtype=np.float32)\n",
    "y_test = tf.convert_to_tensor(y_test, dtype=np.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "user0 = User()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "[217.73195801]\n",
      "23630.008875846863\n",
      "0.306\n",
      "0.20629999\n",
      "0.2857\n",
      "[0.28277493]\n",
      "1\n",
      "[204.37554932]\n",
      "14458.394026756287\n",
      "0.486\n",
      "0.34044\n",
      "0.5197\n",
      "[0.45213413]\n",
      "2\n",
      "[191.8293457]\n",
      "8378.309129238129\n",
      "0.49\n",
      "0.38983998\n",
      "0.5136\n",
      "[0.63154793]\n",
      "3\n",
      "[171.77983398]\n",
      "5007.408725261688\n",
      "0.59000003\n",
      "0.46746\n",
      "0.7199\n",
      "[0.80591583]\n",
      "4\n",
      "[144.98615723]\n",
      "3227.3154668807983\n",
      "0.62\n",
      "0.48813\n",
      "0.7353\n",
      "[0.97880387]\n",
      "5\n",
      "[117.90341797]\n",
      "2286.186225295067\n",
      "0.738\n",
      "0.62477\n",
      "0.8027\n",
      "[1.14725089]\n",
      "6\n",
      "[88.53912354]\n",
      "1768.8520829677582\n",
      "0.80600005\n",
      "0.67753\n",
      "0.8354\n",
      "[1.33288097]\n",
      "7\n",
      "[68.49708252]\n",
      "1456.6455882489681\n",
      "0.85199994\n",
      "0.74540997\n",
      "0.8461\n",
      "[1.50426793]\n",
      "8\n",
      "[52.26917114]\n",
      "1247.4553612172604\n",
      "0.88699996\n",
      "0.79312\n",
      "0.8582\n",
      "[1.67548203]\n",
      "9\n",
      "[43.25117187]\n",
      "1089.4576110839844\n",
      "0.90500003\n",
      "0.79196\n",
      "0.8345\n",
      "[1.85165]\n",
      "10\n",
      "[40.03006897]\n",
      "959.4253695607185\n",
      "0.92399997\n",
      "0.82381\n",
      "0.8577\n",
      "[2.0248251]\n",
      "11\n",
      "[36.90571899]\n",
      "848.9660459160805\n",
      "0.917\n",
      "0.8287299\n",
      "0.8719\n",
      "[2.19609213]\n",
      "12\n",
      "[33.626651]\n",
      "750.620081782341\n",
      "0.936\n",
      "0.8419601\n",
      "0.8673\n",
      "[2.36556005]\n",
      "13\n",
      "[29.44789734]\n",
      "662.5237343907356\n",
      "0.946\n",
      "0.84714\n",
      "0.8732\n",
      "[2.53344417]\n",
      "14\n",
      "[27.3709137]\n",
      "584.5726463496685\n",
      "0.94400007\n",
      "0.84194005\n",
      "0.8807\n",
      "[2.7859242]\n",
      "15\n",
      "[24.68011169]\n",
      "513.1287940442562\n",
      "0.958\n",
      "0.85403\n",
      "0.8813\n",
      "[2.98537827]\n",
      "16\n",
      "[24.97119751]\n",
      "448.9103330075741\n",
      "0.958\n",
      "0.8528501\n",
      "0.8735\n",
      "[3.16994429]\n",
      "17\n",
      "[22.33583527]\n",
      "392.34872598946095\n",
      "0.96799994\n",
      "0.85356\n",
      "0.8868\n",
      "[3.40649343]\n",
      "18\n",
      "[21.51192017]\n",
      "340.66497437655926\n",
      "0.9700001\n",
      "0.86208\n",
      "0.8995\n",
      "[3.58813548]\n",
      "19\n",
      "[21.53492126]\n",
      "295.10292145609856\n",
      "0.962\n",
      "0.86667\n",
      "0.8891\n",
      "[3.77132535]\n",
      "20\n",
      "[21.52137146]\n",
      "260.048301294446\n",
      "0.959\n",
      "0.87018\n",
      "0.8854\n",
      "[3.9341085]\n",
      "21\n",
      "[19.96537933]\n",
      "226.95184123516083\n",
      "0.963\n",
      "0.86878\n",
      "0.8956\n",
      "[4.10420656]\n",
      "22\n",
      "[17.91321259]\n",
      "199.90228089690208\n",
      "0.97299993\n",
      "0.86603004\n",
      "0.8842\n",
      "[4.27915263]\n",
      "23\n",
      "[16.8616333]\n",
      "175.02876406908035\n",
      "0.9610001\n",
      "0.86572\n",
      "0.8903\n",
      "[4.44849873]\n",
      "24\n",
      "[18.55273132]\n",
      "159.22141662240028\n",
      "0.9620001\n",
      "0.87393\n",
      "0.8896\n",
      "[4.6140058]\n",
      "25\n",
      "[13.56513367]\n",
      "144.75222136080265\n",
      "0.978\n",
      "0.88464004\n",
      "0.9026\n",
      "[4.7900486]\n",
      "26\n",
      "[15.16086426]\n",
      "131.10267632454634\n",
      "0.976\n",
      "0.88364\n",
      "0.8995\n",
      "[4.96976852]\n",
      "27\n",
      "[14.32345428]\n",
      "121.08780379593372\n",
      "0.9790001\n",
      "0.88577\n",
      "0.9105\n",
      "[5.13935351]\n",
      "28\n",
      "[11.6093132]\n",
      "113.47265788167715\n",
      "0.985\n",
      "0.88901997\n",
      "0.9099\n",
      "[5.31304765]\n",
      "29\n",
      "[12.27891388]\n",
      "107.10431102663279\n",
      "0.9790001\n",
      "0.88837004\n",
      "0.9057\n",
      "[5.48171353]\n",
      "30\n",
      "[12.51845627]\n",
      "99.93145722150803\n",
      "0.979\n",
      "0.89542997\n",
      "0.9193\n",
      "[5.64687753]\n",
      "31\n",
      "[11.05545654]\n",
      "94.44208526611328\n",
      "0.98\n",
      "0.89202994\n",
      "0.9019\n",
      "[5.81746149]\n",
      "32\n",
      "[14.07907257]\n",
      "91.34478703141212\n",
      "0.972\n",
      "0.90148\n",
      "0.913\n",
      "[6.00184655]\n",
      "33\n",
      "[9.90006027]\n",
      "89.774464905262\n",
      "0.98499995\n",
      "0.88515997\n",
      "0.9152\n",
      "[6.17862058]\n",
      "34\n",
      "[10.38459778]\n",
      "85.62485173344612\n",
      "0.983\n",
      "0.89564\n",
      "0.9135\n",
      "[6.35431266]\n",
      "35\n",
      "[13.09362183]\n",
      "86.01527799665928\n",
      "0.975\n",
      "0.89708006\n",
      "0.9127\n",
      "[6.53134871]\n",
      "36\n",
      "[10.71633759]\n",
      "85.41400650143623\n",
      "0.98100007\n",
      "0.90187997\n",
      "0.9112\n",
      "[6.70138264]\n",
      "37\n",
      "[11.44858551]\n",
      "83.38308161497116\n",
      "0.98\n",
      "0.89398\n",
      "0.9134\n",
      "[6.88135076]\n",
      "38\n",
      "[9.40789719]\n",
      "79.9179358035326\n",
      "0.983\n",
      "0.90579003\n",
      "0.9217\n",
      "[7.05877566]\n",
      "39\n",
      "[9.61459274]\n",
      "77.32716201245785\n",
      "0.987\n",
      "0.9\n",
      "0.9151\n",
      "[7.23067355]\n",
      "40\n",
      "[9.91247559]\n",
      "76.54846904426813\n",
      "0.98\n",
      "0.90102994\n",
      "0.9232\n",
      "[7.40954876]\n",
      "41\n",
      "[9.75804672]\n",
      "77.59023444354534\n",
      "0.983\n",
      "0.90154\n",
      "0.9152\n",
      "[7.59528971]\n",
      "42\n",
      "[8.89425354]\n",
      "77.39530465006828\n",
      "0.98600006\n",
      "0.90078\n",
      "0.918\n",
      "[7.7730875]\n",
      "43\n",
      "[8.65892029]\n",
      "77.9312706887722\n",
      "0.984\n",
      "0.89881\n",
      "0.9147\n",
      "[7.94911361]\n",
      "44\n",
      "[9.92521973]\n",
      "78.30721043795347\n",
      "0.982\n",
      "0.9071501\n",
      "0.9221\n",
      "[8.14161754]\n",
      "45\n",
      "[10.33106842]\n",
      "79.69284529238939\n",
      "0.97700006\n",
      "0.9104099\n",
      "0.9215\n",
      "[8.53214073]\n",
      "46\n",
      "[9.89907074]\n",
      "80.2522527500987\n",
      "0.98800004\n",
      "0.91231\n",
      "0.9254\n",
      "[8.83189678]\n",
      "47\n",
      "[7.18071823]\n",
      "77.21462531387806\n",
      "0.98999995\n",
      "0.91424\n",
      "0.9283\n",
      "[9.12657666]\n",
      "48\n",
      "[8.35642014]\n",
      "74.23412597179413\n",
      "0.98999995\n",
      "0.91441\n",
      "0.9267\n",
      "[9.36156559]\n",
      "49\n",
      "[7.60204315]\n",
      "73.84164438396692\n",
      "0.992\n",
      "0.91829\n",
      "0.9321\n",
      "[9.67015076]\n",
      "50\n",
      "[7.54212799]\n",
      "70.99661947786808\n",
      "0.98999995\n",
      "0.9182501\n",
      "0.9282\n",
      "[10.01103187]\n",
      "51\n",
      "[10.99645081]\n",
      "69.72416470944881\n",
      "0.97699994\n",
      "0.91549\n",
      "0.9233\n",
      "[10.22400093]\n",
      "52\n",
      "[5.74134865]\n",
      "67.8180516064167\n",
      "0.98999995\n",
      "0.9159301\n",
      "0.9246\n",
      "[10.43774176]\n",
      "53\n",
      "[6.98092651]\n",
      "65.7401735112071\n",
      "0.98800004\n",
      "0.91822994\n",
      "0.9302\n",
      "[10.62821293]\n",
      "54\n",
      "[7.15797272]\n",
      "67.25796389579773\n",
      "0.991\n",
      "0.91507995\n",
      "0.9263\n",
      "[10.81557298]\n",
      "55\n",
      "[7.74964752]\n",
      "66.95640569925308\n",
      "0.98600006\n",
      "0.91833\n",
      "0.9304\n",
      "[11.01421404]\n",
      "56\n",
      "[8.6346077]\n",
      "68.71609516441822\n",
      "0.987\n",
      "0.91642\n",
      "0.9293\n",
      "[11.19702005]\n",
      "57\n",
      "[7.43383255]\n",
      "66.25014302134514\n",
      "0.987\n",
      "0.91906005\n",
      "0.9302\n",
      "[11.38921404]\n",
      "58\n",
      "[9.24914246]\n",
      "66.38534377515316\n",
      "0.987\n",
      "0.91864\n",
      "0.9339\n",
      "[11.58881307]\n",
      "59\n",
      "[6.59888458]\n",
      "62.76648008823395\n",
      "0.991\n",
      "0.91788006\n",
      "0.9253\n",
      "[11.79449606]\n",
      "60\n",
      "[6.23861046]\n",
      "62.060506116598845\n",
      "0.98999995\n",
      "0.91986\n",
      "0.9311\n",
      "[11.98411703]\n",
      "61\n",
      "[6.08462563]\n",
      "61.58535373210907\n",
      "0.991\n",
      "0.92194\n",
      "0.9379\n",
      "[12.17042112]\n",
      "62\n",
      "[4.85071411]\n",
      "59.52420984208584\n",
      "0.992\n",
      "0.92012995\n",
      "0.9324\n",
      "[12.36287308]\n",
      "63\n",
      "[5.09843674]\n",
      "60.861533753573895\n",
      "0.99300003\n",
      "0.91853\n",
      "0.9332\n",
      "[12.53963614]\n",
      "64\n",
      "[5.40154266]\n",
      "61.20472923666239\n",
      "0.992\n",
      "0.92724\n",
      "0.934\n",
      "[12.71885419]\n",
      "65\n",
      "[5.81847267]\n",
      "59.53492572158575\n",
      "0.992\n",
      "0.9197799\n",
      "0.9359\n",
      "[12.91751528]\n",
      "66\n",
      "[5.58067741]\n",
      "61.08878364413977\n",
      "0.99300003\n",
      "0.92111\n",
      "0.9284\n",
      "[13.13831449]\n",
      "67\n",
      "[4.48973083]\n",
      "60.64723293483257\n",
      "0.995\n",
      "0.92427003\n",
      "0.9334\n",
      "[13.31930447]\n",
      "68\n",
      "[6.07250595]\n",
      "58.91763761267066\n",
      "0.98999995\n",
      "0.9226\n",
      "0.9354\n",
      "[13.54074526]\n",
      "69\n",
      "[6.34171448]\n",
      "59.49080517143011\n",
      "0.987\n",
      "0.92578995\n",
      "0.9368\n",
      "[13.7367053]\n",
      "70\n",
      "[5.83831978]\n",
      "59.9766210578382\n",
      "0.98999995\n",
      "0.92495\n",
      "0.9382\n",
      "[13.96100521]\n",
      "71\n",
      "[7.04006195]\n",
      "58.958466459065676\n",
      "0.992\n",
      "0.9259101\n",
      "0.9382\n",
      "[14.16416407]\n",
      "72\n",
      "[5.14904633]\n",
      "59.22839639335871\n",
      "0.98999995\n",
      "0.9259001\n",
      "0.9345\n",
      "[14.36651301]\n",
      "73\n",
      "[6.39840965]\n",
      "60.59245193749666\n",
      "0.98999995\n",
      "0.92340994\n",
      "0.9321\n",
      "[14.553509]\n",
      "74\n",
      "[3.59081612]\n",
      "60.709484957158566\n",
      "0.99700004\n",
      "0.92560995\n",
      "0.9355\n",
      "[14.74405408]\n",
      "75\n",
      "[5.82782822]\n",
      "59.067858438938856\n",
      "0.991\n",
      "0.9288799\n",
      "0.9382\n",
      "[14.92955303]\n",
      "76\n",
      "[2.92900848]\n",
      "57.73972427845001\n",
      "0.99799997\n",
      "0.92735994\n",
      "0.9424\n",
      "[15.114815]\n",
      "77\n",
      "[6.03960686]\n",
      "57.28162012249231\n",
      "0.991\n",
      "0.93142\n",
      "0.9402\n",
      "[15.45799804]\n",
      "78\n",
      "[6.53733368]\n",
      "57.33771565556526\n",
      "0.99300003\n",
      "0.92654\n",
      "0.9426\n",
      "[15.64851022]\n",
      "79\n",
      "[4.05040436]\n",
      "55.557364501059055\n",
      "0.996\n",
      "0.92416\n",
      "0.9407\n",
      "[15.8328793]\n",
      "80\n",
      "[7.44584274]\n",
      "56.71529267728329\n",
      "0.992\n",
      "0.9266399\n",
      "0.9344\n",
      "[16.0182333]\n",
      "81\n",
      "[4.05577812]\n",
      "58.29296852648258\n",
      "0.996\n",
      "0.91984\n",
      "0.9328\n",
      "[16.20992732]\n",
      "82\n",
      "[5.79651794]\n",
      "58.93638026714325\n",
      "0.989\n",
      "0.9312\n",
      "0.9359\n",
      "[16.40594029]\n",
      "83\n",
      "[5.17861977]\n",
      "58.83937310054898\n",
      "0.99300003\n",
      "0.92972004\n",
      "0.9389\n",
      "[16.59128428]\n",
      "84\n",
      "[5.05637589]\n",
      "57.578478164970875\n",
      "0.99399996\n",
      "0.92412007\n",
      "0.936\n",
      "[16.78277731]\n",
      "85\n",
      "[3.44006004]\n",
      "55.82778175920248\n",
      "0.995\n",
      "0.92911005\n",
      "0.9334\n",
      "[16.99301243]\n",
      "86\n",
      "[3.89590416]\n",
      "55.51306554675102\n",
      "0.99899995\n",
      "0.92682993\n",
      "0.9384\n",
      "[17.17072654]\n",
      "87\n",
      "[3.81942978]\n",
      "56.43020133674145\n",
      "0.996\n",
      "0.92926997\n",
      "0.9377\n",
      "[17.35604048]\n",
      "88\n",
      "[3.72470512]\n",
      "55.48179715126753\n",
      "0.99799997\n",
      "0.9241401\n",
      "0.9369\n",
      "[17.53517056]\n",
      "89\n",
      "[4.99275551]\n",
      "54.742407746613026\n",
      "0.98999995\n",
      "0.92587996\n",
      "0.9367\n",
      "[17.73535562]\n",
      "90\n",
      "[4.32865868]\n",
      "56.229398012161255\n",
      "0.995\n",
      "0.93073004\n",
      "0.943\n",
      "[17.91704774]\n",
      "91\n",
      "[4.38342514]\n",
      "54.44132949411869\n",
      "0.989\n",
      "0.9309599\n",
      "0.946\n",
      "[18.11470175]\n",
      "92\n",
      "[3.7222374]\n",
      "55.816013388335705\n",
      "0.996\n",
      "0.93251\n",
      "0.9415\n",
      "[18.41778088]\n",
      "93\n",
      "[5.27698364]\n",
      "56.98149003088474\n",
      "0.9940001\n",
      "0.93169993\n",
      "0.9369\n",
      "[18.68992591]\n",
      "94\n",
      "[4.10205269]\n",
      "59.03163459897041\n",
      "0.99300003\n",
      "0.9266099\n",
      "0.9394\n",
      "[19.07919192]\n",
      "95\n",
      "[3.49718208]\n",
      "55.369433261454105\n",
      "0.995\n",
      "0.9329699\n",
      "0.9402\n",
      "[19.26173592]\n",
      "96\n",
      "[4.66257248]\n",
      "53.88176628947258\n",
      "0.991\n",
      "0.92958003\n",
      "0.9427\n",
      "[19.45438981]\n",
      "97\n",
      "[3.50646248]\n",
      "52.97965903580189\n",
      "0.995\n",
      "0.93365\n",
      "0.9438\n",
      "[19.63859677]\n",
      "98\n",
      "[3.69006004]\n",
      "53.642414927482605\n",
      "0.996\n",
      "0.93098\n",
      "0.9374\n",
      "[19.83928776]\n",
      "99\n",
      "[6.26226997]\n",
      "55.110117480158806\n",
      "0.98999995\n",
      "0.9304699\n",
      "0.9417\n",
      "[20.03408957]\n",
      "100\n",
      "[4.75819855]\n",
      "56.56869377940893\n",
      "0.99399996\n",
      "0.93027\n",
      "0.9422\n",
      "[20.22707558]\n",
      "101\n",
      "[4.5044445]\n",
      "55.1632369235158\n",
      "0.9940001\n",
      "0.93048\n",
      "0.946\n",
      "[20.43013644]\n",
      "102\n",
      "[3.44967499]\n",
      "54.43342226743698\n",
      "0.995\n",
      "0.93403006\n",
      "0.9396\n",
      "[20.62571049]\n",
      "103\n",
      "[2.89285069]\n",
      "51.15659585595131\n",
      "0.99899995\n",
      "0.93701\n",
      "0.9445\n",
      "[20.8073535]\n",
      "104\n",
      "[4.80546417]\n",
      "51.311350386589766\n",
      "0.991\n",
      "0.9334901\n",
      "0.9436\n",
      "[20.99956441]\n",
      "105\n",
      "[2.9830019]\n",
      "50.109430111944675\n",
      "0.99700004\n",
      "0.9362\n",
      "0.9458\n",
      "[21.18238425]\n",
      "106\n",
      "[3.31620598]\n",
      "49.84244558215141\n",
      "0.996\n",
      "0.93450004\n",
      "0.9412\n",
      "[21.36222935]\n",
      "107\n",
      "[3.58243408]\n",
      "51.799322329461575\n",
      "0.996\n",
      "0.93126994\n",
      "0.9414\n",
      "[21.57468534]\n",
      "108\n",
      "[4.22161026]\n",
      "53.829619102180004\n",
      "0.9940001\n",
      "0.93269\n",
      "0.9434\n",
      "[21.76994634]\n",
      "109\n",
      "[2.34431858]\n",
      "51.00951397418976\n",
      "0.99799997\n",
      "0.93634003\n",
      "0.9437\n",
      "[21.95272732]\n",
      "110\n",
      "[4.58937454]\n",
      "50.64429125189781\n",
      "0.995\n",
      "0.93499005\n",
      "0.9415\n",
      "[22.14855123]\n",
      "111\n",
      "[4.89205017]\n",
      "52.31131802126765\n",
      "0.9940001\n",
      "0.93337995\n",
      "0.9459\n",
      "[22.3340683]\n",
      "112\n",
      "[2.70998898]\n",
      "50.988125428557396\n",
      "0.99799997\n",
      "0.93457997\n",
      "0.9442\n",
      "[22.53457236]\n",
      "113\n",
      "[3.32475777]\n",
      "48.57486080378294\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.995\n",
      "0.93679\n",
      "0.9472\n",
      "[22.74564934]\n",
      "114\n",
      "[2.20021992]\n",
      "46.80842175707221\n",
      "0.99899995\n",
      "0.93672\n",
      "0.9414\n",
      "[22.94283319]\n",
      "115\n",
      "[3.85789757]\n",
      "48.56205835938454\n",
      "0.9940001\n",
      "0.9369\n",
      "0.9448\n",
      "[23.13861418]\n",
      "116\n",
      "[3.73287506]\n",
      "49.80067418515682\n",
      "0.996\n",
      "0.9332101\n",
      "0.9422\n",
      "[23.33983231]\n",
      "117\n",
      "[2.70709953]\n",
      "49.23527467995882\n",
      "0.99799997\n",
      "0.93569005\n",
      "0.9439\n",
      "[23.75188828]\n",
      "118\n",
      "[3.41192474]\n",
      "50.822378508746624\n",
      "0.996\n",
      "0.93103\n",
      "0.9452\n",
      "[23.96339536]\n",
      "119\n",
      "[3.11661301]\n",
      "54.3407401368022\n",
      "0.99700004\n",
      "0.92923003\n",
      "0.9427\n",
      "[24.15872025]\n",
      "120\n",
      "[2.9601614]\n",
      "54.12069749832153\n",
      "0.99700004\n",
      "0.93429005\n",
      "0.9426\n",
      "[24.34597421]\n",
      "121\n",
      "[3.31604576]\n",
      "53.46098639070988\n",
      "0.995\n",
      "0.9335799\n",
      "0.9437\n",
      "[24.53787422]\n",
      "122\n",
      "[2.40753975]\n",
      "51.59995959699154\n",
      "0.99899995\n",
      "0.9361199\n",
      "0.944\n",
      "[24.76088524]\n",
      "123\n",
      "[3.93249435]\n",
      "51.22491233050823\n",
      "0.99399996\n",
      "0.9353601\n",
      "0.948\n",
      "[24.96357536]\n",
      "124\n",
      "[3.1304697]\n",
      "54.13533728942275\n",
      "0.99799997\n",
      "0.93271005\n",
      "0.9435\n",
      "[25.15136337]\n",
      "125\n",
      "[2.47558365]\n",
      "50.44239918142557\n",
      "0.99799997\n",
      "0.94032\n",
      "0.9445\n",
      "[25.34379435]\n",
      "126\n",
      "[4.46213188]\n",
      "53.0466964058578\n",
      "0.995\n",
      "0.93148994\n",
      "0.9421\n",
      "[25.60042739]\n",
      "127\n",
      "[6.72304459]\n",
      "56.32716837525368\n",
      "0.991\n",
      "0.93539\n",
      "0.946\n",
      "[25.83955026]\n",
      "128\n",
      "[2.97971649]\n",
      "53.329869113862514\n",
      "0.99799997\n",
      "0.9330899\n",
      "0.9458\n",
      "[26.1373384]\n",
      "129\n",
      "[4.09973183]\n",
      "51.375705912709236\n",
      "0.991\n",
      "0.9387\n",
      "0.9463\n",
      "[26.36533928]\n",
      "130\n",
      "[3.63634682]\n",
      "51.665704153478146\n",
      "0.996\n",
      "0.9361501\n",
      "0.9479\n",
      "[26.55927539]\n",
      "131\n",
      "[5.69136238]\n",
      "54.98660796508193\n",
      "0.992\n",
      "0.93803006\n",
      "0.9485\n",
      "[26.90680337]\n",
      "132\n",
      "[3.22744904]\n",
      "56.7385414019227\n",
      "0.995\n",
      "0.93914\n",
      "0.9499\n",
      "[27.12072635]\n",
      "133\n",
      "[4.07369843]\n",
      "56.659076385200024\n",
      "0.991\n",
      "0.93247\n",
      "0.9527\n",
      "[27.31344438]\n",
      "134\n",
      "[2.83606052]\n",
      "54.13362503796816\n",
      "0.996\n",
      "0.93943006\n",
      "0.948\n",
      "[27.57196641]\n",
      "135\n",
      "[2.58079262]\n",
      "50.373023107647896\n",
      "0.996\n",
      "0.9415199\n",
      "0.9505\n",
      "[27.77299953]\n",
      "136\n",
      "[3.26546974]\n",
      "48.66763447597623\n",
      "0.996\n",
      "0.9419999\n",
      "0.9491\n",
      "[28.03494453]\n",
      "137\n",
      "[2.25681076]\n",
      "48.5324696674943\n",
      "0.99899995\n",
      "0.94157\n",
      "0.9493\n",
      "[28.28214955]\n",
      "138\n",
      "[2.86714115]\n",
      "50.518222246319056\n",
      "0.99799997\n",
      "0.93845\n",
      "0.9485\n",
      "[28.47593355]\n",
      "139\n",
      "[2.15090714]\n",
      "50.596515238285065\n",
      "0.99799997\n",
      "0.9391799\n",
      "0.9484\n",
      "[28.70158243]\n",
      "140\n",
      "[3.37393913]\n",
      "49.140965446829796\n",
      "0.99300003\n",
      "0.9417299\n",
      "0.9516\n",
      "[28.86740351]\n",
      "141\n",
      "[3.33125458]\n",
      "51.48994501680136\n",
      "0.996\n",
      "0.94101\n",
      "0.9474\n",
      "[29.0917294]\n",
      "142\n",
      "[2.89128532]\n",
      "50.01839581131935\n",
      "0.996\n",
      "0.93896997\n",
      "0.9527\n",
      "[29.26689148]\n",
      "143\n",
      "[2.00110836]\n",
      "47.41284251026809\n",
      "0.99799997\n",
      "0.94275\n",
      "0.9507\n",
      "[29.43780661]\n",
      "144\n",
      "[4.37748222]\n",
      "46.97135495580733\n",
      "0.99300003\n",
      "0.94254005\n",
      "0.9511\n",
      "[29.6032598]\n",
      "145\n",
      "[4.88628769]\n",
      "49.31080651655793\n",
      "0.996\n",
      "0.93899995\n",
      "0.9513\n",
      "[29.85201573]\n",
      "146\n",
      "[3.58278084]\n",
      "49.236025078222156\n",
      "0.995\n",
      "0.94135\n",
      "0.9507\n",
      "[30.02826285]\n",
      "147\n",
      "[2.03138638]\n",
      "47.05722427740693\n",
      "0.99799997\n",
      "0.94315004\n",
      "0.95\n",
      "[30.25784969]\n",
      "148\n",
      "[3.40600739]\n",
      "45.46590057015419\n",
      "0.996\n",
      "0.94321\n",
      "0.9538\n",
      "[30.46135664]\n",
      "149\n",
      "[2.92949181]\n",
      "46.011121209710836\n",
      "0.99799997\n",
      "0.93856\n",
      "0.9516\n",
      "[30.63555264]\n",
      "150\n",
      "[2.91061268]\n",
      "48.7647527102381\n",
      "0.99700004\n",
      "0.94089\n",
      "0.9467\n",
      "[30.86829162]\n",
      "151\n",
      "[2.65664864]\n",
      "50.01728021167219\n",
      "0.99799997\n",
      "0.93915\n",
      "0.9493\n",
      "[31.14651656]\n",
      "152\n",
      "[2.9535923]\n",
      "49.657961055636406\n",
      "0.995\n",
      "0.94284993\n",
      "0.9505\n",
      "[31.33498168]\n",
      "153\n",
      "[2.17029839]\n",
      "50.46116350032389\n",
      "0.99899995\n",
      "0.94129\n",
      "0.9469\n",
      "[31.50127172]\n",
      "154\n",
      "[2.79492359]\n",
      "48.19844149425626\n",
      "0.996\n",
      "0.9409\n",
      "0.9478\n",
      "[31.68405581]\n",
      "155\n",
      "[2.96102619]\n",
      "49.73868156224489\n",
      "0.996\n",
      "0.94267005\n",
      "0.9521\n",
      "[31.87989378]\n",
      "156\n",
      "[2.82941761]\n",
      "52.750974860042334\n",
      "0.99700004\n",
      "0.9329499\n",
      "0.948\n",
      "[32.06095481]\n",
      "157\n",
      "[1.98545246]\n",
      "51.54552806541324\n",
      "0.99799997\n",
      "0.9369899\n",
      "0.9504\n",
      "[32.31728673]\n",
      "158\n",
      "[3.90125237]\n",
      "52.38239977508783\n",
      "0.996\n",
      "0.93851006\n",
      "0.9465\n",
      "[32.51010156]\n",
      "159\n",
      "[4.07337151]\n",
      "56.303062811493874\n",
      "0.99300003\n",
      "0.9325\n",
      "0.9503\n",
      "[32.70396757]\n",
      "160\n",
      "[4.48475037]\n",
      "56.34006364457309\n",
      "0.99300003\n",
      "0.93478\n",
      "0.9496\n",
      "[32.8911016]\n",
      "161\n",
      "[2.24512863]\n",
      "55.88075163774192\n",
      "0.996\n",
      "0.93533003\n",
      "0.9477\n",
      "[33.07771158]\n",
      "162\n",
      "[2.41077309]\n",
      "53.54412407428026\n",
      "0.996\n",
      "0.93373996\n",
      "0.9469\n",
      "[33.29445457]\n",
      "163\n",
      "[3.14566669]\n",
      "53.47540187463164\n",
      "0.9940001\n",
      "0.93819\n",
      "0.9496\n",
      "[33.48872757]\n",
      "164\n",
      "[2.65637512]\n",
      "52.753199050202966\n",
      "0.996\n",
      "0.93802994\n",
      "0.9495\n",
      "[33.6894505]\n",
      "165\n",
      "[2.1306076]\n",
      "49.365667670965195\n",
      "0.99899995\n",
      "0.94097996\n",
      "0.9465\n",
      "[33.87827253]\n",
      "166\n",
      "[2.65427818]\n",
      "49.61202069371939\n",
      "0.996\n",
      "0.93501997\n",
      "0.9493\n",
      "[34.06703639]\n",
      "167\n",
      "[2.88039017]\n",
      "53.20673168450594\n",
      "0.995\n",
      "0.93722\n",
      "0.9472\n",
      "[34.24509954]\n",
      "168\n",
      "[2.25175381]\n",
      "54.53016130812466\n",
      "0.99700004\n",
      "0.93916\n",
      "0.9498\n",
      "[34.43479156]\n",
      "169\n",
      "[2.17985935]\n",
      "51.29350968822837\n",
      "0.99899995\n",
      "0.9420301\n",
      "0.9503\n",
      "[34.61833262]\n",
      "170\n",
      "[1.61732407]\n",
      "49.80434134975076\n",
      "1.0\n",
      "0.94006\n",
      "0.9495\n",
      "[34.81405044]\n",
      "171\n",
      "[2.84216499]\n",
      "47.806078964844346\n",
      "0.99799997\n",
      "0.94342005\n",
      "0.9505\n",
      "[34.99704957]\n",
      "172\n",
      "[1.97900448]\n",
      "50.139266937971115\n",
      "0.99899995\n",
      "0.9429\n",
      "0.954\n",
      "[35.22042561]\n",
      "173\n",
      "[2.79220467]\n",
      "51.11438889242709\n",
      "0.99700004\n",
      "0.93856\n",
      "0.9479\n",
      "[35.39586639]\n",
      "174\n",
      "[3.49405251]\n",
      "54.2275670170784\n",
      "0.991\n",
      "0.93525\n",
      "0.9482\n",
      "[35.58900642]\n",
      "175\n",
      "[3.22010536]\n",
      "55.11181035079062\n",
      "0.9940001\n",
      "0.93675005\n",
      "0.9509\n",
      "[35.77118039]\n",
      "176\n",
      "[3.06929169]\n",
      "52.895924950018525\n",
      "0.99700004\n",
      "0.94062006\n",
      "0.9496\n",
      "[35.95093036]\n",
      "177\n",
      "[2.26658401]\n",
      "51.74726718477905\n",
      "0.995\n",
      "0.93978995\n",
      "0.9527\n",
      "[36.16624737]\n",
      "178\n",
      "[1.91733627]\n",
      "49.10413328371942\n",
      "0.99799997\n",
      "0.9457299\n",
      "0.9544\n",
      "[36.3509655]\n",
      "179\n",
      "[2.10757408]\n",
      "50.168399792164564\n",
      "0.99799997\n",
      "0.9366501\n",
      "0.9529\n",
      "[36.54198456]\n",
      "180\n",
      "[3.1494894]\n",
      "50.381443344056606\n",
      "0.99700004\n",
      "0.93720996\n",
      "0.9507\n",
      "[36.73262572]\n",
      "181\n",
      "[2.8434536]\n",
      "51.342720836400986\n",
      "0.9940001\n",
      "0.93953\n",
      "0.9515\n",
      "[36.92506671]\n",
      "182\n",
      "[2.0500721]\n",
      "51.61095019429922\n",
      "0.99700004\n",
      "0.94158\n",
      "0.9503\n",
      "[37.15073872]\n",
      "183\n",
      "[5.27773285]\n",
      "53.621896266937256\n",
      "0.99399996\n",
      "0.9364201\n",
      "0.9491\n",
      "[37.34458089]\n",
      "184\n",
      "[1.71150379]\n",
      "53.39228553511202\n",
      "0.99899995\n",
      "0.94380987\n",
      "0.9531\n",
      "[37.72563791]\n",
      "185\n",
      "[1.4109024]\n",
      "48.28340770304203\n",
      "0.99899995\n",
      "0.94530994\n",
      "0.9534\n",
      "[37.89401889]\n",
      "186\n",
      "[3.67776718]\n",
      "50.189452048391104\n",
      "0.99799997\n",
      "0.93877\n",
      "0.9543\n",
      "[38.08365583]\n",
      "187\n",
      "[2.95280819]\n",
      "52.58801943063736\n",
      "0.9940001\n",
      "0.9438801\n",
      "0.9517\n",
      "[38.25834179]\n",
      "188\n",
      "[1.5285573]\n",
      "50.61106823757291\n",
      "1.0\n",
      "0.94083995\n",
      "0.9526\n",
      "[38.43164682]\n",
      "189\n",
      "[2.54227314]\n",
      "49.170790299773216\n",
      "0.99700004\n",
      "0.93927\n",
      "0.9528\n",
      "[38.60546684]\n",
      "190\n",
      "[3.81355629]\n",
      "52.125412687659264\n",
      "0.995\n",
      "0.94287\n",
      "0.9506\n",
      "[38.77188087]\n",
      "191\n",
      "[2.74562263]\n",
      "52.616178296506405\n",
      "0.99300003\n",
      "0.94208\n",
      "0.952\n",
      "[38.95196271]\n",
      "192\n",
      "[1.76655827]\n",
      "52.276378866285086\n",
      "0.99899995\n",
      "0.94129\n",
      "0.9496\n",
      "[39.11591363]\n",
      "193\n",
      "[2.40112438]\n",
      "49.16342247650027\n",
      "0.995\n",
      "0.94549006\n",
      "0.9523\n",
      "[39.2889235]\n",
      "194\n",
      "[2.19617729]\n",
      "50.658022709190845\n",
      "0.99700004\n",
      "0.94449997\n",
      "0.9507\n",
      "[39.45411134]\n",
      "195\n",
      "[2.64751511]\n",
      "49.057544119656086\n",
      "0.99700004\n",
      "0.94157\n",
      "0.9534\n",
      "[39.62231016]\n",
      "196\n",
      "[2.4396492]\n",
      "53.2542180120945\n",
      "0.996\n",
      "0.93803006\n",
      "0.9482\n",
      "[39.79323816]\n",
      "197\n",
      "[5.13970909]\n",
      "54.32294759899378\n",
      "0.98999995\n",
      "0.9443\n",
      "0.9513\n",
      "[39.96312523]\n",
      "198\n",
      "[1.38164883]\n",
      "51.77891394495964\n",
      "1.0\n",
      "0.94628\n",
      "0.955\n",
      "[40.12833834]\n",
      "199\n",
      "[2.37393551]\n",
      "49.33991036564112\n",
      "0.996\n",
      "0.9448099\n",
      "0.95\n",
      "[40.29731822]\n",
      "200\n",
      "[1.92882767]\n",
      "49.542736764997244\n",
      "0.99799997\n",
      "0.9459801\n",
      "0.9558\n",
      "[40.46856213]\n",
      "201\n",
      "[3.56214714]\n",
      "50.49199993163347\n",
      "0.99300003\n",
      "0.94060004\n",
      "0.9542\n",
      "[40.67601609]\n",
      "202\n",
      "[4.50033417]\n",
      "50.31884948909283\n",
      "0.99799997\n",
      "0.94326\n",
      "0.954\n",
      "[40.84587908]\n",
      "203\n",
      "[2.29145508]\n",
      "50.79491230100393\n",
      "0.99799997\n",
      "0.94589996\n",
      "0.9516\n",
      "[41.03024006]\n",
      "204\n",
      "[1.32352848]\n",
      "49.13177864998579\n",
      "1.0\n",
      "0.94267\n",
      "0.9519\n",
      "[41.19950008]\n",
      "205\n",
      "[2.59619236]\n",
      "49.76400086283684\n",
      "0.99799997\n",
      "0.9392699\n",
      "0.9508\n",
      "[41.41468692]\n",
      "206\n",
      "[1.4432312]\n",
      "49.974924800917506\n",
      "1.0\n",
      "0.94512004\n",
      "0.9532\n",
      "[41.58102679]\n",
      "207\n",
      "[2.00331097]\n",
      "47.11602941527963\n",
      "0.99799997\n",
      "0.94400996\n",
      "0.9501\n",
      "[41.7743938]\n",
      "208\n",
      "[2.0025116]\n",
      "46.514613911509514\n",
      "0.99899995\n",
      "0.94328004\n",
      "0.9491\n",
      "[41.95138669]\n",
      "209\n",
      "[1.53166256]\n",
      "45.30013692751527\n",
      "0.99799997\n",
      "0.94891995\n",
      "0.9551\n",
      "[42.11922073]\n",
      "210\n",
      "[2.80854759]\n",
      "48.56690273433924\n",
      "0.99799997\n",
      "0.93998003\n",
      "0.9539\n",
      "[42.28654575]\n",
      "211\n",
      "[3.20919571]\n",
      "54.992455553263426\n",
      "0.99700004\n",
      "0.93894994\n",
      "0.9515\n",
      "[42.45427585]\n",
      "212\n",
      "[1.37549791]\n",
      "53.778819624334574\n",
      "0.99700004\n",
      "0.94387007\n",
      "0.9522\n",
      "[42.61378789]\n",
      "213\n",
      "[1.19562798]\n",
      "49.55142970755696\n",
      "1.0\n",
      "0.94495\n",
      "0.9517\n",
      "[42.78671598]\n",
      "214\n",
      "[4.63561783]\n",
      "49.510639280080795\n",
      "0.996\n",
      "0.94741994\n",
      "0.9554\n",
      "[42.95101905]\n",
      "215\n",
      "[2.6813221]\n",
      "49.82203981652856\n",
      "0.99700004\n",
      "0.94583\n",
      "0.955\n",
      "[43.1264112]\n",
      "216\n",
      "[2.00041809]\n",
      "49.228816490620375\n",
      "0.996\n",
      "0.9437901\n",
      "0.9543\n",
      "[43.30271721]\n",
      "217\n",
      "[2.10175018]\n",
      "49.80676243454218\n",
      "0.99700004\n",
      "0.94448006\n",
      "0.9546\n",
      "[43.47111821]\n",
      "218\n",
      "[2.98202]\n",
      "51.830873019993305\n",
      "0.99700004\n",
      "0.94606\n",
      "0.9536\n",
      "[43.64966726]\n",
      "219\n",
      "[2.72715797]\n",
      "52.80687966197729\n",
      "0.995\n",
      "0.94606\n",
      "0.9549\n",
      "[43.81035519]\n",
      "220\n",
      "[2.15281372]\n",
      "53.75083367154002\n",
      "0.99899995\n",
      "0.9438101\n",
      "0.9504\n",
      "[43.98534012]\n",
      "221\n",
      "[1.71229095]\n",
      "53.50557801872492\n",
      "1.0\n",
      "0.94656\n",
      "0.9548\n",
      "[44.15992212]\n",
      "222\n",
      "[1.58214989]\n",
      "51.22396206483245\n",
      "0.99899995\n",
      "0.94387996\n",
      "0.9554\n",
      "[44.32718921]\n",
      "223\n",
      "[1.73362198]\n",
      "48.14059329405427\n",
      "0.99899995\n",
      "0.94351995\n",
      "0.9556\n",
      "[44.49842525]\n",
      "224\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[3.37239151]\n",
      "51.043652050197124\n",
      "0.995\n",
      "0.94447994\n",
      "0.9566\n",
      "[44.66417909]\n",
      "225\n",
      "[2.02834644]\n",
      "55.78055642917752\n",
      "0.99899995\n",
      "0.93886995\n",
      "0.948\n",
      "[44.82628202]\n",
      "226\n",
      "[2.81708832]\n",
      "55.86224803328514\n",
      "0.99899995\n",
      "0.9410299\n",
      "0.9532\n",
      "[44.98870611]\n",
      "227\n",
      "[1.59816418]\n",
      "55.53050347045064\n",
      "0.99799997\n",
      "0.94537\n",
      "0.9527\n",
      "[45.15517402]\n",
      "228\n",
      "[1.55810194]\n",
      "51.25957961939275\n",
      "0.99899995\n",
      "0.9471\n",
      "0.9526\n",
      "[45.32513809]\n",
      "229\n",
      "[3.03236885]\n",
      "53.319843612611294\n",
      "0.995\n",
      "0.94202006\n",
      "0.9528\n",
      "[45.48643804]\n",
      "230\n",
      "[3.64948959]\n",
      "54.934758234769106\n",
      "0.99799997\n",
      "0.94428\n",
      "0.953\n",
      "[45.65802407]\n",
      "231\n",
      "[1.17959223]\n",
      "52.34524868056178\n",
      "0.99899995\n",
      "0.94769\n",
      "0.9553\n",
      "[45.82523513]\n",
      "232\n",
      "[2.28347073]\n",
      "48.66662632673979\n",
      "0.99700004\n",
      "0.94846\n",
      "0.9561\n",
      "[45.98879409]\n",
      "233\n",
      "[3.12013874]\n",
      "48.63256600499153\n",
      "0.99899995\n",
      "0.9445499\n",
      "0.9538\n",
      "[46.16210103]\n",
      "234\n",
      "[1.60325203]\n",
      "50.54341256432235\n",
      "0.99899995\n",
      "0.94724005\n",
      "0.9545\n",
      "[46.32430696]\n",
      "235\n",
      "[2.35677109]\n",
      "54.00311396829784\n",
      "0.99700004\n",
      "0.94609004\n",
      "0.956\n",
      "[46.49416995]\n",
      "236\n",
      "[1.53716431]\n",
      "54.363920299336314\n",
      "0.99799997\n",
      "0.9452101\n",
      "0.9582\n",
      "[46.6703999]\n",
      "237\n",
      "[2.42854595]\n",
      "56.0031387899071\n",
      "0.99700004\n",
      "0.94540006\n",
      "0.9568\n",
      "[46.839854]\n",
      "238\n",
      "[1.09732981]\n",
      "56.8635638076812\n",
      "1.0\n",
      "0.94791\n",
      "0.9593\n",
      "[47.01172113]\n",
      "239\n",
      "[1.46358948]\n",
      "55.556574733927846\n",
      "0.99799997\n",
      "0.94547004\n",
      "0.9594\n",
      "[47.176332]\n",
      "240\n",
      "[1.13416595]\n",
      "52.7441442203708\n",
      "1.0\n",
      "0.94348\n",
      "0.9583\n",
      "[47.35171795]\n",
      "241\n",
      "[3.89702415]\n",
      "56.500601429492235\n",
      "0.99700004\n",
      "0.94185\n",
      "0.9564\n",
      "[47.51223087]\n",
      "242\n",
      "[1.25763607]\n",
      "55.84791023284197\n",
      "0.99899995\n",
      "0.94582003\n",
      "0.9561\n",
      "[47.67733288]\n",
      "243\n",
      "[1.90501823]\n",
      "53.373935559764504\n",
      "0.995\n",
      "0.94738996\n",
      "0.9569\n",
      "[47.84418082]\n",
      "244\n",
      "[1.84421349]\n",
      "56.65697286184877\n",
      "0.99700004\n",
      "0.93996\n",
      "0.9565\n",
      "[48.01781297]\n",
      "245\n",
      "[1.46922159]\n",
      "55.22156395111233\n",
      "0.99799997\n",
      "0.94396\n",
      "0.957\n",
      "[48.18685389]\n",
      "246\n",
      "[1.57472353]\n",
      "53.2416823245585\n",
      "0.99799997\n",
      "0.94619006\n",
      "0.9574\n",
      "[48.3543911]\n",
      "247\n",
      "[1.64176216]\n",
      "53.95698367245495\n",
      "0.99899995\n",
      "0.94454\n",
      "0.9573\n",
      "[48.51670504]\n",
      "248\n",
      "[4.40987663]\n",
      "56.85992942005396\n",
      "0.99700004\n",
      "0.94516003\n",
      "0.9557\n",
      "[48.67950201]\n",
      "249\n",
      "[0.81964846]\n",
      "53.517230074852705\n",
      "1.0\n",
      "0.94692004\n",
      "0.9579\n",
      "[48.86718082]\n",
      "250\n",
      "[1.50292234]\n",
      "52.11209584958851\n",
      "0.99899995\n",
      "0.94427997\n",
      "0.9565\n",
      "[49.04567099]\n",
      "251\n",
      "[2.51873379]\n",
      "52.36014188081026\n",
      "0.99700004\n",
      "0.9471399\n",
      "0.9571\n",
      "[49.27645206]\n",
      "252\n",
      "[1.54938326]\n",
      "52.218077898956835\n",
      "0.99899995\n",
      "0.9456999\n",
      "0.9575\n",
      "[49.43911695]\n",
      "253\n",
      "[4.53088341]\n",
      "54.81687523704022\n",
      "0.995\n",
      "0.94358\n",
      "0.9569\n",
      "[49.70042396]\n",
      "254\n",
      "[2.33870773]\n",
      "57.08997308835387\n",
      "0.99700004\n",
      "0.94417\n",
      "0.954\n",
      "[49.87998176]\n",
      "255\n",
      "[1.16633415]\n",
      "54.97846610471606\n",
      "1.0\n",
      "0.94554996\n",
      "0.9521\n",
      "[50.06742668]\n",
      "256\n",
      "[2.24044476]\n",
      "55.92118413373828\n",
      "0.996\n",
      "0.94773996\n",
      "0.9573\n",
      "[50.23746252]\n",
      "257\n",
      "[1.40042791]\n",
      "58.16445528715849\n",
      "0.99799997\n",
      "0.94323003\n",
      "0.9582\n",
      "[50.39904356]\n",
      "258\n",
      "[3.89416313]\n",
      "55.441382778808475\n",
      "0.99899995\n",
      "0.94034004\n",
      "0.9557\n",
      "[50.65289855]\n",
      "259\n",
      "[1.70225124]\n",
      "55.41768553853035\n",
      "0.99700004\n",
      "0.93892\n",
      "0.9584\n",
      "[50.8181417]\n",
      "260\n",
      "[2.36476784]\n",
      "57.32987614534795\n",
      "0.99799997\n",
      "0.94441\n",
      "0.9584\n",
      "[51.00479078]\n",
      "261\n",
      "[1.26400213]\n",
      "57.02821504883468\n",
      "1.0\n",
      "0.94654\n",
      "0.9545\n",
      "[51.16654062]\n",
      "262\n",
      "[1.34751644]\n",
      "57.222210969775915\n",
      "0.99799997\n",
      "0.94573003\n",
      "0.9554\n",
      "[51.33963346]\n",
      "263\n",
      "[2.84425163]\n",
      "53.86027503013611\n",
      "0.99700004\n",
      "0.94479007\n",
      "0.9546\n",
      "[51.50839353]\n",
      "264\n",
      "[1.44533958]\n",
      "53.92503594048321\n",
      "0.99899995\n",
      "0.94502\n",
      "0.9586\n",
      "[51.68181872]\n",
      "265\n",
      "[1.47034264]\n",
      "54.40513770375401\n",
      "0.99799997\n",
      "0.94503003\n",
      "0.9565\n",
      "[51.85221267]\n",
      "266\n",
      "[2.26505089]\n",
      "54.134172186255455\n",
      "0.996\n",
      "0.94622993\n",
      "0.9592\n",
      "[52.02662873]\n",
      "267\n",
      "[2.31521206]\n",
      "57.895550097338855\n",
      "0.99799997\n",
      "0.9457601\n",
      "0.9553\n",
      "[52.18796372]\n",
      "268\n",
      "[2.22647877]\n",
      "55.74695045128465\n",
      "0.99700004\n",
      "0.94825\n",
      "0.9569\n",
      "[52.35293579]\n",
      "269\n",
      "[1.22134409]\n",
      "51.47987457178533\n",
      "0.99899995\n",
      "0.94745004\n",
      "0.9567\n",
      "[52.52601695]\n",
      "270\n",
      "[1.00257206]\n",
      "48.92388457804918\n",
      "1.0\n",
      "0.9491099\n",
      "0.957\n",
      "[52.68918204]\n",
      "271\n",
      "[2.00406036]\n",
      "53.68066352419555\n",
      "0.996\n",
      "0.94424\n",
      "0.9592\n",
      "[52.85036206]\n",
      "272\n",
      "[3.64688339]\n",
      "58.66193586215377\n",
      "0.9940001\n",
      "0.94093\n",
      "0.9581\n",
      "[53.02320409]\n",
      "273\n",
      "[1.99954586]\n",
      "58.426887510344386\n",
      "0.9940001\n",
      "0.94280005\n",
      "0.9568\n",
      "[53.18495202]\n",
      "274\n",
      "[2.41477909]\n",
      "59.73961780592799\n",
      "0.99700004\n",
      "0.94061005\n",
      "0.9575\n",
      "[53.35740995]\n",
      "275\n",
      "[1.16944027]\n",
      "58.78768099658191\n",
      "0.99899995\n",
      "0.94718\n",
      "0.955\n",
      "[53.52177691]\n",
      "276\n",
      "[1.65068455]\n",
      "57.51712928339839\n",
      "0.99899995\n",
      "0.94685\n",
      "0.9562\n",
      "[53.70198584]\n",
      "277\n",
      "[1.48991737]\n",
      "58.54199903830886\n",
      "0.99799997\n",
      "0.94397\n",
      "0.957\n",
      "[53.87971592]\n",
      "278\n",
      "[2.02008152]\n",
      "60.70726819336414\n",
      "0.995\n",
      "0.9458\n",
      "0.9567\n",
      "[54.04841685]\n",
      "279\n",
      "[1.42383013]\n",
      "63.00460717454553\n",
      "0.99899995\n",
      "0.93825006\n",
      "0.9566\n",
      "[54.21697974]\n",
      "280\n",
      "[1.36026459]\n",
      "62.36005906574428\n",
      "0.99899995\n",
      "0.9452499\n",
      "0.9573\n",
      "[54.39135075]\n",
      "281\n",
      "[1.72117691]\n",
      "63.48106510564685\n",
      "0.99700004\n",
      "0.94444007\n",
      "0.9574\n",
      "[54.55522656]\n",
      "282\n",
      "[3.52552223]\n",
      "62.86985963769257\n",
      "0.99700004\n",
      "0.94362\n",
      "0.9508\n",
      "[54.73029876]\n",
      "283\n",
      "[1.36977415]\n",
      "61.20079443603754\n",
      "0.99899995\n",
      "0.94356\n",
      "0.9565\n",
      "[54.89965892]\n",
      "284\n",
      "[1.66083698]\n",
      "59.23816276341677\n",
      "0.99899995\n",
      "0.9427\n",
      "0.9563\n",
      "[55.06484604]\n",
      "285\n",
      "[1.29421873]\n",
      "57.282794646918774\n",
      "0.99899995\n",
      "0.94721\n",
      "0.9564\n",
      "[55.22696996]\n",
      "286\n",
      "[2.84518299]\n",
      "60.37048463895917\n",
      "0.99700004\n",
      "0.94583\n",
      "0.9557\n",
      "[55.39627695]\n",
      "287\n",
      "[2.42848377]\n",
      "64.1435512304306\n",
      "0.99700004\n",
      "0.94926006\n",
      "0.9586\n",
      "[55.56020689]\n",
      "288\n",
      "[2.24799309]\n",
      "60.68088394962251\n",
      "0.99899995\n",
      "0.94746\n",
      "0.9569\n",
      "[55.72904396]\n",
      "289\n",
      "[1.50528269]\n",
      "58.477591425180435\n",
      "0.996\n",
      "0.9438499\n",
      "0.9568\n",
      "[55.97140193]\n",
      "290\n",
      "[2.32863331]\n",
      "60.259585342369974\n",
      "0.996\n",
      "0.9441601\n",
      "0.9552\n",
      "[56.13426208]\n",
      "291\n",
      "[3.33348274]\n",
      "59.144553357735276\n",
      "0.99899995\n",
      "0.94646007\n",
      "0.9573\n",
      "[56.29854512]\n",
      "292\n",
      "[1.66059055]\n",
      "59.427198400720954\n",
      "0.99899995\n",
      "0.94639\n",
      "0.9577\n",
      "[56.46162915]\n",
      "293\n",
      "[0.89075413]\n",
      "58.13721872866154\n",
      "0.99899995\n",
      "0.9472\n",
      "0.956\n",
      "[56.64963222]\n",
      "294\n",
      "[2.2086174]\n",
      "57.11110792681575\n",
      "0.99799997\n",
      "0.9490099\n",
      "0.9583\n",
      "[56.82076621]\n",
      "295\n",
      "[1.46161461]\n",
      "57.040056055411696\n",
      "0.99700004\n",
      "0.9469099\n",
      "0.9598\n",
      "[56.98671436]\n",
      "296\n",
      "[3.28541946]\n",
      "58.15780882164836\n",
      "0.99799997\n",
      "0.94755\n",
      "0.9576\n",
      "[57.16327953]\n",
      "297\n",
      "[3.26143913]\n",
      "57.598957508802414\n",
      "0.996\n",
      "0.94666994\n",
      "0.9597\n",
      "[57.32943654]\n",
      "298\n",
      "[3.83670807]\n",
      "56.538880188018084\n",
      "0.99700004\n",
      "0.94755995\n",
      "0.9591\n",
      "[57.5250845]\n",
      "299\n",
      "[1.90721931]\n",
      "55.35624815430492\n",
      "0.99799997\n",
      "0.94701004\n",
      "0.9533\n",
      "[57.69229364]\n",
      "300\n",
      "[1.20077772]\n",
      "54.11305484548211\n",
      "0.99799997\n",
      "0.9462601\n",
      "0.9574\n",
      "[57.85951662]\n",
      "301\n",
      "[1.28453732]\n",
      "53.66070043668151\n",
      "0.99799997\n",
      "0.94872\n",
      "0.9599\n",
      "[58.03273058]\n",
      "302\n",
      "[2.35324268]\n",
      "54.921457790303975\n",
      "0.996\n",
      "0.94721\n",
      "0.9608\n",
      "[58.21160173]\n",
      "303\n",
      "[3.09700012]\n",
      "58.35084549430758\n",
      "0.99700004\n",
      "0.94466\n",
      "0.9578\n",
      "[58.5030818]\n",
      "304\n",
      "[1.14315901]\n",
      "56.631145771592855\n",
      "0.99899995\n",
      "0.94509\n",
      "0.9588\n",
      "[58.70162988]\n",
      "305\n",
      "[1.14740763]\n",
      "53.936690881848335\n",
      "0.99899995\n",
      "0.94773006\n",
      "0.9602\n",
      "[58.86244202]\n",
      "306\n",
      "[1.16041412]\n",
      "53.21807885542512\n",
      "0.99899995\n",
      "0.9481901\n",
      "0.9582\n",
      "[59.02819991]\n",
      "307\n",
      "[1.30243883]\n",
      "52.087126857601106\n",
      "0.99799997\n",
      "0.94741\n",
      "0.9557\n",
      "[59.19145203]\n",
      "308\n",
      "[2.22699623]\n",
      "53.540517648682\n",
      "0.99799997\n",
      "0.94726\n",
      "0.9549\n",
      "[59.35295916]\n",
      "309\n",
      "[2.28994141]\n",
      "55.092950243502855\n",
      "0.996\n",
      "0.94796\n",
      "0.9566\n",
      "[59.52719212]\n",
      "310\n",
      "[1.20580282]\n",
      "55.9573838301003\n",
      "1.0\n",
      "0.94497\n",
      "0.957\n",
      "[59.69693422]\n",
      "311\n",
      "[1.44472666]\n",
      "58.45344592630863\n",
      "0.99899995\n",
      "0.94219\n",
      "0.9601\n",
      "[59.86048841]\n",
      "312\n",
      "[1.52540474]\n",
      "57.21298289485276\n",
      "0.99700004\n",
      "0.94610995\n",
      "0.9598\n",
      "[60.03127861]\n",
      "313\n",
      "[1.13917513]\n",
      "57.25070471689105\n",
      "0.99899995\n",
      "0.9481899\n",
      "0.9612\n",
      "[60.20545578]\n",
      "314\n",
      "[1.38466129]\n",
      "55.42637219745666\n",
      "0.99799997\n",
      "0.9424499\n",
      "0.9615\n",
      "[60.37844086]\n",
      "315\n",
      "[1.26089039]\n",
      "53.028140616137534\n",
      "0.99899995\n",
      "0.94833994\n",
      "0.9606\n",
      "[60.54192376]\n",
      "316\n",
      "[1.56491241]\n",
      "55.105846267193556\n",
      "0.99700004\n",
      "0.94968\n",
      "0.96\n",
      "[60.71534586]\n",
      "317\n",
      "[0.87530184]\n",
      "56.067333434242755\n",
      "0.99899995\n",
      "0.95032007\n",
      "0.9628\n",
      "[60.88967395]\n",
      "318\n",
      "[0.68718815]\n",
      "53.11638931278139\n",
      "1.0\n",
      "0.94889003\n",
      "0.9615\n",
      "[61.06679678]\n",
      "319\n",
      "[3.12173996]\n",
      "51.41252146847546\n",
      "0.99799997\n",
      "0.94987\n",
      "0.9603\n",
      "[61.24571586]\n",
      "320\n",
      "[1.59699287]\n",
      "53.52579043665901\n",
      "0.99799997\n",
      "0.94644004\n",
      "0.9599\n",
      "[61.41300488]\n",
      "321\n",
      "[1.2179636]\n",
      "56.14512321911752\n",
      "0.99899995\n",
      "0.94190997\n",
      "0.9586\n",
      "[61.57524872]\n",
      "322\n",
      "[1.95958748]\n",
      "59.32270253263414\n",
      "0.996\n",
      "0.93846\n",
      "0.9538\n",
      "[61.7379787]\n",
      "323\n",
      "[1.23300304]\n",
      "60.50824546441436\n",
      "0.99799997\n",
      "0.94606\n",
      "0.9563\n",
      "[61.90040469]\n",
      "324\n",
      "[1.58615427]\n",
      "61.50543432496488\n",
      "0.99799997\n",
      "0.93926\n",
      "0.9598\n",
      "[62.06928873]\n",
      "325\n",
      "[1.43388491]\n",
      "60.88878519833088\n",
      "0.99799997\n",
      "0.94602\n",
      "0.9571\n",
      "[62.23147774]\n",
      "326\n",
      "[1.92305889]\n",
      "59.42771517857909\n",
      "0.99799997\n",
      "0.9485901\n",
      "0.9591\n",
      "[62.56365275]\n",
      "327\n",
      "[1.23667707]\n",
      "57.640429803170264\n",
      "0.99799997\n",
      "0.94996005\n",
      "0.9606\n",
      "[62.72941995]\n",
      "328\n",
      "[1.46073685]\n",
      "59.014646116644144\n",
      "0.99799997\n",
      "0.94575995\n",
      "0.96\n",
      "[62.89202285]\n",
      "329\n",
      "[1.29095268]\n",
      "60.41390824317932\n",
      "0.99700004\n",
      "0.9434701\n",
      "0.9574\n",
      "[63.08159566]\n",
      "330\n",
      "[0.96870508]\n",
      "56.345046426635236\n",
      "0.99799997\n",
      "0.94767\n",
      "0.9577\n",
      "[63.24868655]\n",
      "331\n",
      "[3.79570198]\n",
      "56.69611394777894\n",
      "0.995\n",
      "0.94406\n",
      "0.958\n",
      "[63.42637873]\n",
      "332\n",
      "[1.45257101]\n",
      "56.57466150727123\n",
      "0.99899995\n",
      "0.94668007\n",
      "0.9581\n",
      "[63.59141159]\n",
      "333\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2.49525814]\n",
      "54.39882067218423\n",
      "0.996\n",
      "0.94881\n",
      "0.9584\n",
      "[63.77611661]\n",
      "334\n",
      "[1.62368164]\n",
      "56.258307468146086\n",
      "0.9940001\n",
      "0.94689\n",
      "0.9597\n",
      "[63.95060849]\n",
      "335\n",
      "[1.19859734]\n",
      "59.217905377969146\n",
      "1.0\n",
      "0.94068\n",
      "0.9613\n",
      "[64.12568951]\n",
      "336\n",
      "[1.37370977]\n",
      "57.44150183349848\n",
      "0.99899995\n",
      "0.94613\n",
      "0.9606\n",
      "[64.2887454]\n",
      "337\n",
      "[0.91312218]\n",
      "55.26763051562011\n",
      "0.99899995\n",
      "0.95105\n",
      "0.9613\n",
      "[64.45451641]\n",
      "338\n",
      "[3.2227993]\n",
      "61.751393876969814\n",
      "0.99799997\n",
      "0.94473\n",
      "0.9609\n",
      "[64.7089293]\n",
      "339\n",
      "[1.30165577]\n",
      "61.21965132839978\n",
      "0.99899995\n",
      "0.94703007\n",
      "0.9603\n",
      "[64.96657729]\n",
      "340\n",
      "[1.78543606]\n",
      "58.368591072037816\n",
      "0.99700004\n",
      "0.94521\n",
      "0.9591\n",
      "[65.12941337]\n",
      "341\n",
      "[1.73032913]\n",
      "59.02312741521746\n",
      "0.99899995\n",
      "0.94498\n",
      "0.9594\n",
      "[65.30445623]\n",
      "342\n",
      "[0.84720936]\n",
      "58.91858387738466\n",
      "1.0\n",
      "0.94234\n",
      "0.9589\n",
      "[65.4686172]\n",
      "343\n",
      "[1.55051184]\n",
      "54.30709665454924\n",
      "0.99899995\n",
      "0.9494399\n",
      "0.96\n",
      "[65.6304183]\n",
      "344\n",
      "[1.93435974]\n",
      "56.87930281460285\n",
      "0.996\n",
      "0.9422\n",
      "0.9605\n",
      "[65.80669117]\n",
      "345\n",
      "[1.21041565]\n",
      "60.239072451367974\n",
      "0.99799997\n",
      "0.94337\n",
      "0.9578\n",
      "[65.96848416]\n",
      "346\n",
      "[2.22297115]\n",
      "61.49078891798854\n",
      "0.99799997\n",
      "0.9443299\n",
      "0.9562\n",
      "[66.13001204]\n",
      "347\n",
      "[0.95752449]\n",
      "62.77339859306812\n",
      "1.0\n",
      "0.94446003\n",
      "0.9553\n",
      "[66.29501796]\n",
      "348\n",
      "[1.68445435]\n",
      "63.96317297779024\n",
      "0.99799997\n",
      "0.94540006\n",
      "0.9602\n",
      "[66.45787096]\n",
      "349\n",
      "[1.15480604]\n",
      "64.08456594031304\n",
      "1.0\n",
      "0.94793\n",
      "0.9596\n",
      "[66.62818408]\n",
      "350\n",
      "[1.49141722]\n",
      "62.06230383878574\n",
      "0.99700004\n",
      "0.94726\n",
      "0.9595\n",
      "[66.79488611]\n",
      "351\n",
      "[1.936092]\n",
      "58.0447258092463\n",
      "0.99799997\n",
      "0.9453699\n",
      "0.9562\n",
      "[66.95995021]\n",
      "352\n",
      "[2.85782928]\n",
      "61.002610405907035\n",
      "0.99899995\n",
      "0.95142\n",
      "0.9564\n",
      "[67.12142634]\n",
      "353\n",
      "[1.94264507]\n",
      "60.210813065990806\n",
      "0.99899995\n",
      "0.94664\n",
      "0.9581\n",
      "[67.29426527]\n",
      "354\n",
      "[1.06040659]\n",
      "58.577332543674856\n",
      "0.99799997\n",
      "0.94838\n",
      "0.9606\n",
      "[67.4625864]\n",
      "355\n",
      "[2.81349068]\n",
      "63.13909133616835\n",
      "0.99700004\n",
      "0.94472\n",
      "0.9609\n",
      "[67.63152933]\n",
      "356\n",
      "[1.62212048]\n",
      "63.79763182159513\n",
      "0.99700004\n",
      "0.94512\n",
      "0.9606\n",
      "[67.80245042]\n",
      "357\n",
      "[1.54392099]\n",
      "64.54917380493134\n",
      "0.99799997\n",
      "0.94637\n",
      "0.9531\n",
      "[67.96328235]\n",
      "358\n",
      "[1.63846722]\n",
      "65.02540142275393\n",
      "0.99799997\n",
      "0.94369\n",
      "0.9505\n",
      "[68.13343453]\n",
      "359\n",
      "[3.44178467]\n",
      "66.8605077136308\n",
      "0.99799997\n",
      "0.9448\n",
      "0.958\n",
      "[68.42843461]\n",
      "360\n",
      "[1.31816864]\n",
      "68.86654143594205\n",
      "0.99899995\n",
      "0.94393003\n",
      "0.9593\n",
      "[68.60659051]\n",
      "361\n",
      "[1.39634228]\n",
      "66.49711732659489\n",
      "0.99899995\n",
      "0.94594\n",
      "0.9574\n",
      "[68.78133941]\n",
      "362\n",
      "[1.2570982]\n",
      "67.34530556434765\n",
      "0.99799997\n",
      "0.9469999\n",
      "0.9591\n",
      "[68.94623351]\n",
      "363\n",
      "[1.21112223]\n",
      "64.3750074300915\n",
      "0.99899995\n",
      "0.95071\n",
      "0.9599\n",
      "[69.11462951]\n",
      "364\n",
      "[1.4131712]\n",
      "61.64185249526054\n",
      "0.99899995\n",
      "0.94878995\n",
      "0.9597\n",
      "[69.33858657]\n",
      "365\n",
      "[1.25733175]\n",
      "61.53230171697214\n",
      "0.99899995\n",
      "0.94763005\n",
      "0.9607\n",
      "[69.51103354]\n",
      "366\n",
      "[1.34835358]\n",
      "65.28607029374689\n",
      "0.99899995\n",
      "0.94594\n",
      "0.9602\n",
      "[69.76975942]\n",
      "367\n",
      "[1.58734388]\n",
      "64.99551716633141\n",
      "0.99899995\n",
      "0.94773996\n",
      "0.96\n",
      "[69.95325041]\n",
      "368\n",
      "[5.08239174]\n",
      "64.71391534432769\n",
      "0.996\n",
      "0.94747\n",
      "0.9592\n",
      "[70.12545753]\n",
      "369\n",
      "[1.6907896]\n",
      "67.66484532505274\n",
      "0.996\n",
      "0.9455701\n",
      "0.9589\n",
      "[70.29838467]\n",
      "370\n",
      "[1.40116482]\n",
      "65.2827787771821\n",
      "0.99799997\n",
      "0.94582\n",
      "0.9594\n",
      "[70.4751246]\n",
      "371\n",
      "[0.96982517]\n",
      "66.33102631336078\n",
      "0.99899995\n",
      "0.9474899\n",
      "0.9597\n",
      "[70.65237761]\n",
      "372\n",
      "[1.49414988]\n",
      "67.12807553447783\n",
      "0.99899995\n",
      "0.94474\n",
      "0.9596\n",
      "[70.84974051]\n",
      "373\n",
      "[4.19325714]\n",
      "64.58057215437293\n",
      "0.99700004\n",
      "0.94585\n",
      "0.9577\n",
      "[71.02260447]\n",
      "374\n",
      "[1.87710934]\n",
      "68.65458145737648\n",
      "0.99700004\n",
      "0.94361\n",
      "0.956\n",
      "[71.19926238]\n",
      "375\n",
      "[3.86526299]\n",
      "68.55766189470887\n",
      "0.995\n",
      "0.94548\n",
      "0.9558\n",
      "[71.36622834]\n",
      "376\n",
      "[1.33732424]\n",
      "68.20268451422453\n",
      "1.0\n",
      "0.94937\n",
      "0.96\n",
      "[71.54408336]\n",
      "377\n",
      "[0.84993725]\n",
      "65.7835299372673\n",
      "1.0\n",
      "0.94785994\n",
      "0.9573\n",
      "[71.71318841]\n",
      "378\n",
      "[1.4951746]\n",
      "63.239618480205536\n",
      "0.99799997\n",
      "0.9459101\n",
      "0.9597\n",
      "[71.88371134]\n",
      "379\n",
      "[1.58835411]\n",
      "63.922618087381124\n",
      "0.99700004\n",
      "0.9394\n",
      "0.9557\n",
      "[72.06277537]\n",
      "380\n",
      "[2.15386333]\n",
      "68.41341174393892\n",
      "0.996\n",
      "0.94535\n",
      "0.9575\n",
      "[72.23226023]\n",
      "381\n",
      "[1.44112663]\n",
      "72.37453763186932\n",
      "0.99899995\n",
      "0.94474\n",
      "0.9602\n",
      "[72.40963435]\n",
      "382\n",
      "[1.36747694]\n",
      "69.13615152332932\n",
      "0.99700004\n",
      "0.95132\n",
      "0.9602\n",
      "[72.64130425]\n",
      "383\n",
      "[0.73829546]\n",
      "64.88959386665374\n",
      "1.0\n",
      "0.95063007\n",
      "0.9595\n",
      "[72.81378818]\n",
      "384\n",
      "[1.1525959]\n",
      "62.23041402967647\n",
      "0.99899995\n",
      "0.9483\n",
      "0.9591\n",
      "[72.98751831]\n",
      "385\n",
      "[2.19595051]\n",
      "64.73179375100881\n",
      "0.99700004\n",
      "0.94909\n",
      "0.9603\n",
      "[73.15588641]\n",
      "386\n",
      "[2.58154011]\n",
      "70.33178420457989\n",
      "0.99799997\n",
      "0.94211996\n",
      "0.9591\n",
      "[73.33030939]\n",
      "387\n",
      "[2.03156509]\n",
      "69.86248107254505\n",
      "0.99799997\n",
      "0.9415299\n",
      "0.959\n",
      "[73.50542331]\n",
      "388\n",
      "[3.77273827]\n",
      "65.70787915913388\n",
      "0.996\n",
      "0.94523\n",
      "0.9593\n",
      "[73.67984343]\n",
      "389\n",
      "[3.49551048]\n",
      "68.09229012322612\n",
      "0.99799997\n",
      "0.9452899\n",
      "0.9597\n",
      "[73.85155344]\n",
      "390\n",
      "[1.36106625]\n",
      "67.57845366653055\n",
      "0.99899995\n",
      "0.9461199\n",
      "0.9597\n",
      "[74.02863455]\n",
      "391\n",
      "[1.12690563]\n",
      "66.33879371918738\n",
      "0.99899995\n",
      "0.94475\n",
      "0.9593\n",
      "[74.21749568]\n",
      "392\n",
      "[2.90267086]\n",
      "64.25700446777046\n",
      "0.99799997\n",
      "0.94994\n",
      "0.9586\n",
      "[74.39248657]\n",
      "393\n",
      "[1.37946911]\n",
      "62.37491411296651\n",
      "0.99899995\n",
      "0.9502199\n",
      "0.9599\n",
      "[74.5621736]\n",
      "394\n",
      "[1.3404788]\n",
      "60.903594695497304\n",
      "0.99899995\n",
      "0.9464399\n",
      "0.9605\n",
      "[74.73283243]\n",
      "395\n",
      "[4.62683487]\n",
      "60.2035845243372\n",
      "0.995\n",
      "0.94879997\n",
      "0.9613\n",
      "[74.90081739]\n",
      "396\n",
      "[5.16757698]\n",
      "64.85187185835093\n",
      "0.996\n",
      "0.94642\n",
      "0.9608\n",
      "[75.06168842]\n",
      "397\n",
      "[1.20932388]\n",
      "65.3236689446494\n",
      "0.99799997\n",
      "0.9427401\n",
      "0.9563\n",
      "[75.23538256]\n",
      "398\n",
      "[1.43760166]\n",
      "63.25097974576056\n",
      "0.99899995\n",
      "0.9454301\n",
      "0.9582\n",
      "[75.4212935]\n",
      "399\n",
      "[1.11409616]\n",
      "65.59078993275762\n",
      "1.0\n",
      "0.94540006\n",
      "0.9582\n",
      "[75.72988749]\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "abs_weights_diff = []\n",
    "abs_biases_diff = []\n",
    "Train_Acc = []\n",
    "Test_Acc = []\n",
    "CrE_Train = []\n",
    "CrE_Test = []\n",
    "\n",
    "\n",
    "mb_size = 100\n",
    "n_epochs = 400\n",
    "\n",
    "compTime = np.zeros([n_epochs,1])\n",
    "\n",
    "acc_train = np.zeros([n_epochs,1])\n",
    "acc_test = np.zeros([n_epochs,1])\n",
    "acc_test_max = np.zeros([n_epochs,1])\n",
    "total_loss = np.zeros([n_epochs,1])\n",
    "acc_test_user0 = np.zeros([n_epochs,1])\n",
    "alpha=0.01#0.01\n",
    "ii = 0\n",
    "localIter = 10\n",
    "for k in range(n_epochs):\n",
    "        print(k)\n",
    "        if k > 0:\n",
    "            compTime[k] = compTime[k-1]\n",
    "            \n",
    "        batch_x = []\n",
    "        batch_y = []\n",
    "        for i in range(no_users):\n",
    "            batch_xx , batch_yy = mini_batches(x_train_k[i],y_train_k[i],  mb_size)\n",
    "            batch_x.append(batch_xx)\n",
    "            batch_y.append(batch_yy)\n",
    "        \n",
    "                \n",
    "        # Run the optimization to update W  \n",
    "        #for i in range(no_users):\n",
    "        for i in range(0,no_users,2):\n",
    "            if(i==0):\n",
    "                flag = 0\n",
    "                t = time.time()\n",
    "                for kk in range(localIter):\n",
    "                    loss = run_optimization(batch_x[i], batch_y[i], users[i], users[i], users[i+1], rho, flag)\n",
    "                elapsed = time.time() - t\n",
    "                compTime[k]=compTime[k]+elapsed\n",
    "            elif (i == no_users-1):\n",
    "                flag = 1\n",
    "                for kk in range(localIter):\n",
    "                    loss = run_optimization(batch_x[i], batch_y[i], users[i], users[i-1], users[i], rho, flag)\n",
    "            else:\n",
    "                flag = 2\n",
    "                for kk in range(localIter):\n",
    "                    loss = run_optimization(batch_x[i], batch_y[i], users[i], users[i-1], users[i+1], rho, flag)\n",
    "                    \n",
    "            total_loss[k] = total_loss[k] + loss\n",
    "            \n",
    "        for i in range(1,no_users,2):\n",
    "            if(i==0):\n",
    "                flag = 0\n",
    "                for kk in range(localIter):\n",
    "                    loss = run_optimization(batch_x[i], batch_y[i], users[i], users[i], users[i+1], rho, flag)\n",
    "            elif (i == no_users-1):\n",
    "                flag = 1\n",
    "                for kk in range(localIter):\n",
    "                    loss = run_optimization(batch_x[i], batch_y[i], users[i], users[i-1], users[i], rho, flag)\n",
    "            else:\n",
    "                flag = 2\n",
    "                for kk in range(localIter):\n",
    "                    loss = run_optimization(batch_x[i], batch_y[i], users[i], users[i-1], users[i+1], rho, flag)\n",
    "            \n",
    "            \n",
    "            \n",
    "            total_loss[k] = total_loss[k] + loss\n",
    "        \n",
    "        total_loss[k] = total_loss[k]/no_users\n",
    "        print(total_loss[k])\n",
    "        \n",
    "        primal_residual=0\n",
    "        for i in range(no_users):\n",
    "            W1 = tf.reshape(users[i].weights['W1'],[784*n_hidden_1, 1])\n",
    "            W2 = tf.reshape(users[i].weights['W2'],[n_hidden_1*n_hidden_2, 1])\n",
    "            W3 = tf.reshape(users[i].weights['W3'],[10*n_hidden_2, 1])\n",
    "            if (i==0):\n",
    "                W1_r = tf.reshape(users[i+1].weights['W1'],[784*n_hidden_1, 1])\n",
    "                W2_r = tf.reshape(users[i+1].weights['W2'],[n_hidden_1*n_hidden_2, 1])\n",
    "                W3_r = tf.reshape(users[i+1].weights['W3'],[10*n_hidden_2, 1])\n",
    "                \n",
    "                yy1 = (users[i].lambda_W1_r + alpha*rho*(W1 - W1_r)).numpy()\n",
    "                yy2 = (users[i].lambda_W2_r + alpha*rho*(W2 - W2_r)).numpy()\n",
    "                yy3 = (users[i].lambda_W3_r + alpha*rho*(W3 - W3_r)).numpy()\n",
    "                \n",
    "                users[i].lambda_W1_r.assign(yy1) \n",
    "                users[i].lambda_W2_r.assign(yy2) \n",
    "                users[i].lambda_W3_r.assign(yy3)\n",
    "                \n",
    "                primal_residual = primal_residual +tf.reduce_sum(tf.math.square(W1 - W1_r)).numpy()\n",
    "                primal_residual = primal_residual +tf.reduce_sum(tf.math.square(W2 - W2_r)).numpy()\n",
    "                primal_residual = primal_residual +tf.reduce_sum(tf.math.square(W3 - W3_r)).numpy()\n",
    "            \n",
    "            elif (i==no_users-1):\n",
    "                W1_l = tf.reshape(users[i-1].weights['W1'],[784*n_hidden_1, 1])\n",
    "                W2_l = tf.reshape(users[i-1].weights['W2'],[n_hidden_1*n_hidden_2, 1])\n",
    "                W3_l = tf.reshape(users[i-1].weights['W3'],[10*n_hidden_2, 1])\n",
    "                \n",
    "                yy1 = (users[i].lambda_W1_l - alpha*rho*(W1 - W1_l)).numpy()\n",
    "                yy2 = (users[i].lambda_W2_l - alpha*rho*(W2 - W2_l)).numpy()\n",
    "                yy3 = (users[i].lambda_W3_l - alpha*rho*(W3 - W3_l)).numpy()\n",
    "                \n",
    "                users[i].lambda_W1_l.assign(yy1) \n",
    "                users[i].lambda_W2_l.assign(yy2) \n",
    "                users[i].lambda_W3_l.assign(yy3)\n",
    "                \n",
    "                primal_residual = primal_residual +tf.reduce_sum(tf.math.square(W1 - W1_l)).numpy()\n",
    "                primal_residual = primal_residual +tf.reduce_sum(tf.math.square(W2 - W2_l)).numpy()\n",
    "                primal_residual = primal_residual +tf.reduce_sum(tf.math.square(W3 - W3_l)).numpy()\n",
    "            \n",
    "            else:\n",
    "                W1_r = tf.reshape(users[i+1].weights['W1'],[784*n_hidden_1, 1])\n",
    "                W2_r = tf.reshape(users[i+1].weights['W2'],[n_hidden_1*n_hidden_2, 1])\n",
    "                W3_r = tf.reshape(users[i+1].weights['W3'],[10*n_hidden_2, 1])\n",
    "                \n",
    "                W1_l = tf.reshape(users[i-1].weights['W1'],[784*n_hidden_1, 1])\n",
    "                W2_l = tf.reshape(users[i-1].weights['W2'],[n_hidden_1*n_hidden_2, 1])\n",
    "                W3_l = tf.reshape(users[i-1].weights['W3'],[10*n_hidden_2, 1])\n",
    "                \n",
    "                \n",
    "                yy1 = (users[i].lambda_W1_r + alpha*rho*(W1 - W1_r)).numpy()\n",
    "                yy2 = (users[i].lambda_W2_r + alpha*rho*(W2 - W2_r)).numpy()\n",
    "                yy3 = (users[i].lambda_W3_r + alpha*rho*(W3 - W3_r)).numpy()\n",
    "                \n",
    "                yy4 = (users[i].lambda_W1_l - alpha*rho*(W1 - W1_l)).numpy()\n",
    "                yy5 = (users[i].lambda_W2_l - alpha*rho*(W2 - W2_l)).numpy()\n",
    "                yy6 = (users[i].lambda_W3_l - alpha*rho*(W3 - W3_l)).numpy()\n",
    "                \n",
    "                users[i].lambda_W1_r.assign(yy1) \n",
    "                users[i].lambda_W2_r.assign(yy2) \n",
    "                users[i].lambda_W3_r.assign(yy3)\n",
    "            \n",
    "                users[i].lambda_W1_l.assign(yy4) \n",
    "                users[i].lambda_W2_l.assign(yy5) \n",
    "                users[i].lambda_W3_l.assign(yy6)\n",
    "                \n",
    "                primal_residual = primal_residual +tf.reduce_sum(tf.math.square(W1 - W1_r)).numpy()\n",
    "                primal_residual = primal_residual +tf.reduce_sum(tf.math.square(W2 - W2_r)).numpy()\n",
    "                primal_residual = primal_residual +tf.reduce_sum(tf.math.square(W3 - W3_r)).numpy()\n",
    "                \n",
    "                primal_residual = primal_residual +tf.reduce_sum(tf.math.square(W1 - W1_l)).numpy()\n",
    "                primal_residual = primal_residual +tf.reduce_sum(tf.math.square(W2 - W2_l)).numpy()\n",
    "                primal_residual = primal_residual +tf.reduce_sum(tf.math.square(W3 - W3_l)).numpy()\n",
    "                \n",
    "            \n",
    "            #users[i].lambda_W1.assign(yy1) \n",
    "            #users[i].lambda_W2.assign(yy2) \n",
    "            #users[i].lambda_W3.assign(yy3)\n",
    "            \n",
    "        print(primal_residual)\n",
    "        \n",
    "        \n",
    "        train_acc = []\n",
    "        test_acc = []\n",
    "        for j in range(no_users):  \n",
    "            train_pred = users[j].neural_net(batch_x[j])\n",
    "            train_acc.append(accuracy(train_pred , batch_y[j]))\n",
    "            test_pred = users[j].neural_net(x_test)\n",
    "            test_acc.append(accuracy(test_pred , y_test))\n",
    "            \n",
    "        avgAcc_Train = np.mean(train_acc)\n",
    "        avgAcc_Test = np.mean(test_acc)\n",
    "        max_Test = np.max(test_acc)\n",
    "        print(avgAcc_Train)\n",
    "        print(avgAcc_Test)\n",
    "        print(max_Test)\n",
    "        print(compTime[k])\n",
    "        acc_train[k] = avgAcc_Train\n",
    "        acc_test[k] = avgAcc_Test\n",
    "        acc_test_max[k] = max_Test\n",
    "            \n",
    "        #user0.weights['W1'] = tf.reshape(central_modal[0],[784,n_hidden_1])\n",
    "        #user0.weights['W2'] = tf.reshape(central_modal[1],[n_hidden_1,n_hidden_2])\n",
    "        #user0.weights['W3'] = tf.reshape(central_modal[2],[n_hidden_2, 10])\n",
    "        \n",
    "        #test_pred = user0.neural_net(x_test)\n",
    "        #test_acc_user0 = accuracy(test_pred, y_test).numpy()\n",
    "        #acc_test_user0[k]=test_acc_user0\n",
    "        #print(test_acc_user0)\n",
    "            \n",
    "\n",
    "#np.save('loss_analog_4096_3.npy', total_loss)\n",
    "\n",
    "#np.save('acc_test_digital_4096_user0_3.npy', acc_test_user0)\n",
    "\n",
    "#np.save('loss_digital_40960_5.npy', total_loss)\n",
    "#np.save('acc_train_digital_40960_5.npy', acc_train)\n",
    "#np.save('acc_test_digital_40960_5.npy', acc_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#np.save('gadmm_dnn_train.npy', acc_train)\n",
    "#np.save('gadmm_dnn_mean_test.npy', acc_test)\n",
    "np.save('gadmm_dnn_max_test.npy', acc_test)\n",
    "#np.save('gadmm_dnn_compTime.npy', compTime)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}