conv1d-generic-clean.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:49:48.292829Z",
     "start_time": "2020-09-15T00:49:48.201726Z"
    }
   },
   "outputs": [],
   "source": [
    "%load_ext nb_black\n",
    "%load_ext autoreload\n",
    "\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:49:49.574964Z",
     "start_time": "2020-09-15T00:49:48.293969Z"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "from pathlib import Path\n",
    "\n",
    "from requests import get\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.metrics import roc_auc_score, log_loss\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from tensorflow.keras.utils import plot_model\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "\n",
    "import logging\n",
    "\n",
    "logging.basicConfig(level=logging.WARN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:49:49.766392Z",
     "start_time": "2020-09-15T00:49:49.576238Z"
    }
   },
   "outputs": [],
   "source": [
    "from thc_net.explainable_model.input_utils import preproc_dataset\n",
    "from thc_net.explainable_model.model import build_model\n",
    "from thc_net.explainable_model.random_utils import setup_seed, SEED\n",
    "from sklearn.model_selection import StratifiedShuffleSplit\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.pyplot import imshow\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:49:49.789555Z",
     "start_time": "2020-09-15T00:49:49.767586Z"
    }
   },
   "outputs": [],
   "source": [
    "setup_seed()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:49:49.821952Z",
     "start_time": "2020-09-15T00:49:49.790598Z"
    }
   },
   "outputs": [],
   "source": [
    "def download(url, out, force=False, verify=True):\n",
    "    out.parent.mkdir(parents=True, exist_ok=True)\n",
    "    if force and out.exists():\n",
    "        print(f\"Removing file at {str(out)}\")\n",
    "        out.unlink()\n",
    "\n",
    "    if out.exists():\n",
    "        print(\"File already exists.\")\n",
    "        return\n",
    "    print(f\"Downloading {url} at {str(out)} ...\")\n",
    "    # open in binary mode\n",
    "    with out.open(mode=\"wb\") as file:\n",
    "        # get request\n",
    "        response = get(url, verify=verify)\n",
    "        for chunk in response.iter_content(100000):\n",
    "            # write to file\n",
    "            file.write(chunk)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:49:49.866842Z",
     "start_time": "2020-09-15T00:49:49.823008Z"
    }
   },
   "outputs": [],
   "source": [
    "def plot_history(history):\n",
    "    loss_list = [s for s in history.history.keys() if \"loss\" in s and \"val\" not in s]\n",
    "    val_loss_list = [s for s in history.history.keys() if \"loss\" in s and \"val\" in s]\n",
    "    acc_list = [s for s in history.history.keys() if \"AUC\" in s and \"val\" not in s]\n",
    "    val_acc_list = [s for s in history.history.keys() if \"AUC\" in s and \"val\" in s]\n",
    "\n",
    "    if len(loss_list) == 0:\n",
    "        print(\"Loss is missing in history\")\n",
    "        return\n",
    "\n",
    "    ## As loss always exists\n",
    "    epochs = range(1, len(history.history[loss_list[0]]) + 1)\n",
    "\n",
    "    ## Loss\n",
    "    plt.figure(1)\n",
    "    for l in loss_list:\n",
    "        plt.plot(\n",
    "            epochs,\n",
    "            history.history[l],\n",
    "            \"b\",\n",
    "            label=\"Training loss (\"\n",
    "            + str(str(format(history.history[l][-1], \".5f\")) + \")\"),\n",
    "        )\n",
    "    for l in val_loss_list:\n",
    "        plt.plot(\n",
    "            epochs,\n",
    "            history.history[l],\n",
    "            \"g\",\n",
    "            label=\"Validation loss (\"\n",
    "            + str(str(format(history.history[l][-1], \".5f\")) + \")\"),\n",
    "        )\n",
    "\n",
    "    plt.title(\"Loss\")\n",
    "    plt.xlabel(\"Epochs\")\n",
    "    plt.ylabel(\"Loss\")\n",
    "    plt.legend()\n",
    "\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:49:49.891028Z",
     "start_time": "2020-09-15T00:49:49.867842Z"
    }
   },
   "outputs": [],
   "source": [
    "dataset_name = \"portoseguro\"\n",
    "filename = \"train_bench.csv\"\n",
    "target = \"target\"\n",
    "ids = []"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "dataset_name = \"rl\"\n",
    "filename = \"train_bench.csv\"\n",
    "target = \"target\"\n",
    "ids = []"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "dataset_name = \"amazon\"\n",
    "filename = \"train_bench.csv\"\n",
    "target = \"ACTION\"\n",
    "ids = []"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "dataset_name = \"santander-customer-satisfaction\"\n",
    "filename = \"train_bench.csv\"\n",
    "target = \"TARGET\"\n",
    "ids = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:49:49.916377Z",
     "start_time": "2020-09-15T00:49:49.892757Z"
    }
   },
   "outputs": [],
   "source": [
    "dataset_name = \"road-safety\"\n",
    "filename = \"train_bench.csv\"\n",
    "target = \"Sex_of_Driver_df_res\"\n",
    "ids = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:49:49.941359Z",
     "start_time": "2020-09-15T00:49:49.917642Z"
    }
   },
   "outputs": [],
   "source": [
    "dataset_name = \"open-payments\"\n",
    "filename = \"train_bench.csv\"\n",
    "target = \"status\"\n",
    "ids = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:49:49.966021Z",
     "start_time": "2020-09-15T00:49:49.942327Z"
    }
   },
   "outputs": [],
   "source": [
    "dataset_name = \"census-income\"\n",
    "filename = \"train_bench.csv\"\n",
    "target = \"taxable income amount\"\n",
    "ids = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:49:49.989980Z",
     "start_time": "2020-09-15T00:49:49.967110Z"
    }
   },
   "outputs": [],
   "source": [
    "dataset_name = \"albert\"\n",
    "filename = \"train_bench.csv\"\n",
    "target = \"target\"\n",
    "ids = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:49:50.014738Z",
     "start_time": "2020-09-15T00:49:49.990961Z"
    }
   },
   "outputs": [],
   "source": [
    "dataset_name = \"cat-in-the-dat-ii\"\n",
    "filename = \"train_bench.csv\"\n",
    "target = \"target\"\n",
    "ids = [\"id\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:49:50.041756Z",
     "start_time": "2020-09-15T00:49:50.015975Z"
    }
   },
   "outputs": [],
   "source": [
    "dataset_name = \"bnp-cardif\"\n",
    "filename = \"train_bench.csv\"\n",
    "target = \"target\"\n",
    "ids = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:49:50.066868Z",
     "start_time": "2020-09-15T00:49:50.042850Z"
    }
   },
   "outputs": [],
   "source": [
    "dataset_name = \"homesite-quote-conversion\"\n",
    "filename = \"train_bench.csv\"\n",
    "target = \"QuoteConversion_Flag\"\n",
    "ids = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:49:50.091094Z",
     "start_time": "2020-09-15T00:49:50.067915Z"
    }
   },
   "outputs": [],
   "source": [
    "dataset_name = \"give-me-some-credit\"\n",
    "filename = \"train_bench.csv\"\n",
    "target = \"SeriousDlqin2yrs\"\n",
    "ids = [\"Unamed\", \"age\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:49:50.114994Z",
     "start_time": "2020-09-15T00:49:50.091926Z"
    }
   },
   "outputs": [],
   "source": [
    "dataset_name = \"bank-marketing\"\n",
    "filename = \"train_bench.csv\"\n",
    "target = \"y\"\n",
    "ids = []"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "dataset_name = \"sarcasm\"\n",
    "filename = \"train_bench.csv\"\n",
    "target = \"label\"\n",
    "ids = [\n",
    "    \"date\",\n",
    "    \"created_utc\",\n",
    "    \"author\",\n",
    "    \"subreddit\",\n",
    "    \"score\",\n",
    "    \"ups\",\n",
    "    \"downs\",\n",
    "    \"parent_comment\",\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:49:50.138553Z",
     "start_time": "2020-09-15T00:49:50.115999Z"
    }
   },
   "outputs": [],
   "source": [
    "out = Path(os.getcwd()) / \"data\" / dataset_name / filename"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:49:50.236384Z",
     "start_time": "2020-09-15T00:49:50.139509Z"
    }
   },
   "outputs": [],
   "source": [
    "train = pd.read_csv(out)\n",
    "train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:49:50.259734Z",
     "start_time": "2020-09-15T00:49:50.237216Z"
    }
   },
   "outputs": [],
   "source": [
    "train.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:49:50.292726Z",
     "start_time": "2020-09-15T00:49:50.260690Z"
    }
   },
   "outputs": [],
   "source": [
    "if \"Set\" not in train.columns:\n",
    "    print(\"Building tailored column\")\n",
    "    train_valid_index, test_index = next(\n",
    "        StratifiedShuffleSplit(n_splits=1, test_size=0.1, random_state=SEED).split(\n",
    "            range(train[target].shape[0]), train[target].values\n",
    "        )\n",
    "    )\n",
    "    train_index, valid_index = next(\n",
    "        StratifiedShuffleSplit(n_splits=1, test_size=0.1, random_state=SEED).split(\n",
    "            train_valid_index, train[target].values[train_valid_index]\n",
    "        )\n",
    "    )\n",
    "    train[\"Set\"] = \"train\"\n",
    "    train[\"Set\"][valid_index] = \"valid\"\n",
    "    train[\"Set\"][test_index] = \"test\"\n",
    "    # train.to_csv((out.parent / \"train_bench.csv\").as_posix(), index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:49:50.329479Z",
     "start_time": "2020-09-15T00:49:50.293669Z"
    }
   },
   "outputs": [],
   "source": [
    "train_indices = train[train.Set == \"train\"].index\n",
    "valid_indices = train[train.Set == \"valid\"].index\n",
    "test_indices = train[train.Set == \"test\"].index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:49:50.351570Z",
     "start_time": "2020-09-15T00:49:50.330628Z"
    }
   },
   "outputs": [],
   "source": [
    "# train[\"comment\"] = train[\"comment\"].fillna(\"unknown\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:49:50.375478Z",
     "start_time": "2020-09-15T00:49:50.354397Z"
    }
   },
   "outputs": [],
   "source": [
    "# max_len = np.percentile(np.vectorize(len)(train[\"comment\"].values), 90)\n",
    "# max_len"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:49:50.398025Z",
     "start_time": "2020-09-15T00:49:50.376866Z"
    }
   },
   "outputs": [],
   "source": [
    "# train[\"comment\"] = train[\"comment\"].str.slice(0, int(max_len))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:49:50.419520Z",
     "start_time": "2020-09-15T00:49:50.399057Z"
    }
   },
   "outputs": [],
   "source": [
    "# train[\"parent_comment\"] = train[\"parent_comment\"].fillna(\"unknown\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:49:50.442449Z",
     "start_time": "2020-09-15T00:49:50.420502Z"
    }
   },
   "outputs": [],
   "source": [
    "# max_len = np.percentile(np.vectorize(len)(train[\"parent_comment\"].values), 90)\n",
    "# max_len"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:49:50.464562Z",
     "start_time": "2020-09-15T00:49:50.443378Z"
    }
   },
   "outputs": [],
   "source": [
    "# train[\"parent_comment\"] = train[\"parent_comment\"].str.slice(0, int(max_len))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:49:51.038468Z",
     "start_time": "2020-09-15T00:49:50.465697Z"
    }
   },
   "outputs": [],
   "source": [
    "input_train, params = preproc_dataset(train.loc[train_indices], target, ids + [\"Set\"])\n",
    "params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:49:51.061166Z",
     "start_time": "2020-09-15T00:49:51.039320Z"
    }
   },
   "outputs": [],
   "source": [
    "len(train_indices)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:49:51.173047Z",
     "start_time": "2020-09-15T00:49:51.062100Z"
    }
   },
   "outputs": [],
   "source": [
    "input_valid, _ = preproc_dataset(\n",
    "    train.loc[valid_indices], target, ids + [\"Set\"], params\n",
    ")\n",
    "input_test, _ = preproc_dataset(train.loc[test_indices], target, ids + [\"Set\"], params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:49:51.196057Z",
     "start_time": "2020-09-15T00:49:51.173890Z"
    }
   },
   "outputs": [],
   "source": [
    "target_encoder = LabelEncoder()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:49:51.229573Z",
     "start_time": "2020-09-15T00:49:51.197038Z"
    }
   },
   "outputs": [],
   "source": [
    "train[target] = target_encoder.fit_transform(train[target].values.reshape(-1))\n",
    "y_train = train[target].values[train_indices]\n",
    "y_valid = train[target].values[valid_indices]\n",
    "y_test = train[target].values[test_indices]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:49:51.258671Z",
     "start_time": "2020-09-15T00:49:51.230532Z"
    }
   },
   "outputs": [],
   "source": [
    "params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:49:51.879658Z",
     "start_time": "2020-09-15T00:49:51.259616Z"
    }
   },
   "outputs": [],
   "source": [
    "model = build_model(params, lconv_dim=[16], lconv_num_dim=[16],)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:49:51.903426Z",
     "start_time": "2020-09-15T00:49:51.880654Z"
    }
   },
   "outputs": [],
   "source": [
    "model.get_layer(\"output\")._build_input_shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:49:51.928804Z",
     "start_time": "2020-09-15T00:49:51.904292Z"
    }
   },
   "outputs": [],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:49:51.951551Z",
     "start_time": "2020-09-15T00:49:51.929739Z"
    }
   },
   "outputs": [],
   "source": [
    "# !pip install pydot graphviz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:49:51.975849Z",
     "start_time": "2020-09-15T00:49:51.952398Z"
    }
   },
   "outputs": [],
   "source": [
    "plot_model(\n",
    "    model,\n",
    "    # to_file=\"model.png\",\n",
    "    show_shapes=True,\n",
    "    show_layer_names=True,\n",
    "    rankdir=\"TB\",\n",
    "    expand_nested=False,\n",
    "    dpi=96,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:49:51.997916Z",
     "start_time": "2020-09-15T00:49:51.976823Z"
    }
   },
   "outputs": [],
   "source": [
    "#!pip install pydot graphviz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:49:52.021057Z",
     "start_time": "2020-09-15T00:49:51.998901Z"
    }
   },
   "outputs": [],
   "source": [
    "y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:49:52.045667Z",
     "start_time": "2020-09-15T00:49:52.021910Z"
    }
   },
   "outputs": [],
   "source": [
    "counts = np.unique(y_train, return_counts=True)[1]\n",
    "counts = counts.sum() / counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:49:52.070689Z",
     "start_time": "2020-09-15T00:49:52.046630Z"
    }
   },
   "outputs": [],
   "source": [
    "class_weight = {\n",
    "    0: counts[0],\n",
    "    1: counts[1],\n",
    "}\n",
    "class_weight"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-14T10:13:28.824624Z",
     "start_time": "2020-07-14T10:13:28.800384Z"
    }
   },
   "source": [
    "class_weight = {\n",
    "    0: 1,\n",
    "    1: 1,\n",
    "}\n",
    "class_weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:50:05.792688Z",
     "start_time": "2020-09-15T00:49:52.071499Z"
    }
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "history = model.fit(\n",
    "    input_train,\n",
    "    y_train.reshape(-1, 1),\n",
    "    epochs=2000,\n",
    "    batch_size=1024,\n",
    "    validation_data=(input_valid, y_valid.reshape(-1, 1),),\n",
    "    verbose=1,\n",
    "    callbacks=[EarlyStopping(monitor=\"val_loss\", patience=20, verbose=1)],\n",
    "    class_weight=class_weight\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:50:05.918003Z",
     "start_time": "2020-09-15T00:50:05.793647Z"
    }
   },
   "outputs": [],
   "source": [
    "plot_history(history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:50:06.214247Z",
     "start_time": "2020-09-15T00:50:05.918889Z"
    }
   },
   "outputs": [],
   "source": [
    "model_auc = roc_auc_score(\n",
    "    y_true=y_valid, y_score=model.predict(input_valid).reshape(-1),\n",
    ")\n",
    "model_auc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:50:06.236400Z",
     "start_time": "2020-09-15T00:50:06.215177Z"
    }
   },
   "outputs": [],
   "source": [
    "# 0.945180869596132 => embeddign\n",
    "# More cats 0.9499529383532423\n",
    "# 0.9510084257679181 => 32 => 16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:50:06.259648Z",
     "start_time": "2020-09-15T00:50:06.237311Z"
    }
   },
   "outputs": [],
   "source": [
    "# porto seguro 0.6232119567809998\n",
    "# homesite-quote-conversion 0.9523864163447144\n",
    "# cat in dat II 0.7664026194097782\n",
    "# albert 0.7204053242303463\n",
    "# census-income 0.937909089697099\n",
    "# open payments 0.9259053788101843\n",
    "# give-me-some-credit 0.8579704956990437\n",
    "# bank-marketing 0.7783336895486428"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:50:06.376527Z",
     "start_time": "2020-09-15T00:50:06.260604Z"
    }
   },
   "outputs": [],
   "source": [
    "model_auc = roc_auc_score(y_true=y_test, y_score=model.predict(input_test).reshape(-1),)\n",
    "model_auc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:50:06.399891Z",
     "start_time": "2020-09-15T00:50:06.377665Z"
    }
   },
   "outputs": [],
   "source": [
    "# 0.9438800811761906 => embeddign\n",
    "# More cats 0.9501807921285899\n",
    "# 0.9500337101038149 => 32 => 16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:50:06.423122Z",
     "start_time": "2020-09-15T00:50:06.400794Z"
    }
   },
   "outputs": [],
   "source": [
    "# porto seguro 0.6271530652897266\n",
    "# homesite-quote-conversion 0.9551354942949817\n",
    "# cat in dat II 0.759840080492485\n",
    "# albert 0.7160934658045495\n",
    "# census-income 0.9349738109864048\n",
    "# open payments 0.9274364959503505\n",
    "# give-me-some-credit 0.8559444719192335\n",
    "# bank-marketing 0.804706871986707"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:50:06.446660Z",
     "start_time": "2020-09-15T00:50:06.424050Z"
    }
   },
   "outputs": [],
   "source": [
    "from thc_net.explainable_model.model import predict, encode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:50:06.704952Z",
     "start_time": "2020-09-15T00:50:06.447569Z"
    }
   },
   "outputs": [],
   "source": [
    "probs, explanations = predict(model, input_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:50:06.957454Z",
     "start_time": "2020-09-15T00:50:06.705883Z"
    }
   },
   "outputs": [],
   "source": [
    "probs, encoded_output = encode(model, input_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:50:06.980809Z",
     "start_time": "2020-09-15T00:50:06.958427Z"
    }
   },
   "outputs": [],
   "source": [
    "y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:50:07.004097Z",
     "start_time": "2020-09-15T00:50:06.981722Z"
    }
   },
   "outputs": [],
   "source": [
    "encoded_output.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:50:07.027975Z",
     "start_time": "2020-09-15T00:50:07.005060Z"
    }
   },
   "outputs": [],
   "source": [
    "probs[0], explanations[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:50:07.057974Z",
     "start_time": "2020-09-15T00:50:07.028890Z"
    }
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# plt.rcdefaults()\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "def explain_plot(importances, columns):\n",
    "    selection = np.argsort(-np.absolute(importances))[:10]\n",
    "    # indexes = np.argsort(importances)\n",
    "    performance = importances[selection]\n",
    "    #     print(performance.shape)512, 256, 128, 64, 32, 1\n",
    "    y_pos = np.arange(performance.shape[0])\n",
    "\n",
    "    plt.barh(y_pos, performance, align=\"center\", alpha=0.5)\n",
    "    plt.yticks(y_pos, columns[selection])\n",
    "    # plt.xlabel('Usage')\n",
    "    plt.title(\"Feature importance\")\n",
    "\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:50:07.083133Z",
     "start_time": "2020-09-15T00:50:07.058855Z"
    }
   },
   "outputs": [],
   "source": [
    "all_cols = np.array(params[\"bool_cols\"] + params[\"num_cols\"] + params[\"cat_cols\"])\n",
    "all_cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:50:07.209814Z",
     "start_time": "2020-09-15T00:50:07.084058Z"
    }
   },
   "outputs": [],
   "source": [
    "explain_plot(explanations[0], all_cols)\n",
    "probs[0].item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:50:09.309472Z",
     "start_time": "2020-09-15T00:50:07.210679Z"
    }
   },
   "outputs": [],
   "source": [
    "for i in range(20):\n",
    "    explain_plot(explanations[i], all_cols)\n",
    "    print(probs[i].item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:50:09.333289Z",
     "start_time": "2020-09-15T00:50:09.310387Z"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:50:09.357827Z",
     "start_time": "2020-09-15T00:50:09.334361Z"
    }
   },
   "outputs": [],
   "source": [
    "np.unique(y_test, return_counts=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:50:09.380938Z",
     "start_time": "2020-09-15T00:50:09.358684Z"
    }
   },
   "outputs": [],
   "source": [
    "y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:50:09.404429Z",
     "start_time": "2020-09-15T00:50:09.381906Z"
    }
   },
   "outputs": [],
   "source": [
    "target_encoder.classes_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-15T00:50:09.530187Z",
     "start_time": "2020-09-15T00:50:09.405362Z"
    }
   },
   "outputs": [],
   "source": [
    "confusion_matrix(\n",
    "    y_true=y_test,\n",
    "    y_pred=model.predict(input_test).reshape(-1) >= 0.5,\n",
    "    # labels=target_encoder.classes_,\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}